d703487 · 33a0b0e · d71c35b · 47df140 · df7e69c
@@ -99,6 +99,7 @@
Machine routes are not pages and have no entry: `/audit.csv`,
- `/system` — `pages/system.ts` · the machine as it is now: cores, load, memory, processes · `/flow`, `/` · none seen; updates in place over `/system/stream`, no reload script.
- `/resilience` — `pages/resilience.ts` · each protection in `docs/THREATS.md` against the last gate's tests · `/settings/resilience` (similar name, different content) · none seen.
- `/maintenance` — `pages/maintenance.ts` · prepare the forge for a power-off and end it · `/resilience`, `/settings/backups` · none seen.
- `/admin/limits` — `pages/admin-limits.ts` · every limit with value, default and reason · `/settings/capacity` (also a numbers-you-can-change page) · none seen.
- `/settings/capacity` — `pages/capacity.ts` · how many runs and checks go at once · `/admin/limits`, `/settings/usage` · none seen.
- `/settings/usage` — `pages/usage.ts` · route patterns by use, unused first · `/settings/telemetry`, `/board/returns` · none seen. Not about cost, despite the name.
@@ -0,0 +1,44 @@
# Maintenance: shut down for a move and come back by itself (#9453)
**Prepare.** An admin opens `/maintenance` (linked from Resilience), the terminal's Maintenance command
(`src/tui/screens/maintenance.ts`: `p` prepare, `c` check again, `e` end; admins only), or calls
`POST /api/v1/admin/maintenance/prepare` (admin-scope token) or MCP
`maintenance {action: "prepare"}`. The forge then:
1. holds releases (an owner hold, `keeper/hold.ts`);
2. pauses the crew: the dispatcher starts nothing while `maintenance.json`
   stands in the data directory; vitals reads the queue as held by "planned maintenance";
3. takes a fresh snapshot (`actions/backups_now.ts`);
4. reports the drain: `GET /api/v1/admin/maintenance` says `draining` while runs
   or checks still go, `ready` when the backup is taken and nothing runs. Power
   off on `ready`; a SIGTERM drains the rest as in a release.
The page says plainly what stops: releases, new crew runs, then the forge, its
apps and previews when the host powers off.
**Come back.** On boot `resumeAtBoot` acts only when the host has restarted since the prepare
(`/proc/sys/kernel/random/boot_id` differs from the one stored); a release roll of the process on
the same host boot keeps what the admin prepared. It then lifts the hold, unpauses the crew and
records a line (shown on `/maintenance`, in the status, and as the
`maintenance.ended` event) naming what was verified. An admin can also press
"End maintenance now". The tunnel, runner and nightly timers are the forge's
own boot: they start with the process, and the nightly jobs show on Resilience.
No Kubernetes is asked anything; the state is one file.
**While it is off.** `bun scripts/standby.ts 8080` serves the calm 503 page
(`src/maintenance/page.ts`) from the droplet; point the tunnel's or
Cloudflare's origin at it for the move, and back after. A Cloudflare Worker can
return the same HTML with status 503 and `Retry-After`.
**Size of a standby.** For reads or a whole forge: disk for the data (38 GB now,
apps 13 GB) plus one snapshot and headroom, so 64 GB; 4 GB of memory free for the
forge alone, 8 GB with apps running. The 16 GB droplet cannot hold it.
## Not done
- No standby takeover: nothing replicates data to a second machine, so the
  droplet can only show the page. That needs the snapshot mirror restored onto a
  machine of the size above, and is a separate issue.
- The page is not pushed to Cloudflare automatically; `standby.ts` is started by hand.
- The admin is told on the page, event and status; no mail or push.
- Where the kernel gives no boot id, any restart ends maintenance.
- The return checks the database, hold, dispatcher, apps on record and nightly lines; it does not probe the tunnel or the runner.
@@ -0,0 +1,24 @@
/**
 * The maintenance page, served from somewhere that stays up (the droplet):
 * `bun scripts/standby.ts [port]` answers every request with the calm page
 * and a 503, so a tunnel or Cloudflare origin pointed here shows visitors
 * "back soon", not an error. MAINTENANCE_REASON and MAINTENANCE_SINCE are
 * optional. Zero dependencies; reads no forge data.
 */
import { MAINTENANCE_HEADERS, maintenanceHtml } from "../src/maintenance/page.ts";
const port = Number(process.argv[2] ?? process.env["PORT"] ?? 8080);
const body = maintenanceHtml({
  ...(process.env["MAINTENANCE_REASON"] ? { reason: process.env["MAINTENANCE_REASON"] } : {}),
  ...(process.env["MAINTENANCE_SINCE"] ? { since: process.env["MAINTENANCE_SINCE"] } : {}),
});
Bun.serve({
  port,
  fetch(req) {
    const path = new URL(req.url).pathname;
    if (path === "/healthz") return new Response("standby", { status: 200 });
    return new Response(req.method === "HEAD" ? null : body, { status: 503, headers: MAINTENANCE_HEADERS });
  },
});
console.log(`standby maintenance page on :${port}`);
@@ -0,0 +1,9 @@
import type { Route } from "../shared.ts";
export const route: Route = {
  method: "get",
  path: "/api/v1/admin/maintenance",
  summary: "Where planned maintenance stands: { phase: idle | draining | ready, since?, reason?, backup?, working, last?, says }; ready means releases are held, the crew is paused, a fresh backup is taken and nothing runs",
  auth: true,
  refusals: { not_found: "only a forge administrator sees this route" },
};
@@ -40,6 +40,7 @@
export { route as getApiV1AdminDrills } from "./getApiV1AdminDrills.ts";
export { route as getApiV1AdminInvites } from "./getApiV1AdminInvites.ts";
export { route as getApiV1AdminKpisProbe } from "./getApiV1AdminKpisProbe.ts";
export { route as getApiV1AdminLimits } from "./getApiV1AdminLimits.ts";
export { route as getApiV1AdminMaintenance } from "./getApiV1AdminMaintenance.ts";
export { route as getApiV1AdminPeople } from "./getApiV1AdminPeople.ts";
export { route as getApiV1AdminPinned } from "./getApiV1AdminPinned.ts";
export { route as getApiV1AdminProfile } from "./getApiV1AdminProfile.ts";
@@ -218,6 +219,8 @@
export { route as postApiV1AdminAccountsUsernameSuspend } from "./postApiV1Admin
export { route as postApiV1AdminAccountsUsernameUnsuspend } from "./postApiV1AdminAccountsUsernameUnsuspend.ts";
export { route as postApiV1AdminAccountsUsernameVerify } from "./postApiV1AdminAccountsUsernameVerify.ts";
export { route as postApiV1AdminInvites } from "./postApiV1AdminInvites.ts";
export { route as postApiV1AdminMaintenancePrepare } from "./postApiV1AdminMaintenancePrepare.ts";
export { route as postApiV1AdminMaintenanceResume } from "./postApiV1AdminMaintenanceResume.ts";
export { route as postApiV1AdminPinned } from "./postApiV1AdminPinned.ts";
export { route as postApiV1AdminPinnedNameCheck } from "./postApiV1AdminPinnedNameCheck.ts";
export { route as postApiV1AdminPinnedNameRollback } from "./postApiV1AdminPinnedNameRollback.ts";
@@ -0,0 +1,10 @@
import type { Route } from "../shared.ts";
export const route: Route = {
  method: "post",
  path: "/api/v1/admin/maintenance/prepare",
  summary: "Prepare the forge for a power-off: hold releases, pause the crew, take a fresh backup, report the drain; needs an admin-scope token. Answers the status as GET does",
  auth: true,
  body: { reason: "string" },
  refusals: { not_found: "only a forge administrator may", invalid: "the reason is at most 200 characters (422)" },
};
@@ -0,0 +1,9 @@
import type { Route } from "../shared.ts";
export const route: Route = {
  method: "post",
  path: "/api/v1/admin/maintenance/resume",
  summary: "End maintenance now: lift the release hold and let the crew start; needs an admin-scope token. The forge does this by itself at boot",
  auth: true,
  refusals: { not_found: "only a forge administrator may", conflict: "the forge is not in maintenance (409)" },
};
@@ -192,6 +192,8 @@
export function classifiedAudience(kind: string): EventAudience | undefined {
    kind.startsWith("drill.") ||
    // A snapshot moving is the forge owner's too, and has no repository.
    kind.startsWith("backup.") ||
    // Preparing for a power-off and ending it are the forge owner's (`maintenance/service.ts`).
    kind.startsWith("maintenance.") ||
    // The weekly model review's changes and results are the owner's (`actions/model_review.ts`).
    kind.startsWith("crew.model_") ||
    // A child forge answering no one at its public address is the owner's to hear (`spawn/reach.ts`).
@@ -96,6 +96,7 @@
export const SYSTEM_NAMES: readonly string[] = [
  "organizations",
  "now",
  "resilience",
  "maintenance",
  "origins",
  "robots.txt",
  "favicon.ico",
@@ -48,6 +48,7 @@
import { noteHeld, type HoldCause } from "../runs/held.ts";
import { decideAndRecord } from "./ledger.ts";
import { bugsFirst } from "../flow/bugs.ts";
import { byDials } from "./team_share.ts";
import { inMaintenance } from "../maintenance/state.ts";
import { gateQueue } from "../flow/valve_gate.ts";
import { ARCHIVED_WHY, getRepoById, type Repo } from "../core/repos.ts";
import { listReviews } from "../core/reviews.ts";
@@ -1527,6 +1528,13 @@
function pump(
  const gated = startOrder(app);
  // What this pass holds by policy, for vitals (`runs/held.ts`, #9403).
  const held = new Map<string, HoldCause>();
  // Planned maintenance (`maintenance/service.ts`): nothing new starts, and
  // a queued run waits as it does behind a shut valve.
  if (inMaintenance(app.config.dataDir)) {
    for (const q of [...gated.order, ...gated.held]) held.set(q.id, "maintenance");
    noteHeld(app.db, held);
    return;
  }
  for (const shut of gated.held) {
    held.set(shut.id, "valve");
    if (noted.has(shut.id)) continue;
@@ -349,6 +349,7 @@
import { labelsRoutes } from "./api/labels.ts";
import { visionRoutes } from "./api/vision.ts";
import { tokenRoutes } from "./api/tokens.ts";
import { accountAdmin, adminRoutes } from "./api/admin.ts";
import { maintenanceRoutes } from "./api/maintenance.ts";
import { flagsRoutes } from "./api/flags.ts";
import { issueRoutes, labelsHandler } from "./api/issues.ts";
import { FORGE_NOT_CARRIED } from "../origins/words.ts";
@@ -1226,6 +1227,7 @@
export function apiRoutes(router: Router<Handler>): void {
  userRoutes(router);
  forgeOriginRoutes(router);
  adminRoutes(router);
  maintenanceRoutes(router);
  flagsRoutes(router);
  // Open read hooks (#1789): the owner, or an agent holding `telemetry:read`.
@@ -0,0 +1,42 @@
/** `/api/v1/admin/maintenance`: what the Maintenance page and MCP `maintenance` do, for a forge admin's admin-scope credential. */
import { prepare, resumeNow, status } from "../../maintenance/service.ts";
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import { crossOrigin, type Handler } from "../context.ts";
import { apiError, boundedBody, forbidden, json, notFound, requireScope, unauthorized } from "../respond.ts";
import type { Router } from "../router.ts";
/** A reason is at most 200 characters; room for them escaped. */
const BODY_MAX = 4096;
export function maintenanceRoutes(router: Router<Handler>): void {
  router.add("GET", "/api/v1/admin/maintenance", (ctx) => {
    notePhase(ctx.request, "session", takeSessionTime(ctx.request));
    if (!ctx.auth.user) return unauthorized();
    if (!ctx.auth.user.isAdmin) return notFound();
    const denied = requireScope(ctx, "ro");
    if (denied) return denied;
    return json(status(ctx.app));
  });
  router.add("POST", "/api/v1/admin/maintenance/prepare", async (ctx) => {
    if (!ctx.auth.user) return unauthorized();
    if (!ctx.auth.user.isAdmin) return notFound();
    const denied = requireScope(ctx, "admin");
      
        sentinelcrew
        5d ago
        outdated
      
      High: the body is parsed with request.json() and no cap on bytes actually read. Use the repo's capped reader and refuse past the limit.
    
    if (denied) return denied;
    if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden();
    const input = await boundedBody(ctx.request, BODY_MAX);
    if (input === "too_large") return apiError("too_large", 413, "The request is too large.");
    const reason = typeof input["reason"] === "string" ? input["reason"] : "";
    const r = await prepare(ctx.app, ctx.auth.user, reason);
    return r.ok ? json(r.value) : apiError(r.error.code === "invalid" ? "invalid" : "not_found", r.error.code === "invalid" ? 422 : 404, r.error.says);
  });
  router.add("POST", "/api/v1/admin/maintenance/resume", (ctx) => {
    if (!ctx.auth.user) return unauthorized();
    if (!ctx.auth.user.isAdmin) return notFound();
    const denied = requireScope(ctx, "admin");
    if (denied) return denied;
    if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden();
    const r = resumeNow(ctx.app, ctx.auth.user);
    return r.ok ? json(r.value) : apiError(r.error.code, r.error.code === "conflict" ? 409 : 404, r.error.says);
  });
}
@@ -86,6 +86,7 @@
export { tool as land_past_gate } from "./land_past_gate.ts";
export { tool as lift } from "./lift.ts";
export { tool as limb } from "./limb.ts";
export { tool as limits } from "./limits.ts";
export { tool as maintenance } from "./maintenance.ts";
export { tool as make_it_yours } from "./make_it_yours.ts";
export { tool as map } from "./map.ts";
export { tool as merge } from "./merge.ts";
@@ -0,0 +1,28 @@
import { prepare, resumeNow, status } from "../../../maintenance/service.ts";
import { scopeAllows } from "../../../core/tokens.ts";
import { good, bad, type Tool } from "../shared.ts";
export const tool: Tool = {
  name: "maintenance",
  doc: "`action` is `status` (default), `prepare` or `resume`; `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says }`",
  description: "Prepare the forge for a power-off, check the drain, or end maintenance (admin).",
  inputSchema: {
    type: "object",
    properties: { action: { type: "string", enum: ["status", "prepare", "resume"] }, reason: { type: "string", maxLength: 200 } },
  },
  writes: true,
  async run(ctx, user, args) {
    if (!user.isAdmin) return bad("not_found", "no such tool here");
    const action = args["action"] ?? "status";
    if (action === "status") return good(status(ctx.app));
    if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare or resume");
    if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token");
    if (action === "resume") {
      const r = resumeNow(ctx.app, user);
      return r.ok ? good(r.value) : bad(r.error.code, r.error.says);
    }
    const reason = typeof args["reason"] === "string" ? args["reason"] : "";
    const r = await prepare(ctx.app, user, reason);
    return r.ok ? good(r.value) : bad(r.error.code, r.error.says);
  },
};
@@ -26,6 +26,7 @@
import { prerenderComments } from "../web/prerender.ts";
import { systemPage, systemStream } from "../web/pages/system.ts";
import { contributePage } from "../web/pages/contribute.ts";
import { pipelinePage } from "../web/pages/pipeline.ts";
import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
import { nightlyOutputPage, resiliencePage } from "../web/pages/resilience.ts";
import { limitsPage, limitsSet, quotasSet, signupSet, inviteMint, inviteRevoke } from "../web/pages/admin-limits.ts";
import { measureSubmit } from "./measure.ts";
@@ -898,6 +899,9 @@
export function buildRouter(): Router<Handler> {
  pinnedRoutes(router);
  // Each THREATS.md protection against the last gate; literal, above /:owner.
  router.add("GET", "/resilience", resiliencePage);
  router.add("GET", "/maintenance", maintenancePage);
  router.add("POST", "/maintenance/prepare", maintenancePrepare);
  router.add("POST", "/maintenance/resume", maintenanceResume);
  router.add("GET", "/resilience/nightly/:job", nightlyOutputPage);
  // A selection's spend and runs over time; literal, above /:owner.
  router.add("GET", "/fleet/graph", fleetGraphPage);
@@ -72,6 +72,7 @@
import { peopleOnCrews } from "./core/agents.ts";
import { startNotifier } from "./crew/notify.ts";
import { expireCrewHolds } from "./actions/holds.ts";
import { startClogTick } from "./flow/clog_tick.ts";
import { resumeAtBoot } from "./maintenance/service.ts";
import { startKeeper } from "./flow/keeper.ts";
import { sweepStaleAsks } from "./crew/stale_asks.ts";
import { recordAskLedger } from "./web/studio/asks.ts";
@@ -673,6 +674,8 @@
const processTick = (): void => {
const stopClogTick = startClogTick(app.db, app.bus, (error) => app.log.warn("clog tick failed", { error: String(error) }), 60_000);
// The throughput keeper (#7645) names what binds landings each minute and lifts what is its to lift.
const stopKeeper = startKeeper(app, () => sweepAssigned(app));
// A forge that went down prepared for maintenance comes back by itself.
resumeAtBoot(app);
// An agent's hold that has stood 24 h is released each minute (#6918).
const holdTimer = setInterval(() => {
  try {
@@ -0,0 +1,31 @@
/**
 * The page visitors see while the forge is off: calm, static, escaped, and
 * answerable by anything (the droplet's `scripts/standby.ts`, or a Cloudflare
 * Worker serving the same text). No script, no external asset.
 */
const esc = (s: string): string =>
  s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
export interface PageInput {
  /** What the owner said, one line; may be empty. */
  readonly reason?: string;
  /** ISO time the forge went into maintenance, when known. */
  readonly since?: string;
}
export function maintenanceHtml(input: PageInput = {}): string {
  const reason = (input.reason ?? "").replace(/[\u0000-\u001f\u007f]+/g, " ").trim().slice(0, 200);
  const since = input.since !== undefined && /^[0-9T:.\-Z+]{10,30}$/.test(input.since) ? input.since : "";
  return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta http-equiv="refresh" content="60"><title>Back soon</title><style>body{font:16px/1.5 system-ui,sans-serif;margin:0;display:grid;min-height:100vh;place-items:center;background:#fafafa;color:#222}main{max-width:32rem;padding:1.5rem}h1{font-size:1.5rem;margin:0 0 .5rem}p{margin:.5rem 0}.m{color:#666}@media(prefers-color-scheme:dark){body{background:#151515;color:#eee}.m{color:#aaa}}</style></head><body><main><h1>Back soon</h1><p>The forge is down for planned maintenance and will be back soon.</p>${reason === "" ? "" : `<p>${esc(reason)}</p>`}${since === "" ? "" : `<p class="m">Down since ${esc(since)}.</p>`}<p class="m">This page refreshes itself every minute.</p></main></body></html>`;
      
        sentinelcrew
        5d ago
        outdated
      
      Principle 8 and truth: the text says the forge is 'being moved to new hardware', which is false for most maintenance. Say plainly: 'The forge is down for planned maintenance and will be back soon.'
    
}
/** The headers the maintenance page is always served with: 503 so nothing caches it as the real site. */
export const MAINTENANCE_HEADERS: Readonly<Record<string, string>> = {
  "content-type": "text/html; charset=utf-8",
  "retry-after": "300",
  "cache-control": "no-store",
  "content-security-policy": "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'",
  "x-content-type-options": "nosniff",
  "referrer-policy": "no-referrer",
};
@@ -0,0 +1,156 @@
/**
 * Planned maintenance: an admin asks the forge to get ready for a power-off
 * (`prepare`), and on the next boot it comes back by itself (`resume`).
 *
 * prepare: hold releases (an owner hold, `keeper/hold.ts`), pause the crew
 * (the dispatcher starts nothing while the state file stands), take a fresh
 * snapshot, then report what still runs. The host may power off when
 * `status().ready` is true; a SIGTERM drains the rest as it does for a roll.
 * resume: lift the hold, let the crew start again, verify, and keep a line
 * saying what is healthy. No Kubernetes is asked anything.
 */
import { readFileSync } from "node:fs";
import type { App } from "../app.ts";
import { listApps } from "../core/apps.ts";
import { nightlyLines } from "../nightly/says.ts";
import { listActive } from "../core/runs.ts";
import type { User } from "../core/users.ts";
import { lift, MAX_REASON, setHold } from "../keeper/hold.ts";
import { err, ok, type Result } from "../kernel/result.ts";
import { running } from "../kernel/version.ts";
import { snapshotNow } from "../actions/backups_now.ts";
import { readState, writeState, type Maintenance, type Returned } from "./state.ts";
const isAdmin = (u: User | undefined): u is User => u?.isAdmin === true && u.kind !== "agent";
export interface Status {
  readonly phase: "idle" | "draining" | "ready";
  readonly since?: string;
  readonly reason?: string;
  readonly backup?: Maintenance["backup"];
  /** Crew runs and checks still queued or running. */
  readonly working: number;
  readonly last?: Returned;
  /** Plainly: what stops, or what is about to. */
  readonly says: string;
}
/** What stops when maintenance is prepared, one row each; the page and the terminal screen say it the same. */
export const ABOUT_TO_STOP_ROWS: readonly { readonly what: string; readonly stops: string }[] = [
  { what: "Releases", stops: "held now; none are cut." },
  { what: "The crew", stops: "no run starts; running ones finish." },
  { what: "The forge", stops: "stops with the host, with its apps and previews." },
];
/** The host's boot id, or undefined where the kernel does not say: it changes only when the host restarts. */
export function hostBootId(): string | undefined {
  try {
    const id = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
    return id === "" ? undefined : id;
  } catch {
    return undefined;
  }
}
export function status(app: App): Status {
  const { current, last } = readState(app.config.dataDir);
  if (current === undefined)
    return { phase: "idle", working: 0, ...(last ? { last } : {}), says: last ? last.says : "Not in maintenance." };
  const working = listActive(app.db, "agent").length + listActive(app.db, "check").filter((r) => r.state === "running").length;
  const ready = current.backup.ok && working === 0;
  return {
    phase: ready ? "ready" : "draining",
    since: current.since,
    reason: current.reason,
    backup: current.backup,
    working,
    says: ready
      ? "Ready: releases are held, the crew is paused, a fresh backup is taken and nothing is running. The host can power off."
      : !current.backup.ok
        ? `Not ready: ${current.backup.says}`
        : `Draining: ${working} crew run${working === 1 ? "" : "s"} or check${working === 1 ? "" : "s"} still going; the host can power off when it reads 0.`,
  };
}
export type Refusal = { readonly code: "not_found" | "invalid" | "conflict"; readonly says: string };
export async function prepare(app: App, actor: User | undefined, reasonIn: string): Promise<Result<Status, Refusal>> {
  if (!isAdmin(actor)) return err({ code: "not_found", says: "Only a forge admin prepares maintenance." });
  const reason = reasonIn.replace(/[\u0000-\u001f\u007f]+/g, " ").trim();
  if (reason.length > 200) return err({ code: "invalid", says: "The reason is at most 200 characters." });
  const dataDir = app.config.dataDir;
  const prior = readState(dataDir);
  if (prior.current !== undefined) {
    // Asking again retakes the backup and reads the drain; the hold stands once.
    const taken = await snapshotNow(app, actor);
    writeState(dataDir, { ...prior, current: { ...prior.current, backup: { ok: taken.ok, says: taken.ok ? taken.value.says : taken.error.says } } });
    return ok(status(app));
  }
  const hold = setHold(app.db, {
    kind: "owner",
    version: running().version,
    reason: `maintenance${reason === "" ? "" : `: ${reason}`}`.slice(0, MAX_REASON),
    at: Date.now(),
  });
  const now = new Date().toISOString();
  // Written before the backup: the crew pauses at once, and a crash mid-backup still resumes at boot.
  const boot = hostBootId();
  const base: Maintenance = { since: now, by: actor.username, reason, holdId: hold.id, backup: { ok: false, says: "the backup has not finished" }, ...(boot === undefined ? {} : { bootId: boot }) };
  writeState(dataDir, { ...(prior.last ? { last: prior.last } : {}), current: base });
  app.bus.append({ kind: "maintenance.prepared", actorId: actor.id, payload: { hold: hold.id, reason } });
  const taken = await snapshotNow(app, actor);
  writeState(dataDir, {
    ...(prior.last ? { last: prior.last } : {}),
    current: { ...base, backup: { ok: taken.ok, says: taken.ok ? taken.value.says : taken.error.says } },
  });
  return ok(status(app));
}
/** Come back: lift our hold, start the crew, and record what was checked. `by` names who or what did it. */
export function resume(app: App, by: string): Result<Returned, Refusal> {
  const dataDir = app.config.dataDir;
  const { current } = readState(dataDir);
  if (current === undefined) return err({ code: "conflict", says: "The forge is not in maintenance." });
  const now = Date.now();
  const lifted = lift(app.db, current.holdId, by, now);
  const verified = [
    `database open, version ${running().version}`,
    lifted ? "release hold lifted" : "release hold was already lifted",
    "crew dispatcher unpaused",
    `${listApps(app.db).length} app(s) on record, started by the forge's own boot`,
    `nightly jobs: ${nightlyLines(app.config.dataDir).join(" / ")}`,
    `${listActive(app.db, "agent").length} crew run(s) and ${listActive(app.db, "check").length} check(s) waiting to start`,
  ];
  const back: Returned = {
    at: new Date(now).toISOString(),
    downSince: current.since,
    says: `Back at ${new Date(now).toISOString()} after maintenance begun ${current.since}: ${verified.join("; ")}.`,
    verified,
  };
  writeState(dataDir, { last: back });
  app.bus.append({ kind: "maintenance.ended", payload: { by, hold: current.holdId, verified } });
  return ok(back);
}
/**
 * At boot: a forge that went down prepared comes back by itself, once the host
 * has restarted. A restart of the process alone (a release roll) on the same
 * host boot keeps what the admin prepared. Where the boot id is unknown, resume:
 * a crew left paused is worse than a prepare that must be asked again.
 */
export function resumeAtBoot(app: App, bootId: string | undefined = hostBootId()): void {
  const { current } = readState(app.config.dataDir);
  if (current === undefined) return;
  if (bootId !== undefined && current.bootId === bootId) {
    app.log.info("maintenance stands: the process restarted, the host did not");
    return;
  }
  const r = resume(app, "boot");
  if (r.ok) app.log.info("maintenance ended at boot", { says: r.value.says });
}
export function resumeNow(app: App, actor: User | undefined): Result<Returned, Refusal> {
  if (!isAdmin(actor)) return err({ code: "not_found", says: "Only a forge admin ends maintenance." });
  return resume(app, actor.username);
}
@@ -0,0 +1,70 @@
/**
 * The one file of maintenance state, `maintenance.json` under the data
 * directory: it survives the power-off because the data does. Plain data in,
 * plain data out; a missing or unreadable file reads as "not in maintenance",
 * so a damaged file can never keep the crew paused.
 */
import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
export interface Backup {
  readonly ok: boolean;
  readonly says: string;
}
export interface Maintenance {
  readonly since: string;
  readonly by: string;
  /** One line the maintenance page shows, 0 to 200 characters. */
  readonly reason: string;
  /** The release hold this request placed (`keeper/hold.ts`); lifted when the forge returns. */
  readonly holdId: number;
  readonly backup: Backup;
  /** The host's boot id when prepared: a different one at boot means the host was powered off. */
  readonly bootId?: string;
}
export interface Returned {
  readonly at: string;
  readonly downSince: string;
  readonly says: string;
  readonly verified: readonly string[];
}
export interface Stored {
  readonly current?: Maintenance;
  /** The last return, kept so an admin is told when it is back and what is healthy. */
  readonly last?: Returned;
}
const file = (dataDir: string): string => join(dataDir, "maintenance.json");
export function readState(dataDir: string): Stored {
  try {
    if (!existsSync(file(dataDir))) return {};
    const raw: unknown = JSON.parse(readFileSync(file(dataDir), "utf8"));
    if (typeof raw !== "object" || raw === null) return {};
    const o = raw as Stored;
    const cur = o.current;
    const good =
      cur !== undefined &&
      typeof cur.since === "string" &&
      typeof cur.holdId === "number" &&
      typeof cur.backup?.ok === "boolean";
    return { ...(good ? { current: cur } : {}), ...(o.last !== undefined && typeof o.last.at === "string" ? { last: o.last } : {}) };
  } catch {
    return {};
  }
}
export function writeState(dataDir: string, state: Stored): void {
  const tmp = `${file(dataDir)}.tmp`;
  writeFileSync(tmp, JSON.stringify(state), { mode: 0o600 });
  renameSync(tmp, file(dataDir));
}
      
        sentinelcrew
        5d ago
        outdated
      
      clearState has no caller. Remove it (dead code).
    
/** Is the forge in maintenance? The dispatcher asks on every pass. */
export function inMaintenance(dataDir: string): boolean {
  return readState(dataDir).current !== undefined;
}
@@ -17,13 +17,14 @@
import type { Database } from "bun:sqlite";
export const HELD_FRESH_MS = 2 * 60_000;
/** Why the dispatcher held a run. */
export type HoldCause = "valve" | "pace" | "limit";
export type HoldCause = "valve" | "pace" | "limit" | "maintenance";
/** How vitals names each cause: "2 by a shut valve". */
export const HOLD_WORDS: Readonly<Record<HoldCause, string>> = {
  valve: "a shut valve",
  pace: "the week's pace",
  limit: "an account limit",
  maintenance: "planned maintenance",
};
interface Pass {
@@ -33,6 +33,7 @@
import { entry as homePins } from "./home_pins.ts";
import { entry as invites } from "./invites.ts";
import { entry as limits } from "./limits.ts";
import { entry as loopStalls } from "./loopstalls.ts";
import { entry as maintenance } from "./maintenance.ts";
import { modelCeilingEntry, modelGrantEntry } from "./model.ts";
import { entry as orgGrants } from "./orggrants.ts";
import { entry as orgTeams } from "./orgteams.ts";
@@ -82,6 +83,7 @@
export const ENTRIES: readonly PaletteEntry[] = [
  loopStalls,
  telemetry,
  resilience,
  maintenance,
  valves,
  access,
  edgeOnly,
@@ -0,0 +1,140 @@
/**
 * Maintenance, a forge admin's, in the terminal (#9453): the /maintenance
 * page's facts and actions through the same service (`maintenance/service.ts`).
 * `p` asks for a reason shown to visitors, then confirms, then prepares;
 * `c` checks again and retakes the backup; `e` ends maintenance after a
 * confirm. Anyone else is offered nothing and, were the screen reached, is
 * refused by the service. Enter, Esc or q closes it.
 */
import { ABOUT_TO_STOP_ROWS, prepare, resumeNow, status } from "../../maintenance/service.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import type { Session } from "../session.ts";
import { wrap } from "../term/width.ts";
import { dialog } from "../ui/widgets.ts";
import { confirmScreen, later, promptScreen, runAsync } from "./common.ts";
import type { PaletteEntry } from "./entries.ts";
const WIDTH = 72;
const INNER = WIDTH - 6;
export function maintenanceLines(f: Session): string[] {
  const s = status(f.app);
  const facts =
    s.phase === "idle"
      ? ABOUT_TO_STOP_ROWS.flatMap((r) => wrap(`${r.what}: ${r.stops}`, INNER))
      : [
          ...wrap(`Since ${s.since ?? ""}${s.reason ? `: ${s.reason}` : ""}`, INNER),
          ...wrap(`Backup: ${s.backup?.says ?? ""}`, INNER),
          `Still working: ${s.working}`,
        ];
  return [...wrap(s.says, INNER), "", ...facts];
}
export function maintenanceScreen(f: Session, ctx: Ctx): Screen {
  const mayAct = (): boolean => f.user?.isAdmin === true && f.user.kind !== "agent";
  let busy = false;
  let note = "";
  const say = (text: string): void => {
    note = text;
    ctx.invalidate();
  };
  const run = (what: () => Promise<string> | string): void => {
    busy = true;
    say("Working…");
    runAsync(ctx, async () => {
      try {
        say(await what());
      } finally {
        busy = false;
        ctx.invalidate();
      }
    });
  };
  const doPrepare = (reason: string): void =>
    run(async () => {
      const r = await prepare(f.app, f.user, reason);
      return r.ok ? r.value.says : r.error.says;
    });
  const askPrepare = (): Screen =>
    promptScreen(ctx, screen, {
      title: "Prepare for maintenance",
      label: "Shown to visitors (optional, at most 200 characters)",
      hint: "Enter goes on to the confirm.",
      onSubmit: (reason) => {
        if (reason.length > 200) return "At most 200 characters.";
        return later(ctx, () =>
          confirmScreen(ctx, screen, {
            title: "Prepare for maintenance?",
            body: ["The crew and releases stop now; the forge,", "its apps and previews stop at power-off."],
            actions: [{ label: "Cancel", variant: "ghost" }, { label: "Prepare", variant: "danger" }],
            onChoose: (i) => {
              if (i === 1) doPrepare(reason);
            },
          }),
        );
      },
    });
  const askEnd = (): Screen =>
    confirmScreen(ctx, screen, {
      title: "End maintenance now?",
      body: ["Releases are let go and the crew starts again."],
      actions: [{ label: "Cancel", variant: "ghost" }, { label: "End maintenance" }],
      onChoose: (i) => {
        if (i !== 1) return;
        const r = resumeNow(f.app, f.user);
        say(r.ok ? r.value.says : r.error.says);
      },
    });
  const screen: Screen = {
    title: "Maintenance",
    get keys() {
      const idle = status(f.app).phase === "idle";
      return [
        ...(!mayAct() || busy
          ? []
          : idle
            ? [binding(["p"], "prepare for maintenance")]
            : [binding(["c"], "check again, retake the backup"), binding(["e"], "end maintenance")]),
        binding(["enter", "esc", "q"], "close"),
      ];
    },
    render(area, s) {
      const buttons = dialog(s, area, ctx.theme.c, {
        title: "Maintenance",
        body: [...(note === "" ? [] : [...wrap(note, INNER), ""]), ...maintenanceLines(f)],
        actions: [{ label: "Close" }],
        selected: 0,
        width: WIDTH,
      });
      buttons.forEach((rect) => ctx.hit(rect, () => ctx.pop()));
    },
    key(e) {
      if (e.ctrl || e.alt) return true;
      const idle = status(f.app).phase === "idle";
      if (e.key === "enter" || e.key === "esc" || e.key === "q") ctx.pop();
      else if (!mayAct() || busy) return true;
      else if (e.key === "p" && idle) ctx.push(askPrepare());
      else if (e.key === "c" && !idle) doPrepare("");
      else if (e.key === "e" && !idle) ctx.push(askEnd());
      return true;
    },
  };
  return screen;
}
/** Prepare the forge for a power-off, check the drain, or end it: a site admin's, as /maintenance is. */
export const entry: PaletteEntry = {
  id: "maintenance",
  title: "Maintenance",
  hint: "prepare the forge for a power-off, check the drain, or end maintenance",
  after: ["resilience", "capacity", "flow", "sentinel"],
  offered: (session) => session.user?.isAdmin === true && session.user.kind !== "agent",
  run: (session, ctx) => ctx.push(maintenanceScreen(session, ctx)),
};
@@ -0,0 +1,74 @@
/**
 * /maintenance: an admin's page to prepare the forge for a power-off and to
 * end it. It says plainly what stops, what the drain still waits for, and,
 * once the forge is back, when and what was verified.
 */
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import { ABOUT_TO_STOP_ROWS, prepare, resumeNow, status } from "../../maintenance/service.ts";
import type { Ctx } from "../../http/context.ts";
import { notFoundPage, signInFirst } from "../chrome.ts";
import { field as formField, readForm } from "../form.ts";
import { html, type Html } from "../html.ts";
import { flashTo, page } from "../layout.ts";
import { btn, field } from "../ui.ts";
function admin(ctx: Ctx): Response | undefined {
  if (!ctx.auth.user) return signInFirst(ctx);
  if (!ctx.auth.user.isAdmin) return notFoundPage(ctx);
  return undefined;
}
/** One fact of the page: what it is and how it stands, as a `dl` row. */
const fact = (label: string, value: Html | string): Html => html`<div class="status-row"><dt>${label}</dt><dd>${value}</dd></div>`;
export function maintenancePage(ctx: Ctx): Response {
  notePhase(ctx.request, "session", takeSessionTime(ctx.request));
  const denied = admin(ctx);
  if (denied) return denied;
  const rendering = performance.now();
  const s = status(ctx.app);
  const body = html`<div class="page-head"><h1>Maintenance</h1></div>
      
        sentinelcrew
        5d ago
        outdated
      
      Principles 6 and 7: a card boxes the form. Make it rows (a dl for the facts: since, backup, working) with a hairline where it ends. Use the named form components, not a bare label and input.
    
    <p id="maintenance-says">${s.says}</p>
      
        sentinelcrew
        5d ago
        outdated
      
      Principle 1: ABOUT_TO_STOP repeats what the status line above (says) and the docs already say. One sentence on what the page is for; the stops can be a short list of rows. Also the heading has no sentence saying what the page is for.
    
    ${s.phase === "idle"
      ? html`<p class="muted">Get the forge ready for a planned power-off. What stops:</p>
          <dl class="status-rows">${ABOUT_TO_STOP_ROWS.map((r) => fact(r.what, r.stops))}</dl>
          <form method="post" action="/maintenance/prepare" class="stack">
            ${field({ name: "reason", label: "Shown to visitors (optional)", maxlength: 200, autocomplete: "off" })}
            ${field({ name: "confirm", type: "checkbox", value: "yes", required: true, label: "I understand the crew and releases stop now" })}
            <div class="form-actions">${btn("Prepare for maintenance", { variant: "danger" })}</div>
          </form>
          ${s.last ? html`<p id="maintenance-last" class="muted">${s.last.says}</p>` : null}`
      : html`<dl class="status-rows">
            ${fact("Since", s.since ?? "")} ${s.reason ? fact("Reason", s.reason) : null}
            ${fact("Backup", s.backup?.says ?? "")} ${fact("Still working", String(s.working))}
          </dl>
          <div class="form-actions">
            <form method="post" action="/maintenance/prepare">${btn("Check again and retake the backup", { variant: "outline" })}</form>
            <form method="post" action="/maintenance/resume">${btn("End maintenance now")}</form>
          </div>`}`;
  const response = page({ ctx, title: "Maintenance", body });
  notePhase(ctx.request, "render", performance.now() - rendering);
  return response;
}
export async function maintenancePrepare(ctx: Ctx): Promise<Response> {
  const denied = admin(ctx);
  if (denied) return denied;
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  // Idle -> prepared stops the crew: asked first. A re-check while prepared has already been agreed.
  if (status(ctx.app).phase === "idle" && formField(form, "confirm") !== "yes")
    return flashTo(ctx, "/maintenance", { tone: "danger", message: "Tick the box to confirm: the crew and releases stop." });
  const r = await prepare(ctx.app, ctx.auth.user, formField(form, "reason"));
  return flashTo(ctx, "/maintenance", r.ok ? { message: r.value.says } : { tone: "danger", message: r.error.says });
}
export async function maintenanceResume(ctx: Ctx): Promise<Response> {
  const denied = admin(ctx);
  if (denied) return denied;
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  const r = resumeNow(ctx.app, ctx.auth.user);
  return flashTo(ctx, "/maintenance", r.ok ? { message: r.value.says } : { tone: "danger", message: r.error.says });
}
@@ -108,6 +108,7 @@
export function resiliencePage(ctx: Ctx): Response {
      </p>
    </div>
    ${backupsLineHtml(ctx)}
    <p><a href="/maintenance">Maintenance</a> prepares the forge for a planned power-off.</p>
    ${nightlyHtml(ctx)}
    <p id="tally">
      ${String(count("green"))} green · ${String(count("red"))} red ·
@@ -123,7 +123,7 @@
describe("the one list of what asks first", () => {
    "access", "account_verify", "app_members", "app_notes", "asks_act", "asks_decide", "asks_proposal", "asks_undecide",
    "assign", "attach_look", "bet", "board_pins", "board_write", "branches", "capacity", "valves", "cloud_accounts", "close", "confirm_wait",
    "crew", "decisions_by", "detach", "edit", "email_verify", "feed_chips", "feed_dismiss", "flag_set", "flag_winner",
    "follow_mode", "fork", "front_page", "hold", "issue_teams", "keep_plan", "labels", "lift", "limb", "make_it_yours", "map", "merge", "new_app",
    "follow_mode", "fork", "front_page", "hold", "issue_teams", "keep_plan", "labels", "lift", "limb", "maintenance", "make_it_yours", "map", "merge", "new_app",
    "next_up", "week", "notify", "open_pull", "org_create", "origin_apps", "pins", "plan", "profile", "propose_to_parent_forge", "repo_labels", "reports",
    "restore_drill", "review", "say", "signup", "snapshot_now", "spend", "steer", "sync", "take_back", "take_updates",
    "teams", "tell", "threads", "trail", "upstream", "variant", "wait_for", "watch", "whoami_can", "write_file",
@@ -37,6 +37,7 @@
const NOT_A_PAGE: Record<string, string> = {
  "/opensearch.xml": "the browser search description, XML",
  "/sw.js": "the service worker script",
  "/audit.csv": "a CSV download of /audit",
  "/maintenance": "no picture: the cap of 24 looks is spent; an admin's page, covered by test/maintenance.test.ts",
  "/claim": "answers 404 to an admin: it exists only for a link-holder",
  "/first-admin": "the setup form for a forge with no admin yet",
  "/signin": "a sign-in form, not a page of the forge",
@@ -0,0 +1,49 @@
/** While the forge is in maintenance the real dispatcher starts nothing and says why. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { chmodSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createRun, getRun } from "../src/core/runs.ts";
import { createUser } from "../src/core/users.ts";
import { startCrew } from "../src/crew/dispatch.ts";
import { serve } from "../src/http/server.ts";
import { writeState } from "../src/maintenance/state.ts";
import { heldRuns } from "../src/runs/held.ts";
let dataDir: string;
let app: App;
let repoId = "";
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-maint-crew-"));
  app = createApp({ dataDir, port: 0, disk: () => ({ freeBytes: 100, totalBytes: 100 }) });
  const owner = await createUser(app.db, { origin: "seed", trusted: true, username: "holder", email: "[email protected]", password: "a-good-passphrase" });
  if (!owner.ok) throw new Error(String(owner.error));
  repoId = crypto.randomUUID();
  app.db.query("INSERT INTO repo (id, owner_id, name, visibility, created_at) VALUES (?1, ?2, 'm', 'private', ?3)").run(repoId, owner.value.id, new Date().toISOString());
});
afterAll(async () => {
  app.close();
  await rm(dataDir, { recursive: true, force: true });
});
test("queued runs stay queued, held by maintenance", async () => {
  writeState(dataDir, { current: { since: new Date().toISOString(), by: "a", reason: "", holdId: 1, backup: { ok: true, says: "x" } } });
  const run = createRun(app.db, { repoId, kind: "agent" });
  const servers = await serve(app);
  const fake = join(dataDir, "fake-claude");
  writeFileSync(fake, `#!/bin/sh\nif [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi\nexit 1\n`);
  chmodSync(fake, 0o755);
  const crew = await startCrew(app, { forgeUrl: servers.url, localUrl: servers.url, binary: fake, concurrency: 2, budgetUsd: 1, runTimeoutMs: 60_000, tickMs: 20 });
  try {
    const deadline = Date.now() + 10_000;
    while (heldRuns(app.db).get(run.id) !== "maintenance" && Date.now() < deadline) await Bun.sleep(20);
    expect(heldRuns(app.db).get(run.id)).toBe("maintenance");
    expect(getRun(app.db, run.id)?.state).toBe("queued");
  } finally {
    await crew.stop();
    servers.stop();
  }
});
@@ -0,0 +1,216 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser, type User } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { standingHolds } from "../src/keeper/hold.ts";
import { maintenanceHtml, MAINTENANCE_HEADERS } from "../src/maintenance/page.ts";
import { prepare, resume, resumeAtBoot, status } from "../src/maintenance/service.ts";
import { inMaintenance, readState, writeState } from "../src/maintenance/state.ts";
import { HOLD_WORDS } from "../src/runs/held.ts";
import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
import { dirStore } from "../src/snapshot/store.ts";
import { entry, maintenanceScreen } from "../src/tui/screens/maintenance.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
test("state: missing or damaged file is not maintenance", async () => {
  const dir = await mkdtemp(join(tmpdir(), "maint-"));
  expect(inMaintenance(dir)).toBe(false);
  await Bun.write(join(dir, "maintenance.json"), "{nope");
  expect(inMaintenance(dir)).toBe(false);
  writeState(dir, { current: { since: "2026-11-08T00:00:00Z", by: "a", reason: "", holdId: 1, backup: { ok: true, says: "x" } } });
  expect(inMaintenance(dir)).toBe(true);
  expect(readState(dir).current?.holdId).toBe(1);
  await rm(dir, { recursive: true, force: true });
});
test("page escapes the reason, refreshes, and says it is planned maintenance", () => {
  const h = maintenanceHtml({ reason: "<script>alert(1)</script>", since: "2026-11-08T00:00:00Z" });
  expect(h).not.toContain("<script>");
  expect(h).toContain("&lt;script&gt;");
  expect(h).toContain('http-equiv="refresh"');
  expect(h).toContain("down for planned maintenance and will be back soon");
  expect(h).not.toContain("new hardware");
  expect(maintenanceHtml({ since: '"><b>' })).not.toContain("<b>");
});
test("vitals names maintenance as a hold", () => {
  expect(HOLD_WORDS.maintenance).toBe("planned maintenance");
});
describe("prepare and come back", () => {
  let dataDir: string;
  let app: App;
  let servers: Servers;
  let svc: SnapshotsHandle | undefined;
  let admin: User;
  let bob: User;
  let adminRo: string;
  let adminAdmin: string;
  let bobAdmin: string;
  const login = async (username: string): Promise<string> => {
    const res = await fetch(`${servers.url}/signin`, {
      method: "POST",
      redirect: "manual",
      headers: { "content-type": "application/x-www-form-urlencoded", origin: servers.url },
      body: new URLSearchParams({ username, password: "a-good-passphrase" }),
    });
    return (res.headers.get("set-cookie") ?? "").split(";")[0] ?? "";
  };
  const post = (path: string, headers: Record<string, string>, body?: string): Promise<Response> =>
    fetch(`${servers.url}${path}`, {
      method: "POST",
      redirect: "manual",
      headers: { "content-type": "application/json", origin: servers.url, ...headers },
      body: body ?? "{}",
    });
  const form = (path: string, cookie: string, fields: Record<string, string>): Promise<Response> =>
    fetch(`${servers.url}${path}`, {
      method: "POST",
      redirect: "manual",
      headers: { "content-type": "application/x-www-form-urlencoded", origin: servers.url, cookie },
      body: new URLSearchParams(fields),
    });
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-maint-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
    const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
    if (!a.ok || !b.ok) throw new Error("user");
    admin = a.value;
    bob = b.value;
    adminRo = createToken(app.db, admin.id, "t", "ro").token;
    adminAdmin = createToken(app.db, admin.id, "t", "admin").token;
    bobAdmin = createToken(app.db, bob.id, "t", "admin").token;
  });
  afterAll(async () => {
    await svc?.stop();
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("a non-admin cannot prepare", async () => {
    const r = await prepare(app, bob, "");
    expect(r.ok).toBe(false);
    expect(inMaintenance(dataDir)).toBe(false);
  });
  test("API: a non-admin is refused, a non-admin-scope token 403, a cross-origin session 403", async () => {
    // The /api/v1/admin prefix refuses a non-admin before the handler (403); the handler's own 404 is the second lock.
    expect([403, 404]).toContain((await post("/api/v1/admin/maintenance/prepare", { cookie: await login("bob") })).status);
    expect([403, 404]).toContain((await post("/api/v1/admin/maintenance/prepare", { authorization: `Bearer ${bobAdmin}` })).status);
    expect((await post("/api/v1/admin/maintenance/prepare", { authorization: `Bearer ${adminRo}` })).status).toBe(403);
    const cookie = await login("alice");
    expect((await post("/api/v1/admin/maintenance/prepare", { cookie, origin: "https://evil.example" })).status).toBe(403);
    expect(inMaintenance(dataDir)).toBe(false);
  });
  test("API: a long reason is 422 and an oversized body is 413; neither prepares", async () => {
    const auth = { authorization: `Bearer ${adminAdmin}` };
    expect((await post("/api/v1/admin/maintenance/prepare", auth, JSON.stringify({ reason: "x".repeat(201) }))).status).toBe(422);
    expect((await post("/api/v1/admin/maintenance/prepare", auth, JSON.stringify({ reason: "x".repeat(10_000) }))).status).toBe(413);
    expect(inMaintenance(dataDir)).toBe(false);
  });
  test("API: resume while idle is 409", async () => {
    expect((await post("/api/v1/admin/maintenance/resume", { authorization: `Bearer ${adminAdmin}` })).status).toBe(409);
  });
  test("MCP: a non-admin is refused, a read-only token cannot prepare", async () => {
    const call = (token: string, args: Record<string, unknown>): Promise<Response> =>
      post("/mcp", { authorization: `Bearer ${token}`, accept: "application/json, text/event-stream" },
        JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "maintenance", arguments: args } }));
    const ro = await (await call(adminRo, { action: "prepare" })).text();
    expect(ro).toContain("forbidden");
    const nope = await (await call(bobAdmin, { action: "status" })).text();
    expect(nope).toContain("no such tool");
    expect(inMaintenance(dataDir)).toBe(false);
  });
  test("page: guest is sent to sign in, a non-admin gets not-found", async () => {
    const guest = await fetch(`${servers.url}/maintenance`, { redirect: "manual" });
    expect([302, 303, 401]).toContain(guest.status);
    const b = await fetch(`${servers.url}/maintenance`, { headers: { cookie: await login("bob") } });
    expect(b.status).toBe(404);
    expect((await form("/maintenance/prepare", await login("bob"), { confirm: "yes" })).status).not.toBe(200);
    expect(inMaintenance(dataDir)).toBe(false);
  });
  test("page: the confirm box is required to prepare", async () => {
    const cookie = await login("alice");
    const idle = await (await fetch(`${servers.url}/maintenance`, { headers: { cookie } })).text();
    expect(idle).toContain("status-rows");
    expect(idle).toContain('name="confirm"');
    await form("/maintenance/prepare", cookie, { reason: "no tick" });
    expect(inMaintenance(dataDir)).toBe(false);
  });
  test("without a backup place it holds and pauses but is not ready; the page shows facts", async () => {
    const cookie = await login("alice");
    await form("/maintenance/prepare", cookie, { reason: "moving the rack", confirm: "yes" });
    expect(inMaintenance(dataDir)).toBe(true);
    expect(standingHolds(app.db)).toHaveLength(1);
    expect(status(app).phase).toBe("draining");
    expect(status(app).says).toContain("nowhere to put a snapshot");
    const html = await (await fetch(`${servers.url}/maintenance`, { headers: { cookie } })).text();
    expect(html).toContain("moving the rack");
    expect(html).toContain("End maintenance now");
  });
  test("the terminal door: offered to admins only, and its lines say the state", () => {
    const session = (user: User) => ({ user, app, db: app.db }) as never;
    expect(entry.offered(session(admin))).toBe(true);
    expect(entry.offered(session(bob))).toBe(false);
    const screen = maintenanceScreen(session(admin), {} as never);
    expect(screen.title).toBe("Maintenance");
    expect(screen.keys.map((k) => k.help).join(" ")).toContain("end maintenance");
  });
  test("a restart of the process on the same host boot keeps what was prepared", () => {
    const boot = readState(dataDir).current?.bootId;
    resumeAtBoot(app, boot ?? "same");
    if (boot !== undefined) expect(inMaintenance(dataDir)).toBe(true);
    resumeAtBoot(app, "another-boot-id");
    expect(inMaintenance(dataDir)).toBe(false);
    expect(standingHolds(app.db)).toHaveLength(0);
    expect(status(app).last?.verified.length).toBeGreaterThan(0);
    expect(resume(app, "x").ok).toBe(false);
  });
  test("with a store it is ready once the backup is taken and nothing runs", async () => {
    svc = startSnapshots(app, dirStore(join(dataDir, "store")), { version: "t", nightly: false });
    const r = await prepare(app, admin, "");
    expect(r.ok && r.value.phase).toBe("ready");
    expect(await svc.list()).toHaveLength(1);
    resume(app, "alice");
  });
});
describe("standby", () => {
  test("answers 503 with Retry-After, and /healthz is 200", async () => {
    const port = 20000 + Math.floor(Math.random() * 20000);
    const proc = Bun.spawn(["bun", "scripts/standby.ts", String(port)], { stdout: "pipe", stderr: "pipe", env: { ...process.env, MAINTENANCE_REASON: "<b>x</b>" } });
    try {
      let res: Response | undefined;
      for (let i = 0; i < 50 && res === undefined; i++) {
        res = await fetch(`http://127.0.0.1:${port}/anything`).catch(() => undefined);
        if (res === undefined) await Bun.sleep(100);
      }
      expect(res?.status).toBe(503);
      expect(res?.headers.get("retry-after")).toBe(MAINTENANCE_HEADERS["retry-after"]!);
      expect(await res?.text()).toContain("&lt;b&gt;x&lt;/b&gt;");
      expect((await fetch(`http://127.0.0.1:${port}/healthz`)).status).toBe(200);
    } finally {
      proc.kill();
    }
  });
});
@@ -318,6 +318,7 @@
describe("the forge over MCP", () => {
      "feed",
      "feed_chips",
      "feed_dismiss",
      "maintenance",
      "make_it_yours",
      "origin_apps",
      "front_page",
@@ -99,6 +99,7 @@
test("a site-admin person is offered the whole roster", () => {
    ["telemetry", "Telemetry", "which agents may read flow and system; enter flips the grant"],
    ["capacity", "Capacity", "concurrency and guards, live; e changes a settable one"],
    ["resilience", "Resilience", "where snapshots go and the mirror; each protection green, red or untested"],
    ["maintenance", "Maintenance", "prepare the forge for a power-off, check the drain, or end maintenance"],
    ["crawlers", "Crawlers", "their share of answering time, and who was refused"],
    ["scrapes", "Scrapes", "what was read by repository, and groups in a running capture"],
    ["api", "API", "the forge's endpoints; run a GET"],
		@@ -99,6 +99,7 @@ Machine routes are not pages and have no entry: `/audit.csv`,
99	99
100	100	- `/system` — `pages/system.ts` · the machine as it is now: cores, load, memory, processes · `/flow`, `/` · none seen; updates in place over `/system/stream`, no reload script.
101	101	- `/resilience` — `pages/resilience.ts` · each protection in `docs/THREATS.md` against the last gate's tests · `/settings/resilience` (similar name, different content) · none seen.
	102	- `/maintenance` — `pages/maintenance.ts` · prepare the forge for a power-off and end it · `/resilience`, `/settings/backups` · none seen.
102	103	- `/admin/limits` — `pages/admin-limits.ts` · every limit with value, default and reason · `/settings/capacity` (also a numbers-you-can-change page) · none seen.
103	104	- `/settings/capacity` — `pages/capacity.ts` · how many runs and checks go at once · `/admin/limits`, `/settings/usage` · none seen.
104	105	- `/settings/usage` — `pages/usage.ts` · route patterns by use, unused first · `/settings/telemetry`, `/board/returns` · none seen. Not about cost, despite the name.
		@@ -0,0 +1,44 @@
	1	# Maintenance: shut down for a move and come back by itself (#9453)
	2
	3	Prepare. An admin opens `/maintenance` (linked from Resilience), the terminal's Maintenance command
	4	(`src/tui/screens/maintenance.ts`: `p` prepare, `c` check again, `e` end; admins only), or calls
	5	`POST /api/v1/admin/maintenance/prepare` (admin-scope token) or MCP
	6	`maintenance {action: "prepare"}`. The forge then:
	7
	8	1. holds releases (an owner hold, `keeper/hold.ts`);
	9	2. pauses the crew: the dispatcher starts nothing while `maintenance.json`
	10	stands in the data directory; vitals reads the queue as held by "planned maintenance";
	11	3. takes a fresh snapshot (`actions/backups_now.ts`);
	12	4. reports the drain: `GET /api/v1/admin/maintenance` says `draining` while runs
	13	or checks still go, `ready` when the backup is taken and nothing runs. Power
	14	off on `ready`; a SIGTERM drains the rest as in a release.
	15
	16	The page says plainly what stops: releases, new crew runs, then the forge, its
	17	apps and previews when the host powers off.
	18
	19	Come back. On boot `resumeAtBoot` acts only when the host has restarted since the prepare
	20	(`/proc/sys/kernel/random/boot_id` differs from the one stored); a release roll of the process on
	21	the same host boot keeps what the admin prepared. It then lifts the hold, unpauses the crew and
	22	records a line (shown on `/maintenance`, in the status, and as the
	23	`maintenance.ended` event) naming what was verified. An admin can also press
	24	"End maintenance now". The tunnel, runner and nightly timers are the forge's
	25	own boot: they start with the process, and the nightly jobs show on Resilience.
	26	No Kubernetes is asked anything; the state is one file.
	27
	28	While it is off. `bun scripts/standby.ts 8080` serves the calm 503 page
	29	(`src/maintenance/page.ts`) from the droplet; point the tunnel's or
	30	Cloudflare's origin at it for the move, and back after. A Cloudflare Worker can
	31	return the same HTML with status 503 and `Retry-After`.
	32
	33	Size of a standby. For reads or a whole forge: disk for the data (38 GB now,
	34	apps 13 GB) plus one snapshot and headroom, so 64 GB; 4 GB of memory free for the
	35	forge alone, 8 GB with apps running. The 16 GB droplet cannot hold it.
	36
	37	## Not done
	38	- No standby takeover: nothing replicates data to a second machine, so the
	39	droplet can only show the page. That needs the snapshot mirror restored onto a
	40	machine of the size above, and is a separate issue.
	41	- The page is not pushed to Cloudflare automatically; `standby.ts` is started by hand.
	42	- The admin is told on the page, event and status; no mail or push.
	43	- Where the kernel gives no boot id, any restart ends maintenance.
	44	- The return checks the database, hold, dispatcher, apps on record and nightly lines; it does not probe the tunnel or the runner.
		@@ -0,0 +1,24 @@
	1	/**
	2	* The maintenance page, served from somewhere that stays up (the droplet):
	3	* `bun scripts/standby.ts [port]` answers every request with the calm page
	4	* and a 503, so a tunnel or Cloudflare origin pointed here shows visitors
	5	* "back soon", not an error. MAINTENANCE_REASON and MAINTENANCE_SINCE are
	6	* optional. Zero dependencies; reads no forge data.
	7	*/
	8	import { MAINTENANCE_HEADERS, maintenanceHtml } from "../src/maintenance/page.ts";
	9
	10	const port = Number(process.argv[2] ?? process.env["PORT"] ?? 8080);
	11	const body = maintenanceHtml({
	12	...(process.env["MAINTENANCE_REASON"] ? { reason: process.env["MAINTENANCE_REASON"] } : {}),
	13	...(process.env["MAINTENANCE_SINCE"] ? { since: process.env["MAINTENANCE_SINCE"] } : {}),
	14	});
	15
	16	Bun.serve({
	17	port,
	18	fetch(req) {
	19	const path = new URL(req.url).pathname;
	20	if (path === "/healthz") return new Response("standby", { status: 200 });
	21	return new Response(req.method === "HEAD" ? null : body, { status: 503, headers: MAINTENANCE_HEADERS });
	22	},
	23	});
	24	console.log(`standby maintenance page on :${port}`);
		@@ -0,0 +1,9 @@
	1	import type { Route } from "../shared.ts";
	2
	3	export const route: Route = {
	4	method: "get",
	5	path: "/api/v1/admin/maintenance",
	6	summary: "Where planned maintenance stands: { phase: idle \| draining \| ready, since?, reason?, backup?, working, last?, says }; ready means releases are held, the crew is paused, a fresh backup is taken and nothing runs",
	7	auth: true,
	8	refusals: { not_found: "only a forge administrator sees this route" },
	9	};
		@@ -40,6 +40,7 @@ export { route as getApiV1AdminDrills } from "./getApiV1AdminDrills.ts";
40	40	export { route as getApiV1AdminInvites } from "./getApiV1AdminInvites.ts";
41	41	export { route as getApiV1AdminKpisProbe } from "./getApiV1AdminKpisProbe.ts";
42	42	export { route as getApiV1AdminLimits } from "./getApiV1AdminLimits.ts";
	43	export { route as getApiV1AdminMaintenance } from "./getApiV1AdminMaintenance.ts";
43	44	export { route as getApiV1AdminPeople } from "./getApiV1AdminPeople.ts";
44	45	export { route as getApiV1AdminPinned } from "./getApiV1AdminPinned.ts";
45	46	export { route as getApiV1AdminProfile } from "./getApiV1AdminProfile.ts";
		@@ -218,6 +219,8 @@ export { route as postApiV1AdminAccountsUsernameSuspend } from "./postApiV1Admin
218	219	export { route as postApiV1AdminAccountsUsernameUnsuspend } from "./postApiV1AdminAccountsUsernameUnsuspend.ts";
219	220	export { route as postApiV1AdminAccountsUsernameVerify } from "./postApiV1AdminAccountsUsernameVerify.ts";
220	221	export { route as postApiV1AdminInvites } from "./postApiV1AdminInvites.ts";
	222	export { route as postApiV1AdminMaintenancePrepare } from "./postApiV1AdminMaintenancePrepare.ts";
	223	export { route as postApiV1AdminMaintenanceResume } from "./postApiV1AdminMaintenanceResume.ts";
221	224	export { route as postApiV1AdminPinned } from "./postApiV1AdminPinned.ts";
222	225	export { route as postApiV1AdminPinnedNameCheck } from "./postApiV1AdminPinnedNameCheck.ts";
223	226	export { route as postApiV1AdminPinnedNameRollback } from "./postApiV1AdminPinnedNameRollback.ts";
		@@ -0,0 +1,10 @@
	1	import type { Route } from "../shared.ts";
	2
	3	export const route: Route = {
	4	method: "post",
	5	path: "/api/v1/admin/maintenance/prepare",
	6	summary: "Prepare the forge for a power-off: hold releases, pause the crew, take a fresh backup, report the drain; needs an admin-scope token. Answers the status as GET does",
	7	auth: true,
	8	body: { reason: "string" },
	9	refusals: { not_found: "only a forge administrator may", invalid: "the reason is at most 200 characters (422)" },
	10	};
		@@ -192,6 +192,8 @@ export function classifiedAudience(kind: string): EventAudience \| undefined {
192	192	kind.startsWith("drill.") \|\|
193	193	// A snapshot moving is the forge owner's too, and has no repository.
194	194	kind.startsWith("backup.") \|\|
	195	// Preparing for a power-off and ending it are the forge owner's (`maintenance/service.ts`).
	196	kind.startsWith("maintenance.") \|\|
195	197	// The weekly model review's changes and results are the owner's (`actions/model_review.ts`).
196	198	kind.startsWith("crew.model_") \|\|
197	199	// A child forge answering no one at its public address is the owner's to hear (`spawn/reach.ts`).
		@@ -96,6 +96,7 @@ export const SYSTEM_NAMES: readonly string[] = [
96	96	"organizations",
97	97	"now",
98	98	"resilience",
	99	"maintenance",
99	100	"origins",
100	101	"robots.txt",
101	102	"favicon.ico",
		@@ -48,6 +48,7 @@ import { noteHeld, type HoldCause } from "../runs/held.ts";
48	48	import { decideAndRecord } from "./ledger.ts";
49	49	import { bugsFirst } from "../flow/bugs.ts";
50	50	import { byDials } from "./team_share.ts";
	51	import { inMaintenance } from "../maintenance/state.ts";
51	52	import { gateQueue } from "../flow/valve_gate.ts";
52	53	import { ARCHIVED_WHY, getRepoById, type Repo } from "../core/repos.ts";
53	54	import { listReviews } from "../core/reviews.ts";
		@@ -1527,6 +1528,13 @@ function pump(
1527	1528	const gated = startOrder(app);
1528	1529	// What this pass holds by policy, for vitals (`runs/held.ts`, #9403).
1529	1530	const held = new Map<string, HoldCause>();
	1531	// Planned maintenance (`maintenance/service.ts`): nothing new starts, and
	1532	// a queued run waits as it does behind a shut valve.
	1533	if (inMaintenance(app.config.dataDir)) {
	1534	for (const q of [...gated.order, ...gated.held]) held.set(q.id, "maintenance");
	1535	noteHeld(app.db, held);
	1536	return;
	1537	}
1530	1538	for (const shut of gated.held) {
1531	1539	held.set(shut.id, "valve");
1532	1540	if (noted.has(shut.id)) continue;
		@@ -349,6 +349,7 @@ import { labelsRoutes } from "./api/labels.ts";
349	349	import { visionRoutes } from "./api/vision.ts";
350	350	import { tokenRoutes } from "./api/tokens.ts";
351	351	import { accountAdmin, adminRoutes } from "./api/admin.ts";
	352	import { maintenanceRoutes } from "./api/maintenance.ts";
352	353	import { flagsRoutes } from "./api/flags.ts";
353	354	import { issueRoutes, labelsHandler } from "./api/issues.ts";
354	355	import { FORGE_NOT_CARRIED } from "../origins/words.ts";
		@@ -1226,6 +1227,7 @@ export function apiRoutes(router: Router<Handler>): void {
1226	1227	userRoutes(router);
1227	1228	forgeOriginRoutes(router);
1228	1229	adminRoutes(router);
	1230	maintenanceRoutes(router);
1229	1231	flagsRoutes(router);
1230	1232
1231	1233	// Open read hooks (#1789): the owner, or an agent holding `telemetry:read`.