Public: anyone can read this; the crew can change it.

Maintenance from the forge: request a shutdown, keep a page up, and come back by itself #9557

Parked opened this in answer to
Closed without merging

scout/forge:scout/9453-maintenance-from-the-forge-request-a-shu → main 3 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
27 files changed +628 −2
docs/design/audit.md +1
@@ -99,6 +99,7 @@ Machine routes are not pages and have no entry: `/audit.csv`,
9999
100100 - `/system` — `pages/system.ts` · the machine as it is now: cores, load, memory, processes · `/flow`, `/` · none seen; updates in place over `/system/stream`, no reload script.
101101 - `/resilience` — `pages/resilience.ts` · each protection in `docs/THREATS.md` against the last gate's tests · `/settings/resilience` (similar name, different content) · none seen.
102 - `/maintenance` — `pages/maintenance.ts` · prepare the forge for a power-off and end it · `/resilience`, `/settings/backups` · none seen.
102103 - `/admin/limits` — `pages/admin-limits.ts` · every limit with value, default and reason · `/settings/capacity` (also a numbers-you-can-change page) · none seen.
103104 - `/settings/capacity` — `pages/capacity.ts` · how many runs and checks go at once · `/admin/limits`, `/settings/usage` · none seen.
104105 - `/settings/usage` — `pages/usage.ts` · route patterns by use, unused first · `/settings/telemetry`, `/board/returns` · none seen. Not about cost, despite the name.
docs/features/maintenance.md added +40
@@ -0,0 +1,40 @@
1 # Maintenance: shut down for a move and come back by itself (#9453)
2
3 **Prepare.** An admin opens `/maintenance` (linked from Resilience), or calls
4 `POST /api/v1/admin/maintenance/prepare` (admin-scope token) or MCP
5 `maintenance {action: "prepare"}`. The forge then:
6
7 1. holds releases (an owner hold, `keeper/hold.ts`);
8 2. pauses the crew: the dispatcher starts nothing while `maintenance.json`
9 stands in the data directory; vitals reads the queue as held by "planned maintenance";
10 3. takes a fresh snapshot (`actions/backups_now.ts`);
11 4. reports the drain: `GET /api/v1/admin/maintenance` says `draining` while runs
12 or checks still go, `ready` when the backup is taken and nothing runs. Power
13 off on `ready`; a SIGTERM drains the rest as in a release.
14
15 The page says plainly what stops: releases, new crew runs, then the forge, its
16 apps and previews when the host powers off.
17
18 **Come back.** On boot `resumeAtBoot` lifts the hold, unpauses the crew and
19 records a line (shown on `/maintenance`, in the status, and as the
20 `maintenance.ended` event) naming what was verified. An admin can also press
21 "End maintenance now". The tunnel, runner and nightly timers are the forge's
22 own boot: they start with the process, and the nightly jobs show on Resilience.
23 No Kubernetes is asked anything; the state is one file.
24
25 **While it is off.** `bun scripts/standby.ts 8080` serves the calm 503 page
26 (`src/maintenance/page.ts`) from the droplet; point the tunnel's or
27 Cloudflare's origin at it for the move, and back after. A Cloudflare Worker can
28 return the same HTML with status 503 and `Retry-After`.
29
30 **Size of a standby.** For reads or a whole forge: disk for the data (38 GB now,
31 apps 13 GB) plus one snapshot and headroom, so 64 GB; 4 GB of memory free for the
32 forge alone, 8 GB with apps running. The 16 GB droplet cannot hold it.
33
34 ## Not done
35 - No standby takeover: nothing replicates data to a second machine, so the
36 droplet can only show the page. That needs the snapshot mirror restored onto a
37 machine of the size above, and is a separate issue.
38 - The page is not pushed to Cloudflare automatically; `standby.ts` is started by hand.
39 - The admin is told on the page, event and status; no mail or push.
40 - A restart of the forge while prepared (not a power-off) also ends maintenance.
scripts/standby.ts added +24
@@ -0,0 +1,24 @@
1 /**
2 * The maintenance page, served from somewhere that stays up (the droplet):
3 * `bun scripts/standby.ts [port]` answers every request with the calm page
4 * and a 503, so a tunnel or Cloudflare origin pointed here shows visitors
5 * "back soon", not an error. MAINTENANCE_REASON and MAINTENANCE_SINCE are
6 * optional. Zero dependencies; reads no forge data.
7 */
8 import { MAINTENANCE_HEADERS, maintenanceHtml } from "../src/maintenance/page.ts";
9
10 const port = Number(process.argv[2] ?? process.env["PORT"] ?? 8080);
11 const body = maintenanceHtml({
12 ...(process.env["MAINTENANCE_REASON"] ? { reason: process.env["MAINTENANCE_REASON"] } : {}),
13 ...(process.env["MAINTENANCE_SINCE"] ? { since: process.env["MAINTENANCE_SINCE"] } : {}),
14 });
15
16 Bun.serve({
17 port,
18 fetch(req) {
19 const path = new URL(req.url).pathname;
20 if (path === "/healthz") return new Response("standby", { status: 200 });
21 return new Response(req.method === "HEAD" ? null : body, { status: 503, headers: MAINTENANCE_HEADERS });
22 },
23 });
24 console.log(`standby maintenance page on :${port}`);
src/contract/openapi/routes/getApiV1AdminMaintenance.ts added +9
@@ -0,0 +1,9 @@
1 import type { Route } from "../shared.ts";
2
3 export const route: Route = {
4 method: "get",
5 path: "/api/v1/admin/maintenance",
6 summary: "Where planned maintenance stands: { phase: idle | draining | ready, since?, reason?, backup?, working, last?, says }; ready means releases are held, the crew is paused, a fresh backup is taken and nothing runs",
7 auth: true,
8 refusals: { not_found: "only a forge administrator sees this route" },
9 };
src/contract/openapi/routes/index.ts +3
@@ -40,6 +40,7 @@ export { route as getApiV1AdminDrills } from "./getApiV1AdminDrills.ts";
4040 export { route as getApiV1AdminInvites } from "./getApiV1AdminInvites.ts";
4141 export { route as getApiV1AdminKpisProbe } from "./getApiV1AdminKpisProbe.ts";
4242 export { route as getApiV1AdminLimits } from "./getApiV1AdminLimits.ts";
43 export { route as getApiV1AdminMaintenance } from "./getApiV1AdminMaintenance.ts";
4344 export { route as getApiV1AdminPeople } from "./getApiV1AdminPeople.ts";
4445 export { route as getApiV1AdminPinned } from "./getApiV1AdminPinned.ts";
4546 export { route as getApiV1AdminProfile } from "./getApiV1AdminProfile.ts";
@@ -218,6 +219,8 @@ export { route as postApiV1AdminAccountsUsernameSuspend } from "./postApiV1Admin
218219 export { route as postApiV1AdminAccountsUsernameUnsuspend } from "./postApiV1AdminAccountsUsernameUnsuspend.ts";
219220 export { route as postApiV1AdminAccountsUsernameVerify } from "./postApiV1AdminAccountsUsernameVerify.ts";
220221 export { route as postApiV1AdminInvites } from "./postApiV1AdminInvites.ts";
222 export { route as postApiV1AdminMaintenancePrepare } from "./postApiV1AdminMaintenancePrepare.ts";
223 export { route as postApiV1AdminMaintenanceResume } from "./postApiV1AdminMaintenanceResume.ts";
221224 export { route as postApiV1AdminPinned } from "./postApiV1AdminPinned.ts";
222225 export { route as postApiV1AdminPinnedNameCheck } from "./postApiV1AdminPinnedNameCheck.ts";
223226 export { route as postApiV1AdminPinnedNameRollback } from "./postApiV1AdminPinnedNameRollback.ts";
src/contract/openapi/routes/postApiV1AdminMaintenancePrepare.ts added +10
@@ -0,0 +1,10 @@
1 import type { Route } from "../shared.ts";
2
3 export const route: Route = {
4 method: "post",
5 path: "/api/v1/admin/maintenance/prepare",
6 summary: "Prepare the forge for a power-off: hold releases, pause the crew, take a fresh backup, report the drain; needs an admin-scope token. Answers the status as GET does",
7 auth: true,
8 body: { reason: "string" },
9 refusals: { not_found: "only a forge administrator may", invalid: "the reason is at most 200 characters (422)" },
10 };
src/contract/openapi/routes/postApiV1AdminMaintenanceResume.ts added +9
@@ -0,0 +1,9 @@
1 import type { Route } from "../shared.ts";
2
3 export const route: Route = {
4 method: "post",
5 path: "/api/v1/admin/maintenance/resume",
6 summary: "End maintenance now: lift the release hold and let the crew start; needs an admin-scope token. The forge does this by itself at boot",
7 auth: true,
8 refusals: { not_found: "only a forge administrator may", conflict: "the forge is not in maintenance (409)" },
9 };
src/core/event_audience.ts +2
@@ -189,6 +189,8 @@ export function classifiedAudience(kind: string): EventAudience | undefined {
189189 kind.startsWith("drill.") ||
190190 // A snapshot moving is the forge owner's too, and has no repository.
191191 kind.startsWith("backup.") ||
192 // Preparing for a power-off and ending it are the forge owner's (`maintenance/service.ts`).
193 kind.startsWith("maintenance.") ||
192194 // The weekly model review's changes and results are the owner's (`actions/model_review.ts`).
193195 kind.startsWith("crew.model_") ||
194196 // A child forge answering no one at its public address is the owner's to hear (`spawn/reach.ts`).
src/core/reserved.ts +1
@@ -96,6 +96,7 @@ export const SYSTEM_NAMES: readonly string[] = [
9696 "organizations",
9797 "now",
9898 "resilience",
99 "maintenance",
99100 "origins",
100101 "robots.txt",
101102 "favicon.ico",
src/crew/dispatch.ts +8
@@ -47,6 +47,7 @@ import { noteHeld, type HoldCause } from "../runs/held.ts";
4747 import { decideAndRecord } from "./ledger.ts";
4848 import { bugsFirst } from "../flow/bugs.ts";
4949 import { byDials } from "./team_share.ts";
50 import { inMaintenance } from "../maintenance/state.ts";
5051 import { gateQueue } from "../flow/valve_gate.ts";
5152 import { ARCHIVED_WHY, getRepoById, type Repo } from "../core/repos.ts";
5253 import { listReviews } from "../core/reviews.ts";
@@ -1514,6 +1515,13 @@ function pump(
15141515 const gated = startOrder(app);
15151516 // What this pass holds by policy, for vitals (`runs/held.ts`, #9403).
15161517 const held = new Map<string, HoldCause>();
1518 // Planned maintenance (`maintenance/service.ts`): nothing new starts, and
1519 // a queued run waits as it does behind a shut valve.
1520 if (inMaintenance(app.config.dataDir)) {
1521 for (const q of [...gated.order, ...gated.held]) held.set(q.id, "maintenance");
1522 noteHeld(app.db, held);
1523 return;
1524 }
15171525 for (const shut of gated.held) {
15181526 held.set(shut.id, "valve");
15191527 if (noted.has(shut.id)) continue;
src/http/api.ts +2
@@ -347,6 +347,7 @@ import { labelsRoutes } from "./api/labels.ts";
347347 import { visionRoutes } from "./api/vision.ts";
348348 import { tokenRoutes } from "./api/tokens.ts";
349349 import { accountAdmin, adminRoutes } from "./api/admin.ts";
350 import { maintenanceRoutes } from "./api/maintenance.ts";
350351 import { flagsRoutes } from "./api/flags.ts";
351352 import { issueRoutes, labelsHandler } from "./api/issues.ts";
352353 import { FORGE_NOT_CARRIED } from "../origins/words.ts";
@@ -1224,6 +1225,7 @@ export function apiRoutes(router: Router<Handler>): void {
12241225 userRoutes(router);
12251226 forgeOriginRoutes(router);
12261227 adminRoutes(router);
1228 maintenanceRoutes(router);
12271229 flagsRoutes(router);
12281230
12291231 // Open read hooks (#1789): the owner, or an agent holding `telemetry:read`.
src/http/api/maintenance.ts added +38
@@ -0,0 +1,38 @@
1 /** `/api/v1/admin/maintenance`: what the Maintenance page and MCP `maintenance` do, for a forge admin's admin-scope credential. */
2
3 import { prepare, resumeNow, status } from "../../maintenance/service.ts";
4 import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
5 import { crossOrigin, type Handler } from "../context.ts";
6 import { apiError, forbidden, json, notFound, requireScope, unauthorized } from "../respond.ts";
7 import type { Router } from "../router.ts";
8
9 export function maintenanceRoutes(router: Router<Handler>): void {
10 router.add("GET", "/api/v1/admin/maintenance", (ctx) => {
11 notePhase(ctx.request, "session", takeSessionTime(ctx.request));
12 if (!ctx.auth.user) return unauthorized();
13 if (!ctx.auth.user.isAdmin) return notFound();
14 const denied = requireScope(ctx, "ro");
15 if (denied) return denied;
16 return json(status(ctx.app));
17 });
18 router.add("POST", "/api/v1/admin/maintenance/prepare", async (ctx) => {
19 if (!ctx.auth.user) return unauthorized();
20 if (!ctx.auth.user.isAdmin) return notFound();
21 const denied = requireScope(ctx, "admin");
22 if (denied) return denied;
23 if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden();
24 const input: unknown = await ctx.request.json().catch(() => ({}));
25 const reason = typeof input === "object" && input !== null && typeof (input as { reason?: unknown }).reason === "string" ? (input as { reason: string }).reason : "";
26 const r = await prepare(ctx.app, ctx.auth.user, reason);
27 return r.ok ? json(r.value) : apiError(r.error.code === "invalid" ? "invalid" : "not_found", r.error.code === "invalid" ? 422 : 404, r.error.says);
28 });
29 router.add("POST", "/api/v1/admin/maintenance/resume", (ctx) => {
30 if (!ctx.auth.user) return unauthorized();
31 if (!ctx.auth.user.isAdmin) return notFound();
32 const denied = requireScope(ctx, "admin");
33 if (denied) return denied;
34 if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden();
35 const r = resumeNow(ctx.app, ctx.auth.user);
36 return r.ok ? json(r.value) : apiError(r.error.code, r.error.code === "conflict" ? 409 : 404, r.error.says);
37 });
38 }
src/http/mcp/tools/index.ts +1
@@ -86,6 +86,7 @@ export { tool as land_past_gate } from "./land_past_gate.ts";
8686 export { tool as lift } from "./lift.ts";
8787 export { tool as limb } from "./limb.ts";
8888 export { tool as limits } from "./limits.ts";
89 export { tool as maintenance } from "./maintenance.ts";
8990 export { tool as make_it_yours } from "./make_it_yours.ts";
9091 export { tool as map } from "./map.ts";
9192 export { tool as merge } from "./merge.ts";
src/http/mcp/tools/maintenance.ts added +28
@@ -0,0 +1,28 @@
1 import { prepare, resumeNow, status } from "../../../maintenance/service.ts";
2 import { scopeAllows } from "../../../core/tokens.ts";
3 import { good, bad, type Tool } from "../shared.ts";
4
5 export const tool: Tool = {
6 name: "maintenance",
7 doc: "`action` is `status` (default), `prepare` or `resume`; `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says }`",
8 description: "Prepare the forge for a power-off, check the drain, or end maintenance (admin).",
9 inputSchema: {
10 type: "object",
11 properties: { action: { type: "string", enum: ["status", "prepare", "resume"] }, reason: { type: "string", maxLength: 200 } },
12 },
13 writes: true,
14 async run(ctx, user, args) {
15 if (!user.isAdmin) return bad("not_found", "no such tool here");
16 const action = args["action"] ?? "status";
17 if (action === "status") return good(status(ctx.app));
18 if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare or resume");
19 if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token");
20 if (action === "resume") {
21 const r = resumeNow(ctx.app, user);
22 return r.ok ? good(r.value) : bad(r.error.code, r.error.says);
23 }
24 const reason = typeof args["reason"] === "string" ? args["reason"] : "";
25 const r = await prepare(ctx.app, user, reason);
26 return r.ok ? good(r.value) : bad(r.error.code, r.error.says);
27 },
28 };
src/http/server.ts +4
@@ -25,6 +25,7 @@ import { prerenderComments } from "../web/prerender.ts";
2525 import { systemPage, systemStream } from "../web/pages/system.ts";
2626 import { contributePage } from "../web/pages/contribute.ts";
2727 import { pipelinePage } from "../web/pages/pipeline.ts";
28 import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts";
2829 import { nightlyOutputPage, resiliencePage } from "../web/pages/resilience.ts";
2930 import { limitsPage, limitsSet, quotasSet, signupSet, inviteMint, inviteRevoke } from "../web/pages/admin-limits.ts";
3031 import { measureSubmit } from "./measure.ts";
@@ -889,6 +890,9 @@ export function buildRouter(): Router<Handler> {
889890 pinnedRoutes(router);
890891 // Each THREATS.md protection against the last gate; literal, above /:owner.
891892 router.add("GET", "/resilience", resiliencePage);
893 router.add("GET", "/maintenance", maintenancePage);
894 router.add("POST", "/maintenance/prepare", maintenancePrepare);
895 router.add("POST", "/maintenance/resume", maintenanceResume);
892896 router.add("GET", "/resilience/nightly/:job", nightlyOutputPage);
893897 // A selection's spend and runs over time; literal, above /:owner.
894898 router.add("GET", "/fleet/graph", fleetGraphPage);
src/main.ts +3
@@ -70,6 +70,7 @@ import { peopleOnCrews } from "./core/agents.ts";
7070 import { startNotifier } from "./crew/notify.ts";
7171 import { expireCrewHolds } from "./actions/holds.ts";
7272 import { startClogTick } from "./flow/clog_tick.ts";
73 import { resumeAtBoot } from "./maintenance/service.ts";
7374 import { startKeeper } from "./flow/keeper.ts";
7475 import { sweepStaleAsks } from "./crew/stale_asks.ts";
7576 import { recordAskLedger } from "./web/studio/asks.ts";
@@ -671,6 +672,8 @@ const processTick = (): void => {
671672 const stopClogTick = startClogTick(app.db, app.bus, (error) => app.log.warn("clog tick failed", { error: String(error) }), 60_000);
672673 // The throughput keeper (#7645) names what binds landings each minute and lifts what is its to lift.
673674 const stopKeeper = startKeeper(app, () => sweepAssigned(app));
675 // A forge that went down prepared for maintenance comes back by itself.
676 resumeAtBoot(app);
674677 // An agent's hold that has stood 24 h is released each minute (#6918).
675678 const holdTimer = setInterval(() => {
676679 try {
src/maintenance/page.ts added +31
@@ -0,0 +1,31 @@
1 /**
2 * The page visitors see while the forge is off: calm, static, escaped, and
3 * answerable by anything (the droplet's `scripts/standby.ts`, or a Cloudflare
4 * Worker serving the same text). No script, no external asset.
5 */
6
7 const esc = (s: string): string =>
8 s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
9
10 export interface PageInput {
11 /** What the owner said, one line; may be empty. */
12 readonly reason?: string;
13 /** ISO time the forge went into maintenance, when known. */
14 readonly since?: string;
15 }
16
17 export function maintenanceHtml(input: PageInput = {}): string {
18 const reason = (input.reason ?? "").replace(/[\u0000-\u001f\u007f]+/g, " ").trim().slice(0, 200);
19 const since = input.since !== undefined && /^[0-9T:.\-Z+]{10,30}$/.test(input.since) ? input.since : "";
20 return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta http-equiv="refresh" content="60"><title>Back soon</title><style>body{font:16px/1.5 system-ui,sans-serif;margin:0;display:grid;min-height:100vh;place-items:center;background:#fafafa;color:#222}main{max-width:32rem;padding:1.5rem}h1{font-size:1.5rem;margin:0 0 .5rem}p{margin:.5rem 0}.m{color:#666}@media(prefers-color-scheme:dark){body{background:#151515;color:#eee}.m{color:#aaa}}</style></head><body><main><h1>Back soon</h1><p>The forge is being moved to new hardware. Nothing is lost: everything was backed up first, and it comes back by itself.</p>${reason === "" ? "" : `<p>${esc(reason)}</p>`}${since === "" ? "" : `<p class="m">Down since ${esc(since)}.</p>`}<p class="m">This page refreshes itself every minute.</p></main></body></html>`;
21 }
22
23 /** The headers the maintenance page is always served with: 503 so nothing caches it as the real site. */
24 export const MAINTENANCE_HEADERS: Readonly<Record<string, string>> = {
25 "content-type": "text/html; charset=utf-8",
26 "retry-after": "300",
27 "cache-control": "no-store",
28 "content-security-policy": "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'",
29 "x-content-type-options": "nosniff",
30 "referrer-policy": "no-referrer",
31 };
src/maintenance/service.ts added +126
@@ -0,0 +1,126 @@
1 /**
2 * Planned maintenance: an admin asks the forge to get ready for a power-off
3 * (`prepare`), and on the next boot it comes back by itself (`resume`).
4 *
5 * prepare: hold releases (an owner hold, `keeper/hold.ts`), pause the crew
6 * (the dispatcher starts nothing while the state file stands), take a fresh
7 * snapshot, then report what still runs. The host may power off when
8 * `status().ready` is true; a SIGTERM drains the rest as it does for a roll.
9 * resume: lift the hold, let the crew start again, verify, and keep a line
10 * saying what is healthy. No Kubernetes is asked anything.
11 */
12
13 import type { App } from "../app.ts";
14 import { listActive } from "../core/runs.ts";
15 import type { User } from "../core/users.ts";
16 import { lift, MAX_REASON, setHold } from "../keeper/hold.ts";
17 import { err, ok, type Result } from "../kernel/result.ts";
18 import { running } from "../kernel/version.ts";
19 import { snapshotNow } from "../actions/backups_now.ts";
20 import { readState, writeState, type Maintenance, type Returned } from "./state.ts";
21
22 const isAdmin = (u: User | undefined): u is User => u?.isAdmin === true && u.kind !== "agent";
23
24 export interface Status {
25 readonly phase: "idle" | "draining" | "ready";
26 readonly since?: string;
27 readonly reason?: string;
28 readonly backup?: Maintenance["backup"];
29 /** Crew runs and checks still queued or running. */
30 readonly working: number;
31 readonly last?: Returned;
32 /** Plainly: what stops, or what is about to. */
33 readonly says: string;
34 }
35
36 export const ABOUT_TO_STOP =
37 "Preparing stops: releases (none are cut), the crew (no run starts; running ones finish or are handed back queued), and then the forge itself when the host powers off. Apps and previews stop with it. Visitors see a maintenance page if one is served elsewhere (docs/features/maintenance.md).";
38
39 export function status(app: App): Status {
40 const { current, last } = readState(app.config.dataDir);
41 if (current === undefined)
42 return { phase: "idle", working: 0, ...(last ? { last } : {}), says: last ? last.says : "Not in maintenance." };
43 const working = listActive(app.db, "agent").length + listActive(app.db, "check").filter((r) => r.state === "running").length;
44 const ready = current.backup.ok && working === 0;
45 return {
46 phase: ready ? "ready" : "draining",
47 since: current.since,
48 reason: current.reason,
49 backup: current.backup,
50 working,
51 says: ready
52 ? "Ready: releases are held, the crew is paused, a fresh backup is taken and nothing is running. The host can power off."
53 : !current.backup.ok
54 ? `Not ready: ${current.backup.says}`
55 : `Draining: ${working} crew run${working === 1 ? "" : "s"} or check${working === 1 ? "" : "s"} still going; the host can power off when it reads 0.`,
56 };
57 }
58
59 export type Refusal = { readonly code: "not_found" | "invalid" | "conflict"; readonly says: string };
60
61 export async function prepare(app: App, actor: User | undefined, reasonIn: string): Promise<Result<Status, Refusal>> {
62 if (!isAdmin(actor)) return err({ code: "not_found", says: "Only a forge admin prepares maintenance." });
63 const reason = reasonIn.replace(/[\u0000-\u001f\u007f]+/g, " ").trim();
64 if (reason.length > 200) return err({ code: "invalid", says: "The reason is at most 200 characters." });
65 const dataDir = app.config.dataDir;
66 const prior = readState(dataDir);
67 if (prior.current !== undefined) {
68 // Asking again retakes the backup and reads the drain; the hold stands once.
69 const taken = await snapshotNow(app, actor);
70 writeState(dataDir, { ...prior, current: { ...prior.current, backup: { ok: taken.ok, says: taken.ok ? taken.value.says : taken.error.says } } });
71 return ok(status(app));
72 }
73 const hold = setHold(app.db, {
74 kind: "owner",
75 version: running().version,
76 reason: `maintenance${reason === "" ? "" : `: ${reason}`}`.slice(0, MAX_REASON),
77 at: Date.now(),
78 });
79 const now = new Date().toISOString();
80 // Written before the backup: the crew pauses at once, and a crash mid-backup still resumes at boot.
81 const base: Maintenance = { since: now, by: actor.username, reason, holdId: hold.id, backup: { ok: false, says: "the backup has not finished" } };
82 writeState(dataDir, { ...(prior.last ? { last: prior.last } : {}), current: base });
83 app.bus.append({ kind: "maintenance.prepared", actorId: actor.id, payload: { hold: hold.id, reason } });
84 const taken = await snapshotNow(app, actor);
85 writeState(dataDir, {
86 ...(prior.last ? { last: prior.last } : {}),
87 current: { ...base, backup: { ok: taken.ok, says: taken.ok ? taken.value.says : taken.error.says } },
88 });
89 return ok(status(app));
90 }
91
92 /** Come back: lift our hold, start the crew, and record what was checked. `by` names who or what did it. */
93 export function resume(app: App, by: string): Result<Returned, Refusal> {
94 const dataDir = app.config.dataDir;
95 const { current } = readState(dataDir);
96 if (current === undefined) return err({ code: "conflict", says: "The forge is not in maintenance." });
97 const now = Date.now();
98 const lifted = lift(app.db, current.holdId, by, now);
99 const verified = [
100 `database open, version ${running().version}`,
101 lifted ? "release hold lifted" : "release hold was already lifted",
102 "crew dispatcher unpaused",
103 `${listActive(app.db, "agent").length} crew run(s) and ${listActive(app.db, "check").length} check(s) waiting to start`,
104 ];
105 const back: Returned = {
106 at: new Date(now).toISOString(),
107 downSince: current.since,
108 says: `Back at ${new Date(now).toISOString()} after maintenance begun ${current.since}: ${verified.join("; ")}.`,
109 verified,
110 };
111 writeState(dataDir, { last: back });
112 app.bus.append({ kind: "maintenance.ended", payload: { by, hold: current.holdId, verified } });
113 return ok(back);
114 }
115
116 /** At boot: a forge that went down prepared comes back by itself. */
117 export function resumeAtBoot(app: App): void {
118 if (readState(app.config.dataDir).current === undefined) return;
119 const r = resume(app, "boot");
120 if (r.ok) app.log.info("maintenance ended at boot", { says: r.value.says });
121 }
122
123 export function resumeNow(app: App, actor: User | undefined): Result<Returned, Refusal> {
124 if (!isAdmin(actor)) return err({ code: "not_found", says: "Only a forge admin ends maintenance." });
125 return resume(app, actor.username);
126 }
src/maintenance/state.ts added +72
@@ -0,0 +1,72 @@
1 /**
2 * The one file of maintenance state, `maintenance.json` under the data
3 * directory: it survives the power-off because the data does. Plain data in,
4 * plain data out; a missing or unreadable file reads as "not in maintenance",
5 * so a damaged file can never keep the crew paused.
6 */
7
8 import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
9 import { join } from "node:path";
10
11 export interface Backup {
12 readonly ok: boolean;
13 readonly says: string;
14 }
15
16 export interface Maintenance {
17 readonly since: string;
18 readonly by: string;
19 /** One line the maintenance page shows, 0 to 200 characters. */
20 readonly reason: string;
21 /** The release hold this request placed (`keeper/hold.ts`); lifted when the forge returns. */
22 readonly holdId: number;
23 readonly backup: Backup;
24 }
25
26 export interface Returned {
27 readonly at: string;
28 readonly downSince: string;
29 readonly says: string;
30 readonly verified: readonly string[];
31 }
32
33 export interface Stored {
34 readonly current?: Maintenance;
35 /** The last return, kept so an admin is told when it is back and what is healthy. */
36 readonly last?: Returned;
37 }
38
39 const file = (dataDir: string): string => join(dataDir, "maintenance.json");
40
41 export function readState(dataDir: string): Stored {
42 try {
43 if (!existsSync(file(dataDir))) return {};
44 const raw: unknown = JSON.parse(readFileSync(file(dataDir), "utf8"));
45 if (typeof raw !== "object" || raw === null) return {};
46 const o = raw as Stored;
47 const cur = o.current;
48 const good =
49 cur !== undefined &&
50 typeof cur.since === "string" &&
51 typeof cur.holdId === "number" &&
52 typeof cur.backup?.ok === "boolean";
53 return { ...(good ? { current: cur } : {}), ...(o.last !== undefined && typeof o.last.at === "string" ? { last: o.last } : {}) };
54 } catch {
55 return {};
56 }
57 }
58
59 export function writeState(dataDir: string, state: Stored): void {
60 const tmp = `${file(dataDir)}.tmp`;
61 writeFileSync(tmp, JSON.stringify(state), { mode: 0o600 });
62 renameSync(tmp, file(dataDir));
63 }
64
65 export function clearState(dataDir: string): void {
66 rmSync(file(dataDir), { force: true });
67 }
68
69 /** Is the forge in maintenance? The dispatcher asks on every pass. */
70 export function inMaintenance(dataDir: string): boolean {
71 return readState(dataDir).current !== undefined;
72 }
src/runs/held.ts +2 −1
@@ -17,13 +17,14 @@ import type { Database } from "bun:sqlite";
1717 export const HELD_FRESH_MS = 2 * 60_000;
1818
1919 /** Why the dispatcher held a run. */
20 export type HoldCause = "valve" | "pace" | "limit";
20 export type HoldCause = "valve" | "pace" | "limit" | "maintenance";
2121
2222 /** How vitals names each cause: "2 by a shut valve". */
2323 export const HOLD_WORDS: Readonly<Record<HoldCause, string>> = {
2424 valve: "a shut valve",
2525 pace: "the week's pace",
2626 limit: "an account limit",
27 maintenance: "planned maintenance",
2728 };
2829
2930 interface Pass {
src/web/pages/maintenance.ts added +66
@@ -0,0 +1,66 @@
1 /**
2 * /maintenance: an admin's page to prepare the forge for a power-off and to
3 * end it. It says plainly what stops, what the drain still waits for, and,
4 * once the forge is back, when and what was verified.
5 */
6
7 import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
8 import { ABOUT_TO_STOP, prepare, resumeNow, status } from "../../maintenance/service.ts";
9 import type { Ctx } from "../../http/context.ts";
10 import { notFoundPage, signInFirst } from "../chrome.ts";
11 import { field, readForm } from "../form.ts";
12 import { html } from "../html.ts";
13 import { flashTo, page } from "../layout.ts";
14 import { btn, card } from "../ui.ts";
15
16 function admin(ctx: Ctx): Response | undefined {
17 if (!ctx.auth.user) return signInFirst(ctx);
18 if (!ctx.auth.user.isAdmin) return notFoundPage(ctx);
19 return undefined;
20 }
21
22 export function maintenancePage(ctx: Ctx): Response {
23 notePhase(ctx.request, "session", takeSessionTime(ctx.request));
24 const denied = admin(ctx);
25 if (denied) return denied;
26 const rendering = performance.now();
27 const s = status(ctx.app);
28 const body = html`<div class="page-head"><h1>Maintenance</h1></div>
29 <p id="maintenance-says">${s.says}</p>
30 ${s.phase === "idle"
31 ? card(html`<form method="post" action="/maintenance/prepare">
32 <p>${ABOUT_TO_STOP}</p>
33 <label for="reason">Shown to visitors (optional)</label>
34 <input id="reason" name="reason" maxlength="200" autocomplete="off" />
35 <label><input type="checkbox" name="confirm" value="yes" required /> I understand the crew and releases stop now</label>
36 ${btn("Prepare for maintenance", { variant: "danger" })}
37 </form>
38 ${s.last ? html`<p id="maintenance-last" class="muted">${s.last.says}</p>` : null}`)
39 : card(html`<p>Since ${s.since}. Backup: ${s.backup?.says}</p>
40 <form method="post" action="/maintenance/prepare">${btn("Check again and retake the backup", { variant: "outline" })}</form>
41 <form method="post" action="/maintenance/resume">${btn("End maintenance now")}</form>`)}`;
42 const response = page({ ctx, title: "Maintenance", body });
43 notePhase(ctx.request, "render", performance.now() - rendering);
44 return response;
45 }
46
47 export async function maintenancePrepare(ctx: Ctx): Promise<Response> {
48 const denied = admin(ctx);
49 if (denied) return denied;
50 const form = await readForm(ctx, "admin");
51 if (form instanceof Response) return form;
52 // Idle -> prepared stops the crew: asked first. A re-check while prepared has already been agreed.
53 if (status(ctx.app).phase === "idle" && field(form, "confirm") !== "yes")
54 return flashTo(ctx, "/maintenance", { tone: "danger", message: "Tick the box to confirm: the crew and releases stop." });
55 const r = await prepare(ctx.app, ctx.auth.user, field(form, "reason"));
56 return flashTo(ctx, "/maintenance", r.ok ? { message: r.value.says } : { tone: "danger", message: r.error.says });
57 }
58
59 export async function maintenanceResume(ctx: Ctx): Promise<Response> {
60 const denied = admin(ctx);
61 if (denied) return denied;
62 const form = await readForm(ctx, "admin");
63 if (form instanceof Response) return form;
64 const r = resumeNow(ctx.app, ctx.auth.user);
65 return flashTo(ctx, "/maintenance", r.ok ? { message: r.value.says } : { tone: "danger", message: r.error.says });
66 }
src/web/pages/resilience.ts +1
@@ -107,6 +107,7 @@ export function resiliencePage(ctx: Ctx): Response {
107107 </p>
108108 </div>
109109 ${backupsLineHtml(ctx)}
110 <p><a href="/maintenance">Maintenance</a> prepares the forge for a planned power-off.</p>
110111 ${nightlyHtml(ctx)}
111112 <p id="tally">
112113 ${String(count("green"))} green · ${String(count("red"))} red ·
test/conversation-confirm.test.ts +1 −1
@@ -123,7 +123,7 @@ describe("the one list of what asks first", () => {
123123 "access", "account_verify", "app_members", "app_notes", "asks_act", "asks_decide", "asks_proposal", "asks_undecide",
124124 "assign", "attach_look", "bet", "board_pins", "board_write", "branches", "capacity", "valves", "cloud_accounts", "close", "confirm_wait",
125125 "crew", "decisions_by", "detach", "edit", "email_verify", "feed_chips", "feed_dismiss", "flag_set", "flag_winner",
126 "follow_mode", "fork", "front_page", "hold", "issue_teams", "keep_plan", "labels", "lift", "limb", "make_it_yours", "map", "merge", "new_app",
126 "follow_mode", "fork", "front_page", "hold", "issue_teams", "keep_plan", "labels", "lift", "limb", "maintenance", "make_it_yours", "map", "merge", "new_app",
127127 "next_up", "week", "notify", "open_pull", "org_create", "origin_apps", "pins", "plan", "profile", "propose_to_parent_forge", "repo_labels", "reports",
128128 "restore_drill", "review", "say", "signup", "snapshot_now", "spend", "steer", "sync", "take_back", "take_updates",
129129 "teams", "tell", "threads", "trail", "upstream", "variant", "wait_for", "watch", "whoami_can", "write_file",
test/forge-looks.test.ts +1
@@ -37,6 +37,7 @@ const NOT_A_PAGE: Record<string, string> = {
3737 "/opensearch.xml": "the browser search description, XML",
3838 "/sw.js": "the service worker script",
3939 "/audit.csv": "a CSV download of /audit",
40 "/maintenance": "no picture: the cap of 24 looks is spent; an admin's page, covered by test/maintenance.test.ts",
4041 "/claim": "answers 404 to an admin: it exists only for a link-holder",
4142 "/first-admin": "the setup form for a forge with no admin yet",
4243 "/signin": "a sign-in form, not a page of the forge",
test/maintenance-crew.test.ts added +49
@@ -0,0 +1,49 @@
1 /** While the forge is in maintenance the real dispatcher starts nothing and says why. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { chmodSync, writeFileSync } from "node:fs";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp, type App } from "../src/app.ts";
8 import { createRun, getRun } from "../src/core/runs.ts";
9 import { createUser } from "../src/core/users.ts";
10 import { startCrew } from "../src/crew/dispatch.ts";
11 import { serve } from "../src/http/server.ts";
12 import { writeState } from "../src/maintenance/state.ts";
13 import { heldRuns } from "../src/runs/held.ts";
14
15 let dataDir: string;
16 let app: App;
17 let repoId = "";
18
19 beforeAll(async () => {
20 dataDir = await mkdtemp(join(tmpdir(), "forge-maint-crew-"));
21 app = createApp({ dataDir, port: 0, disk: () => ({ freeBytes: 100, totalBytes: 100 }) });
22 const owner = await createUser(app.db, { origin: "seed", trusted: true, username: "holder", email: "[email protected]", password: "a-good-passphrase" });
23 if (!owner.ok) throw new Error(String(owner.error));
24 repoId = crypto.randomUUID();
25 app.db.query("INSERT INTO repo (id, owner_id, name, visibility, created_at) VALUES (?1, ?2, 'm', 'private', ?3)").run(repoId, owner.value.id, new Date().toISOString());
26 });
27 afterAll(async () => {
28 app.close();
29 await rm(dataDir, { recursive: true, force: true });
30 });
31
32 test("queued runs stay queued, held by maintenance", async () => {
33 writeState(dataDir, { current: { since: new Date().toISOString(), by: "a", reason: "", holdId: 1, backup: { ok: true, says: "x" } } });
34 const run = createRun(app.db, { repoId, kind: "agent" });
35 const servers = await serve(app);
36 const fake = join(dataDir, "fake-claude");
37 writeFileSync(fake, `#!/bin/sh\nif [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi\nexit 1\n`);
38 chmodSync(fake, 0o755);
39 const crew = await startCrew(app, { forgeUrl: servers.url, localUrl: servers.url, binary: fake, concurrency: 2, budgetUsd: 1, runTimeoutMs: 60_000, tickMs: 20 });
40 try {
41 const deadline = Date.now() + 10_000;
42 while (heldRuns(app.db).get(run.id) !== "maintenance" && Date.now() < deadline) await Bun.sleep(20);
43 expect(heldRuns(app.db).get(run.id)).toBe("maintenance");
44 expect(getRun(app.db, run.id)?.state).toBe("queued");
45 } finally {
46 await crew.stop();
47 servers.stop();
48 }
49 });
test/maintenance.test.ts added +95
@@ -0,0 +1,95 @@
1 import { expect, test } from "bun:test";
2 import { mkdtempSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { maintenanceHtml } from "../src/maintenance/page.ts";
6 import { inMaintenance, readState, writeState } from "../src/maintenance/state.ts";
7 import { HOLD_WORDS } from "../src/runs/held.ts";
8
9 test("state: missing or damaged file is not maintenance", async () => {
10 const dir = mkdtempSync(join(tmpdir(), "maint-"));
11 expect(inMaintenance(dir)).toBe(false);
12 await Bun.write(join(dir, "maintenance.json"), "{nope");
13 expect(inMaintenance(dir)).toBe(false);
14 writeState(dir, { current: { since: "2026-11-08T00:00:00Z", by: "a", reason: "", holdId: 1, backup: { ok: true, says: "x" } } });
15 expect(inMaintenance(dir)).toBe(true);
16 expect(readState(dir).current?.holdId).toBe(1);
17 });
18
19 test("page escapes the reason and refreshes", () => {
20 const h = maintenanceHtml({ reason: "<script>alert(1)</script>", since: "2026-11-08T00:00:00Z" });
21 expect(h).not.toContain("<script>");
22 expect(h).toContain("&lt;script&gt;");
23 expect(h).toContain('http-equiv="refresh"');
24 expect(maintenanceHtml({ since: '"><b>' })).not.toContain("<b>");
25 });
26
27 test("vitals names maintenance as a hold", () => {
28 expect(HOLD_WORDS.maintenance).toBe("planned maintenance");
29 });
30
31 import { afterAll, beforeAll, describe } from "bun:test";
32 import { mkdtemp, rm } from "node:fs/promises";
33 import { createApp, type App } from "../src/app.ts";
34 import { createUser } from "../src/core/users.ts";
35 import { standingHolds } from "../src/keeper/hold.ts";
36 import { prepare, resume, resumeAtBoot, status } from "../src/maintenance/service.ts";
37 import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
38 import { dirStore } from "../src/snapshot/store.ts";
39 import { quietLogs } from "./helpers/globals.ts";
40
41 quietLogs();
42
43 describe("prepare and come back", () => {
44 let dataDir: string;
45 let app: App;
46 let svc: SnapshotsHandle | undefined;
47 let admin: any;
48 let bob: any;
49
50 beforeAll(async () => {
51 dataDir = await mkdtemp(join(tmpdir(), "forge-maint-"));
52 app = createApp({ dataDir, port: 0 });
53 const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
54 const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
55 if (!a.ok || !b.ok) throw new Error("user");
56 admin = a.value;
57 bob = b.value;
58 });
59 afterAll(async () => {
60 await svc?.stop();
61 app.close();
62 await rm(dataDir, { recursive: true, force: true });
63 });
64
65 test("a non-admin cannot prepare", async () => {
66 const r = await prepare(app, bob, "");
67 expect(r.ok).toBe(false);
68 expect(inMaintenance(dataDir)).toBe(false);
69 });
70
71 test("without a backup place it holds and pauses but is not ready", async () => {
72 const r = await prepare(app, admin, "moving the rack");
73 expect(r.ok).toBe(true);
74 expect(inMaintenance(dataDir)).toBe(true);
75 expect(standingHolds(app.db)).toHaveLength(1);
76 expect(status(app).phase).toBe("draining");
77 expect(status(app).says).toContain("nowhere to put a snapshot");
78 });
79
80 test("boot ends it: hold lifted, crew unpaused, the return is told", () => {
81 resumeAtBoot(app);
82 expect(inMaintenance(dataDir)).toBe(false);
83 expect(standingHolds(app.db)).toHaveLength(0);
84 expect(status(app).last?.verified.length).toBeGreaterThan(0);
85 expect(resume(app, "x").ok).toBe(false);
86 });
87
88 test("with a store it is ready once the backup is taken and nothing runs", async () => {
89 svc = startSnapshots(app, dirStore(join(dataDir, "store")), { version: "t", nightly: false });
90 const r = await prepare(app, admin, "");
91 expect(r.ok && r.value.phase).toBe("ready");
92 expect(await svc.list()).toHaveLength(1);
93 resume(app, "alice");
94 });
95 });
test/mcp.test.ts +1
@@ -318,6 +318,7 @@ describe("the forge over MCP", () => {
318318 "feed",
319319 "feed_chips",
320320 "feed_dismiss",
321 "maintenance",
321322 "make_it_yours",
322323 "origin_apps",
323324 "front_page",

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs