Public: anyone can read this; the crew can change it.
Maintenance from the forge: request a shutdown, keep a page up, and come back by itself #9557
scout/forge:scout/9453-maintenance-from-the-forge-request-a-shu → main 3 commits fork
27 files
changed +628
−2
docs/design/audit.md +1
| @@ -99,6 +99,7 @@ Machine routes are not pages and have no entry: `/audit.csv`, | ||
| 99 | 99 | |
| 100 | 100 | - `/system` — `pages/system.ts` · the machine as it is now: cores, load, memory, processes · `/flow`, `/` · none seen; updates in place over `/system/stream`, no reload script. |
| 101 | 101 | - `/resilience` — `pages/resilience.ts` · each protection in `docs/THREATS.md` against the last gate's tests · `/settings/resilience` (similar name, different content) · none seen. |
| 102 | - `/maintenance` — `pages/maintenance.ts` · prepare the forge for a power-off and end it · `/resilience`, `/settings/backups` · none seen. | |
| 102 | 103 | - `/admin/limits` — `pages/admin-limits.ts` · every limit with value, default and reason · `/settings/capacity` (also a numbers-you-can-change page) · none seen. |
| 103 | 104 | - `/settings/capacity` — `pages/capacity.ts` · how many runs and checks go at once · `/admin/limits`, `/settings/usage` · none seen. |
| 104 | 105 | - `/settings/usage` — `pages/usage.ts` · route patterns by use, unused first · `/settings/telemetry`, `/board/returns` · none seen. Not about cost, despite the name. |
docs/features/maintenance.md added +40
| @@ -0,0 +1,40 @@ | ||
| 1 | # Maintenance: shut down for a move and come back by itself (#9453) | |
| 2 | ||
| 3 | **Prepare.** An admin opens `/maintenance` (linked from Resilience), or calls | |
| 4 | `POST /api/v1/admin/maintenance/prepare` (admin-scope token) or MCP | |
| 5 | `maintenance {action: "prepare"}`. The forge then: | |
| 6 | ||
| 7 | 1. holds releases (an owner hold, `keeper/hold.ts`); | |
| 8 | 2. pauses the crew: the dispatcher starts nothing while `maintenance.json` | |
| 9 | stands in the data directory; vitals reads the queue as held by "planned maintenance"; | |
| 10 | 3. takes a fresh snapshot (`actions/backups_now.ts`); | |
| 11 | 4. reports the drain: `GET /api/v1/admin/maintenance` says `draining` while runs | |
| 12 | or checks still go, `ready` when the backup is taken and nothing runs. Power | |
| 13 | off on `ready`; a SIGTERM drains the rest as in a release. | |
| 14 | ||
| 15 | The page says plainly what stops: releases, new crew runs, then the forge, its | |
| 16 | apps and previews when the host powers off. | |
| 17 | ||
| 18 | **Come back.** On boot `resumeAtBoot` lifts the hold, unpauses the crew and | |
| 19 | records a line (shown on `/maintenance`, in the status, and as the | |
| 20 | `maintenance.ended` event) naming what was verified. An admin can also press | |
| 21 | "End maintenance now". The tunnel, runner and nightly timers are the forge's | |
| 22 | own boot: they start with the process, and the nightly jobs show on Resilience. | |
| 23 | No Kubernetes is asked anything; the state is one file. | |
| 24 | ||
| 25 | **While it is off.** `bun scripts/standby.ts 8080` serves the calm 503 page | |
| 26 | (`src/maintenance/page.ts`) from the droplet; point the tunnel's or | |
| 27 | Cloudflare's origin at it for the move, and back after. A Cloudflare Worker can | |
| 28 | return the same HTML with status 503 and `Retry-After`. | |
| 29 | ||
| 30 | **Size of a standby.** For reads or a whole forge: disk for the data (38 GB now, | |
| 31 | apps 13 GB) plus one snapshot and headroom, so 64 GB; 4 GB of memory free for the | |
| 32 | forge alone, 8 GB with apps running. The 16 GB droplet cannot hold it. | |
| 33 | ||
| 34 | ## Not done | |
| 35 | - No standby takeover: nothing replicates data to a second machine, so the | |
| 36 | droplet can only show the page. That needs the snapshot mirror restored onto a | |
| 37 | machine of the size above, and is a separate issue. | |
| 38 | - The page is not pushed to Cloudflare automatically; `standby.ts` is started by hand. | |
| 39 | - The admin is told on the page, event and status; no mail or push. | |
| 40 | - A restart of the forge while prepared (not a power-off) also ends maintenance. |
scripts/standby.ts added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | /** | |
| 2 | * The maintenance page, served from somewhere that stays up (the droplet): | |
| 3 | * `bun scripts/standby.ts [port]` answers every request with the calm page | |
| 4 | * and a 503, so a tunnel or Cloudflare origin pointed here shows visitors | |
| 5 | * "back soon", not an error. MAINTENANCE_REASON and MAINTENANCE_SINCE are | |
| 6 | * optional. Zero dependencies; reads no forge data. | |
| 7 | */ | |
| 8 | import { MAINTENANCE_HEADERS, maintenanceHtml } from "../src/maintenance/page.ts"; | |
| 9 | ||
| 10 | const port = Number(process.argv[2] ?? process.env["PORT"] ?? 8080); | |
| 11 | const body = maintenanceHtml({ | |
| 12 | ...(process.env["MAINTENANCE_REASON"] ? { reason: process.env["MAINTENANCE_REASON"] } : {}), | |
| 13 | ...(process.env["MAINTENANCE_SINCE"] ? { since: process.env["MAINTENANCE_SINCE"] } : {}), | |
| 14 | }); | |
| 15 | ||
| 16 | Bun.serve({ | |
| 17 | port, | |
| 18 | fetch(req) { | |
| 19 | const path = new URL(req.url).pathname; | |
| 20 | if (path === "/healthz") return new Response("standby", { status: 200 }); | |
| 21 | return new Response(req.method === "HEAD" ? null : body, { status: 503, headers: MAINTENANCE_HEADERS }); | |
| 22 | }, | |
| 23 | }); | |
| 24 | console.log(`standby maintenance page on :${port}`); |
src/contract/openapi/routes/getApiV1AdminMaintenance.ts added +9
| @@ -0,0 +1,9 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { | |
| 4 | method: "get", | |
| 5 | path: "/api/v1/admin/maintenance", | |
| 6 | summary: "Where planned maintenance stands: { phase: idle | draining | ready, since?, reason?, backup?, working, last?, says }; ready means releases are held, the crew is paused, a fresh backup is taken and nothing runs", | |
| 7 | auth: true, | |
| 8 | refusals: { not_found: "only a forge administrator sees this route" }, | |
| 9 | }; |
src/contract/openapi/routes/index.ts +3
| @@ -40,6 +40,7 @@ export { route as getApiV1AdminDrills } from "./getApiV1AdminDrills.ts"; | ||
| 40 | 40 | export { route as getApiV1AdminInvites } from "./getApiV1AdminInvites.ts"; |
| 41 | 41 | export { route as getApiV1AdminKpisProbe } from "./getApiV1AdminKpisProbe.ts"; |
| 42 | 42 | export { route as getApiV1AdminLimits } from "./getApiV1AdminLimits.ts"; |
| 43 | export { route as getApiV1AdminMaintenance } from "./getApiV1AdminMaintenance.ts"; | |
| 43 | 44 | export { route as getApiV1AdminPeople } from "./getApiV1AdminPeople.ts"; |
| 44 | 45 | export { route as getApiV1AdminPinned } from "./getApiV1AdminPinned.ts"; |
| 45 | 46 | export { route as getApiV1AdminProfile } from "./getApiV1AdminProfile.ts"; |
| @@ -218,6 +219,8 @@ export { route as postApiV1AdminAccountsUsernameSuspend } from "./postApiV1Admin | ||
| 218 | 219 | export { route as postApiV1AdminAccountsUsernameUnsuspend } from "./postApiV1AdminAccountsUsernameUnsuspend.ts"; |
| 219 | 220 | export { route as postApiV1AdminAccountsUsernameVerify } from "./postApiV1AdminAccountsUsernameVerify.ts"; |
| 220 | 221 | export { route as postApiV1AdminInvites } from "./postApiV1AdminInvites.ts"; |
| 222 | export { route as postApiV1AdminMaintenancePrepare } from "./postApiV1AdminMaintenancePrepare.ts"; | |
| 223 | export { route as postApiV1AdminMaintenanceResume } from "./postApiV1AdminMaintenanceResume.ts"; | |
| 221 | 224 | export { route as postApiV1AdminPinned } from "./postApiV1AdminPinned.ts"; |
| 222 | 225 | export { route as postApiV1AdminPinnedNameCheck } from "./postApiV1AdminPinnedNameCheck.ts"; |
| 223 | 226 | export { route as postApiV1AdminPinnedNameRollback } from "./postApiV1AdminPinnedNameRollback.ts"; |
src/contract/openapi/routes/postApiV1AdminMaintenancePrepare.ts added +10
| @@ -0,0 +1,10 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { | |
| 4 | method: "post", | |
| 5 | path: "/api/v1/admin/maintenance/prepare", | |
| 6 | summary: "Prepare the forge for a power-off: hold releases, pause the crew, take a fresh backup, report the drain; needs an admin-scope token. Answers the status as GET does", | |
| 7 | auth: true, | |
| 8 | body: { reason: "string" }, | |
| 9 | refusals: { not_found: "only a forge administrator may", invalid: "the reason is at most 200 characters (422)" }, | |
| 10 | }; |
src/contract/openapi/routes/postApiV1AdminMaintenanceResume.ts added +9
| @@ -0,0 +1,9 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { | |
| 4 | method: "post", | |
| 5 | path: "/api/v1/admin/maintenance/resume", | |
| 6 | summary: "End maintenance now: lift the release hold and let the crew start; needs an admin-scope token. The forge does this by itself at boot", | |
| 7 | auth: true, | |
| 8 | refusals: { not_found: "only a forge administrator may", conflict: "the forge is not in maintenance (409)" }, | |
| 9 | }; |
src/core/event_audience.ts +2
| @@ -189,6 +189,8 @@ export function classifiedAudience(kind: string): EventAudience | undefined { | ||
| 189 | 189 | kind.startsWith("drill.") || |
| 190 | 190 | // A snapshot moving is the forge owner's too, and has no repository. |
| 191 | 191 | kind.startsWith("backup.") || |
| 192 | // Preparing for a power-off and ending it are the forge owner's (`maintenance/service.ts`). | |
| 193 | kind.startsWith("maintenance.") || | |
| 192 | 194 | // The weekly model review's changes and results are the owner's (`actions/model_review.ts`). |
| 193 | 195 | kind.startsWith("crew.model_") || |
| 194 | 196 | // A child forge answering no one at its public address is the owner's to hear (`spawn/reach.ts`). |
src/core/reserved.ts +1
| @@ -96,6 +96,7 @@ export const SYSTEM_NAMES: readonly string[] = [ | ||
| 96 | 96 | "organizations", |
| 97 | 97 | "now", |
| 98 | 98 | "resilience", |
| 99 | "maintenance", | |
| 99 | 100 | "origins", |
| 100 | 101 | "robots.txt", |
| 101 | 102 | "favicon.ico", |
src/crew/dispatch.ts +8
| @@ -47,6 +47,7 @@ import { noteHeld, type HoldCause } from "../runs/held.ts"; | ||
| 47 | 47 | import { decideAndRecord } from "./ledger.ts"; |
| 48 | 48 | import { bugsFirst } from "../flow/bugs.ts"; |
| 49 | 49 | import { byDials } from "./team_share.ts"; |
| 50 | import { inMaintenance } from "../maintenance/state.ts"; | |
| 50 | 51 | import { gateQueue } from "../flow/valve_gate.ts"; |
| 51 | 52 | import { ARCHIVED_WHY, getRepoById, type Repo } from "../core/repos.ts"; |
| 52 | 53 | import { listReviews } from "../core/reviews.ts"; |
| @@ -1514,6 +1515,13 @@ function pump( | ||
| 1514 | 1515 | const gated = startOrder(app); |
| 1515 | 1516 | // What this pass holds by policy, for vitals (`runs/held.ts`, #9403). |
| 1516 | 1517 | const held = new Map<string, HoldCause>(); |
| 1518 | // Planned maintenance (`maintenance/service.ts`): nothing new starts, and | |
| 1519 | // a queued run waits as it does behind a shut valve. | |
| 1520 | if (inMaintenance(app.config.dataDir)) { | |
| 1521 | for (const q of [...gated.order, ...gated.held]) held.set(q.id, "maintenance"); | |
| 1522 | noteHeld(app.db, held); | |
| 1523 | return; | |
| 1524 | } | |
| 1517 | 1525 | for (const shut of gated.held) { |
| 1518 | 1526 | held.set(shut.id, "valve"); |
| 1519 | 1527 | if (noted.has(shut.id)) continue; |
src/http/api.ts +2
| @@ -347,6 +347,7 @@ import { labelsRoutes } from "./api/labels.ts"; | ||
| 347 | 347 | import { visionRoutes } from "./api/vision.ts"; |
| 348 | 348 | import { tokenRoutes } from "./api/tokens.ts"; |
| 349 | 349 | import { accountAdmin, adminRoutes } from "./api/admin.ts"; |
| 350 | import { maintenanceRoutes } from "./api/maintenance.ts"; | |
| 350 | 351 | import { flagsRoutes } from "./api/flags.ts"; |
| 351 | 352 | import { issueRoutes, labelsHandler } from "./api/issues.ts"; |
| 352 | 353 | import { FORGE_NOT_CARRIED } from "../origins/words.ts"; |
| @@ -1224,6 +1225,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 1224 | 1225 | userRoutes(router); |
| 1225 | 1226 | forgeOriginRoutes(router); |
| 1226 | 1227 | adminRoutes(router); |
| 1228 | maintenanceRoutes(router); | |
| 1227 | 1229 | flagsRoutes(router); |
| 1228 | 1230 | |
| 1229 | 1231 | // Open read hooks (#1789): the owner, or an agent holding `telemetry:read`. |
src/http/api/maintenance.ts added +38
| @@ -0,0 +1,38 @@ | ||
| 1 | /** `/api/v1/admin/maintenance`: what the Maintenance page and MCP `maintenance` do, for a forge admin's admin-scope credential. */ | |
| 2 | ||
| 3 | import { prepare, resumeNow, status } from "../../maintenance/service.ts"; | |
| 4 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; | |
| 5 | import { crossOrigin, type Handler } from "../context.ts"; | |
| 6 | import { apiError, forbidden, json, notFound, requireScope, unauthorized } from "../respond.ts"; | |
| 7 | import type { Router } from "../router.ts"; | |
| 8 | ||
| 9 | export function maintenanceRoutes(router: Router<Handler>): void { | |
| 10 | router.add("GET", "/api/v1/admin/maintenance", (ctx) => { | |
| 11 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); | |
| 12 | if (!ctx.auth.user) return unauthorized(); | |
| 13 | if (!ctx.auth.user.isAdmin) return notFound(); | |
| 14 | const denied = requireScope(ctx, "ro"); | |
| 15 | if (denied) return denied; | |
| 16 | return json(status(ctx.app)); | |
| 17 | }); | |
| 18 | router.add("POST", "/api/v1/admin/maintenance/prepare", async (ctx) => { | |
| 19 | if (!ctx.auth.user) return unauthorized(); | |
| 20 | if (!ctx.auth.user.isAdmin) return notFound(); | |
| 21 | const denied = requireScope(ctx, "admin"); | |
| 22 | if (denied) return denied; | |
| 23 | if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden(); | |
| 24 | const input: unknown = await ctx.request.json().catch(() => ({})); | |
| 25 | const reason = typeof input === "object" && input !== null && typeof (input as { reason?: unknown }).reason === "string" ? (input as { reason: string }).reason : ""; | |
| 26 | const r = await prepare(ctx.app, ctx.auth.user, reason); | |
| 27 | return r.ok ? json(r.value) : apiError(r.error.code === "invalid" ? "invalid" : "not_found", r.error.code === "invalid" ? 422 : 404, r.error.says); | |
| 28 | }); | |
| 29 | router.add("POST", "/api/v1/admin/maintenance/resume", (ctx) => { | |
| 30 | if (!ctx.auth.user) return unauthorized(); | |
| 31 | if (!ctx.auth.user.isAdmin) return notFound(); | |
| 32 | const denied = requireScope(ctx, "admin"); | |
| 33 | if (denied) return denied; | |
| 34 | if (ctx.auth.via === "session" && crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden(); | |
| 35 | const r = resumeNow(ctx.app, ctx.auth.user); | |
| 36 | return r.ok ? json(r.value) : apiError(r.error.code, r.error.code === "conflict" ? 409 : 404, r.error.says); | |
| 37 | }); | |
| 38 | } |
src/http/mcp/tools/index.ts +1
| @@ -86,6 +86,7 @@ export { tool as land_past_gate } from "./land_past_gate.ts"; | ||
| 86 | 86 | export { tool as lift } from "./lift.ts"; |
| 87 | 87 | export { tool as limb } from "./limb.ts"; |
| 88 | 88 | export { tool as limits } from "./limits.ts"; |
| 89 | export { tool as maintenance } from "./maintenance.ts"; | |
| 89 | 90 | export { tool as make_it_yours } from "./make_it_yours.ts"; |
| 90 | 91 | export { tool as map } from "./map.ts"; |
| 91 | 92 | export { tool as merge } from "./merge.ts"; |
src/http/mcp/tools/maintenance.ts added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | import { prepare, resumeNow, status } from "../../../maintenance/service.ts"; | |
| 2 | import { scopeAllows } from "../../../core/tokens.ts"; | |
| 3 | import { good, bad, type Tool } from "../shared.ts"; | |
| 4 | ||
| 5 | export const tool: Tool = { | |
| 6 | name: "maintenance", | |
| 7 | doc: "`action` is `status` (default), `prepare` or `resume`; `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says }`", | |
| 8 | description: "Prepare the forge for a power-off, check the drain, or end maintenance (admin).", | |
| 9 | inputSchema: { | |
| 10 | type: "object", | |
| 11 | properties: { action: { type: "string", enum: ["status", "prepare", "resume"] }, reason: { type: "string", maxLength: 200 } }, | |
| 12 | }, | |
| 13 | writes: true, | |
| 14 | async run(ctx, user, args) { | |
| 15 | if (!user.isAdmin) return bad("not_found", "no such tool here"); | |
| 16 | const action = args["action"] ?? "status"; | |
| 17 | if (action === "status") return good(status(ctx.app)); | |
| 18 | if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare or resume"); | |
| 19 | if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token"); | |
| 20 | if (action === "resume") { | |
| 21 | const r = resumeNow(ctx.app, user); | |
| 22 | return r.ok ? good(r.value) : bad(r.error.code, r.error.says); | |
| 23 | } | |
| 24 | const reason = typeof args["reason"] === "string" ? args["reason"] : ""; | |
| 25 | const r = await prepare(ctx.app, user, reason); | |
| 26 | return r.ok ? good(r.value) : bad(r.error.code, r.error.says); | |
| 27 | }, | |
| 28 | }; |
src/http/server.ts +4
| @@ -25,6 +25,7 @@ import { prerenderComments } from "../web/prerender.ts"; | ||
| 25 | 25 | import { systemPage, systemStream } from "../web/pages/system.ts"; |
| 26 | 26 | import { contributePage } from "../web/pages/contribute.ts"; |
| 27 | 27 | import { pipelinePage } from "../web/pages/pipeline.ts"; |
| 28 | import { maintenancePage, maintenancePrepare, maintenanceResume } from "../web/pages/maintenance.ts"; | |
| 28 | 29 | import { nightlyOutputPage, resiliencePage } from "../web/pages/resilience.ts"; |
| 29 | 30 | import { limitsPage, limitsSet, quotasSet, signupSet, inviteMint, inviteRevoke } from "../web/pages/admin-limits.ts"; |
| 30 | 31 | import { measureSubmit } from "./measure.ts"; |
| @@ -889,6 +890,9 @@ export function buildRouter(): Router<Handler> { | ||
| 889 | 890 | pinnedRoutes(router); |
| 890 | 891 | // Each THREATS.md protection against the last gate; literal, above /:owner. |
| 891 | 892 | router.add("GET", "/resilience", resiliencePage); |
| 893 | router.add("GET", "/maintenance", maintenancePage); | |
| 894 | router.add("POST", "/maintenance/prepare", maintenancePrepare); | |
| 895 | router.add("POST", "/maintenance/resume", maintenanceResume); | |
| 892 | 896 | router.add("GET", "/resilience/nightly/:job", nightlyOutputPage); |
| 893 | 897 | // A selection's spend and runs over time; literal, above /:owner. |
| 894 | 898 | router.add("GET", "/fleet/graph", fleetGraphPage); |
src/main.ts +3
| @@ -70,6 +70,7 @@ import { peopleOnCrews } from "./core/agents.ts"; | ||
| 70 | 70 | import { startNotifier } from "./crew/notify.ts"; |
| 71 | 71 | import { expireCrewHolds } from "./actions/holds.ts"; |
| 72 | 72 | import { startClogTick } from "./flow/clog_tick.ts"; |
| 73 | import { resumeAtBoot } from "./maintenance/service.ts"; | |
| 73 | 74 | import { startKeeper } from "./flow/keeper.ts"; |
| 74 | 75 | import { sweepStaleAsks } from "./crew/stale_asks.ts"; |
| 75 | 76 | import { recordAskLedger } from "./web/studio/asks.ts"; |
| @@ -671,6 +672,8 @@ const processTick = (): void => { | ||
| 671 | 672 | const stopClogTick = startClogTick(app.db, app.bus, (error) => app.log.warn("clog tick failed", { error: String(error) }), 60_000); |
| 672 | 673 | // The throughput keeper (#7645) names what binds landings each minute and lifts what is its to lift. |
| 673 | 674 | const stopKeeper = startKeeper(app, () => sweepAssigned(app)); |
| 675 | // A forge that went down prepared for maintenance comes back by itself. | |
| 676 | resumeAtBoot(app); | |
| 674 | 677 | // An agent's hold that has stood 24 h is released each minute (#6918). |
| 675 | 678 | const holdTimer = setInterval(() => { |
| 676 | 679 | try { |
src/maintenance/page.ts added +31
| @@ -0,0 +1,31 @@ | ||
| 1 | /** | |
| 2 | * The page visitors see while the forge is off: calm, static, escaped, and | |
| 3 | * answerable by anything (the droplet's `scripts/standby.ts`, or a Cloudflare | |
| 4 | * Worker serving the same text). No script, no external asset. | |
| 5 | */ | |
| 6 | ||
| 7 | const esc = (s: string): string => | |
| 8 | s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "&9;"); | |
| 9 | ||
| 10 | export interface PageInput { | |
| 11 | /** What the owner said, one line; may be empty. */ | |
| 12 | readonly reason?: string; | |
| 13 | /** ISO time the forge went into maintenance, when known. */ | |
| 14 | readonly since?: string; | |
| 15 | } | |
| 16 | ||
| 17 | export function maintenanceHtml(input: PageInput = {}): string { | |
| 18 | const reason = (input.reason ?? "").replace(/[\u0000-\u001f\u007f]+/g, " ").trim().slice(0, 200); | |
| 19 | const since = input.since !== undefined && /^[0-9T:.\-Z+]{10,30}$/.test(input.since) ? input.since : ""; | |
| 20 | return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta http-equiv="refresh" content="60"><title>Back soon</title><style>body{font:16px/1.5 system-ui,sans-serif;margin:0;display:grid;min-height:100vh;place-items:center;background:#fafafa;color:#222}main{max-width:32rem;padding:1.5rem}h1{font-size:1.5rem;margin:0 0 .5rem}p{margin:.5rem 0}.m{color:#666}@media(prefers-color-scheme:dark){body{background:#151515;color:#eee}.m{color:#aaa}}</style></head><body><main><h1>Back soon</h1><p>The forge is being moved to new hardware. Nothing is lost: everything was backed up first, and it comes back by itself.</p>${reason === "" ? "" : `<p>${esc(reason)}</p>`}${since === "" ? "" : `<p class="m">Down since ${esc(since)}.</p>`}<p class="m">This page refreshes itself every minute.</p></main></body></html>`; | |
| 21 | } | |
| 22 | ||
| 23 | /** The headers the maintenance page is always served with: 503 so nothing caches it as the real site. */ | |
| 24 | export const MAINTENANCE_HEADERS: Readonly<Record<string, string>> = { | |
| 25 | "content-type": "text/html; charset=utf-8", | |
| 26 | "retry-after": "300", | |
| 27 | "cache-control": "no-store", | |
| 28 | "content-security-policy": "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'", | |
| 29 | "x-content-type-options": "nosniff", | |
| 30 | "referrer-policy": "no-referrer", | |
| 31 | }; |
src/maintenance/service.ts added +126
| @@ -0,0 +1,126 @@ | ||
| 1 | /** | |
| 2 | * Planned maintenance: an admin asks the forge to get ready for a power-off | |
| 3 | * (`prepare`), and on the next boot it comes back by itself (`resume`). | |
| 4 | * | |
| 5 | * prepare: hold releases (an owner hold, `keeper/hold.ts`), pause the crew | |
| 6 | * (the dispatcher starts nothing while the state file stands), take a fresh | |
| 7 | * snapshot, then report what still runs. The host may power off when | |
| 8 | * `status().ready` is true; a SIGTERM drains the rest as it does for a roll. | |
| 9 | * resume: lift the hold, let the crew start again, verify, and keep a line | |
| 10 | * saying what is healthy. No Kubernetes is asked anything. | |
| 11 | */ | |
| 12 | ||
| 13 | import type { App } from "../app.ts"; | |
| 14 | import { listActive } from "../core/runs.ts"; | |
| 15 | import type { User } from "../core/users.ts"; | |
| 16 | import { lift, MAX_REASON, setHold } from "../keeper/hold.ts"; | |
| 17 | import { err, ok, type Result } from "../kernel/result.ts"; | |
| 18 | import { running } from "../kernel/version.ts"; | |
| 19 | import { snapshotNow } from "../actions/backups_now.ts"; | |
| 20 | import { readState, writeState, type Maintenance, type Returned } from "./state.ts"; | |
| 21 | ||
| 22 | const isAdmin = (u: User | undefined): u is User => u?.isAdmin === true && u.kind !== "agent"; | |
| 23 | ||
| 24 | export interface Status { | |
| 25 | readonly phase: "idle" | "draining" | "ready"; | |
| 26 | readonly since?: string; | |
| 27 | readonly reason?: string; | |
| 28 | readonly backup?: Maintenance["backup"]; | |
| 29 | /** Crew runs and checks still queued or running. */ | |
| 30 | readonly working: number; | |
| 31 | readonly last?: Returned; | |
| 32 | /** Plainly: what stops, or what is about to. */ | |
| 33 | readonly says: string; | |
| 34 | } | |
| 35 | ||
| 36 | export const ABOUT_TO_STOP = | |
| 37 | "Preparing stops: releases (none are cut), the crew (no run starts; running ones finish or are handed back queued), and then the forge itself when the host powers off. Apps and previews stop with it. Visitors see a maintenance page if one is served elsewhere (docs/features/maintenance.md)."; | |
| 38 | ||
| 39 | export function status(app: App): Status { | |
| 40 | const { current, last } = readState(app.config.dataDir); | |
| 41 | if (current === undefined) | |
| 42 | return { phase: "idle", working: 0, ...(last ? { last } : {}), says: last ? last.says : "Not in maintenance." }; | |
| 43 | const working = listActive(app.db, "agent").length + listActive(app.db, "check").filter((r) => r.state === "running").length; | |
| 44 | const ready = current.backup.ok && working === 0; | |
| 45 | return { | |
| 46 | phase: ready ? "ready" : "draining", | |
| 47 | since: current.since, | |
| 48 | reason: current.reason, | |
| 49 | backup: current.backup, | |
| 50 | working, | |
| 51 | says: ready | |
| 52 | ? "Ready: releases are held, the crew is paused, a fresh backup is taken and nothing is running. The host can power off." | |
| 53 | : !current.backup.ok | |
| 54 | ? `Not ready: ${current.backup.says}` | |
| 55 | : `Draining: ${working} crew run${working === 1 ? "" : "s"} or check${working === 1 ? "" : "s"} still going; the host can power off when it reads 0.`, | |
| 56 | }; | |
| 57 | } | |
| 58 | ||
| 59 | export type Refusal = { readonly code: "not_found" | "invalid" | "conflict"; readonly says: string }; | |
| 60 | ||
| 61 | export async function prepare(app: App, actor: User | undefined, reasonIn: string): Promise<Result<Status, Refusal>> { | |
| 62 | if (!isAdmin(actor)) return err({ code: "not_found", says: "Only a forge admin prepares maintenance." }); | |
| 63 | const reason = reasonIn.replace(/[\u0000-\u001f\u007f]+/g, " ").trim(); | |
| 64 | if (reason.length > 200) return err({ code: "invalid", says: "The reason is at most 200 characters." }); | |
| 65 | const dataDir = app.config.dataDir; | |
| 66 | const prior = readState(dataDir); | |
| 67 | if (prior.current !== undefined) { | |
| 68 | // Asking again retakes the backup and reads the drain; the hold stands once. | |
| 69 | const taken = await snapshotNow(app, actor); | |
| 70 | writeState(dataDir, { ...prior, current: { ...prior.current, backup: { ok: taken.ok, says: taken.ok ? taken.value.says : taken.error.says } } }); | |
| 71 | return ok(status(app)); | |
| 72 | } | |
| 73 | const hold = setHold(app.db, { | |
| 74 | kind: "owner", | |
| 75 | version: running().version, | |
| 76 | reason: `maintenance${reason === "" ? "" : `: ${reason}`}`.slice(0, MAX_REASON), | |
| 77 | at: Date.now(), | |
| 78 | }); | |
| 79 | const now = new Date().toISOString(); | |
| 80 | // Written before the backup: the crew pauses at once, and a crash mid-backup still resumes at boot. | |
| 81 | const base: Maintenance = { since: now, by: actor.username, reason, holdId: hold.id, backup: { ok: false, says: "the backup has not finished" } }; | |
| 82 | writeState(dataDir, { ...(prior.last ? { last: prior.last } : {}), current: base }); | |
| 83 | app.bus.append({ kind: "maintenance.prepared", actorId: actor.id, payload: { hold: hold.id, reason } }); | |
| 84 | const taken = await snapshotNow(app, actor); | |
| 85 | writeState(dataDir, { | |
| 86 | ...(prior.last ? { last: prior.last } : {}), | |
| 87 | current: { ...base, backup: { ok: taken.ok, says: taken.ok ? taken.value.says : taken.error.says } }, | |
| 88 | }); | |
| 89 | return ok(status(app)); | |
| 90 | } | |
| 91 | ||
| 92 | /** Come back: lift our hold, start the crew, and record what was checked. `by` names who or what did it. */ | |
| 93 | export function resume(app: App, by: string): Result<Returned, Refusal> { | |
| 94 | const dataDir = app.config.dataDir; | |
| 95 | const { current } = readState(dataDir); | |
| 96 | if (current === undefined) return err({ code: "conflict", says: "The forge is not in maintenance." }); | |
| 97 | const now = Date.now(); | |
| 98 | const lifted = lift(app.db, current.holdId, by, now); | |
| 99 | const verified = [ | |
| 100 | `database open, version ${running().version}`, | |
| 101 | lifted ? "release hold lifted" : "release hold was already lifted", | |
| 102 | "crew dispatcher unpaused", | |
| 103 | `${listActive(app.db, "agent").length} crew run(s) and ${listActive(app.db, "check").length} check(s) waiting to start`, | |
| 104 | ]; | |
| 105 | const back: Returned = { | |
| 106 | at: new Date(now).toISOString(), | |
| 107 | downSince: current.since, | |
| 108 | says: `Back at ${new Date(now).toISOString()} after maintenance begun ${current.since}: ${verified.join("; ")}.`, | |
| 109 | verified, | |
| 110 | }; | |
| 111 | writeState(dataDir, { last: back }); | |
| 112 | app.bus.append({ kind: "maintenance.ended", payload: { by, hold: current.holdId, verified } }); | |
| 113 | return ok(back); | |
| 114 | } | |
| 115 | ||
| 116 | /** At boot: a forge that went down prepared comes back by itself. */ | |
| 117 | export function resumeAtBoot(app: App): void { | |
| 118 | if (readState(app.config.dataDir).current === undefined) return; | |
| 119 | const r = resume(app, "boot"); | |
| 120 | if (r.ok) app.log.info("maintenance ended at boot", { says: r.value.says }); | |
| 121 | } | |
| 122 | ||
| 123 | export function resumeNow(app: App, actor: User | undefined): Result<Returned, Refusal> { | |
| 124 | if (!isAdmin(actor)) return err({ code: "not_found", says: "Only a forge admin ends maintenance." }); | |
| 125 | return resume(app, actor.username); | |
| 126 | } |
src/maintenance/state.ts added +72
| @@ -0,0 +1,72 @@ | ||
| 1 | /** | |
| 2 | * The one file of maintenance state, `maintenance.json` under the data | |
| 3 | * directory: it survives the power-off because the data does. Plain data in, | |
| 4 | * plain data out; a missing or unreadable file reads as "not in maintenance", | |
| 5 | * so a damaged file can never keep the crew paused. | |
| 6 | */ | |
| 7 | ||
| 8 | import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; | |
| 9 | import { join } from "node:path"; | |
| 10 | ||
| 11 | export interface Backup { | |
| 12 | readonly ok: boolean; | |
| 13 | readonly says: string; | |
| 14 | } | |
| 15 | ||
| 16 | export interface Maintenance { | |
| 17 | readonly since: string; | |
| 18 | readonly by: string; | |
| 19 | /** One line the maintenance page shows, 0 to 200 characters. */ | |
| 20 | readonly reason: string; | |
| 21 | /** The release hold this request placed (`keeper/hold.ts`); lifted when the forge returns. */ | |
| 22 | readonly holdId: number; | |
| 23 | readonly backup: Backup; | |
| 24 | } | |
| 25 | ||
| 26 | export interface Returned { | |
| 27 | readonly at: string; | |
| 28 | readonly downSince: string; | |
| 29 | readonly says: string; | |
| 30 | readonly verified: readonly string[]; | |
| 31 | } | |
| 32 | ||
| 33 | export interface Stored { | |
| 34 | readonly current?: Maintenance; | |
| 35 | /** The last return, kept so an admin is told when it is back and what is healthy. */ | |
| 36 | readonly last?: Returned; | |
| 37 | } | |
| 38 | ||
| 39 | const file = (dataDir: string): string => join(dataDir, "maintenance.json"); | |
| 40 | ||
| 41 | export function readState(dataDir: string): Stored { | |
| 42 | try { | |
| 43 | if (!existsSync(file(dataDir))) return {}; | |
| 44 | const raw: unknown = JSON.parse(readFileSync(file(dataDir), "utf8")); | |
| 45 | if (typeof raw !== "object" || raw === null) return {}; | |
| 46 | const o = raw as Stored; | |
| 47 | const cur = o.current; | |
| 48 | const good = | |
| 49 | cur !== undefined && | |
| 50 | typeof cur.since === "string" && | |
| 51 | typeof cur.holdId === "number" && | |
| 52 | typeof cur.backup?.ok === "boolean"; | |
| 53 | return { ...(good ? { current: cur } : {}), ...(o.last !== undefined && typeof o.last.at === "string" ? { last: o.last } : {}) }; | |
| 54 | } catch { | |
| 55 | return {}; | |
| 56 | } | |
| 57 | } | |
| 58 | ||
| 59 | export function writeState(dataDir: string, state: Stored): void { | |
| 60 | const tmp = `${file(dataDir)}.tmp`; | |
| 61 | writeFileSync(tmp, JSON.stringify(state), { mode: 0o600 }); | |
| 62 | renameSync(tmp, file(dataDir)); | |
| 63 | } | |
| 64 | ||
| 65 | export function clearState(dataDir: string): void { | |
| 66 | rmSync(file(dataDir), { force: true }); | |
| 67 | } | |
| 68 | ||
| 69 | /** Is the forge in maintenance? The dispatcher asks on every pass. */ | |
| 70 | export function inMaintenance(dataDir: string): boolean { | |
| 71 | return readState(dataDir).current !== undefined; | |
| 72 | } |
src/runs/held.ts +2 −1
| @@ -17,13 +17,14 @@ import type { Database } from "bun:sqlite"; | ||
| 17 | 17 | export const HELD_FRESH_MS = 2 * 60_000; |
| 18 | 18 | |
| 19 | 19 | /** Why the dispatcher held a run. */ |
| 20 | export type HoldCause = "valve" | "pace" | "limit"; | |
| 20 | export type HoldCause = "valve" | "pace" | "limit" | "maintenance"; | |
| 21 | 21 | |
| 22 | 22 | /** How vitals names each cause: "2 by a shut valve". */ |
| 23 | 23 | export const HOLD_WORDS: Readonly<Record<HoldCause, string>> = { |
| 24 | 24 | valve: "a shut valve", |
| 25 | 25 | pace: "the week's pace", |
| 26 | 26 | limit: "an account limit", |
| 27 | maintenance: "planned maintenance", | |
| 27 | 28 | }; |
| 28 | 29 | |
| 29 | 30 | interface Pass { |
src/web/pages/maintenance.ts added +66
| @@ -0,0 +1,66 @@ | ||
| 1 | /** | |
| 2 | * /maintenance: an admin's page to prepare the forge for a power-off and to | |
| 3 | * end it. It says plainly what stops, what the drain still waits for, and, | |
| 4 | * once the forge is back, when and what was verified. | |
| 5 | */ | |
| 6 | ||
| 7 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; | |
| 8 | import { ABOUT_TO_STOP, prepare, resumeNow, status } from "../../maintenance/service.ts"; | |
| 9 | import type { Ctx } from "../../http/context.ts"; | |
| 10 | import { notFoundPage, signInFirst } from "../chrome.ts"; | |
| 11 | import { field, readForm } from "../form.ts"; | |
| 12 | import { html } from "../html.ts"; | |
| 13 | import { flashTo, page } from "../layout.ts"; | |
| 14 | import { btn, card } from "../ui.ts"; | |
| 15 | ||
| 16 | function admin(ctx: Ctx): Response | undefined { | |
| 17 | if (!ctx.auth.user) return signInFirst(ctx); | |
| 18 | if (!ctx.auth.user.isAdmin) return notFoundPage(ctx); | |
| 19 | return undefined; | |
| 20 | } | |
| 21 | ||
| 22 | export function maintenancePage(ctx: Ctx): Response { | |
| 23 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); | |
| 24 | const denied = admin(ctx); | |
| 25 | if (denied) return denied; | |
| 26 | const rendering = performance.now(); | |
| 27 | const s = status(ctx.app); | |
| 28 | const body = html`<div class="page-head"><h1>Maintenance</h1></div> | |
| 29 | <p id="maintenance-says">${s.says}</p> | |
| 30 | ${s.phase === "idle" | |
| 31 | ? card(html`<form method="post" action="/maintenance/prepare"> | |
| 32 | <p>${ABOUT_TO_STOP}</p> | |
| 33 | <label for="reason">Shown to visitors (optional)</label> | |
| 34 | <input id="reason" name="reason" maxlength="200" autocomplete="off" /> | |
| 35 | <label><input type="checkbox" name="confirm" value="yes" required /> I understand the crew and releases stop now</label> | |
| 36 | ${btn("Prepare for maintenance", { variant: "danger" })} | |
| 37 | </form> | |
| 38 | ${s.last ? html`<p id="maintenance-last" class="muted">${s.last.says}</p>` : null}`) | |
| 39 | : card(html`<p>Since ${s.since}. Backup: ${s.backup?.says}</p> | |
| 40 | <form method="post" action="/maintenance/prepare">${btn("Check again and retake the backup", { variant: "outline" })}</form> | |
| 41 | <form method="post" action="/maintenance/resume">${btn("End maintenance now")}</form>`)}`; | |
| 42 | const response = page({ ctx, title: "Maintenance", body }); | |
| 43 | notePhase(ctx.request, "render", performance.now() - rendering); | |
| 44 | return response; | |
| 45 | } | |
| 46 | ||
| 47 | export async function maintenancePrepare(ctx: Ctx): Promise<Response> { | |
| 48 | const denied = admin(ctx); | |
| 49 | if (denied) return denied; | |
| 50 | const form = await readForm(ctx, "admin"); | |
| 51 | if (form instanceof Response) return form; | |
| 52 | // Idle -> prepared stops the crew: asked first. A re-check while prepared has already been agreed. | |
| 53 | if (status(ctx.app).phase === "idle" && field(form, "confirm") !== "yes") | |
| 54 | return flashTo(ctx, "/maintenance", { tone: "danger", message: "Tick the box to confirm: the crew and releases stop." }); | |
| 55 | const r = await prepare(ctx.app, ctx.auth.user, field(form, "reason")); | |
| 56 | return flashTo(ctx, "/maintenance", r.ok ? { message: r.value.says } : { tone: "danger", message: r.error.says }); | |
| 57 | } | |
| 58 | ||
| 59 | export async function maintenanceResume(ctx: Ctx): Promise<Response> { | |
| 60 | const denied = admin(ctx); | |
| 61 | if (denied) return denied; | |
| 62 | const form = await readForm(ctx, "admin"); | |
| 63 | if (form instanceof Response) return form; | |
| 64 | const r = resumeNow(ctx.app, ctx.auth.user); | |
| 65 | return flashTo(ctx, "/maintenance", r.ok ? { message: r.value.says } : { tone: "danger", message: r.error.says }); | |
| 66 | } |
src/web/pages/resilience.ts +1
| @@ -107,6 +107,7 @@ export function resiliencePage(ctx: Ctx): Response { | ||
| 107 | 107 | </p> |
| 108 | 108 | </div> |
| 109 | 109 | ${backupsLineHtml(ctx)} |
| 110 | <p><a href="/maintenance">Maintenance</a> prepares the forge for a planned power-off.</p> | |
| 110 | 111 | ${nightlyHtml(ctx)} |
| 111 | 112 | <p id="tally"> |
| 112 | 113 | ${String(count("green"))} green · ${String(count("red"))} red · |
test/conversation-confirm.test.ts +1 −1
| @@ -123,7 +123,7 @@ describe("the one list of what asks first", () => { | ||
| 123 | 123 | "access", "account_verify", "app_members", "app_notes", "asks_act", "asks_decide", "asks_proposal", "asks_undecide", |
| 124 | 124 | "assign", "attach_look", "bet", "board_pins", "board_write", "branches", "capacity", "valves", "cloud_accounts", "close", "confirm_wait", |
| 125 | 125 | "crew", "decisions_by", "detach", "edit", "email_verify", "feed_chips", "feed_dismiss", "flag_set", "flag_winner", |
| 126 | "follow_mode", "fork", "front_page", "hold", "issue_teams", "keep_plan", "labels", "lift", "limb", "make_it_yours", "map", "merge", "new_app", | |
| 126 | "follow_mode", "fork", "front_page", "hold", "issue_teams", "keep_plan", "labels", "lift", "limb", "maintenance", "make_it_yours", "map", "merge", "new_app", | |
| 127 | 127 | "next_up", "week", "notify", "open_pull", "org_create", "origin_apps", "pins", "plan", "profile", "propose_to_parent_forge", "repo_labels", "reports", |
| 128 | 128 | "restore_drill", "review", "say", "signup", "snapshot_now", "spend", "steer", "sync", "take_back", "take_updates", |
| 129 | 129 | "teams", "tell", "threads", "trail", "upstream", "variant", "wait_for", "watch", "whoami_can", "write_file", |
test/forge-looks.test.ts +1
| @@ -37,6 +37,7 @@ const NOT_A_PAGE: Record<string, string> = { | ||
| 37 | 37 | "/opensearch.xml": "the browser search description, XML", |
| 38 | 38 | "/sw.js": "the service worker script", |
| 39 | 39 | "/audit.csv": "a CSV download of /audit", |
| 40 | "/maintenance": "no picture: the cap of 24 looks is spent; an admin's page, covered by test/maintenance.test.ts", | |
| 40 | 41 | "/claim": "answers 404 to an admin: it exists only for a link-holder", |
| 41 | 42 | "/first-admin": "the setup form for a forge with no admin yet", |
| 42 | 43 | "/signin": "a sign-in form, not a page of the forge", |
test/maintenance-crew.test.ts added +49
| @@ -0,0 +1,49 @@ | ||
| 1 | /** While the forge is in maintenance the real dispatcher starts nothing and says why. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { chmodSync, writeFileSync } from "node:fs"; | |
| 4 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { createApp, type App } from "../src/app.ts"; | |
| 8 | import { createRun, getRun } from "../src/core/runs.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import { startCrew } from "../src/crew/dispatch.ts"; | |
| 11 | import { serve } from "../src/http/server.ts"; | |
| 12 | import { writeState } from "../src/maintenance/state.ts"; | |
| 13 | import { heldRuns } from "../src/runs/held.ts"; | |
| 14 | ||
| 15 | let dataDir: string; | |
| 16 | let app: App; | |
| 17 | let repoId = ""; | |
| 18 | ||
| 19 | beforeAll(async () => { | |
| 20 | dataDir = await mkdtemp(join(tmpdir(), "forge-maint-crew-")); | |
| 21 | app = createApp({ dataDir, port: 0, disk: () => ({ freeBytes: 100, totalBytes: 100 }) }); | |
| 22 | const owner = await createUser(app.db, { origin: "seed", trusted: true, username: "holder", email: "[email protected]", password: "a-good-passphrase" }); | |
| 23 | if (!owner.ok) throw new Error(String(owner.error)); | |
| 24 | repoId = crypto.randomUUID(); | |
| 25 | app.db.query("INSERT INTO repo (id, owner_id, name, visibility, created_at) VALUES (?1, ?2, 'm', 'private', ?3)").run(repoId, owner.value.id, new Date().toISOString()); | |
| 26 | }); | |
| 27 | afterAll(async () => { | |
| 28 | app.close(); | |
| 29 | await rm(dataDir, { recursive: true, force: true }); | |
| 30 | }); | |
| 31 | ||
| 32 | test("queued runs stay queued, held by maintenance", async () => { | |
| 33 | writeState(dataDir, { current: { since: new Date().toISOString(), by: "a", reason: "", holdId: 1, backup: { ok: true, says: "x" } } }); | |
| 34 | const run = createRun(app.db, { repoId, kind: "agent" }); | |
| 35 | const servers = await serve(app); | |
| 36 | const fake = join(dataDir, "fake-claude"); | |
| 37 | writeFileSync(fake, `#!/bin/sh\nif [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi\nexit 1\n`); | |
| 38 | chmodSync(fake, 0o755); | |
| 39 | const crew = await startCrew(app, { forgeUrl: servers.url, localUrl: servers.url, binary: fake, concurrency: 2, budgetUsd: 1, runTimeoutMs: 60_000, tickMs: 20 }); | |
| 40 | try { | |
| 41 | const deadline = Date.now() + 10_000; | |
| 42 | while (heldRuns(app.db).get(run.id) !== "maintenance" && Date.now() < deadline) await Bun.sleep(20); | |
| 43 | expect(heldRuns(app.db).get(run.id)).toBe("maintenance"); | |
| 44 | expect(getRun(app.db, run.id)?.state).toBe("queued"); | |
| 45 | } finally { | |
| 46 | await crew.stop(); | |
| 47 | servers.stop(); | |
| 48 | } | |
| 49 | }); |
test/maintenance.test.ts added +95
| @@ -0,0 +1,95 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { maintenanceHtml } from "../src/maintenance/page.ts"; | |
| 6 | import { inMaintenance, readState, writeState } from "../src/maintenance/state.ts"; | |
| 7 | import { HOLD_WORDS } from "../src/runs/held.ts"; | |
| 8 | ||
| 9 | test("state: missing or damaged file is not maintenance", async () => { | |
| 10 | const dir = mkdtempSync(join(tmpdir(), "maint-")); | |
| 11 | expect(inMaintenance(dir)).toBe(false); | |
| 12 | await Bun.write(join(dir, "maintenance.json"), "{nope"); | |
| 13 | expect(inMaintenance(dir)).toBe(false); | |
| 14 | writeState(dir, { current: { since: "2026-11-08T00:00:00Z", by: "a", reason: "", holdId: 1, backup: { ok: true, says: "x" } } }); | |
| 15 | expect(inMaintenance(dir)).toBe(true); | |
| 16 | expect(readState(dir).current?.holdId).toBe(1); | |
| 17 | }); | |
| 18 | ||
| 19 | test("page escapes the reason and refreshes", () => { | |
| 20 | const h = maintenanceHtml({ reason: "<script>alert(1)</script>", since: "2026-11-08T00:00:00Z" }); | |
| 21 | expect(h).not.toContain("<script>"); | |
| 22 | expect(h).toContain("<script>"); | |
| 23 | expect(h).toContain('http-equiv="refresh"'); | |
| 24 | expect(maintenanceHtml({ since: '"><b>' })).not.toContain("<b>"); | |
| 25 | }); | |
| 26 | ||
| 27 | test("vitals names maintenance as a hold", () => { | |
| 28 | expect(HOLD_WORDS.maintenance).toBe("planned maintenance"); | |
| 29 | }); | |
| 30 | ||
| 31 | import { afterAll, beforeAll, describe } from "bun:test"; | |
| 32 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 33 | import { createApp, type App } from "../src/app.ts"; | |
| 34 | import { createUser } from "../src/core/users.ts"; | |
| 35 | import { standingHolds } from "../src/keeper/hold.ts"; | |
| 36 | import { prepare, resume, resumeAtBoot, status } from "../src/maintenance/service.ts"; | |
| 37 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; | |
| 38 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 39 | import { quietLogs } from "./helpers/globals.ts"; | |
| 40 | ||
| 41 | quietLogs(); | |
| 42 | ||
| 43 | describe("prepare and come back", () => { | |
| 44 | let dataDir: string; | |
| 45 | let app: App; | |
| 46 | let svc: SnapshotsHandle | undefined; | |
| 47 | let admin: any; | |
| 48 | let bob: any; | |
| 49 | ||
| 50 | beforeAll(async () => { | |
| 51 | dataDir = await mkdtemp(join(tmpdir(), "forge-maint-")); | |
| 52 | app = createApp({ dataDir, port: 0 }); | |
| 53 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 54 | const b = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" }); | |
| 55 | if (!a.ok || !b.ok) throw new Error("user"); | |
| 56 | admin = a.value; | |
| 57 | bob = b.value; | |
| 58 | }); | |
| 59 | afterAll(async () => { | |
| 60 | await svc?.stop(); | |
| 61 | app.close(); | |
| 62 | await rm(dataDir, { recursive: true, force: true }); | |
| 63 | }); | |
| 64 | ||
| 65 | test("a non-admin cannot prepare", async () => { | |
| 66 | const r = await prepare(app, bob, ""); | |
| 67 | expect(r.ok).toBe(false); | |
| 68 | expect(inMaintenance(dataDir)).toBe(false); | |
| 69 | }); | |
| 70 | ||
| 71 | test("without a backup place it holds and pauses but is not ready", async () => { | |
| 72 | const r = await prepare(app, admin, "moving the rack"); | |
| 73 | expect(r.ok).toBe(true); | |
| 74 | expect(inMaintenance(dataDir)).toBe(true); | |
| 75 | expect(standingHolds(app.db)).toHaveLength(1); | |
| 76 | expect(status(app).phase).toBe("draining"); | |
| 77 | expect(status(app).says).toContain("nowhere to put a snapshot"); | |
| 78 | }); | |
| 79 | ||
| 80 | test("boot ends it: hold lifted, crew unpaused, the return is told", () => { | |
| 81 | resumeAtBoot(app); | |
| 82 | expect(inMaintenance(dataDir)).toBe(false); | |
| 83 | expect(standingHolds(app.db)).toHaveLength(0); | |
| 84 | expect(status(app).last?.verified.length).toBeGreaterThan(0); | |
| 85 | expect(resume(app, "x").ok).toBe(false); | |
| 86 | }); | |
| 87 | ||
| 88 | test("with a store it is ready once the backup is taken and nothing runs", async () => { | |
| 89 | svc = startSnapshots(app, dirStore(join(dataDir, "store")), { version: "t", nightly: false }); | |
| 90 | const r = await prepare(app, admin, ""); | |
| 91 | expect(r.ok && r.value.phase).toBe("ready"); | |
| 92 | expect(await svc.list()).toHaveLength(1); | |
| 93 | resume(app, "alice"); | |
| 94 | }); | |
| 95 | }); |
test/mcp.test.ts +1
| @@ -318,6 +318,7 @@ describe("the forge over MCP", () => { | ||
| 318 | 318 | "feed", |
| 319 | 319 | "feed_chips", |
| 320 | 320 | "feed_dismiss", |
| 321 | "maintenance", | |
| 321 | 322 | "make_it_yours", |
| 322 | 323 | "origin_apps", |
| 323 | 324 | "front_page", |
new above
offline