Public: anyone can read this; the crew can change it.
/maintenance shows, for each duty, when it last ran and what it found, in one line each #10952
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10939-maintenance-shows-for-each-duty-when-it → main 5 commits fork 2 threads · 2 unresolved
7 files
changed +115
−3
docs/features/maintenance.md +6
| @@ -16,6 +16,12 @@ | ||
| 16 | 16 | The page says plainly what stops: releases, new crew runs, then the forge, its |
| 17 | 17 | apps and previews when the host powers off. |
| 18 | 18 | |
| 19 | **Duties.** `/maintenance`, the terminal's Maintenance screen and the `duties` of MCP `maintenance {action: "status"}` | |
| 20 | list each nightly job and the radar in one line (`src/maintenance/duties.ts`, one function for all three doors): | |
| 21 | when it last ran (UTC), what it found ("nothing found" when it filed nothing), and "overdue". A duty is | |
| 22 | overdue when now is past its last slot plus one day plus 6 hours of grace (a night may run long); one | |
| 23 | that never ran says so and gives its slot, and is not overdue. | |
| 24 | ||
| 19 | 25 | **Come back.** On boot `resumeAtBoot` acts only when the host has restarted since the prepare |
| 20 | 26 | (`/proc/sys/kernel/random/boot_id` differs from the one stored); a release roll of the process on |
| 21 | 27 | the same host boot keeps what the admin prepared. It then lifts the hold, unpauses the crew and |
src/http/mcp/tools/maintenance.ts +3 −2
| @@ -1,11 +1,12 @@ | ||
| 1 | 1 | import { prepare, resumeNow, status } from "../../../maintenance/service.ts"; |
| 2 | import { duties } from "../../../maintenance/duties.ts"; | |
| 2 | 3 | import { statusWithStandby } from "../../../maintenance/standby.ts"; |
| 3 | 4 | import { scopeAllows } from "../../../core/tokens.ts"; |
| 4 | 5 | import { good, bad, type Tool } from "../shared.ts"; |
| 5 | 6 | |
| 6 | 7 | export const tool: Tool = { |
| 7 | 8 | name: "maintenance", |
| 8 | doc: "`action` is `status` (default), `prepare` or `resume`; `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says, standby: { state, says } }`", | |
| 9 | doc: "`action` is `status` (default), `prepare` or `resume`; `reason` (prepare only, at most 200 characters) is shown to visitors; a forge admin, and prepare and resume need an admin-scope token; answers the status `{ phase, since?, reason?, backup?, working, says, standby: { state, says }, duties }`", | |
| 9 | 10 | description: "Prepare the forge for a power-off, check the drain, or end maintenance (admin).", |
| 10 | 11 | inputSchema: { |
| 11 | 12 | type: "object", |
| @@ -15,7 +16,7 @@ export const tool: Tool = { | ||
| 15 | 16 | async run(ctx, user, args) { |
| 16 | 17 | if (!user.isAdmin) return bad("not_found", "no such tool here"); |
| 17 | 18 | const action = args["action"] ?? "status"; |
| 18 | if (action === "status") return good(await statusWithStandby(status(ctx.app))); | |
| 19 | if (action === "status") return good({ ...(await statusWithStandby(status(ctx.app))), duties: duties(ctx.app.config.dataDir).map((d) => d.says) }); | |
| 19 | 20 | if (action !== "prepare" && action !== "resume") return bad("invalid", "action is status, prepare or resume"); |
| 20 | 21 | if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) return bad("forbidden", "maintenance needs an admin-scope token"); |
| 21 | 22 | if (action === "resume") { |
src/maintenance/duties.ts added +58
| @@ -0,0 +1,58 @@ | ||
| 1 | /** | |
| 2 | * What each maintenance duty (the nightly jobs and the radar) last did, one | |
| 3 | * line apiece: when it last ran in the UI's clock (UTC, as `nightly/says.ts`), | |
| 4 | * what it found, and "overdue" in plain words once it is past its period | |
| 5 | * plus a grace. Plain data in, plain data out; the page and the MCP read it. | |
| 6 | */ | |
| 7 | ||
| 8 | import { JOBS } from "../nightly/jobs.ts"; | |
| 9 | import { RADAR_NAME, RADAR_SLOT } from "../nightly/radar.ts"; | |
| 10 | import { readLast, type Last } from "../nightly/stamps.ts"; | |
| 11 | import type { Slot } from "../nightly/slots.ts"; | |
| 12 | ||
| 13 | const DAY = 86_400_000; | |
| 14 | /** A night that began its slot may run long (the hunt is twenty half-hour slices): overdue only past this much more. */ | |
| 15 | export const GRACE_MS = 6 * 3_600_000; | |
| 16 | ||
| 17 | export interface Duty { | |
| 18 | readonly name: string; | |
| 19 | /** "never" before the first run; otherwise the last start, `YYYY-MM-DD HH:MM UTC`. */ | |
| 20 | readonly lastRun: string; | |
| 21 | /** What it found, one line; "nothing found" when it filed nothing. */ | |
| 22 | readonly found: string; | |
| 23 | readonly overdue: boolean; | |
| 24 | /** What the row says after the name, without the overdue mark. */ | |
| 25 | readonly detail: string; | |
| 26 | /** The row in words, the same in the terminal and in the MCP answer. */ | |
| 27 | readonly says: string; | |
| 28 | } | |
| 29 | ||
| 30 | const when = (ms: number): string => `${new Date(ms).toISOString().slice(0, 16).replace("T", " ")} UTC`; | |
| 31 | ||
| 32 | const oneLine = (s: string): string => { | |
| 33 | const t = s.replace(/\s+/g, " ").trim().replace(/\.$/, ""); | |
| 34 | return t === "" || /^(filed|found) nothing$/i.test(t) || /^read \d+ repositor(y|ies); filed nothing$/i.test(t) ? "nothing found" : t.slice(0, 160); | |
| 35 | }; | |
| 36 | ||
| 37 | export function dutyOf(name: string, slot: Slot, last: Last | undefined, now: number): Duty { | |
| 38 | if (last === undefined) { | |
| 39 | const at = `${String(slot.hour).padStart(2, "0")}:${String(slot.minute).padStart(2, "0")} UTC`; | |
| 40 | return { name, lastRun: "never", found: "nothing found", overdue: false, detail: `never run; its slot is ${at} each night`, says: `${name}: never run; its slot is ${at} each night.` }; | |
| 41 | } | |
| 42 | const overdue = now > last.slot + DAY + GRACE_MS; | |
| 43 | const found = last.finishedAt === undefined ? "not finished" : oneLine(last.says); | |
| 44 | return { | |
| 45 | name, | |
| 46 | lastRun: when(last.startedAt), | |
| 47 | found, | |
| 48 | overdue, | |
| 49 | detail: `last ran ${when(last.startedAt)}; ${found}`, | |
| 50 | says: `${name}: last ran ${when(last.startedAt)}; ${found}${overdue ? "; overdue" : ""}.`, | |
| 51 | }; | |
| 52 | } | |
| 53 | ||
| 54 | export function duties(dataDir: string, now: number = Date.now()): Duty[] { | |
| 55 | return [...JOBS.map((j) => ({ name: j.name as string, slot: j.slot })), { name: RADAR_NAME, slot: RADAR_SLOT }].map((d) => | |
| 56 | dutyOf(d.name, d.slot, readLast(dataDir, d.name), now), | |
| 57 | ); | |
| 58 | } |
src/tui/screens/maintenance.ts +3 −1
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | * refused by the service. Enter, Esc or q closes it. |
| 8 | 8 | */ |
| 9 | 9 | |
| 10 | import { duties } from "../../maintenance/duties.ts"; | |
| 10 | 11 | import { ABOUT_TO_STOP_ROWS, prepare, resumeNow, status } from "../../maintenance/service.ts"; |
| 11 | 12 | import { probeStandby, withStandby, type Standby } from "../../maintenance/standby.ts"; |
| 12 | 13 | import type { Ctx, Screen } from "../app/app.ts"; |
| @@ -42,7 +43,8 @@ export function maintenanceLines(f: Session, standby?: Standby): string[] { | ||
| 42 | 43 | ...[...last.checks].sort((a, b) => Number(a.ok) - Number(b.ok)).flatMap((c) => wrap(`${c.ok ? "ok " : "FAILED"} ${c.name}: ${c.says}`, INNER)), |
| 43 | 44 | ] |
| 44 | 45 | : []; |
| 45 | return [...wrap(s.says, INNER), "", ...facts, ...checks]; | |
| 46 | const dutyLines = duties(f.app.config.dataDir).flatMap((d) => wrap(d.says, INNER)); | |
| 47 | return [...wrap(s.says, INNER), "", ...facts, ...checks, "", ...dutyLines]; | |
| 46 | 48 | } |
| 47 | 49 | |
| 48 | 50 | export function maintenanceScreen(f: Session, ctx: Ctx): Screen { |
src/web/pages/maintenance.ts +4
| @@ -5,6 +5,7 @@ | ||
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | 7 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; |
| 8 | import { duties } from "../../maintenance/duties.ts"; | |
| 8 | 9 | import { statusWithStandby } from "../../maintenance/standby.ts"; |
| 9 | 10 | import { ABOUT_TO_STOP_ROWS, prepare, resumeNow, status } from "../../maintenance/service.ts"; |
| 10 | 11 | import type { Ctx } from "../../http/context.ts"; |
| @@ -31,6 +32,9 @@ export async function maintenancePage(ctx: Ctx): Promise<Response> { | ||
| 31 | 32 | const s = await statusWithStandby(status(ctx.app)); |
| 32 | 33 | const body = html`<div class="page-head"><h1>Maintenance</h1></div> |
| 33 | 34 | <p id="maintenance-says">${s.says}</p> |
| 35 | <dl id="maintenance-duties" class="status-rows"> | |
outdated
Principle 1/7: the 'Duties' h2 plus a muted |
||
| 36 | ${duties(ctx.app.config.dataDir).map((d) => fact(d.name, html`${d.detail}${d.overdue ? html` ${badge("overdue", { tone: "danger" })}` : null}`))} | |
| 37 | </dl> | |
| 34 | 38 | ${s.phase === "idle" ? html`<p id="maintenance-standby" class="muted">${s.standby.says}</p>` : null} |
| 35 | 39 | ${s.phase === "idle" |
| 36 | 40 | ? html`<p class="muted">Get the forge ready for a planned power-off. What stops:</p> |
test/maintenance-duties.test.ts added +37
| @@ -0,0 +1,37 @@ | ||
| 1 | /** /maintenance and the MCP say, per duty, when it last ran, what it found and when it is overdue. */ | |
| 2 | import { expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { duties, GRACE_MS } from "../src/maintenance/duties.ts"; | |
| 7 | import { writeLast } from "../src/nightly/stamps.ts"; | |
| 8 | ||
| 9 | const DAY = 86_400_000; | |
| 10 | const at = (iso: string): number => Date.parse(iso); | |
| 11 | ||
| 12 | test("never run, fresh and overdue, with a fake clock", async () => { | |
| 13 | const dir = await mkdtemp(join(tmpdir(), "forge-duties-")); | |
| 14 | try { | |
| 15 | const now = at("2026-11-10T12:00:00Z"); | |
| 16 | const never = duties(dir, now); | |
| 17 | expect(never.map((d) => d.name)).toEqual(["flake-hunt", "outside-gate", "pack-compat", "radar"]); | |
| 18 | expect(never.every((d) => d.lastRun === "never" && !d.overdue)).toBe(true); | |
| 19 | expect(never[0]!.says).toBe("flake-hunt: never run; its slot is 02:17 UTC each night."); | |
| 20 | ||
| 21 | const slot = at("2026-11-10T05:07:00Z"); | |
| 22 | writeLast(dir, "radar", { slot, startedAt: slot + 1000, finishedAt: slot + 5000, says: "read 3 repositories; filed nothing." }); | |
| 23 | writeLast(dir, "flake-hunt", { slot: at("2026-11-10T02:17:00Z"), startedAt: at("2026-11-10T02:17:30Z"), finishedAt: at("2026-11-10T04:00:00Z"), says: "filed 2 flakes." }); | |
| 24 | const fresh = duties(dir, now); | |
| 25 | expect(fresh[3]!.says).toBe("radar: last ran 2026-11-10 05:07 UTC; nothing found."); | |
| 26 | expect(fresh[0]!.says).toBe("flake-hunt: last ran 2026-11-10 02:17 UTC; filed 2 flakes."); | |
| 27 | expect(fresh[3]!.overdue).toBe(false); | |
| 28 | ||
| 29 | const later = slot + DAY + GRACE_MS + 1; | |
| 30 | const late = duties(dir, later); | |
| 31 | expect(late[3]!.overdue).toBe(true); | |
| 32 | expect(late[3]!.says).toBe("radar: last ran 2026-11-10 05:07 UTC; nothing found; overdue."); | |
| 33 | expect(duties(dir, slot + DAY + GRACE_MS)[3]!.overdue).toBe(false); | |
| 34 | } finally { | |
| 35 | await rm(dir, { recursive: true, force: true }); | |
| 36 | } | |
| 37 | }); |
test/maintenance.test.ts +4
| @@ -158,13 +158,17 @@ describe("prepare and come back", () => { | ||
| 158 | 158 | const html = await (await fetch(`${servers.url}/maintenance`, { headers: { cookie } })).text(); |
| 159 | 159 | expect(html).toContain('id="maintenance-standby"'); |
| 160 | 160 | expect(html).toContain("Standby answering"); |
| 161 | expect(html).toContain('id="maintenance-duties"'); | |
| 162 | expect(html).toMatch(/<dt>radar<\/dt>\s*<dd>never run/); | |
| 161 | 163 | const api = (await (await fetch(`${servers.url}/api/v1/admin/maintenance`, { headers: { authorization: `Bearer ${adminRo}` } })).json()) as { standby: { state: string } }; |
| 162 | 164 | expect(api.standby.state).toBe("answering"); |
| 163 | 165 | const mcp = await (await post("/mcp", { authorization: `Bearer ${adminAdmin}`, accept: "application/json, text/event-stream" }, |
| 164 | 166 | JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "maintenance", arguments: { action: "status" } } }))).text(); |
| 165 | 167 | expect(mcp).toContain("answering"); |
| 168 | expect(mcp).toContain("radar: never run"); | |
| 166 | 169 | const lines = maintenanceLines({ user: admin, app, db: app.db } as never, api.standby as never).join("\n"); |
| 167 | 170 | expect(lines).toContain("Standby answering"); |
| 171 | expect(lines).toContain("radar: never run"); | |
| 168 | 172 | } finally { |
| 169 | 173 | if (before === undefined) delete process.env["MAINTENANCE_STANDBY_URL"]; |
| 170 | 174 | else process.env["MAINTENANCE_STANDBY_URL"] = before; |
Reads stamp files synchronously per request on both doors; fine at four duties, but the terminal door must reuse this same function, not a copy.