Public: anyone can read this; the crew can change it.
Resilience in the terminal: the failure-domain map and what needs the owner, over SSH #2208
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1236-resilience-in-the-terminal-the-failure-d → main 2 commits fork 2 threads · 2 unresolved
5 files
changed +266
−1
README.md +1
| @@ -689,6 +689,7 @@ last kept drill (when, green or red with its reason, seconds until back) or | ||
| 689 | 689 | the sentence on why it is not drilled. In the terminal it is the **Failure domains** |
| 690 | 690 | command (`ctrl+k`; the **Resilience** command is the THREATS.md one), shown to an admin only and read through the same |
| 691 | 691 | `listResilience` action. Only the latest drill per domain is shown. |
| 692 | The **Owner's needs** command (admin only, `ownerView`) is the owner's one screen: the forge's needs as the board strip says them, where snapshots go (and any mirror) with the newest snapshot, and every domain with what the owner does. | |
| 692 | 693 | |
| 693 | 694 | A red drill, or none in eight days, is one line for the forge's admin on the |
| 694 | 695 | board's forge strip, in the terminal's dashboard (same `forgeNeeds`), and in |
src/actions/resilience.ts +44
| @@ -10,6 +10,8 @@ import { lastDrill, type LastDrill } from "../core/domain_drills.ts"; | ||
| 10 | 10 | import type { User } from "../core/users.ts"; |
| 11 | 11 | import { DOMAINS } from "../kernel/domains.ts"; |
| 12 | 12 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 13 | import { snapshotsOf } from "../snapshot/service.ts"; | |
| 14 | import { forgeNeeds } from "./health.ts"; | |
| 13 | 15 | |
| 14 | 16 | export interface ResilienceRow { |
| 15 | 17 | readonly key: string; |
| @@ -42,3 +44,45 @@ export function listResilience( | ||
| 42 | 44 | })), |
| 43 | 45 | ); |
| 44 | 46 | } |
| 47 | ||
| 48 | /** Where snapshots go and the newest one, as Settings → Backups says them. */ | |
| 49 | export interface SnapshotView { | |
| 50 | /** The store's name (never a secret), or null when the forge has none. */ | |
| 51 | readonly where: string | null; | |
| 52 | readonly mirror: string | null; | |
| 53 | /** The newest snapshot's finish time and size, or null when none is kept (or the store cannot be read). */ | |
| 54 | readonly last: { readonly at: string; readonly bytes: number } | null; | |
| 55 | } | |
| 56 | ||
| 57 | export interface OwnerView { | |
| 58 | readonly needs: string[]; | |
| 59 | readonly snapshots: SnapshotView; | |
| 60 | readonly domains: (ResilienceRow & { readonly owner: string })[]; | |
| 61 | } | |
| 62 | ||
| 63 | /** What needs the owner: the forge's needs, its snapshot store, and every failure domain. */ | |
| 64 | export async function ownerView( | |
| 65 | app: App, | |
| 66 | actor: User | undefined, | |
| 67 | ): Promise<Result<OwnerView, "not_found">> { | |
| 68 | const listed = listResilience(app, actor); | |
| 69 | if (!listed.ok) return listed; | |
| 70 | const handle = snapshotsOf(app); | |
| 71 | let last: SnapshotView["last"] = null; | |
| 72 | try { | |
| 73 | const newest = (await handle?.list())?.[0]; | |
| 74 | if (newest !== undefined) last = { at: newest.finishedAt, bytes: newest.bytes }; | |
| 75 | } catch { | |
| 76 | // an unreadable store is a nothing-to-show, the domain row says what to do | |
| 77 | } | |
| 78 | const owner = new Map(DOMAINS.map((d) => [d.key, d.owner])); | |
| 79 | return ok({ | |
| 80 | needs: forgeNeeds(app, actor) ?? [], | |
| 81 | snapshots: { | |
| 82 | where: handle?.store?.where ?? null, | |
| 83 | mirror: handle?.mirror?.where ?? null, | |
| 84 | last, | |
| 85 | }, | |
| 86 | domains: listed.value.map((d) => ({ ...d, owner: owner.get(d.key) ?? "" })), | |
| 87 | }); | |
| 88 | } |
src/tui/forge.ts +9
| @@ -45,6 +45,7 @@ import { flagsScreen } from "./screens/flags.ts"; | ||
| 45 | 45 | import { usageScreen } from "./screens/usage.ts"; |
| 46 | 46 | import { quotasScreen } from "./screens/quotas.ts"; |
| 47 | 47 | import { domainsScreen } from "./screens/domains.ts"; |
| 48 | import { ownerScreen } from "./screens/owner.ts"; | |
| 48 | 49 | import { resilienceLines, resilienceScreen } from "./screens/resilience.ts"; |
| 49 | 50 | import { flowLines, flowScreen } from "./screens/flow.ts"; |
| 50 | 51 | import { memoryScreen, systemLines, systemScreen } from "./screens/system.ts"; |
| @@ -370,6 +371,14 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] { | ||
| 370 | 371 | hint: "failure domains and their last drill", |
| 371 | 372 | run: () => ctx.push(domainsScreen(session, ctx)), |
| 372 | 373 | }); |
| 374 | // What needs the owner: needs, snapshot store, every failure domain (#1236). | |
| 375 | if (session.user?.isAdmin === true && session.user.kind !== "agent") | |
| 376 | commands.push({ | |
| 377 | id: "owner", | |
| 378 | title: "Owner's needs", | |
| 379 | hint: "what needs the owner, snapshots, failure domains", | |
| 380 | run: () => ctx.push(ownerScreen(session, ctx)), | |
| 381 | }); | |
| 373 | 382 | commands.push({ |
| 374 | 383 | id: "family", |
| 375 | 384 | title: "Family", |
src/tui/screens/owner.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** | |
| 2 | * What needs the owner, in the terminal, a forge admin's: the forge's needs, | |
| 3 | * where snapshots go and the newest one, and every failure domain with what | |
| 4 | * the owner does — `ownerView`, the action behind Settings → Failure domains | |
| 5 | * and Backups. Anyone else gets an empty screen that says nothing exists. | |
| 6 | */ | |
| 7 | ||
| 8 | import { ownerView, type OwnerView } from "../../actions/resilience.ts"; | |
| 9 | import type { Ctx, Screen } from "../app/app.ts"; | |
| 10 | import { binding } from "../app/keys.ts"; | |
| 11 | import { fill, len, split } from "../ui/layout.ts"; | |
| 12 | import { style } from "../term/style.ts"; | |
| 13 | import { truncate } from "../term/width.ts"; | |
| 14 | import type { Session } from "../session.ts"; | |
| 15 | import { ListState, drawList, type ListRow } from "./common.ts"; | |
| 16 | ||
| 17 | export function ownerRows(v: OwnerView): ListRow[] { | |
| 18 | const needs: ListRow[] = | |
The act-needs row and the snapshot variants (none yet / kept / mirrored to) have no test. Only the no-store case is asserted. |
||
| 19 | v.needs.length === 0 | |
| 20 | ? [{ title: "Needs", secondary: "all fine", meta: "Nothing asks for the owner now." }] | |
| 21 | : v.needs.map((n) => ({ title: "Needs", secondary: "act", meta: n })); | |
| 22 | const s = v.snapshots; | |
| 23 | const snaps: ListRow = { | |
| 24 | title: "Snapshots", | |
| 25 | secondary: s.where === null ? "none" : s.last === null ? "none yet" : "kept", | |
| 26 | meta: | |
| 27 | s.where === null | |
| 28 | ? "This forge was given nowhere to put a snapshot." | |
| 29 | : `To ${s.where}${s.mirror === null ? ", mirrored nowhere" : `, mirrored to ${s.mirror}`} · ${ | |
| 30 | s.last === null | |
| 31 | ? "no snapshot read back" | |
| 32 | : `last ${s.last.at.slice(0, 16).replace("T", " ")} UTC, ${s.last.bytes} bytes` | |
| 33 | }`, | |
| 34 | }; | |
| 35 | const domains: ListRow[] = v.domains.map((d) => ({ | |
| 36 | title: d.name, | |
| 37 | secondary: `back in ~${d.backMinutes} min`, | |
| 38 | meta: `${d.breaks} · Owner: ${d.owner}`, | |
| 39 | })); | |
| 40 | return [...needs, snaps, ...domains]; | |
| 41 | } | |
| 42 | ||
| 43 | export function ownerScreen(f: Session, ctx: Ctx): Screen { | |
| 44 | const state = new ListState(); | |
| 45 | let view: OwnerView | undefined; | |
| 46 | let loaded = false; | |
| 47 | ||
| 48 | const screen: Screen = { | |
| 49 | title: "What needs the owner", | |
| 50 | keys: [binding(["j", "k"], "move cursor")], | |
| 51 | async load() { | |
| 52 | const got = await ownerView(f.app, f.user); | |
| 53 | view = got.ok ? got.value : undefined; | |
| 54 | loaded = true; | |
| 55 | state.clamp(view === undefined ? 0 : ownerRows(view).length); | |
| 56 | ctx.invalidate(); | |
| 57 | }, | |
| 58 | render(area, s) { | |
| 59 | const c = ctx.theme.c; | |
| 60 | const [head, list] = split(area, "v", [len(1), fill(1)]); | |
| 61 | const note = !loaded ? "Loading…" : view === undefined ? "Nothing here." : "The forge's needs, its snapshots, and each failure domain"; | |
| 62 | s.text(head!.x, head!.y, truncate(note, head!.w), style(c.mutedFg, c.bg), head!.x + head!.w); | |
| 63 | drawList(s, list!, c, state, view === undefined ? [] : ownerRows(view)); | |
| 64 | }, | |
| 65 | key(e) { | |
| 66 | return state.key(e, view === undefined ? 0 : ownerRows(view).length); | |
| 67 | }, | |
| 68 | }; | |
| 69 | return screen; | |
| 70 | } |
test/resilience.test.ts +142 −1
| @@ -10,9 +10,17 @@ import { join } from "node:path"; | ||
| 10 | 10 | import { createApp, type App } from "../src/app.ts"; |
| 11 | 11 | import { fileResults, keepForgeGate, keepGate, readGate } from "../src/checks/gate_last.ts"; |
| 12 | 12 | import { parseThreats, standing } from "../src/core/threats.ts"; |
| 13 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 14 | 14 | import type { Ctx, Screen } from "../src/tui/app/app.ts"; |
| 15 | 15 | import { forgeCommands, Session } from "../src/tui/forge.ts"; |
| 16 | import { startSnapshots } from "../src/snapshot/service.ts"; | |
| 17 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 18 | import { createForgeApp } from "../src/tui/forge.ts"; | |
| 19 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; | |
| 20 | import { settleApp } from "./helpers/tui.ts"; | |
| 21 | import { ownerView } from "../src/actions/resilience.ts"; | |
| 22 | import { DOMAINS } from "../src/kernel/domains.ts"; | |
| 23 | import { ownerRows } from "../src/tui/screens/owner.ts"; | |
| 16 | 24 | import { resilienceLines } from "../src/tui/screens/resilience.ts"; |
| 17 | 25 | import { serve, type Servers } from "../src/http/server.ts"; |
| 18 | 26 | |
| @@ -160,4 +168,137 @@ describe("resilience", () => { | ||
| 160 | 168 | expect(resilienceLines(dataDir).join("\n")).toContain("red"); |
| 161 | 169 | expect(command(false).found).toBeUndefined(); |
| 162 | 170 | }); |
| 171 | ||
| 172 | test("the owner's screen: needs, snapshots and every domain for an admin; nothing for the rest, and no command", async () => { | |
| 173 | const user = (n: string) => app.db.query("SELECT * FROM user WHERE username = ?1").get(n) as never; | |
| 174 | const alice = { ...(user("alice") as object), isAdmin: true } as never; | |
| 175 | const seen = await ownerView(app, alice); | |
| 176 | if (!seen.ok) throw new Error("admin refused"); | |
| 177 | expect(seen.value.domains.map((d) => d.key)).toEqual(DOMAINS.map((d) => d.key)); | |
| 178 | expect(seen.value.snapshots.where).toBeNull(); | |
| 179 | const text = ownerRows(seen.value).map((r) => `${r.title} ${r.meta}`).join("\n"); | |
| 180 | expect(text).toContain("nowhere to put a snapshot"); | |
| 181 | for (const d of DOMAINS) expect(text).toContain(d.name); | |
| 182 | expect(text).toContain(DOMAINS[0]!.owner); | |
| 183 | ||
| 184 | const bob = { ...(user("bob") as object), isAdmin: false } as never; | |
| 185 | expect((await ownerView(app, bob)).ok).toBe(false); | |
| 186 | expect((await ownerView(app, undefined)).ok).toBe(false); | |
| 187 | ||
| 188 | const command = (u: unknown) => { | |
| 189 | const pushed: Screen[] = []; | |
| 190 | const ctx = { push: (x: Screen) => pushed.push(x) } as unknown as Ctx; | |
| 191 | const session = new Session(app, { sshUsername: "git", remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user: u as never }); | |
| 192 | return { found: forgeCommands(session)(ctx).find((c) => c.id === "owner"), pushed }; | |
| 193 | }; | |
| 194 | const a = command(alice); | |
| 195 | a.found!.run(); | |
| 196 | expect(a.pushed.map((x) => x.title)).toEqual(["What needs the owner"]); | |
| 197 | expect(command(bob).found).toBeUndefined(); | |
| 198 | }); | |
| 199 | }); | |
| 200 | ||
| 201 | describe("the owner's screen with snapshots", () => { | |
| 202 | let dir: string; | |
| 203 | let guard: string; | |
| 204 | let forge: App; | |
| 205 | ||
| 206 | beforeAll(async () => { | |
| 207 | dir = await mkdtemp(join(tmpdir(), "forge-owner-")); | |
| 208 | guard = await mkdtemp(join(tmpdir(), "forge-owner-guard-")); | |
| 209 | process.env["FORGE_GUARD_DIR"] = guard; | |
| 210 | forge = createApp({ dataDir: dir, port: 0 }); | |
| 211 | const made = await createUser(forge.db, { | |
| 212 | trusted: true, | |
| 213 | username: "alice", | |
| 214 | email: "[email protected]", | |
| 215 | password: "a-good-passphrase", | |
| 216 | adminIfFirst: true, | |
| 217 | }); | |
| 218 | if (!made.ok) throw new Error(made.error); | |
| 219 | }); | |
| 220 | afterAll(async () => { | |
| 221 | delete process.env["FORGE_GUARD_DIR"]; | |
| 222 | forge.close(); | |
| 223 | await rm(dir, { recursive: true, force: true }); | |
| 224 | await rm(guard, { recursive: true, force: true }); | |
| 225 | }); | |
| 226 | ||
| 227 | const alice = () => getUserByName(forge.db, "alice")!; | |
| 228 | const view = async () => { | |
| 229 | const got = await ownerView(forge, alice()); | |
| 230 | if (!got.ok) throw new Error("admin refused"); | |
| 231 | return got.value; | |
| 232 | }; | |
| 233 | ||
| 234 | test("a store with a newest snapshot and a mirror: where, mirror, last, and the shared-volume need as an act row", async () => { | |
| 235 | const store = dirStore(join(dir, "snapshots")); | |
| 236 | const noMirror = startSnapshots(forge, store, { version: "t", nightly: false }); | |
| 237 | try { | |
| 238 | const taken = await noMirror.now(); | |
| 239 | const v = await view(); | |
| 240 | expect(v.snapshots.where).toBe(store.where); | |
| 241 | expect(v.snapshots.mirror).toBeNull(); | |
| 242 | expect(v.snapshots.last).toEqual({ at: taken.finishedAt, bytes: taken.bytes }); | |
| 243 | const rows = ownerRows(v); | |
| 244 | const act = rows.filter((r) => r.title === "Needs" && r.secondary === "act"); | |
| 245 | expect(act.map((r) => r.meta).join("\n")).toContain("Snapshots are kept on the forge's own volume"); | |
| 246 | const snap = rows.find((r) => r.title === "Snapshots")!; | |
| 247 | expect(snap.secondary).toBe("kept"); | |
| 248 | expect(snap.meta).toContain(`To ${store.where}, mirrored nowhere`); | |
| 249 | expect(snap.meta).toContain(`last ${taken.finishedAt.slice(0, 16).replace("T", " ")} UTC, ${taken.bytes} bytes`); | |
| 250 | } finally { | |
| 251 | await noMirror.stop(); | |
| 252 | } | |
| 253 | ||
| 254 | const mirror = dirStore(join(dir, "mirror")); | |
| 255 | const mirrored = startSnapshots(forge, store, { version: "t", nightly: false, mirror, mirrorEveryMs: 3_600_000 }); | |
| 256 | try { | |
| 257 | const v = await view(); | |
| 258 | expect(v.snapshots.mirror).toBe(mirror.where); | |
| 259 | const rows = ownerRows(v); | |
| 260 | expect(rows.find((r) => r.title === "Snapshots")!.meta).toContain(`mirrored to ${mirror.where}`); | |
| 261 | expect(rows.some((r) => r.secondary === "act")).toBe(false); | |
| 262 | expect(rows[0]).toMatchObject({ title: "Needs", secondary: "all fine" }); | |
| 263 | } finally { | |
| 264 | await mirrored.stop(); | |
| 265 | } | |
| 266 | }); | |
| 267 | ||
| 268 | test("a store whose list() throws gives last=null and no crash; an empty store reads none yet", async () => { | |
| 269 | const broken: Store = { ...dirStore(join(dir, "broken")), list: () => Promise.reject(new Error("bucket gone")) }; | |
| 270 | const svc = startSnapshots(forge, broken, { version: "t", nightly: false }); | |
| 271 | try { | |
| 272 | const v = await view(); | |
| 273 | expect(v.snapshots.where).toBe(broken.where); | |
| 274 | expect(v.snapshots.last).toBeNull(); | |
| 275 | expect(ownerRows(v).find((r) => r.title === "Snapshots")!.secondary).toBe("none yet"); | |
| 276 | } finally { | |
| 277 | await svc.stop(); | |
| 278 | } | |
| 279 | }); | |
| 280 | ||
| 281 | test("the screen loads and renders the rows in the terminal", async () => { | |
| 282 | const svc = startSnapshots(forge, dirStore(join(dir, "snapshots")), { version: "t", nightly: false }); | |
| 283 | try { | |
| 284 | const terminal = headlessTerminal(140, 32); | |
| 285 | const user = alice(); | |
| 286 | const { app } = createForgeApp(forge, { sshUsername: "t", remoteAddress: "127.0.0.1", user }, { terminal, exitHooks: false }); | |
| 287 | app.start(); | |
| 288 | const press = async (...keys: string[]) => { | |
| 289 | for (const k of keys) app.dispatch({ type: "key", key: k, ctrl: false, alt: false, shift: false }); | |
| 290 | await settleApp(app); | |
| 291 | }; | |
| 292 | app.dispatch({ type: "key", key: "k", ctrl: true, alt: false, shift: false }); | |
| 293 | await press(..."Owner's needs", "enter"); | |
| 294 | await settleApp(app); | |
| 295 | const text = app.screen.toText(); | |
| 296 | expect(text).toContain("The forge's needs, its snapshots"); | |
| 297 | expect(text).toContain("Needs"); | |
| 298 | expect(text).toContain("Snapshots"); | |
| 299 | app.stop(); | |
| 300 | } finally { | |
| 301 | await svc.stop(); | |
| 302 | } | |
| 303 | }); | |
| 163 | 304 | }); |
new above
offline
The try/catch that swallows a store failure and the mirror/last values are untested. Add a test with a store whose list() rejects, and one with a real newest snapshot.