Public: anyone can read this; the crew can change it.
The restore drill result is on /system as one line: last pass, how long, and the age of the snapshot restored #13057
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13052-the-restore-drill-result-is-on-system-as → main 4 commits fork 1 thread · 1 unresolved
8 files
changed +215
−8
docs/design/attention-budget.json +5 −5
| @@ -173,16 +173,16 @@ | ||
| 173 | 173 | }, |
| 174 | 174 | "system": { |
| 175 | 175 | "390": { |
| 176 | "words": 230, | |
| 177 | "blocks": 75, | |
| 176 | "words": 234, | |
| 177 | "blocks": 77, | |
| 178 | 178 | "colours": 0 |
| 179 | 179 | }, |
| 180 | 180 | "1280": { |
| 181 | "words": 230, | |
| 182 | "blocks": 75, | |
| 181 | "words": 234, | |
| 182 | "blocks": 77, | |
| 183 | 183 | "colours": 0 |
| 184 | 184 | }, |
| 185 | "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull." | |
| 185 | "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull. Words +4, blocks +2 (#13052): the Restore section's heading and its one sentence (\"Restore never proven.\" on an empty forge) stay on first view, so the owner sees whether a restore works without a click." | |
| 186 | 186 | }, |
| 187 | 187 | "explore": { |
| 188 | 188 | "390": { |
docs/features/resilience-backups.md +27
| @@ -31,3 +31,30 @@ The first of these that is set wins (`src/snapshot/destination.ts`): | ||
| 31 | 31 | |
| 32 | 32 | The line describes the store that the running process holds. After you save |
| 33 | 33 | a destination, the next use resolves it again and the line names it. |
| 34 | ||
| 35 | ## When a restore was last proven (#13052) | |
| 36 | ||
| 37 | A backup is only as good as the last restore that worked. The forge's | |
| 38 | restore drill restores the newest snapshot into a scratch directory and | |
| 39 | checks it; this line says when that last passed, so an owner sees it | |
| 40 | without running one. | |
| 41 | ||
| 42 | ### Where to read it | |
| 43 | ||
| 44 | - **`/system`** (admins): the **Restore** section is one sentence. | |
| 45 | - **The terminal's System screen**: the same sentence, under the forge's | |
| 46 | health line. | |
| 47 | ||
| 48 | ### What it says | |
| 49 | ||
| 50 | - `Restore never proven.` until a drill has passed. | |
| 51 | - `Restore proven 2 days ago, 45 seconds, snapshot 3 hours old.` after one: | |
| 52 | when it passed, how long it took (seconds or minutes, never | |
| 53 | milliseconds), and how old the restored snapshot was then. | |
| 54 | - A drill that failed after the last pass is added: | |
| 55 | `…; the last drill failed 2 hours ago at restore.` | |
| 56 | ||
| 57 | ### Prove one now | |
| 58 | ||
| 59 | Open `/settings/backups` and run a drill, or call the MCP tool | |
| 60 | `restore_drill`. Reload `/system`: the line reads `Restore proven just now`. |
src/snapshot/drills.ts +12 −3
| @@ -85,9 +85,18 @@ export function recordDrill(db: Database, result: DrillResult): void { | ||
| 85 | 85 | |
| 86 | 86 | /** The newest drill, or undefined when none has run. */ |
| 87 | 87 | export function lastDrill(db: Database): DrillResult | undefined { |
| 88 | const row = db | |
| 89 | .query<Row, []>("SELECT * FROM drill ORDER BY id DESC LIMIT 1") | |
| 90 | .get(); | |
| 88 | return asResult(db.query<Row, []>("SELECT * FROM drill ORDER BY id DESC LIMIT 1").get()); | |
| 89 | } | |
| 90 | ||
| 91 | /** | |
| 92 | * The newest drill that passed, or undefined when none ever has: when a | |
| 93 | * restore was last proven. | |
| 94 | */ | |
| 95 | export function lastPass(db: Database): DrillResult | undefined { | |
| 96 | return asResult(db.query<Row, []>("SELECT * FROM drill WHERE verdict = 'green' ORDER BY id DESC LIMIT 1").get()); | |
| 97 | } | |
| 98 | ||
| 99 | function asResult(row: Row | null): DrillResult | undefined { | |
| 91 | 100 | if (row === null) return undefined; |
| 92 | 101 | const kept = parsePhases(row.phases); |
| 93 | 102 | return { |
src/snapshot/restore_line.ts added +61
| @@ -0,0 +1,61 @@ | ||
| 1 | /** | |
| 2 | * The restore drill as one line (#13052): when a restore was last proven, | |
| 3 | * how long it took, and how old the snapshot it restored was, so the owner | |
| 4 | * sees a restore works without asking. `/system` prints it. | |
| 5 | * | |
| 6 | * Plain data in (the newest passing drill and the newest drill), plain | |
| 7 | * words out; no clock but the one given. | |
| 8 | */ | |
| 9 | ||
| 10 | import type { DrillResult } from "./drill.ts"; | |
| 11 | import { agoWords } from "../kernel/ago.ts"; | |
| 12 | import { snapshotStart } from "./snapshot.ts"; | |
| 13 | ||
| 14 | /** | |
| 15 | * A drill's length as a person says it: seconds under a minute and a half, | |
| 16 | * else minutes; never milliseconds. | |
| 17 | */ | |
| 18 | export function drillDuration(ms: number): string { | |
| 19 | if (!Number.isFinite(ms) || ms < 0) return "an unknown time"; | |
| 20 | if (ms < 1_500) return "1 second"; | |
| 21 | if (ms < 90_000) return `${Math.round(ms / 1000)} seconds`; | |
| 22 | const min = Math.round(ms / 60_000); | |
| 23 | return `${min} minute${min === 1 ? "" : "s"}`; | |
| 24 | } | |
| 25 | ||
| 26 | /** | |
| 27 | * How old the snapshot was when the drill restored it, from its id; | |
| 28 | * undefined when the id is no time. | |
| 29 | */ | |
| 30 | export function snapshotAgeAt(id: string | null, atIso: string): string | undefined { | |
| 31 | const start = id === null ? undefined : snapshotStart(id); | |
| 32 | const at = Date.parse(atIso); | |
| 33 | if (start === undefined || !Number.isFinite(at)) return undefined; | |
| 34 | const minutes = Math.max(0, Math.round((at - start.getTime()) / 60_000)); | |
| 35 | const of = (n: number, unit: string): string => `${n} ${unit}${n === 1 ? "" : "s"}`; | |
| 36 | if (minutes < 60) return of(minutes, "minute"); | |
| 37 | if (minutes < 48 * 60) return of(Math.round(minutes / 60), "hour"); | |
| 38 | return of(Math.round(minutes / (24 * 60)), "day"); | |
| 39 | } | |
| 40 | ||
| 41 | export interface RestoreLine { | |
| 42 | readonly proven: boolean; | |
| 43 | readonly sentence: string; | |
| 44 | } | |
| 45 | ||
| 46 | /** | |
| 47 | * "restore proven 2 days ago, 45 seconds, snapshot 3 hours old", or "never | |
| 48 | * proven"; a newer failed drill is said after it, since a pass that is no | |
| 49 | * longer the latest word is not the whole story. | |
| 50 | */ | |
| 51 | export function restoreLine(pass: DrillResult | undefined, last: DrillResult | undefined, now: number): RestoreLine { | |
| 52 | const failed = last !== undefined && last.verdict === "red" && (pass === undefined || last.at > pass.at) | |
| 53 | ? `; the last drill failed ${agoWords(last.at, now)}${last.failedStep ? ` at ${last.failedStep}` : ""}` | |
| 54 | : ""; | |
| 55 | if (pass === undefined) return { proven: false, sentence: `Restore never proven${failed}.` }; | |
| 56 | const age = snapshotAgeAt(pass.snapshotId, pass.at); | |
| 57 | return { | |
| 58 | proven: true, | |
| 59 | sentence: `Restore proven ${agoWords(pass.at, now)}, ${drillDuration(pass.ms)}, ${age === undefined ? "snapshot age not recorded" : `snapshot ${age} old`}${failed}.`, | |
| 60 | }; | |
| 61 | } |
src/tui/screens/system.ts +8
| @@ -57,6 +57,8 @@ import type { PaletteEntry } from "./entries.ts"; | ||
| 57 | 57 | import { shapeWords } from "../../engine/mode.ts"; |
| 58 | 58 | import { readHealth } from "../../actions/health.ts"; |
| 59 | 59 | import { egressReportedLine } from "../../system/egress_line.ts"; |
| 60 | import { restoreLine } from "../../snapshot/restore_line.ts"; | |
| 61 | import { lastDrill, lastPass } from "../../snapshot/drills.ts"; | |
| 60 | 62 | import { featureHealth } from "../../system/feature_health.ts"; |
| 61 | 63 | import { readerPaces } from "../../system/telemetry_reads.ts"; |
| 62 | 64 | import { live as liveMachine } from "../../system/live.ts"; |
| @@ -507,6 +509,11 @@ export function egressLines(session: Session): string[] { | ||
| 507 | 509 | return said === null ? [] : [said]; |
| 508 | 510 | } |
| 509 | 511 | |
| 512 | /** When a restore was last proven, one line, as /system's Restore section says it (#13052). */ | |
| 513 | export function restoreLines(session: Session, now = Date.now()): string[] { | |
| 514 | return [restoreLine(lastPass(session.db), lastDrill(session.db), now).sentence]; | |
| 515 | } | |
| 516 | ||
| 510 | 517 | /** The machine, as /system says it, and which agents may read it: the owner's, a person (#1789). */ |
| 511 | 518 | export const entry: PaletteEntry = { |
| 512 | 519 | id: "system", |
| @@ -521,6 +528,7 @@ export const entry: PaletteEntry = { | ||
| 521 | 528 | ctx.push(systemScreen(ctx, [ |
| 522 | 529 | ...forgeLines(readHealth(session.app, session.user)), |
| 523 | 530 | ...egressLines(session), |
| 531 | ...restoreLines(session), | |
| 524 | 532 | ...(who.length === 0 ? [] : ["", ...who]), |
| 525 | 533 | ...engineLines(session.app.config.engine), |
| 526 | 534 | ...(self.length === 0 ? [] : ["", ...self]), |
src/web/pages/system.ts +9
| @@ -56,6 +56,8 @@ import { lastFootprint, footprintFacts, footprintVerdict, BUDGET, type Footprint | ||
| 56 | 56 | import { lastProof } from "../../release/prove.ts"; |
| 57 | 57 | import { BREAK_STEPS, MACHINES, declaredFooting, footing, offMachine, whereWords, type Footing, type Machine } from "../../system/footing.ts"; |
| 58 | 58 | import { laneWords, readOffsiteLane, type OffsiteLane } from "../../snapshot/offsite_lane.ts"; |
| 59 | import { restoreLine, type RestoreLine } from "../../snapshot/restore_line.ts"; | |
| 60 | import { lastDrill, lastPass } from "../../snapshot/drills.ts"; | |
| 59 | 61 | import { stalePullsSection } from "./maintenance.ts"; |
| 60 | 62 | import { shapeWords } from "../../engine/mode.ts"; |
| 61 | 63 | import { badge, chartBox, disclosure, meter, pill, row, time } from "../ui.ts"; |
| @@ -400,6 +402,12 @@ export function offsiteSection(l: OffsiteLane): Html { | ||
| 400 | 402 | <p class="small"><a href="/settings/backups">Backups</a> runs a drill; so does the MCP <code>restore_drill</code>.</p>`, { id: "sys-offsite-more" })}</section>`; |
| 401 | 403 | } |
| 402 | 404 | |
| 405 | /** When a restore was last proven (#13052): one sentence, `snapshot/restore_line.ts`'s. */ | |
| 406 | export function restoreSection(l: RestoreLine): Html { | |
| 407 | return html`<section class="settings-block" id="sys-restore" data-proven="${String(l.proven)}"><h2>Restore</h2> | |
| 408 | <p>${l.sentence}</p></section>`; | |
| 409 | } | |
| 410 | ||
| 403 | 411 | /** The last hour's stalls by route, most milliseconds first; the rows are `stallHour`'s. */ |
| 404 | 412 | export function stallSection(rows: readonly RouteStall[], worst: Html = html``): Html { |
| 405 | 413 | return html`<section class="settings-block"><h2>Main-thread stalls, last hour</h2>${ |
| @@ -653,6 +661,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> { | ||
| 653 | 661 | ${maintenanceSection(timePhase(ctx.request, "maintenance", () => weekRows(ctx.app.db, duties.map((d) => d.name))))} |
| 654 | 662 | ${footprintSection(lastFootprint() ?? lastProof(ctx.app.config.dataDir)?.footprint ?? null)} |
| 655 | 663 | ${footingSection(standsOn)} |
| 664 | ${restoreSection(timePhase(ctx.request, "restore", () => restoreLine(lastPass(ctx.app.db), lastDrill(ctx.app.db), Date.now())))} | |
| 656 | 665 | ${offsiteSection(timePhase(ctx.request, "offsite", () => readOffsiteLane(ctx.app.config.dataDir, ctx.app.db)))} |
| 657 | 666 | ${featuresSection(timePhase(ctx.request, "features", () => featureHealth(ctx.app, reader)))} |
| 658 | 667 | ${lagSection(liveOf(ctx.app).lagMinutes.readings())} |
test/restore-line.test.ts added +92
| @@ -0,0 +1,92 @@ | ||
| 1 | /** | |
| 2 | * The restore drill as one line on /system (#13052): a pass says when, how | |
| 3 | * long and how old the snapshot was; a failure after it is said too; no | |
| 4 | * record says "never proven". Durations are seconds or minutes, never ms. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import type { DrillResult } from "../src/snapshot/drill.ts"; | |
| 12 | import { lastDrill, lastPass, recordDrill } from "../src/snapshot/drills.ts"; | |
| 13 | import { drillDuration, restoreLine, snapshotAgeAt } from "../src/snapshot/restore_line.ts"; | |
| 14 | import { restoreSection } from "../src/web/pages/system.ts"; | |
| 15 | import { restoreLines } from "../src/tui/screens/system.ts"; | |
| 16 | import type { Session } from "../src/tui/session.ts"; | |
| 17 | ||
| 18 | const drill = (over: Partial<DrillResult>): DrillResult => | |
| 19 | ({ | |
| 20 | at: "2026-11-13T08:00:00.000Z", | |
| 21 | snapshotId: "2026-11-13T05-00-00Z", | |
| 22 | ms: 45_000, | |
| 23 | counts: { repositories: 1, issues: 1, users: 1, pulls: 0, apps: 0 }, | |
| 24 | health: "ok", | |
| 25 | verdict: "green", | |
| 26 | failedStep: null, | |
| 27 | reason: null, | |
| 28 | phases: [], | |
| 29 | budgetMs: 600_000, | |
| 30 | ...over, | |
| 31 | }) as DrillResult; | |
| 32 | ||
| 33 | const now = Date.parse("2026-11-15T08:00:00Z"); | |
| 34 | ||
| 35 | test("durations are seconds or minutes, never milliseconds", () => { | |
| 36 | expect(drillDuration(340)).toBe("1 second"); | |
| 37 | expect(drillDuration(45_000)).toBe("45 seconds"); | |
| 38 | expect(drillDuration(180_000)).toBe("3 minutes"); | |
| 39 | expect(drillDuration(60_000 * 61)).toBe("61 minutes"); | |
| 40 | expect(snapshotAgeAt("2026-11-13T05-00-00Z", "2026-11-13T08:00:00Z")).toBe("3 hours"); | |
| 41 | expect(snapshotAgeAt("not-a-time", "2026-11-13T08:00:00Z")).toBeUndefined(); | |
| 42 | }); | |
| 43 | ||
| 44 | test("a pass says when, how long and the snapshot's age", () => { | |
| 45 | const l = restoreLine(drill({}), drill({}), now); | |
| 46 | expect(l).toEqual({ proven: true, sentence: "Restore proven 2 days ago, 45 seconds, snapshot 3 hours old." }); | |
| 47 | expect(l.sentence).not.toMatch(/\bms\b|45000/); | |
| 48 | }); | |
| 49 | ||
| 50 | test("a failure after the pass is said; a failure with no pass is never proven", () => { | |
| 51 | const red = drill({ at: "2026-11-15T06:00:00.000Z", verdict: "red", failedStep: "restore", ms: 600_000 }); | |
| 52 | expect(restoreLine(drill({}), red, now).sentence).toBe( | |
| 53 | "Restore proven 2 days ago, 45 seconds, snapshot 3 hours old; the last drill failed 2 hours ago at restore.", | |
| 54 | ); | |
| 55 | expect(restoreLine(undefined, red, now)).toEqual({ proven: false, sentence: "Restore never proven; the last drill failed 2 hours ago at restore." }); | |
| 56 | }); | |
| 57 | ||
| 58 | test("no record is never proven", () => { | |
| 59 | expect(restoreLine(undefined, undefined, now)).toEqual({ proven: false, sentence: "Restore never proven." }); | |
| 60 | expect(String(restoreSection(restoreLine(undefined, undefined, now)))).toContain('data-proven="false"'); | |
| 61 | }); | |
| 62 | ||
| 63 | describe("read from the drill table", () => { | |
| 64 | let root = ""; | |
| 65 | let app: App; | |
| 66 | beforeAll(() => { | |
| 67 | root = mkdtempSync(join(tmpdir(), "restore-line-")); | |
| 68 | app = createApp({ dataDir: join(root, "data"), port: 0 }); | |
| 69 | }); | |
| 70 | afterAll(() => { | |
| 71 | app.db.close(); | |
| 72 | rmSync(root, { recursive: true, force: true }); | |
| 73 | }); | |
| 74 | ||
| 75 | test("the terminal's System screen says never proven until a drill passes", () => { | |
| 76 | expect(restoreLines({ db: app.db } as Session, now)).toEqual(["Restore never proven."]); | |
| 77 | }); | |
| 78 | ||
| 79 | test("lastPass skips a newer red drill; none kept reads as none", () => { | |
| 80 | expect(lastPass(app.db)).toBeUndefined(); | |
| 81 | recordDrill(app.db, drill({})); | |
| 82 | recordDrill(app.db, drill({ at: "2026-11-15T06:00:00.000Z", verdict: "red", failedStep: "open", reason: "no db" })); | |
| 83 | expect(lastPass(app.db)?.at).toBe("2026-11-13T08:00:00.000Z"); | |
| 84 | expect(lastDrill(app.db)?.verdict).toBe("red"); | |
| 85 | const html = String(restoreSection(restoreLine(lastPass(app.db), lastDrill(app.db), now))); | |
| 86 | expect(html).toContain('id="sys-restore" data-proven="true"'); | |
| 87 | expect(html).toContain("Restore proven 2 days ago, 45 seconds, snapshot 3 hours old; the last drill failed 2 hours ago at open."); | |
| 88 | expect(restoreLines({ db: app.db } as Session, now)).toEqual([ | |
| 89 | "Restore proven 2 days ago, 45 seconds, snapshot 3 hours old; the last drill failed 2 hours ago at open.", | |
| 90 | ]); | |
| 91 | }); | |
| 92 | }); |
test/system-parts.test.ts +1
| @@ -57,6 +57,7 @@ test("one <h2> per part, each in a settings-block, and no card", async () => { | ||
| 57 | 57 | "Maintenance, last 7 days", |
| 58 | 58 | "What the forge costs", |
| 59 | 59 | "What the forge stands on", |
| 60 | "Restore", | |
| 60 | 61 | "Off-site backup", |
| 61 | 62 | "Loop lag, last 20 minutes", |
| 62 | 63 | "Main-thread stalls, last hour", |
new above
offline
Principle 1: heading says Restore and the sentence starts Restore again. Also data-proven has no consumer.