Public: anyone can read this; the crew can change it.

The restore drill result is on /system as one line: last pass, how long, and the age of the snapshot restored #13057

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13052-the-restore-drill-result-is-on-system-as → main 4 commits fork 1 thread · 1 unresolved

  • Merged. scout/13052-the-restore-drill-result-is-on-system-as was merged into main as 6640bb9.

Review

Sign in to review this change.
8 files changed +215 −8
docs/design/attention-budget.json +5 −5
@@ -173,16 +173,16 @@
173173 },
174174 "system": {
175175 "390": {
176 "words": 230,
177 "blocks": 75,
176 "words": 234,
177 "blocks": 77,
178178 "colours": 0
179179 },
180180 "1280": {
181 "words": 230,
182 "blocks": 75,
181 "words": 234,
182 "blocks": 77,
183183 "colours": 0
184184 },
185 "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull."
185 "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull. Words +4, blocks +2 (#13052): the Restore section's heading and its one sentence (\"Restore never proven.\" on an empty forge) stay on first view, so the owner sees whether a restore works without a click."
186186 },
187187 "explore": {
188188 "390": {
docs/features/resilience-backups.md +27
@@ -31,3 +31,30 @@ The first of these that is set wins (`src/snapshot/destination.ts`):
3131
3232 The line describes the store that the running process holds. After you save
3333 a destination, the next use resolves it again and the line names it.
34
35 ## When a restore was last proven (#13052)
36
37 A backup is only as good as the last restore that worked. The forge's
38 restore drill restores the newest snapshot into a scratch directory and
39 checks it; this line says when that last passed, so an owner sees it
40 without running one.
41
42 ### Where to read it
43
44 - **`/system`** (admins): the **Restore** section is one sentence.
45 - **The terminal's System screen**: the same sentence, under the forge's
46 health line.
47
48 ### What it says
49
50 - `Restore never proven.` until a drill has passed.
51 - `Restore proven 2 days ago, 45 seconds, snapshot 3 hours old.` after one:
52 when it passed, how long it took (seconds or minutes, never
53 milliseconds), and how old the restored snapshot was then.
54 - A drill that failed after the last pass is added:
55 `…; the last drill failed 2 hours ago at restore.`
56
57 ### Prove one now
58
59 Open `/settings/backups` and run a drill, or call the MCP tool
60 `restore_drill`. Reload `/system`: the line reads `Restore proven just now`.
src/snapshot/drills.ts +12 −3
@@ -85,9 +85,18 @@ export function recordDrill(db: Database, result: DrillResult): void {
8585
8686 /** The newest drill, or undefined when none has run. */
8787 export function lastDrill(db: Database): DrillResult | undefined {
88 const row = db
89 .query<Row, []>("SELECT * FROM drill ORDER BY id DESC LIMIT 1")
90 .get();
88 return asResult(db.query<Row, []>("SELECT * FROM drill ORDER BY id DESC LIMIT 1").get());
89 }
90
91 /**
92 * The newest drill that passed, or undefined when none ever has: when a
93 * restore was last proven.
94 */
95 export function lastPass(db: Database): DrillResult | undefined {
96 return asResult(db.query<Row, []>("SELECT * FROM drill WHERE verdict = 'green' ORDER BY id DESC LIMIT 1").get());
97 }
98
99 function asResult(row: Row | null): DrillResult | undefined {
91100 if (row === null) return undefined;
92101 const kept = parsePhases(row.phases);
93102 return {
src/snapshot/restore_line.ts added +61
@@ -0,0 +1,61 @@
1 /**
2 * The restore drill as one line (#13052): when a restore was last proven,
3 * how long it took, and how old the snapshot it restored was, so the owner
4 * sees a restore works without asking. `/system` prints it.
5 *
6 * Plain data in (the newest passing drill and the newest drill), plain
7 * words out; no clock but the one given.
8 */
9
10 import type { DrillResult } from "./drill.ts";
11 import { agoWords } from "../kernel/ago.ts";
12 import { snapshotStart } from "./snapshot.ts";
13
14 /**
15 * A drill's length as a person says it: seconds under a minute and a half,
16 * else minutes; never milliseconds.
17 */
18 export function drillDuration(ms: number): string {
19 if (!Number.isFinite(ms) || ms < 0) return "an unknown time";
20 if (ms < 1_500) return "1 second";
21 if (ms < 90_000) return `${Math.round(ms / 1000)} seconds`;
22 const min = Math.round(ms / 60_000);
23 return `${min} minute${min === 1 ? "" : "s"}`;
24 }
25
26 /**
27 * How old the snapshot was when the drill restored it, from its id;
28 * undefined when the id is no time.
29 */
30 export function snapshotAgeAt(id: string | null, atIso: string): string | undefined {
31 const start = id === null ? undefined : snapshotStart(id);
32 const at = Date.parse(atIso);
33 if (start === undefined || !Number.isFinite(at)) return undefined;
34 const minutes = Math.max(0, Math.round((at - start.getTime()) / 60_000));
35 const of = (n: number, unit: string): string => `${n} ${unit}${n === 1 ? "" : "s"}`;
36 if (minutes < 60) return of(minutes, "minute");
37 if (minutes < 48 * 60) return of(Math.round(minutes / 60), "hour");
38 return of(Math.round(minutes / (24 * 60)), "day");
39 }
40
41 export interface RestoreLine {
42 readonly proven: boolean;
43 readonly sentence: string;
44 }
45
46 /**
47 * "restore proven 2 days ago, 45 seconds, snapshot 3 hours old", or "never
48 * proven"; a newer failed drill is said after it, since a pass that is no
49 * longer the latest word is not the whole story.
50 */
51 export function restoreLine(pass: DrillResult | undefined, last: DrillResult | undefined, now: number): RestoreLine {
52 const failed = last !== undefined && last.verdict === "red" && (pass === undefined || last.at > pass.at)
53 ? `; the last drill failed ${agoWords(last.at, now)}${last.failedStep ? ` at ${last.failedStep}` : ""}`
54 : "";
55 if (pass === undefined) return { proven: false, sentence: `Restore never proven${failed}.` };
56 const age = snapshotAgeAt(pass.snapshotId, pass.at);
57 return {
58 proven: true,
59 sentence: `Restore proven ${agoWords(pass.at, now)}, ${drillDuration(pass.ms)}, ${age === undefined ? "snapshot age not recorded" : `snapshot ${age} old`}${failed}.`,
60 };
61 }
src/tui/screens/system.ts +8
@@ -57,6 +57,8 @@ import type { PaletteEntry } from "./entries.ts";
5757 import { shapeWords } from "../../engine/mode.ts";
5858 import { readHealth } from "../../actions/health.ts";
5959 import { egressReportedLine } from "../../system/egress_line.ts";
60 import { restoreLine } from "../../snapshot/restore_line.ts";
61 import { lastDrill, lastPass } from "../../snapshot/drills.ts";
6062 import { featureHealth } from "../../system/feature_health.ts";
6163 import { readerPaces } from "../../system/telemetry_reads.ts";
6264 import { live as liveMachine } from "../../system/live.ts";
@@ -507,6 +509,11 @@ export function egressLines(session: Session): string[] {
507509 return said === null ? [] : [said];
508510 }
509511
512 /** When a restore was last proven, one line, as /system's Restore section says it (#13052). */
513 export function restoreLines(session: Session, now = Date.now()): string[] {
514 return [restoreLine(lastPass(session.db), lastDrill(session.db), now).sentence];
515 }
516
510517 /** The machine, as /system says it, and which agents may read it: the owner's, a person (#1789). */
511518 export const entry: PaletteEntry = {
512519 id: "system",
@@ -521,6 +528,7 @@ export const entry: PaletteEntry = {
521528 ctx.push(systemScreen(ctx, [
522529 ...forgeLines(readHealth(session.app, session.user)),
523530 ...egressLines(session),
531 ...restoreLines(session),
524532 ...(who.length === 0 ? [] : ["", ...who]),
525533 ...engineLines(session.app.config.engine),
526534 ...(self.length === 0 ? [] : ["", ...self]),
src/web/pages/system.ts +9
@@ -56,6 +56,8 @@ import { lastFootprint, footprintFacts, footprintVerdict, BUDGET, type Footprint
5656 import { lastProof } from "../../release/prove.ts";
5757 import { BREAK_STEPS, MACHINES, declaredFooting, footing, offMachine, whereWords, type Footing, type Machine } from "../../system/footing.ts";
5858 import { laneWords, readOffsiteLane, type OffsiteLane } from "../../snapshot/offsite_lane.ts";
59 import { restoreLine, type RestoreLine } from "../../snapshot/restore_line.ts";
60 import { lastDrill, lastPass } from "../../snapshot/drills.ts";
5961 import { stalePullsSection } from "./maintenance.ts";
6062 import { shapeWords } from "../../engine/mode.ts";
6163 import { badge, chartBox, disclosure, meter, pill, row, time } from "../ui.ts";
@@ -400,6 +402,12 @@ export function offsiteSection(l: OffsiteLane): Html {
400402 <p class="small"><a href="/settings/backups">Backups</a> runs a drill; so does the MCP <code>restore_drill</code>.</p>`, { id: "sys-offsite-more" })}</section>`;
401403 }
402404
405 /** When a restore was last proven (#13052): one sentence, `snapshot/restore_line.ts`'s. */
406 export function restoreSection(l: RestoreLine): Html {
407 return html`<section class="settings-block" id="sys-restore" data-proven="${String(l.proven)}"><h2>Restore</h2>

Principle 1: heading says Restore and the sentence starts Restore again. Also data-proven has no consumer.

408 <p>${l.sentence}</p></section>`;
409 }
410
403411 /** The last hour's stalls by route, most milliseconds first; the rows are `stallHour`'s. */
404412 export function stallSection(rows: readonly RouteStall[], worst: Html = html``): Html {
405413 return html`<section class="settings-block"><h2>Main-thread stalls, last hour</h2>${
@@ -653,6 +661,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> {
653661 ${maintenanceSection(timePhase(ctx.request, "maintenance", () => weekRows(ctx.app.db, duties.map((d) => d.name))))}
654662 ${footprintSection(lastFootprint() ?? lastProof(ctx.app.config.dataDir)?.footprint ?? null)}
655663 ${footingSection(standsOn)}
664 ${restoreSection(timePhase(ctx.request, "restore", () => restoreLine(lastPass(ctx.app.db), lastDrill(ctx.app.db), Date.now())))}
656665 ${offsiteSection(timePhase(ctx.request, "offsite", () => readOffsiteLane(ctx.app.config.dataDir, ctx.app.db)))}
657666 ${featuresSection(timePhase(ctx.request, "features", () => featureHealth(ctx.app, reader)))}
658667 ${lagSection(liveOf(ctx.app).lagMinutes.readings())}
test/restore-line.test.ts added +92
@@ -0,0 +1,92 @@
1 /**
2 * The restore drill as one line on /system (#13052): a pass says when, how
3 * long and how old the snapshot was; a failure after it is said too; no
4 * record says "never proven". Durations are seconds or minutes, never ms.
5 */
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { mkdtempSync, rmSync } from "node:fs";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import type { DrillResult } from "../src/snapshot/drill.ts";
12 import { lastDrill, lastPass, recordDrill } from "../src/snapshot/drills.ts";
13 import { drillDuration, restoreLine, snapshotAgeAt } from "../src/snapshot/restore_line.ts";
14 import { restoreSection } from "../src/web/pages/system.ts";
15 import { restoreLines } from "../src/tui/screens/system.ts";
16 import type { Session } from "../src/tui/session.ts";
17
18 const drill = (over: Partial<DrillResult>): DrillResult =>
19 ({
20 at: "2026-11-13T08:00:00.000Z",
21 snapshotId: "2026-11-13T05-00-00Z",
22 ms: 45_000,
23 counts: { repositories: 1, issues: 1, users: 1, pulls: 0, apps: 0 },
24 health: "ok",
25 verdict: "green",
26 failedStep: null,
27 reason: null,
28 phases: [],
29 budgetMs: 600_000,
30 ...over,
31 }) as DrillResult;
32
33 const now = Date.parse("2026-11-15T08:00:00Z");
34
35 test("durations are seconds or minutes, never milliseconds", () => {
36 expect(drillDuration(340)).toBe("1 second");
37 expect(drillDuration(45_000)).toBe("45 seconds");
38 expect(drillDuration(180_000)).toBe("3 minutes");
39 expect(drillDuration(60_000 * 61)).toBe("61 minutes");
40 expect(snapshotAgeAt("2026-11-13T05-00-00Z", "2026-11-13T08:00:00Z")).toBe("3 hours");
41 expect(snapshotAgeAt("not-a-time", "2026-11-13T08:00:00Z")).toBeUndefined();
42 });
43
44 test("a pass says when, how long and the snapshot's age", () => {
45 const l = restoreLine(drill({}), drill({}), now);
46 expect(l).toEqual({ proven: true, sentence: "Restore proven 2 days ago, 45 seconds, snapshot 3 hours old." });
47 expect(l.sentence).not.toMatch(/\bms\b|45000/);
48 });
49
50 test("a failure after the pass is said; a failure with no pass is never proven", () => {
51 const red = drill({ at: "2026-11-15T06:00:00.000Z", verdict: "red", failedStep: "restore", ms: 600_000 });
52 expect(restoreLine(drill({}), red, now).sentence).toBe(
53 "Restore proven 2 days ago, 45 seconds, snapshot 3 hours old; the last drill failed 2 hours ago at restore.",
54 );
55 expect(restoreLine(undefined, red, now)).toEqual({ proven: false, sentence: "Restore never proven; the last drill failed 2 hours ago at restore." });
56 });
57
58 test("no record is never proven", () => {
59 expect(restoreLine(undefined, undefined, now)).toEqual({ proven: false, sentence: "Restore never proven." });
60 expect(String(restoreSection(restoreLine(undefined, undefined, now)))).toContain('data-proven="false"');
61 });
62
63 describe("read from the drill table", () => {
64 let root = "";
65 let app: App;
66 beforeAll(() => {
67 root = mkdtempSync(join(tmpdir(), "restore-line-"));
68 app = createApp({ dataDir: join(root, "data"), port: 0 });
69 });
70 afterAll(() => {
71 app.db.close();
72 rmSync(root, { recursive: true, force: true });
73 });
74
75 test("the terminal's System screen says never proven until a drill passes", () => {
76 expect(restoreLines({ db: app.db } as Session, now)).toEqual(["Restore never proven."]);
77 });
78
79 test("lastPass skips a newer red drill; none kept reads as none", () => {
80 expect(lastPass(app.db)).toBeUndefined();
81 recordDrill(app.db, drill({}));
82 recordDrill(app.db, drill({ at: "2026-11-15T06:00:00.000Z", verdict: "red", failedStep: "open", reason: "no db" }));
83 expect(lastPass(app.db)?.at).toBe("2026-11-13T08:00:00.000Z");
84 expect(lastDrill(app.db)?.verdict).toBe("red");
85 const html = String(restoreSection(restoreLine(lastPass(app.db), lastDrill(app.db), now)));
86 expect(html).toContain('id="sys-restore" data-proven="true"');
87 expect(html).toContain("Restore proven 2 days ago, 45 seconds, snapshot 3 hours old; the last drill failed 2 hours ago at open.");
88 expect(restoreLines({ db: app.db } as Session, now)).toEqual([
89 "Restore proven 2 days ago, 45 seconds, snapshot 3 hours old; the last drill failed 2 hours ago at open.",
90 ]);
91 });
92 });
test/system-parts.test.ts +1
@@ -57,6 +57,7 @@ test("one <h2> per part, each in a settings-block, and no card", async () => {
5757 "Maintenance, last 7 days",
5858 "What the forge costs",
5959 "What the forge stands on",
60 "Restore",
6061 "Off-site backup",
6162 "Loop lag, last 20 minutes",
6263 "Main-thread stalls, last hour",

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs