Public: anyone can read this; the crew can change it.

The restore drill restores one hosted forge from its backup file and proves its database and repositories open #14181

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13919-the-restore-drill-restores-one-hosted-fo → main 10 commits fork

  • Merged. torque/13919-the-restore-drill-restores-one-hosted-fo was merged into main as 430fbe2.

Review

Sign in to review this change.
11 files changed +216 −7
docs/features/hosted-backup.md +4
@@ -34,6 +34,10 @@ A hosted forge's directory is the forge's own to write, and its host reads it fo
3434
3535 Each restore drill, after the host itself, restores one hosted forge alone from the same snapshot (the next one each day and each week, `src/snapshot/drill_hosted.ts`) and proves it opens: its ledger reads and names it, its database passes SQLite's integrity check, and every repository its copy holds comes back and passes `git fsck`. The verdict says which forge ("…passed, with hosted forge glint (1 of 17)"). One that does not come back fails the drill at the step `forge`, by name ([RECOVERY.md](../RECOVERY.md)).
3636
37 - Web: `/settings/backups`, press Run a drill; when it ends, the last-drill line says "Hosted forge NAME (SIZE, 1 of N) restored alone in 3.2 s: its database passed the integrity check and its K repositories git fsck." Terminal: the Resilience screen's "Last drill:" line under the verdict says the same, and so does the recovery issue's comment (`- hosted:`).
38 - A forge that hosts none, or whose snapshot left every hosted forge out, says in one line that the hosted restore was skipped; the drill stays green.
39 - Red at `forge`: the reason names the forge and what did not hold (a missing chunk, the integrity check, a repository's `git fsck`, its ledger); it is paged once like every red drill.
40
3741 ## Restore one forge
3842
3943 Into an empty data directory; `children/<bodyId>/` and `spawn/<name>.json` are written there:
src/snapshot/drill.ts +12 −2
@@ -190,6 +190,11 @@ export interface DrillResult {
190190 * the snapshot held none, or the drill failed before it.
191191 */
192192 readonly hosted?: HostedProof | undefined;
193 /**
194 * True when every earlier step passed and the snapshot held no hosted forge whole to restore, so
195 * that step was skipped (#13919); absent otherwise, and on drills kept before it.
196 */
197 readonly hostedSkipped?: true | undefined;
193198 }
194199
195200 /** The drill process's wall time and peak resident memory, and the memory limit it ran under. */
@@ -253,6 +258,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
253258 let bootStart: number | undefined;
254259 let health: DrillResult["health"] = "unreached";
255260 let hosted: HostedProof | undefined;
261 let hostedSkipped = false;
256262 let bootEnd: number | undefined;
257263 try {
258264 // One clock for the whole drill, so a hang in any step ends it red.
@@ -351,12 +357,15 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
351357 const pick = drillPick(held, clock());
352358 if (pick !== undefined) {
353359 step = "forge";
354 hosted = await within(
360 const from = tick();
361 const proof = await within(
355362 (deps.prove ?? proveHosted)(store, newest.name, pick, join(scratch, "forge"), { passphrase: deps.passphrase, of: held.length, leftMs: left }),
356363 left(),
357364 budget,
358365 );
359 }
366 // Its restore and checks together, so the card can say it in seconds (#13919).
367 hosted = { ...proof, ms: Math.round(tick() - from) };
368 } else hostedSkipped = true;
360369 } catch (error) {
361370 reason =
362371 error instanceof StepTimeout && step === "restore" && progress !== undefined
@@ -402,6 +411,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
402411 offSite: isOffBox(deps.store),
403412 ...(format === undefined ? {} : { format }),
404413 ...(hosted === undefined ? {} : { hosted }),
414 ...(hostedSkipped ? { hostedSkipped: true as const } : {}),
405415 };
406416 }
407417 }
src/snapshot/drill_hosted.ts +5
@@ -28,6 +28,11 @@ export interface HostedProof {
2828 readonly repos: number;
2929 /** The bytes its restore wrote. */
3030 readonly bytes: number;
31 /**
32 * Its restore and checks, wall time in ms, as the drill timed them (#13919); absent from a proof
33 * before the drill times it, and on drills kept before that.
34 */
35 readonly ms?: number | undefined;
3136 }
3237
3338 /** No fsck of one repository is given longer than this. */
src/snapshot/drill_hosted_words.ts added +30
@@ -0,0 +1,30 @@
1 /**
2 * What the last restore drill says of its hosted step (#13919), one line, in the same words on the
3 * Backups card, the Resilience screen and the recovery issue's comment: the forge it restored
4 * alone, its size and its restore time in seconds; or that it was skipped for want of one. A drill
5 * that failed there is red at the step `forge` and its reason names the forge; this says nothing
6 * more of it. Control characters in a kept name print as `?`; each surface still escapes the line.
7 */
8
9 import { printable } from "./bucket.ts";
10 import { formatBytes, type DrillResult } from "./drill.ts";
11
12 /** Milliseconds as seconds with one decimal: 3240 → "3.2 s". */
13 export function seconds(ms: number): string {
14 return `${(Math.max(0, ms) / 1000).toFixed(1)} s`;
15 }
16
17 /**
18 * The line, or undefined when the drill reached no hosted step (it failed before, or predates it).
19 */
20 export function hostedDrillWords(d: Pick<DrillResult, "hosted" | "hostedSkipped">): string | undefined {
21 const h = d.hosted;
22 if (h !== undefined) {
23 const took = h.ms === undefined ? "" : ` in ${seconds(h.ms)}`;
24 const of = h.of > 1 ? `, 1 of ${h.of}` : "";
25 const repos = h.repos === 1 ? "its 1 repository" : `its ${h.repos} repositories`;
26 return `Hosted forge ${printable(h.name)} (${formatBytes(h.bytes)}${of}) restored alone${took}: its database passed the integrity check and ${repos} git fsck.`;
27 }
28 if (d.hostedSkipped === true) return "No hosted forge was whole in the snapshot, so the hosted restore was skipped.";
29 return undefined;
30 }
src/snapshot/drills.ts +13 −2
@@ -66,9 +66,18 @@ export function parseHosted(raw: string | null): { hosted?: HostedProof } {
6666 const h = (JSON.parse(raw ?? "null") as { hosted?: unknown } | null)?.hosted as Record<string, unknown> | undefined;
6767 if (typeof h !== "object" || h === null) return {};
6868 const whole = (v: unknown): v is number => typeof v === "number" && Number.isSafeInteger(v) && v >= 0;
69 const { name, of, repos, bytes } = h;
69 const { name, of, repos, bytes, ms } = h;
7070 if (typeof name !== "string" || name.length > 100 || !whole(of) || !whole(repos) || !whole(bytes)) return {};
71 return { hosted: { name, of, repos, bytes } };
71 return { hosted: { name, of, repos, bytes, ...(whole(ms) ? { ms } : {}) } };
72 } catch {
73 return {};
74 }
75 }
76
77 /** Whether a kept drill skipped its hosted step for want of a hosted forge (#13919). */
78 export function parseHostedSkipped(raw: string | null): { hostedSkipped?: true } {
79 try {
80 return (JSON.parse(raw ?? "null") as { hostedSkipped?: unknown } | null)?.hostedSkipped === true ? { hostedSkipped: true } : {};
7281 } catch {
7382 return {};
7483 }
@@ -121,6 +130,7 @@ export function recordDrill(db: Database, result: DrillResult): void {
121130 ...(result.from === undefined ? {} : { from: result.from, offSite: result.offSite === true }),
122131 ...(result.format === undefined ? {} : { format: result.format }),
123132 ...(result.hosted === undefined ? {} : { hosted: result.hosted }),
133 ...(result.hostedSkipped === true ? { hostedSkipped: true } : {}),
124134 ...(result.cost === undefined
125135 ? {}
126136 : { cost: { elapsedMs: Math.round(result.cost.elapsedMs), peakBytes: Math.round(result.cost.peakBytes), limitBytes: Math.round(result.cost.limitBytes) } }),
@@ -166,5 +176,6 @@ function asResult(row: Row | null): DrillResult | undefined {
166176 ...parseFrom(row.phases),
167177 ...parseCost(row.phases),
168178 ...parseHosted(row.phases),
179 ...parseHostedSkipped(row.phases),
169180 };
170181 }
src/snapshot/verdict.ts +5 −1
@@ -9,9 +9,12 @@
99
1010 import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
1111 import { join } from "node:path";
12 import { hostedDrillWords } from "./drill_hosted_words.ts";
1213 import { describeTimes, formatMinutes, formatSpan, timeByPart, type DrillResult, type DrillStep } from "./drill.ts";
1314
14 const STEPS: readonly DrillStep[] = ["fetch", "restore", "open", "count", "unseal", "health"];
15 // Every step `runDrill` can fail at: a step not here makes the kept verdict unreadable, and a red
16 // drill at it would never page.
17 const STEPS: readonly DrillStep[] = ["fetch", "restore", "open", "count", "unseal", "health", "forge"];
1518
1619 const lastFile = (dataDir: string): string => join(dataDir, "drill", "last.json");
1720
@@ -168,6 +171,7 @@ export function verdictComment(r: DrillResult): string {
168171 `- users: ${r.counts.users}`,
169172 `- pulls: ${r.counts.pulls}`,
170173 `- apps: ${r.counts.apps}`,
174 ...(hostedDrillWords(r) === undefined ? [] : [`- hosted: ${hostedDrillWords(r)}`]),
171175 ...(pass || !r.reason ? [] : [`- why: ${r.reason}`]),
172176 ].join("\n");
173177 }
src/tui/screens/resilience.ts +12
@@ -43,6 +43,7 @@ import { hostedWords, keptWords, readHosted } from "../../snapshot/hosted.ts";
4343 import { backupsLine } from "../../snapshot/backups_line.ts";
4444 import { backupVerdict } from "../../snapshot/bucket.ts";
4545 import { lastDrill } from "../../snapshot/drills.ts";
46 import { hostedDrillWords } from "../../snapshot/drill_hosted_words.ts";
4647 import { resolveSnapshots } from "../../snapshot/destination.ts";
4748 import { bodiesOf } from "../../snapshot/store.ts";
4849 import { snapshotsOf } from "../../snapshot/service.ts";
@@ -119,9 +120,20 @@ export function verdictLines(f: Session): string[] {
119120 ...wrap(` ${v.checkLine}`, INNER),
120121 ...v.lanes.flatMap((l) => wrap(` ${l.state === "red" ? "!" : " "} ${l.says}`, INNER).concat(wrap(` ${l.line}${l.bodies === undefined ? "" : `; ${l.bodies}`}`, INNER))),
121122 ...[...v.reasons, ...v.unknowns].flatMap((r) => wrap(` - ${r}`, INNER)),
123 ...hostedDrillLines(f),
122124 ];
123125 }
124126
127 /**
128 * The last drill's hosted step in the Backups card's words (#13919): the forge it restored, its
129 * size and seconds, or that it was skipped; none when that drill reached no hosted step.
130 */
131 export function hostedDrillLines(f: Pick<Session, "app">): string[] {
132 const d = lastDrill(f.app.db);
133 const said = d === undefined ? undefined : hostedDrillWords(d);
134 return said === undefined ? [] : wrap(` Last drill: ${said}`, INNER);
135 }
136
125137 const STEPS: readonly { key: keyof DestinationInput; label: string; placeholder?: string; mask?: true }[] = [
126138 { key: "endpoint", label: "Endpoint, an http(s) URL", placeholder: "https://s3.example.com" },
127139 { key: "bucket", label: "Bucket" },
src/web/pages/backups.ts +5 −1
@@ -31,6 +31,7 @@ import { lastDrill } from "../../snapshot/drills.ts";
3131 import { backupVerdict } from "../../snapshot/bucket.ts";
3232 import { bodiesOf, type Store } from "../../snapshot/store.ts";
3333 import { hostedWords } from "../../snapshot/hosted.ts";
34 import { hostedDrillWords } from "../../snapshot/drill_hosted_words.ts";
3435 import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts";
3536 import { drillStatus, mirrorNow, snapshotNow, startDrillNow } from "../../actions/backups_now.ts";
3637 import { lastSnapshotOutcome } from "../../snapshot/outcome.ts";
@@ -132,6 +133,9 @@ function drillLine(ctx: Ctx): Html {
132133 d === undefined
133134 ? ""
134135 : html`<dl class="status-rows" id="drill-cost">${costFacts(d).map((f) => html`<div class="status-row"><dt>${f.label}</dt><dd>${f.value}</dd></div>`)}</dl>`;
136 // The hosted forge it restored alone, by name, size and seconds, or that it skipped (#13919).
137 const said = d === undefined ? undefined : hostedDrillWords(d);
138 const hosted = said === undefined ? "" : html` <span data-drill-hosted>${said}</span>`;
135139 const line =
136140 d === undefined
137141 ? html`<p class="break">No restore drill has run yet.</p>`
@@ -139,7 +143,7 @@ function drillLine(ctx: Ctx): Html {
139143 ? html`<p class="break">
140144 ${badge("Ready", { tone: "success", icon: "check" })} Last drill: ${time(d.at)}, ${d.ms} ms, all counts matched —
141145 snapshot <code>${d.snapshotId ?? ""}</code>, key opened —
142 ${d.counts.users} users, ${d.counts.repositories} repositories, ${d.counts.issues} issues, ${d.counts.pulls} pulls, ${d.counts.apps} apps.${timing
146 ${d.counts.users} users, ${d.counts.repositories} repositories, ${d.counts.issues} issues, ${d.counts.pulls} pulls, ${d.counts.apps} apps.${hosted}${timing
143147 ? html` <span class="muted small">${timing}.</span>`
144148 : ""}
145149 </p>`
test/drill-hosted-card.test.ts added +119
@@ -0,0 +1,119 @@
1 /**
2 * The restore drill's hosted step, said (#13919): the Backups card, the Resilience screen and the
3 * recovery comment name the forge it restored, its size and its restore time in seconds; a drill
4 * with no hosted forge says it skipped the step in one line; a drill red at `forge` is a verdict
5 * the pager can read, so it pages.
6 */
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { mkdtempSync, rmSync } from "node:fs";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { serve, type Servers } from "../src/http/server.ts";
14 import type { DrillResult } from "../src/snapshot/drill.ts";
15 import { hostedDrillWords, seconds } from "../src/snapshot/drill_hosted_words.ts";
16 import { lastDrill, recordDrill } from "../src/snapshot/drills.ts";
17 import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
18 import { dirStore } from "../src/snapshot/store.ts";
19 import { drillNeed, lastVerdict, recordVerdict, verdictComment } from "../src/snapshot/verdict.ts";
20 import { hostedDrillLines } from "../src/tui/screens/resilience.ts";
21 import { quietLogs } from "./helpers/globals.ts";
22
23 quietLogs();
24
25 let root = "";
26 let app: App;
27 let servers: Servers;
28 let svc: SnapshotsHandle;
29 let cookie = "";
30
31 const green: DrillResult = {
32 at: "2026-11-15T08:10:00.000Z", snapshotId: "s", ms: 60_000, counts: { repositories: 1, issues: 0, users: 1, pulls: 0, apps: 0 },
33 health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 600_000, reason: null,
34 };
35 // 3 MiB = 3 × 1024² bytes; 3240 ms = 3.2 s.
36 const restored: DrillResult = { ...green, hosted: { name: "glint", of: 4, repos: 2, bytes: 3 * 1024 ** 2, ms: 3_240 } };
37 const LINE = "Hosted forge glint (3 MiB, 1 of 4) restored alone in 3.2 s: its database passed the integrity check and its 2 repositories git fsck.";
38 const SKIP = "No hosted forge was whole in the snapshot, so the hosted restore was skipped.";
39
40 beforeAll(async () => {
41 root = mkdtempSync(join(tmpdir(), "forge-drill-hosted-card-"));
42 app = createApp({ dataDir: root, port: 0 });
43 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
44 if (!u.ok) throw new Error(u.error);
45 // The Backups card shows the drill only where snapshots are kept.
46 svc = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false });
47 servers = await serve(app);
48 const r = await fetch(`${servers.url}/signin`, {
49 method: "POST",
50 headers: { origin: servers.url },
51 body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
52 redirect: "manual",
53 });
54 cookie = (r.headers.get("set-cookie") ?? "").split(";")[0]!;
55 });
56
57 afterAll(async () => {
58 servers.stop();
59 await svc.stop();
60 app.close();
61 rmSync(root, { recursive: true, force: true });
62 });
63
64 const cardSays = async (): Promise<string | undefined> => {
65 const page = await (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text();
66 return /<span data-drill-hosted>([^<]*)<\/span>/.exec(page)?.[1];
67 };
68
69 test("seconds have one decimal and never go negative", () => {
70 expect([seconds(3_240), seconds(0), seconds(59_960), seconds(-5)]).toEqual(["3.2 s", "0.0 s", "60.0 s", "0.0 s"]);
71 });
72
73 test("the words name the forge, its size and seconds; one forge alone and one repository read as such", () => {
74 expect(hostedDrillWords(restored)).toBe(LINE);
75 expect(hostedDrillWords({ hosted: { name: "solo", of: 1, repos: 1, bytes: 512, ms: 900 } })).toBe(
76 "Hosted forge solo (512 B) restored alone in 0.9 s: its database passed the integrity check and its 1 repository git fsck.",
77 );
78 // A proof kept before the drill timed it says no time rather than a wrong one.
79 expect(hostedDrillWords({ hosted: { name: "old", of: 1, repos: 0, bytes: 2048 } })).toBe(
80 "Hosted forge old (2 KiB) restored alone: its database passed the integrity check and its 0 repositories git fsck.",
81 );
82 // A control character in a kept name cannot move the terminal's cursor.
83 expect(hostedDrillWords({ hosted: { name: "a\u001b[2Jb", of: 1, repos: 0, bytes: 0, ms: 0 } })).toStartWith("Hosted forge a?[2Jb (");
84 expect(hostedDrillWords({ hostedSkipped: true })).toBe(SKIP);
85 expect(hostedDrillWords(green)).toBeUndefined();
86 });
87
88 test("a drill that restored a hosted forge keeps its seconds, and the card and the terminal say it", async () => {
89 recordDrill(app.db, restored);
90 expect(lastDrill(app.db)?.hosted).toEqual(restored.hosted!);
91 expect(await cardSays()).toBe(LINE);
92 expect(hostedDrillLines({ app }).join(" ").replace(/\s+/g, " ").trim()).toBe(`Last drill: ${LINE}`);
93 expect(verdictComment(restored)).toContain(`- hosted: ${LINE}`);
94 });
95
96 test("a drill with no hosted forge says it skipped the step, in one line", async () => {
97 recordDrill(app.db, { ...green, hostedSkipped: true });
98 expect(lastDrill(app.db)?.hostedSkipped).toBe(true);
99 expect(await cardSays()).toBe(SKIP);
100 expect(hostedDrillLines({ app }).join(" ").replace(/\s+/g, " ").trim()).toBe(`Last drill: ${SKIP}`);
101 });
102
103 test("a drill kept before the hosted step says nothing of it", async () => {
104 recordDrill(app.db, green);
105 expect(await cardSays()).toBeUndefined();
106 expect(hostedDrillLines({ app })).toEqual([]);
107 expect(verdictComment(green)).not.toContain("hosted");
108 });
109
110 test("a drill red at the hosted step is a verdict the pager reads, naming the forge", () => {
111 const red: DrillResult = { ...green, verdict: "red", failedStep: "forge", health: "ok", reason: "hosted forge glint: its database does not pass the integrity check" };
112 recordVerdict(root, red);
113 const kept = lastVerdict(root);
114 expect(kept?.failedStep).toBe("forge");
115 // Paged once: a red drill is one need, keyed by its time, so the next look at it is not news.
116 const need = drillNeed(kept, Date.parse("2026-11-15T09:00:00Z"), () => undefined);
117 expect(need?.sentence).toContain("hosted forge glint");
118 expect(drillNeed(lastVerdict(root), Date.parse("2026-11-15T10:00:00Z"), () => undefined)?.key).toBe(need?.key);
119 });
test/snapshot-drill.test.ts +3
@@ -148,6 +148,9 @@ describe("the restore drill", () => {
148148 );
149149 expect(manifest.files.find((f: { path: string }) => f.path === "forge.db").chunks.length).toBeGreaterThan(1);
150150 expect(r.counts).toEqual({ repositories: 1, issues: 2, users: 1, pulls: 0, apps: 1 });
151 // This forge hosts none: the hosted step is skipped, and said so (#13919).
152 expect(r.hosted).toBeUndefined();
153 expect(r.hostedSkipped).toBe(true);
151154 expect(r.ms).toBeGreaterThanOrEqual(0);
152155 expect(scratchLeft()).toEqual([]);
153156 });
test/snapshot-hosted-chunks.test.ts +8 −1
@@ -31,6 +31,7 @@ import { judge, readLane, type LaneRead } from "../src/snapshot/bucket.ts";
3131 import { CHUNK_PREFIX, chunkKey } from "../src/snapshot/chunks.ts";
3232 import { runDrill } from "../src/snapshot/drill.ts";
3333 import { drillPick } from "../src/snapshot/drill_hosted.ts";
34 import { hostedDrillWords } from "../src/snapshot/drill_hosted_words.ts";
3435 import { restoreHosted } from "../src/snapshot/restore.ts";
3536 import { chunksNamed, pruneSnapshots, takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts";
3637 import type { Store } from "../src/snapshot/store.ts";
@@ -260,7 +261,12 @@ describe("two nights of a host with hosted forges", () => {
260261 const result = await runDrill({ store, dataDir: fresh("drill"), passphrase: PASS, now: () => at, boot: async () => ({ ok: true, reason: "" }) });
261262 expect(result.reason).toBeNull();
262263 expect(result.verdict).toBe("green");
263 expect(result.hosted).toEqual({ name: "alpha", of: 1, repos: 1, bytes: m2.hosted![0]!.bytes });
264 const { ms, ...proof } = result.hosted!;
265 expect(proof).toEqual({ name: "alpha", of: 1, repos: 1, bytes: m2.hosted![0]!.bytes });
266 // Its restore and checks are timed, and the card says them in seconds (#13919).
267 expect(Number.isSafeInteger(ms) && ms! >= 0).toBe(true);
268 expect(result.hostedSkipped).toBeUndefined();
269 expect(hostedDrillWords(result)).toMatch(/^Hosted forge alpha \(\d+(\.\d)? (B|KiB|MiB)\) restored alone in \d+\.\d s: its database passed the integrity check and its 1 repository git fsck\.$/);
264270 // The board says the forge the drill brought back.
265271 const now = at.getTime() + 3_600_000;
266272 const lane = await readLane("primary", store, now);
@@ -278,6 +284,7 @@ describe("two nights of a host with hosted forges", () => {
278284 expect([red.verdict, red.failedStep]).toEqual(["red", "forge"]);
279285 expect(red.reason).toMatch(new RegExp(`missing chunk ${lost} of forge.db`));
280286 expect(red.hosted).toBeUndefined();
287 expect(red.hostedSkipped).toBeUndefined();
281288 });
282289
283290 test("the drill's forge turns by the day and by the week, and never to one that was left out", () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs