Public: anyone can read this; the crew can change it.
The restore drill restores one hosted forge from its backup file and proves its database and repositories open #14181
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13919-the-restore-drill-restores-one-hosted-fo → main 10 commits fork
11 files
changed +216
−7
docs/features/hosted-backup.md +4
| @@ -34,6 +34,10 @@ A hosted forge's directory is the forge's own to write, and its host reads it fo | ||
| 34 | 34 | |
| 35 | 35 | Each restore drill, after the host itself, restores one hosted forge alone from the same snapshot (the next one each day and each week, `src/snapshot/drill_hosted.ts`) and proves it opens: its ledger reads and names it, its database passes SQLite's integrity check, and every repository its copy holds comes back and passes `git fsck`. The verdict says which forge ("…passed, with hosted forge glint (1 of 17)"). One that does not come back fails the drill at the step `forge`, by name ([](../RECOVERY.md)). |
| 36 | 36 | |
| 37 | - Web: `/settings/backups`, press Run a drill; when it ends, the last-drill line says "Hosted forge NAME (SIZE, 1 of N) restored alone in 3.2 s: its database passed the integrity check and its K repositories git fsck." Terminal: the Resilience screen's "Last drill:" line under the verdict says the same, and so does the recovery issue's comment (`- hosted:`). | |
| 38 | - A forge that hosts none, or whose snapshot left every hosted forge out, says in one line that the hosted restore was skipped; the drill stays green. | |
| 39 | - Red at `forge`: the reason names the forge and what did not hold (a missing chunk, the integrity check, a repository's `git fsck`, its ledger); it is paged once like every red drill. | |
| 40 | ||
| 37 | 41 | ## Restore one forge |
| 38 | 42 | |
| 39 | 43 | Into an empty data directory; `children/<bodyId>/` and `spawn/<name>.json` are written there: |
src/snapshot/drill.ts +12 −2
| @@ -190,6 +190,11 @@ export interface DrillResult { | ||
| 190 | 190 | * the snapshot held none, or the drill failed before it. |
| 191 | 191 | */ |
| 192 | 192 | readonly hosted?: HostedProof | undefined; |
| 193 | /** | |
| 194 | * True when every earlier step passed and the snapshot held no hosted forge whole to restore, so | |
| 195 | * that step was skipped (#13919); absent otherwise, and on drills kept before it. | |
| 196 | */ | |
| 197 | readonly hostedSkipped?: true | undefined; | |
| 193 | 198 | } |
| 194 | 199 | |
| 195 | 200 | /** The drill process's wall time and peak resident memory, and the memory limit it ran under. */ |
| @@ -253,6 +258,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 253 | 258 | let bootStart: number | undefined; |
| 254 | 259 | let health: DrillResult["health"] = "unreached"; |
| 255 | 260 | let hosted: HostedProof | undefined; |
| 261 | let hostedSkipped = false; | |
| 256 | 262 | let bootEnd: number | undefined; |
| 257 | 263 | try { |
| 258 | 264 | // One clock for the whole drill, so a hang in any step ends it red. |
| @@ -351,12 +357,15 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 351 | 357 | const pick = drillPick(held, clock()); |
| 352 | 358 | if (pick !== undefined) { |
| 353 | 359 | step = "forge"; |
| 354 | hosted = await within( | |
| 360 | const from = tick(); | |
| 361 | const proof = await within( | |
| 355 | 362 | (deps.prove ?? proveHosted)(store, newest.name, pick, join(scratch, "forge"), { passphrase: deps.passphrase, of: held.length, leftMs: left }), |
| 356 | 363 | left(), |
| 357 | 364 | budget, |
| 358 | 365 | ); |
| 359 | } | |
| 366 | // Its restore and checks together, so the card can say it in seconds (#13919). | |
| 367 | hosted = { ...proof, ms: Math.round(tick() - from) }; | |
| 368 | } else hostedSkipped = true; | |
| 360 | 369 | } catch (error) { |
| 361 | 370 | reason = |
| 362 | 371 | error instanceof StepTimeout && step === "restore" && progress !== undefined |
| @@ -402,6 +411,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 402 | 411 | offSite: isOffBox(deps.store), |
| 403 | 412 | ...(format === undefined ? {} : { format }), |
| 404 | 413 | ...(hosted === undefined ? {} : { hosted }), |
| 414 | ...(hostedSkipped ? { hostedSkipped: true as const } : {}), | |
| 405 | 415 | }; |
| 406 | 416 | } |
| 407 | 417 | } |
src/snapshot/drill_hosted.ts +5
| @@ -28,6 +28,11 @@ export interface HostedProof { | ||
| 28 | 28 | readonly repos: number; |
| 29 | 29 | /** The bytes its restore wrote. */ |
| 30 | 30 | readonly bytes: number; |
| 31 | /** | |
| 32 | * Its restore and checks, wall time in ms, as the drill timed them (#13919); absent from a proof | |
| 33 | * before the drill times it, and on drills kept before that. | |
| 34 | */ | |
| 35 | readonly ms?: number | undefined; | |
| 31 | 36 | } |
| 32 | 37 | |
| 33 | 38 | /** No fsck of one repository is given longer than this. */ |
src/snapshot/drill_hosted_words.ts added +30
| @@ -0,0 +1,30 @@ | ||
| 1 | /** | |
| 2 | * What the last restore drill says of its hosted step (#13919), one line, in the same words on the | |
| 3 | * Backups card, the Resilience screen and the recovery issue's comment: the forge it restored | |
| 4 | * alone, its size and its restore time in seconds; or that it was skipped for want of one. A drill | |
| 5 | * that failed there is red at the step `forge` and its reason names the forge; this says nothing | |
| 6 | * more of it. Control characters in a kept name print as `?`; each surface still escapes the line. | |
| 7 | */ | |
| 8 | ||
| 9 | import { printable } from "./bucket.ts"; | |
| 10 | import { formatBytes, type DrillResult } from "./drill.ts"; | |
| 11 | ||
| 12 | /** Milliseconds as seconds with one decimal: 3240 → "3.2 s". */ | |
| 13 | export function seconds(ms: number): string { | |
| 14 | return `${(Math.max(0, ms) / 1000).toFixed(1)} s`; | |
| 15 | } | |
| 16 | ||
| 17 | /** | |
| 18 | * The line, or undefined when the drill reached no hosted step (it failed before, or predates it). | |
| 19 | */ | |
| 20 | export function hostedDrillWords(d: Pick<DrillResult, "hosted" | "hostedSkipped">): string | undefined { | |
| 21 | const h = d.hosted; | |
| 22 | if (h !== undefined) { | |
| 23 | const took = h.ms === undefined ? "" : ` in ${seconds(h.ms)}`; | |
| 24 | const of = h.of > 1 ? `, 1 of ${h.of}` : ""; | |
| 25 | const repos = h.repos === 1 ? "its 1 repository" : `its ${h.repos} repositories`; | |
| 26 | return `Hosted forge ${printable(h.name)} (${formatBytes(h.bytes)}${of}) restored alone${took}: its database passed the integrity check and ${repos} git fsck.`; | |
| 27 | } | |
| 28 | if (d.hostedSkipped === true) return "No hosted forge was whole in the snapshot, so the hosted restore was skipped."; | |
| 29 | return undefined; | |
| 30 | } |
src/snapshot/drills.ts +13 −2
| @@ -66,9 +66,18 @@ export function parseHosted(raw: string | null): { hosted?: HostedProof } { | ||
| 66 | 66 | const h = (JSON.parse(raw ?? "null") as { hosted?: unknown } | null)?.hosted as Record<string, unknown> | undefined; |
| 67 | 67 | if (typeof h !== "object" || h === null) return {}; |
| 68 | 68 | const whole = (v: unknown): v is number => typeof v === "number" && Number.isSafeInteger(v) && v >= 0; |
| 69 | const { name, of, repos, bytes } = h; | |
| 69 | const { name, of, repos, bytes, ms } = h; | |
| 70 | 70 | if (typeof name !== "string" || name.length > 100 || !whole(of) || !whole(repos) || !whole(bytes)) return {}; |
| 71 | return { hosted: { name, of, repos, bytes } }; | |
| 71 | return { hosted: { name, of, repos, bytes, ...(whole(ms) ? { ms } : {}) } }; | |
| 72 | } catch { | |
| 73 | return {}; | |
| 74 | } | |
| 75 | } | |
| 76 | ||
| 77 | /** Whether a kept drill skipped its hosted step for want of a hosted forge (#13919). */ | |
| 78 | export function parseHostedSkipped(raw: string | null): { hostedSkipped?: true } { | |
| 79 | try { | |
| 80 | return (JSON.parse(raw ?? "null") as { hostedSkipped?: unknown } | null)?.hostedSkipped === true ? { hostedSkipped: true } : {}; | |
| 72 | 81 | } catch { |
| 73 | 82 | return {}; |
| 74 | 83 | } |
| @@ -121,6 +130,7 @@ export function recordDrill(db: Database, result: DrillResult): void { | ||
| 121 | 130 | ...(result.from === undefined ? {} : { from: result.from, offSite: result.offSite === true }), |
| 122 | 131 | ...(result.format === undefined ? {} : { format: result.format }), |
| 123 | 132 | ...(result.hosted === undefined ? {} : { hosted: result.hosted }), |
| 133 | ...(result.hostedSkipped === true ? { hostedSkipped: true } : {}), | |
| 124 | 134 | ...(result.cost === undefined |
| 125 | 135 | ? {} |
| 126 | 136 | : { cost: { elapsedMs: Math.round(result.cost.elapsedMs), peakBytes: Math.round(result.cost.peakBytes), limitBytes: Math.round(result.cost.limitBytes) } }), |
| @@ -166,5 +176,6 @@ function asResult(row: Row | null): DrillResult | undefined { | ||
| 166 | 176 | ...parseFrom(row.phases), |
| 167 | 177 | ...parseCost(row.phases), |
| 168 | 178 | ...parseHosted(row.phases), |
| 179 | ...parseHostedSkipped(row.phases), | |
| 169 | 180 | }; |
| 170 | 181 | } |
src/snapshot/verdict.ts +5 −1
| @@ -9,9 +9,12 @@ | ||
| 9 | 9 | |
| 10 | 10 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| 11 | 11 | import { join } from "node:path"; |
| 12 | import { hostedDrillWords } from "./drill_hosted_words.ts"; | |
| 12 | 13 | import { describeTimes, formatMinutes, formatSpan, timeByPart, type DrillResult, type DrillStep } from "./drill.ts"; |
| 13 | 14 | |
| 14 | const STEPS: readonly DrillStep[] = ["fetch", "restore", "open", "count", "unseal", "health"]; | |
| 15 | // Every step `runDrill` can fail at: a step not here makes the kept verdict unreadable, and a red | |
| 16 | // drill at it would never page. | |
| 17 | const STEPS: readonly DrillStep[] = ["fetch", "restore", "open", "count", "unseal", "health", "forge"]; | |
| 15 | 18 | |
| 16 | 19 | const lastFile = (dataDir: string): string => join(dataDir, "drill", "last.json"); |
| 17 | 20 | |
| @@ -168,6 +171,7 @@ export function verdictComment(r: DrillResult): string { | ||
| 168 | 171 | `- users: ${r.counts.users}`, |
| 169 | 172 | `- pulls: ${r.counts.pulls}`, |
| 170 | 173 | `- apps: ${r.counts.apps}`, |
| 174 | ...(hostedDrillWords(r) === undefined ? [] : [`- hosted: ${hostedDrillWords(r)}`]), | |
| 171 | 175 | ...(pass || !r.reason ? [] : [`- why: ${r.reason}`]), |
| 172 | 176 | ].join("\n"); |
| 173 | 177 | } |
src/tui/screens/resilience.ts +12
| @@ -43,6 +43,7 @@ import { hostedWords, keptWords, readHosted } from "../../snapshot/hosted.ts"; | ||
| 43 | 43 | import { backupsLine } from "../../snapshot/backups_line.ts"; |
| 44 | 44 | import { backupVerdict } from "../../snapshot/bucket.ts"; |
| 45 | 45 | import { lastDrill } from "../../snapshot/drills.ts"; |
| 46 | import { hostedDrillWords } from "../../snapshot/drill_hosted_words.ts"; | |
| 46 | 47 | import { resolveSnapshots } from "../../snapshot/destination.ts"; |
| 47 | 48 | import { bodiesOf } from "../../snapshot/store.ts"; |
| 48 | 49 | import { snapshotsOf } from "../../snapshot/service.ts"; |
| @@ -119,9 +120,20 @@ export function verdictLines(f: Session): string[] { | ||
| 119 | 120 | ...wrap(` ${v.checkLine}`, INNER), |
| 120 | 121 | ...v.lanes.flatMap((l) => wrap(` ${l.state === "red" ? "!" : " "} ${l.says}`, INNER).concat(wrap(` ${l.line}${l.bodies === undefined ? "" : `; ${l.bodies}`}`, INNER))), |
| 121 | 122 | ...[...v.reasons, ...v.unknowns].flatMap((r) => wrap(` - ${r}`, INNER)), |
| 123 | ...hostedDrillLines(f), | |
| 122 | 124 | ]; |
| 123 | 125 | } |
| 124 | 126 | |
| 127 | /** | |
| 128 | * The last drill's hosted step in the Backups card's words (#13919): the forge it restored, its | |
| 129 | * size and seconds, or that it was skipped; none when that drill reached no hosted step. | |
| 130 | */ | |
| 131 | export function hostedDrillLines(f: Pick<Session, "app">): string[] { | |
| 132 | const d = lastDrill(f.app.db); | |
| 133 | const said = d === undefined ? undefined : hostedDrillWords(d); | |
| 134 | return said === undefined ? [] : wrap(` Last drill: ${said}`, INNER); | |
| 135 | } | |
| 136 | ||
| 125 | 137 | const STEPS: readonly { key: keyof DestinationInput; label: string; placeholder?: string; mask?: true }[] = [ |
| 126 | 138 | { key: "endpoint", label: "Endpoint, an http(s) URL", placeholder: "https://s3.example.com" }, |
| 127 | 139 | { key: "bucket", label: "Bucket" }, |
src/web/pages/backups.ts +5 −1
| @@ -31,6 +31,7 @@ import { lastDrill } from "../../snapshot/drills.ts"; | ||
| 31 | 31 | import { backupVerdict } from "../../snapshot/bucket.ts"; |
| 32 | 32 | import { bodiesOf, type Store } from "../../snapshot/store.ts"; |
| 33 | 33 | import { hostedWords } from "../../snapshot/hosted.ts"; |
| 34 | import { hostedDrillWords } from "../../snapshot/drill_hosted_words.ts"; | |
| 34 | 35 | import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts"; |
| 35 | 36 | import { drillStatus, mirrorNow, snapshotNow, startDrillNow } from "../../actions/backups_now.ts"; |
| 36 | 37 | import { lastSnapshotOutcome } from "../../snapshot/outcome.ts"; |
| @@ -132,6 +133,9 @@ function drillLine(ctx: Ctx): Html { | ||
| 132 | 133 | d === undefined |
| 133 | 134 | ? "" |
| 134 | 135 | : html`<dl class="status-rows" id="drill-cost">${costFacts(d).map((f) => html`<div class="status-row"><dt>${f.label}</dt><dd>${f.value}</dd></div>`)}</dl>`; |
| 136 | // The hosted forge it restored alone, by name, size and seconds, or that it skipped (#13919). | |
| 137 | const said = d === undefined ? undefined : hostedDrillWords(d); | |
| 138 | const hosted = said === undefined ? "" : html` <span data-drill-hosted>${said}</span>`; | |
| 135 | 139 | const line = |
| 136 | 140 | d === undefined |
| 137 | 141 | ? html`<p class="break">No restore drill has run yet.</p>` |
| @@ -139,7 +143,7 @@ function drillLine(ctx: Ctx): Html { | ||
| 139 | 143 | ? html`<p class="break"> |
| 140 | 144 | ${badge("Ready", { tone: "success", icon: "check" })} Last drill: ${time(d.at)}, ${d.ms} ms, all counts matched — |
| 141 | 145 | snapshot <code>${d.snapshotId ?? ""}</code>, key opened — |
| 142 | ${d.counts.users} users, ${d.counts.repositories} repositories, ${d.counts.issues} issues, ${d.counts.pulls} pulls, ${d.counts.apps} apps.${timing | |
| 146 | ${d.counts.users} users, ${d.counts.repositories} repositories, ${d.counts.issues} issues, ${d.counts.pulls} pulls, ${d.counts.apps} apps.${hosted}${timing | |
| 143 | 147 | ? html` <span class="muted small">${timing}.</span>` |
| 144 | 148 | : ""} |
| 145 | 149 | </p>` |
test/drill-hosted-card.test.ts added +119
| @@ -0,0 +1,119 @@ | ||
| 1 | /** | |
| 2 | * The restore drill's hosted step, said (#13919): the Backups card, the Resilience screen and the | |
| 3 | * recovery comment name the forge it restored, its size and its restore time in seconds; a drill | |
| 4 | * with no hosted forge says it skipped the step in one line; a drill red at `forge` is a verdict | |
| 5 | * the pager can read, so it pages. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 8 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 14 | import type { DrillResult } from "../src/snapshot/drill.ts"; | |
| 15 | import { hostedDrillWords, seconds } from "../src/snapshot/drill_hosted_words.ts"; | |
| 16 | import { lastDrill, recordDrill } from "../src/snapshot/drills.ts"; | |
| 17 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; | |
| 18 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 19 | import { drillNeed, lastVerdict, recordVerdict, verdictComment } from "../src/snapshot/verdict.ts"; | |
| 20 | import { hostedDrillLines } from "../src/tui/screens/resilience.ts"; | |
| 21 | import { quietLogs } from "./helpers/globals.ts"; | |
| 22 | ||
| 23 | quietLogs(); | |
| 24 | ||
| 25 | let root = ""; | |
| 26 | let app: App; | |
| 27 | let servers: Servers; | |
| 28 | let svc: SnapshotsHandle; | |
| 29 | let cookie = ""; | |
| 30 | ||
| 31 | const green: DrillResult = { | |
| 32 | at: "2026-11-15T08:10:00.000Z", snapshotId: "s", ms: 60_000, counts: { repositories: 1, issues: 0, users: 1, pulls: 0, apps: 0 }, | |
| 33 | health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 600_000, reason: null, | |
| 34 | }; | |
| 35 | // 3 MiB = 3 × 1024² bytes; 3240 ms = 3.2 s. | |
| 36 | const restored: DrillResult = { ...green, hosted: { name: "glint", of: 4, repos: 2, bytes: 3 * 1024 ** 2, ms: 3_240 } }; | |
| 37 | const LINE = "Hosted forge glint (3 MiB, 1 of 4) restored alone in 3.2 s: its database passed the integrity check and its 2 repositories git fsck."; | |
| 38 | const SKIP = "No hosted forge was whole in the snapshot, so the hosted restore was skipped."; | |
| 39 | ||
| 40 | beforeAll(async () => { | |
| 41 | root = mkdtempSync(join(tmpdir(), "forge-drill-hosted-card-")); | |
| 42 | app = createApp({ dataDir: root, port: 0 }); | |
| 43 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 44 | if (!u.ok) throw new Error(u.error); | |
| 45 | // The Backups card shows the drill only where snapshots are kept. | |
| 46 | svc = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false }); | |
| 47 | servers = await serve(app); | |
| 48 | const r = await fetch(`${servers.url}/signin`, { | |
| 49 | method: "POST", | |
| 50 | headers: { origin: servers.url }, | |
| 51 | body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }), | |
| 52 | redirect: "manual", | |
| 53 | }); | |
| 54 | cookie = (r.headers.get("set-cookie") ?? "").split(";")[0]!; | |
| 55 | }); | |
| 56 | ||
| 57 | afterAll(async () => { | |
| 58 | servers.stop(); | |
| 59 | await svc.stop(); | |
| 60 | app.close(); | |
| 61 | rmSync(root, { recursive: true, force: true }); | |
| 62 | }); | |
| 63 | ||
| 64 | const cardSays = async (): Promise<string | undefined> => { | |
| 65 | const page = await (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text(); | |
| 66 | return /<span data-drill-hosted>([^<]*)<\/span>/.exec(page)?.[1]; | |
| 67 | }; | |
| 68 | ||
| 69 | test("seconds have one decimal and never go negative", () => { | |
| 70 | expect([seconds(3_240), seconds(0), seconds(59_960), seconds(-5)]).toEqual(["3.2 s", "0.0 s", "60.0 s", "0.0 s"]); | |
| 71 | }); | |
| 72 | ||
| 73 | test("the words name the forge, its size and seconds; one forge alone and one repository read as such", () => { | |
| 74 | expect(hostedDrillWords(restored)).toBe(LINE); | |
| 75 | expect(hostedDrillWords({ hosted: { name: "solo", of: 1, repos: 1, bytes: 512, ms: 900 } })).toBe( | |
| 76 | "Hosted forge solo (512 B) restored alone in 0.9 s: its database passed the integrity check and its 1 repository git fsck.", | |
| 77 | ); | |
| 78 | // A proof kept before the drill timed it says no time rather than a wrong one. | |
| 79 | expect(hostedDrillWords({ hosted: { name: "old", of: 1, repos: 0, bytes: 2048 } })).toBe( | |
| 80 | "Hosted forge old (2 KiB) restored alone: its database passed the integrity check and its 0 repositories git fsck.", | |
| 81 | ); | |
| 82 | // A control character in a kept name cannot move the terminal's cursor. | |
| 83 | expect(hostedDrillWords({ hosted: { name: "a\u001b[2Jb", of: 1, repos: 0, bytes: 0, ms: 0 } })).toStartWith("Hosted forge a?[2Jb ("); | |
| 84 | expect(hostedDrillWords({ hostedSkipped: true })).toBe(SKIP); | |
| 85 | expect(hostedDrillWords(green)).toBeUndefined(); | |
| 86 | }); | |
| 87 | ||
| 88 | test("a drill that restored a hosted forge keeps its seconds, and the card and the terminal say it", async () => { | |
| 89 | recordDrill(app.db, restored); | |
| 90 | expect(lastDrill(app.db)?.hosted).toEqual(restored.hosted!); | |
| 91 | expect(await cardSays()).toBe(LINE); | |
| 92 | expect(hostedDrillLines({ app }).join(" ").replace(/\s+/g, " ").trim()).toBe(`Last drill: ${LINE}`); | |
| 93 | expect(verdictComment(restored)).toContain(`- hosted: ${LINE}`); | |
| 94 | }); | |
| 95 | ||
| 96 | test("a drill with no hosted forge says it skipped the step, in one line", async () => { | |
| 97 | recordDrill(app.db, { ...green, hostedSkipped: true }); | |
| 98 | expect(lastDrill(app.db)?.hostedSkipped).toBe(true); | |
| 99 | expect(await cardSays()).toBe(SKIP); | |
| 100 | expect(hostedDrillLines({ app }).join(" ").replace(/\s+/g, " ").trim()).toBe(`Last drill: ${SKIP}`); | |
| 101 | }); | |
| 102 | ||
| 103 | test("a drill kept before the hosted step says nothing of it", async () => { | |
| 104 | recordDrill(app.db, green); | |
| 105 | expect(await cardSays()).toBeUndefined(); | |
| 106 | expect(hostedDrillLines({ app })).toEqual([]); | |
| 107 | expect(verdictComment(green)).not.toContain("hosted"); | |
| 108 | }); | |
| 109 | ||
| 110 | test("a drill red at the hosted step is a verdict the pager reads, naming the forge", () => { | |
| 111 | const red: DrillResult = { ...green, verdict: "red", failedStep: "forge", health: "ok", reason: "hosted forge glint: its database does not pass the integrity check" }; | |
| 112 | recordVerdict(root, red); | |
| 113 | const kept = lastVerdict(root); | |
| 114 | expect(kept?.failedStep).toBe("forge"); | |
| 115 | // Paged once: a red drill is one need, keyed by its time, so the next look at it is not news. | |
| 116 | const need = drillNeed(kept, Date.parse("2026-11-15T09:00:00Z"), () => undefined); | |
| 117 | expect(need?.sentence).toContain("hosted forge glint"); | |
| 118 | expect(drillNeed(lastVerdict(root), Date.parse("2026-11-15T10:00:00Z"), () => undefined)?.key).toBe(need?.key); | |
| 119 | }); |
test/snapshot-drill.test.ts +3
| @@ -148,6 +148,9 @@ describe("the restore drill", () => { | ||
| 148 | 148 | ); |
| 149 | 149 | expect(manifest.files.find((f: { path: string }) => f.path === "forge.db").chunks.length).toBeGreaterThan(1); |
| 150 | 150 | expect(r.counts).toEqual({ repositories: 1, issues: 2, users: 1, pulls: 0, apps: 1 }); |
| 151 | // This forge hosts none: the hosted step is skipped, and said so (#13919). | |
| 152 | expect(r.hosted).toBeUndefined(); | |
| 153 | expect(r.hostedSkipped).toBe(true); | |
| 151 | 154 | expect(r.ms).toBeGreaterThanOrEqual(0); |
| 152 | 155 | expect(scratchLeft()).toEqual([]); |
| 153 | 156 | }); |
test/snapshot-hosted-chunks.test.ts +8 −1
| @@ -31,6 +31,7 @@ import { judge, readLane, type LaneRead } from "../src/snapshot/bucket.ts"; | ||
| 31 | 31 | import { CHUNK_PREFIX, chunkKey } from "../src/snapshot/chunks.ts"; |
| 32 | 32 | import { runDrill } from "../src/snapshot/drill.ts"; |
| 33 | 33 | import { drillPick } from "../src/snapshot/drill_hosted.ts"; |
| 34 | import { hostedDrillWords } from "../src/snapshot/drill_hosted_words.ts"; | |
| 34 | 35 | import { restoreHosted } from "../src/snapshot/restore.ts"; |
| 35 | 36 | import { chunksNamed, pruneSnapshots, takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts"; |
| 36 | 37 | import type { Store } from "../src/snapshot/store.ts"; |
| @@ -260,7 +261,12 @@ describe("two nights of a host with hosted forges", () => { | ||
| 260 | 261 | const result = await runDrill({ store, dataDir: fresh("drill"), passphrase: PASS, now: () => at, boot: async () => ({ ok: true, reason: "" }) }); |
| 261 | 262 | expect(result.reason).toBeNull(); |
| 262 | 263 | expect(result.verdict).toBe("green"); |
| 263 | expect(result.hosted).toEqual({ name: "alpha", of: 1, repos: 1, bytes: m2.hosted![0]!.bytes }); | |
| 264 | const { ms, ...proof } = result.hosted!; | |
| 265 | expect(proof).toEqual({ name: "alpha", of: 1, repos: 1, bytes: m2.hosted![0]!.bytes }); | |
| 266 | // Its restore and checks are timed, and the card says them in seconds (#13919). | |
| 267 | expect(Number.isSafeInteger(ms) && ms! >= 0).toBe(true); | |
| 268 | expect(result.hostedSkipped).toBeUndefined(); | |
| 269 | expect(hostedDrillWords(result)).toMatch(/^Hosted forge alpha \(\d+(\.\d)? (B|KiB|MiB)\) restored alone in \d+\.\d s: its database passed the integrity check and its 1 repository git fsck\.$/); | |
| 264 | 270 | // The board says the forge the drill brought back. |
| 265 | 271 | const now = at.getTime() + 3_600_000; |
| 266 | 272 | const lane = await readLane("primary", store, now); |
| @@ -278,6 +284,7 @@ describe("two nights of a host with hosted forges", () => { | ||
| 278 | 284 | expect([red.verdict, red.failedStep]).toEqual(["red", "forge"]); |
| 279 | 285 | expect(red.reason).toMatch(new RegExp(`missing chunk ${lost} of forge.db`)); |
| 280 | 286 | expect(red.hosted).toBeUndefined(); |
| 287 | expect(red.hostedSkipped).toBeUndefined(); | |
| 281 | 288 | }); |
| 282 | 289 | |
| 283 | 290 | test("the drill's forge turns by the day and by the week, and never to one that was left out", () => { |
new above
offline