Public: anyone can read this; the crew can change it.
A failed hosted-forge restore in the drill is one named failure and pages once #14197
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14186-a-failed-hosted-forge-restore-in-the-dri → main 3 commits fork
3 files
changed +169
−1
src/snapshot/drill.ts +13
| @@ -219,6 +219,15 @@ export async function answersRoot(base: string): Promise<void> { | ||
| 219 | 219 | |
| 220 | 220 | class StepTimeout extends Error {} |
| 221 | 221 | |
| 222 | /** | |
| 223 | * A failure at step `forge`, read as `hosted forge <name>: <what did not hold>` (#14186), whether | |
| 224 | * the proof said the name itself or the restore under it, or the budget, did not. | |
| 225 | */ | |
| 226 | export function namedForge(name: string, what: string): string { | |
| 227 | const prefix = `hosted forge ${name}: `; | |
| 228 | return what.startsWith(prefix) ? what : `${prefix}${what}`; | |
| 229 | } | |
| 230 | ||
| 222 | 231 | /** Where a restore got to, for people: "bundles (37 of 212 cloned, 1.8 GiB fetched)". */ |
| 223 | 232 | export function describeProgress(p: Progress): string { |
| 224 | 233 | const unit = p.phase === "bundles" ? ["cloned", "fetched"] : ["done", "so far"]; |
| @@ -259,6 +268,8 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 259 | 268 | let health: DrillResult["health"] = "unreached"; |
| 260 | 269 | let hosted: HostedProof | undefined; |
| 261 | 270 | let hostedSkipped = false; |
| 271 | // The hosted forge being proved, so its failure names it whatever threw. | |
| 272 | let proving: string | undefined; | |
| 262 | 273 | let bootEnd: number | undefined; |
| 263 | 274 | try { |
| 264 | 275 | // One clock for the whole drill, so a hang in any step ends it red. |
| @@ -357,6 +368,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 357 | 368 | const pick = drillPick(held, clock()); |
| 358 | 369 | if (pick !== undefined) { |
| 359 | 370 | step = "forge"; |
| 371 | proving = pick.name; | |
| 360 | 372 | const from = tick(); |
| 361 | 373 | const proof = await within( |
| 362 | 374 | (deps.prove ?? proveHosted)(store, newest.name, pick, join(scratch, "forge"), { passphrase: deps.passphrase, of: held.length, leftMs: left, ticks: tick }), |
| @@ -375,6 +387,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 375 | 387 | : error instanceof Error |
| 376 | 388 | ? error.message |
| 377 | 389 | : String(error); |
| 390 | if (step === "forge" && proving !== undefined) reason = namedForge(proving, error instanceof StepTimeout ? `timed out: ${error.message}` : reason); | |
| 378 | 391 | return finish(step); |
| 379 | 392 | } finally { |
| 380 | 393 | if (scratch !== undefined) { |
test/drill-hosted-proof.test.ts +3 −1
| @@ -109,8 +109,10 @@ describe("proveHosted", () => { | ||
| 109 | 109 | test("a whole hosted forge restores and reports its name, repositories and bytes", async () => { |
| 110 | 110 | const { store, entry } = await snapshotOf("alpha", whole("alpha")); |
| 111 | 111 | const dir = fresh(); |
| 112 | const proof = await proveHosted(store, SNAP, entry, dir, { of: 3 }); | |
| 112 | // Its time is read from the real clock (#14189): whole ms, checked for shape only. | |
| 113 | const { ms, ...proof } = await proveHosted(store, SNAP, entry, dir, { of: 3 }); | |
| 113 | 114 | expect(proof).toEqual({ name: "alpha", of: 3, repos: 1, bytes: entry.bytes }); |
| 115 | expect(Number.isSafeInteger(ms) && ms! >= 0).toBe(true); | |
| 114 | 116 | expect(readdirSync(join(dir, "repos", "alpha"))).toEqual(["notes.git"]); |
| 115 | 117 | }); |
| 116 | 118 |
test/drill-hosted-red.test.ts added +153
| @@ -0,0 +1,153 @@ | ||
| 1 | /** | |
| 2 | * A hosted forge that does not prove in the restore drill (#14186): with an injected `prove` | |
| 3 | * (deps.prove) that throws, the drill is red at step `forge` with the reason `hosted forge <name>: | |
| 4 | * <what did not hold>`, stops there, and the forge's admin is paged once for it, not again when the | |
| 5 | * same drill is read again. The host's own restore is a fixture database written in place, so no | |
| 6 | * real snapshot is taken. | |
| 7 | */ | |
| 8 | import { Database } from "bun:sqlite"; | |
| 9 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 10 | import { mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { createApp, type App } from "../src/app.ts"; | |
| 14 | import { createUser } from "../src/core/users.ts"; | |
| 15 | import { setNotifyUrl, startNotifier } from "../src/crew/notify.ts"; | |
| 16 | import { namedForge, runDrill, type DrillDeps } from "../src/snapshot/drill.ts"; | |
| 17 | import type { HostedEntry } from "../src/snapshot/hosted.ts"; | |
| 18 | import { SNAPSHOT_PREFIX, type Manifest } from "../src/snapshot/snapshot.ts"; | |
| 19 | import type { Store } from "../src/snapshot/store.ts"; | |
| 20 | import { recordVerdict, seedFirstSeen } from "../src/snapshot/verdict.ts"; | |
| 21 | ||
| 22 | const SNAP = "2026-11-16T08-00-00Z"; | |
| 23 | const AT = new Date("2026-11-16T12:00:00Z"); | |
| 24 | const entry: HostedEntry = { name: "alpha", at: "2026-11-16T08:00:00Z", repos: 1, bytes: 10, key: "none" }; | |
| 25 | ||
| 26 | const manifest: Manifest = { | |
| 27 | format: 2, | |
| 28 | name: SNAP, | |
| 29 | startedAt: entry.at, | |
| 30 | finishedAt: entry.at, | |
| 31 | version: "t", | |
| 32 | repos: 0, | |
| 33 | apps: 0, | |
| 34 | files: [], | |
| 35 | bytes: 0, | |
| 36 | hosted: [entry], | |
| 37 | }; | |
| 38 | ||
| 39 | /** A store holding one snapshot's manifest, naming hosted forge alpha; nothing else. */ | |
| 40 | function storeOf(): Store { | |
| 41 | const objects = new Map<string, Uint8Array>([[`${SNAPSHOT_PREFIX}${SNAP}/manifest.json`, new TextEncoder().encode(JSON.stringify(manifest))]]); | |
| 42 | return { | |
| 43 | where: "memory", | |
| 44 | put: async (key, body) => void objects.set(key, typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body)), | |
| 45 | get: async (key) => objects.get(key), | |
| 46 | list: async (prefix) => [...objects.keys()].filter((k) => k.startsWith(prefix)), | |
| 47 | names: async () => [SNAP], | |
| 48 | delete: async (key) => void objects.delete(key), | |
| 49 | }; | |
| 50 | } | |
| 51 | ||
| 52 | /** The host's restore: an empty forge's database, enough for open, count and unseal to pass. */ | |
| 53 | const restore: NonNullable<DrillDeps["restore"]> = async (_store, _name, target) => { | |
| 54 | mkdirSync(target, { recursive: true }); | |
| 55 | const db = new Database(join(target, "forge.db")); | |
| 56 | for (const t of ["repo", "issue", "user", "pull", "app"]) db.run(`CREATE TABLE ${t} (id INTEGER PRIMARY KEY)`); | |
| 57 | db.run("CREATE TABLE secret (name TEXT, scope TEXT, value_enc TEXT)"); | |
| 58 | db.run("CREATE TABLE app_env (app_id TEXT, name TEXT, value_enc TEXT)"); | |
| 59 | db.close(); | |
| 60 | return { manifest, files: 1, repos: 0, key: "missing", sealedMissing: [] }; | |
| 61 | }; | |
| 62 | ||
| 63 | let root: string; | |
| 64 | let made = 0; | |
| 65 | const deps = (prove: DrillDeps["prove"], extra: Partial<DrillDeps> = {}): DrillDeps => ({ | |
| 66 | store: storeOf(), | |
| 67 | dataDir: join(root, `drill-${++made}`), | |
| 68 | now: () => AT, | |
| 69 | restore, | |
| 70 | boot: async () => ({ ok: true, reason: "" }), | |
| 71 | prove, | |
| 72 | ...extra, | |
| 73 | }); | |
| 74 | const scratchLeft = (d: DrillDeps): string[] => readdirSync(join(d.dataDir, "drill")); | |
| 75 | ||
| 76 | beforeAll(() => { | |
| 77 | root = mkdtempSync(join(tmpdir(), "forge-drill-hosted-red-")); | |
| 78 | }); | |
| 79 | afterAll(() => rmSync(root, { recursive: true, force: true })); | |
| 80 | ||
| 81 | describe("a hosted forge that does not prove in the drill", () => { | |
| 82 | test("is red at step forge, naming the forge and what did not hold, and the drill stops there", async () => { | |
| 83 | let asked = 0; | |
| 84 | const d = deps(async () => { | |
| 85 | asked++; | |
| 86 | throw new Error("hosted forge alpha: its database does not pass the integrity check"); | |
| 87 | }); | |
| 88 | const r = await runDrill(d); | |
| 89 | expect(asked).toBe(1); | |
| 90 | expect(r.verdict).toBe("red"); | |
| 91 | expect(r.failedStep).toBe("forge"); | |
| 92 | // Said once, not "hosted forge alpha: hosted forge alpha: …". | |
| 93 | expect(r.reason).toBe("hosted forge alpha: its database does not pass the integrity check"); | |
| 94 | // The host itself came back: the steps before it held. | |
| 95 | expect(r.health).toBe("ok"); | |
| 96 | expect(r.hosted).toBeUndefined(); | |
| 97 | expect(r.hostedSkipped).toBeUndefined(); | |
| 98 | expect(scratchLeft(d)).toEqual([]); | |
| 99 | }); | |
| 100 | ||
| 101 | test("a throw from under the proof that does not name the forge is named for it", async () => { | |
| 102 | const r = await runDrill(deps(async () => { | |
| 103 | throw new Error("missing chunk abc of forge.db"); | |
| 104 | })); | |
| 105 | expect([r.verdict, r.failedStep, r.reason]).toEqual(["red", "forge", "hosted forge alpha: missing chunk abc of forge.db"]); | |
| 106 | }); | |
| 107 | ||
| 108 | test("a proof that hangs past the drill's own budget is red at forge, naming the forge", async () => { | |
| 109 | const r = await runDrill(deps(() => new Promise(() => {}), { timeoutMs: 50 })); | |
| 110 | expect([r.verdict, r.failedStep]).toEqual(["red", "forge"]); | |
| 111 | expect(r.reason).toBe("hosted forge alpha: timed out: no answer within its 50 ms budget for a 0 B snapshot"); | |
| 112 | }); | |
| 113 | ||
| 114 | test("namedForge prefixes once", () => { | |
| 115 | expect(namedForge("beta", "x")).toBe("hosted forge beta: x"); | |
| 116 | expect(namedForge("beta", "hosted forge beta: x")).toBe("hosted forge beta: x"); | |
| 117 | expect(namedForge("beta", "hosted forge alpha: x")).toBe("hosted forge beta: hosted forge alpha: x"); | |
| 118 | }); | |
| 119 | ||
| 120 | test("the owner is paged once for it, not again on the same drill's re-read", async () => { | |
| 121 | const appDir = join(root, "forge"); | |
| 122 | const app: App = createApp({ dataDir: appDir, port: 0 }); | |
| 123 | try { | |
| 124 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 125 | if (!u.ok) throw new Error(u.error); | |
| 126 | seedFirstSeen(appDir, AT.getTime() - 30 * 86_400_000); | |
| 127 | setNotifyUrl(app.db, u.value.id, "https://ntfy.example.com/topic"); | |
| 128 | const sent: string[] = []; | |
| 129 | const n = startNotifier(app, { | |
| 130 | forgeUrl: "https://forge.example.com", | |
| 131 | now: () => AT.getTime() + 60_000, | |
| 132 | send: async (_u, body) => (sent.push(body), true), | |
| 133 | }); | |
| 134 | await n.settled(); | |
| 135 | const r = await runDrill(deps(async () => { | |
| 136 | throw new Error("hosted forge alpha: 0 repositories came back, its copy holds 1"); | |
| 137 | })); | |
| 138 | recordVerdict(appDir, r); | |
| 139 | app.bus.append({ kind: "drill.finished", payload: { verdict: r.verdict, step: r.failedStep } }); | |
| 140 | await n.settled(); | |
| 141 | expect(sent).toHaveLength(1); | |
| 142 | expect(sent[0]).toContain("failed at forge: hosted forge alpha: 0 repositories came back, its copy holds 1"); | |
| 143 | // The same drill read again, by another event or the board: no second page. | |
| 144 | app.bus.append({ kind: "drill.finished", payload: { verdict: r.verdict, step: r.failedStep } }); | |
| 145 | app.bus.append({ kind: "drill.finished", payload: {} }); | |
| 146 | await n.settled(); | |
| 147 | expect(sent).toHaveLength(1); | |
| 148 | await n.close(); | |
| 149 | } finally { | |
| 150 | app.close(); | |
| 151 | } | |
| 152 | }); | |
| 153 | }); |
new above
offline