Public: anyone can read this; the crew can change it.

A failed hosted-forge restore in the drill is one named failure and pages once #14197

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14186-a-failed-hosted-forge-restore-in-the-dri → main 3 commits fork

  • Merged. mason/14186-a-failed-hosted-forge-restore-in-the-dri was merged into main as d954eb4.

Review

Sign in to review this change.
3 files changed +169 −1
src/snapshot/drill.ts +13
@@ -219,6 +219,15 @@ export async function answersRoot(base: string): Promise<void> {
219219
220220 class StepTimeout extends Error {}
221221
222 /**
223 * A failure at step `forge`, read as `hosted forge <name>: <what did not hold>` (#14186), whether
224 * the proof said the name itself or the restore under it, or the budget, did not.
225 */
226 export function namedForge(name: string, what: string): string {
227 const prefix = `hosted forge ${name}: `;
228 return what.startsWith(prefix) ? what : `${prefix}${what}`;
229 }
230
222231 /** Where a restore got to, for people: "bundles (37 of 212 cloned, 1.8 GiB fetched)". */
223232 export function describeProgress(p: Progress): string {
224233 const unit = p.phase === "bundles" ? ["cloned", "fetched"] : ["done", "so far"];
@@ -259,6 +268,8 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
259268 let health: DrillResult["health"] = "unreached";
260269 let hosted: HostedProof | undefined;
261270 let hostedSkipped = false;
271 // The hosted forge being proved, so its failure names it whatever threw.
272 let proving: string | undefined;
262273 let bootEnd: number | undefined;
263274 try {
264275 // One clock for the whole drill, so a hang in any step ends it red.
@@ -357,6 +368,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
357368 const pick = drillPick(held, clock());
358369 if (pick !== undefined) {
359370 step = "forge";
371 proving = pick.name;
360372 const from = tick();
361373 const proof = await within(
362374 (deps.prove ?? proveHosted)(store, newest.name, pick, join(scratch, "forge"), { passphrase: deps.passphrase, of: held.length, leftMs: left, ticks: tick }),
@@ -375,6 +387,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> {
375387 : error instanceof Error
376388 ? error.message
377389 : String(error);
390 if (step === "forge" && proving !== undefined) reason = namedForge(proving, error instanceof StepTimeout ? `timed out: ${error.message}` : reason);
378391 return finish(step);
379392 } finally {
380393 if (scratch !== undefined) {
test/drill-hosted-proof.test.ts +3 −1
@@ -109,8 +109,10 @@ describe("proveHosted", () => {
109109 test("a whole hosted forge restores and reports its name, repositories and bytes", async () => {
110110 const { store, entry } = await snapshotOf("alpha", whole("alpha"));
111111 const dir = fresh();
112 const proof = await proveHosted(store, SNAP, entry, dir, { of: 3 });
112 // Its time is read from the real clock (#14189): whole ms, checked for shape only.
113 const { ms, ...proof } = await proveHosted(store, SNAP, entry, dir, { of: 3 });
113114 expect(proof).toEqual({ name: "alpha", of: 3, repos: 1, bytes: entry.bytes });
115 expect(Number.isSafeInteger(ms) && ms! >= 0).toBe(true);
114116 expect(readdirSync(join(dir, "repos", "alpha"))).toEqual(["notes.git"]);
115117 });
116118
test/drill-hosted-red.test.ts added +153
@@ -0,0 +1,153 @@
1 /**
2 * A hosted forge that does not prove in the restore drill (#14186): with an injected `prove`
3 * (deps.prove) that throws, the drill is red at step `forge` with the reason `hosted forge <name>:
4 * <what did not hold>`, stops there, and the forge's admin is paged once for it, not again when the
5 * same drill is read again. The host's own restore is a fixture database written in place, so no
6 * real snapshot is taken.
7 */
8 import { Database } from "bun:sqlite";
9 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
10 import { mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp, type App } from "../src/app.ts";
14 import { createUser } from "../src/core/users.ts";
15 import { setNotifyUrl, startNotifier } from "../src/crew/notify.ts";
16 import { namedForge, runDrill, type DrillDeps } from "../src/snapshot/drill.ts";
17 import type { HostedEntry } from "../src/snapshot/hosted.ts";
18 import { SNAPSHOT_PREFIX, type Manifest } from "../src/snapshot/snapshot.ts";
19 import type { Store } from "../src/snapshot/store.ts";
20 import { recordVerdict, seedFirstSeen } from "../src/snapshot/verdict.ts";
21
22 const SNAP = "2026-11-16T08-00-00Z";
23 const AT = new Date("2026-11-16T12:00:00Z");
24 const entry: HostedEntry = { name: "alpha", at: "2026-11-16T08:00:00Z", repos: 1, bytes: 10, key: "none" };
25
26 const manifest: Manifest = {
27 format: 2,
28 name: SNAP,
29 startedAt: entry.at,
30 finishedAt: entry.at,
31 version: "t",
32 repos: 0,
33 apps: 0,
34 files: [],
35 bytes: 0,
36 hosted: [entry],
37 };
38
39 /** A store holding one snapshot's manifest, naming hosted forge alpha; nothing else. */
40 function storeOf(): Store {
41 const objects = new Map<string, Uint8Array>([[`${SNAPSHOT_PREFIX}${SNAP}/manifest.json`, new TextEncoder().encode(JSON.stringify(manifest))]]);
42 return {
43 where: "memory",
44 put: async (key, body) => void objects.set(key, typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body)),
45 get: async (key) => objects.get(key),
46 list: async (prefix) => [...objects.keys()].filter((k) => k.startsWith(prefix)),
47 names: async () => [SNAP],
48 delete: async (key) => void objects.delete(key),
49 };
50 }
51
52 /** The host's restore: an empty forge's database, enough for open, count and unseal to pass. */
53 const restore: NonNullable<DrillDeps["restore"]> = async (_store, _name, target) => {
54 mkdirSync(target, { recursive: true });
55 const db = new Database(join(target, "forge.db"));
56 for (const t of ["repo", "issue", "user", "pull", "app"]) db.run(`CREATE TABLE ${t} (id INTEGER PRIMARY KEY)`);
57 db.run("CREATE TABLE secret (name TEXT, scope TEXT, value_enc TEXT)");
58 db.run("CREATE TABLE app_env (app_id TEXT, name TEXT, value_enc TEXT)");
59 db.close();
60 return { manifest, files: 1, repos: 0, key: "missing", sealedMissing: [] };
61 };
62
63 let root: string;
64 let made = 0;
65 const deps = (prove: DrillDeps["prove"], extra: Partial<DrillDeps> = {}): DrillDeps => ({
66 store: storeOf(),
67 dataDir: join(root, `drill-${++made}`),
68 now: () => AT,
69 restore,
70 boot: async () => ({ ok: true, reason: "" }),
71 prove,
72 ...extra,
73 });
74 const scratchLeft = (d: DrillDeps): string[] => readdirSync(join(d.dataDir, "drill"));
75
76 beforeAll(() => {
77 root = mkdtempSync(join(tmpdir(), "forge-drill-hosted-red-"));
78 });
79 afterAll(() => rmSync(root, { recursive: true, force: true }));
80
81 describe("a hosted forge that does not prove in the drill", () => {
82 test("is red at step forge, naming the forge and what did not hold, and the drill stops there", async () => {
83 let asked = 0;
84 const d = deps(async () => {
85 asked++;
86 throw new Error("hosted forge alpha: its database does not pass the integrity check");
87 });
88 const r = await runDrill(d);
89 expect(asked).toBe(1);
90 expect(r.verdict).toBe("red");
91 expect(r.failedStep).toBe("forge");
92 // Said once, not "hosted forge alpha: hosted forge alpha: …".
93 expect(r.reason).toBe("hosted forge alpha: its database does not pass the integrity check");
94 // The host itself came back: the steps before it held.
95 expect(r.health).toBe("ok");
96 expect(r.hosted).toBeUndefined();
97 expect(r.hostedSkipped).toBeUndefined();
98 expect(scratchLeft(d)).toEqual([]);
99 });
100
101 test("a throw from under the proof that does not name the forge is named for it", async () => {
102 const r = await runDrill(deps(async () => {
103 throw new Error("missing chunk abc of forge.db");
104 }));
105 expect([r.verdict, r.failedStep, r.reason]).toEqual(["red", "forge", "hosted forge alpha: missing chunk abc of forge.db"]);
106 });
107
108 test("a proof that hangs past the drill's own budget is red at forge, naming the forge", async () => {
109 const r = await runDrill(deps(() => new Promise(() => {}), { timeoutMs: 50 }));
110 expect([r.verdict, r.failedStep]).toEqual(["red", "forge"]);
111 expect(r.reason).toBe("hosted forge alpha: timed out: no answer within its 50 ms budget for a 0 B snapshot");
112 });
113
114 test("namedForge prefixes once", () => {
115 expect(namedForge("beta", "x")).toBe("hosted forge beta: x");
116 expect(namedForge("beta", "hosted forge beta: x")).toBe("hosted forge beta: x");
117 expect(namedForge("beta", "hosted forge alpha: x")).toBe("hosted forge beta: hosted forge alpha: x");
118 });
119
120 test("the owner is paged once for it, not again on the same drill's re-read", async () => {
121 const appDir = join(root, "forge");
122 const app: App = createApp({ dataDir: appDir, port: 0 });
123 try {
124 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
125 if (!u.ok) throw new Error(u.error);
126 seedFirstSeen(appDir, AT.getTime() - 30 * 86_400_000);
127 setNotifyUrl(app.db, u.value.id, "https://ntfy.example.com/topic");
128 const sent: string[] = [];
129 const n = startNotifier(app, {
130 forgeUrl: "https://forge.example.com",
131 now: () => AT.getTime() + 60_000,
132 send: async (_u, body) => (sent.push(body), true),
133 });
134 await n.settled();
135 const r = await runDrill(deps(async () => {
136 throw new Error("hosted forge alpha: 0 repositories came back, its copy holds 1");
137 }));
138 recordVerdict(appDir, r);
139 app.bus.append({ kind: "drill.finished", payload: { verdict: r.verdict, step: r.failedStep } });
140 await n.settled();
141 expect(sent).toHaveLength(1);
142 expect(sent[0]).toContain("failed at forge: hosted forge alpha: 0 repositories came back, its copy holds 1");
143 // The same drill read again, by another event or the board: no second page.
144 app.bus.append({ kind: "drill.finished", payload: { verdict: r.verdict, step: r.failedStep } });
145 app.bus.append({ kind: "drill.finished", payload: {} });
146 await n.settled();
147 expect(sent).toHaveLength(1);
148 await n.close();
149 } finally {
150 app.close();
151 }
152 });
153 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs