trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns14g rEventsFlows

Recovery, part 2: snapshots are mirrored off the box within the hour, and a stale mirror is a board line and a push #1994

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/1987-recovery-part-2-snapshots-are-mirrored-o → main 8 commits fork 5 threads · 5 unresolved

  • Merged. scout/1987-recovery-part-2-snapshots-are-mirrored-o was merged into main as 4c981fa.

Review

Sign in to review this change.
13 files changed +957 −2
docs/SNAPSHOT.md +18
@@ -67,6 +67,24 @@ Delete the old ones from the bucket if the bucket is not trusted.
6767 - Given neither, no snapshots are taken, and the Backups page says so
6868 instead of showing an empty list that looks like a clean bill.
6969
70 ## The off-box mirror
71
72 A second S3-compatible bucket (Backblaze B2, say) that every snapshot is
73 copied to, so the snapshots do not die with the machine. Seal it on the
74 Backups page (`SNAPSHOT_MIRROR_S3_*`, write-only like the destination) or
75 give `FORGE_MIRROR_S3_ENDPOINT`, `_BUCKET`, `_ACCESS_KEY`, `_SECRET_KEY` and
76 optionally `_REGION`. A pass runs right after each snapshot, thirty seconds
77 after boot and every hour: it copies the newest fourteen snapshots the mirror
78 lacks (manifest last, so a half copy is never listed; objects already there
79 are skipped), then prunes the mirror to fourteen. What it found is kept in
80 `mirror/state.json` under the data directory. The Backups page shows the
81 mirror's reachability, its newest snapshot and when it was last confirmed
82 level. A pass that fails (store unreachable, or a snapshot that did not land)
83 or a mirror not confirmed level for a day is one card on the board
84 (`mirror`) and one push to the admin, the path the drill uses. Not done: the
85 mirror is copied from the primary store, not written in parallel, so a snapshot
86 that never landed on the primary is not seen by it.
87
7088 The store is `Bun.S3Client` or a directory behind one small interface
7189 (`src/snapshot/store.ts`): put, get, list, delete. Nothing else.
7290
scripts/gate.ts +2
@@ -192,6 +192,8 @@ export const GATE: readonly string[] = [
192192 "test/drill-board-line.test.ts",
193193 "test/snapshot-drill-page.test.ts",
194194 "test/snapshot-store.test.ts",
195 "test/snapshot-mirror.test.ts",
196 "test/snapshot-mirror-page.test.ts",
195197 "test/migrate.test.ts",
196198 "test/schema-additive.test.ts",
197199 "test/schema-migrations.test.ts",
src/actions/mirror.ts added +94
@@ -0,0 +1,94 @@
1 /**
2 * The off-box mirror, as an admin sets it. The same fields as the
3 * destination (`actions/destination.ts`), validated the same way and sealed
4 * as forge-level secrets under their own names; nothing is read back to a
5 * person except which are set. A blank field keeps what was sealed.
6 */
7
8 import type { App } from "../app.ts";
9 import { putSecret } from "./secrets.ts";
10 import { invalidField, type DestinationInput } from "./destination.ts";
11 import {
12 clearMirrorSealed,
13 mirrorSealedSet,
14 MIRROR_SEALED,
15 recordMirror,
16 } from "../snapshot/mirror.ts";
17 import { snapshotsOf } from "../snapshot/service.ts";
18 import type { User } from "../core/users.ts";
19 import { err, ok, type Result } from "../kernel/result.ts";
20
21 const isAdmin = (actor: User | undefined): boolean => actor?.isAdmin === true;
22
23 function forgetState(app: App): void {
24 try {
25 recordMirror(app.config.dataDir, undefined);
26 } catch {
27 // The next pass overwrites it.
28 }
29 }
30
31 /** Seal what was given (an error is `not_found` or a sentence for the page); says what is still missing. */
32 export function saveMirror(
33 app: App,
34 actor: User | undefined,
35 input: DestinationInput,
36 ): Result<string, string> {
37 if (!isAdmin(actor)) return err("not_found");
38 const bad = invalidField(input);
39 if (bad) return err(bad);
40 const fields: [string, string][] = [
41 [MIRROR_SEALED.endpoint, input.endpoint],
42 [MIRROR_SEALED.bucket, input.bucket],
43 [MIRROR_SEALED.region, input.region],
44 [MIRROR_SEALED.accessKey, input.accessKey],
45 [MIRROR_SEALED.secretKey, input.secretKey],
46 ];
47 for (const [name, value] of fields) {
48 if (value === "") continue;
49 if (!putSecret(app, actor, "forge", "", name, value).ok)
50 return err("That could not be sealed.");
51 }
52 // A new target is not judged by the old one's last pass.
53 const svc = snapshotsOf(app);
54 svc?.forget();
55 if (fields.some(([, v]) => v !== "")) {
56 forgetState(app);
57 void svc?.mirrorFresh().catch(() => {});
58 }
59 const have = mirrorSealedSet(app.db);
60 const missing = [
61 [MIRROR_SEALED.endpoint, "endpoint"],
62 [MIRROR_SEALED.bucket, "bucket"],
63 [MIRROR_SEALED.accessKey, "access key"],
64 [MIRROR_SEALED.secretKey, "secret key"],
65 ]
66 .filter(([n]) => !have.includes(n!))
67 .map(([, label]) => label);
68 return ok(
69 missing.length === 0
70 ? "Mirror sealed; it is brought level within the hour."
71 : `Sealed, but not in effect until the ${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} set.`,
72 );
73 }
74
sentinelcrew outdated

If a pass against the old mirror is in flight, this joins it instead of clearing. The old pass then records a failing state after the clear, and the board line stays until the next hourly pass. Clear the state file here directly, or start a fresh pass once the in-flight one ends.
75 export function forgetMirror(
76 app: App,
77 actor: User | undefined,
78 ): Result<void, "not_found"> {
79 if (!isAdmin(actor)) return err("not_found");
80 clearMirrorSealed(app.db);
81 app.bus.append({
82 kind: "secret.deleted",
83 actorId: actor!.id,
84 payload: { name: "SNAPSHOT_MIRROR_S3_*", scope: "forge" },
85 });
86 const svc = snapshotsOf(app);
87 svc?.forget();
88 forgetState(app);
89 // A pass against the old mirror may still be running and would write its
90 // failure after this: a fresh pass starts when it ends, and with no mirror
91 // clears the state the board reads — no mirror, no line.
92 void svc?.mirrorFresh().catch(() => {});
93 return ok();
94 }
src/crew/notify.ts +15
@@ -30,6 +30,7 @@ import { getUserSetting, setUserSetting } from "../core/settings.ts";
3030 import { getUserById, type User } from "../core/users.ts";
3131 import { landedLines } from "../core/watches.ts";
3232 import { err, ok, type Result } from "../kernel/result.ts";
33 import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
3334 import { drillNeed, firstSeen, lastVerdict } from "../snapshot/verdict.ts";
3435 import { askLinesFor } from "../web/studio/asks.ts";
3536
@@ -167,6 +168,19 @@ export function drillLines(
167168 return [{ id, line: need.sentence, items: [id] }];
168169 }
169170
171 /** The admin hears of a snapshot that did not land off the box, or a stale mirror, once per cause. */
172 export function mirrorLines(
173 app: App,
174 user: User,
175 now = Date.now(),
176 ): { id: string; line: string; items: readonly string[] }[] {
177 if (!user.isAdmin || user.kind !== "user") return [];
178 const need = mirrorNeed(lastMirror(app.config.dataDir), now);
179 if (need === undefined) return [];
180 const id = `mirror:${need.key}`;
181 return [{ id, line: need.sentence, items: [id] }];
182 }
183
170184 export interface Notifier {
171185 /** Resolves when the work queued so far is done. For tests and shutdown. */
172186 settled(): Promise<void>;
@@ -222,6 +236,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
222236 ...landedLines(app.db, user, now()),
223237 ...mailCeilingLines(app, user, now()),
224238 ...drillLines(app, user, now()),
239 ...mirrorLines(app, user, now()),
225240 ];
226241 const before = seen.get(userId);
227242 // What is gone may ask again later, and then it is new again. Seen by
src/http/server.ts +6
@@ -113,6 +113,9 @@ import {
113113 backupsClear,
114114 backupsDestination,
115115 backupsDrill,
116 backupsMirror,
117 backupsMirrorClear,
118 backupsMirrorNow,
116119 backupsNow,
117120 backupsPage,
118121 backupsTest,
@@ -727,6 +730,9 @@ export function buildRouter(): Router<Handler> {
727730 router.add("POST", "/settings/backups/destination", backupsDestination);
728731 router.add("POST", "/settings/backups/test", backupsTest);
729732 router.add("POST", "/settings/backups/clear", backupsClear);
733 router.add("POST", "/settings/backups/mirror", backupsMirror);
734 router.add("POST", "/settings/backups/mirror/clear", backupsMirrorClear);
735 router.add("POST", "/settings/backups/mirror/now", backupsMirrorNow);
730736 router.add("GET", "/settings/accounts", accountsPage);
731737 router.add("GET", "/settings/accounts/:username", personSignInsPage);
732738 router.add(
src/kernel/vitals.ts +5
@@ -21,6 +21,7 @@ import {
2121 } from "../core/runs.ts";
2222 import { STALE_AFTER_MS } from "../runs/lease.ts";
2323 import { BUDGETS } from "./budgets.ts";
24 import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
2425 import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts";
2526 import { heldChecks, redSentence } from "../checks/red_main.ts";
2627
@@ -318,6 +319,10 @@ export function vitals(input: VitalsInput): Vitals {
318319 if (owed !== undefined)
319320 need("drill", owed.sentence, owed.since, "/settings/backups");
320321
322 // A snapshot that did not land off the box, or a mirror a day behind, is a card until it does.
323 const behind = mirrorNeed(lastMirror(input.dataDir), now);
324 if (behind !== undefined) need("mirror", behind.sentence, behind.since, "/settings/backups");
325
321326 const apps = { serving: 0, asleep: 0, previews: 0 };
322327 for (const e of input.envs ?? []) {
323328 if (e.env !== "production") apps.previews++;
src/main.ts +3
@@ -31,6 +31,7 @@ import { startLooks } from "./looks/service.ts";
3131 import { startProbes } from "./contract/probes.ts";
3232 import { startSnapshots } from "./snapshot/service.ts";
3333 import { resolveDestination } from "./snapshot/destination.ts";
34 import { resolveMirror } from "./snapshot/mirror.ts";
3435 import { releaseOrphans } from "./core/run_secret.ts";
3536 import { failAbandonedRuns } from "./core/runs.ts";
3637 import { DRAIN_LINGER_MAX_S, drainLingerMs, startDraining } from "./http/health.ts";
@@ -327,6 +328,8 @@ const snapshots = startSnapshots(
327328 })?.store,
328329 {
329330 version: running().version,
331 mirror: () =>
332 resolveMirror({ db: app.db, dataDir: app.config.dataDir, bus: app.bus })?.store,
330333 passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined,
331334 },
332335 );
src/snapshot/mirror.ts added +269
@@ -0,0 +1,269 @@
1 /**
2 * The off-box mirror: a second S3-compatible bucket (Backblaze B2, say) that
3 * every snapshot is copied to, so losing the machine does not lose the
4 * snapshots with it. Its keys are sealed in the forge like the Backups
5 * destination (`SNAPSHOT_MIRROR_S3_*`), or given as `FORGE_MIRROR_S3_*`.
6 *
7 * A pass copies each of the newest snapshots the mirror lacks, the manifest
8 * last, so a half-copied snapshot is never listed; objects already there are
9 * skipped, so a pass that died resumes. What the last pass found is kept in
10 * one small file under the data directory (`mirror/state.json`), so the
11 * board and the push read it without touching the network. Plain data in,
12 * plain data out.
13 */
14
15 import type { Database } from "bun:sqlite";
16 import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
17 import { join } from "node:path";
18 import type { EventBus } from "../core/events.ts";
19 import { deleteSecret, resolveSecrets } from "../core/secret_store.ts";
20 import { KEEP, listSnapshots, pruneSnapshots, SNAPSHOT_PREFIX } from "./snapshot.ts";
21 import { s3Store, type S3Target, type Store } from "./store.ts";
22
23 /** The sealed names, one per field of the form. */
24 export const MIRROR_SEALED = {
25 endpoint: "SNAPSHOT_MIRROR_S3_ENDPOINT",
26 bucket: "SNAPSHOT_MIRROR_S3_BUCKET",
27 region: "SNAPSHOT_MIRROR_S3_REGION",
28 accessKey: "SNAPSHOT_MIRROR_S3_ACCESS_KEY",
29 secretKey: "SNAPSHOT_MIRROR_S3_SECRET_KEY",
30 } as const;
31 export const MIRROR_NAMES: readonly string[] = Object.values(MIRROR_SEALED);
32
33 /** A mirror not confirmed in sync for this long is one line on the board. */
34 export const MIRROR_STALE_MS = 24 * 3_600_000;
35 /** How often the forge looks for snapshots the mirror lacks. */
36 export const MIRROR_EVERY_MS = 3_600_000;
37 /** Objects copied at once: each is held in memory while it moves. */
38 const COPY_AT_ONCE = 4;
39
40 export interface MirrorSources {
41 readonly db?: Database | undefined;
42 readonly dataDir?: string | undefined;
43 readonly bus?: EventBus | undefined;
44 readonly env?: Record<string, string | undefined>;
45 }
46
47 export interface Mirror {
48 readonly source: "sealed" | "env";
49 readonly store: Store;
50 }
51
52 /** The mirror's target: sealed in this forge, else `FORGE_MIRROR_S3_*`; undefined when neither is whole. */
53 export function mirrorTarget(s: MirrorSources = {}): { source: Mirror["source"]; target: S3Target } | undefined {
54 if (s.db && s.dataDir !== undefined && s.bus) {
55 const v = resolveSecrets(s.db, s.dataDir, s.bus, MIRROR_NAMES, {});
56 const endpoint = v[MIRROR_SEALED.endpoint];
57 const bucket = v[MIRROR_SEALED.bucket];
58 const accessKeyId = v[MIRROR_SEALED.accessKey];
59 const secretAccessKey = v[MIRROR_SEALED.secretKey];
60 if (endpoint && bucket && accessKeyId && secretAccessKey)
61 return {
62 source: "sealed",
63 target: { endpoint, bucket, accessKeyId, secretAccessKey, region: v[MIRROR_SEALED.region] },
64 };
65 }
66 const env = s.env ?? process.env;
67 const endpoint = env["FORGE_MIRROR_S3_ENDPOINT"]?.trim();
68 const bucket = env["FORGE_MIRROR_S3_BUCKET"]?.trim();
69 const accessKeyId = env["FORGE_MIRROR_S3_ACCESS_KEY"]?.trim();
70 const secretAccessKey = env["FORGE_MIRROR_S3_SECRET_KEY"]?.trim();
71 if (!endpoint || !bucket || !accessKeyId || !secretAccessKey) return undefined;
72 return {
73 source: "env",
74 target: { endpoint, bucket, accessKeyId, secretAccessKey, region: env["FORGE_MIRROR_S3_REGION"]?.trim() || undefined },
75 };
76 }
77
78 export function resolveMirror(s: MirrorSources = {}): Mirror | undefined {
79 const t = mirrorTarget(s);
80 return t === undefined ? undefined : { source: t.source, store: s3Store(t.target) };
81 }
82
83 /** Which of the sealed mirror names are set, without opening any. */
84 export function mirrorSealedSet(db: Database): string[] {
85 return db
86 .query<{ name: string }, []>(
87 `SELECT name FROM secret WHERE scope = 'forge' AND scope_id = ''
88 AND name LIKE 'SNAPSHOT\\_MIRROR\\_S3\\_%' ESCAPE '\\'`,
89 )
90 .all()
91 .map((r) => r.name);
92 }
93
94 export function clearMirrorSealed(db: Database): void {
95 for (const name of MIRROR_NAMES) deleteSecret(db, "forge", "", name);
96 }
97
98 export interface MirrorState {
99 readonly where: string;
100 /** When the last pass began. */
101 readonly attemptAt: string;
102 /** Whether the last pass could list the mirror. */
103 readonly reachable: boolean;
104 /** The last time the mirror was confirmed to hold the newest snapshot. */
105 readonly syncedAt?: string;
106 /** The newest snapshot the mirror holds, and when it finished. */
107 readonly newest?: string;
108 readonly newestAt?: string;
109 /** Set while passes are failing: since when, which snapshot, and why. */
110 readonly failingSince?: string;
111 readonly failed?: string;
112 readonly error?: string;
113 /** Set (and nothing recorded) when the primary, not the mirror, could not be read. */
114 readonly sourceError?: string;
115 }
116
117 /** A failure on the primary side of a copy: the mirror is not to blame for it. */
118 class SourceError extends Error {}
119
120 /** The primary, with every failure marked as its own. */
121 function marked(src: Store): Store {
122 const mark = <A extends unknown[], R>(fn: (...a: A) => Promise<R>) => async (...a: A): Promise<R> => {
123 try {
124 return await fn(...a);
125 } catch (error) {
126 throw new SourceError(error instanceof Error ? error.message : String(error));
127 }
128 };
129 return { ...src, where: src.where, get: mark(src.get.bind(src)), list: mark(src.list.bind(src)), names: mark(src.names.bind(src)) };
130 }
131
132 const stateFile = (dataDir: string): string => join(dataDir, "mirror", "state.json");
133
134 export function recordMirror(dataDir: string, state: MirrorState | undefined): void {
135 if (state === undefined) {
136 rmSync(stateFile(dataDir), { force: true });
137 return;
138 }
139 mkdirSync(join(dataDir, "mirror"), { recursive: true });
140 const tmp = `${stateFile(dataDir)}.tmp`;
141 writeFileSync(tmp, JSON.stringify(state));
142 renameSync(tmp, stateFile(dataDir));
143 }
144
145 /** What the last pass found; undefined when no mirror has been tried or the file is not one. */
146 export function lastMirror(dataDir: string): MirrorState | undefined {
147 try {
148 const s = JSON.parse(readFileSync(stateFile(dataDir), "utf8")) as MirrorState;
149 if (typeof s.where !== "string" || typeof s.attemptAt !== "string" || typeof s.reachable !== "boolean")
150 return undefined;
151 return s;
152 } catch {
153 return undefined;
154 }
155 }
156
157 async function copyOne(src: Store, dst: Store, name: string): Promise<void> {
158 const prefix = `${SNAPSHOT_PREFIX}${name}/`;
159 const manifest = `${prefix}manifest.json`;
160 const have = new Set(await dst.list(prefix));
161 const todo = (await src.list(prefix)).filter((k) => k !== manifest && !have.has(k));
162 for (let i = 0; i < todo.length; i += COPY_AT_ONCE)
163 await Promise.all(
164 todo.slice(i, i + COPY_AT_ONCE).map(async (key) => {
165 const body = await src.get(key);
166 if (body === undefined) throw new Error(`${key} vanished from the source`);
167 await dst.put(key, body);
168 }),
169 );
170 const body = await src.get(manifest);
171 if (body === undefined) throw new Error(`${manifest} vanished from the source`);
172 await dst.put(manifest, body);
173 }
174
175 /**
176 * One pass: bring the mirror level with the source, prune it to `keep`, and
177 * keep what came of it. Never throws; a failure is the returned state.
178 */
sentinelcrew outdated

If the source is unreachable this throws while reachable is already true and failed is undefined, so the line at :234 says a snapshot "did not land on the off-box mirror" when the primary is the side that is down. Record which side failed, and either word the sentence for that side or leave a primary outage to the destination's own line.
179 export async function mirrorPass(
180 dataDir: string,
181 primary: Store,
182 dst: Store,
183 options: { now?: number; keep?: number } = {},
184 ): Promise<MirrorState> {
185 const now = options.now ?? Date.now();
186 const keep = options.keep ?? KEEP;
187 const at = new Date(now).toISOString();
188 const src = marked(primary);
189 const prev = lastMirror(dataDir);
190 let reachable = false;
191 let failed: string | undefined;
192 let state: MirrorState;
193 try {
194 const names = new Set(await dst.names(SNAPSHOT_PREFIX));
195 reachable = true;
196 const wanted = (await listSnapshots(src)).slice(0, keep);
197 for (const m of [...wanted].reverse()) {
198 if (names.has(m.name) && (await dst.get(`${SNAPSHOT_PREFIX}${m.name}/manifest.json`)) !== undefined)
199 continue;
200 failed = m.name;
201 await copyOne(src, dst, m.name);
202 failed = undefined;
203 }
204 await pruneSnapshots(dst, keep);
205 const newest = wanted[0];
206 state = {
207 where: dst.where,
208 attemptAt: at,
209 reachable: true,
210 syncedAt: at,
211 ...(newest === undefined ? {} : { newest: newest.name, newestAt: newest.finishedAt }),
212 };
213 } catch (error) {
214 const reason = error instanceof Error ? error.message : String(error);
215 if (error instanceof SourceError) {
216 // The primary is down: the destination has its own line, and nothing
217 // here says the mirror failed. What was recorded stands.
218 return { ...(prev ?? { where: dst.where, attemptAt: at, reachable }), sourceError: reason.slice(0, 200) };
219 }
220 state = {
221 where: dst.where,
222 attemptAt: at,
223 reachable,
224 ...(prev?.syncedAt === undefined ? {} : { syncedAt: prev.syncedAt }),
225 ...(prev?.newest === undefined ? {} : { newest: prev.newest }),
226 ...(prev?.newestAt === undefined ? {} : { newestAt: prev.newestAt }),
227 failingSince: prev?.failingSince ?? at,
228 ...(failed === undefined ? {} : { failed }),
229 error: reason.slice(0, 200),
230 };
231 }
232 try {
233 recordMirror(dataDir, state);
234 } catch {
235 // A disk that will not take the file must not turn a copied snapshot into a failure.
236 }
237 return state;
238 }
239
240 export interface MirrorNeed {
241 /** Holds as long as the same cause does: a new failure or a new silence is a new key. */
242 readonly key: string;
243 readonly sentence: string;
244 readonly since: string;
245 }
246
247 /**
248 * The one line the mirror owes the board: a pass that failed (the store
249 * unreachable, or a snapshot that did not land), or a mirror not confirmed
250 * in sync for a day. A mirror in sync: none.
251 */
252 export function mirrorNeed(s: MirrorState | undefined, now: number): MirrorNeed | undefined {
253 if (s === undefined) return undefined;
254 if (s.failingSince !== undefined)
255 return {
256 key: `fail:${s.failingSince}`,
257 sentence: s.reachable
258 ? `Snapshot ${s.failed ?? "(unknown)"} did not land on the off-box mirror ${s.where}: ${s.error ?? "unknown reason"}.`
259 : `The off-box mirror ${s.where} is unreachable: ${s.error ?? "unknown reason"}.`,
260 since: s.failingSince,
261 };
262 const at = Date.parse(s.syncedAt ?? "");
263 if (Number.isFinite(at) && now - at <= MIRROR_STALE_MS) return undefined;
264 return {
265 key: `stale:${s.syncedAt ?? s.attemptAt}`,
266 sentence: `The off-box mirror ${s.where} has not been confirmed in sync for over a day${s.syncedAt ? `; last on ${s.syncedAt.slice(0, 10)}` : ""}.`,
267 since: s.syncedAt ? new Date(at + MIRROR_STALE_MS).toISOString() : s.attemptAt,
268 };
269 }
src/snapshot/service.ts +76
@@ -21,6 +21,7 @@ import {
2121 } from "./snapshot.ts";
2222 import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
2323 import { recordDrill } from "./drills.ts";
24 import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts";
2425 import type { Store } from "./store.ts";
2526
2627 const log = createLog("snapshots");
@@ -43,6 +44,14 @@ export interface SnapshotsHandle {
4344 readonly last: () => { manifest?: Manifest; error?: string; at?: string };
4445 /** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
4546 readonly drill: () => Promise<DrillResult>;
47 /** The off-box mirror's store; undefined when the forge has none. */
48 readonly mirror: Store | undefined;
49 /** Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no mirror. */
50 readonly mirrorNow: () => Promise<MirrorState | undefined>;
51 /** Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it now is. */
52 readonly mirrorFresh: () => Promise<MirrorState | undefined>;
53 /** What the last mirror pass found. */
54 readonly mirrored: () => MirrorState | undefined;
4655 /** Forget the resolved store; the next use resolves again (a new destination was saved). */
4756 readonly forget: () => void;
4857 readonly stop: () => Promise<void>;
@@ -76,6 +85,10 @@ export function startSnapshots(
7685 destination: Store | (() => Store | undefined),
7786 options: {
7887 readonly version: string;
88 /** The off-box mirror, resolved like the destination. */
89 readonly mirror?: Store | (() => Store | undefined) | undefined;
90 /** Test seam: how often the mirror is looked at. */
91 readonly mirrorEveryMs?: number | undefined;
7992 readonly nightly?: boolean;
8093 readonly passphrase?: string | undefined;
8194 /** Test seam: the drill itself. */
@@ -92,6 +105,13 @@ export function startSnapshots(
92105 resolved ??= { store: destination() };
93106 return resolved.store;
94107 };
108 let resolvedMirror: { store: Store | undefined } | undefined;
109 const currentMirror = (): Store | undefined => {
110 const m = options.mirror;
111 if (typeof m !== "function") return m;
112 resolvedMirror ??= { store: m() };
113 return resolvedMirror.store;
114 };
95115 let stopped = false;
96116 let inFlight: Promise<Manifest> | undefined;
97117 let running: Progress | undefined;
@@ -123,6 +143,9 @@ export function startSnapshots(
123143 await pruneSnapshots(store, KEEP);
124144 return manifest;
125145 });
146 // The mirror hears of every snapshot at once; a failure there is its own
147 // line on the board and never the snapshot's.
148 run.then(() => mirrorNow()).catch(() => {});
126149 inFlight = run.then(
127150 (m) => {
128151 last = { manifest: m, at: m.finishedAt };
@@ -137,6 +160,37 @@ export function startSnapshots(
137160 return inFlight;
138161 };
139162
163 // Single-flight: two passes at once would copy the same objects twice.
164 let mirroring: Promise<MirrorState | undefined> | undefined;
165 const mirrorNow = (): Promise<MirrorState | undefined> => {
166 mirroring ??= (async () => {
167 await (inFlight ?? Promise.resolve()).catch(() => {});
168 const dst = currentMirror();
169 const src = current();
170 if (dst === undefined || src === undefined) {
171 try {
172 recordMirror(app.config.dataDir, undefined);
173 } catch (error) {
174 log.warn("mirror: state not cleared", { error: String(error) });
175 }
176 return undefined;
177 }
178 const state = await mirrorPass(app.config.dataDir, src, dst);
179 if (state.sourceError !== undefined) log.error("mirror: primary unreadable", { error: state.sourceError });
180 else if (state.failingSince === undefined) log.info("mirror level", { where: state.where, newest: state.newest ?? "none" });
181 else log.error("mirror failed", { where: state.where, reachable: state.reachable, error: state.error ?? "" });
182 return state;
183 })().finally(() => {
184 mirroring = undefined;
185 });
186 return mirroring;
187 };
188
189 const mirrorFresh = async (): Promise<MirrorState | undefined> => {
190 while (mirroring !== undefined) await mirroring.catch(() => {});
191 return mirrorNow();
192 };
193
140194 // Single-flight: a restore of every repository is heavy, so a second call
141195 // while one runs joins it instead of filling the disk with another.
142196 let drilling: Promise<DrillResult> | undefined;
@@ -192,6 +246,19 @@ export function startSnapshots(
192246 };
193247 scheduleDrill();
194248
249 // Within the hour: look for snapshots the mirror lacks, soon after boot and then hourly.
250 let mirrorTimer: ReturnType<typeof setTimeout> | undefined;
251 const everyMs = options.mirrorEveryMs ?? MIRROR_EVERY_MS;
252 const scheduleMirror = (first: boolean): void => {
253 if (options.nightly === false || stopped) return;
254 mirrorTimer = setTimeout(() => {
255 mirrorNow()
256 .catch(() => {})
257 .finally(() => scheduleMirror(false));
258 }, first ? Math.min(30_000, everyMs) : everyMs);
259 };
260 scheduleMirror(true);
261
195262 const handle: SnapshotsHandle = {
196263 get store() {
197264 return current();
@@ -205,13 +272,22 @@ export function startSnapshots(
205272 },
206273 last: () => last,
207274 progress: () => running,
275 get mirror() {
276 return currentMirror();
277 },
278 mirrorNow,
279 mirrorFresh,
280 mirrored: () => lastMirror(app.config.dataDir),
208281 forget: () => {
209282 resolved = undefined;
283 resolvedMirror = undefined;
210284 },
211285 stop: async () => {
212286 stopped = true;
213287 if (timer !== undefined) clearTimeout(timer);
214288 if (drillTimer !== undefined) clearTimeout(drillTimer);
289 if (mirrorTimer !== undefined) clearTimeout(mirrorTimer);
290 if (mirroring !== undefined) await mirroring.catch(() => {});
215291 if (inFlight !== undefined) await inFlight.catch(() => {});
216292 if (handles.get(app) === handle) handles.delete(app);
217293 },
src/web/pages/backups.ts +105 −1
@@ -14,6 +14,8 @@ import {
1414 testDestination,
1515 type DestinationInput,
1616 } from "../../actions/destination.ts";
17 import { forgetMirror, saveMirror } from "../../actions/mirror.ts";
18 import { mirrorSealedSet, mirrorNeed, MIRROR_SEALED } from "../../snapshot/mirror.ts";
1719 import { resolveDestination, sealedSet } from "../../snapshot/destination.ts";
1820 import { keyOf } from "../../snapshot/snapshot.ts";
1921 import { lastDrill } from "../../snapshot/drills.ts";
@@ -115,7 +117,7 @@ export async function backupsPage(ctx: Ctx): Promise<Response> {
115117 const denied = requireAdmin(ctx);
116118 if (denied) return denied;
117119 const svc = snapshotsOf(ctx.app);
118 const form = destinationCard(ctx);
120 const form = html`${destinationCard(ctx)}${mirrorCard(ctx)}`;
119121 if (svc === undefined || svc.store === undefined)
120122 return shell(
121123 ctx,
@@ -314,6 +316,61 @@ function destinationCard(ctx: Ctx): Html {
314316 );
315317 }
316318
319 /** The mirror: its age and reachability from the last pass, and the write-only form that seals it. */
320 function mirrorCard(ctx: Ctx): Html {
321 const svc = snapshotsOf(ctx.app);
322 const state = svc?.mirrored();
323 const configured = svc?.mirror !== undefined;
324 const need = mirrorNeed(state, Date.now());
325 const set = mirrorSealedSet(ctx.app.db);
326 const field_ = (name: string, label: string, sealed: string, type = "text"): Html =>
327 html`<label class="field"
328 ><span>${label}</span
329 ><input
330 name="${name}"
331 type="${type}"
332 autocomplete="off"
333 spellcheck="false"
334 placeholder="${set.includes(sealed) ? "set — leave blank to keep" : ""}"
335 /></label>`;
336 const status = !configured
337 ? html`<p>No off-box mirror: snapshots live only where the destination is.
338 Seal a second bucket below (or set <code>FORGE_MIRROR_S3_*</code>) and
339 every snapshot is copied there within the hour.</p>`
340 : state === undefined
341 ? html`<p>Mirroring to <code>${svc!.mirror!.where}</code>; the first pass has not run yet.</p>`
342 : html`<p class="${need ? "backups-error" : ""}">
343 ${icon(need ? "alert" : "check")} <code>${state.where}</code> is
344 ${state.reachable ? "reachable" : "unreachable"} (checked
345 ${time(state.attemptAt)}).
346 ${state.newest
347 ? html`It holds <code>${state.newest}</code>, taken
348 ${time(state.newestAt ?? "")}; confirmed level ${time(state.syncedAt ?? "")}.`
349 : html`It holds no snapshot yet.`}
350 ${need ? html`<br />${need.sentence}` : ""}
351 </p>`;
352 return card(
353 html`${status}
354 <form method="post" action="/settings/backups/mirror" class="stack">
355 ${field_("endpoint", "Endpoint", MIRROR_SEALED.endpoint)}
sentinelcrew outdated

Low: use MIRROR_SEALED.endpoint and the others in place of these literals, so the placeholder check cannot drift from the names saveMirror seals.
356 ${field_("bucket", "Bucket", MIRROR_SEALED.bucket)}
357 ${field_("region", "Region (optional)", MIRROR_SEALED.region)}
358 ${field_("access_key", "Access key", MIRROR_SEALED.accessKey)}
359 ${field_("secret_key", "Secret key", MIRROR_SEALED.secretKey, "password")}
360 <div class="row-actions">
361 ${btn("Seal mirror", { type: "submit", variant: "primary", icon: "lock" })}
362 <button class="btn" type="submit" formaction="/settings/backups/mirror/now" formnovalidate>
363 Mirror now
364 </button>
365 <button class="btn" type="submit" formaction="/settings/backups/mirror/clear" formnovalidate>
366 Clear sealed mirror
367 </button>
368 </div>
369 </form>`,
370 { head: html`${icon("download")}<h2>Off-box mirror</h2>` },
371 );
372 }
373
317374 function destinationInput(form: Form): DestinationInput {
318375 return {
319376 endpoint: field(form, "endpoint").trim(),
@@ -427,3 +484,50 @@ export async function backupsDrill(ctx: Ctx): Promise<Response> {
427484 });
428485 }
429486 }
487
488 /** POST /settings/backups/mirror — seal the mirror. */
489 export async function backupsMirror(ctx: Ctx): Promise<Response> {
sentinelcrew

This route, /mirror/clear and /mirror/now are new admin writes with no test. Cover them: a guest or non-admin is refused, a cross-origin post gets 403, a seal stores the names without echoing the values, a blank field keeps what was sealed, and clear removes the names and the board card.
490 const denied = requireAdmin(ctx);
491 if (denied) return denied;
492 const form = await readForm(ctx, "admin");
493 if (form instanceof Response) return form;
494 const saved = saveMirror(ctx.app, ctx.auth.user, destinationInput(form));
495 return flashTo(
496 ctx,
497 "/settings/backups",
498 saved.ok ? { message: saved.value } : { tone: "danger", message: saved.error },
499 );
500 }
501
502 /** POST /settings/backups/mirror/clear — forget the sealed mirror. */
503 export async function backupsMirrorClear(ctx: Ctx): Promise<Response> {
504 const denied = requireAdmin(ctx);
505 if (denied) return denied;
506 const form = await readForm(ctx, "admin");
507 if (form instanceof Response) return form;
508 forgetMirror(ctx.app, ctx.auth.user);
509 return flashTo(ctx, "/settings/backups", { message: "The sealed mirror is cleared." });
510 }
511
512 /** POST /settings/backups/mirror/now — bring the mirror level, and say what came of it. */
513 export async function backupsMirrorNow(ctx: Ctx): Promise<Response> {
514 const denied = requireAdmin(ctx);
515 if (denied) return denied;
516 const form = await readForm(ctx, "admin");
517 if (form instanceof Response) return form;
518 const state = await snapshotsOf(ctx.app)?.mirrorNow();
519 if (state === undefined)
520 return flashTo(ctx, "/settings/backups", { tone: "danger", message: "This forge has no mirror to bring level." });
521 if (state.sourceError !== undefined)
522 return flashTo(ctx, "/settings/backups", {
523 tone: "danger",
524 message: `The snapshots could not be read from the destination, so the mirror was not touched: ${state.sourceError}`,
525 });
526 return flashTo(
527 ctx,
528 "/settings/backups",
529 state.failingSince === undefined
530 ? { message: `The mirror is level${state.newest ? ` at ${state.newest}` : ""}.` }
531 : { tone: "danger", message: `The mirror is not level: ${state.error ?? "unknown reason"}` },
532 );
533 }
test/rehearsal.test.ts +1 −1
@@ -114,7 +114,7 @@ describe("the verdict", () => {
114114 describe("a real roll of two forge processes on one data directory", () => {
115115 const short = (lingerS: number): RehearsalArgs => ({
116116 ...parseArgs([], `${import.meta.dir}/..`),
117 seconds: 8,
117 seconds: 20,
118118 probeMs: 300,
119119 lingerS,
120120 concurrency: 2,
test/snapshot-mirror-page.test.ts added +142
@@ -0,0 +1,142 @@
1 /**
2 * The mirror's admin routes: a guest and a non-admin are refused and nothing
3 * is sealed; a cross-origin post is 403; a seal stores the five names and the
4 * page never shows a value; a blank field keeps what was sealed; clear removes
5 * the names and, once the pass settles, the board card.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdtempSync, rmSync } from "node:fs";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { serve, type Servers } from "../src/http/server.ts";
14 import { vitals } from "../src/kernel/vitals.ts";
15 import { lastMirror, mirrorSealedSet, MIRROR_NAMES, recordMirror, resolveMirror } from "../src/snapshot/mirror.ts";
16 import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
17 import { dirStore } from "../src/snapshot/store.ts";
18 import { quietLogs } from "./helpers/globals.ts";
19
20 quietLogs();
21
22 const ENDPOINT = "http://127.0.0.1:1";
23 const SECRET = "s3cr3t-mirror-xyz";
24 const ACCESS = "AKIAMIRROREXAMPLE";
25 const full = { endpoint: ENDPOINT, bucket: "off-box", region: "eu-1", access_key: ACCESS, secret_key: SECRET };
26
27 describe("the mirror's admin routes", () => {
28 let root = "";
29 let app: App;
30 let svc: SnapshotsHandle;
31 let servers: Servers;
32
33 const login = async (username: string, password: string) => {
34 const r = await fetch(`${servers.url}/signin`, {
35 method: "POST",
36 headers: { origin: servers.url },
37 body: new URLSearchParams({ username, password }),
38 redirect: "manual",
39 });
40 return (r.headers.get("set-cookie") ?? "").split(";")[0]!;
41 };
42 const post = (path: string, body: Record<string, string>, cookie = "", origin = servers.url) =>
43 fetch(`${servers.url}/settings/backups${path}`, {
44 method: "POST",
45 headers: { cookie, origin },
46 body: new URLSearchParams(body),
47 redirect: "manual",
48 });
49 const page = async (cookie: string) =>
50 (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text();
51 const card = () =>
52 vitals({ db: app.db, dataDir: app.config.dataDir, signals: app.signals, now: Date.now() }).degraded.filter(
53 (d) => d.cause === "mirror",
54 );
55 const sealed = () => mirrorSealedSet(app.db).sort();
56
57 beforeAll(async () => {
58 root = mkdtempSync(join(tmpdir(), "forge-mirrorpage-"));
59 app = createApp({ dataDir: root, port: 0 });
60 for (const [username, password, adminIfFirst] of [
61 ["alice", "a-good-passphrase", true],
62 ["bob", "another-good-passphrase", false],
63 ] as const) {
64 const u = await createUser(app.db, {
65 trusted: true,
66 username,
67 email: `${username}@forge.local`,
68 password,
69 adminIfFirst,
70 });
71 if (!u.ok) throw new Error(u.error);
72 }
73 svc = startSnapshots(app, dirStore(join(root, "store")), {
74 version: "t",
75 nightly: false,
76 // A directory stands in for the bucket whenever the names are sealed: no network.
77 mirror: () =>
78 resolveMirror({ db: app.db, dataDir: root, bus: app.bus, env: {} }) === undefined
79 ? undefined
80 : dirStore(join(root, "offbox")),
81 });
82 servers = await serve(app);
83 });
84 afterAll(async () => {
85 servers.stop();
86 await svc.stop();
87 app.close();
88 rmSync(root, { recursive: true, force: true });
89 });
90
91 test("a guest and a non-admin are refused, and nothing is sealed", async () => {
92 const bob = await login("bob", "another-good-passphrase");
93 for (const path of ["/mirror", "/mirror/clear", "/mirror/now"]) {
94 const guest = await post(path, full);
95 expect(guest.status).toBe(303);
96 expect(guest.headers.get("location")).toContain("/signin");
97 expect((await post(path, full, bob)).status).toBe(404);
98 }
99 expect(sealed()).toEqual([]);
100 });
101
102 test("a cross-origin post is refused and seals nothing", async () => {
103 const alice = await login("alice", "a-good-passphrase");
104 for (const path of ["/mirror", "/mirror/clear", "/mirror/now"])
105 expect((await post(path, full, alice, "https://evil.example")).status).toBe(403);
106 expect(sealed()).toEqual([]);
107 });
108
109 test("an admin seals five names, and the page never shows a value", async () => {
110 const alice = await login("alice", "a-good-passphrase");
111 expect((await post("/mirror", full, alice)).status).toBe(303);
112 expect(sealed()).toEqual([...MIRROR_NAMES].sort());
113 const html = await page(alice);
114 expect(html).toContain("set — leave blank to keep");
115 for (const v of [SECRET, ACCESS, "off-box", "eu-1"]) expect(html).not.toContain(`value="${v}"`);
116 expect(html).not.toContain(SECRET);
117 expect(html).not.toContain(ACCESS);
118 });
119
120 test("a blank field keeps what was sealed, and a changed target drops the old pass", async () => {
121 const alice = await login("alice", "a-good-passphrase");
122 await svc.mirrorFresh();
123 recordMirror(root, { where: "old/bucket", attemptAt: new Date().toISOString(), reachable: true, syncedAt: new Date().toISOString() });
124 expect((await post("/mirror", { ...full, endpoint: "", bucket: "other-bucket", access_key: "", secret_key: "" }, alice)).status).toBe(303);
125 expect(sealed()).toEqual([...MIRROR_NAMES].sort());
126 await svc.mirrorFresh();
127 expect(lastMirror(root)?.where).not.toBe("old/bucket");
128 expect(lastMirror(root)?.reachable).toBe(true);
129 });
130
131 test("clear removes the names, and once the pass settles the card is gone", async () => {
132 const alice = await login("alice", "a-good-passphrase");
133 recordMirror(root, { where: "x", attemptAt: new Date().toISOString(), reachable: false, failingSince: new Date().toISOString(), error: "refused" });
134 expect(card()).toHaveLength(1);
135 expect((await post("/mirror/clear", {}, alice)).status).toBe(303);
136 expect(sealed()).toEqual([]);
137 await svc.mirrorFresh();
138 expect(card()).toHaveLength(0);
139 expect(lastMirror(root)).toBeUndefined();
140 expect(await page(alice)).not.toContain("set — leave blank to keep");
141 });
142 });
test/snapshot-mirror.test.ts added +221
@@ -0,0 +1,221 @@
1 /**
2 * The off-box mirror, against a stand-in S3 on a loopback port: a snapshot
3 * lands in both buckets; a mirror that cannot be reached is one line on the
4 * board and one push; so is a mirror a day out of date; and a mirror
5 * brought back catches up, resumes, and clears the line.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdtempSync } from "node:fs";
9 import { rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { createUser } from "../src/core/users.ts";
14 import { setNotifyUrl, startNotifier } from "../src/crew/notify.ts";
15 import { vitals } from "../src/kernel/vitals.ts";
16 import { listSnapshots, SNAPSHOT_PREFIX } from "../src/snapshot/snapshot.ts";
17 import {
18 lastMirror,
19 MIRROR_STALE_MS,
20 mirrorNeed,
21 mirrorPass,
22 recordMirror,
23 resolveMirror,
24 } from "../src/snapshot/mirror.ts";
25 import { startSnapshots } from "../src/snapshot/service.ts";
26 import { dirStore, type Store } from "../src/snapshot/store.ts";
27
28 /** A directory as a bucket that can be switched off: while `down`, every call fails like an unreachable host. */
29 function bucket(dir: string): { store: () => Store; down: boolean } {
30 const inner = dirStore(dir);
31 const me = { down: false, store: (): Store => flaky };
32 const guard = <A extends unknown[], R>(fn: (...a: A) => Promise<R>) => async (...a: A): Promise<R> => {
33 if (me.down) throw new Error("connection refused");
34 return fn(...a);
35 };
36 const flaky: Store = {
37 where: inner.where,
38 put: guard(inner.put),
39 get: guard(inner.get),
40 list: guard(inner.list),
41 names: guard(inner.names),
42 delete: guard(inner.delete),
43 };
44 return me;
45 }
46
47 const T0 = Date.parse("2026-10-01T00:00:00Z");
48
49 describe("the off-box mirror", () => {
50 let root: string;
51 let app: App;
52 let admin: { id: string };
53 let primary: ReturnType<typeof bucket>;
54 let offbox: ReturnType<typeof bucket>;
55 const src = () => primary.store();
56 const dst = () => offbox.store();
57 const board = (now: number) =>
58 vitals({ db: app.db, dataDir: app.config.dataDir, signals: app.signals, now }).degraded.filter(
59 (d) => d.cause === "mirror",
60 );
61
62 beforeAll(async () => {
63 root = mkdtempSync(join(tmpdir(), "forge-mirror-"));
64 primary = bucket(join(root, "primary-dir"));
65 offbox = bucket(join(root, "offbox-dir"));
66 app = createApp({ dataDir: root, port: 0 });
67 const u = await createUser(app.db, {
68 trusted: true,
69 username: "alice",
70 email: "[email protected]",
71 password: "a-good-passphrase",
72 adminIfFirst: true,
73 });
74 if (!u.ok) throw new Error(u.error);
75 admin = u.value;
76 });
77 afterAll(async () => {
78 await rm(root, { recursive: true, force: true });
79 });
80
81 test("a snapshot lands in both buckets, and the page's facts say so", async () => {
82 const svc = startSnapshots(app, src(), { version: "t", nightly: false, mirror: dst() });
83 const m = await svc.now();
84 const state = await svc.mirrorNow();
85 expect(state?.reachable).toBe(true);
86 expect(state?.failingSince).toBeUndefined();
87 const there = await listSnapshots(dst());
88 expect(there.map((x) => x.name)).toEqual([m.name]);
89 // Every file the manifest names is in the off-box bucket, byte for byte.
90 for (const f of m.files) {
91 const a = await src().get(`${SNAPSHOT_PREFIX}${m.name}/${f.path}`);
92 const b = await dst().get(`${SNAPSHOT_PREFIX}${m.name}/${f.path}`);
93 expect(a).toBeDefined();
94 expect(Buffer.from(b!).equals(Buffer.from(a!))).toBe(true);
95 }
96 expect(svc.mirrored()?.newest).toBe(m.name);
97 expect(board(Date.now())).toHaveLength(0);
98 await svc.stop();
99 });
100
101 test("an unreachable mirror is one line on the board and one push", async () => {
102 const sent: string[] = [];
103 setNotifyUrl(app.db, admin.id, "https://ntfy.example.com/topic");
104 let clock = Date.now();
105 // The notifier's first look is its baseline: it starts on a healthy mirror.
106 recordMirror(root, { where: "x", attemptAt: new Date(clock).toISOString(), reachable: true, syncedAt: new Date(clock).toISOString() });
107 const n = startNotifier(app, {
108 forgeUrl: "https://forge.example.com",
109 now: () => clock,
110 send: async (_u, body) => (sent.push(body), true),
111 clockMs: 20,
112 });
113 await n.settled();
114 offbox.down = true;
115 const svc = startSnapshots(app, src(), { version: "t", nightly: false, mirror: dst() });
116 await svc.now();
117 const state = await svc.mirrorNow();
118 expect(state?.reachable).toBe(false);
119 const line = board(Date.now());
120 expect(line).toHaveLength(1);
121 expect(line[0]!.sentence).toContain("unreachable");
122 expect(line[0]!.href).toBe("/settings/backups");
123 // A second failing pass is the same cause, not a second line.
124 const first = lastMirror(root)!.failingSince;
125 await svc.mirrorNow();
126 expect(lastMirror(root)!.failingSince).toBe(first);
127 expect(board(Date.now())).toHaveLength(1);
128
129 // The clock wakes the notifier a few times: one push, however many ticks.
130 for (let i = 0; i < 4; i++) {
131 clock += 60_000;
132 await Bun.sleep(40);
133 await n.settled();
134 }
135 n.stop();
136 expect(sent).toHaveLength(1);
137 expect(sent[0]).toContain("off-box mirror");
138 expect(sent[0]).toContain("unreachable");
139 await svc.stop();
140 });
141
142 test("a mirror that comes back catches up and the line goes", async () => {
143 offbox.down = false;
144 const svc = startSnapshots(app, src(), { version: "t", nightly: false, mirror: dst() });
145 const state = await svc.mirrorNow();
146 expect(state?.failingSince).toBeUndefined();
147 expect((await listSnapshots(dst())).length).toBe((await listSnapshots(src())).length);
148 expect(board(Date.now())).toHaveLength(0);
149 await svc.stop();
150 });
151
152 test("a snapshot that did not land says which, and a half-copied one is never listed", async () => {
153 const s = src();
154 const m = (await listSnapshots(s))[0]!;
155 const broken = dirStore(join(root, "dst-dir"));
156 let puts = 0;
157 const flaky = {
158 ...broken,
159 put: async (k: string, b: Uint8Array | string) => {
160 if (++puts > 1) throw new Error("quota exceeded");
161 await broken.put(k, b);
162 },
163 };
164 const state = await mirrorPass(root, s, flaky, { now: Date.now() });
165 expect(state.reachable).toBe(true);
166 expect(state.failed).toBeDefined();
167 expect(mirrorNeed(state, Date.now())?.sentence).toContain("did not land");
168 expect(await listSnapshots(broken)).toEqual([]);
169 expect(await broken.get(`${SNAPSHOT_PREFIX}${m.name}/manifest.json`)).toBeUndefined();
170 // Reset to the good mirror for what follows.
171 await mirrorPass(root, s, dst());
172 });
173
174 test("a primary that is down is not blamed on the mirror: no failure recorded, no line", async () => {
175 recordMirror(root, undefined);
176 primary.down = true;
177 const state = await mirrorPass(root, src(), dst());
178 primary.down = false;
179 expect(state.sourceError).toContain("connection refused");
180 expect(state.failingSince).toBeUndefined();
181 expect(lastMirror(root)).toBeUndefined();
182 expect(board(Date.now())).toHaveLength(0);
183 // With a mirror already level, the primary going down leaves its state alone.
184 const level = await mirrorPass(root, src(), dst());
185 primary.down = true;
186 const again = await mirrorPass(root, src(), dst());
187 primary.down = false;
188 expect(again.sourceError).toBeDefined();
189 expect(lastMirror(root)).toEqual(level);
190 expect(board(Date.now())).toHaveLength(0);
191 });
192
193 test("a mirror not confirmed level for a day is a line, once, until it is level again", () => {
194 const synced = new Date(T0).toISOString();
195 const state = { where: "b2/offbox", attemptAt: synced, reachable: true, syncedAt: synced, newest: "n" };
196 expect(mirrorNeed(state, T0 + MIRROR_STALE_MS)).toBeUndefined();
197 const stale = mirrorNeed(state, T0 + MIRROR_STALE_MS + 1);
198 expect(stale?.sentence).toContain("over a day");
199 expect(stale?.key).toBe(`stale:${synced}`);
200 recordMirror(root, state);
201 expect(board(T0 + MIRROR_STALE_MS + 1)).toHaveLength(1);
202 expect(board(T0 + 1000)).toHaveLength(0);
203 recordMirror(root, undefined);
204 expect(board(T0 + 5 * MIRROR_STALE_MS)).toHaveLength(0);
205 });
206
207 test("the mirror comes from the environment when nothing is sealed, and from nowhere otherwise", () => {
208 expect(resolveMirror({ env: {} })).toBeUndefined();
209 expect(resolveMirror({ env: { FORGE_MIRROR_S3_ENDPOINT: "http://x", FORGE_MIRROR_S3_BUCKET: "b" } })).toBeUndefined();
210 const m = resolveMirror({
211 env: {
212 FORGE_MIRROR_S3_ENDPOINT: "https://s3.b2.example.com",
213 FORGE_MIRROR_S3_BUCKET: "offbox",
214 FORGE_MIRROR_S3_ACCESS_KEY: "a",
215 FORGE_MIRROR_S3_SECRET_KEY: "s",
216 },
217 });
218 expect(m?.source).toBe("env");
219 expect(m?.store.where).toBe("s3.b2.example.com/offbox");
220 });
221 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs