Public: anyone can read this; the crew can change it.
Sentinel: backup was degraded 575 minutes today: Primary (s3.us-east-005.backblazeb2.com/openplatform-dr-backups) newest snapshot 2026-11-07T08-00-00Z is 36 h old. The … #10172
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10167-sentinel-backup-was-degraded-575-minutes → main 1 commit fork
2 files
changed +34
−2
src/snapshot/service.ts +15 −2
| @@ -102,6 +102,8 @@ const DRILL_HOUR_UTC = 9; | ||
| 102 | 102 | const CATCH_UP_MS = 5 * 60_000; |
| 103 | 103 | /** A snapshot due on boot is taken this long after it, once the forge is serving. */ |
| 104 | 104 | const SNAPSHOT_CATCH_UP_MS = 2 * 60_000; |
| 105 | /** A snapshot still owed is looked for again this often; one boot's look is not the last (#10167). */ | |
| 106 | const SNAPSHOT_CHECK_MS = 30 * 60_000; | |
| 105 | 107 | |
| 106 | 108 | export function untilDrill(now = Date.now()): number { |
| 107 | 109 | const next = new Date(now); |
| @@ -141,6 +143,7 @@ export function startSnapshots( | ||
| 141 | 143 | readonly bootRedFloorMs?: number | undefined; |
| 142 | 144 | /** Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read. */ |
| 143 | 145 | readonly snapshotCatchUpMs?: number | undefined; |
| 146 | readonly snapshotCheckMs?: number | undefined; | |
| 144 | 147 | readonly bucketEveryMs?: number | undefined; |
| 145 | 148 | readonly bucketFirstMs?: number | undefined; |
| 146 | 149 | /** Test seam: the clock `takeIfDue` judges by. */ |
| @@ -438,8 +441,18 @@ export function startSnapshots( | ||
| 438 | 441 | }; |
| 439 | 442 | scheduleMirror(true); |
| 440 | 443 | |
| 441 | const catchUpSnapshot = | |
| 442 | options.nightly === false ? undefined : setTimeout(() => void takeIfDue(), options.snapshotCatchUpMs ?? SNAPSHOT_CATCH_UP_MS); | |
| 444 | // Not once: a store unreadable at the boot's look, or a snapshot cut short | |
| 445 | // again, left the night's snapshot owed until the next night (#10167). | |
| 446 | let catchUpSnapshot: ReturnType<typeof setTimeout> | undefined; | |
| 447 | const scheduleCatchUp = (ms: number): void => { | |
| 448 | if (options.nightly === false || stopped) return; | |
| 449 | catchUpSnapshot = setTimeout(() => { | |
| 450 | takeIfDue() | |
| 451 | .catch(() => {}) | |
| 452 | .finally(() => scheduleCatchUp(options.snapshotCheckMs ?? SNAPSHOT_CHECK_MS)); | |
| 453 | }, ms); | |
| 454 | }; | |
| 455 | scheduleCatchUp(options.snapshotCatchUpMs ?? SNAPSHOT_CATCH_UP_MS); | |
| 443 | 456 | |
| 444 | 457 | // The bucket is read by this process, on its own clock, whatever any job says. |
| 445 | 458 | const bucket = watchBucket( |
test/backup-truth.test.ts +19
| @@ -203,6 +203,25 @@ test("a restart with a snapshot lost mid-run takes one shortly after boot", asyn | ||
| 203 | 203 | } |
| 204 | 204 | }, 30_000); |
| 205 | 205 | |
| 206 | test("a store unreadable at the boot's look is looked at again, and the owed snapshot taken", async () => { | |
| 207 | const dir = await mkdtemp(join(tmpdir(), "forge-again-")); | |
| 208 | const real = dirStore(dir); | |
| 209 | let broken = true; | |
| 210 | const store = { ...real, names: (p: string) => (broken ? Promise.reject(new Error("down")) : real.names(p)) }; | |
| 211 | const svc = startSnapshots(app, store, { version: "t", snapshotCatchUpMs: 20, snapshotCheckMs: 50, bucketEveryMs: 3_600_000, bucketFirstMs: 3_600_000 }); | |
| 212 | try { | |
| 213 | await Bun.sleep(300); | |
| 214 | expect((await listSnapshots(real)).length).toBe(0); | |
| 215 | broken = false; | |
| 216 | const until = Date.now() + 20_000; | |
| 217 | while ((await listSnapshots(real)).length === 0 && Date.now() < until) await Bun.sleep(50); | |
| 218 | expect((await listSnapshots(real)).length).toBe(1); | |
| 219 | } finally { | |
| 220 | await svc.stop(); | |
| 221 | await rm(dir, { recursive: true, force: true }); | |
| 222 | } | |
| 223 | }, 30_000); | |
| 224 | ||
| 206 | 225 | test("a boot with the night's snapshot in the store takes none; a day later it takes one", async () => { |
| 207 | 226 | const dir = await mkdtemp(join(tmpdir(), "forge-kept-")); |
| 208 | 227 | const store = dirStore(dir); |
new above
offline