Public: anyone can read this; the crew can change it.
Sentinel: backup was degraded 349 minutes today: Primary (s3.us-east-005.backblazeb2.com/openplatform-dr-backups) newest snapshot 2026-11-07T08-00-00Z is 36 h old. The … #10237
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10221-sentinel-backup-was-degraded-349-minutes → main 1 commit fork 2 threads · 2 unresolved
2 files
changed +26
−2
src/snapshot/service.ts +12 −2
| @@ -449,9 +449,16 @@ export function startSnapshots( | ||
| 449 | 449 | catchUpSnapshot = setTimeout(() => { |
| 450 | 450 | takeIfDue() |
| 451 | 451 | .catch(() => {}) |
| 452 | .finally(() => scheduleCatchUp(options.snapshotCheckMs ?? SNAPSHOT_CHECK_MS)); | |
| 452 | .finally(() => { | |
| 453 | bootLookAt = undefined; | |
| 454 | scheduleCatchUp(options.snapshotCheckMs ?? SNAPSHOT_CHECK_MS); | |
| 455 | }); | |
| 453 | 456 | }, ms); |
| 454 | 457 | }; |
| 458 | // Until the boot's look at the store is done, a snapshot may be about to | |
| 459 | // start; the keeper counts that as one running, so a swap does not land in | |
| 460 | // the gap between a boot and the catch-up and cut it short again (#9675). | |
| 461 | let bootLookAt: number | undefined = options.nightly === false ? undefined : Date.now(); | |
| 455 | 462 | scheduleCatchUp(options.snapshotCatchUpMs ?? SNAPSHOT_CATCH_UP_MS); |
| 456 | 463 | |
| 457 | 464 | // The bucket is read by this process, on its own clock, whatever any job says. |
| @@ -488,7 +495,10 @@ export function startSnapshots( | ||
| 488 | 495 | }, |
| 489 | 496 | last: () => last, |
| 490 | 497 | progress: () => running, |
| 491 | takingMs: () => (startedAt === undefined ? undefined : Date.now() - startedAt), | |
| 498 | takingMs: () => { | |
| 499 | const since = startedAt ?? bootLookAt; | |
| 500 | return since === undefined ? undefined : Date.now() - since; | |
| 501 | }, | |
| 492 | 502 | get mirror() { |
| 493 | 503 | return currentMirror(); |
| 494 | 504 | }, |
test/snapshot.test.ts +14
| @@ -611,4 +611,18 @@ describe("a snapshot's uploads", () => { | ||
| 611 | 611 | await svc.stop(); |
| 612 | 612 | } |
| 613 | 613 | }, 15_000); |
| 614 | ||
| 615 | test("until the boot's look at the store is done, the service counts as taking, so no release lands in that gap", async () => { | |
| 616 | const { store } = slowStore(0); | |
| 617 | const svc = startSnapshots(app, store, { version: "t", snapshotCatchUpMs: 30, snapshotCheckMs: 60_000 }); | |
| 618 | try { | |
| 619 | expect(svc.takingMs()).toBeGreaterThanOrEqual(0); | |
| 620 | await Bun.sleep(400); | |
Fixed sleeps make this timing-dependent under load. Poll takingMs() until undefined with a deadline instead. |
||
| 621 | await svc.now().catch(() => {}); | |
| 622 | await Bun.sleep(50); | |
| 623 | expect(svc.takingMs()).toBeUndefined(); | |
| 624 | } finally { | |
| 625 | await svc.stop(); | |
| 626 | } | |
| 627 | }, 15_000); | |
| 614 | 628 | }); |
new above
offline
bootLookAt is only cleared when the catch-up fires; if stop() runs first, takingMs() stays set. Clear it in stop().