Public: anyone can read this; the crew can change it.
Sentinel: backup was degraded 69 minutes today: Primary (s3.us-east-005.backblazeb2.com/openplatform-dr-backups (sealed)) holds no snapshot. The backup check last ran … #13737
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13732-sentinel-backup-was-degraded-69-minutes → main 6 commits fork 2 threads · 2 unresolved
4 files
changed +204
−34
src/snapshot/service.ts +49 −11
| @@ -70,7 +70,10 @@ export interface SnapshotsHandle { | ||
| 70 | 70 | readonly now: () => Promise<Manifest>; |
| 71 | 71 | /** How far the snapshot being taken is; undefined when none is. */ |
| 72 | 72 | readonly progress: () => Progress | undefined; |
| 73 | /** How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits on it (#9653). */ | |
| 73 | /** | |
| 74 | * How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits | |
| 75 | * on it (#9653). | |
| 76 | */ | |
| 74 | 77 | readonly takingMs: () => number | undefined; |
| 75 | 78 | /** The newest first, from the store. */ |
| 76 | 79 | readonly list: () => Promise<Manifest[]>; |
| @@ -85,13 +88,22 @@ export interface SnapshotsHandle { | ||
| 85 | 88 | readonly startDrill: () => { readonly started: boolean; readonly state: DrillState | { readonly id: string; readonly phase: "waiting" } }; |
| 86 | 89 | /** Where the running drill stands; undefined when none runs. */ |
| 87 | 90 | readonly drillState: () => DrillState | { readonly id: string; readonly phase: "waiting" } | undefined; |
| 88 | /** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */ | |
| 91 | /** | |
| 92 | * Drill if this week's slot has no drill yet, or the last one has stood red long enough; says | |
| 93 | * whether it did. | |
| 94 | */ | |
| 89 | 95 | readonly drillIfDue: () => Promise<boolean>; |
| 90 | 96 | /** The off-box mirror's store; undefined when the forge has none. */ |
| 91 | 97 | readonly mirror: Store | undefined; |
| 92 | /** Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no mirror. */ | |
| 98 | /** | |
| 99 | * Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no | |
| 100 | * mirror. | |
| 101 | */ | |
| 93 | 102 | readonly mirrorNow: () => Promise<MirrorState | undefined>; |
| 94 | /** Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it now is. */ | |
| 103 | /** | |
| 104 | * Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it | |
| 105 | * now is. | |
| 106 | */ | |
| 95 | 107 | readonly mirrorFresh: () => Promise<MirrorState | undefined>; |
| 96 | 108 | /** What the last mirror pass found. */ |
| 97 | 109 | readonly mirrored: () => MirrorState | undefined; |
| @@ -99,7 +111,9 @@ export interface SnapshotsHandle { | ||
| 99 | 111 | readonly forget: () => void; |
| 100 | 112 | /** Read the bucket now, as the watch does; resolves when the read is kept. */ |
| 101 | 113 | readonly checkBucket: () => Promise<void>; |
| 102 | /** Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests. */ | |
| 114 | /** | |
| 115 | * Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests. | |
| 116 | */ | |
| 103 | 117 | readonly takeIfDue: () => Promise<boolean>; |
| 104 | 118 | readonly stop: () => Promise<void>; |
| 105 | 119 | } |
| @@ -114,7 +128,9 @@ const DRILL_HOUR_UTC = 9; | ||
| 114 | 128 | const CATCH_UP_MS = 5 * 60_000; |
| 115 | 129 | /** A snapshot due on boot is taken this long after it, once the forge is serving. */ |
| 116 | 130 | const SNAPSHOT_CATCH_UP_MS = 2 * 60_000; |
| 117 | /** A snapshot still owed is looked for again this often; one boot's look is not the last (#10167). */ | |
| 131 | /** | |
| 132 | * A snapshot still owed is looked for again this often; one boot's look is not the last (#10167). | |
| 133 | */ | |
| 118 | 134 | const SNAPSHOT_CHECK_MS = 30 * 60_000; |
| 119 | 135 | /** A snapshot held for a crew run on one CPU looks again this often. */ |
| 120 | 136 | const HOLD_RETRY_MS = 30_000; |
| @@ -149,7 +165,10 @@ export function startSnapshots( | ||
| 149 | 165 | /** Test seam: how soon a failed snapshot is tried again. */ |
| 150 | 166 | readonly retryMs?: number | undefined; |
| 151 | 167 | readonly passphrase?: string | undefined; |
| 152 | /** Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it stands. */ | |
| 168 | /** | |
| 169 | * Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it | |
| 170 | * stands. | |
| 171 | */ | |
| 153 | 172 | readonly drillWaitMs?: number | undefined; |
| 154 | 173 | /** Test seam: the drill itself. */ |
| 155 | 174 | readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined; |
| @@ -157,14 +176,19 @@ export function startSnapshots( | ||
| 157 | 176 | readonly catchUpMs?: number | undefined; |
| 158 | 177 | readonly redRetryMs?: number | undefined; |
| 159 | 178 | readonly bootRedFloorMs?: number | undefined; |
| 160 | /** Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read. */ | |
| 179 | /** | |
| 180 | * Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read. | |
| 181 | */ | |
| 161 | 182 | readonly snapshotCatchUpMs?: number | undefined; |
| 162 | 183 | readonly snapshotCheckMs?: number | undefined; |
| 163 | 184 | readonly bucketEveryMs?: number | undefined; |
| 164 | 185 | readonly bucketFirstMs?: number | undefined; |
| 165 | 186 | /** Test seam: the clock `takeIfDue` judges by. */ |
| 166 | 187 | readonly now?: (() => number) | undefined; |
| 167 | /** Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks again, and how long it waits at most. */ | |
| 188 | /** | |
| 189 | * Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks | |
| 190 | * again, and how long it waits at most. | |
| 191 | */ | |
| 168 | 192 | readonly cpus?: (() => number) | undefined; |
| 169 | 193 | readonly activeCrewRuns?: (() => number) | undefined; |
| 170 | 194 | readonly holdRetryMs?: number | undefined; |
| @@ -329,7 +353,10 @@ export function startSnapshots( | ||
| 329 | 353 | log.warn("snapshot catch-up: store unreadable, not taking one now", { error: String(error) }); |
| 330 | 354 | return false; |
| 331 | 355 | } |
| 332 | if (stopped || active > 0 || !snapshotDue(newest?.finishedAt, (options.now ?? Date.now)()) || crewHolds()) return false; | |
| 356 | if (stopped || active > 0 || !snapshotDue(newest?.finishedAt, (options.now ?? Date.now)())) return false; | |
| 357 | // A store with no snapshot at all is a forge with no backup: that is not | |
| 358 | // deferred for a crew run (#13732); only a late one waits for the core. | |
| 359 | if (newest !== undefined && crewHolds()) return false; | |
| 333 | 360 | log.info("snapshot due or lost; taking one", { newest: newest?.name ?? "none" }); |
| 334 | 361 | await take().catch(() => {}); |
| 335 | 362 | return true; |
| @@ -475,6 +502,17 @@ export function startSnapshots( | ||
| 475 | 502 | } |
| 476 | 503 | }; |
| 477 | 504 | |
| 505 | /** Does the store hold any snapshot; an unreadable store is taken to, so the hold stays. */ | |
| 506 | const storeHoldsSnapshot = async (): Promise<boolean> => { | |
| 507 | const store = current(); | |
| 508 | if (store === undefined) return true; | |
| 509 | try { | |
| 510 | return (await listSnapshots(store)).length > 0; | |
| 511 | } catch { | |
| 512 | return true; | |
| 513 | } | |
| 514 | }; | |
| 515 | ||
| 478 | 516 | // One timer for the next snapshot: the nightly, or sooner after a failure. |
| 479 | 517 | // Every run that ends sets it again, whoever asked for the run. |
| 480 | 518 | let timer: ReturnType<typeof setTimeout> | undefined; |
| @@ -484,7 +522,7 @@ export function startSnapshots( | ||
| 484 | 522 | if (options.nightly === false || stopped) return; |
| 485 | 523 | timer = setTimeout(() => { |
| 486 | 524 | void during("snapshot", async () => { |
| 487 | if (active === 0 && crewHolds()) return arm(holdRetryMs); | |
| 525 | if (active === 0 && (await storeHoldsSnapshot()) && crewHolds()) return arm(holdRetryMs); | |
| 488 | 526 | await take(); |
| 489 | 527 | }, undefined, app).catch(() => {}); |
| 490 | 528 | }, ms); |
test/helpers/snapshot-seed.ts added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | /** | |
| 2 | * A store that already holds one snapshot, finished two days ago: the next is late, but the store | |
| 3 | * is not empty, so the one-core hold applies to it (#13732). The manifest is written under a fixed | |
| 4 | * name two days back, so a snapshot taken now is always a second one, never the seeded one again. | |
| 5 | */ | |
| 6 | import { SNAPSHOT_PREFIX, snapshotName, type Manifest } from "../../src/snapshot/snapshot.ts"; | |
| 7 | import type { Store } from "../../src/snapshot/store.ts"; | |
| 8 | ||
| 9 | const SEED_AGE_MS = 2 * 86_400_000; | |
| 10 | ||
| 11 | /** Seeds `store` and says the seeded snapshot's name. */ | |
| 12 | export async function seedSnapshot(store: Store, now: number = Date.now()): Promise<string> { | |
| 13 | const at = new Date(now - SEED_AGE_MS); | |
| 14 | const name = snapshotName(at); | |
| 15 | const manifest: Manifest = { | |
| 16 | format: 2, | |
| 17 | name, | |
| 18 | startedAt: at.toISOString(), | |
| 19 | finishedAt: at.toISOString(), | |
| 20 | version: "seed", | |
| 21 | repos: 0, | |
| 22 | apps: 0, | |
| 23 | files: [], | |
| 24 | bytes: 0, | |
| 25 | }; | |
| 26 | await store.put(`${SNAPSHOT_PREFIX}${name}/manifest.json`, JSON.stringify(manifest)); | |
| 27 | return name; | |
| 28 | } |
test/one-core-hold.test.ts +14 −4
| @@ -13,6 +13,7 @@ import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts | ||
| 13 | 13 | import { dirStore } from "../src/snapshot/store.ts"; |
| 14 | 14 | import { holdClock } from "../src/core/one_core.ts"; |
| 15 | 15 | import { quietLogs } from "./helpers/globals.ts"; |
| 16 | import { seedSnapshot } from "./helpers/snapshot-seed.ts"; | |
| 16 | 17 | |
| 17 | 18 | quietLogs(); |
| 18 | 19 | |
| @@ -162,23 +163,26 @@ function snapshotRuns(cpus: number, crew: () => number, extra: { holdMaxMs?: num | ||
| 162 | 163 | |
| 163 | 164 | test("1 CPU with a crew run holds a due snapshot, and takes it once the run is gone", async () => { |
| 164 | 165 | let crew = 1; |
| 166 | const seed = await seedSnapshot(dirStore(join(dir, "store"))); | |
| 165 | 167 | const s = snapshotRuns(1, () => crew); |
| 166 | 168 | expect(await s.takeIfDue()).toBe(false); |
| 167 | expect((await s.list()).length).toBe(0); | |
| 169 | expect((await s.list()).map((m) => m.name)).toEqual([seed]); | |
| 168 | 170 | crew = 0; |
| 169 | 171 | expect(await s.takeIfDue()).toBe(true); |
| 170 | expect(s.last().manifest).toBeDefined(); | |
| 172 | expect((await s.list()).length).toBe(2); | |
| 173 | expect(s.last().manifest?.name).not.toBe(seed); | |
| 171 | 174 | }); |
| 172 | 175 | |
| 173 | 176 | test("1 CPU: a snapshot held past the maximum wait is taken anyway", async () => { |
| 174 | let now = 1_000; | |
| 177 | await seedSnapshot(dirStore(join(dir, "store"))); | |
| 178 | let now = Date.now(); | |
| 175 | 179 | const s = snapshotRuns(1, () => 1, { holdMaxMs: 5_000, now: () => now }); |
| 176 | 180 | expect(await s.takeIfDue()).toBe(false); |
| 177 | 181 | now += 4_999; |
| 178 | 182 | expect(await s.takeIfDue()).toBe(false); |
| 179 | 183 | now += 1; |
| 180 | 184 | expect(await s.takeIfDue()).toBe(true); |
| 181 | expect(s.last().manifest).toBeDefined(); | |
| 185 | expect((await s.list()).length).toBe(2); | |
| 182 | 186 | }); |
| 183 | 187 | |
| 184 | 188 | test("2 CPUs take a due snapshot at once, without asking about crew runs", async () => { |
| @@ -188,3 +192,9 @@ test("2 CPUs take a due snapshot at once, without asking about crew runs", async | ||
| 188 | 192 | expect(s.last().manifest).toBeDefined(); |
| 189 | 193 | expect(asked).toBe(0); |
| 190 | 194 | }); |
| 195 | ||
| 196 | test("1 CPU: a store with no snapshot at all is not held for a crew run (#13732)", async () => { | |
| 197 | const s = snapshotRuns(1, () => 1); | |
| 198 | expect(await s.takeIfDue()).toBe(true); | |
| 199 | expect((await s.list()).length).toBe(1); | |
| 200 | }); |
test/snapshot-one-core.test.ts +113 −19
| @@ -12,7 +12,8 @@ import { join } from "node:path"; | ||
| 12 | 12 | import { createApp, type App } from "../src/app.ts"; |
| 13 | 13 | import { MAX_DEFER_MS, startSnapshots } from "../src/snapshot/service.ts"; |
| 14 | 14 | import { listSnapshots } from "../src/snapshot/snapshot.ts"; |
| 15 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 15 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 16 | import { seedSnapshot } from "./helpers/snapshot-seed.ts"; | |
| 16 | 17 | |
| 17 | 18 | let root: string; |
| 18 | 19 | let app: App; |
| @@ -39,18 +40,46 @@ async function setup(cpus: number, runs: { n: number }, clock: { t: number }) { | ||
| 39 | 40 | return { store, snaps }; |
| 40 | 41 | } |
| 41 | 42 | |
| 43 | /** Like `setup`, on a store that holds a snapshot from two days ago: late, not empty, so held. */ | |
| 44 | async function seeded(cpus: number, runs: { n: number }, c: { t: number }) { | |
| 45 | const store = dirStore(join(root, `store-${n++}`)); | |
| 46 | const seed = await seedSnapshot(store, c.t); | |
| 47 | const snaps = startSnapshots(app, store, { version: "t", nightly: false, now: () => c.t, cpus: () => cpus, activeCrewRuns: () => runs.n }); | |
| 48 | return { store, snaps, seed }; | |
| 49 | } | |
outdated
Weakened assertion: toBeGreaterThan(0) cannot tell a new snapshot from the seeded one. Assert the exact count or that the newest snapshot changed. |
||
| 50 | ||
| 51 | /** Every 50 ms until `ok`, for at most five seconds; a wait that never came is a failure. */ | |
| 52 | async function until(ok: () => Promise<boolean>): Promise<void> { | |
| 53 | for (let i = 0; i < 100 && !(await ok()); i++) { | |
| 54 | await Bun.sleep(50); // real-timer: the nightly timer under test is a real setTimeout armed 20 ms out | |
| 55 | } | |
| 56 | expect(await ok()).toBe(true); | |
| 57 | } | |
| 58 | ||
| 42 | 59 | const clock = () => ({ t: Date.parse("2026-11-12T09:00:00Z") }); |
| 43 | 60 | |
| 44 | 61 | test("1 CPU with a crew run active defers a due snapshot, then takes it once the run is gone", async () => { |
| 45 | 62 | const runs = { n: 1 }; |
| 46 | 63 | const c = clock(); |
| 47 | const { store, snaps } = await setup(1, runs, c); | |
| 64 | const { store, snaps, seed } = await seeded(1, runs, c); | |
| 48 | 65 | try { |
| 49 | 66 | expect(await snaps.takeIfDue()).toBe(false); |
| 50 | expect(await listSnapshots(store)).toHaveLength(0); | |
| 67 | expect((await listSnapshots(store)).map((m) => m.name)).toEqual([seed]); | |
| 51 | 68 | c.t += 5 * 60_000; |
| 52 | 69 | expect(await snaps.takeIfDue()).toBe(false); |
| 53 | 70 | runs.n = 0; |
| 71 | expect(await snaps.takeIfDue()).toBe(true); | |
| 72 | const after = await listSnapshots(store); | |
| 73 | expect(after).toHaveLength(2); | |
| 74 | expect(after[0]!.name).not.toBe(seed); | |
| 75 | } finally { | |
| 76 | await snaps.stop(); | |
| 77 | } | |
| 78 | }); | |
| 79 | ||
| 80 | test("a store with no snapshot at all is not made to wait for a crew run (#13732)", async () => { | |
| 81 | const { store, snaps } = await setup(1, { n: 1 }, clock()); | |
| 82 | try { | |
| 54 | 83 | expect(await snaps.takeIfDue()).toBe(true); |
| 55 | 84 | expect(await listSnapshots(store)).toHaveLength(1); |
| 56 | 85 | } finally { |
| @@ -71,37 +100,102 @@ test("2 CPUs take a due snapshot at once, run or not", async () => { | ||
| 71 | 100 | test("the cap: a run that never ends defers a snapshot at most one hour, then it is taken", async () => { |
| 72 | 101 | expect(MAX_DEFER_MS).toBe(3_600_000); |
| 73 | 102 | const c = clock(); |
| 74 | const { store, snaps } = await setup(1, { n: 1 }, c); | |
| 103 | const { store, snaps, seed } = await seeded(1, { n: 1 }, c); | |
| 75 | 104 | try { |
| 76 | 105 | expect(await snaps.takeIfDue()).toBe(false); |
| 77 | 106 | c.t += MAX_DEFER_MS - 1; |
| 78 | 107 | expect(await snaps.takeIfDue()).toBe(false); |
| 79 | 108 | c.t += 1; |
| 80 | 109 | expect(await snaps.takeIfDue()).toBe(true); |
| 110 | const after = await listSnapshots(store); | |
| 111 | expect(after).toHaveLength(2); | |
| 112 | expect(after[0]!.name).not.toBe(seed); | |
| 113 | } finally { | |
| 114 | await snaps.stop(); | |
| 115 | } | |
| 116 | }); | |
| 117 | ||
| 118 | /** Crew runs that count how often the snapshot asks; `asked(k)` settles on the k-th ask. */ | |
| 119 | function askedRuns(n: number) { | |
| 120 | const waiting: { k: number; done: () => void }[] = []; | |
| 121 | const runs = { | |
| 122 | n, | |
| 123 | asks: 0, | |
| 124 | ask: (): number => { | |
| 125 | runs.asks++; | |
| 126 | for (const w of waiting.filter((x) => x.k <= runs.asks)) w.done(); | |
| 127 | return runs.n; | |
| 128 | }, | |
| 129 | asked: (k: number): Promise<void> => | |
| 130 | k <= runs.asks ? Promise.resolve() : new Promise((done) => waiting.push({ k, done })), | |
| 131 | }; | |
| 132 | return runs; | |
| 133 | } | |
| 134 | ||
| 135 | /** The nightly timer due in 20 ms on one CPU; a held snapshot looks again every 20 ms. */ | |
| 136 | function nightly(store: Store, runs: { n: number; ask?: () => number }) { | |
| 137 | return startSnapshots(app, store, { | |
| 138 | version: "t", | |
| 139 | nightlyInMs: 20, | |
| 140 | holdRetryMs: 20, | |
| 141 | snapshotCheckMs: 3_600_000, | |
| 142 | snapshotCatchUpMs: 3_600_000, | |
| 143 | catchUpMs: 3_600_000, | |
| 144 | mirrorEveryMs: 3_600_000, | |
| 145 | cpus: () => 1, | |
| 146 | activeCrewRuns: runs.ask ?? (() => runs.n), | |
| 147 | }); | |
| 148 | } | |
| 149 | ||
| 150 | test("the nightly timer on an empty store takes a snapshot at once, crew run or not (#13732)", async () => { | |
| 151 | const store = dirStore(join(root, `store-${n++}`)); | |
| 152 | const snaps = nightly(store, { n: 1 }); | |
| 153 | try { | |
| 154 | await until(async () => (await listSnapshots(store)).length > 0); | |
| 81 | 155 | expect(await listSnapshots(store)).toHaveLength(1); |
| 82 | 156 | } finally { |
| 83 | 157 | await snaps.stop(); |
| 84 | 158 | } |
| 85 | 159 | }); |
| 86 | 160 | |
| 87 | test("the nightly timer: a held core re-arms it without taking; the next tick after the run takes", async () => { | |
| 88 | const runs = { n: 1 }; | |
| 161 | test("the nightly timer on a seeded store re-arms while the core is held; the next tick after the run takes", async () => { | |
| 162 | const runs = askedRuns(1); | |
| 89 | 163 | const store = dirStore(join(root, `store-${n++}`)); |
| 90 | const snaps = startSnapshots(app, store, { | |
| 91 | version: "t", | |
| 92 | nightlyInMs: 20, | |
| 93 | snapshotCheckMs: 20, | |
| 94 | snapshotCatchUpMs: 3_600_000, | |
| 95 | catchUpMs: 3_600_000, | |
| 96 | cpus: () => 1, | |
| 97 | activeCrewRuns: () => runs.n, | |
| 98 | }); | |
| 164 | const seed = await seedSnapshot(store); | |
| 165 | let puts = 0; | |
| 166 | const counted: Store = { ...store, put: (k, b) => (puts++, store.put(k, b)) }; | |
| 167 | const snaps = nightly(counted, runs); | |
| 99 | 168 | try { |
| 100 | await Bun.sleep(150); | |
| 101 | expect(await listSnapshots(store)).toHaveLength(0); | |
| 169 | // Three ticks go by held: nothing is written. | |
| 170 | await runs.asked(3); | |
| 171 | expect(puts).toBe(0); | |
| 172 | expect((await listSnapshots(store)).map((m) => m.name)).toEqual([seed]); | |
| 173 | // A re-armed timer, not a spent one, sees the run end. | |
| 102 | 174 | runs.n = 0; |
| 103 | for (let i = 0; i < 100 && (await listSnapshots(store)).length === 0; i++) await Bun.sleep(50); | |
| 104 | expect(await listSnapshots(store)).toHaveLength(1); | |
| 175 | await until(async () => (await listSnapshots(store)).length > 1); | |
| 176 | const after = await listSnapshots(store); | |
| 177 | expect(after).toHaveLength(2); | |
| 178 | expect(after[0]!.name).not.toBe(seed); | |
| 179 | } finally { | |
| 180 | await snaps.stop(); | |
| 181 | } | |
| 182 | }); | |
| 183 | ||
| 184 | test("the nightly timer keeps the hold when the store cannot be read: unreadable is not empty", async () => { | |
| 185 | const inner = dirStore(join(root, `store-${n++}`)); | |
| 186 | let puts = 0; | |
| 187 | const store: Store = { | |
| 188 | ...inner, | |
| 189 | names: () => Promise.reject(new Error("store unreadable")), | |
| 190 | put: (k, b) => (puts++, inner.put(k, b)), | |
| 191 | }; | |
| 192 | const runs = askedRuns(1); | |
| 193 | const snaps = nightly(store, runs); | |
| 194 | try { | |
| 195 | // Three ticks ask about crew runs, so the unreadable store was taken as held each time. | |
| 196 | await runs.asked(3); | |
| 197 | expect(puts).toBe(0); | |
| 198 | expect(snaps.progress()).toBeUndefined(); | |
| 105 | 199 | } finally { |
| 106 | 200 | await snaps.stop(); |
| 107 | 201 | } |
new above
offline
No test fails if this storeHoldsSnapshot guard is removed. Add a nightly-timer test (empty store is taken, seeded store re-arms, unreadable store keeps the hold).