Public: anyone can read this; the crew can change it.

Sentinel: backup was degraded 69 minutes today: Primary (s3.us-east-005.backblazeb2.com/openplatform-dr-backups (sealed)) holds no snapshot. The backup check last ran … #13737

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13732-sentinel-backup-was-degraded-69-minutes → main 6 commits fork 2 threads · 2 unresolved

  • Merged. scout/13732-sentinel-backup-was-degraded-69-minutes was merged into main as 3dfbc54.

Review

Sign in to review this change.
4 files changed +204 −34
src/snapshot/service.ts +49 −11
@@ -70,7 +70,10 @@ export interface SnapshotsHandle {
7070 readonly now: () => Promise<Manifest>;
7171 /** How far the snapshot being taken is; undefined when none is. */
7272 readonly progress: () => Progress | undefined;
73 /** How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits on it (#9653). */
73 /**
74 * How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits
75 * on it (#9653).
76 */
7477 readonly takingMs: () => number | undefined;
7578 /** The newest first, from the store. */
7679 readonly list: () => Promise<Manifest[]>;
@@ -85,13 +88,22 @@ export interface SnapshotsHandle {
8588 readonly startDrill: () => { readonly started: boolean; readonly state: DrillState | { readonly id: string; readonly phase: "waiting" } };
8689 /** Where the running drill stands; undefined when none runs. */
8790 readonly drillState: () => DrillState | { readonly id: string; readonly phase: "waiting" } | undefined;
88 /** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */
91 /**
92 * Drill if this week's slot has no drill yet, or the last one has stood red long enough; says
93 * whether it did.
94 */
8995 readonly drillIfDue: () => Promise<boolean>;
9096 /** The off-box mirror's store; undefined when the forge has none. */
9197 readonly mirror: Store | undefined;
92 /** Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no mirror. */
98 /**
99 * Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no
100 * mirror.
101 */
93102 readonly mirrorNow: () => Promise<MirrorState | undefined>;
94 /** Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it now is. */
103 /**
104 * Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it
105 * now is.
106 */
95107 readonly mirrorFresh: () => Promise<MirrorState | undefined>;
96108 /** What the last mirror pass found. */
97109 readonly mirrored: () => MirrorState | undefined;
@@ -99,7 +111,9 @@ export interface SnapshotsHandle {
99111 readonly forget: () => void;
100112 /** Read the bucket now, as the watch does; resolves when the read is kept. */
101113 readonly checkBucket: () => Promise<void>;
102 /** Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests. */
114 /**
115 * Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests.
116 */
103117 readonly takeIfDue: () => Promise<boolean>;
104118 readonly stop: () => Promise<void>;
105119 }
@@ -114,7 +128,9 @@ const DRILL_HOUR_UTC = 9;
114128 const CATCH_UP_MS = 5 * 60_000;
115129 /** A snapshot due on boot is taken this long after it, once the forge is serving. */
116130 const SNAPSHOT_CATCH_UP_MS = 2 * 60_000;
117 /** A snapshot still owed is looked for again this often; one boot's look is not the last (#10167). */
131 /**
132 * A snapshot still owed is looked for again this often; one boot's look is not the last (#10167).
133 */
118134 const SNAPSHOT_CHECK_MS = 30 * 60_000;
119135 /** A snapshot held for a crew run on one CPU looks again this often. */
120136 const HOLD_RETRY_MS = 30_000;
@@ -149,7 +165,10 @@ export function startSnapshots(
149165 /** Test seam: how soon a failed snapshot is tried again. */
150166 readonly retryMs?: number | undefined;
151167 readonly passphrase?: string | undefined;
152 /** Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it stands. */
168 /**
169 * Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it
170 * stands.
171 */
153172 readonly drillWaitMs?: number | undefined;
154173 /** Test seam: the drill itself. */
155174 readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined;
@@ -157,14 +176,19 @@ export function startSnapshots(
157176 readonly catchUpMs?: number | undefined;
158177 readonly redRetryMs?: number | undefined;
159178 readonly bootRedFloorMs?: number | undefined;
160 /** Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read. */
179 /**
180 * Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read.
181 */
161182 readonly snapshotCatchUpMs?: number | undefined;
162183 readonly snapshotCheckMs?: number | undefined;
163184 readonly bucketEveryMs?: number | undefined;
164185 readonly bucketFirstMs?: number | undefined;
165186 /** Test seam: the clock `takeIfDue` judges by. */
166187 readonly now?: (() => number) | undefined;
167 /** Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks again, and how long it waits at most. */
188 /**
189 * Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks
190 * again, and how long it waits at most.
191 */
168192 readonly cpus?: (() => number) | undefined;
169193 readonly activeCrewRuns?: (() => number) | undefined;
170194 readonly holdRetryMs?: number | undefined;
@@ -329,7 +353,10 @@ export function startSnapshots(
329353 log.warn("snapshot catch-up: store unreadable, not taking one now", { error: String(error) });
330354 return false;
331355 }
332 if (stopped || active > 0 || !snapshotDue(newest?.finishedAt, (options.now ?? Date.now)()) || crewHolds()) return false;
356 if (stopped || active > 0 || !snapshotDue(newest?.finishedAt, (options.now ?? Date.now)())) return false;
357 // A store with no snapshot at all is a forge with no backup: that is not
358 // deferred for a crew run (#13732); only a late one waits for the core.
359 if (newest !== undefined && crewHolds()) return false;
333360 log.info("snapshot due or lost; taking one", { newest: newest?.name ?? "none" });
334361 await take().catch(() => {});
335362 return true;
@@ -475,6 +502,17 @@ export function startSnapshots(
475502 }
476503 };
477504
505 /** Does the store hold any snapshot; an unreadable store is taken to, so the hold stays. */
506 const storeHoldsSnapshot = async (): Promise<boolean> => {
507 const store = current();
508 if (store === undefined) return true;
509 try {
510 return (await listSnapshots(store)).length > 0;
511 } catch {
512 return true;
513 }
514 };
515
478516 // One timer for the next snapshot: the nightly, or sooner after a failure.
479517 // Every run that ends sets it again, whoever asked for the run.
480518 let timer: ReturnType<typeof setTimeout> | undefined;
@@ -484,7 +522,7 @@ export function startSnapshots(
484522 if (options.nightly === false || stopped) return;
485523 timer = setTimeout(() => {
486524 void during("snapshot", async () => {
487 if (active === 0 && crewHolds()) return arm(holdRetryMs);
525 if (active === 0 && (await storeHoldsSnapshot()) && crewHolds()) return arm(holdRetryMs);

No test fails if this storeHoldsSnapshot guard is removed. Add a nightly-timer test (empty store is taken, seeded store re-arms, unreadable store keeps the hold).

488526 await take();
489527 }, undefined, app).catch(() => {});
490528 }, ms);
test/helpers/snapshot-seed.ts added +28
@@ -0,0 +1,28 @@
1 /**
2 * A store that already holds one snapshot, finished two days ago: the next is late, but the store
3 * is not empty, so the one-core hold applies to it (#13732). The manifest is written under a fixed
4 * name two days back, so a snapshot taken now is always a second one, never the seeded one again.
5 */
6 import { SNAPSHOT_PREFIX, snapshotName, type Manifest } from "../../src/snapshot/snapshot.ts";
7 import type { Store } from "../../src/snapshot/store.ts";
8
9 const SEED_AGE_MS = 2 * 86_400_000;
10
11 /** Seeds `store` and says the seeded snapshot's name. */
12 export async function seedSnapshot(store: Store, now: number = Date.now()): Promise<string> {
13 const at = new Date(now - SEED_AGE_MS);
14 const name = snapshotName(at);
15 const manifest: Manifest = {
16 format: 2,
17 name,
18 startedAt: at.toISOString(),
19 finishedAt: at.toISOString(),
20 version: "seed",
21 repos: 0,
22 apps: 0,
23 files: [],
24 bytes: 0,
25 };
26 await store.put(`${SNAPSHOT_PREFIX}${name}/manifest.json`, JSON.stringify(manifest));
27 return name;
28 }
test/one-core-hold.test.ts +14 −4
@@ -13,6 +13,7 @@ import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts
1313 import { dirStore } from "../src/snapshot/store.ts";
1414 import { holdClock } from "../src/core/one_core.ts";
1515 import { quietLogs } from "./helpers/globals.ts";
16 import { seedSnapshot } from "./helpers/snapshot-seed.ts";
1617
1718 quietLogs();
1819
@@ -162,23 +163,26 @@ function snapshotRuns(cpus: number, crew: () => number, extra: { holdMaxMs?: num
162163
163164 test("1 CPU with a crew run holds a due snapshot, and takes it once the run is gone", async () => {
164165 let crew = 1;
166 const seed = await seedSnapshot(dirStore(join(dir, "store")));
165167 const s = snapshotRuns(1, () => crew);
166168 expect(await s.takeIfDue()).toBe(false);
167 expect((await s.list()).length).toBe(0);
169 expect((await s.list()).map((m) => m.name)).toEqual([seed]);
168170 crew = 0;
169171 expect(await s.takeIfDue()).toBe(true);
170 expect(s.last().manifest).toBeDefined();
172 expect((await s.list()).length).toBe(2);
173 expect(s.last().manifest?.name).not.toBe(seed);
171174 });
172175
173176 test("1 CPU: a snapshot held past the maximum wait is taken anyway", async () => {
174 let now = 1_000;
177 await seedSnapshot(dirStore(join(dir, "store")));
178 let now = Date.now();
175179 const s = snapshotRuns(1, () => 1, { holdMaxMs: 5_000, now: () => now });
176180 expect(await s.takeIfDue()).toBe(false);
177181 now += 4_999;
178182 expect(await s.takeIfDue()).toBe(false);
179183 now += 1;
180184 expect(await s.takeIfDue()).toBe(true);
181 expect(s.last().manifest).toBeDefined();
185 expect((await s.list()).length).toBe(2);
182186 });
183187
184188 test("2 CPUs take a due snapshot at once, without asking about crew runs", async () => {
@@ -188,3 +192,9 @@ test("2 CPUs take a due snapshot at once, without asking about crew runs", async
188192 expect(s.last().manifest).toBeDefined();
189193 expect(asked).toBe(0);
190194 });
195
196 test("1 CPU: a store with no snapshot at all is not held for a crew run (#13732)", async () => {
197 const s = snapshotRuns(1, () => 1);
198 expect(await s.takeIfDue()).toBe(true);
199 expect((await s.list()).length).toBe(1);
200 });
test/snapshot-one-core.test.ts +113 −19
@@ -12,7 +12,8 @@ import { join } from "node:path";
1212 import { createApp, type App } from "../src/app.ts";
1313 import { MAX_DEFER_MS, startSnapshots } from "../src/snapshot/service.ts";
1414 import { listSnapshots } from "../src/snapshot/snapshot.ts";
15 import { dirStore } from "../src/snapshot/store.ts";
15 import { dirStore, type Store } from "../src/snapshot/store.ts";
16 import { seedSnapshot } from "./helpers/snapshot-seed.ts";
1617
1718 let root: string;
1819 let app: App;
@@ -39,18 +40,46 @@ async function setup(cpus: number, runs: { n: number }, clock: { t: number }) {
3940 return { store, snaps };
4041 }
4142
43 /** Like `setup`, on a store that holds a snapshot from two days ago: late, not empty, so held. */
44 async function seeded(cpus: number, runs: { n: number }, c: { t: number }) {
45 const store = dirStore(join(root, `store-${n++}`));
46 const seed = await seedSnapshot(store, c.t);
47 const snaps = startSnapshots(app, store, { version: "t", nightly: false, now: () => c.t, cpus: () => cpus, activeCrewRuns: () => runs.n });
48 return { store, snaps, seed };
49 }
outdated

Weakened assertion: toBeGreaterThan(0) cannot tell a new snapshot from the seeded one. Assert the exact count or that the newest snapshot changed.

50
51 /** Every 50 ms until `ok`, for at most five seconds; a wait that never came is a failure. */
52 async function until(ok: () => Promise<boolean>): Promise<void> {
53 for (let i = 0; i < 100 && !(await ok()); i++) {
54 await Bun.sleep(50); // real-timer: the nightly timer under test is a real setTimeout armed 20 ms out
55 }
56 expect(await ok()).toBe(true);
57 }
58
4259 const clock = () => ({ t: Date.parse("2026-11-12T09:00:00Z") });
4360
4461 test("1 CPU with a crew run active defers a due snapshot, then takes it once the run is gone", async () => {
4562 const runs = { n: 1 };
4663 const c = clock();
47 const { store, snaps } = await setup(1, runs, c);
64 const { store, snaps, seed } = await seeded(1, runs, c);
4865 try {
4966 expect(await snaps.takeIfDue()).toBe(false);
50 expect(await listSnapshots(store)).toHaveLength(0);
67 expect((await listSnapshots(store)).map((m) => m.name)).toEqual([seed]);
5168 c.t += 5 * 60_000;
5269 expect(await snaps.takeIfDue()).toBe(false);
5370 runs.n = 0;
71 expect(await snaps.takeIfDue()).toBe(true);
72 const after = await listSnapshots(store);
73 expect(after).toHaveLength(2);
74 expect(after[0]!.name).not.toBe(seed);
75 } finally {
76 await snaps.stop();
77 }
78 });
79
80 test("a store with no snapshot at all is not made to wait for a crew run (#13732)", async () => {
81 const { store, snaps } = await setup(1, { n: 1 }, clock());
82 try {
5483 expect(await snaps.takeIfDue()).toBe(true);
5584 expect(await listSnapshots(store)).toHaveLength(1);
5685 } finally {
@@ -71,37 +100,102 @@ test("2 CPUs take a due snapshot at once, run or not", async () => {
71100 test("the cap: a run that never ends defers a snapshot at most one hour, then it is taken", async () => {
72101 expect(MAX_DEFER_MS).toBe(3_600_000);
73102 const c = clock();
74 const { store, snaps } = await setup(1, { n: 1 }, c);
103 const { store, snaps, seed } = await seeded(1, { n: 1 }, c);
75104 try {
76105 expect(await snaps.takeIfDue()).toBe(false);
77106 c.t += MAX_DEFER_MS - 1;
78107 expect(await snaps.takeIfDue()).toBe(false);
79108 c.t += 1;
80109 expect(await snaps.takeIfDue()).toBe(true);
110 const after = await listSnapshots(store);
111 expect(after).toHaveLength(2);
112 expect(after[0]!.name).not.toBe(seed);
113 } finally {
114 await snaps.stop();
115 }
116 });
117
118 /** Crew runs that count how often the snapshot asks; `asked(k)` settles on the k-th ask. */
119 function askedRuns(n: number) {
120 const waiting: { k: number; done: () => void }[] = [];
121 const runs = {
122 n,
123 asks: 0,
124 ask: (): number => {
125 runs.asks++;
126 for (const w of waiting.filter((x) => x.k <= runs.asks)) w.done();
127 return runs.n;
128 },
129 asked: (k: number): Promise<void> =>
130 k <= runs.asks ? Promise.resolve() : new Promise((done) => waiting.push({ k, done })),
131 };
132 return runs;
133 }
134
135 /** The nightly timer due in 20 ms on one CPU; a held snapshot looks again every 20 ms. */
136 function nightly(store: Store, runs: { n: number; ask?: () => number }) {
137 return startSnapshots(app, store, {
138 version: "t",
139 nightlyInMs: 20,
140 holdRetryMs: 20,
141 snapshotCheckMs: 3_600_000,
142 snapshotCatchUpMs: 3_600_000,
143 catchUpMs: 3_600_000,
144 mirrorEveryMs: 3_600_000,
145 cpus: () => 1,
146 activeCrewRuns: runs.ask ?? (() => runs.n),
147 });
148 }
149
150 test("the nightly timer on an empty store takes a snapshot at once, crew run or not (#13732)", async () => {
151 const store = dirStore(join(root, `store-${n++}`));
152 const snaps = nightly(store, { n: 1 });
153 try {
154 await until(async () => (await listSnapshots(store)).length > 0);
81155 expect(await listSnapshots(store)).toHaveLength(1);
82156 } finally {
83157 await snaps.stop();
84158 }
85159 });
86160
87 test("the nightly timer: a held core re-arms it without taking; the next tick after the run takes", async () => {
88 const runs = { n: 1 };
161 test("the nightly timer on a seeded store re-arms while the core is held; the next tick after the run takes", async () => {
162 const runs = askedRuns(1);
89163 const store = dirStore(join(root, `store-${n++}`));
90 const snaps = startSnapshots(app, store, {
91 version: "t",
92 nightlyInMs: 20,
93 snapshotCheckMs: 20,
94 snapshotCatchUpMs: 3_600_000,
95 catchUpMs: 3_600_000,
96 cpus: () => 1,
97 activeCrewRuns: () => runs.n,
98 });
164 const seed = await seedSnapshot(store);
165 let puts = 0;
166 const counted: Store = { ...store, put: (k, b) => (puts++, store.put(k, b)) };
167 const snaps = nightly(counted, runs);
99168 try {
100 await Bun.sleep(150);
101 expect(await listSnapshots(store)).toHaveLength(0);
169 // Three ticks go by held: nothing is written.
170 await runs.asked(3);
171 expect(puts).toBe(0);
172 expect((await listSnapshots(store)).map((m) => m.name)).toEqual([seed]);
173 // A re-armed timer, not a spent one, sees the run end.
102174 runs.n = 0;
103 for (let i = 0; i < 100 && (await listSnapshots(store)).length === 0; i++) await Bun.sleep(50);
104 expect(await listSnapshots(store)).toHaveLength(1);
175 await until(async () => (await listSnapshots(store)).length > 1);
176 const after = await listSnapshots(store);
177 expect(after).toHaveLength(2);
178 expect(after[0]!.name).not.toBe(seed);
179 } finally {
180 await snaps.stop();
181 }
182 });
183
184 test("the nightly timer keeps the hold when the store cannot be read: unreadable is not empty", async () => {
185 const inner = dirStore(join(root, `store-${n++}`));
186 let puts = 0;
187 const store: Store = {
188 ...inner,
189 names: () => Promise.reject(new Error("store unreadable")),
190 put: (k, b) => (puts++, inner.put(k, b)),
191 };
192 const runs = askedRuns(1);
193 const snaps = nightly(store, runs);
194 try {
195 // Three ticks ask about crew runs, so the unreadable store was taken as held each time.
196 await runs.asked(3);
197 expect(puts).toBe(0);
198 expect(snaps.progress()).toBeUndefined();
105199 } finally {
106200 await snaps.stop();
107201 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs