Public: anyone can read this; the crew can change it.
Sentinel: backup was degraded 41 minutes today: Primary (s3.us-east-005.backblazeb2.com/openplatform-dr-backups) newest snapshot 2026-11-07T08-00-00Z is 26 h old. The … #9675
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/9653-sentinel-backup-was-degraded-41-minutes → main 1 commit fork
7 files
changed +56
−1
docs/features/release-keeper.md +2
| @@ -13,6 +13,8 @@ keyboard. It does what the steward's `release.sh`, `verify-after.sh` and | ||
| 13 | 13 | - no keeper release is rolling or verifying; |
| 14 | 14 | - the newest tag is live, or the keeper rolled it back; |
| 15 | 15 | - no merge-queue gate is running; |
| 16 | - no snapshot is being taken, unless it has run two hours and is stuck | |
| 17 | (a release drains the process the snapshot runs in; #9653); | |
| 16 | 18 | - 30 minutes have passed since the last cut; |
| 17 | 19 | - no hold stands. |
| 18 | 20 |
src/keeper/cadence.ts +13
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | * ∧ the live version is the newest tag's, or the keeper rolled that |
| 7 | 7 | * tag back (nothing still promoting) |
| 8 | 8 | * ∧ no merge-queue gate is running (the queue is not mid-landing) |
| 9 | * ∧ no snapshot is being taken, unless it has run `SNAPSHOT_WAIT_MS` | |
| 9 | 10 | * ∧ now − the last cut ≥ the minimum interval |
| 10 | 11 | * ∧ no hold stands (a failed verify's, or an owner's) |
| 11 | 12 | * |
| @@ -16,6 +17,14 @@ | ||
| 16 | 17 | /** Two releases are at least this far apart: 30 minutes. */ |
| 17 | 18 | export const MIN_INTERVAL_MS = 30 * 60_000; |
| 18 | 19 | |
| 20 | /** | |
| 21 | * A release drains the process a snapshot runs in, so a cut waits for the | |
| 22 | * snapshot (#9653: swaps every few minutes cut the nightly short all | |
| 23 | * morning, and the newest snapshot went 27 h old). One running longer than | |
| 24 | * this is stuck, and holds releases no more. | |
| 25 | */ | |
| 26 | export const SNAPSHOT_WAIT_MS = 2 * 3_600_000; | |
| 27 | ||
| 19 | 28 | export interface CadenceInput { |
| 20 | 29 | /** main's head commit. */ |
| 21 | 30 | readonly main: string; |
| @@ -29,6 +38,8 @@ export interface CadenceInput { | ||
| 29 | 38 | readonly inFlight: boolean; |
| 30 | 39 | /** Merge-queue gates running now. */ |
| 31 | 40 | readonly gating: number; |
| 41 | /** How long the snapshot being taken has run, ms; undefined when none is. */ | |
| 42 | readonly snapshotMs?: number | undefined; | |
| 32 | 43 | /** When the keeper last cut, epoch ms. */ |
| 33 | 44 | readonly lastCutAt: number | undefined; |
| 34 | 45 | /** The reason of a standing hold, when one stands. */ |
| @@ -56,6 +67,8 @@ export function decideCut(input: CadenceInput): Cadence { | ||
| 56 | 67 | if (input.inFlight) return no("a release is still rolling or being verified"); |
| 57 | 68 | if (input.live !== input.newest.version && input.settled !== true) return no(`${input.newest.version} is not live yet (live: ${input.live})`); |
| 58 | 69 | if (input.gating > 0) return no(`the merge queue is landing (${input.gating} gate${input.gating === 1 ? "" : "s"} running)`); |
| 70 | if (input.snapshotMs !== undefined && input.snapshotMs < SNAPSHOT_WAIT_MS) | |
| 71 | return no(`a snapshot is being taken (${Math.floor(input.snapshotMs / 60_000)} min in); a release would cut it short`); | |
| 59 | 72 | const wait = input.minIntervalMs ?? MIN_INTERVAL_MS; |
| 60 | 73 | if (input.lastCutAt !== undefined && input.now - input.lastCutAt < wait) |
| 61 | 74 | return no(`the last cut was ${Math.floor((input.now - input.lastCutAt) / 60_000)} min ago; the interval is ${wait / 60_000} min`); |
src/keeper/keeper.ts +3
| @@ -53,6 +53,8 @@ export interface KeeperPorts { | ||
| 53 | 53 | readonly newest: () => Promise<{ version: string; sha: string } | undefined>; |
| 54 | 54 | /** Merge-queue gates running now. */ |
| 55 | 55 | readonly gating: () => number; |
| 56 | /** How long the snapshot being taken has run, ms; undefined when none is (#9653). */ | |
| 57 | readonly snapshotting?: () => number | undefined; | |
| 56 | 58 | readonly deployer: Deployer; |
| 57 | 59 | /** Make the annotated tag `v<version>` on `sha` and send it where the release workflow starts; why not. */ |
| 58 | 60 | readonly tag: (version: string, sha: string) => Promise<string | undefined>; |
| @@ -199,6 +201,7 @@ export async function tick(p: KeeperPorts): Promise<string> { | ||
| 199 | 201 | settled: newest !== undefined && rolledBack(p.db, newest.version), |
| 200 | 202 | inFlight: false, |
| 201 | 203 | gating: p.gating(), |
| 204 | snapshotMs: p.snapshotting?.(), | |
| 202 | 205 | lastCutAt: last === undefined ? undefined : Date.parse(last.cutAt), |
| 203 | 206 | hold: blocking(standingHolds(p.db)), |
| 204 | 207 | now: p.now(), |
src/keeper/wire.ts +2
| @@ -30,6 +30,7 @@ import { running } from "../kernel/version.ts"; | ||
| 30 | 30 | import { compareVersions, readTags, recordFor } from "../release/shipped.ts"; |
| 31 | 31 | import { checkoutRelease, preflightRelease } from "../release/checkout.ts"; |
| 32 | 32 | import { handover } from "../release/swap.ts"; |
| 33 | import { snapshotsOf } from "../snapshot/service.ts"; | |
| 33 | 34 | import { kubeDeployer } from "./deploy.ts"; |
| 34 | 35 | import { swapper } from "./swapper.ts"; |
| 35 | 36 | import type { KeeperPorts } from "./keeper.ts"; |
| @@ -121,6 +122,7 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str | ||
| 121 | 122 | return sha.ok && OID.test(sha.value) ? { version: top.version, sha: sha.value } : undefined; |
| 122 | 123 | }, |
| 123 | 124 | gating: gatesInFlight, |
| 125 | snapshotting: () => snapshotsOf(app)?.takingMs(), | |
| 124 | 126 | deployer, |
| 125 | 127 | async tag(version, sha) { |
| 126 | 128 | if (!VERSION.test(version) || !OID.test(sha)) return "not a version and a commit"; |
src/snapshot/service.ts +6
| @@ -65,6 +65,8 @@ export interface SnapshotsHandle { | ||
| 65 | 65 | readonly now: () => Promise<Manifest>; |
| 66 | 66 | /** How far the snapshot being taken is; undefined when none is. */ |
| 67 | 67 | readonly progress: () => Progress | undefined; |
| 68 | /** How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits on it (#9653). */ | |
| 69 | readonly takingMs: () => number | undefined; | |
| 68 | 70 | /** The newest first, from the store. */ |
| 69 | 71 | readonly list: () => Promise<Manifest[]>; |
| 70 | 72 | /** The last one taken by this process, and what it said. */ |
| @@ -166,6 +168,7 @@ export function startSnapshots( | ||
| 166 | 168 | let inFlight: Promise<Manifest> | undefined; |
| 167 | 169 | let active = 0; |
| 168 | 170 | let running: Progress | undefined; |
| 171 | let startedAt: number | undefined; | |
| 169 | 172 | let last: { manifest?: Manifest; error?: string; at?: string } = {}; |
| 170 | 173 | |
| 171 | 174 | const take = (resume?: Date): Promise<Manifest> => { |
| @@ -180,6 +183,7 @@ export function startSnapshots( | ||
| 180 | 183 | if (store === undefined) throw new Nowhere(); |
| 181 | 184 | offBox = isOffBox(store); |
| 182 | 185 | running = { objects: 0, bytes: 0, elapsedMs: 0 }; |
| 186 | startedAt = Date.now(); | |
| 183 | 187 | // Admin-only, no repository, nothing /backups does not already show. |
| 184 | 188 | app.bus.append({ kind: "backup.started" }); |
| 185 | 189 | let toldAt = Date.now(); |
| @@ -202,6 +206,7 @@ export function startSnapshots( | ||
| 202 | 206 | resume, |
| 203 | 207 | ).finally(() => { |
| 204 | 208 | running = undefined; |
| 209 | startedAt = undefined; | |
| 205 | 210 | }); |
| 206 | 211 | await pruneSnapshots(store, KEEP); |
| 207 | 212 | return manifest; |
| @@ -470,6 +475,7 @@ export function startSnapshots( | ||
| 470 | 475 | }, |
| 471 | 476 | last: () => last, |
| 472 | 477 | progress: () => running, |
| 478 | takingMs: () => (startedAt === undefined ? undefined : Date.now() - startedAt), | |
| 473 | 479 | get mirror() { |
| 474 | 480 | return currentMirror(); |
| 475 | 481 | }, |
test/release-keeper-cadence.test.ts +14 −1
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | // When the release keeper cuts (#7557): each rule refuses on its own, and a cut is a tag, never a commit to main. |
| 2 | 2 | import { describe, expect, test } from "bun:test"; |
| 3 | import { decideCut, MIN_INTERVAL_MS, nextVersion, type CadenceInput } from "../src/keeper/cadence.ts"; | |
| 3 | import { decideCut, MIN_INTERVAL_MS, nextVersion, SNAPSHOT_WAIT_MS, type CadenceInput } from "../src/keeper/cadence.ts"; | |
| 4 | 4 | import { tick } from "../src/keeper/keeper.ts"; |
| 5 | 5 | import { listKeeperReleases, recordCut } from "../src/keeper/record.ts"; |
| 6 | 6 | import { A, B, keeperForge } from "./helpers/keeper.ts"; |
| @@ -41,6 +41,19 @@ describe("cadence", () => { | ||
| 41 | 41 | expect(why({ lastCutAt: NOW - MIN_INTERVAL_MS + 60_000 })).toBe("the last cut was 29 min ago; the interval is 30 min"); |
| 42 | 42 | expect(why({ hold: "owner: freeze for the demo" })).toBe("held: owner: freeze for the demo"); |
| 43 | 43 | expect(why({ newest: undefined })).toContain("no release tag"); |
| 44 | expect(why({ snapshotMs: 42 * 60_000 })).toBe("a snapshot is being taken (42 min in); a release would cut it short"); | |
| 45 | }); | |
| 46 | ||
| 47 | test("a snapshot in flight holds the cut, until it has run long enough to be stuck (#9653)", async () => { | |
| 48 | expect(decideCut({ ...ready, snapshotMs: 0 }).cut).toBe(false); | |
| 49 | expect(decideCut({ ...ready, snapshotMs: SNAPSHOT_WAIT_MS - 1 }).cut).toBe(false); | |
| 50 | expect(decideCut({ ...ready, snapshotMs: SNAPSHOT_WAIT_MS })).toEqual({ cut: true, version: "0.7.284" }); | |
| 51 | expect(decideCut({ ...ready, snapshotMs: undefined })).toEqual({ cut: true, version: "0.7.284" }); | |
| 52 | const f = await keeperForge(); | |
| 53 | const held = await tick({ ...f.ports, snapshotting: () => 10 * 60_000 }); | |
| 54 | expect(held).toBe("a snapshot is being taken (10 min in); a release would cut it short"); | |
| 55 | expect(f.world.tags).toEqual([]); | |
| 56 | expect(await tick({ ...f.ports, snapshotting: () => undefined })).toContain("cut 0.7.284"); | |
| 44 | 57 | }); |
| 45 | 58 | |
| 46 | 59 | test("a tag the keeper rolled back is settled: the fix after it may be cut", () => { |
test/snapshot.test.ts +16
| @@ -595,4 +595,20 @@ describe("a snapshot's uploads", () => { | ||
| 595 | 595 | await svc.stop(); |
| 596 | 596 | } |
| 597 | 597 | }, 15_000); |
| 598 | ||
| 599 | test("the service says how long the snapshot in flight has run, for the release keeper to wait on (#9653)", async () => { | |
| 600 | const { store } = slowStore(40); | |
| 601 | const svc = startSnapshots(app, store, { version: "t", nightly: false }); | |
| 602 | try { | |
| 603 | expect(svc.takingMs()).toBeUndefined(); | |
| 604 | const taking = svc.now(); | |
| 605 | await Bun.sleep(30); | |
| 606 | const ms = svc.takingMs(); | |
| 607 | expect(ms).toBeGreaterThanOrEqual(0); | |
| 608 | await taking; | |
| 609 | expect(svc.takingMs()).toBeUndefined(); | |
| 610 | } finally { | |
| 611 | await svc.stop(); | |
| 612 | } | |
| 613 | }, 15_000); | |
| 598 | 614 | }); |