Public: anyone can read this; the crew can change it.
Sentinel: snapshot-failed was degraded 23 minutes today: The last snapshot failed: could not put [redacted].chunk: … #13736
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13733-sentinel-snapshot-failed-was-degraded-23 → main 6 commits fork
6 files
changed +246
−35
docs/SNAPSHOT.md +7 −2
| @@ -200,7 +200,10 @@ The store is `Bun.S3Client` or a directory behind one small interface | ||
| 200 | 200 | is tried five times; an attempt that has not answered by its deadline counts |
| 201 | 201 | as failed. A put's deadline follows its size: its bytes at a floor of |
| 202 | 202 | 5 Mbit/s, plus 60 s. A fixed 180 s once failed every attempt at a 334 MB |
| 203 | database behind a 15 Mbit/s uplink (#2536). | |
| 203 | database behind a 15 Mbit/s uplink (#2536). An attempt past its deadline is | |
| 204 | aborted, not just abandoned: a put goes as a presigned PUT through `fetch` | |
| 205 | with the attempt's signal, because `client.write` takes none, and an | |
| 206 | abandoned upload went on sending beside the tries after it (#13733). | |
| 204 | 207 | |
| 205 | 208 | ## Backup lanes |
| 206 | 209 | |
| @@ -219,7 +222,9 @@ Its outcome is kept in `snapshot/state.json` under the data directory, so a | ||
| 219 | 222 | restart forgets nothing. A failure shows straight away as a card on the board |
| 220 | 223 | (`snapshot-failed`, pointing at Backups), in the terminal's dashboard among |
| 221 | 224 | the forge's needs, and as a red line on the Backups page with the reason and |
| 222 | since when. The forge tries again within the hour, not the next night. The | |
| 225 | since when. The forge tries again after ten minutes (`SNAPSHOT_RETRY_MS`, capped at the | |
| 226 | next nightly), not the next night: a store that is down gets six tries an hour, each of five | |
| 227 | attempts per put. The | |
| 223 | 228 | next snapshot that lands, nightly or on demand, clears both. |
| 224 | 229 | |
| 225 | 230 | A forge given nowhere to put a snapshot has not failed: nothing is kept, no |
src/snapshot/outcome.ts +19 −4
| @@ -35,7 +35,10 @@ export interface SnapshotOutcome { | ||
| 35 | 35 | /** The nights that landed, newest last, at most `NIGHTS_KEPT`: what each took, for |
| 36 | 36 | * `growth.ts`. */ |
| 37 | 37 | readonly nights?: readonly Night[]; |
| 38 | /** Whether the store it landed in is off the machine (a bucket, not a directory); absent when not known. */ | |
| 38 | /** | |
| 39 | * Whether the store it landed in is off the machine (a bucket, not a directory); absent when not | |
| 40 | * known. | |
| 41 | */ | |
| 39 | 42 | readonly offBox?: boolean; |
| 40 | 43 | /** |
| 41 | 44 | * When a snapshot last landed off the machine; kept across landings in a local directory, |
| @@ -71,7 +74,10 @@ export function lastSnapshotOutcome(dataDir: string): SnapshotOutcome | undefine | ||
| 71 | 74 | } |
| 72 | 75 | } |
| 73 | 76 | |
| 74 | /** When this outcome's last off-box landing was: the kept time, or the landing itself when it was off the machine. */ | |
| 77 | /** | |
| 78 | * When this outcome's last off-box landing was: the kept time, or the landing itself when it was | |
| 79 | * off the machine. | |
| 80 | */ | |
| 75 | 81 | export function offBoxAtOf(s: SnapshotOutcome | undefined): string | undefined { |
| 76 | 82 | if (s === undefined) return undefined; |
| 77 | 83 | if (s.offBoxAt !== undefined) return s.offBoxAt; |
| @@ -84,7 +90,9 @@ function nightsWith(nights: readonly Night[] | undefined, night: Night): Night[] | ||
| 84 | 90 | return [...(nights ?? []).filter((n) => n.at.slice(0, 10) !== day), night].slice(-NIGHTS_KEPT); |
| 85 | 91 | } |
| 86 | 92 | |
| 87 | /** Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept. */ | |
| 93 | /** | |
| 94 | * Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept. | |
| 95 | */ | |
| 88 | 96 | export function recordSnapshotOutcome( |
| 89 | 97 | dataDir: string, |
| 90 | 98 | result: |
| @@ -147,6 +155,13 @@ function write(dataDir: string, next: SnapshotOutcome): void { | ||
| 147 | 155 | renameSync(tmp, stateFile(dataDir)); |
| 148 | 156 | } |
| 149 | 157 | |
| 158 | /** | |
| 159 | * After a failed snapshot the next try comes this soon (or the nightly, if sooner). A flaky store | |
| 160 | * (one stalled put, 2026-11-15) used to hold the card for the hour before the retry; ten minutes | |
| 161 | * clears a flake and still gives a dead store only six tries an hour. | |
| 162 | */ | |
| 163 | export const SNAPSHOT_RETRY_MS = 10 * 60_000; | |
| 164 | ||
| 150 | 165 | /** The card a failing snapshot is, or undefined while the last one landed. */ |
| 151 | 166 | export function snapshotNeed( |
| 152 | 167 | s: SnapshotOutcome | undefined, |
| @@ -154,7 +169,7 @@ export function snapshotNeed( | ||
| 154 | 169 | if (s?.failingSince === undefined) return undefined; |
| 155 | 170 | const last = s.landedAt ? ` The newest that landed is from ${s.landedAt.slice(0, 10)}.` : ""; |
| 156 | 171 | return { |
| 157 | sentence: `The last snapshot failed: ${(s.error ?? "unknown reason").replace(/\.$/, "")}.${last} The forge tries again within the hour.`, | |
| 172 | sentence: `The last snapshot failed: ${(s.error ?? "unknown reason").replace(/\.$/, "")}.${last} The forge tries again in ${SNAPSHOT_RETRY_MS / 60_000} minutes.`, | |
| 158 | 173 | since: s.failingSince, |
| 159 | 174 | }; |
| 160 | 175 | } |
src/snapshot/service.ts +35 −13
| @@ -34,7 +34,7 @@ import { cutDrill, runDrillInChild, sweepDrills, type DrillState } from "./drill | ||
| 34 | 34 | import { lastMirror, mirrorOffThread, mirrorTarget, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts"; |
| 35 | 35 | import { isOffBox } from "./off_box.ts"; |
| 36 | 36 | import { failureCause } from "./cause.ts"; |
| 37 | import { noteOffBox, recordSnapshotOutcome } from "./outcome.ts"; | |
| 37 | import { noteOffBox, recordSnapshotOutcome, SNAPSHOT_RETRY_MS } from "./outcome.ts"; | |
| 38 | 38 | import { remakeOf, s3FromEnv, type Store } from "./store.ts"; |
| 39 | 39 | import { verifySnapshot } from "./verify.ts"; |
| 40 | 40 | import { during } from "../perf/jobs.ts"; |
| @@ -43,8 +43,6 @@ const log = createLog("snapshots"); | ||
| 43 | 43 | |
| 44 | 44 | /** When the nightly snapshot is taken, in UTC: before the probe round. */ |
| 45 | 45 | const NIGHTLY_HOUR_UTC = 8; |
| 46 | /** After a failed snapshot, the next try is this soon (or the nightly, if sooner). */ | |
| 47 | export const RETRY_MS = 3_600_000; | |
| 48 | 46 | /** A due snapshot waits for a crew run on a 1-CPU forge at most this long, then is taken anyway. */ |
| 49 | 47 | export const MAX_DEFER_MS = 3_600_000; |
| 50 | 48 | /** A moving snapshot tells the page how far it is no oftener than this. */ |
| @@ -70,7 +68,10 @@ export interface SnapshotsHandle { | ||
| 70 | 68 | readonly now: () => Promise<Manifest>; |
| 71 | 69 | /** How far the snapshot being taken is; undefined when none is. */ |
| 72 | 70 | readonly progress: () => Progress | undefined; |
| 73 | /** How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits on it (#9653). */ | |
| 71 | /** | |
| 72 | * How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits | |
| 73 | * on it (#9653). | |
| 74 | */ | |
| 74 | 75 | readonly takingMs: () => number | undefined; |
| 75 | 76 | /** The newest first, from the store. */ |
| 76 | 77 | readonly list: () => Promise<Manifest[]>; |
| @@ -85,13 +86,22 @@ export interface SnapshotsHandle { | ||
| 85 | 86 | readonly startDrill: () => { readonly started: boolean; readonly state: DrillState | { readonly id: string; readonly phase: "waiting" } }; |
| 86 | 87 | /** Where the running drill stands; undefined when none runs. */ |
| 87 | 88 | readonly drillState: () => DrillState | { readonly id: string; readonly phase: "waiting" } | undefined; |
| 88 | /** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */ | |
| 89 | /** | |
| 90 | * Drill if this week's slot has no drill yet, or the last one has stood red long enough; says | |
| 91 | * whether it did. | |
| 92 | */ | |
| 89 | 93 | readonly drillIfDue: () => Promise<boolean>; |
| 90 | 94 | /** The off-box mirror's store; undefined when the forge has none. */ |
| 91 | 95 | readonly mirror: Store | undefined; |
| 92 | /** Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no mirror. */ | |
| 96 | /** | |
| 97 | * Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no | |
| 98 | * mirror. | |
| 99 | */ | |
| 93 | 100 | readonly mirrorNow: () => Promise<MirrorState | undefined>; |
| 94 | /** Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it now is. */ | |
| 101 | /** | |
| 102 | * Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it | |
| 103 | * now is. | |
| 104 | */ | |
| 95 | 105 | readonly mirrorFresh: () => Promise<MirrorState | undefined>; |
| 96 | 106 | /** What the last mirror pass found. */ |
| 97 | 107 | readonly mirrored: () => MirrorState | undefined; |
| @@ -99,7 +109,9 @@ export interface SnapshotsHandle { | ||
| 99 | 109 | readonly forget: () => void; |
| 100 | 110 | /** Read the bucket now, as the watch does; resolves when the read is kept. */ |
| 101 | 111 | readonly checkBucket: () => Promise<void>; |
| 102 | /** Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests. */ | |
| 112 | /** | |
| 113 | * Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests. | |
| 114 | */ | |
| 103 | 115 | readonly takeIfDue: () => Promise<boolean>; |
| 104 | 116 | readonly stop: () => Promise<void>; |
| 105 | 117 | } |
| @@ -114,7 +126,9 @@ const DRILL_HOUR_UTC = 9; | ||
| 114 | 126 | const CATCH_UP_MS = 5 * 60_000; |
| 115 | 127 | /** A snapshot due on boot is taken this long after it, once the forge is serving. */ |
| 116 | 128 | const SNAPSHOT_CATCH_UP_MS = 2 * 60_000; |
| 117 | /** A snapshot still owed is looked for again this often; one boot's look is not the last (#10167). */ | |
| 129 | /** | |
| 130 | * A snapshot still owed is looked for again this often; one boot's look is not the last (#10167). | |
| 131 | */ | |
| 118 | 132 | const SNAPSHOT_CHECK_MS = 30 * 60_000; |
| 119 | 133 | /** A snapshot held for a crew run on one CPU looks again this often. */ |
| 120 | 134 | const HOLD_RETRY_MS = 30_000; |
| @@ -149,7 +163,10 @@ export function startSnapshots( | ||
| 149 | 163 | /** Test seam: how soon a failed snapshot is tried again. */ |
| 150 | 164 | readonly retryMs?: number | undefined; |
| 151 | 165 | readonly passphrase?: string | undefined; |
| 152 | /** Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it stands. */ | |
| 166 | /** | |
| 167 | * Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it | |
| 168 | * stands. | |
| 169 | */ | |
| 153 | 170 | readonly drillWaitMs?: number | undefined; |
| 154 | 171 | /** Test seam: the drill itself. */ |
| 155 | 172 | readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined; |
| @@ -157,14 +174,19 @@ export function startSnapshots( | ||
| 157 | 174 | readonly catchUpMs?: number | undefined; |
| 158 | 175 | readonly redRetryMs?: number | undefined; |
| 159 | 176 | readonly bootRedFloorMs?: number | undefined; |
| 160 | /** Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read. */ | |
| 177 | /** | |
| 178 | * Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read. | |
| 179 | */ | |
| 161 | 180 | readonly snapshotCatchUpMs?: number | undefined; |
| 162 | 181 | readonly snapshotCheckMs?: number | undefined; |
| 163 | 182 | readonly bucketEveryMs?: number | undefined; |
| 164 | 183 | readonly bucketFirstMs?: number | undefined; |
| 165 | 184 | /** Test seam: the clock `takeIfDue` judges by. */ |
| 166 | 185 | readonly now?: (() => number) | undefined; |
| 167 | /** Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks again, and how long it waits at most. */ | |
| 186 | /** | |
| 187 | * Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks | |
| 188 | * again, and how long it waits at most. | |
| 189 | */ | |
| 168 | 190 | readonly cpus?: (() => number) | undefined; |
| 169 | 191 | readonly activeCrewRuns?: (() => number) | undefined; |
| 170 | 192 | readonly holdRetryMs?: number | undefined; |
| @@ -314,7 +336,7 @@ export function startSnapshots( | ||
| 314 | 336 | }, |
| 315 | 337 | }); |
| 316 | 338 | app.bus.append({ kind: "backup.finished", payload: { ok: false } }); |
| 317 | arm(Math.min(options.retryMs ?? RETRY_MS, untilNightly())); | |
| 339 | arm(Math.min(options.retryMs ?? SNAPSHOT_RETRY_MS, untilNightly())); | |
| 318 | 340 | throw error; |
| 319 | 341 | }, |
| 320 | 342 | ); |
src/snapshot/store.ts +45 −13
| @@ -15,7 +15,12 @@ export interface Store { | ||
| 15 | 15 | readonly where: string; |
| 16 | 16 | /** The directory it writes under, when it is one; a bucket has none. */ |
| 17 | 17 | readonly dir?: string; |
| 18 | put(key: string, body: Uint8Array | string): Promise<void>; | |
| 18 | /** | |
| 19 | * `signal` aborts the upload: a put given up at its deadline must stop sending, or its bytes | |
| 20 | * keep the line busy under the attempt that replaces it (#13733). A store that cannot abort | |
| 21 | * ignores it. | |
| 22 | */ | |
| 23 | put(key: string, body: Uint8Array | string, signal?: AbortSignal): Promise<void>; | |
| 19 | 24 | get(key: string): Promise<Uint8Array | undefined>; |
| 20 | 25 | /** Every key under `prefix`, in no particular order. */ |
| 21 | 26 | list(prefix: string): Promise<string[]>; |
| @@ -34,7 +39,9 @@ export interface Store { | ||
| 34 | 39 | * this out. |
| 35 | 40 | */ |
| 36 | 41 | storedKey?(key: string): Promise<string>; |
| 37 | /** True when every object is encrypted before it reaches the store (the off-site lane, #11805). */ | |
| 42 | /** | |
| 43 | * True when every object is encrypted before it reaches the store (the off-site lane, #11805). | |
| 44 | */ | |
| 38 | 45 | readonly sealed?: boolean; |
| 39 | 46 | /** |
| 40 | 47 | * What the line is giving this store's puts (`line.ts`); a store that does not measure leaves it |
| @@ -102,12 +109,18 @@ export function s3FromEnv( | ||
| 102 | 109 | |
| 103 | 110 | /** The slowest uplink a put may assume: 5 Mbit/s, a third of the home line it runs behind. */ |
| 104 | 111 | export const FLOOR_BYTES_PER_S = 5_000_000 / 8; |
| 105 | /** What every attempt gets on top of its bytes' time: the handshake, the server's thought, a queue ahead of it. */ | |
| 112 | /** | |
| 113 | * What every attempt gets on top of its bytes' time: the handshake, the server's thought, a queue | |
| 114 | * ahead of it. | |
| 115 | */ | |
| 106 | 116 | export const SLACK_MS = 60_000; |
| 107 | 117 | /** How long a read, a listing or a delete may go unanswered. */ |
| 108 | 118 | export const CALL_DEADLINE_MS = 180_000; |
| 109 | 119 | |
| 110 | /** How long a put of `bytes` may take before it counts as failed: its bytes at the floor rate, plus slack. */ | |
| 120 | /** | |
| 121 | * How long a put of `bytes` may take before it counts as failed: its bytes at the floor rate, plus | |
| 122 | * slack. | |
| 123 | */ | |
| 111 | 124 | export const deadlineFor = ( |
| 112 | 125 | bytes: number, |
| 113 | 126 | floorBytesPerS = FLOOR_BYTES_PER_S, |
| @@ -181,7 +194,7 @@ export function withRetries( | ||
| 181 | 194 | async function tried<T>( |
| 182 | 195 | what: string, |
| 183 | 196 | deadlineMs: number | (() => number), |
| 184 | fn: () => Promise<T>, | |
| 197 | fn: (signal: AbortSignal) => Promise<T>, | |
| 185 | 198 | leftMs?: () => number, |
| 186 | 199 | ): Promise<T> { |
| 187 | 200 | let last: unknown; |
| @@ -198,7 +211,7 @@ export function withRetries( | ||
| 198 | 211 | } |
| 199 | 212 | made++; |
| 200 | 213 | try { |
| 201 | return await answered(fn(), Math.min(own, share)); | |
| 214 | return await answered(fn, Math.min(own, share)); | |
| 202 | 215 | } catch (error) { |
| 203 | 216 | last = error; |
| 204 | 217 | if (n + 1 < attempts) await sleep(baseMs * 2 ** n); |
| @@ -215,11 +228,11 @@ export function withRetries( | ||
| 215 | 228 | tried( |
| 216 | 229 | `put ${key}`, |
| 217 | 230 | () => putMs(body), |
| 218 | async () => { | |
| 231 | async (signal) => { | |
| 219 | 232 | const end = line.begin(); |
| 220 | 233 | let landed = 0; |
| 221 | 234 | try { |
| 222 | await inner.put(key, body); | |
| 235 | await inner.put(key, body, signal); | |
| 223 | 236 | landed = sizeOf(body); |
| 224 | 237 | } finally { |
| 225 | 238 | end(landed); |
| @@ -240,14 +253,21 @@ export function withRetries( | ||
| 240 | 253 | |
| 241 | 254 | /** |
| 242 | 255 | * The attempt's answer, or a rejection once `ms` have passed without one. |
| 243 | * The abandoned attempt is left to settle on its own, its outcome ignored. | |
| 256 | * At the deadline the attempt's signal is aborted, so a store that can | |
| 257 | * stops sending (#13733: an abandoned put went on uploading beside the | |
| 258 | * fourteen tries after it); its outcome is ignored either way. | |
| 244 | 259 | */ |
| 245 | function answered<T>(attempt: Promise<T>, ms: number): Promise<T> { | |
| 260 | function answered<T>(start: (signal: AbortSignal) => Promise<T>, ms: number): Promise<T> { | |
| 246 | 261 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 247 | 262 | const shown = ms >= 1000 ? `${Math.round(ms / 1000)} s` : `${ms} ms`; |
| 263 | const control = new AbortController(); | |
| 248 | 264 | const deadline = new Promise<never>((_, reject) => { |
| 249 | 265 | timer = setTimeout(() => reject(new Error(`no answer in ${shown}`)), ms); |
| 250 | 266 | }); |
| 267 | // The deadline's own rejection aborts the attempt, so the timer stays the | |
| 268 | // plain reject that test/timers-named.test.ts allows by name. | |
| 269 | deadline.catch((reason: unknown) => control.abort(reason)); | |
| 270 | const attempt = start(control.signal); | |
| 251 | 271 | attempt.catch(() => {}); |
| 252 | 272 | return Promise.race([attempt, deadline]).finally(() => clearTimeout(timer)); |
| 253 | 273 | } |
| @@ -261,7 +281,11 @@ export function s3Store(target: S3Target, attempts?: number): Store { | ||
| 261 | 281 | export const s3Where = (target: Pick<S3Target, "endpoint" | "bucket">): string => |
| 262 | 282 | `${target.endpoint.replace(/^https?:\/\//, "")}/${target.bucket}`; |
| 263 | 283 | |
| 264 | function rawS3Store(target: S3Target): Store { | |
| 284 | /** How long a put's signed URL is good for: longer than any put's deadline. */ | |
| 285 | const PUT_URL_SECONDS = 6 * 3600; | |
| 286 | ||
| 287 | /** The bucket as a store, each call once: `s3Store` is this with retries. Exported for tests. */ | |
| 288 | export function rawS3Store(target: S3Target): Store { | |
| 265 | 289 | const client = new Bun.S3Client({ |
| 266 | 290 | endpoint: target.endpoint, |
| 267 | 291 | bucket: target.bucket, |
| @@ -271,8 +295,16 @@ function rawS3Store(target: S3Target): Store { | ||
| 271 | 295 | }); |
| 272 | 296 | return { |
| 273 | 297 | where: s3Where(target), |
| 274 | async put(key, body) { | |
| 275 | await client.write(key, body); | |
| 298 | // A presigned PUT through fetch, not `client.write`: only fetch takes a signal, and an | |
| 299 | // upload given up at its deadline must stop (#13733). The URL is signed here, no network. | |
| 300 | async put(key, body, signal) { | |
| 301 | const url = client.presign(key, { method: "PUT", expiresIn: PUT_URL_SECONDS }); | |
| 302 | const res = await fetch(url, { method: "PUT", body, ...(signal === undefined ? {} : { signal }) }); | |
| 303 | if (!res.ok) { | |
| 304 | const said = (await res.text().catch(() => "")).match(/<Code>([^<]{1,80})<\/Code>/)?.[1]; | |
| 305 | throw new Error(`the bucket answered ${res.status}${said === undefined ? "" : ` ${said}`}`); | |
| 306 | } | |
| 307 | await res.body?.cancel().catch(() => {}); | |
| 276 | 308 | }, |
| 277 | 309 | async get(key) { |
| 278 | 310 | const file = client.file(key); |
test/snapshot-outcome.test.ts +46 −3
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * A failed snapshot is seen the same morning (#2536): it is kept under the |
| 3 | 3 | * data directory, vitals carries it as a card and Backups as a red line, |
| 4 | * both pointing at the reason, and the forge tries again within the hour | |
| 4 | * both pointing at the reason, and the forge tries again in ten minutes | |
| 5 | 5 | * rather than the next night. A snapshot that lands clears it. |
| 6 | 6 | */ |
| 7 | 7 | |
| @@ -14,7 +14,7 @@ import { forgeNeeds, forgeVitals } from "../src/actions/health.ts"; | ||
| 14 | 14 | import { createApp, type App } from "../src/app.ts"; |
| 15 | 15 | import { createUser, type User } from "../src/core/users.ts"; |
| 16 | 16 | import { serve, type Servers } from "../src/http/server.ts"; |
| 17 | import { lastSnapshotOutcome, recordSnapshotOutcome, snapshotNeed } from "../src/snapshot/outcome.ts"; | |
| 17 | import { lastSnapshotOutcome, recordSnapshotOutcome, snapshotNeed, SNAPSHOT_RETRY_MS } from "../src/snapshot/outcome.ts"; | |
| 18 | 18 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 19 | 19 | import { dirStore, type Store } from "../src/snapshot/store.ts"; |
| 20 | 20 | |
| @@ -38,7 +38,7 @@ describe("the outcome kept", () => { | ||
| 38 | 38 | const need = snapshotNeed(lastSnapshotOutcome(dir)); |
| 39 | 39 | expect(need?.since).toBe("2026-10-17T08:45:00.000Z"); |
| 40 | 40 | expect(need?.sentence).toBe( |
| 41 | "The last snapshot failed: could not put forge.db: no answer in 180 s. The newest that landed is from 2026-10-16. The forge tries again within the hour.", | |
| 41 | "The last snapshot failed: could not put forge.db: no answer in 180 s. The newest that landed is from 2026-10-16. The forge tries again in 10 minutes.", | |
| 42 | 42 | ); |
| 43 | 43 | recordSnapshotOutcome(dir, { landed: "2026-10-18T09-00-00Z" }, "2026-10-18T09:20:00.000Z"); |
| 44 | 44 | expect(snapshotNeed(lastSnapshotOutcome(dir))).toBeUndefined(); |
| @@ -160,3 +160,46 @@ describe("a forge given nowhere to put a snapshot", () => { | ||
| 160 | 160 | } |
| 161 | 161 | }); |
| 162 | 162 | }); |
| 163 | ||
| 164 | describe("the next try after a failed snapshot (#13733)", () => { | |
| 165 | /** Fail one snapshot with the clock at `at`; the delay the forge then armed, ms. */ | |
| 166 | async function armedAfterFailure(at: string): Promise<number> { | |
| 167 | const root = mkdtempSync(join(tmpdir(), "forge-retry-")); | |
| 168 | const app = createApp({ dataDir: join(root, "data"), port: 0 }); | |
| 169 | const realNow = Date.now; | |
| 170 | const realSet = globalThis.setTimeout; | |
| 171 | const delays: number[] = []; | |
| 172 | const store: Store = { | |
| 173 | ...dirStore(join(root, "store")), | |
| 174 | put: async () => { | |
| 175 | throw new Error("no answer in 66 s"); | |
| 176 | }, | |
| 177 | }; | |
| 178 | // Only the nightly timer is read; the fixed nightlyInMs keeps the first arm out of the way. | |
| 179 | const svc = startSnapshots(app, store, { version: "t", nightlyInMs: 1e9 }); | |
| 180 | try { | |
| 181 | Date.now = () => Date.parse(at); | |
| 182 | globalThis.setTimeout = ((fn: () => void, ms?: number, ...rest: unknown[]) => { | |
| 183 | if (typeof ms === "number" && ms >= 60_000) delays.push(ms); | |
| 184 | return realSet(fn, ms, ...rest); | |
| 185 | }) as typeof setTimeout; | |
| 186 | await expect(svc.now()).rejects.toThrow(/no answer in 66 s/); | |
| 187 | return delays[delays.length - 1]!; | |
| 188 | } finally { | |
| 189 | Date.now = realNow; | |
| 190 | globalThis.setTimeout = realSet; | |
| 191 | svc.stop(); | |
| 192 | app.close(); | |
| 193 | await rm(root, { recursive: true, force: true }); | |
| 194 | } | |
| 195 | } | |
| 196 | ||
| 197 | test("is SNAPSHOT_RETRY_MS away by default, not the hour", async () => { | |
| 198 | expect(SNAPSHOT_RETRY_MS).toBe(600_000); | |
| 199 | expect(await armedAfterFailure("2026-11-15T10:00:00Z")).toBe(SNAPSHOT_RETRY_MS); | |
| 200 | }); | |
| 201 | ||
| 202 | test("is capped at the nightly when that is sooner", async () => { | |
| 203 | expect(await armedAfterFailure("2026-11-15T07:55:00Z")).toBe(5 * 60_000); | |
| 204 | }); | |
| 205 | }); |
test/snapshot-put-abort.test.ts added +94
| @@ -0,0 +1,94 @@ | ||
| 1 | /** | |
| 2 | * A put given up at its deadline stops sending (#13733). A snapshot failed for | |
| 3 | * 83 minutes with "no answer in 63 s" fifteen times over one chunk: each try | |
| 4 | * that timed out was abandoned but kept uploading, so every retry shared the | |
| 5 | * line with the ones before it. The deadline now aborts the attempt's request. | |
| 6 | */ | |
| 7 | import { describe, expect, test } from "bun:test"; | |
| 8 | import { rawS3Store, s3Store, withRetries, type Store } from "../src/snapshot/store.ts"; | |
| 9 | import { bypassProxy, stubBucket } from "./helpers/stub-bucket.ts"; | |
| 10 | ||
| 11 | describe("a put past its deadline", () => { | |
| 12 | test("has its signal aborted, and the next attempt gets a fresh one", async () => { | |
| 13 | const signals: AbortSignal[] = []; | |
| 14 | const inner: Store = { | |
| 15 | where: "stalls once", | |
| 16 | async put(_key, _body, signal) { | |
| 17 | signals.push(signal!); | |
| 18 | if (signals.length === 1) await new Promise<void>(() => {}); | |
| 19 | }, | |
| 20 | get: async () => undefined, | |
| 21 | list: async () => [], | |
| 22 | names: async () => [], | |
| 23 | delete: async () => {}, | |
| 24 | }; | |
| 25 | const store = withRetries(inner, { attempts: 3, baseMs: 1, deadlineMs: 20, sleep: async () => {} }); | |
| 26 | await store.put("chunks/a.chunk", new Uint8Array(4)); | |
| 27 | expect(signals.length).toBe(2); | |
| 28 | expect(signals[0]!.aborted).toBe(true); | |
| 29 | expect(String(signals[0]!.reason)).toContain("no answer in 20 ms"); | |
| 30 | expect(signals[1]!.aborted).toBe(false); | |
| 31 | }); | |
| 32 | ||
| 33 | test("closes the request to the bucket, through the real client", async () => { | |
| 34 | const restore = bypassProxy(); | |
| 35 | const arrived = Promise.withResolvers<void>(); | |
| 36 | const closed = Promise.withResolvers<boolean>(); | |
| 37 | const server = Bun.serve({ | |
| 38 | port: 0, | |
| 39 | hostname: "127.0.0.1", | |
| 40 | fetch(req) { | |
| 41 | req.signal.addEventListener("abort", () => closed.resolve(true)); | |
| 42 | arrived.resolve(); | |
| 43 | return new Promise<Response>(() => {}); | |
| 44 | }, | |
| 45 | }); | |
| 46 | try { | |
| 47 | const store = rawS3Store({ endpoint: `http://127.0.0.1:${server.port}`, bucket: "offsite", accessKeyId: "a", secretAccessKey: "b" }); | |
| 48 | const control = new AbortController(); | |
| 49 | const put = store.put("chunks/b.chunk", new Uint8Array(1024), control.signal); | |
| 50 | put.catch(() => {}); | |
| 51 | await arrived.promise; | |
| 52 | control.abort(new Error("no answer in 50 ms")); | |
| 53 | await expect(put).rejects.toThrow(); | |
| 54 | expect(await closed.promise).toBe(true); | |
| 55 | } finally { | |
| 56 | server.stop(true); | |
| 57 | restore(); | |
| 58 | } | |
| 59 | }); | |
| 60 | }); | |
| 61 | ||
| 62 | describe("a put through the bucket", () => { | |
| 63 | test("lands the bytes under the key", async () => { | |
| 64 | const restore = bypassProxy(); | |
| 65 | const objects = new Map<string, Uint8Array>(); | |
| 66 | const seen: string[] = []; | |
| 67 | const server = stubBucket(objects, seen); | |
| 68 | try { | |
| 69 | const store = s3Store({ endpoint: `http://127.0.0.1:${server.port}`, bucket: "offsite", accessKeyId: "a", secretAccessKey: "b" }); | |
| 70 | await store.put("chunks/c d.chunk", new Uint8Array([1, 2, 3])); | |
| 71 | expect(seen).toEqual(["PUT chunks/c d.chunk"]); | |
| 72 | expect([...objects.get("chunks/c d.chunk")!]).toEqual([1, 2, 3]); | |
| 73 | } finally { | |
| 74 | server.stop(true); | |
| 75 | restore(); | |
| 76 | } | |
| 77 | }); | |
| 78 | ||
| 79 | test("says what the bucket answered when it refuses", async () => { | |
| 80 | const restore = bypassProxy(); | |
| 81 | const server = Bun.serve({ | |
| 82 | port: 0, | |
| 83 | hostname: "127.0.0.1", | |
| 84 | fetch: () => new Response("<Error><Code>AccessDenied</Code></Error>", { status: 403 }), | |
| 85 | }); | |
| 86 | try { | |
| 87 | const store = rawS3Store({ endpoint: `http://127.0.0.1:${server.port}`, bucket: "offsite", accessKeyId: "a", secretAccessKey: "b" }); | |
| 88 | await expect(store.put("chunks/e.chunk", "x")).rejects.toThrow("the bucket answered 403 AccessDenied"); | |
| 89 | } finally { | |
| 90 | server.stop(true); | |
| 91 | restore(); | |
| 92 | } | |
| 93 | }); | |
| 94 | }); |
new above
offline