Public: anyone can read this; the crew can change it.
A backup finishes on a slow line: no fixed time budget for the whole snapshot, fewer chunks at once, and a failed run's chunks count #13745
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13743-a-backup-finishes-on-a-slow-line-no-fixe → main 3 commits fork 2 threads · 2 unresolved
10 files
changed +421
−21
docs/features/README.md +1
| @@ -442,6 +442,7 @@ Read with the guide page [Backups and moving](../guide-backups-moving.md). | ||
| 442 | 442 | - [](upstream-landing.md) |
| 443 | 443 | - [](follow-an-origin.md) |
| 444 | 444 | - [](resilience-backups.md) |
| 445 | - [](slow-line-backups.md) | |
| 445 | 446 | - [](offsite-lane.md) |
| 446 | 447 | - [](origin-guard.md) |
| 447 | 448 | - [](forge-archive.md) |
docs/features/slow-line-backups.md added +20
| @@ -0,0 +1,20 @@ | ||
| 1 | # A backup that finishes on a slow line | |
| 2 | ||
| 3 | A snapshot sent over a home uplink that gives the backup 1 to 2 Mbit/s used to fail every | |
| 4 | time: each put got a deadline from a fixed 5 Mbit/s floor (66 s for a 4 MB chunk), eight puts | |
| 5 | shared the line, and the run retried from the top (#13743). Now: | |
| 6 | ||
| 7 | - **Deadline.** A put's deadline is the longer of the floor's and three times its bytes at the | |
| 8 | rate the line has actually landed puts at, over its share (`src/snapshot/line.ts`, | |
| 9 | `withRetries` in `src/snapshot/store.ts`). It is the put's own, never what is left of the | |
| 10 | run. A put that moves nothing still times out. | |
| 11 | - **Puts at once.** One per megabit the line gives, between one and eight; two until the line | |
| 12 | has been measured (`putsAtOnce`). | |
| 13 | - **A failed run keeps what it sent.** Every run lists the chunks the lane holds (for the | |
| 14 | sealed lane, under their hidden names) and sends only the rest, so a killed run resumes | |
| 15 | without re-sending; this was already true of the sealed lane and is now tested. | |
| 16 | - **Progress.** The log line `in progress`, the Backups page, the terminal's Resilience screen | |
| 17 | and the `backup.progress` event say bytes sent, "N of M chunks already there", bytes to go | |
| 18 | (estimated from the last snapshot's size), the rate and when it will be done. | |
| 19 | ||
| 20 | Tests: `test/snapshot-slow-line.test.ts`. |
src/snapshot/line.ts added +86
| @@ -0,0 +1,86 @@ | ||
| 1 | /** | |
| 2 | * What the uplink is actually giving a store (#13743): the bytes its puts landed over the time at | |
| 3 | * least one was in flight, so the rate is the line's whole, whatever the number of puts sharing it. | |
| 4 | * A put's deadline (`store.ts`) and the number of puts at once (`snapshot.ts`) both read it. | |
| 5 | * | |
| 6 | * Failed puts count their time but not their bytes, so a line that has stalled reads slower and | |
| 7 | * the next deadline is longer, not shorter. Halved past `WINDOW_MS` of busy time, so a line that | |
| 8 | * changed (a night shift of the household's own traffic) is believed within the hour. | |
| 9 | * | |
| 10 | * Plain data in, plain data out; the clock is a parameter. | |
| 11 | */ | |
| 12 | ||
| 13 | /** Busy time kept in the average before it is halved. */ | |
| 14 | export const WINDOW_MS = 20 * 60_000; | |
| 15 | /** Busy time and bytes needed before the rate is believed at all. */ | |
| 16 | export const MIN_BUSY_MS = 2_000; | |
| 17 | export const MIN_BYTES = 256 * 1024; | |
| 18 | ||
| 19 | export interface LineMeter { | |
| 20 | /** | |
| 21 | * A put began; returns what to call when it ends, with the bytes it landed (0 when it failed). | |
| 22 | */ | |
| 23 | begin(): (landedBytes: number) => void; | |
| 24 | /** Bytes per second the line has given, or undefined until enough has moved to say. */ | |
| 25 | rate(): number | undefined; | |
| 26 | /** Puts in flight now. */ | |
| 27 | inFlight(): number; | |
| 28 | } | |
| 29 | ||
| 30 | export function lineMeter(now: () => number = () => performance.now()): LineMeter { | |
| 31 | let bytes = 0; | |
| 32 | let busyMs = 0; | |
| 33 | let open = 0; | |
| 34 | let last = now(); | |
| 35 | const tick = (): void => { | |
| 36 | const t = now(); | |
| 37 | if (open > 0) busyMs += Math.max(0, t - last); | |
| 38 | last = t; | |
| 39 | }; | |
| 40 | return { | |
| 41 | begin() { | |
| 42 | tick(); | |
| 43 | open++; | |
| 44 | let ended = false; | |
| 45 | return (landed) => { | |
| 46 | if (ended) return; | |
| 47 | ended = true; | |
| 48 | tick(); | |
| 49 | open--; | |
| 50 | bytes += landed; | |
| 51 | if (busyMs > WINDOW_MS) { | |
| 52 | busyMs /= 2; | |
| 53 | bytes /= 2; | |
| 54 | } | |
| 55 | }; | |
| 56 | }, | |
| 57 | rate() { | |
| 58 | tick(); | |
| 59 | return busyMs >= MIN_BUSY_MS && bytes >= MIN_BYTES ? (bytes / busyMs) * 1000 : undefined; | |
| 60 | }, | |
| 61 | inFlight: () => open, | |
| 62 | }; | |
| 63 | } | |
| 64 | ||
| 65 | /** Each put should have at least this much of the line (1 Mbit/s), or fewer go at once. */ | |
| 66 | export const MIN_PER_PUT_BYTES_PER_S = 125_000; | |
| 67 | /** Puts at once before the line has been measured. */ | |
| 68 | export const UNMEASURED_PUTS = 2; | |
| 69 | ||
| 70 | /** | |
| 71 | * How many puts to have in flight: the line's rate over what each needs, between one and `most`. | |
| 72 | */ | |
| 73 | export function putsAtOnce(rate: number | undefined, most: number): number { | |
| 74 | if (rate === undefined) return Math.min(UNMEASURED_PUTS, most); | |
| 75 | return Math.max(1, Math.min(most, Math.floor(rate / MIN_PER_PUT_BYTES_PER_S))); | |
| 76 | } | |
| 77 | ||
| 78 | /** | |
| 79 | * How long a put of `bytes` may take with `sharing` puts on a line giving `rate`: its bytes at its | |
| 80 | * share, three times over, since a line is not steady. Zero when there is no rate yet: the floor | |
| 81 | * (`deadlineFor`) stands alone then. | |
| 82 | */ | |
| 83 | export function measuredDeadlineMs(bytes: number, rate: number | undefined, sharing: number): number { | |
| 84 | if (rate === undefined || !(rate > 0)) return 0; | |
| 85 | return Math.ceil(((bytes * Math.max(1, sharing)) / rate) * 3000); | |
| 86 | } |
src/snapshot/sealed_store.ts +1
| @@ -177,6 +177,7 @@ function over(inner: Store, keys: () => Promise<SealKeys>): Store { | ||
| 177 | 177 | await inner.delete(await at(key)); |
| 178 | 178 | }, |
| 179 | 179 | storedKey: async (key) => hiddenKey(await keys(), key), |
| 180 | ...(inner.line === undefined ? {} : { line: inner.line }), | |
| 180 | 181 | }; |
| 181 | 182 | const bounded = inner.bounded; |
| 182 | 183 | return bounded === undefined ? store : { ...store, bounded: (leftMs) => over(bounded.call(inner, leftMs), keys) }; |
src/snapshot/service.ts +12
| @@ -208,6 +208,16 @@ export function startSnapshots( | ||
| 208 | 208 | let startedAt: number | undefined; |
| 209 | 209 | let last: { manifest?: Manifest; error?: string; at?: string } = {}; |
| 210 | 210 | |
| 211 | // What "to go" is judged against: the size of this lane's newest snapshot. | |
| 212 | const expectedSize = async (store: Store): Promise<number | undefined> => { | |
| 213 | if (last.manifest !== undefined) return last.manifest.bytes; | |
| 214 | try { | |
| 215 | return (await listSnapshots(store))[0]?.bytes; | |
| 216 | } catch { | |
| 217 | return undefined; | |
| 218 | } | |
| 219 | }; | |
| 220 | ||
| 211 | 221 | const take = (): Promise<Manifest> => { |
| 212 | 222 | // Whether this run's store is off the machine, for home's backup age (#3701). |
| 213 | 223 | let offBox: boolean | undefined; |
| @@ -231,6 +241,7 @@ export function startSnapshots( | ||
| 231 | 241 | // Admin-only, no repository, nothing /backups does not already show. |
| 232 | 242 | app.bus.append({ kind: "backup.started" }); |
| 233 | 243 | let toldAt = Date.now(); |
| 244 | const expectedBytes = await expectedSize(store); | |
| 234 | 245 | const manifest = await takeSnapshot( |
| 235 | 246 | { |
| 236 | 247 | db: app.db, |
| @@ -238,6 +249,7 @@ export function startSnapshots( | ||
| 238 | 249 | reposRoot: app.reposRoot, |
| 239 | 250 | version: options.version, |
| 240 | 251 | passphrase: options.passphrase, |
| 252 | expectedBytes, | |
| 241 | 253 | onLand: (p) => { |
| 242 | 254 | running = p; |
| 243 | 255 | sent = p.bytes; |
src/snapshot/snapshot.ts +80 −13
| @@ -50,6 +50,7 @@ import { countRows, type DrillCounts } from "./counts.ts"; | ||
| 50 | 50 | import { CHUNK_PREFIX, chunkId, chunkKey, chunkSeed, contentChunker, fixedChunker, isChunkId, packChunk, type Chunker } from "./chunks.ts"; |
| 51 | 51 | import type { PackedFile } from "./parts.ts"; |
| 52 | 52 | import { bodiesOf, type Store } from "./store.ts"; |
| 53 | import { putsAtOnce } from "./line.ts"; | |
| 53 | 54 | import { unwrapKey, wrapKey, WRAPPED_KEY } from "./wrap.ts"; |
| 54 | 55 | import { during } from "../perf/jobs.ts"; |
| 55 | 56 | |
| @@ -167,9 +168,50 @@ export interface Progress { | ||
| 167 | 168 | readonly objects: number; |
| 168 | 169 | readonly bytes: number; |
| 169 | 170 | readonly elapsedMs: number; |
| 171 | /** | |
| 172 | * Chunks the store already held, and chunks seen so far (held plus sent): "N of M already there". | |
| 173 | */ | |
| 174 | readonly already?: number; | |
| 175 | readonly seen?: number; | |
| 176 | /** What the line is giving, bytes per second, once measured. */ | |
| 177 | readonly bytesPerS?: number; | |
| 178 | /** | |
| 179 | * Stored bytes still to send, estimated from the last snapshot's size; absent when that is not | |
| 180 | * known. | |
| 181 | */ | |
| 182 | readonly toGoBytes?: number; | |
| 183 | /** When it will be done at this rate, ms from now. */ | |
| 184 | readonly etaMs?: number; | |
| 170 | 185 | } |
| 171 | 186 | |
| 172 | /** Puts in flight at once: latency, not bandwidth, was the hour. */ | |
| 187 | /** What a page shows beside "N objects, X so far": one label and value each; parts not known are left out. */ | |
| 188 | export function progressFacts(p: Progress): { label: string; value: string }[] { | |
| 189 | const rows: { label: string; value: string }[] = []; | |
| 190 | if (p.already !== undefined && p.seen !== undefined && p.seen > 0) rows.push({ label: "Already there", value: `${p.already} of ${p.seen} chunks` }); | |
| 191 | if (p.toGoBytes !== undefined) rows.push({ label: "Still to send", value: fmtBytes(p.toGoBytes) }); | |
| 192 | if (p.bytesPerS !== undefined) rows.push({ label: "Line speed", value: `${(p.bytesPerS * 8 / 1_000_000).toFixed(2)} Mbit/s` }); | |
| 193 | if (p.etaMs !== undefined) rows.push({ label: "Done in about", value: fmtDuration(p.etaMs) }); | |
| 194 | return rows; | |
| 195 | } | |
| 196 | ||
| 197 | /** Progress as a short sentence for the log and the terminal: what is sent, what is left, how long. */ | |
| 198 | export function progressWords(p: Progress): string { | |
| 199 | const parts = [`${fmtBytes(p.bytes)} sent`]; | |
| 200 | if (p.toGoBytes !== undefined) parts.push(`${fmtBytes(p.toGoBytes)} left`); | |
| 201 | if (p.etaMs !== undefined) parts.push(`done in about ${fmtDuration(p.etaMs)}`); | |
| 202 | return parts.join(", "); | |
| 203 | } | |
| 204 | ||
| 205 | const fmtBytes = (n: number): string => (n < 1_048_576 ? `${(n / 1024).toFixed(0)} KiB` : n < 1_073_741_824 ? `${(n / 1_048_576).toFixed(1)} MiB` : `${(n / 1_073_741_824).toFixed(2)} GiB`); | |
| 206 | const fmtDuration = (ms: number): string => { | |
| 207 | const min = Math.max(1, Math.round(ms / 60_000)); | |
| 208 | return min < 90 ? `${min} min` : `${(min / 60).toFixed(1)} h`; | |
| 209 | }; | |
| 210 | ||
| 211 | /** | |
| 212 | * Puts in flight at most: latency, not bandwidth, was the hour on a fast line. On a slow one fewer | |
| 213 | * go at once (`putsAtOnce`): each needs about a megabit of its own, or its deadline is a guess. | |
| 214 | */ | |
| 173 | 215 | export const IN_FLIGHT = 8; |
| 174 | 216 | /** Bytes held in memory by puts in flight; past this the next chunk waits. */ |
| 175 | 217 | export const IN_FLIGHT_BYTES = 64 * 1024 * 1024; |
| @@ -273,6 +315,8 @@ export interface Snapshotter { | ||
| 273 | 315 | readonly contentCut?: { readonly min: number; readonly bits: number; readonly max: number } | undefined; |
| 274 | 316 | /** Override of a bundle's deadline by the repository's bytes, for tests. */ |
| 275 | 317 | readonly bundleDeadlineMs?: ((bytes: number) => number) | undefined; |
| 318 | /** Plain bytes of the last snapshot, any lane: what "to go" is judged against. */ | |
| 319 | readonly expectedBytes?: number | undefined; | |
| 276 | 320 | } |
| 277 | 321 | |
| 278 | 322 | /** |
| @@ -341,28 +385,50 @@ export async function takeSnapshot( | ||
| 341 | 385 | objects: p.objects, |
| 342 | 386 | bytes: p.bytes, |
| 343 | 387 | elapsed: `${Math.round(p.elapsedMs / 1000)}s`, |
| 388 | says: progressWords(p), | |
| 344 | 389 | })); |
| 345 | 390 | let landed = 0; |
| 346 | 391 | let landedBytes = 0; |
| 347 | 392 | let sinceReport = 0; |
| 348 | const say = (): void => { | |
| 349 | sinceReport = 0; | |
| 350 | report({ | |
| 393 | // Plain bytes cut so far, and the stored bytes handed to the queue (landed or not), for "to go". | |
| 394 | let readBytes = 0; | |
| 395 | let queuedBytes = 0; | |
| 396 | const now = (): Progress => { | |
| 397 | const elapsedMs = Date.now() - started; | |
| 398 | const rate = store.line?.rate() ?? (landedBytes > 0 && elapsedMs > 5_000 ? (landedBytes / elapsedMs) * 1000 : undefined); | |
| 399 | const expected = forge.expectedBytes; | |
| 400 | const toGo = | |
| 401 | expected !== undefined && expected > 0 && readBytes > 0 | |
| 402 | ? Math.round(Math.max(0, queuedBytes - landedBytes) + Math.max(0, expected - readBytes) * (queuedBytes / readBytes)) | |
| 403 | : undefined; | |
| 404 | return { | |
| 351 | 405 | objects: landed, |
| 352 | 406 | bytes: landedBytes, |
| 353 | elapsedMs: Date.now() - started, | |
| 354 | }); | |
| 407 | elapsedMs, | |
| 408 | already: found, | |
| 409 | seen: found + sentChunks, | |
| 410 | ...(rate === undefined ? {} : { bytesPerS: Math.round(rate) }), | |
| 411 | ...(toGo === undefined ? {} : { toGoBytes: toGo }), | |
| 412 | ...(toGo === undefined || rate === undefined || rate <= 0 ? {} : { etaMs: Math.round((toGo / rate) * 1000) }), | |
| 413 | }; | |
| 414 | }; | |
| 415 | const say = (): void => { | |
| 416 | sinceReport = 0; | |
| 417 | report(now()); | |
| 355 | 418 | }; |
| 356 | 419 | const ticker = setInterval(() => during("snapshot-report", say), everyMs); |
| 357 | 420 | const pending = new Set<Promise<void>>(); |
| 358 | 421 | let failure: Error | undefined; |
| 359 | 422 | let held = 0; |
| 423 | let sentChunks = 0; | |
| 424 | // How many puts may be in flight now: the line's own rate sets it (`putsAtOnce`). | |
| 425 | const atOnce = (): number => (store.line === undefined ? IN_FLIGHT : putsAtOnce(store.line.rate(), IN_FLIGHT)); | |
| 360 | 426 | // Hands an object to the queue: waits for a free slot, then lets the put |
| 361 | 427 | // run on its own while the caller reads on. |
| 362 | 428 | const put = async (key: string, body: Uint8Array): Promise<void> => { |
| 363 | 429 | const bytes = body.byteLength; |
| 364 | 430 | while ( |
| 365 | (pending.size >= IN_FLIGHT || | |
| 431 | (pending.size >= atOnce() || | |
| 366 | 432 | (pending.size > 0 && held + bytes > IN_FLIGHT_BYTES)) && |
| 367 | 433 | failure === undefined |
| 368 | 434 | ) |
| @@ -374,11 +440,7 @@ export async function takeSnapshot( | ||
| 374 | 440 | () => { |
| 375 | 441 | landed++; |
| 376 | 442 | landedBytes += bytes; |
| 377 | forge.onLand?.({ | |
| 378 | objects: landed, | |
| 379 | bytes: landedBytes, | |
| 380 | elapsedMs: Date.now() - started, | |
| 381 | }); | |
| 443 | forge.onLand?.(now()); | |
| 382 | 444 | if (++sinceReport >= everyObjects) say(); |
| 383 | 445 | }, |
| 384 | 446 | (error) => { |
| @@ -404,10 +466,15 @@ export async function takeSnapshot( | ||
| 404 | 466 | const key = await storedChunk(store, id); |
| 405 | 467 | if (there.has(key)) { |
| 406 | 468 | found++; |
| 469 | readBytes += plain.byteLength; | |
| 407 | 470 | return id; |
| 408 | 471 | } |
| 409 | 472 | there.add(key); |
| 410 | await put(chunkKey(id), await packChunk(plain)); | |
| 473 | const body = await packChunk(plain); | |
| 474 | readBytes += plain.byteLength; | |
| 475 | queuedBytes += body.byteLength; | |
| 476 | sentChunks++; | |
| 477 | await put(chunkKey(id), body); | |
| 411 | 478 | return id; |
| 412 | 479 | }; |
| 413 | 480 | let seed: Uint8Array | undefined; |
src/snapshot/store.ts +41 −6
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | |
| 9 | 9 | import { mkdirSync, readdirSync, rmSync, statSync } from "node:fs"; |
| 10 | 10 | import { dirname, join, relative } from "node:path"; |
| 11 | import { lineMeter, measuredDeadlineMs, type LineMeter } from "./line.ts"; | |
| 11 | 12 | |
| 12 | 13 | export interface Store { |
| 13 | 14 | /** What this store is, for a page and a log; never a secret. */ |
| @@ -35,6 +36,11 @@ export interface Store { | ||
| 35 | 36 | storedKey?(key: string): Promise<string>; |
| 36 | 37 | /** True when every object is encrypted before it reaches the store (the off-site lane, #11805). */ |
| 37 | 38 | readonly sealed?: boolean; |
| 39 | /** | |
| 40 | * What the line is giving this store's puts (`line.ts`); a store that does not measure leaves it | |
| 41 | * out. | |
| 42 | */ | |
| 43 | readonly line?: LineMeter; | |
| 38 | 44 | } |
| 39 | 45 | |
| 40 | 46 | /** |
| @@ -129,6 +135,11 @@ const sizeOf = (body: Uint8Array | string): number => | ||
| 129 | 135 | * `floorBytesPerS` and `slackMs` scale the size rule, for tests that run a |
| 130 | 136 | * slow link at a thousand times its speed. |
| 131 | 137 | * |
| 138 | * A put's deadline also follows the line (#13743): its bytes at the rate the store's puts have | |
| 139 | * actually landed at (`line.ts`), over its share of the line, three times, when that is longer | |
| 140 | * than the floor rule's. The deadline is the put's own and never the run's: `bounded` alone cuts | |
| 141 | * it to a budget. | |
| 142 | * | |
| 132 | 143 | * `bounded(leftMs)` is the same store inside someone else's budget (the |
| 133 | 144 | * restore drill's): each attempt's deadline is cut to an even share of |
| 134 | 145 | * what is left after the waits still to come, and none starts once |
| @@ -144,14 +155,22 @@ export function withRetries( | ||
| 144 | 155 | floorBytesPerS?: number; |
| 145 | 156 | slackMs?: number; |
| 146 | 157 | sleep?: (ms: number) => Promise<void>; |
| 158 | line?: LineMeter; | |
| 147 | 159 | } = {}, |
| 148 | 160 | ): Store { |
| 149 | 161 | const attempts = Math.max(1, options.attempts ?? 5); |
| 150 | 162 | const baseMs = options.baseMs ?? 500; |
| 151 | 163 | const fixed = options.deadlineMs; |
| 152 | 164 | const callMs = fixed ?? CALL_DEADLINE_MS; |
| 153 | const putMs = (body: Uint8Array | string): number => | |
| 154 | fixed ?? deadlineFor(sizeOf(body), options.floorBytesPerS, options.slackMs); | |
| 165 | const line = options.line ?? lineMeter(); | |
| 166 | const slackMs = options.slackMs ?? SLACK_MS; | |
| 167 | // Read at each attempt: the line may have been measured since the last one. | |
| 168 | const putMs = (body: Uint8Array | string): number => { | |
| 169 | if (fixed !== undefined) return fixed; | |
| 170 | const bytes = sizeOf(body); | |
| 171 | const floor = deadlineFor(bytes, options.floorBytesPerS, slackMs); | |
| 172 | return Math.max(floor, slackMs + measuredDeadlineMs(bytes, line.rate(), line.inFlight() + 1)); | |
| 173 | }; | |
| 155 | 174 | const sleep = options.sleep ?? ((ms) => Bun.sleep(ms)); |
| 156 | 175 | // The waits still to come once attempt `n` has failed, through the last. |
| 157 | 176 | const waitsFrom = (n: number): number => { |
| @@ -161,7 +180,7 @@ export function withRetries( | ||
| 161 | 180 | }; |
| 162 | 181 | async function tried<T>( |
| 163 | 182 | what: string, |
| 164 | deadlineMs: number, | |
| 183 | deadlineMs: number | (() => number), | |
| 165 | 184 | fn: () => Promise<T>, |
| 166 | 185 | leftMs?: () => number, |
| 167 | 186 | ): Promise<T> { |
| @@ -170,15 +189,16 @@ export function withRetries( | ||
| 170 | 189 | for (let n = 0; n < attempts; n++) { |
| 171 | 190 | // Bounded, each attempt gets an even share of what is left after the |
| 172 | 191 | // waits to come, so deadlines × attempts never exceed it. |
| 192 | const own = typeof deadlineMs === "number" ? deadlineMs : deadlineMs(); | |
| 173 | 193 | const share = |
| 174 | leftMs === undefined ? deadlineMs : Math.floor((leftMs() - waitsFrom(n)) / (attempts - n)); | |
| 194 | leftMs === undefined ? own : Math.floor((leftMs() - waitsFrom(n)) / (attempts - n)); | |
| 175 | 195 | if (share < 1) { |
| 176 | 196 | last ??= new Error("no time left in its budget"); |
| 177 | 197 | break; |
| 178 | 198 | } |
| 179 | 199 | made++; |
| 180 | 200 | try { |
| 181 | return await answered(fn(), Math.min(deadlineMs, share)); | |
| 201 | return await answered(fn(), Math.min(own, share)); | |
| 182 | 202 | } catch (error) { |
| 183 | 203 | last = error; |
| 184 | 204 | if (n + 1 < attempts) await sleep(baseMs * 2 ** n); |
| @@ -192,13 +212,28 @@ export function withRetries( | ||
| 192 | 212 | where: inner.where, |
| 193 | 213 | ...(inner.dir === undefined ? {} : { dir: inner.dir }), |
| 194 | 214 | put: (key, body) => |
| 195 | tried(`put ${key}`, putMs(body), () => inner.put(key, body), leftMs), | |
| 215 | tried( | |
| 216 | `put ${key}`, | |
| 217 | () => putMs(body), | |
| 218 | async () => { | |
| 219 | const end = line.begin(); | |
| 220 | let landed = 0; | |
| 221 | try { | |
| 222 | await inner.put(key, body); | |
| 223 | landed = sizeOf(body); | |
| 224 | } finally { | |
| 225 | end(landed); | |
| 226 | } | |
| 227 | }, | |
| 228 | leftMs, | |
| 229 | ), | |
| 196 | 230 | get: (key) => tried(`get ${key}`, callMs, () => inner.get(key), leftMs), |
| 197 | 231 | list: (prefix) => tried(`list ${prefix}`, callMs, () => inner.list(prefix), leftMs), |
| 198 | 232 | names: (prefix) => |
| 199 | 233 | tried(`names ${prefix}`, callMs, () => inner.names(prefix), leftMs), |
| 200 | 234 | delete: (key) => tried(`delete ${key}`, callMs, () => inner.delete(key), leftMs), |
| 201 | 235 | bounded: (left) => view(left), |
| 236 | line, | |
| 202 | 237 | }); |
| 203 | 238 | return view(); |
| 204 | 239 | } |
src/tui/screens/resilience.ts +3
| @@ -24,6 +24,7 @@ | ||
| 24 | 24 | * `Flaky:` and `Red:` issues link to. |
| 25 | 25 | */ |
| 26 | 26 | |
| 27 | import { progressWords } from "../../snapshot/snapshot.ts"; | |
| 27 | 28 | import { followDrill, mirrorNow, recentSnapshots, snapshotNow, startDrillNow } from "../../actions/backups_now.ts"; |
| 28 | 29 | import { readResilience } from "../../checks/resilience.ts"; |
| 29 | 30 | import { |
| @@ -297,6 +298,8 @@ export function resilienceScreen(f: Session, ctx: Ctx): Screen { | ||
| 297 | 298 | out.push(""); |
| 298 | 299 | out.push(...wrap(destinationLine(f), INNER), ...wrap(mirrorLine(f), INNER)); |
| 299 | 300 | if (mayChange()) out.push("d sets where snapshots go, x forgets it.", "m sets the mirror, f forgets it.", "t tests where snapshots go."); |
| 301 | const running = snapshotsOf(f.app)?.progress(); | |
| 302 | if (running !== undefined) out.push("", ...wrap(`A snapshot is being taken: ${progressWords(running)}.`, INNER)); | |
| 300 | 303 | out.push("", "Snapshots, newest first:"); |
| 301 | 304 | if (snapshots === undefined) out.push(" reading…"); |
| 302 | 305 | else if (snapshots === "unreadable") out.push(" the snapshots could not be read"); |
src/web/pages/backups.ts +3 −2
| @@ -23,7 +23,7 @@ import { forgetMirror, saveMirror } from "../../actions/mirror.ts"; | ||
| 23 | 23 | import { listResilience, type ResilienceRow } from "../../actions/resilience.ts"; |
| 24 | 24 | import { mirrorSealedSet, mirrorNeed, MIRROR_SEALED } from "../../snapshot/mirror.ts"; |
| 25 | 25 | import { resolveSnapshots, sealedSet } from "../../snapshot/destination.ts"; |
| 26 | import { keyOf } from "../../snapshot/snapshot.ts"; | |
| 26 | import { keyOf, progressFacts } from "../../snapshot/snapshot.ts"; | |
| 27 | 27 | import { formatBytes, formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts"; |
| 28 | 28 | import { costFacts } from "../../snapshot/drill_cost.ts"; |
| 29 | 29 | import { lastDrill } from "../../snapshot/drills.ts"; |
| @@ -364,7 +364,8 @@ async function movingPart( | ||
| 364 | 364 | ${badge("Running", { icon: "clock", variant: "secondary" })} A snapshot is being taken: ${p.objects} objects, |
| 365 | 365 | ${fileSize(p.bytes)} so far, ${Math.round(p.elapsedMs / 1000)} s |
| 366 | 366 | in. |
| 367 | </p>`, | |
| 367 | </p> | |
| 368 | ${progressFacts(p).length === 0 ? null : html`<dl class="status-rows" id="snapshot-progress">${progressFacts(p).map((f) => html`<div class="status-row"><dt>${f.label}</dt><dd>${f.value}</dd></div>`)}</dl>`}`, | |
| 368 | 369 | )} |
| 369 | 370 | ${when( |
| 370 | 371 | failing, |
test/snapshot-slow-line.test.ts added +174
| @@ -0,0 +1,174 @@ | ||
| 1 | /** | |
| 2 | * A backup finishes on a slow line (#13743). Behind a 1.2 to 1.75 Mbit/s share of a home uplink a | |
| 3 | * 4 MB chunk took a minute and a half, the fixed rule gave it 66 s, and eight puts at once each got | |
| 4 | * a fraction of the line: a first sealed snapshot could never finish. Proved here: | |
| 5 | * | |
| 6 | * - the line meter reads the rate puts actually landed at, over the time any was in flight; | |
| 7 | * - a put's deadline follows its size and that rate, over the floor's, and a stalled one still | |
| 8 | * fails; | |
| 9 | * - the number of puts at once follows the rate: one or two on a slow line; | |
| 10 | * - a run cut short keeps what it sent: the next sends only the rest, and says "N of M already | |
| 11 | * there"; | |
| 12 | * - progress names bytes to go, the rate and when it will be done. | |
| 13 | */ | |
| 14 | import { Database } from "bun:sqlite"; | |
| 15 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 16 | import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs"; | |
| 17 | import { rm } from "node:fs/promises"; | |
| 18 | import { tmpdir } from "node:os"; | |
| 19 | import { join } from "node:path"; | |
| 20 | import { createApp, type App } from "../src/app.ts"; | |
| 21 | import { lineMeter, measuredDeadlineMs, putsAtOnce } from "../src/snapshot/line.ts"; | |
| 22 | import { progressFacts, progressWords, takeSnapshot, type Progress } from "../src/snapshot/snapshot.ts"; | |
| 23 | import { dirStore, withRetries, type Store } from "../src/snapshot/store.ts"; | |
| 24 | import { quietLogs } from "./helpers/globals.ts"; | |
| 25 | ||
| 26 | quietLogs(); | |
| 27 | ||
| 28 | test("the line meter says nothing until enough moved, then bytes over busy time", () => { | |
| 29 | let t = 0; | |
| 30 | const m = lineMeter(() => t); | |
| 31 | expect(m.rate()).toBeUndefined(); | |
| 32 | const end = m.begin(); | |
| 33 | t = 10_000; | |
| 34 | end(1_000_000); | |
| 35 | expect(m.rate()).toBe(100_000); | |
| 36 | // Idle time is not the line's slowness. | |
| 37 | t = 1_000_000; | |
| 38 | expect(m.rate()).toBe(100_000); | |
| 39 | // A failed put spends time and lands nothing: the rate falls. | |
| 40 | const failed = m.begin(); | |
| 41 | t += 10_000; | |
| 42 | failed(0); | |
| 43 | expect(m.rate()).toBe(50_000); | |
| 44 | }); | |
| 45 | ||
| 46 | test("puts at once follow the rate: one or two on a slow line, the most on a fast one", () => { | |
| 47 | expect(putsAtOnce(undefined, 8)).toBe(2); | |
| 48 | expect(putsAtOnce(200_000, 8)).toBe(1); | |
| 49 | expect(putsAtOnce(400_000, 8)).toBe(3); | |
| 50 | expect(putsAtOnce(50_000_000, 8)).toBe(8); | |
| 51 | expect(measuredDeadlineMs(4_000_000, undefined, 1)).toBe(0); | |
| 52 | // 4 MB alone on 200 kB/s is 20 s; three times over is a minute; sharing doubles it. | |
| 53 | expect(measuredDeadlineMs(4_000_000, 200_000, 1)).toBe(60_000); | |
| 54 | expect(measuredDeadlineMs(4_000_000, 200_000, 2)).toBe(120_000); | |
| 55 | }); | |
| 56 | ||
| 57 | function slowPut(ms: number): Store { | |
| 58 | const inner = dirStore(mkdtempSync(join(tmpdir(), "forge-slow-"))); | |
| 59 | return { ...inner, put: async (k, b) => (await Bun.sleep(ms), inner.put(k, b)) }; // real-timer: simulates a slow put on the real clock | |
| 60 | } | |
| 61 | ||
| 62 | test("a put that outlasts the floor's deadline lands when the measured rate says it needs the time", async () => { | |
| 63 | const body = new Uint8Array(100); | |
| 64 | const floor = { floorBytesPerS: 1e9, slackMs: 40, attempts: 1, baseMs: 1 }; | |
| 65 | // Without a measurement the floor stands alone: 150 ms against 40 ms fails. | |
| 66 | await expect(withRetries(slowPut(150), floor).put("a", body)).rejects.toThrow(/no answer in 4\d ms/); | |
| 67 | // The line gave 1,000 B/s: a hundred bytes alone need 300 ms with margin. | |
| 68 | const measured = { rate: () => 1000 as number | undefined, inFlight: () => 0, begin: () => () => {} }; | |
| 69 | await withRetries(slowPut(150), { ...floor, line: measured }).put("a", body); | |
| 70 | // A put that moves nothing still times out, at the measured deadline. | |
| 71 | await expect(withRetries(slowPut(2000), { ...floor, line: measured }).put("a", body)).rejects.toThrow(/no answer in 3\d\d ms/); | |
| 72 | }); | |
| 73 | ||
| 74 | let root: string; | |
| 75 | let app: App; | |
| 76 | beforeAll(() => { | |
| 77 | root = mkdtempSync(join(tmpdir(), "forge-slowline-")); | |
| 78 | app = createApp({ dataDir: join(root, "live"), port: 0 }); | |
| 79 | writeFileSync(join(root, "live", "big.bin"), ""); | |
| 80 | mkdirSync(join(root, "live", "ssh"), { recursive: true }); | |
| 81 | const words = new Uint32Array(1024 * 256); | |
| 82 | let x = 12345; | |
| 83 | for (let i = 0; i < words.length; i++) { | |
| 84 | x ^= x << 13; | |
| 85 | x ^= x >>> 17; | |
| 86 | x ^= x << 5; | |
| 87 | words[i] = x >>> 0; | |
| 88 | } | |
| 89 | writeFileSync(join(root, "live", "ssh", "pack.bin"), new Uint8Array(words.buffer)); | |
| 90 | }); | |
| 91 | afterAll(async () => { | |
| 92 | app.db.close(); | |
| 93 | await rm(root, { recursive: true, force: true }); | |
| 94 | }); | |
| 95 | ||
| 96 | const forge = (extra: Record<string, unknown> = {}) => ({ | |
| 97 | db: app.db as Database, | |
| 98 | dataDir: app.config.dataDir, | |
| 99 | reposRoot: app.reposRoot, | |
| 100 | version: "t", | |
| 101 | contentCut: { min: 16 * 1024, bits: 15, max: 64 * 1024 }, | |
| 102 | ...extra, | |
| 103 | }); | |
| 104 | ||
| 105 | /** A directory store that counts puts and the most at once, on a line that reads as `rate`. */ | |
| 106 | function metered(rate: number | undefined, failAfter = Infinity, inner = dirStore(mkdtempSync(join(tmpdir(), "forge-metered-")))) { | |
| 107 | const seen = { puts: [] as string[], most: 0, open: 0 }; | |
| 108 | const store: Store = { | |
| 109 | ...inner, | |
| 110 | line: { rate: () => rate, inFlight: () => seen.open, begin: () => () => {} }, | |
| 111 | async put(key, body) { | |
| 112 | if (key.startsWith("chunks/")) { | |
| 113 | if (seen.puts.length >= failAfter) throw new Error("the line dropped"); | |
| 114 | seen.puts.push(key); | |
| 115 | } | |
| 116 | seen.open++; | |
| 117 | seen.most = Math.max(seen.most, seen.open); | |
| 118 | await Bun.sleep(3); // real-timer: lets the in-flight puts interleave | |
| 119 | seen.open--; | |
| 120 | await inner.put(key, body); | |
| 121 | }, | |
| 122 | }; | |
| 123 | return { store, seen }; | |
| 124 | } | |
| 125 | ||
| 126 | test("on a slow line one put goes at a time; on a fast one several", async () => { | |
| 127 | const slow = metered(150_000); | |
| 128 | await takeSnapshot(forge(), slow.store, new Date("2026-11-01T08:00:00Z")); | |
| 129 | expect(slow.seen.puts.length).toBeGreaterThan(8); | |
| 130 | expect(slow.seen.most).toBe(1); | |
| 131 | const fast = metered(50_000_000); | |
| 132 | await takeSnapshot(forge(), fast.store, new Date("2026-11-01T08:00:00Z")); | |
| 133 | expect(fast.seen.most).toBeGreaterThan(2); | |
| 134 | }); | |
| 135 | ||
| 136 | test("a run that fails keeps what it sent: the next sends only the rest and says how many were there", async () => { | |
| 137 | const first = metered(undefined, 6); | |
| 138 | await expect(takeSnapshot(forge(), first.store, new Date("2026-11-01T08:00:00Z"))).rejects.toThrow(/could not put/); | |
| 139 | const kept = first.seen.puts.length; | |
| 140 | expect(kept).toBeGreaterThanOrEqual(5); | |
| 141 | const seen: Progress[] = []; | |
| 142 | // The same bucket, its fault gone: the chunks the failed run sent are there. | |
| 143 | const store = metered(undefined, Infinity, dirStore(first.store.dir!)).store; | |
| 144 | const m = await takeSnapshot( | |
| 145 | { ...forge({ expectedBytes: 1_300_000 }), progressEvery: { objects: 1, ms: 60_000 }, onProgress: (p: Progress) => seen.push(p) }, | |
| 146 | store, | |
| 147 | new Date("2026-11-01T09:00:00Z"), | |
| 148 | ); | |
| 149 | expect(m.sent?.held).toBeGreaterThanOrEqual(kept - 1); | |
| 150 | const last = seen.at(-1)!; | |
| 151 | expect(last.already).toBeGreaterThan(0); | |
| 152 | expect(progressFacts(last).find((f) => f.label === "Already there")?.value).toMatch(/\d+ of \d+ chunks/); | |
| 153 | }); | |
| 154 | ||
| 155 | test("progress names bytes to go, the rate, and when it will be done", () => { | |
| 156 | const p = { | |
| 157 | objects: 40, | |
| 158 | bytes: 160_000_000, | |
| 159 | elapsedMs: 3_600_000, | |
| 160 | already: 3, | |
| 161 | seen: 43, | |
| 162 | bytesPerS: 200_000, | |
| 163 | toGoBytes: 900_000_000, | |
| 164 | etaMs: 4_500_000, | |
| 165 | }; | |
| 166 | const facts = Object.fromEntries(progressFacts(p).map((f) => [f.label, f.value])); | |
| 167 | expect(facts["Already there"]).toBe("3 of 43 chunks"); | |
| 168 | expect(facts["Still to send"]).toBe("858.3 MiB"); | |
| 169 | expect(facts["Line speed"]).toBe("1.60 Mbit/s"); | |
| 170 | expect(facts["Done in about"]).toBe("75 min"); | |
| 171 | expect(progressWords(p)).toBe("152.6 MiB sent, 858.3 MiB left, done in about 75 min"); | |
| 172 | // What is not known is left out, not guessed. | |
| 173 | expect(progressFacts({ objects: 1, bytes: 10, elapsedMs: 1 }).map((f) => f.label)).toEqual([]); | |
| 174 | }); |
new above
offline