Public: anyone can read this; the crew can change it.
One BlobStore: promote src/snapshot/store.ts to a shared module with streaming, range reads and presigned URLs #4544
mason/forge:mason/4489-one-blobstore-promote-src-snapshot-store → main 3 commits fork
49 files
changed +1036
−334
CLAUDE.md +3
| @@ -201,6 +201,9 @@ src/ | ||
| 201 | 201 | actions/); actions/fleet.ts plans, applies, keeps the receipt |
| 202 | 202 | (core/fleet_ops.ts) and undoes |
| 203 | 203 | runs/ log (one file per run, appended live, tailed by listeners) |
| 204 | blob/ store (the one BlobStore, docs/BLOBS.md: a bucket over S3 or a | |
| 205 | directory; streamed puts, ranges, presigned URLs, a class | |
| 206 | hint; snapshots use its small `Store` shape) | |
| 204 | 207 | crew/ agents as child processes: brief (the system + task prompts: |
| 205 | 208 | the golden path, written down), claude (the headless adapter: |
| 206 | 209 | argv, stream-json, outcome), sandbox (what the agent's shell |
docs/BLOBS.md added +47
| @@ -0,0 +1,47 @@ | ||
| 1 | # Blobs | |
| 2 | ||
| 3 | Big, immutable things go to one store, `src/blob/store.ts` (step 2 of | |
| 4 | #4481). Its keys are paths and its contents are bytes; it has no other | |
| 5 | idea of structure. The refs and metadata that name those bytes stay in | |
| 6 | the database. | |
| 7 | ||
| 8 | ## Two shapes, one store | |
| 9 | ||
| 10 | - **`Store`** is the small shape snapshots were written against: put, get, | |
| 11 | list, names, delete. Snapshots, the mirror and the drill still take | |
| 12 | exactly this, unchanged (`docs/SNAPSHOT.md`). | |
| 13 | - **`BlobStore`** is the same store, with what a large file needs: | |
| 14 | - `put(key, Blob)`: a `Bun.file` is streamed from disk, never read whole | |
| 15 | into memory. It can be retried, because it is read anew each time. | |
| 16 | - `putStream(key, stream)`: a body of unknown length. A bucket takes it | |
| 17 | multipart. A directory writes it under a temporary name and renames it | |
| 18 | into place, so a reader never sees half of it. It is tried only once, | |
| 19 | because a stream cannot be replayed. | |
| 20 | - `size(key)`, `getRange(key, { start, end })` and `getStream(key, range?)`. | |
| 21 | `end` is exclusive. A range past the end of the object is empty, and an | |
| 22 | absent key is `undefined`, the same answer on a bucket as on a | |
| 23 | directory. | |
| 24 | - `presign(key, { method: "GET" | "PUT", expiresInS })`: a URL straight to | |
| 25 | the bucket, signed locally with no request. A directory has none, so its | |
| 26 | caller serves the bytes itself. | |
| 27 | - A **storage-class hint** on every put: `hot` (the default, which sends | |
| 28 | nothing), `infrequent` (`STANDARD_IA`, which R2 and S3 accept) or | |
| 29 | `archive` (`DEEP_ARCHIVE`, S3 only). | |
| 30 | ||
| 31 | `s3Store` (any S3-compatible endpoint: R2, S3, B2) and `dirStore` are both | |
| 32 | `BlobStore`s. `withRetries` keeps one a `BlobStore`: sizes, ranges and | |
| 33 | opening a stream are retried the way a get is, and a `Blob` put gets a | |
| 34 | deadline that follows its size. | |
| 35 | ||
| 36 | ## At the boundary | |
| 37 | ||
| 38 | A key with `..` or `.` segments, a leading `/` or a control byte is | |
| 39 | refused. So is a range that is not whole, non-negative bytes in order, and | |
| 40 | a presign that is not a GET or PUT valid for 1 s to 7 days. The bucket's | |
| 41 | secret never appears in a presigned URL; only its signature does. | |
| 42 | ||
| 43 | ## Not here yet | |
| 44 | ||
| 45 | Archiving finished run logs and attachments onto the store, big git blobs | |
| 46 | by OID and the cold-tier lifecycle are later steps of #4481, each with | |
| 47 | its own issue. |
docs/SNAPSHOT.md +1 −1
| @@ -115,7 +115,7 @@ mirror is copied from the primary store, not written in parallel, so a snapshot | ||
| 115 | 115 | that never landed on the primary is not seen by it. |
| 116 | 116 | |
| 117 | 117 | The store is `Bun.S3Client` or a directory behind one small interface |
| 118 | (`src/snapshot/store.ts`): put, get, list, delete. Nothing else. Every call | |
| 118 | (`src/blob/store.ts`, shared; see `docs/BLOBS.md`): put, get, list, delete. Nothing else. Every call | |
| 119 | 119 | is tried five times; an attempt that has not answered by its deadline counts |
| 120 | 120 | as failed. A put's deadline follows its size: its bytes at a floor of |
| 121 | 121 | 5 Mbit/s, plus 60 s. A fixed 180 s once failed every attempt at a 334 MB |
src/actions/destination.ts +1 −1
| @@ -14,7 +14,7 @@ import { | ||
| 14 | 14 | SEALED, |
| 15 | 15 | testTarget, |
| 16 | 16 | } from "../snapshot/destination.ts"; |
| 17 | import { s3FromEnv, type S3Target } from "../snapshot/store.ts"; | |
| 17 | import { s3FromEnv, type S3Target } from "../blob/store.ts"; | |
| 18 | 18 | import { snapshotsOf } from "../snapshot/service.ts"; |
| 19 | 19 | import type { User } from "../core/users.ts"; |
| 20 | 20 | import { err, ok, type Result } from "../kernel/result.ts"; |
src/blob/store.ts added +538
| @@ -0,0 +1,538 @@ | ||
| 1 | /** | |
| 2 | * Where the forge's big, immutable things go (docs/BLOBS.md). Two places, | |
| 3 | * one shape: a bucket over S3 (the platform gives every app one, mirrored | |
| 4 | * off the machine by lanes the forge never sees; R2, S3 and B2 all speak | |
| 5 | * it) or a directory (a forge on a desk, a test). Keys are paths; the | |
| 6 | * store keeps no other idea of structure, so a snapshot is exactly the | |
| 7 | * files it lists and a restore is exactly reading them back. | |
| 8 | * | |
| 9 | * `Store` is the small shape snapshots were written against; `BlobStore` | |
| 10 | * is the same store with what a large file needs: a body streamed in from | |
| 11 | * disk or a pipe, a byte range read back, a URL a client fetches from the | |
| 12 | * bucket itself, and a hint of how cold the object may be kept. | |
| 13 | */ | |
| 14 | ||
| 15 | import { mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs"; | |
| 16 | import { dirname, join, relative } from "node:path"; | |
| 17 | ||
| 18 | /** | |
| 19 | * How often an object is expected to be read back, as a hint the bucket | |
| 20 | * may honour: `hot` (the default, sends nothing), `infrequent` (R2 and S3 | |
| 21 | * Infrequent Access; a 30-day minimum), `archive` (S3 Glacier Deep | |
| 22 | * Archive; S3 only, 180-day minimum, hours to read back). A directory has | |
| 23 | * no classes and ignores it. | |
| 24 | */ | |
| 25 | export type StorageClass = "hot" | "infrequent" | "archive"; | |
| 26 | ||
| 27 | export interface PutOptions { | |
| 28 | readonly storageClass?: StorageClass; | |
| 29 | /** The object's media type; `application/octet-stream` when left out. */ | |
| 30 | readonly type?: string; | |
| 31 | } | |
| 32 | ||
| 33 | /** Bytes `start` up to, not including, `end` (the object's end when left out). */ | |
| 34 | export interface ByteRange { | |
| 35 | readonly start: number; | |
| 36 | readonly end?: number; | |
| 37 | } | |
| 38 | ||
| 39 | /** A URL that lets its holder GET or PUT one key, until it expires. */ | |
| 40 | export interface PresignRequest { | |
| 41 | readonly method: "GET" | "PUT"; | |
| 42 | /** Seconds the URL stays good: 1 to 604800 (seven days, S3's limit). */ | |
| 43 | readonly expiresInS: number; | |
| 44 | /** For a PUT, the class the object is written in. */ | |
| 45 | readonly storageClass?: StorageClass; | |
| 46 | } | |
| 47 | ||
| 48 | export interface Store { | |
| 49 | /** What this store is, for a page and a log; never a secret. */ | |
| 50 | readonly where: string; | |
| 51 | /** The directory it writes under, when it is one; a bucket has none. */ | |
| 52 | readonly dir?: string; | |
| 53 | put(key: string, body: Uint8Array | string, options?: PutOptions): Promise<void>; | |
| 54 | get(key: string): Promise<Uint8Array | undefined>; | |
| 55 | /** Every key under `prefix`, in no particular order. */ | |
| 56 | list(prefix: string): Promise<string[]>; | |
| 57 | /** The next path segment under `prefix`: the "directories", not the keys. */ | |
| 58 | names(prefix: string): Promise<string[]>; | |
| 59 | delete(key: string): Promise<void>; | |
| 60 | /** | |
| 61 | * The same store, every call of which gives up within `leftMs()`: its | |
| 62 | * deadlines and retries together never outlast what is left. A store | |
| 63 | * that does not retry has nothing to bound and leaves this out. | |
| 64 | */ | |
| 65 | bounded?(leftMs: () => number): Store; | |
| 66 | } | |
| 67 | ||
| 68 | export interface BlobStore extends Store { | |
| 69 | /** | |
| 70 | * A `Blob` (a `Bun.file` most of all) is streamed from where it lies, | |
| 71 | * never read whole into memory, and can be tried again: it is read anew. | |
| 72 | */ | |
| 73 | put(key: string, body: Uint8Array | string | Blob, options?: PutOptions): Promise<void>; | |
| 74 | /** | |
| 75 | * A body of unknown length, streamed in (multipart on a bucket). Tried | |
| 76 | * once: a stream cannot be read twice, so its caller holds the source. | |
| 77 | */ | |
| 78 | putStream( | |
| 79 | key: string, | |
| 80 | body: ReadableStream<Uint8Array>, | |
| 81 | options?: PutOptions, | |
| 82 | ): Promise<void>; | |
| 83 | /** The object's size in bytes; undefined when there is no such key. */ | |
| 84 | size(key: string): Promise<number | undefined>; | |
| 85 | /** Some of the object's bytes; empty past its end, undefined when absent. */ | |
| 86 | getRange(key: string, range: ByteRange): Promise<Uint8Array | undefined>; | |
| 87 | /** The object, or a range of it, as a stream; undefined when absent. */ | |
| 88 | getStream( | |
| 89 | key: string, | |
| 90 | range?: ByteRange, | |
| 91 | ): Promise<ReadableStream<Uint8Array> | undefined>; | |
| 92 | /** | |
| 93 | * A URL straight to the object, signed here with no request made. A | |
| 94 | * directory has none and leaves this out: its caller serves the bytes. | |
| 95 | */ | |
| 96 | presign?(key: string, request: PresignRequest): string; | |
| 97 | bounded?(leftMs: () => number): BlobStore; | |
| 98 | } | |
| 99 | ||
| 100 | /** Whether `store` is a whole `BlobStore`, not only the snapshot shape. */ | |
| 101 | export const isBlobStore = (store: Store): store is BlobStore => | |
| 102 | "getRange" in store && "putStream" in store && "size" in store; | |
| 103 | ||
| 104 | /** The longest a presigned URL may live: S3's own limit. */ | |
| 105 | export const PRESIGN_MAX_S = 7 * 24 * 60 * 60; | |
| 106 | ||
| 107 | /** Refuse a range that is not whole, non-negative bytes in order. */ | |
| 108 | export function checkRange(range: ByteRange): void { | |
| 109 | const { start, end } = range; | |
| 110 | if (!Number.isSafeInteger(start) || start < 0) | |
| 111 | throw new Error(`refusing range start ${start}`); | |
| 112 | if (end !== undefined && (!Number.isSafeInteger(end) || end < start)) | |
| 113 | throw new Error(`refusing range end ${end}`); | |
| 114 | } | |
| 115 | ||
| 116 | /** Refuse a presign that is not a GET or PUT for a whole number of seconds within S3's limit. */ | |
| 117 | export function checkPresign(request: PresignRequest): void { | |
| 118 | if (request.method !== "GET" && request.method !== "PUT") | |
| 119 | throw new Error(`refusing presign method ${String(request.method)}`); | |
| 120 | const s = request.expiresInS; | |
| 121 | if (!Number.isSafeInteger(s) || s < 1 || s > PRESIGN_MAX_S) | |
| 122 | throw new Error(`refusing presign expiry ${s}`); | |
| 123 | } | |
| 124 | ||
| 125 | /** | |
| 126 | * Refuse a key that is not a plain relative path: empty, absolute, with a | |
| 127 | * `..` or `.` segment, or a control byte. A directory | |
| 128 | * would resolve one outside its root; a bucket would sign it as given. | |
| 129 | */ | |
| 130 | export function checkKey(key: string): void { | |
| 131 | if ( | |
| 132 | key === "" || | |
| 133 | key.startsWith("/") || | |
| 134 | key.split("/").some((s) => s === ".." || s === ".") || | |
| 135 | /[\u0000-\u001f\u007f]/.test(key) | |
| 136 | ) | |
| 137 | throw new Error(`refusing key ${JSON.stringify(key)}`); | |
| 138 | } | |
| 139 | ||
| 140 | /** The S3 `x-amz-storage-class` a hint asks for; none for `hot`, so a default put sends exactly what it always did. */ | |
| 141 | export const s3StorageClass = ( | |
| 142 | hint: StorageClass | undefined, | |
| 143 | ): "STANDARD_IA" | "DEEP_ARCHIVE" | undefined => | |
| 144 | hint === "infrequent" ? "STANDARD_IA" : hint === "archive" ? "DEEP_ARCHIVE" : undefined; | |
| 145 | ||
| 146 | /** The S3 credentials as the platform hands them to an app. */ | |
| 147 | export interface S3Target { | |
| 148 | readonly endpoint: string; | |
| 149 | readonly bucket: string; | |
| 150 | readonly accessKeyId: string; | |
| 151 | readonly secretAccessKey: string; | |
| 152 | readonly region?: string | undefined; | |
| 153 | } | |
| 154 | ||
| 155 | /** Read the platform's `S3_*` variables; undefined when the app has no bucket. */ | |
| 156 | export function s3FromEnv( | |
| 157 | env: Record<string, string | undefined> = process.env, | |
| 158 | ): S3Target | undefined { | |
| 159 | const endpoint = env["S3_ENDPOINT"]?.trim(); | |
| 160 | const bucket = env["S3_BUCKET"]?.trim(); | |
| 161 | const accessKeyId = env["S3_ACCESS_KEY"]?.trim(); | |
| 162 | const secretAccessKey = env["S3_SECRET_KEY"]?.trim(); | |
| 163 | if (!endpoint || !bucket || !accessKeyId || !secretAccessKey) | |
| 164 | return undefined; | |
| 165 | return { | |
| 166 | endpoint, | |
| 167 | bucket, | |
| 168 | accessKeyId, | |
| 169 | secretAccessKey, | |
| 170 | region: env["S3_REGION"]?.trim() || undefined, | |
| 171 | }; | |
| 172 | } | |
| 173 | ||
| 174 | /** The slowest uplink a put may assume: 5 Mbit/s, a third of the home line it runs behind. */ | |
| 175 | export const FLOOR_BYTES_PER_S = 5_000_000 / 8; | |
| 176 | /** What every attempt gets on top of its bytes' time: the handshake, the server's thought, a queue ahead of it. */ | |
| 177 | export const SLACK_MS = 60_000; | |
| 178 | /** How long a read, a listing or a delete may go unanswered. */ | |
| 179 | export const CALL_DEADLINE_MS = 180_000; | |
| 180 | ||
| 181 | /** How long a put of `bytes` may take before it counts as failed: its bytes at the floor rate, plus slack. */ | |
| 182 | export const deadlineFor = ( | |
| 183 | bytes: number, | |
| 184 | floorBytesPerS = FLOOR_BYTES_PER_S, | |
| 185 | slackMs = SLACK_MS, | |
| 186 | ): number => slackMs + Math.ceil((bytes / floorBytesPerS) * 1000); | |
| 187 | ||
| 188 | const sizeOf = (body: Uint8Array | string | Blob): number => | |
| 189 | typeof body === "string" | |
| 190 | ? Buffer.byteLength(body) | |
| 191 | : body instanceof Blob | |
| 192 | ? body.size | |
| 193 | : body.byteLength; | |
| 194 | ||
| 195 | /** | |
| 196 | * A store that tries again. Object stores answer a burst, a flake or a | |
| 197 | * moment of maintenance with a 5xx — Backblaze says so in its own docs — | |
| 198 | * and one such answer in a thousand puts used to fail the whole snapshot | |
| 199 | * with "S3Error: an unexpected error has occurred" and no key named. Every | |
| 200 | * call is tried `attempts` times with doubling waits, and what finally | |
| 201 | * fails says which key it was. An attempt that has not answered within its | |
| 202 | * deadline counts as failed too: a connection that died quietly never | |
| 203 | * settles on its own, and once held a snapshot on its first object for an | |
| 204 | * hour with the network fine (2026-10-13), so the next attempt opens a | |
| 205 | * fresh request instead. | |
| 206 | * | |
| 207 | * A put's deadline follows its size (`deadlineFor`): a fixed 180 s once | |
| 208 | * failed every attempt at a 334 MB database behind a 15 Mbit/s uplink | |
| 209 | * (2026-10-17). `deadlineMs` fixes every deadline instead, for tests; | |
| 210 | * `floorBytesPerS` and `slackMs` scale the size rule, for tests that run a | |
| 211 | * slow link at a thousand times its speed. | |
| 212 | * | |
| 213 | * `bounded(leftMs)` is the same store inside someone else's budget (the | |
| 214 | * restore drill's): each attempt's deadline is cut to an even share of | |
| 215 | * what is left after the waits still to come, and none starts once | |
| 216 | * nothing is, so a stalled store fails within the budget instead of | |
| 217 | * holding five 180 s attempts past it. | |
| 218 | * | |
| 219 | * A `BlobStore` comes back a `BlobStore`: its sizes and ranges are tried | |
| 220 | * as a get is, opening a stream is tried but not the reading of it, and a | |
| 221 | * `putStream` is passed through once with its key named when it fails (a | |
| 222 | * stream cannot be replayed). Presigning makes no request, so is not tried. | |
| 223 | */ | |
| 224 | export interface RetryOptions { | |
| 225 | attempts?: number; | |
| 226 | baseMs?: number; | |
| 227 | deadlineMs?: number; | |
| 228 | floorBytesPerS?: number; | |
| 229 | slackMs?: number; | |
| 230 | sleep?: (ms: number) => Promise<void>; | |
| 231 | } | |
| 232 | export function withRetries(inner: BlobStore, options?: RetryOptions): BlobStore; | |
| 233 | export function withRetries(inner: Store, options?: RetryOptions): Store; | |
| 234 | export function withRetries(inner: Store, options: RetryOptions = {}): Store { | |
| 235 | const attempts = Math.max(1, options.attempts ?? 5); | |
| 236 | const baseMs = options.baseMs ?? 500; | |
| 237 | const fixed = options.deadlineMs; | |
| 238 | const callMs = fixed ?? CALL_DEADLINE_MS; | |
| 239 | const putMs = (body: Uint8Array | string | Blob): number => | |
| 240 | fixed ?? deadlineFor(sizeOf(body), options.floorBytesPerS, options.slackMs); | |
| 241 | const sleep = options.sleep ?? ((ms) => Bun.sleep(ms)); | |
| 242 | // The waits still to come once attempt `n` has failed, through the last. | |
| 243 | const waitsFrom = (n: number): number => { | |
| 244 | let ms = 0; | |
| 245 | for (let k = n; k + 1 < attempts; k++) ms += baseMs * 2 ** k; | |
| 246 | return ms; | |
| 247 | }; | |
| 248 | async function tried<T>( | |
| 249 | what: string, | |
| 250 | deadlineMs: number, | |
| 251 | fn: () => Promise<T>, | |
| 252 | leftMs?: () => number, | |
| 253 | ): Promise<T> { | |
| 254 | let last: unknown; | |
| 255 | let made = 0; | |
| 256 | for (let n = 0; n < attempts; n++) { | |
| 257 | // Bounded, each attempt gets an even share of what is left after the | |
| 258 | // waits to come, so deadlines × attempts never exceed it. | |
| 259 | const share = | |
| 260 | leftMs === undefined ? deadlineMs : Math.floor((leftMs() - waitsFrom(n)) / (attempts - n)); | |
| 261 | if (share < 1) { | |
| 262 | last ??= new Error("no time left in its budget"); | |
| 263 | break; | |
| 264 | } | |
| 265 | made++; | |
| 266 | try { | |
| 267 | return await answered(fn(), Math.min(deadlineMs, share)); | |
| 268 | } catch (error) { | |
| 269 | last = error; | |
| 270 | if (n + 1 < attempts) await sleep(baseMs * 2 ** n); | |
| 271 | } | |
| 272 | } | |
| 273 | throw new Error( | |
| 274 | `${what} failed after ${made} attempt${made === 1 ? "" : "s"}: ${String(last instanceof Error ? last.message : last)}`, | |
| 275 | ); | |
| 276 | } | |
| 277 | const view = (leftMs?: () => number): Store => ({ | |
| 278 | where: inner.where, | |
| 279 | ...(inner.dir === undefined ? {} : { dir: inner.dir }), | |
| 280 | put: (key, body, put) => | |
| 281 | tried(`put ${key}`, putMs(body), () => inner.put(key, body, put), leftMs), | |
| 282 | get: (key) => tried(`get ${key}`, callMs, () => inner.get(key), leftMs), | |
| 283 | list: (prefix) => tried(`list ${prefix}`, callMs, () => inner.list(prefix), leftMs), | |
| 284 | names: (prefix) => | |
| 285 | tried(`names ${prefix}`, callMs, () => inner.names(prefix), leftMs), | |
| 286 | delete: (key) => tried(`delete ${key}`, callMs, () => inner.delete(key), leftMs), | |
| 287 | bounded: (left) => view(left), | |
| 288 | }); | |
| 289 | if (!isBlobStore(inner)) return view(); | |
| 290 | const blobView = (leftMs?: () => number): BlobStore => { | |
| 291 | const presign = inner.presign?.bind(inner); | |
| 292 | return { | |
| 293 | ...view(leftMs), | |
| 294 | put: (key, body, put) => | |
| 295 | tried(`put ${key}`, putMs(body), () => inner.put(key, body, put), leftMs), | |
| 296 | putStream: (key, body, put) => | |
| 297 | inner.putStream(key, body, put).catch((error: unknown) => { | |
| 298 | throw new Error( | |
| 299 | `put ${key} failed: ${String(error instanceof Error ? error.message : error)}`, | |
| 300 | ); | |
| 301 | }), | |
| 302 | size: (key) => tried(`size ${key}`, callMs, () => inner.size(key), leftMs), | |
| 303 | getRange: (key, range) => | |
| 304 | tried(`get ${key}`, callMs, () => inner.getRange(key, range), leftMs), | |
| 305 | getStream: (key, range) => | |
| 306 | tried(`get ${key}`, callMs, () => inner.getStream(key, range), leftMs), | |
| 307 | ...(presign === undefined ? {} : { presign }), | |
| 308 | bounded: (left) => blobView(left), | |
| 309 | }; | |
| 310 | }; | |
| 311 | return blobView(); | |
| 312 | } | |
| 313 | ||
| 314 | /** | |
| 315 | * The attempt's answer, or a rejection once `ms` have passed without one. | |
| 316 | * The abandoned attempt is left to settle on its own, its outcome ignored. | |
| 317 | */ | |
| 318 | function answered<T>(attempt: Promise<T>, ms: number): Promise<T> { | |
| 319 | let timer: ReturnType<typeof setTimeout> | undefined; | |
| 320 | const shown = ms >= 1000 ? `${Math.round(ms / 1000)} s` : `${ms} ms`; | |
| 321 | const deadline = new Promise<never>((_, reject) => { | |
| 322 | timer = setTimeout(() => reject(new Error(`no answer in ${shown}`)), ms); | |
| 323 | }); | |
| 324 | attempt.catch(() => {}); | |
| 325 | return Promise.race([attempt, deadline]).finally(() => clearTimeout(timer)); | |
| 326 | } | |
| 327 | ||
| 328 | export function s3Store(target: S3Target): BlobStore { | |
| 329 | return withRetries(rawS3Store(target)); | |
| 330 | } | |
| 331 | ||
| 332 | /** The S3 options a put sends: none at all for a default put, as before. */ | |
| 333 | const s3PutOptions = (put: PutOptions | undefined) => { | |
| 334 | const storageClass = s3StorageClass(put?.storageClass); | |
| 335 | return { | |
| 336 | ...(storageClass === undefined ? {} : { storageClass }), | |
| 337 | ...(put?.type === undefined ? {} : { type: put.type }), | |
| 338 | }; | |
| 339 | }; | |
| 340 | ||
| 341 | function rawS3Store(target: S3Target): BlobStore { | |
| 342 | const client = new Bun.S3Client({ | |
| 343 | endpoint: target.endpoint, | |
| 344 | bucket: target.bucket, | |
| 345 | accessKeyId: target.accessKeyId, | |
| 346 | secretAccessKey: target.secretAccessKey, | |
| 347 | ...(target.region === undefined ? {} : { region: target.region }), | |
| 348 | }); | |
| 349 | // One HEAD: a missing key is the client's `NoSuchKey`, anything else a failure. | |
| 350 | const size = async (key: string): Promise<number | undefined> => { | |
| 351 | try { | |
| 352 | return (await client.file(key).stat()).size; | |
| 353 | } catch (error) { | |
| 354 | if ((error as { code?: unknown }).code === "NoSuchKey") return undefined; | |
| 355 | throw error; | |
| 356 | } | |
| 357 | }; | |
| 358 | // The object's bytes in `range`, clamped to its size: a bucket answers a | |
| 359 | // range starting past the end with 416, a file with nothing. | |
| 360 | const ranged = async (key: string, range: ByteRange) => { | |
| 361 | checkKey(key); | |
| 362 | checkRange(range); | |
| 363 | const n = await size(key); | |
| 364 | if (n === undefined) return undefined; | |
| 365 | const start = Math.min(range.start, n); | |
| 366 | const end = Math.min(range.end ?? n, n); | |
| 367 | return { file: client.file(key).slice(start, end), empty: start >= end }; | |
| 368 | }; | |
| 369 | return { | |
| 370 | where: `${target.endpoint.replace(/^https?:\/\//, "")}/${target.bucket}`, | |
| 371 | async put(key, body, put) { | |
| 372 | await client.write(key, body, s3PutOptions(put)); | |
| 373 | }, | |
| 374 | async putStream(key, body, put) { | |
| 375 | checkKey(key); | |
| 376 | await client.write(key, new Response(body), s3PutOptions(put)); | |
| 377 | }, | |
| 378 | async get(key) { | |
| 379 | const file = client.file(key); | |
| 380 | if (!(await file.exists())) return undefined; | |
| 381 | return new Uint8Array(await file.arrayBuffer()); | |
| 382 | }, | |
| 383 | size(key) { | |
| 384 | checkKey(key); | |
| 385 | return size(key); | |
| 386 | }, | |
| 387 | async getRange(key, range) { | |
| 388 | const at = await ranged(key, range); | |
| 389 | if (at === undefined) return undefined; | |
| 390 | return at.empty ? new Uint8Array(0) : new Uint8Array(await at.file.arrayBuffer()); | |
| 391 | }, | |
| 392 | async getStream(key, range = { start: 0 }) { | |
| 393 | const at = await ranged(key, range); | |
| 394 | if (at === undefined) return undefined; | |
| 395 | return at.empty ? new Blob([]).stream() : at.file.stream(); | |
| 396 | }, | |
| 397 | presign(key, request) { | |
| 398 | checkKey(key); | |
| 399 | checkPresign(request); | |
| 400 | const storageClass = | |
| 401 | request.method === "PUT" ? s3StorageClass(request.storageClass) : undefined; | |
| 402 | return client.presign(key, { | |
| 403 | method: request.method, | |
| 404 | expiresIn: request.expiresInS, | |
| 405 | ...(storageClass === undefined ? {} : { storageClass }), | |
| 406 | }); | |
| 407 | }, | |
| 408 | async list(prefix) { | |
| 409 | const keys: string[] = []; | |
| 410 | let token: string | undefined; | |
| 411 | do { | |
| 412 | const page = await client.list({ | |
| 413 | prefix, | |
| 414 | maxKeys: 1000, | |
| 415 | ...(token === undefined ? {} : { continuationToken: token }), | |
| 416 | }); | |
| 417 | for (const item of page.contents ?? []) keys.push(item.key); | |
| 418 | token = page.isTruncated ? page.nextContinuationToken : undefined; | |
| 419 | } while (token !== undefined); | |
| 420 | return keys; | |
| 421 | }, | |
| 422 | async names(prefix) { | |
| 423 | const out: string[] = []; | |
| 424 | let token: string | undefined; | |
| 425 | do { | |
| 426 | const page = await client.list({ | |
| 427 | prefix, | |
| 428 | delimiter: "/", | |
| 429 | maxKeys: 1000, | |
| 430 | ...(token === undefined ? {} : { continuationToken: token }), | |
| 431 | }); | |
| 432 | for (const p of page.commonPrefixes ?? []) | |
| 433 | out.push(p.prefix.slice(prefix.length).replace(/\/$/, "")); | |
| 434 | token = page.isTruncated ? page.nextContinuationToken : undefined; | |
| 435 | } while (token !== undefined); | |
| 436 | return out; | |
| 437 | }, | |
| 438 | async delete(key) { | |
| 439 | await client.delete(key); | |
| 440 | }, | |
| 441 | }; | |
| 442 | } | |
| 443 | ||
| 444 | /** | |
| 445 | * A directory as a store: `key` is a path under it, nothing else. It has | |
| 446 | * no URL to presign and no storage classes; a streamed put lands under a | |
| 447 | * temporary name and is renamed into place, so a reader never sees half. | |
| 448 | */ | |
| 449 | export function dirStore(root: string): BlobStore { | |
| 450 | const at = (key: string): string => { | |
| 451 | if (key.startsWith("/") || key.split("/").includes("..")) | |
| 452 | throw new Error(`refusing key ${key}`); | |
| 453 | return join(root, key); | |
| 454 | }; | |
| 455 | const walk = (dir: string, out: string[]): void => { | |
| 456 | let entries: string[]; | |
| 457 | try { | |
| 458 | entries = readdirSync(dir); | |
| 459 | } catch { | |
| 460 | return; | |
| 461 | } | |
| 462 | for (const name of entries) { | |
| 463 | const path = join(dir, name); | |
| 464 | if (statSync(path).isDirectory()) walk(path, out); | |
| 465 | else out.push(relative(root, path)); | |
| 466 | } | |
| 467 | }; | |
| 468 | const file = async (key: string) => { | |
| 469 | checkKey(key); | |
| 470 | const f = Bun.file(at(key)); | |
| 471 | return (await f.exists()) ? f : undefined; | |
| 472 | }; | |
| 473 | return { | |
| 474 | where: root, | |
| 475 | dir: root, | |
| 476 | async put(key, body) { | |
| 477 | const path = at(key); | |
| 478 | mkdirSync(dirname(path), { recursive: true }); | |
| 479 | await Bun.write(path, body); | |
| 480 | }, | |
| 481 | async putStream(key, body) { | |
| 482 | checkKey(key); | |
| 483 | const path = at(key); | |
| 484 | mkdirSync(dirname(path), { recursive: true }); | |
| 485 | const part = `${path}.part-${process.pid}-${Bun.nanoseconds()}`; | |
| 486 | // Pumped by hand: `Bun.write(part, new Response(body))` never settles | |
| 487 | // when the stream errors (Bun 1.3), and a broken pipe must fail the put. | |
| 488 | const sink = Bun.file(part).writer(); | |
| 489 | try { | |
| 490 | try { | |
| 491 | for await (const chunk of body) sink.write(chunk); | |
| 492 | } finally { | |
| 493 | await sink.end(); | |
| 494 | } | |
| 495 | renameSync(part, path); | |
| 496 | } catch (error) { | |
| 497 | rmSync(part, { force: true }); | |
| 498 | throw error; | |
| 499 | } | |
| 500 | }, | |
| 501 | async get(key) { | |
| 502 | const file = Bun.file(at(key)); | |
| 503 | if (!(await file.exists())) return undefined; | |
| 504 | return new Uint8Array(await file.arrayBuffer()); | |
| 505 | }, | |
| 506 | async size(key) { | |
| 507 | return (await file(key))?.size; | |
| 508 | }, | |
| 509 | async getRange(key, range) { | |
| 510 | checkRange(range); | |
| 511 | const f = await file(key); | |
| 512 | if (f === undefined) return undefined; | |
| 513 | return new Uint8Array(await f.slice(range.start, range.end).arrayBuffer()); | |
| 514 | }, | |
| 515 | async getStream(key, range = { start: 0 }) { | |
| 516 | checkRange(range); | |
| 517 | const f = await file(key); | |
| 518 | return f?.slice(range.start, range.end).stream(); | |
| 519 | }, | |
| 520 | async list(prefix) { | |
| 521 | const all: string[] = []; | |
| 522 | walk(root, all); | |
| 523 | return all.filter((k) => k.startsWith(prefix)); | |
| 524 | }, | |
| 525 | async names(prefix) { | |
| 526 | try { | |
| 527 | return readdirSync(at(prefix)).filter((n) => | |
| 528 | statSync(join(at(prefix), n)).isDirectory(), | |
| 529 | ); | |
| 530 | } catch { | |
| 531 | return []; | |
| 532 | } | |
| 533 | }, | |
| 534 | async delete(key) { | |
| 535 | rmSync(at(key), { force: true }); | |
| 536 | }, | |
| 537 | }; | |
| 538 | } |
src/chaos/faults.ts +1 −1
| @@ -10,7 +10,7 @@ import { tmpdir } from "node:os"; | ||
| 10 | 10 | import { sweepAbandonedChecks, failAbandonedRuns } from "../core/runs.ts"; |
| 11 | 11 | import { serve } from "../http/server.ts"; |
| 12 | 12 | import { takeSnapshot } from "../snapshot/snapshot.ts"; |
| 13 | import { dirStore } from "../snapshot/store.ts"; | |
| 13 | import { dirStore } from "../blob/store.ts"; | |
| 14 | 14 | import { dbFile, reopen, type World } from "./world.ts"; |
| 15 | 15 | |
| 16 | 16 | export interface Fault { |
src/chaos/invariants.ts +1 −1
| @@ -10,7 +10,7 @@ import { Database } from "bun:sqlite"; | ||
| 10 | 10 | import { repoDir } from "../git/repos.ts"; |
| 11 | 11 | import { restoreSnapshot } from "../snapshot/restore.ts"; |
| 12 | 12 | import { takeSnapshot } from "../snapshot/snapshot.ts"; |
| 13 | import { dirStore } from "../snapshot/store.ts"; | |
| 13 | import { dirStore } from "../blob/store.ts"; | |
| 14 | 14 | import { SEALED_VALUE, type World } from "./world.ts"; |
| 15 | 15 | |
| 16 | 16 | export interface Invariant { |
src/snapshot/destination.ts +1 −1
| @@ -19,7 +19,7 @@ import { | ||
| 19 | 19 | s3Store, |
| 20 | 20 | type S3Target, |
| 21 | 21 | type Store, |
| 22 | } from "./store.ts"; | |
| 22 | } from "../blob/store.ts"; | |
| 23 | 23 | |
| 24 | 24 | export type DestinationSource = "sealed" | "dir" | "env"; |
| 25 | 25 |
src/snapshot/drill.ts +1 −1
| @@ -20,7 +20,7 @@ import { forgetKey, open } from "../kernel/secret.ts"; | ||
| 20 | 20 | import { countRows, mismatched, type DrillCounts } from "./counts.ts"; |
| 21 | 21 | import { restoreSnapshot, type Progress } from "./restore.ts"; |
| 22 | 22 | import { listSnapshots } from "./snapshot.ts"; |
| 23 | import type { Store } from "./store.ts"; | |
| 23 | import type { Store } from "../blob/store.ts"; | |
| 24 | 24 | |
| 25 | 25 | /** The floor of every drill's budget, and the whole budget for finding the newest snapshot. */ |
| 26 | 26 | export const DRILL_TIMEOUT_MS = 10 * 60_000; |
src/snapshot/mirror.ts +1 −1
| @@ -18,7 +18,7 @@ import { join } from "node:path"; | ||
| 18 | 18 | import type { EventBus } from "../core/events.ts"; |
| 19 | 19 | import { deleteSecret, resolveSecrets } from "../core/secret_store.ts"; |
| 20 | 20 | import { KEEP, listSnapshots, pruneSnapshots, SNAPSHOT_PREFIX } from "./snapshot.ts"; |
| 21 | import { s3Store, type S3Target, type Store } from "./store.ts"; | |
| 21 | import { s3Store, type S3Target, type Store } from "../blob/store.ts"; | |
| 22 | 22 | |
| 23 | 23 | /** The sealed names, one per field of the form. */ |
| 24 | 24 | export const MIRROR_SEALED = { |
src/snapshot/off_box.ts +1 −1
| @@ -12,7 +12,7 @@ | ||
| 12 | 12 | |
| 13 | 13 | import { lastMirror } from "./mirror.ts"; |
| 14 | 14 | import { lastSnapshotOutcome } from "./outcome.ts"; |
| 15 | import type { Store } from "./store.ts"; | |
| 15 | import type { Store } from "../blob/store.ts"; | |
| 16 | 16 | |
| 17 | 17 | /** The host of a bucket's `where` (`host[:port]/bucket`), lower-cased, without its port or brackets. */ |
| 18 | 18 | function hostOf(where: string): string { |
src/snapshot/restore.ts +1 −1
| @@ -48,7 +48,7 @@ import { | ||
| 48 | 48 | import { unwrapKey, WRAPPED_KEY } from "./wrap.ts"; |
| 49 | 49 | import { isPackedFile, partPaths, unpacker, type PackedFile, type Unpacker } from "./parts.ts"; |
| 50 | 50 | import { resolveDestination } from "./destination.ts"; |
| 51 | import type { Store } from "./store.ts"; | |
| 51 | import type { Store } from "../blob/store.ts"; | |
| 52 | 52 | |
| 53 | 53 | export interface Restored { |
| 54 | 54 | readonly manifest: Manifest; |
src/snapshot/service.ts +1 −1
| @@ -27,7 +27,7 @@ import { lastDrill, recordDrill } from "./drills.ts"; | ||
| 27 | 27 | import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts"; |
| 28 | 28 | import { isOffBox } from "./off_box.ts"; |
| 29 | 29 | import { noteOffBox, recordSnapshotOutcome } from "./outcome.ts"; |
| 30 | import type { Store } from "./store.ts"; | |
| 30 | import type { Store } from "../blob/store.ts"; | |
| 31 | 31 | |
| 32 | 32 | const log = createLog("snapshots"); |
| 33 | 33 |
src/snapshot/snapshot.ts +1 −1
| @@ -36,7 +36,7 @@ import { repoDir } from "../git/repos.ts"; | ||
| 36 | 36 | import { createLog } from "../kernel/log.ts"; |
| 37 | 37 | import { countRows, type DrillCounts } from "./counts.ts"; |
| 38 | 38 | import { pack, partPath, PART_BYTES, type PackedFile } from "./parts.ts"; |
| 39 | import type { Store } from "./store.ts"; | |
| 39 | import type { Store } from "../blob/store.ts"; | |
| 40 | 40 | import { unwrapKey, wrapKey, WRAPPED_KEY } from "./wrap.ts"; |
| 41 | 41 | |
| 42 | 42 | const log = createLog("snapshot"); |
src/snapshot/store.ts deleted −292
| @@ -1,292 +0,0 @@ | ||
| 1 | /** | |
| 2 | * Where snapshots go. Two places, one shape: a bucket over S3 (the platform | |
| 3 | * gives every app one, mirrored off the machine by lanes the forge never | |
| 4 | * sees) or a directory (a forge on a desk, a test). Keys are paths; the | |
| 5 | * store keeps no other idea of structure, so a snapshot is exactly the | |
| 6 | * files it lists and a restore is exactly reading them back. | |
| 7 | */ | |
| 8 | ||
| 9 | import { mkdirSync, readdirSync, rmSync, statSync } from "node:fs"; | |
| 10 | import { dirname, join, relative } from "node:path"; | |
| 11 | ||
| 12 | export interface Store { | |
| 13 | /** What this store is, for a page and a log; never a secret. */ | |
| 14 | readonly where: string; | |
| 15 | /** The directory it writes under, when it is one; a bucket has none. */ | |
| 16 | readonly dir?: string; | |
| 17 | put(key: string, body: Uint8Array | string): Promise<void>; | |
| 18 | get(key: string): Promise<Uint8Array | undefined>; | |
| 19 | /** Every key under `prefix`, in no particular order. */ | |
| 20 | list(prefix: string): Promise<string[]>; | |
| 21 | /** The next path segment under `prefix`: the "directories", not the keys. */ | |
| 22 | names(prefix: string): Promise<string[]>; | |
| 23 | delete(key: string): Promise<void>; | |
| 24 | /** | |
| 25 | * The same store, every call of which gives up within `leftMs()`: its | |
| 26 | * deadlines and retries together never outlast what is left. A store | |
| 27 | * that does not retry has nothing to bound and leaves this out. | |
| 28 | */ | |
| 29 | bounded?(leftMs: () => number): Store; | |
| 30 | } | |
| 31 | ||
| 32 | /** The S3 credentials as the platform hands them to an app. */ | |
| 33 | export interface S3Target { | |
| 34 | readonly endpoint: string; | |
| 35 | readonly bucket: string; | |
| 36 | readonly accessKeyId: string; | |
| 37 | readonly secretAccessKey: string; | |
| 38 | readonly region?: string | undefined; | |
| 39 | } | |
| 40 | ||
| 41 | /** Read the platform's `S3_*` variables; undefined when the app has no bucket. */ | |
| 42 | export function s3FromEnv( | |
| 43 | env: Record<string, string | undefined> = process.env, | |
| 44 | ): S3Target | undefined { | |
| 45 | const endpoint = env["S3_ENDPOINT"]?.trim(); | |
| 46 | const bucket = env["S3_BUCKET"]?.trim(); | |
| 47 | const accessKeyId = env["S3_ACCESS_KEY"]?.trim(); | |
| 48 | const secretAccessKey = env["S3_SECRET_KEY"]?.trim(); | |
| 49 | if (!endpoint || !bucket || !accessKeyId || !secretAccessKey) | |
| 50 | return undefined; | |
| 51 | return { | |
| 52 | endpoint, | |
| 53 | bucket, | |
| 54 | accessKeyId, | |
| 55 | secretAccessKey, | |
| 56 | region: env["S3_REGION"]?.trim() || undefined, | |
| 57 | }; | |
| 58 | } | |
| 59 | ||
| 60 | /** The slowest uplink a put may assume: 5 Mbit/s, a third of the home line it runs behind. */ | |
| 61 | export const FLOOR_BYTES_PER_S = 5_000_000 / 8; | |
| 62 | /** What every attempt gets on top of its bytes' time: the handshake, the server's thought, a queue ahead of it. */ | |
| 63 | export const SLACK_MS = 60_000; | |
| 64 | /** How long a read, a listing or a delete may go unanswered. */ | |
| 65 | export const CALL_DEADLINE_MS = 180_000; | |
| 66 | ||
| 67 | /** How long a put of `bytes` may take before it counts as failed: its bytes at the floor rate, plus slack. */ | |
| 68 | export const deadlineFor = ( | |
| 69 | bytes: number, | |
| 70 | floorBytesPerS = FLOOR_BYTES_PER_S, | |
| 71 | slackMs = SLACK_MS, | |
| 72 | ): number => slackMs + Math.ceil((bytes / floorBytesPerS) * 1000); | |
| 73 | ||
| 74 | const sizeOf = (body: Uint8Array | string): number => | |
| 75 | typeof body === "string" ? Buffer.byteLength(body) : body.byteLength; | |
| 76 | ||
| 77 | /** | |
| 78 | * A store that tries again. Object stores answer a burst, a flake or a | |
| 79 | * moment of maintenance with a 5xx — Backblaze says so in its own docs — | |
| 80 | * and one such answer in a thousand puts used to fail the whole snapshot | |
| 81 | * with "S3Error: an unexpected error has occurred" and no key named. Every | |
| 82 | * call is tried `attempts` times with doubling waits, and what finally | |
| 83 | * fails says which key it was. An attempt that has not answered within its | |
| 84 | * deadline counts as failed too: a connection that died quietly never | |
| 85 | * settles on its own, and once held a snapshot on its first object for an | |
| 86 | * hour with the network fine (2026-10-13), so the next attempt opens a | |
| 87 | * fresh request instead. | |
| 88 | * | |
| 89 | * A put's deadline follows its size (`deadlineFor`): a fixed 180 s once | |
| 90 | * failed every attempt at a 334 MB database behind a 15 Mbit/s uplink | |
| 91 | * (2026-10-17). `deadlineMs` fixes every deadline instead, for tests; | |
| 92 | * `floorBytesPerS` and `slackMs` scale the size rule, for tests that run a | |
| 93 | * slow link at a thousand times its speed. | |
| 94 | * | |
| 95 | * `bounded(leftMs)` is the same store inside someone else's budget (the | |
| 96 | * restore drill's): each attempt's deadline is cut to an even share of | |
| 97 | * what is left after the waits still to come, and none starts once | |
| 98 | * nothing is, so a stalled store fails within the budget instead of | |
| 99 | * holding five 180 s attempts past it. | |
| 100 | */ | |
| 101 | export function withRetries( | |
| 102 | inner: Store, | |
| 103 | options: { | |
| 104 | attempts?: number; | |
| 105 | baseMs?: number; | |
| 106 | deadlineMs?: number; | |
| 107 | floorBytesPerS?: number; | |
| 108 | slackMs?: number; | |
| 109 | sleep?: (ms: number) => Promise<void>; | |
| 110 | } = {}, | |
| 111 | ): Store { | |
| 112 | const attempts = Math.max(1, options.attempts ?? 5); | |
| 113 | const baseMs = options.baseMs ?? 500; | |
| 114 | const fixed = options.deadlineMs; | |
| 115 | const callMs = fixed ?? CALL_DEADLINE_MS; | |
| 116 | const putMs = (body: Uint8Array | string): number => | |
| 117 | fixed ?? deadlineFor(sizeOf(body), options.floorBytesPerS, options.slackMs); | |
| 118 | const sleep = options.sleep ?? ((ms) => Bun.sleep(ms)); | |
| 119 | // The waits still to come once attempt `n` has failed, through the last. | |
| 120 | const waitsFrom = (n: number): number => { | |
| 121 | let ms = 0; | |
| 122 | for (let k = n; k + 1 < attempts; k++) ms += baseMs * 2 ** k; | |
| 123 | return ms; | |
| 124 | }; | |
| 125 | async function tried<T>( | |
| 126 | what: string, | |
| 127 | deadlineMs: number, | |
| 128 | fn: () => Promise<T>, | |
| 129 | leftMs?: () => number, | |
| 130 | ): Promise<T> { | |
| 131 | let last: unknown; | |
| 132 | let made = 0; | |
| 133 | for (let n = 0; n < attempts; n++) { | |
| 134 | // Bounded, each attempt gets an even share of what is left after the | |
| 135 | // waits to come, so deadlines × attempts never exceed it. | |
| 136 | const share = | |
| 137 | leftMs === undefined ? deadlineMs : Math.floor((leftMs() - waitsFrom(n)) / (attempts - n)); | |
| 138 | if (share < 1) { | |
| 139 | last ??= new Error("no time left in its budget"); | |
| 140 | break; | |
| 141 | } | |
| 142 | made++; | |
| 143 | try { | |
| 144 | return await answered(fn(), Math.min(deadlineMs, share)); | |
| 145 | } catch (error) { | |
| 146 | last = error; | |
| 147 | if (n + 1 < attempts) await sleep(baseMs * 2 ** n); | |
| 148 | } | |
| 149 | } | |
| 150 | throw new Error( | |
| 151 | `${what} failed after ${made} attempt${made === 1 ? "" : "s"}: ${String(last instanceof Error ? last.message : last)}`, | |
| 152 | ); | |
| 153 | } | |
| 154 | const view = (leftMs?: () => number): Store => ({ | |
| 155 | where: inner.where, | |
| 156 | ...(inner.dir === undefined ? {} : { dir: inner.dir }), | |
| 157 | put: (key, body) => | |
| 158 | tried(`put ${key}`, putMs(body), () => inner.put(key, body), leftMs), | |
| 159 | get: (key) => tried(`get ${key}`, callMs, () => inner.get(key), leftMs), | |
| 160 | list: (prefix) => tried(`list ${prefix}`, callMs, () => inner.list(prefix), leftMs), | |
| 161 | names: (prefix) => | |
| 162 | tried(`names ${prefix}`, callMs, () => inner.names(prefix), leftMs), | |
| 163 | delete: (key) => tried(`delete ${key}`, callMs, () => inner.delete(key), leftMs), | |
| 164 | bounded: (left) => view(left), | |
| 165 | }); | |
| 166 | return view(); | |
| 167 | } | |
| 168 | ||
| 169 | /** | |
| 170 | * The attempt's answer, or a rejection once `ms` have passed without one. | |
| 171 | * The abandoned attempt is left to settle on its own, its outcome ignored. | |
| 172 | */ | |
| 173 | function answered<T>(attempt: Promise<T>, ms: number): Promise<T> { | |
| 174 | let timer: ReturnType<typeof setTimeout> | undefined; | |
| 175 | const shown = ms >= 1000 ? `${Math.round(ms / 1000)} s` : `${ms} ms`; | |
| 176 | const deadline = new Promise<never>((_, reject) => { | |
| 177 | timer = setTimeout(() => reject(new Error(`no answer in ${shown}`)), ms); | |
| 178 | }); | |
| 179 | attempt.catch(() => {}); | |
| 180 | return Promise.race([attempt, deadline]).finally(() => clearTimeout(timer)); | |
| 181 | } | |
| 182 | ||
| 183 | export function s3Store(target: S3Target): Store { | |
| 184 | return withRetries(rawS3Store(target)); | |
| 185 | } | |
| 186 | ||
| 187 | function rawS3Store(target: S3Target): Store { | |
| 188 | const client = new Bun.S3Client({ | |
| 189 | endpoint: target.endpoint, | |
| 190 | bucket: target.bucket, | |
| 191 | accessKeyId: target.accessKeyId, | |
| 192 | secretAccessKey: target.secretAccessKey, | |
| 193 | ...(target.region === undefined ? {} : { region: target.region }), | |
| 194 | }); | |
| 195 | return { | |
| 196 | where: `${target.endpoint.replace(/^https?:\/\//, "")}/${target.bucket}`, | |
| 197 | async put(key, body) { | |
| 198 | await client.write(key, body); | |
| 199 | }, | |
| 200 | async get(key) { | |
| 201 | const file = client.file(key); | |
| 202 | if (!(await file.exists())) return undefined; | |
| 203 | return new Uint8Array(await file.arrayBuffer()); | |
| 204 | }, | |
| 205 | async list(prefix) { | |
| 206 | const keys: string[] = []; | |
| 207 | let token: string | undefined; | |
| 208 | do { | |
| 209 | const page = await client.list({ | |
| 210 | prefix, | |
| 211 | maxKeys: 1000, | |
| 212 | ...(token === undefined ? {} : { continuationToken: token }), | |
| 213 | }); | |
| 214 | for (const item of page.contents ?? []) keys.push(item.key); | |
| 215 | token = page.isTruncated ? page.nextContinuationToken : undefined; | |
| 216 | } while (token !== undefined); | |
| 217 | return keys; | |
| 218 | }, | |
| 219 | async names(prefix) { | |
| 220 | const out: string[] = []; | |
| 221 | let token: string | undefined; | |
| 222 | do { | |
| 223 | const page = await client.list({ | |
| 224 | prefix, | |
| 225 | delimiter: "/", | |
| 226 | maxKeys: 1000, | |
| 227 | ...(token === undefined ? {} : { continuationToken: token }), | |
| 228 | }); | |
| 229 | for (const p of page.commonPrefixes ?? []) | |
| 230 | out.push(p.prefix.slice(prefix.length).replace(/\/$/, "")); | |
| 231 | token = page.isTruncated ? page.nextContinuationToken : undefined; | |
| 232 | } while (token !== undefined); | |
| 233 | return out; | |
| 234 | }, | |
| 235 | async delete(key) { | |
| 236 | await client.delete(key); | |
| 237 | }, | |
| 238 | }; | |
| 239 | } | |
| 240 | ||
| 241 | /** A directory as a store: `key` is a path under it, nothing else. */ | |
| 242 | export function dirStore(root: string): Store { | |
| 243 | const at = (key: string): string => { | |
| 244 | if (key.startsWith("/") || key.split("/").includes("..")) | |
| 245 | throw new Error(`refusing key ${key}`); | |
| 246 | return join(root, key); | |
| 247 | }; | |
| 248 | const walk = (dir: string, out: string[]): void => { | |
| 249 | let entries: string[]; | |
| 250 | try { | |
| 251 | entries = readdirSync(dir); | |
| 252 | } catch { | |
| 253 | return; | |
| 254 | } | |
| 255 | for (const name of entries) { | |
| 256 | const path = join(dir, name); | |
| 257 | if (statSync(path).isDirectory()) walk(path, out); | |
| 258 | else out.push(relative(root, path)); | |
| 259 | } | |
| 260 | }; | |
| 261 | return { | |
| 262 | where: root, | |
| 263 | dir: root, | |
| 264 | async put(key, body) { | |
| 265 | const path = at(key); | |
| 266 | mkdirSync(dirname(path), { recursive: true }); | |
| 267 | await Bun.write(path, body); | |
| 268 | }, | |
| 269 | async get(key) { | |
| 270 | const file = Bun.file(at(key)); | |
| 271 | if (!(await file.exists())) return undefined; | |
| 272 | return new Uint8Array(await file.arrayBuffer()); | |
| 273 | }, | |
| 274 | async list(prefix) { | |
| 275 | const all: string[] = []; | |
| 276 | walk(root, all); | |
| 277 | return all.filter((k) => k.startsWith(prefix)); | |
| 278 | }, | |
| 279 | async names(prefix) { | |
| 280 | try { | |
| 281 | return readdirSync(at(prefix)).filter((n) => | |
| 282 | statSync(join(at(prefix), n)).isDirectory(), | |
| 283 | ); | |
| 284 | } catch { | |
| 285 | return []; | |
| 286 | } | |
| 287 | }, | |
| 288 | async delete(key) { | |
| 289 | rmSync(at(key), { force: true }); | |
| 290 | }, | |
| 291 | }; | |
| 292 | } |
src/web/pages/backups.ts +1 −1
| @@ -26,7 +26,7 @@ import { resolveDestination, sealedSet } from "../../snapshot/destination.ts"; | ||
| 26 | 26 | import { keyOf } from "../../snapshot/snapshot.ts"; |
| 27 | 27 | import { formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts"; |
| 28 | 28 | import { lastDrill } from "../../snapshot/drills.ts"; |
| 29 | import type { Store } from "../../snapshot/store.ts"; | |
| 29 | import type { Store } from "../../blob/store.ts"; | |
| 30 | 30 | import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts"; |
| 31 | 31 | import { lastSnapshotOutcome } from "../../snapshot/outcome.ts"; |
| 32 | 32 | import type { Ctx } from "../../http/context.ts"; |
test/backups-live.test.ts +1 −1
| @@ -13,7 +13,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 13 | 13 | import { listEvents } from "../src/core/events.ts"; |
| 14 | 14 | import { serve, type Servers } from "../src/http/server.ts"; |
| 15 | 15 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; |
| 16 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 16 | import { dirStore } from "../src/blob/store.ts"; | |
| 17 | 17 | import { quietLogs } from "./helpers/globals.ts"; |
| 18 | 18 | |
| 19 | 19 | quietLogs(); |
test/backups-parts.test.ts +1 −1
| @@ -13,7 +13,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 13 | 13 | import { serve, type Servers } from "../src/http/server.ts"; |
| 14 | 14 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; |
| 15 | 15 | import { recordDrill } from "../src/snapshot/drills.ts"; |
| 16 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 16 | import { dirStore } from "../src/blob/store.ts"; | |
| 17 | 17 | import { quietLogs } from "./helpers/globals.ts"; |
| 18 | 18 | |
| 19 | 19 | quietLogs(); |
test/backups-timing.test.ts +1 −1
| @@ -7,7 +7,7 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 7 | 7 | import { createUser } from "../src/core/users.ts"; |
| 8 | 8 | import { serve, type Servers } from "../src/http/server.ts"; |
| 9 | 9 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; |
| 10 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 10 | import { dirStore } from "../src/blob/store.ts"; | |
| 11 | 11 | import { quietLogs } from "./helpers/globals.ts"; |
| 12 | 12 | |
| 13 | 13 | quietLogs(); |
test/blob-store-s3.test.ts added +97
| @@ -0,0 +1,97 @@ | ||
| 1 | /** | |
| 2 | * The bucket side of the BlobStore, on the wire: what `s3Store` asks the | |
| 3 | * bucket for when it puts with a storage class, streams a body, reads a | |
| 4 | * range and presigns a URL. The real Bun.S3Client runs in a child | |
| 5 | * (test/helpers/s3-wire.ts) against a fake bucket on loopback, from an | |
| 6 | * environment built here: the client reads a proxy from its environment | |
| 7 | * once, and a proxy refuses loopback. | |
| 8 | */ | |
| 9 | import { beforeAll, describe, expect, test } from "bun:test"; | |
| 10 | import { join } from "node:path"; | |
| 11 | ||
| 12 | interface Seen { | |
| 13 | method: string; | |
| 14 | path: string; | |
| 15 | range: string | null; | |
| 16 | storageClass: string | null; | |
| 17 | type: string | null; | |
| 18 | } | |
| 19 | type Step<T = unknown> = { value?: T; seen: Seen[] }; | |
| 20 | let out: Record<string, Step> & { presignGet: string; presignPut: string }; | |
| 21 | ||
| 22 | beforeAll(async () => { | |
| 23 | const child = Bun.spawn([process.execPath, join(import.meta.dir, "helpers/s3-wire.ts")], { | |
| 24 | env: { | |
| 25 | PATH: process.env["PATH"] ?? "/usr/bin:/bin", | |
| 26 | HOME: process.env["HOME"] ?? "/tmp", | |
| 27 | ...(process.env["TMPDIR"] === undefined ? {} : { TMPDIR: process.env["TMPDIR"] }), | |
| 28 | }, | |
| 29 | stdout: "pipe", | |
| 30 | stderr: "pipe", | |
| 31 | }); | |
| 32 | const [stdout, stderr, code] = await Promise.all([ | |
| 33 | new Response(child.stdout).text(), | |
| 34 | new Response(child.stderr).text(), | |
| 35 | child.exited, | |
| 36 | ]); | |
| 37 | if (code !== 0) throw new Error(`s3-wire exited ${code}: ${stderr}`); | |
| 38 | out = JSON.parse(stdout); | |
| 39 | }, 30_000); | |
| 40 | ||
| 41 | const requests = (step: string) => | |
| 42 | out[step]!.seen.map((s) => `${s.method} ${s.path}${s.range === null ? "" : ` ${s.range}`}`); | |
| 43 | ||
| 44 | describe("s3Store on the wire", () => { | |
| 45 | test("a default put sends no storage class, exactly as snapshots always have", () => { | |
| 46 | expect(out["putDefault"]!.seen).toEqual([ | |
| 47 | { method: "PUT", path: "/b/logs/a", range: null, storageClass: null, type: "application/octet-stream" }, | |
| 48 | ]); | |
| 49 | }); | |
| 50 | ||
| 51 | test("the storage-class hint becomes the bucket's class, and a type is sent as given", () => { | |
| 52 | expect(out["putInfrequent"]!.seen[0]!.storageClass).toBe("STANDARD_IA"); | |
| 53 | expect(out["putInfrequent"]!.seen[0]!.type).toStartWith("text/plain"); | |
| 54 | expect(out["putFile"]!.seen[0]!.storageClass).toBe("DEEP_ARCHIVE"); | |
| 55 | expect(out["putStream"]!.seen.some((s) => s.storageClass === "STANDARD_IA")).toBe(true); | |
| 56 | }); | |
| 57 | ||
| 58 | test("a Blob and a stream both land whole", () => { | |
| 59 | expect(out["whole"]!.value).toEqual(["from a file", "streamed in"]); | |
| 60 | }); | |
| 61 | ||
| 62 | test("a size is one HEAD, and a missing key is undefined, not an error", () => { | |
| 63 | expect(out["size"]!.value).toBe(11); | |
| 64 | expect(requests("size")).toEqual(["HEAD /b/logs/a"]); | |
| 65 | expect(out["sizeAbsent"]!.value).toBeUndefined(); | |
| 66 | }); | |
| 67 | ||
| 68 | test("a range is one ranged GET with an inclusive end, after the HEAD that bounds it", () => { | |
| 69 | expect(out["range"]!.value).toBe("llo"); | |
| 70 | expect(requests("range")).toEqual(["HEAD /b/logs/a", "GET /b/logs/a bytes=2-4"]); | |
| 71 | expect(out["rangeOpen"]!.value).toBe("world"); | |
| 72 | expect(requests("rangeOpen")).toEqual(["HEAD /b/logs/a", "GET /b/logs/a bytes=6-10"]); | |
| 73 | }); | |
| 74 | ||
| 75 | test("a range past the end is empty without asking (a bucket would answer 416); an absent key is null", () => { | |
| 76 | expect(out["rangePast"]!.value).toBe(""); | |
| 77 | expect(requests("rangePast")).toEqual(["HEAD /b/logs/a"]); | |
| 78 | expect(out["rangeAbsent"]!.value).toBeNull(); | |
| 79 | }); | |
| 80 | ||
| 81 | test("a stream of a range reads only that range", () => { | |
| 82 | expect(out["stream"]!.value).toBe("in"); | |
| 83 | expect(requests("stream")).toContain("GET /b/logs/d bytes=9-10"); | |
| 84 | }); | |
| 85 | ||
| 86 | test("a presigned URL is signed for its method, expiry and class, and names no secret", () => { | |
| 87 | const get = new URL(out.presignGet); | |
| 88 | expect(get.pathname).toBe("/b/logs/a"); | |
| 89 | expect(get.searchParams.get("X-Amz-Expires")).toBe("600"); | |
| 90 | expect(get.searchParams.get("X-Amz-Signature")).toMatch(/^[0-9a-f]{64}$/); | |
| 91 | expect(get.searchParams.get("x-amz-storage-class")).toBeNull(); | |
| 92 | const put = new URL(out.presignPut); | |
| 93 | expect(put.searchParams.get("X-Amz-Expires")).toBe("60"); | |
| 94 | expect(put.searchParams.get("x-amz-storage-class")).toBe("STANDARD_IA"); | |
| 95 | expect(out.presignGet + out.presignPut).not.toContain("secret"); | |
| 96 | }); | |
| 97 | }); |
test/blob-store.test.ts added +187
| @@ -0,0 +1,187 @@ | ||
| 1 | /** | |
| 2 | * The BlobStore over a directory, and its retries: streamed puts that | |
| 3 | * never show half a file, sizes, byte ranges and range streams, keys and | |
| 4 | * ranges refused at the boundary, and `withRetries` keeping a BlobStore a | |
| 5 | * BlobStore. The bucket side is test/blob-store-s3.test.ts. | |
| 6 | */ | |
| 7 | import { afterAll, describe, expect, test } from "bun:test"; | |
| 8 | import { mkdtempSync, readdirSync, rmSync } from "node:fs"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { | |
| 12 | checkPresign, | |
| 13 | checkRange, | |
| 14 | dirStore, | |
| 15 | isBlobStore, | |
| 16 | s3StorageClass, | |
| 17 | withRetries, | |
| 18 | type BlobStore, | |
| 19 | type Store, | |
| 20 | } from "../src/blob/store.ts"; | |
| 21 | ||
| 22 | const root = mkdtempSync(join(tmpdir(), "blob-store-")); | |
| 23 | afterAll(() => rmSync(root, { recursive: true, force: true })); | |
| 24 | const text = (b: Uint8Array | undefined) => | |
| 25 | b === undefined ? undefined : new TextDecoder().decode(b); | |
| 26 | ||
| 27 | describe("a directory as a BlobStore", () => { | |
| 28 | const store = dirStore(join(root, "dir")); | |
| 29 | ||
| 30 | test("a stream lands whole under its key, and no temporary file is left beside it", async () => { | |
| 31 | await store.putStream("runs/1/log", new Blob(["one ", "two ", "three"]).stream()); | |
| 32 | expect(text(await store.get("runs/1/log"))).toBe("one two three"); | |
| 33 | expect(readdirSync(join(root, "dir/runs/1"))).toEqual(["log"]); | |
| 34 | }); | |
| 35 | ||
| 36 | test("a stream that fails leaves nothing under its key", async () => { | |
| 37 | const broken = new ReadableStream<Uint8Array>({ | |
| 38 | pull(c) { | |
| 39 | c.error(new Error("the pipe broke")); | |
| 40 | }, | |
| 41 | }); | |
| 42 | await expect(store.putStream("runs/2/log", broken)).rejects.toThrow("the pipe broke"); | |
| 43 | expect(await store.size("runs/2/log")).toBeUndefined(); | |
| 44 | expect(readdirSync(join(root, "dir/runs/2"))).toEqual([]); | |
| 45 | }); | |
| 46 | ||
| 47 | test("a Blob put lands as its bytes", async () => { | |
| 48 | await store.put("a/blob", new Blob(["from a file"]), { storageClass: "archive" }); | |
| 49 | expect(text(await store.get("a/blob"))).toBe("from a file"); | |
| 50 | }); | |
| 51 | ||
| 52 | test("size, ranges and range streams read what is there; absent is undefined", async () => { | |
| 53 | await store.put("r", "hello world"); | |
| 54 | expect(await store.size("r")).toBe(11); | |
| 55 | expect(await store.size("none")).toBeUndefined(); | |
| 56 | expect(text(await store.getRange("r", { start: 2, end: 5 }))).toBe("llo"); | |
| 57 | expect(text(await store.getRange("r", { start: 6 }))).toBe("world"); | |
| 58 | expect(text(await store.getRange("r", { start: 50 }))).toBe(""); | |
| 59 | expect(await store.getRange("none", { start: 0 })).toBeUndefined(); | |
| 60 | expect(await new Response(await store.getStream("r", { start: 0, end: 5 })).text()).toBe("hello"); | |
| 61 | expect(await new Response(await store.getStream("r")).text()).toBe("hello world"); | |
| 62 | expect(await store.getStream("none")).toBeUndefined(); | |
| 63 | }); | |
| 64 | ||
| 65 | test("a directory has no URL to presign", () => { | |
| 66 | expect(store.presign).toBeUndefined(); | |
| 67 | }); | |
| 68 | ||
| 69 | test("a key that leaves the directory, or carries a control byte, is refused", async () => { | |
| 70 | for (const key of ["../x", "/etc/passwd", "a/../../x", "", "a/./b", "a\nb"]) | |
| 71 | await expect(store.getRange(key, { start: 0 })).rejects.toThrow("refusing key"); | |
| 72 | await expect(store.putStream("../x", new Blob(["x"]).stream())).rejects.toThrow("refusing key"); | |
| 73 | }); | |
| 74 | ||
| 75 | test("a range that is not whole, non-negative bytes in order is refused", async () => { | |
| 76 | for (const range of [{ start: -1 }, { start: 1.5 }, { start: 5, end: 4 }, { start: 0, end: Number.NaN }]) | |
| 77 | await expect(store.getRange("r", range)).rejects.toThrow("refusing range"); | |
| 78 | expect(() => checkRange({ start: 0, end: 0 })).not.toThrow(); | |
| 79 | }); | |
| 80 | }); | |
| 81 | ||
| 82 | describe("the boundary rules a bucket shares", () => { | |
| 83 | test("a presign is a GET or a PUT for 1 s to seven days", () => { | |
| 84 | expect(() => checkPresign({ method: "GET", expiresInS: 1 })).not.toThrow(); | |
| 85 | expect(() => checkPresign({ method: "PUT", expiresInS: 604_800 })).not.toThrow(); | |
| 86 | for (const bad of [ | |
| 87 | { method: "GET", expiresInS: 0 }, | |
| 88 | { method: "GET", expiresInS: 604_801 }, | |
| 89 | { method: "GET", expiresInS: 1.5 }, | |
| 90 | { method: "DELETE", expiresInS: 60 }, | |
| 91 | ]) | |
| 92 | expect(() => checkPresign(bad as never)).toThrow("refusing presign"); | |
| 93 | }); | |
| 94 | ||
| 95 | test("the class hint maps to S3's names, and hot sends none", () => { | |
| 96 | expect(s3StorageClass(undefined)).toBeUndefined(); | |
| 97 | expect(s3StorageClass("hot")).toBeUndefined(); | |
| 98 | expect(s3StorageClass("infrequent")).toBe("STANDARD_IA"); | |
| 99 | expect(s3StorageClass("archive")).toBe("DEEP_ARCHIVE"); | |
| 100 | }); | |
| 101 | }); | |
| 102 | ||
| 103 | describe("withRetries over a BlobStore", () => { | |
| 104 | /** A directory store whose reads fail the first `failures` times each. */ | |
| 105 | function flaky(failures: number): { store: BlobStore; calls: string[] } { | |
| 106 | const inner = dirStore(join(root, `flaky-${failures}`)); | |
| 107 | const calls: string[] = []; | |
| 108 | const left = new Map<string, number>(); | |
| 109 | const gate = (what: string): void => { | |
| 110 | calls.push(what); | |
| 111 | const n = left.get(what) ?? failures; | |
| 112 | if (n > 0) { | |
| 113 | left.set(what, n - 1); | |
| 114 | throw new Error("S3Error: an unexpected error has occurred"); | |
| 115 | } | |
| 116 | }; | |
| 117 | return { | |
| 118 | calls, | |
| 119 | store: { | |
| 120 | ...inner, | |
| 121 | async size(key) { | |
| 122 | gate(`size ${key}`); | |
| 123 | return inner.size(key); | |
| 124 | }, | |
| 125 | async getRange(key, range) { | |
| 126 | gate(`getRange ${key}`); | |
| 127 | return inner.getRange(key, range); | |
| 128 | }, | |
| 129 | async getStream(key, range) { | |
| 130 | gate(`getStream ${key}`); | |
| 131 | return inner.getStream(key, range); | |
| 132 | }, | |
| 133 | async putStream(key, body) { | |
| 134 | gate(`putStream ${key}`); | |
| 135 | return inner.putStream(key, body); | |
| 136 | }, | |
| 137 | }, | |
| 138 | }; | |
| 139 | } | |
| 140 | const quick = { sleep: async () => {} }; | |
| 141 | ||
| 142 | test("it stays a BlobStore; the snapshot shape stays a Store", () => { | |
| 143 | const blob = withRetries(dirStore(join(root, "w")), quick); | |
| 144 | expect(isBlobStore(blob)).toBe(true); | |
| 145 | expect(isBlobStore(blob.bounded!(() => 60_000))).toBe(true); | |
| 146 | const plain: Store = { | |
| 147 | where: "plain", | |
| 148 | put: async () => {}, | |
| 149 | get: async () => undefined, | |
| 150 | list: async () => [], | |
| 151 | names: async () => [], | |
| 152 | delete: async () => {}, | |
| 153 | }; | |
| 154 | expect(isBlobStore(withRetries(plain, quick))).toBe(false); | |
| 155 | }); | |
| 156 | ||
| 157 | test("sizes, ranges and opening a stream are tried again", async () => { | |
| 158 | const f = flaky(2); | |
| 159 | await f.store.put("k", "hello world"); | |
| 160 | const store = withRetries(f.store, quick); | |
| 161 | expect(await store.size("k")).toBe(11); | |
| 162 | expect(text(await store.getRange("k", { start: 0, end: 5 }))).toBe("hello"); | |
| 163 | expect(await new Response(await store.getStream("k", { start: 6 })).text()).toBe("world"); | |
| 164 | expect(f.calls.filter((c) => c === "getRange k")).toHaveLength(3); | |
| 165 | }); | |
| 166 | ||
| 167 | test("a stream is put once, and its failure names the key", async () => { | |
| 168 | const f = flaky(1); | |
| 169 | const store = withRetries(f.store, quick); | |
| 170 | await expect(store.putStream("s", new Blob(["x"]).stream())).rejects.toThrow( | |
| 171 | "put s failed: S3Error", | |
| 172 | ); | |
| 173 | expect(f.calls).toEqual(["putStream s"]); | |
| 174 | }); | |
| 175 | ||
| 176 | test("a Blob put's deadline follows its size, as a byte put's does", async () => { | |
| 177 | const stalled = { | |
| 178 | ...dirStore(join(root, "stalled")), | |
| 179 | put: () => new Promise<void>(() => {}), | |
| 180 | } as BlobStore; | |
| 181 | // 300 bytes at a floor of 1 kB/s and no slack: a 300 ms deadline, named in the failure. | |
| 182 | const store = withRetries(stalled, { ...quick, attempts: 1, floorBytesPerS: 1_000, slackMs: 0 }); | |
| 183 | await expect(store.put("b", new Blob([new Uint8Array(300)]))).rejects.toThrow( | |
| 184 | "put b failed after 1 attempt: no answer in 300 ms", | |
| 185 | ); | |
| 186 | }); | |
| 187 | }); |
test/drill-400mb.test.ts +1 −1
| @@ -27,7 +27,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 27 | 27 | import { drillBudgetMs, runDrill } from "../src/snapshot/drill.ts"; |
| 28 | 28 | import { PART_BYTES, partPath, partPaths, unpack } from "../src/snapshot/parts.ts"; |
| 29 | 29 | import { listSnapshots, takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts"; |
| 30 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 30 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 31 | 31 | import { quietLogs } from "./helpers/globals.ts"; |
| 32 | 32 | |
| 33 | 33 | quietLogs(); |
test/drill-budget-size.test.ts +1 −1
| @@ -18,7 +18,7 @@ import { DRILL_MS_PER_MB, drillBudgetMs, runDrill } from "../src/snapshot/drill. | ||
| 18 | 18 | import { failedSentence } from "../src/snapshot/verdict.ts"; |
| 19 | 19 | import { restoreSnapshot } from "../src/snapshot/restore.ts"; |
| 20 | 20 | import { listSnapshots, takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 21 | import { CALL_DEADLINE_MS, dirStore, withRetries, type Store } from "../src/snapshot/store.ts"; | |
| 21 | import { CALL_DEADLINE_MS, dirStore, withRetries, type Store } from "../src/blob/store.ts"; | |
| 22 | 22 | import { quietLogs } from "./helpers/globals.ts"; |
| 23 | 23 | |
| 24 | 24 | quietLogs(); |
test/drill-parts.test.ts +1 −1
| @@ -10,7 +10,7 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 10 | 10 | import { createUser } from "../src/core/users.ts"; |
| 11 | 11 | import { describeTimes, runDrill, timeByPart } from "../src/snapshot/drill.ts"; |
| 12 | 12 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 13 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 13 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 14 | 14 | import { failedSentence, slowSentence } from "../src/snapshot/verdict.ts"; |
| 15 | 15 | import { quietLogs } from "./helpers/globals.ts"; |
| 16 | 16 |
test/drill-verdict.test.ts +1 −1
| @@ -15,7 +15,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 15 | 15 | import { vitals } from "../src/kernel/vitals.ts"; |
| 16 | 16 | import { bootRestored } from "../src/snapshot/boot.ts"; |
| 17 | 17 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 18 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 18 | import { dirStore } from "../src/blob/store.ts"; | |
| 19 | 19 | import { runDrill } from "../src/snapshot/drill.ts"; |
| 20 | 20 | import { untilDrill } from "../src/snapshot/service.ts"; |
| 21 | 21 | import { lastVerdict, postToRecoveryIssue, reportVerdict, verdictComment } from "../src/snapshot/verdict.ts"; |
test/drill-weekly.test.ts +1 −1
| @@ -16,7 +16,7 @@ import { BOOT_RED_FLOOR_MS, drillDue, lastSlot, RED_RETRY_MS } from "../src/snap | ||
| 16 | 16 | import { lastDrill, recordDrill } from "../src/snapshot/drills.ts"; |
| 17 | 17 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 18 | 18 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 19 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 19 | import { dirStore } from "../src/blob/store.ts"; | |
| 20 | 20 | import { recordVerdict } from "../src/snapshot/verdict.ts"; |
| 21 | 21 | |
| 22 | 22 | const t = (iso: string): number => Date.parse(iso); |
test/helpers/s3-wire.ts added +122
| @@ -0,0 +1,122 @@ | ||
| 1 | /** | |
| 2 | * Run as its own process by test/blob-store-s3.test.ts: a fake S3 bucket on | |
| 3 | * loopback and `s3Store` (the real Bun.S3Client) pointed at it. Every | |
| 4 | * request the client made and every answer it got is printed as one JSON | |
| 5 | * line. Its own process, so the environment it starts from is the one the | |
| 6 | * test built: Bun's S3 client takes its proxy from the environment once, | |
| 7 | * and a proxy refuses loopback. | |
| 8 | */ | |
| 9 | import { s3Store } from "../../src/blob/store.ts"; | |
| 10 | ||
| 11 | interface Seen { | |
| 12 | method: string; | |
| 13 | path: string; | |
| 14 | range: string | null; | |
| 15 | storageClass: string | null; | |
| 16 | type: string | null; | |
| 17 | } | |
| 18 | const seen: Seen[] = []; | |
| 19 | const kept = new Map<string, Uint8Array>(); | |
| 20 | const parts = new Map<number, Uint8Array>(); | |
| 21 | ||
| 22 | const server = Bun.serve({ | |
| 23 | hostname: "127.0.0.1", | |
| 24 | port: 0, | |
| 25 | async fetch(req) { | |
| 26 | const url = new URL(req.url); | |
| 27 | const path = decodeURIComponent(url.pathname); | |
| 28 | seen.push({ | |
| 29 | method: req.method, | |
| 30 | path, | |
| 31 | range: req.headers.get("range"), | |
| 32 | storageClass: req.headers.get("x-amz-storage-class"), | |
| 33 | type: req.headers.get("content-type"), | |
| 34 | }); | |
| 35 | const q = url.searchParams; | |
| 36 | if (req.method === "POST" && q.has("uploads")) | |
| 37 | return new Response( | |
| 38 | `<?xml version="1.0"?><InitiateMultipartUploadResult><UploadId>u</UploadId></InitiateMultipartUploadResult>`, | |
| 39 | ); | |
| 40 | if (req.method === "PUT" && q.has("partNumber")) { | |
| 41 | parts.set(Number(q.get("partNumber")), new Uint8Array(await req.arrayBuffer())); | |
| 42 | return new Response(null, { headers: { etag: `"p${q.get("partNumber")}"` } }); | |
| 43 | } | |
| 44 | if (req.method === "POST" && q.has("uploadId")) { | |
| 45 | const all = [...parts.entries()].sort(([a], [b]) => a - b).map(([, p]) => p); | |
| 46 | kept.set(path, new Uint8Array(Buffer.concat(all))); | |
| 47 | parts.clear(); | |
| 48 | return new Response( | |
| 49 | `<?xml version="1.0"?><CompleteMultipartUploadResult><ETag>"x"</ETag></CompleteMultipartUploadResult>`, | |
| 50 | ); | |
| 51 | } | |
| 52 | if (req.method === "PUT") { | |
| 53 | kept.set(path, new Uint8Array(await req.arrayBuffer())); | |
| 54 | return new Response(null, { headers: { etag: '"x"' } }); | |
| 55 | } | |
| 56 | if (req.method === "DELETE") { | |
| 57 | kept.delete(path); | |
| 58 | return new Response(null, { status: 204 }); | |
| 59 | } | |
| 60 | const body = kept.get(path); | |
| 61 | if (body === undefined) return new Response(null, { status: 404 }); | |
| 62 | if (req.method === "HEAD") | |
| 63 | return new Response(null, { headers: { "content-length": String(body.byteLength) } }); | |
| 64 | const m = /^bytes=(\d+)-(\d*)$/.exec(req.headers.get("range") ?? ""); | |
| 65 | if (m === null) return new Response(body); | |
| 66 | const start = Number(m[1]); | |
| 67 | const end = m[2] === "" ? body.byteLength : Number(m[2]) + 1; | |
| 68 | if (start >= body.byteLength) return new Response(null, { status: 416 }); | |
| 69 | return new Response(body.slice(start, end), { status: 206 }); | |
| 70 | }, | |
| 71 | }); | |
| 72 | ||
| 73 | const store = s3Store({ | |
| 74 | endpoint: `http://127.0.0.1:${server.port}`, | |
| 75 | bucket: "b", | |
| 76 | accessKeyId: "AKIDEXAMPLE", | |
| 77 | secretAccessKey: "secret", | |
| 78 | region: "auto", | |
| 79 | }); | |
| 80 | const text = (b: Uint8Array | undefined) => | |
| 81 | b === undefined ? null : new TextDecoder().decode(b); | |
| 82 | const step = async <T>(f: () => Promise<T>): Promise<{ value: T; seen: Seen[] }> => { | |
| 83 | seen.length = 0; | |
| 84 | const value = await f(); | |
| 85 | return { value, seen: [...seen] }; | |
| 86 | }; | |
| 87 | ||
| 88 | const out = { | |
| 89 | putDefault: await step(() => store.put("logs/a", "hello world")), | |
| 90 | putInfrequent: await step(() => | |
| 91 | store.put("logs/b", "cold", { storageClass: "infrequent", type: "text/plain" }), | |
| 92 | ), | |
| 93 | putFile: await step(() => | |
| 94 | store.put("logs/c", new Blob(["from a file"]), { storageClass: "archive" }), | |
| 95 | ), | |
| 96 | putStream: await step(() => | |
| 97 | store.putStream( | |
| 98 | "logs/d", | |
| 99 | new Blob(["streamed ", "in"]).stream(), | |
| 100 | { storageClass: "infrequent" }, | |
| 101 | ), | |
| 102 | ), | |
| 103 | size: await step(() => store.size("logs/a")), | |
| 104 | sizeAbsent: await step(() => store.size("logs/none")), | |
| 105 | range: await step(async () => text(await store.getRange("logs/a", { start: 2, end: 5 }))), | |
| 106 | rangeOpen: await step(async () => text(await store.getRange("logs/a", { start: 6 }))), | |
| 107 | rangePast: await step(async () => text(await store.getRange("logs/a", { start: 50 }))), | |
| 108 | rangeAbsent: await step(async () => text(await store.getRange("logs/none", { start: 0 }))), | |
| 109 | stream: await step(async () => { | |
| 110 | const s = await store.getStream("logs/d", { start: 9 }); | |
| 111 | return s === undefined ? null : await new Response(s).text(); | |
| 112 | }), | |
| 113 | whole: await step(async () => [text(await store.get("logs/c")), text(await store.get("logs/d"))]), | |
| 114 | presignGet: store.presign!("logs/a", { method: "GET", expiresInS: 600 }), | |
| 115 | presignPut: store.presign!("logs/e", { | |
| 116 | method: "PUT", | |
| 117 | expiresInS: 60, | |
| 118 | storageClass: "infrequent", | |
| 119 | }), | |
| 120 | }; | |
| 121 | console.log(JSON.stringify(out)); | |
| 122 | server.stop(true); |
test/recovery-doc.test.ts +1 −1
| @@ -8,7 +8,7 @@ const doc = read("docs/RECOVERY.md"); | ||
| 8 | 8 | |
| 9 | 9 | test("the doc names every variable the store reads", () => { |
| 10 | 10 | const vars = new Set( |
| 11 | [...read("src/snapshot/store.ts").matchAll(/env\["(S3_[A-Z_]+)"\]/g)].map( | |
| 11 | [...read("src/blob/store.ts").matchAll(/env\["(S3_[A-Z_]+)"\]/g)].map( | |
| 12 | 12 | (m) => m[1]!, |
| 13 | 13 | ), |
| 14 | 14 | ); |
test/resilience.test.ts +1 −1
| @@ -14,7 +14,7 @@ import { createUser, getUserByName } from "../src/core/users.ts"; | ||
| 14 | 14 | import type { Ctx, Screen } from "../src/tui/app/app.ts"; |
| 15 | 15 | import { forgeCommands, Session } from "../src/tui/forge.ts"; |
| 16 | 16 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 17 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 17 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 18 | 18 | import { createForgeApp } from "../src/tui/forge.ts"; |
| 19 | 19 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| 20 | 20 | import { settleApp } from "./helpers/tui.ts"; |
test/restore-card.test.ts +1 −1
| @@ -14,7 +14,7 @@ import { join } from "node:path"; | ||
| 14 | 14 | import { createApp, type App } from "../src/app.ts"; |
| 15 | 15 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 16 | 16 | import { createUser } from "../src/core/users.ts"; |
| 17 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 17 | import { dirStore } from "../src/blob/store.ts"; | |
| 18 | 18 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 19 | 19 | |
| 20 | 20 | const PASS = "correct horse battery staple"; |
test/snapshot-destination.test.ts +1 −1
| @@ -17,7 +17,7 @@ import { | ||
| 17 | 17 | } from "../src/actions/destination.ts"; |
| 18 | 18 | import { createUser, type User } from "../src/core/users.ts"; |
| 19 | 19 | import { resolveDestination } from "../src/snapshot/destination.ts"; |
| 20 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 20 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 21 | 21 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 22 | 22 | import { serve, type Servers } from "../src/http/server.ts"; |
| 23 | 23 | import { quietLogs } from "./helpers/globals.ts"; |
test/snapshot-drill-counts.test.ts +1 −1
| @@ -15,7 +15,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 15 | 15 | import { runDrill } from "../src/snapshot/drill.ts"; |
| 16 | 16 | import { lastDrill, recordDrill } from "../src/snapshot/drills.ts"; |
| 17 | 17 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 18 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 18 | import { dirStore } from "../src/blob/store.ts"; | |
| 19 | 19 | |
| 20 | 20 | describe("the restore drill's counts", () => { |
| 21 | 21 | let root: string; |
test/snapshot-drill-page.test.ts +1 −1
| @@ -12,7 +12,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 12 | 12 | import { serve, type Servers } from "../src/http/server.ts"; |
| 13 | 13 | import type { DrillResult } from "../src/snapshot/drill.ts"; |
| 14 | 14 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; |
| 15 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 15 | import { dirStore } from "../src/blob/store.ts"; | |
| 16 | 16 | import { quietLogs } from "./helpers/globals.ts"; |
| 17 | 17 | |
| 18 | 18 | quietLogs(); |
test/snapshot-drill-phases.test.ts +1 −1
| @@ -15,7 +15,7 @@ import { lastDrill, parsePhases, recordDrill } from "../src/snapshot/drills.ts"; | ||
| 15 | 15 | import { restoreSnapshot } from "../src/snapshot/restore.ts"; |
| 16 | 16 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; |
| 17 | 17 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 18 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 18 | import { dirStore } from "../src/blob/store.ts"; | |
| 19 | 19 | import { quietLogs } from "./helpers/globals.ts"; |
| 20 | 20 | |
| 21 | 21 | quietLogs(); |
test/snapshot-drill.test.ts +1 −1
| @@ -16,7 +16,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 16 | 16 | import { drillBudgetMs, formatBudget, formatBytes, formatMinutes, runDrill } from "../src/snapshot/drill.ts"; |
| 17 | 17 | import { pack, partPath } from "../src/snapshot/parts.ts"; |
| 18 | 18 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 19 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 19 | import { dirStore } from "../src/blob/store.ts"; | |
| 20 | 20 | |
| 21 | 21 | const PASS = "correct horse battery staple"; |
| 22 | 22 |
test/snapshot-mirror-page.test.ts +1 −1
| @@ -14,7 +14,7 @@ import { serve, type Servers } from "../src/http/server.ts"; | ||
| 14 | 14 | import { vitals } from "../src/kernel/vitals.ts"; |
| 15 | 15 | import { lastMirror, mirrorSealedSet, MIRROR_NAMES, recordMirror, resolveMirror } from "../src/snapshot/mirror.ts"; |
| 16 | 16 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; |
| 17 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 17 | import { dirStore } from "../src/blob/store.ts"; | |
| 18 | 18 | import { quietLogs } from "./helpers/globals.ts"; |
| 19 | 19 | |
| 20 | 20 | quietLogs(); |
test/snapshot-mirror.test.ts +1 −1
| @@ -23,7 +23,7 @@ import { | ||
| 23 | 23 | resolveMirror, |
| 24 | 24 | } from "../src/snapshot/mirror.ts"; |
| 25 | 25 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 26 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 26 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 27 | 27 | |
| 28 | 28 | /** A directory as a bucket that can be switched off: while `down`, every call fails like an unreachable host. */ |
| 29 | 29 | function bucket(dir: string): { store: () => Store; down: boolean } { |
test/snapshot-off-box.test.ts +1 −1
| @@ -17,7 +17,7 @@ import { recordMirror } from "../src/snapshot/mirror.ts"; | ||
| 17 | 17 | import { isOffBox, lastOffBox } from "../src/snapshot/off_box.ts"; |
| 18 | 18 | import { lastSnapshotOutcome, noteOffBox, recordSnapshotOutcome } from "../src/snapshot/outcome.ts"; |
| 19 | 19 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 20 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 20 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 21 | 21 | import { kpiForge } from "../src/tui/screens/dashboard.ts"; |
| 22 | 22 | |
| 23 | 23 | describe("isOffBox", () => { |
test/snapshot-outcome.test.ts +1 −1
| @@ -16,7 +16,7 @@ import { createUser, type User } from "../src/core/users.ts"; | ||
| 16 | 16 | import { serve, type Servers } from "../src/http/server.ts"; |
| 17 | 17 | import { lastSnapshotOutcome, recordSnapshotOutcome, snapshotNeed } from "../src/snapshot/outcome.ts"; |
| 18 | 18 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 19 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 19 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 20 | 20 | |
| 21 | 21 | const failed = (app: App) => forgeVitals(app).degraded.find((d) => d.cause === "snapshot-failed"); |
| 22 | 22 |
test/snapshot-restore-clones.test.ts +1 −1
| @@ -13,7 +13,7 @@ import { join } from "node:path"; | ||
| 13 | 13 | import { runGit } from "../src/git/env.ts"; |
| 14 | 14 | import { err } from "../src/kernel/result.ts"; |
| 15 | 15 | import { defaultClones, parseClones, restoreSnapshot } from "../src/snapshot/restore.ts"; |
| 16 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 16 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 17 | 17 | |
| 18 | 18 | const NAME = "2026-10-18T08-00-00Z"; |
| 19 | 19 | const prefix = `snapshots/${NAME}/`; |
test/snapshot-restore-packed.test.ts +1 −1
| @@ -20,7 +20,7 @@ import { createRepo } from "../src/actions/repos.ts"; | ||
| 20 | 20 | import { pack, partPath } from "../src/snapshot/parts.ts"; |
| 21 | 21 | import { restoreSnapshot } from "../src/snapshot/restore.ts"; |
| 22 | 22 | import { takeSnapshot } from "../src/snapshot/snapshot.ts"; |
| 23 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 23 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 24 | 24 | |
| 25 | 25 | const NAME = "2026-10-16T08-00-00Z"; |
| 26 | 26 | const prefix = `snapshots/${NAME}/`; |
test/snapshot-restore-parallel.test.ts +1 −1
| @@ -14,7 +14,7 @@ import { join } from "node:path"; | ||
| 14 | 14 | import type { runGit } from "../src/git/env.ts"; |
| 15 | 15 | import { restoreSnapshot, RESTORE_IN_FLIGHT } from "../src/snapshot/restore.ts"; |
| 16 | 16 | import { IN_FLIGHT } from "../src/snapshot/snapshot.ts"; |
| 17 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 17 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 18 | 18 | |
| 19 | 19 | const NAME = "2026-10-14T08-00-00Z"; |
| 20 | 20 | const prefix = `snapshots/${NAME}/`; |
test/snapshot-restore-progress.test.ts +1 −1
| @@ -10,7 +10,7 @@ import { tmpdir } from "node:os"; | ||
| 10 | 10 | import { join } from "node:path"; |
| 11 | 11 | import { ok } from "../src/kernel/result.ts"; |
| 12 | 12 | import { restoreSnapshot, type Progress } from "../src/snapshot/restore.ts"; |
| 13 | import type { Store } from "../src/snapshot/store.ts"; | |
| 13 | import type { Store } from "../src/blob/store.ts"; | |
| 14 | 14 | |
| 15 | 15 | const NAME = "2026-10-19T08-00-00Z"; |
| 16 | 16 | const prefix = `snapshots/${NAME}/`; |
test/snapshot-restore-streamed.test.ts +1 −1
| @@ -12,7 +12,7 @@ import { tmpdir } from "node:os"; | ||
| 12 | 12 | import { join } from "node:path"; |
| 13 | 13 | import { pack, partPath, unpacker, type PackedFile } from "../src/snapshot/parts.ts"; |
| 14 | 14 | import { restoreSnapshot } from "../src/snapshot/restore.ts"; |
| 15 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 15 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 16 | 16 | |
| 17 | 17 | const NAME = "2026-10-19T08-00-00Z"; |
| 18 | 18 | const prefix = `snapshots/${NAME}/`; |
test/snapshot-store.test.ts +1 −1
| @@ -5,7 +5,7 @@ | ||
| 5 | 5 | * times with doubling waits and, when it still fails, says which key. |
| 6 | 6 | */ |
| 7 | 7 | import { describe, expect, test } from "bun:test"; |
| 8 | import { withRetries, type Store } from "../src/snapshot/store.ts"; | |
| 8 | import { withRetries, type Store } from "../src/blob/store.ts"; | |
| 9 | 9 | |
| 10 | 10 | /** A store that fails the first `failures` calls of each kind, then works. */ |
| 11 | 11 | function flaky(failures: number): { store: Store; calls: string[] } { |
test/snapshot-uplink.test.ts +1 −1
| @@ -14,7 +14,7 @@ | ||
| 14 | 14 | |
| 15 | 15 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 16 | 16 | import { isPackedFile, pack, partPath, partPaths, PART_BYTES, unpack, type PackedFile } from "../src/snapshot/parts.ts"; |
| 17 | import { deadlineFor, FLOOR_BYTES_PER_S, SLACK_MS, withRetries, type Store } from "../src/snapshot/store.ts"; | |
| 17 | import { deadlineFor, FLOOR_BYTES_PER_S, SLACK_MS, withRetries, type Store } from "../src/blob/store.ts"; | |
| 18 | 18 | |
| 19 | 19 | /** |
| 20 | 20 | * How much faster than life the link runs; every deadline shrinks alike. |
test/snapshot.test.ts +1 −1
| @@ -41,7 +41,7 @@ import { | ||
| 41 | 41 | unpackRuns, |
| 42 | 42 | } from "../src/snapshot/snapshot.ts"; |
| 43 | 43 | import { unwrapKey, wrapKey } from "../src/snapshot/wrap.ts"; |
| 44 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 44 | import { dirStore, type Store } from "../src/blob/store.ts"; | |
| 45 | 45 | |
| 46 | 46 | const PASS = "correct horse battery staple"; |
| 47 | 47 |
test/tui-screens.test.ts +1 −1
| @@ -26,7 +26,7 @@ import { | ||
| 26 | 26 | import { addSshKey, signUp } from "../src/actions/users.ts"; |
| 27 | 27 | import { createApp, type App as Forge } from "../src/app.ts"; |
| 28 | 28 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 29 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 29 | import { dirStore } from "../src/blob/store.ts"; | |
| 30 | 30 | import { listEvents } from "../src/core/events.ts"; |
| 31 | 31 | import { getIssue, listComments } from "../src/core/issues.ts"; |
| 32 | 32 | import { getPull } from "../src/core/pulls.ts"; |
test/vitals-snapshots.test.ts +1 −1
| @@ -12,7 +12,7 @@ import { join } from "node:path"; | ||
| 12 | 12 | import { createApp, type App } from "../src/app.ts"; |
| 13 | 13 | import { forgeVitals, snapshotPlace } from "../src/actions/health.ts"; |
| 14 | 14 | import { sharesVolume, SNAPSHOTS_ON_STATE_VOLUME, vitals } from "../src/kernel/vitals.ts"; |
| 15 | import { dirStore, withRetries } from "../src/snapshot/store.ts"; | |
| 15 | import { dirStore, withRetries } from "../src/blob/store.ts"; | |
| 16 | 16 | |
| 17 | 17 | let dataDir: string; |
| 18 | 18 | let app: App; |
new above
offline