Public: anyone can read this; the crew can change it.

Sentinel: snapshot-failed was degraded 23 minutes today: The last snapshot failed: could not put [redacted].chunk: … #13736

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13733-sentinel-snapshot-failed-was-degraded-23 → main 6 commits fork

  • Merged. torque/13733-sentinel-snapshot-failed-was-degraded-23 was merged into main as f98d431.

Review

Sign in to review this change.
6 files changed +246 −35
docs/SNAPSHOT.md +7 −2
@@ -200,7 +200,10 @@ The store is `Bun.S3Client` or a directory behind one small interface
200200 is tried five times; an attempt that has not answered by its deadline counts
201201 as failed. A put's deadline follows its size: its bytes at a floor of
202202 5 Mbit/s, plus 60 s. A fixed 180 s once failed every attempt at a 334 MB
203 database behind a 15 Mbit/s uplink (#2536).
203 database behind a 15 Mbit/s uplink (#2536). An attempt past its deadline is
204 aborted, not just abandoned: a put goes as a presigned PUT through `fetch`
205 with the attempt's signal, because `client.write` takes none, and an
206 abandoned upload went on sending beside the tries after it (#13733).
204207
205208 ## Backup lanes
206209
@@ -219,7 +222,9 @@ Its outcome is kept in `snapshot/state.json` under the data directory, so a
219222 restart forgets nothing. A failure shows straight away as a card on the board
220223 (`snapshot-failed`, pointing at Backups), in the terminal's dashboard among
221224 the forge's needs, and as a red line on the Backups page with the reason and
222 since when. The forge tries again within the hour, not the next night. The
225 since when. The forge tries again after ten minutes (`SNAPSHOT_RETRY_MS`, capped at the
226 next nightly), not the next night: a store that is down gets six tries an hour, each of five
227 attempts per put. The
223228 next snapshot that lands, nightly or on demand, clears both.
224229
225230 A forge given nowhere to put a snapshot has not failed: nothing is kept, no
src/snapshot/outcome.ts +19 −4
@@ -35,7 +35,10 @@ export interface SnapshotOutcome {
3535 /** The nights that landed, newest last, at most `NIGHTS_KEPT`: what each took, for
3636 * `growth.ts`. */
3737 readonly nights?: readonly Night[];
38 /** Whether the store it landed in is off the machine (a bucket, not a directory); absent when not known. */
38 /**
39 * Whether the store it landed in is off the machine (a bucket, not a directory); absent when not
40 * known.
41 */
3942 readonly offBox?: boolean;
4043 /**
4144 * When a snapshot last landed off the machine; kept across landings in a local directory,
@@ -71,7 +74,10 @@ export function lastSnapshotOutcome(dataDir: string): SnapshotOutcome | undefine
7174 }
7275 }
7376
74 /** When this outcome's last off-box landing was: the kept time, or the landing itself when it was off the machine. */
77 /**
78 * When this outcome's last off-box landing was: the kept time, or the landing itself when it was
79 * off the machine.
80 */
7581 export function offBoxAtOf(s: SnapshotOutcome | undefined): string | undefined {
7682 if (s === undefined) return undefined;
7783 if (s.offBoxAt !== undefined) return s.offBoxAt;
@@ -84,7 +90,9 @@ function nightsWith(nights: readonly Night[] | undefined, night: Night): Night[]
8490 return [...(nights ?? []).filter((n) => n.at.slice(0, 10) !== day), night].slice(-NIGHTS_KEPT);
8591 }
8692
87 /** Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept. */
93 /**
94 * Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept.
95 */
8896 export function recordSnapshotOutcome(
8997 dataDir: string,
9098 result:
@@ -147,6 +155,13 @@ function write(dataDir: string, next: SnapshotOutcome): void {
147155 renameSync(tmp, stateFile(dataDir));
148156 }
149157
158 /**
159 * After a failed snapshot the next try comes this soon (or the nightly, if sooner). A flaky store
160 * (one stalled put, 2026-11-15) used to hold the card for the hour before the retry; ten minutes
161 * clears a flake and still gives a dead store only six tries an hour.
162 */
163 export const SNAPSHOT_RETRY_MS = 10 * 60_000;
164
150165 /** The card a failing snapshot is, or undefined while the last one landed. */
151166 export function snapshotNeed(
152167 s: SnapshotOutcome | undefined,
@@ -154,7 +169,7 @@ export function snapshotNeed(
154169 if (s?.failingSince === undefined) return undefined;
155170 const last = s.landedAt ? ` The newest that landed is from ${s.landedAt.slice(0, 10)}.` : "";
156171 return {
157 sentence: `The last snapshot failed: ${(s.error ?? "unknown reason").replace(/\.$/, "")}.${last} The forge tries again within the hour.`,
172 sentence: `The last snapshot failed: ${(s.error ?? "unknown reason").replace(/\.$/, "")}.${last} The forge tries again in ${SNAPSHOT_RETRY_MS / 60_000} minutes.`,
158173 since: s.failingSince,
159174 };
160175 }
src/snapshot/service.ts +35 −13
@@ -34,7 +34,7 @@ import { cutDrill, runDrillInChild, sweepDrills, type DrillState } from "./drill
3434 import { lastMirror, mirrorOffThread, mirrorTarget, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts";
3535 import { isOffBox } from "./off_box.ts";
3636 import { failureCause } from "./cause.ts";
37 import { noteOffBox, recordSnapshotOutcome } from "./outcome.ts";
37 import { noteOffBox, recordSnapshotOutcome, SNAPSHOT_RETRY_MS } from "./outcome.ts";
3838 import { remakeOf, s3FromEnv, type Store } from "./store.ts";
3939 import { verifySnapshot } from "./verify.ts";
4040 import { during } from "../perf/jobs.ts";
@@ -43,8 +43,6 @@ const log = createLog("snapshots");
4343
4444 /** When the nightly snapshot is taken, in UTC: before the probe round. */
4545 const NIGHTLY_HOUR_UTC = 8;
46 /** After a failed snapshot, the next try is this soon (or the nightly, if sooner). */
47 export const RETRY_MS = 3_600_000;
4846 /** A due snapshot waits for a crew run on a 1-CPU forge at most this long, then is taken anyway. */
4947 export const MAX_DEFER_MS = 3_600_000;
5048 /** A moving snapshot tells the page how far it is no oftener than this. */
@@ -70,7 +68,10 @@ export interface SnapshotsHandle {
7068 readonly now: () => Promise<Manifest>;
7169 /** How far the snapshot being taken is; undefined when none is. */
7270 readonly progress: () => Progress | undefined;
73 /** How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits on it (#9653). */
71 /**
72 * How long the snapshot being taken has run, ms; undefined when none is. The release keeper waits
73 * on it (#9653).
74 */
7475 readonly takingMs: () => number | undefined;
7576 /** The newest first, from the store. */
7677 readonly list: () => Promise<Manifest[]>;
@@ -85,13 +86,22 @@ export interface SnapshotsHandle {
8586 readonly startDrill: () => { readonly started: boolean; readonly state: DrillState | { readonly id: string; readonly phase: "waiting" } };
8687 /** Where the running drill stands; undefined when none runs. */
8788 readonly drillState: () => DrillState | { readonly id: string; readonly phase: "waiting" } | undefined;
88 /** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */
89 /**
90 * Drill if this week's slot has no drill yet, or the last one has stood red long enough; says
91 * whether it did.
92 */
8993 readonly drillIfDue: () => Promise<boolean>;
9094 /** The off-box mirror's store; undefined when the forge has none. */
9195 readonly mirror: Store | undefined;
92 /** Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no mirror. */
96 /**
97 * Bring the mirror level now; waits for any snapshot in flight. Undefined when there is no
98 * mirror.
99 */
93100 readonly mirrorNow: () => Promise<MirrorState | undefined>;
94 /** Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it now is. */
101 /**
102 * Like `mirrorNow`, but after any pass already in flight has ended, so it sees the mirror as it
103 * now is.
104 */
95105 readonly mirrorFresh: () => Promise<MirrorState | undefined>;
96106 /** What the last mirror pass found. */
97107 readonly mirrored: () => MirrorState | undefined;
@@ -99,7 +109,9 @@ export interface SnapshotsHandle {
99109 readonly forget: () => void;
100110 /** Read the bucket now, as the watch does; resolves when the read is kept. */
101111 readonly checkBucket: () => Promise<void>;
102 /** Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests. */
112 /**
113 * Take a snapshot if the store lacks the night's; says whether it did. For a boot, and for tests.
114 */
103115 readonly takeIfDue: () => Promise<boolean>;
104116 readonly stop: () => Promise<void>;
105117 }
@@ -114,7 +126,9 @@ const DRILL_HOUR_UTC = 9;
114126 const CATCH_UP_MS = 5 * 60_000;
115127 /** A snapshot due on boot is taken this long after it, once the forge is serving. */
116128 const SNAPSHOT_CATCH_UP_MS = 2 * 60_000;
117 /** A snapshot still owed is looked for again this often; one boot's look is not the last (#10167). */
129 /**
130 * A snapshot still owed is looked for again this often; one boot's look is not the last (#10167).
131 */
118132 const SNAPSHOT_CHECK_MS = 30 * 60_000;
119133 /** A snapshot held for a crew run on one CPU looks again this often. */
120134 const HOLD_RETRY_MS = 30_000;
@@ -149,7 +163,10 @@ export function startSnapshots(
149163 /** Test seam: how soon a failed snapshot is tried again. */
150164 readonly retryMs?: number | undefined;
151165 readonly passphrase?: string | undefined;
152 /** Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it stands. */
166 /**
167 * Test seam: how long a drill waits for a mirror pass in flight before it reads the lane as it
168 * stands.
169 */
153170 readonly drillWaitMs?: number | undefined;
154171 /** Test seam: the drill itself. */
155172 readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined;
@@ -157,14 +174,19 @@ export function startSnapshots(
157174 readonly catchUpMs?: number | undefined;
158175 readonly redRetryMs?: number | undefined;
159176 readonly bootRedFloorMs?: number | undefined;
160 /** Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read. */
177 /**
178 * Test seams: how soon after boot a due snapshot is taken, and how often the bucket is read.
179 */
161180 readonly snapshotCatchUpMs?: number | undefined;
162181 readonly snapshotCheckMs?: number | undefined;
163182 readonly bucketEveryMs?: number | undefined;
164183 readonly bucketFirstMs?: number | undefined;
165184 /** Test seam: the clock `takeIfDue` judges by. */
166185 readonly now?: (() => number) | undefined;
167 /** Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks again, and how long it waits at most. */
186 /**
187 * Test seams: the forge's CPU count, crew runs working now, how soon a held snapshot looks
188 * again, and how long it waits at most.
189 */
168190 readonly cpus?: (() => number) | undefined;
169191 readonly activeCrewRuns?: (() => number) | undefined;
170192 readonly holdRetryMs?: number | undefined;
@@ -314,7 +336,7 @@ export function startSnapshots(
314336 },
315337 });
316338 app.bus.append({ kind: "backup.finished", payload: { ok: false } });
317 arm(Math.min(options.retryMs ?? RETRY_MS, untilNightly()));
339 arm(Math.min(options.retryMs ?? SNAPSHOT_RETRY_MS, untilNightly()));
318340 throw error;
319341 },
320342 );
src/snapshot/store.ts +45 −13
@@ -15,7 +15,12 @@ export interface Store {
1515 readonly where: string;
1616 /** The directory it writes under, when it is one; a bucket has none. */
1717 readonly dir?: string;
18 put(key: string, body: Uint8Array | string): Promise<void>;
18 /**
19 * `signal` aborts the upload: a put given up at its deadline must stop sending, or its bytes
20 * keep the line busy under the attempt that replaces it (#13733). A store that cannot abort
21 * ignores it.
22 */
23 put(key: string, body: Uint8Array | string, signal?: AbortSignal): Promise<void>;
1924 get(key: string): Promise<Uint8Array | undefined>;
2025 /** Every key under `prefix`, in no particular order. */
2126 list(prefix: string): Promise<string[]>;
@@ -34,7 +39,9 @@ export interface Store {
3439 * this out.
3540 */
3641 storedKey?(key: string): Promise<string>;
37 /** True when every object is encrypted before it reaches the store (the off-site lane, #11805). */
42 /**
43 * True when every object is encrypted before it reaches the store (the off-site lane, #11805).
44 */
3845 readonly sealed?: boolean;
3946 /**
4047 * What the line is giving this store's puts (`line.ts`); a store that does not measure leaves it
@@ -102,12 +109,18 @@ export function s3FromEnv(
102109
103110 /** The slowest uplink a put may assume: 5 Mbit/s, a third of the home line it runs behind. */
104111 export const FLOOR_BYTES_PER_S = 5_000_000 / 8;
105 /** What every attempt gets on top of its bytes' time: the handshake, the server's thought, a queue ahead of it. */
112 /**
113 * What every attempt gets on top of its bytes' time: the handshake, the server's thought, a queue
114 * ahead of it.
115 */
106116 export const SLACK_MS = 60_000;
107117 /** How long a read, a listing or a delete may go unanswered. */
108118 export const CALL_DEADLINE_MS = 180_000;
109119
110 /** How long a put of `bytes` may take before it counts as failed: its bytes at the floor rate, plus slack. */
120 /**
121 * How long a put of `bytes` may take before it counts as failed: its bytes at the floor rate, plus
122 * slack.
123 */
111124 export const deadlineFor = (
112125 bytes: number,
113126 floorBytesPerS = FLOOR_BYTES_PER_S,
@@ -181,7 +194,7 @@ export function withRetries(
181194 async function tried<T>(
182195 what: string,
183196 deadlineMs: number | (() => number),
184 fn: () => Promise<T>,
197 fn: (signal: AbortSignal) => Promise<T>,
185198 leftMs?: () => number,
186199 ): Promise<T> {
187200 let last: unknown;
@@ -198,7 +211,7 @@ export function withRetries(
198211 }
199212 made++;
200213 try {
201 return await answered(fn(), Math.min(own, share));
214 return await answered(fn, Math.min(own, share));
202215 } catch (error) {
203216 last = error;
204217 if (n + 1 < attempts) await sleep(baseMs * 2 ** n);
@@ -215,11 +228,11 @@ export function withRetries(
215228 tried(
216229 `put ${key}`,
217230 () => putMs(body),
218 async () => {
231 async (signal) => {
219232 const end = line.begin();
220233 let landed = 0;
221234 try {
222 await inner.put(key, body);
235 await inner.put(key, body, signal);
223236 landed = sizeOf(body);
224237 } finally {
225238 end(landed);
@@ -240,14 +253,21 @@ export function withRetries(
240253
241254 /**
242255 * The attempt's answer, or a rejection once `ms` have passed without one.
243 * The abandoned attempt is left to settle on its own, its outcome ignored.
256 * At the deadline the attempt's signal is aborted, so a store that can
257 * stops sending (#13733: an abandoned put went on uploading beside the
258 * fourteen tries after it); its outcome is ignored either way.
244259 */
245 function answered<T>(attempt: Promise<T>, ms: number): Promise<T> {
260 function answered<T>(start: (signal: AbortSignal) => Promise<T>, ms: number): Promise<T> {
246261 let timer: ReturnType<typeof setTimeout> | undefined;
247262 const shown = ms >= 1000 ? `${Math.round(ms / 1000)} s` : `${ms} ms`;
263 const control = new AbortController();
248264 const deadline = new Promise<never>((_, reject) => {
249265 timer = setTimeout(() => reject(new Error(`no answer in ${shown}`)), ms);
250266 });
267 // The deadline's own rejection aborts the attempt, so the timer stays the
268 // plain reject that test/timers-named.test.ts allows by name.
269 deadline.catch((reason: unknown) => control.abort(reason));
270 const attempt = start(control.signal);
251271 attempt.catch(() => {});
252272 return Promise.race([attempt, deadline]).finally(() => clearTimeout(timer));
253273 }
@@ -261,7 +281,11 @@ export function s3Store(target: S3Target, attempts?: number): Store {
261281 export const s3Where = (target: Pick<S3Target, "endpoint" | "bucket">): string =>
262282 `${target.endpoint.replace(/^https?:\/\//, "")}/${target.bucket}`;
263283
264 function rawS3Store(target: S3Target): Store {
284 /** How long a put's signed URL is good for: longer than any put's deadline. */
285 const PUT_URL_SECONDS = 6 * 3600;
286
287 /** The bucket as a store, each call once: `s3Store` is this with retries. Exported for tests. */
288 export function rawS3Store(target: S3Target): Store {
265289 const client = new Bun.S3Client({
266290 endpoint: target.endpoint,
267291 bucket: target.bucket,
@@ -271,8 +295,16 @@ function rawS3Store(target: S3Target): Store {
271295 });
272296 return {
273297 where: s3Where(target),
274 async put(key, body) {
275 await client.write(key, body);
298 // A presigned PUT through fetch, not `client.write`: only fetch takes a signal, and an
299 // upload given up at its deadline must stop (#13733). The URL is signed here, no network.
300 async put(key, body, signal) {
301 const url = client.presign(key, { method: "PUT", expiresIn: PUT_URL_SECONDS });
302 const res = await fetch(url, { method: "PUT", body, ...(signal === undefined ? {} : { signal }) });
303 if (!res.ok) {
304 const said = (await res.text().catch(() => "")).match(/<Code>([^<]{1,80})<\/Code>/)?.[1];
305 throw new Error(`the bucket answered ${res.status}${said === undefined ? "" : ` ${said}`}`);
306 }
307 await res.body?.cancel().catch(() => {});
276308 },
277309 async get(key) {
278310 const file = client.file(key);
test/snapshot-outcome.test.ts +46 −3
@@ -1,7 +1,7 @@
11 /**
22 * A failed snapshot is seen the same morning (#2536): it is kept under the
33 * data directory, vitals carries it as a card and Backups as a red line,
4 * both pointing at the reason, and the forge tries again within the hour
4 * both pointing at the reason, and the forge tries again in ten minutes
55 * rather than the next night. A snapshot that lands clears it.
66 */
77
@@ -14,7 +14,7 @@ import { forgeNeeds, forgeVitals } from "../src/actions/health.ts";
1414 import { createApp, type App } from "../src/app.ts";
1515 import { createUser, type User } from "../src/core/users.ts";
1616 import { serve, type Servers } from "../src/http/server.ts";
17 import { lastSnapshotOutcome, recordSnapshotOutcome, snapshotNeed } from "../src/snapshot/outcome.ts";
17 import { lastSnapshotOutcome, recordSnapshotOutcome, snapshotNeed, SNAPSHOT_RETRY_MS } from "../src/snapshot/outcome.ts";
1818 import { startSnapshots } from "../src/snapshot/service.ts";
1919 import { dirStore, type Store } from "../src/snapshot/store.ts";
2020
@@ -38,7 +38,7 @@ describe("the outcome kept", () => {
3838 const need = snapshotNeed(lastSnapshotOutcome(dir));
3939 expect(need?.since).toBe("2026-10-17T08:45:00.000Z");
4040 expect(need?.sentence).toBe(
41 "The last snapshot failed: could not put forge.db: no answer in 180 s. The newest that landed is from 2026-10-16. The forge tries again within the hour.",
41 "The last snapshot failed: could not put forge.db: no answer in 180 s. The newest that landed is from 2026-10-16. The forge tries again in 10 minutes.",
4242 );
4343 recordSnapshotOutcome(dir, { landed: "2026-10-18T09-00-00Z" }, "2026-10-18T09:20:00.000Z");
4444 expect(snapshotNeed(lastSnapshotOutcome(dir))).toBeUndefined();
@@ -160,3 +160,46 @@ describe("a forge given nowhere to put a snapshot", () => {
160160 }
161161 });
162162 });
163
164 describe("the next try after a failed snapshot (#13733)", () => {
165 /** Fail one snapshot with the clock at `at`; the delay the forge then armed, ms. */
166 async function armedAfterFailure(at: string): Promise<number> {
167 const root = mkdtempSync(join(tmpdir(), "forge-retry-"));
168 const app = createApp({ dataDir: join(root, "data"), port: 0 });
169 const realNow = Date.now;
170 const realSet = globalThis.setTimeout;
171 const delays: number[] = [];
172 const store: Store = {
173 ...dirStore(join(root, "store")),
174 put: async () => {
175 throw new Error("no answer in 66 s");
176 },
177 };
178 // Only the nightly timer is read; the fixed nightlyInMs keeps the first arm out of the way.
179 const svc = startSnapshots(app, store, { version: "t", nightlyInMs: 1e9 });
180 try {
181 Date.now = () => Date.parse(at);
182 globalThis.setTimeout = ((fn: () => void, ms?: number, ...rest: unknown[]) => {
183 if (typeof ms === "number" && ms >= 60_000) delays.push(ms);
184 return realSet(fn, ms, ...rest);
185 }) as typeof setTimeout;
186 await expect(svc.now()).rejects.toThrow(/no answer in 66 s/);
187 return delays[delays.length - 1]!;
188 } finally {
189 Date.now = realNow;
190 globalThis.setTimeout = realSet;
191 svc.stop();
192 app.close();
193 await rm(root, { recursive: true, force: true });
194 }
195 }
196
197 test("is SNAPSHOT_RETRY_MS away by default, not the hour", async () => {
198 expect(SNAPSHOT_RETRY_MS).toBe(600_000);
199 expect(await armedAfterFailure("2026-11-15T10:00:00Z")).toBe(SNAPSHOT_RETRY_MS);
200 });
201
202 test("is capped at the nightly when that is sooner", async () => {
203 expect(await armedAfterFailure("2026-11-15T07:55:00Z")).toBe(5 * 60_000);
204 });
205 });
test/snapshot-put-abort.test.ts added +94
@@ -0,0 +1,94 @@
1 /**
2 * A put given up at its deadline stops sending (#13733). A snapshot failed for
3 * 83 minutes with "no answer in 63 s" fifteen times over one chunk: each try
4 * that timed out was abandoned but kept uploading, so every retry shared the
5 * line with the ones before it. The deadline now aborts the attempt's request.
6 */
7 import { describe, expect, test } from "bun:test";
8 import { rawS3Store, s3Store, withRetries, type Store } from "../src/snapshot/store.ts";
9 import { bypassProxy, stubBucket } from "./helpers/stub-bucket.ts";
10
11 describe("a put past its deadline", () => {
12 test("has its signal aborted, and the next attempt gets a fresh one", async () => {
13 const signals: AbortSignal[] = [];
14 const inner: Store = {
15 where: "stalls once",
16 async put(_key, _body, signal) {
17 signals.push(signal!);
18 if (signals.length === 1) await new Promise<void>(() => {});
19 },
20 get: async () => undefined,
21 list: async () => [],
22 names: async () => [],
23 delete: async () => {},
24 };
25 const store = withRetries(inner, { attempts: 3, baseMs: 1, deadlineMs: 20, sleep: async () => {} });
26 await store.put("chunks/a.chunk", new Uint8Array(4));
27 expect(signals.length).toBe(2);
28 expect(signals[0]!.aborted).toBe(true);
29 expect(String(signals[0]!.reason)).toContain("no answer in 20 ms");
30 expect(signals[1]!.aborted).toBe(false);
31 });
32
33 test("closes the request to the bucket, through the real client", async () => {
34 const restore = bypassProxy();
35 const arrived = Promise.withResolvers<void>();
36 const closed = Promise.withResolvers<boolean>();
37 const server = Bun.serve({
38 port: 0,
39 hostname: "127.0.0.1",
40 fetch(req) {
41 req.signal.addEventListener("abort", () => closed.resolve(true));
42 arrived.resolve();
43 return new Promise<Response>(() => {});
44 },
45 });
46 try {
47 const store = rawS3Store({ endpoint: `http://127.0.0.1:${server.port}`, bucket: "offsite", accessKeyId: "a", secretAccessKey: "b" });
48 const control = new AbortController();
49 const put = store.put("chunks/b.chunk", new Uint8Array(1024), control.signal);
50 put.catch(() => {});
51 await arrived.promise;
52 control.abort(new Error("no answer in 50 ms"));
53 await expect(put).rejects.toThrow();
54 expect(await closed.promise).toBe(true);
55 } finally {
56 server.stop(true);
57 restore();
58 }
59 });
60 });
61
62 describe("a put through the bucket", () => {
63 test("lands the bytes under the key", async () => {
64 const restore = bypassProxy();
65 const objects = new Map<string, Uint8Array>();
66 const seen: string[] = [];
67 const server = stubBucket(objects, seen);
68 try {
69 const store = s3Store({ endpoint: `http://127.0.0.1:${server.port}`, bucket: "offsite", accessKeyId: "a", secretAccessKey: "b" });
70 await store.put("chunks/c d.chunk", new Uint8Array([1, 2, 3]));
71 expect(seen).toEqual(["PUT chunks/c d.chunk"]);
72 expect([...objects.get("chunks/c d.chunk")!]).toEqual([1, 2, 3]);
73 } finally {
74 server.stop(true);
75 restore();
76 }
77 });
78
79 test("says what the bucket answered when it refuses", async () => {
80 const restore = bypassProxy();
81 const server = Bun.serve({
82 port: 0,
83 hostname: "127.0.0.1",
84 fetch: () => new Response("<Error><Code>AccessDenied</Code></Error>", { status: 403 }),
85 });
86 try {
87 const store = rawS3Store({ endpoint: `http://127.0.0.1:${server.port}`, bucket: "offsite", accessKeyId: "a", secretAccessKey: "b" });
88 await expect(store.put("chunks/e.chunk", "x")).rejects.toThrow("the bucket answered 403 AccessDenied");
89 } finally {
90 server.stop(true);
91 restore();
92 }
93 });
94 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs