Public: anyone can read this; the crew can change it.

Full suite red: test/snapshot.test.ts #12763

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12746-full-suite-red-test-snapshot-test-ts → main 2 commits fork

  • Merged. mason/12746-full-suite-red-test-snapshot-test-ts was merged into main as 3f8d819.

Review

Sign in to review this change.
2 files changed +255 −237
test/snapshot-uploads.test.ts added +253
@@ -0,0 +1,253 @@
1 /**
2 * A snapshot's uploads: how many puts are in flight, how run logs are
3 * bundled, and what the service says while one runs. Split from
4 * snapshot.test.ts so neither file passes the file budget.
5 */
6 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
7 import { mkdirSync, mkdtempSync, readFileSync, utimesSync, writeFileSync } from "node:fs";
8 import { rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { chunkKey } from "../src/snapshot/chunks.ts";
13 import { restoreSnapshot } from "../src/snapshot/restore.ts";
14 import { startSnapshots } from "../src/snapshot/service.ts";
15 import {
16 IN_FLIGHT_BYTES,
17 isRunBundle,
18 packRuns,
19 takeSnapshot,
20 unpackRuns,
21 } from "../src/snapshot/snapshot.ts";
22 import { dirStore, type Store } from "../src/snapshot/store.ts";
23
24 describe("a snapshot's uploads", () => {
25 // Each test gets a forge of its own, so no test reads the logs another wrote.
26 let root: string;
27 let app: App;
28 beforeEach(() => {
29 root = mkdtempSync(join(tmpdir(), "forge-snapshot-queue-"));
30 app = createApp({ dataDir: join(root, "live"), port: 0 });
31 mkdirSync(join(app.config.dataDir, "runs"), { recursive: true });
32 });
33 afterEach(async () => {
34 app.close();
35 await rm(root, { recursive: true, force: true });
36 });
37 const forge = (extra: object = {}) => ({
38 db: app.db,
39 dataDir: app.config.dataDir,
40 reposRoot: app.reposRoot,
41 version: "t",
42 ...extra,
43 });
44 /**
45 * A store that keeps names and, of chunks, no bodies; it counts, and answers after `ms`. `failOn`
46 * is the put (from 1) it refuses.
47 */
48 const slowStore = (ms: number, failOn?: number) => {
49 const seen = { inFlight: 0, peak: 0, bytes: 0, peakBytes: 0, keys: [] as string[], started: [] as string[] };
50 const manifests = new Map<string, Uint8Array>();
51 const store: Store = {
52 ...dirStore(join(root, "unused")),
53 where: "fake",
54 async put(key, body) {
55 const size = typeof body === "string" ? body.length : body.byteLength;
56 seen.started.push(key);
57 seen.inFlight++;
58 seen.bytes += size;
59 seen.peak = Math.max(seen.peak, seen.inFlight);
60 seen.peakBytes = Math.max(seen.peakBytes, seen.bytes);
61 await Bun.sleep(ms); // real-timer: a held put stands in for a slow store so uploads overlap
62 seen.inFlight--;
63 seen.bytes -= size;
64 if (failOn !== undefined && seen.started.length >= failOn && seen.started[failOn - 1] === key)
65 throw new Error("refused");
66 seen.keys.push(key);
67 if (!key.startsWith("chunks/")) manifests.set(key, typeof body === "string" ? new TextEncoder().encode(body) : body);
68 },
69 get: async (key) => manifests.get(key),
70 list: async (prefix) => seen.keys.filter((k) => k.startsWith(prefix)),
71 };
72 return { store, seen };
73 };
74 const logsDir = () => join(app.config.dataDir, "runs");
75 /**
76 * Forty run logs, new-00.log to new-39.log, each last written on a day of its own: forty bundles,
77 * forty chunks.
78 */
79 const newLogs = () => {
80 for (let i = 0; i < 40; i++)
81 old(`new-${String(i).padStart(2, "0")}.log`, `x${i}`, new Date(Date.UTC(2026, 7, 1 + i)).toISOString().slice(0, 10));
82 };
83 const old = (name: string, body: string, day: string) => {
84 const file = join(logsDir(), name);
85 writeFileSync(file, body);
86 const when = new Date(`${day}T12:00:00Z`);
87 utimesSync(file, when, when);
88 };
89
90 test("at most eight puts are in flight, and the manifest keeps its order", async () => {
91 newLogs();
92 const { store, seen } = slowStore(15);
93 const manifest = await takeSnapshot(forge(), store);
94 expect(seen.peak).toBeGreaterThan(1);
95 expect(seen.peak).toBeLessThanOrEqual(8);
96 // Handed over in the manifest's order, whatever order they landed in: each chunk once, the
97 // manifest last.
98 const chunks = [...new Set(manifest.files.flatMap((f) => f.chunks ?? []))];
99 expect(seen.started.slice(0, -1)).toEqual(chunks.map(chunkKey));
100 expect(seen.started.at(-1)).toBe(`snapshots/${manifest.name}/manifest.json`);
101 expect(manifest.files.filter((f) => isRunBundle(f.path)).length).toBe(40);
102 expect(seen.keys.length).toBe(chunks.length + 1);
103 expect(manifest.sent).toEqual({ chunks: chunks.length, bytes: expect.any(Number), held: expect.any(Number) });
104 });
105
106 test("held bytes are bounded too: what is in flight never passes the cap", async () => {
107 // Bytes that do not compress: three logs of 9 MiB, each a chunk of 8 MiB and the rest.
108 for (let i = 0; i < 3; i++) {
109 const big = new Uint8Array(9 * 1024 * 1024);
110 for (let at = 0; at < big.byteLength; at += 65536) crypto.getRandomValues(big.subarray(at, at + 65536));
111 writeFileSync(join(logsDir(), `big-${i}.log`), big);
112 }
113 const { store, seen } = slowStore(30);
114 const manifest = await takeSnapshot(forge(), store);
115 expect(manifest.files.filter((f) => isRunBundle(f.path)).flatMap((f) => f.chunks ?? []).length).toBe(6);
116 expect(seen.peak).toBeLessThanOrEqual(8);
117 expect(seen.peakBytes).toBeGreaterThan(1024 * 1024);
118 expect(seen.peakBytes).toBeLessThanOrEqual(IN_FLIGHT_BYTES);
119 }, 30_000);
120
121 test("a failed put fails the snapshot and names its key", async () => {
122 newLogs();
123 const { store } = slowStore(1, 7);
124 await expect(takeSnapshot(forge(), store)).rejects.toThrow(
125 /could not put chunks\/[0-9a-f]{64}\.chunk: /,
126 );
127 // No manifest: the store lists no snapshot.
128 expect((await store.list("snapshots/")).length).toBe(0);
129 }, 15_000);
130
131 test("a slow store gets a progress line every so many objects", async () => {
132 newLogs();
133 const { store } = slowStore(10);
134 const lines: { objects: number; bytes: number }[] = [];
135 await takeSnapshot(
136 forge({
137 onProgress: (p: { objects: number; bytes: number }) => lines.push(p),
138 progressEvery: { objects: 10, ms: 60_000 },
139 }),
140 store,
141 );
142 expect(lines.length).toBeGreaterThanOrEqual(4);
143 expect(lines[0]!.objects).toBeGreaterThanOrEqual(10);
144 expect(lines.at(-1)!.bytes).toBeGreaterThan(0);
145 // And by the clock when objects are slow to land.
146 const ticks: number[] = [];
147 await takeSnapshot(
148 forge({
149 onProgress: (p: { objects: number }) => ticks.push(p.objects),
150 progressEvery: { objects: 100_000, ms: 20 },
151 }),
152 slowStore(30).store,
153 );
154 expect(ticks.length).toBeGreaterThan(0);
155 });
156
157 test("run logs go up a day to a bundle and restore as the logs they were", async () => {
158 newLogs();
159 old("a.log", "alpha\n", "2026-10-10");
160 old("b.log", "beta\n", "2026-10-10");
161 old("c.log", "gamma\n", "2026-10-11");
162 const store = dirStore(join(root, "bundled"));
163 const manifest = await takeSnapshot(forge(), store);
164 const paths = manifest.files.map((f) => f.path);
165 expect(paths).toContain("runs/2026-10-10.ndjson");
166 expect(paths).toContain("runs/2026-10-11.ndjson");
167 expect(paths).not.toContain("runs/a.log");
168 const target = join(root, "restored");
169 await restoreSnapshot(store, manifest.name, target);
170 expect(readFileSync(join(target, "runs", "a.log"), "utf8")).toBe("alpha\n");
171 expect(readFileSync(join(target, "runs", "b.log"), "utf8")).toBe("beta\n");
172 expect(readFileSync(join(target, "runs", "c.log"), "utf8")).toBe("gamma\n");
173 expect(readFileSync(join(target, "runs", "new-03.log"), "utf8")).toBe("x3");
174 });
175
176 test("a bundle that names a path outside runs is refused", () => {
177 const bad = packRuns([{ path: "../evil", body: new Uint8Array([1]) }]);
178 expect(() => unpackRuns(bad)).toThrow(/names/);
179 });
180
181 test("a bundle is cut before it grows past the cap, and every log still restores", async () => {
182 const big = "z".repeat(1_500_000);
183 for (const n of [1, 2, 3, 4]) old(`big-${n}.log`, big + n, "2026-10-09");
184 const store = dirStore(join(root, "capped"));
185 const manifest = await takeSnapshot(forge(), store);
186 const bundles = manifest.files
187 .map((f) => f.path)
188 .filter((p) => p.startsWith("runs/2026-10-09"));
189 expect(bundles.length).toBeGreaterThan(1);
190 expect(bundles.every(isRunBundle)).toBe(true);
191 const target = join(root, "capped-restored");
192 await restoreSnapshot(store, manifest.name, target);
193 for (const n of [1, 2, 3, 4])
194 expect(readFileSync(join(target, "runs", `big-${n}.log`), "utf8")).toBe(
195 big + n,
196 );
197 }, 15_000);
198
199 test("the service shows a running snapshot's progress while it runs", async () => {
200 newLogs();
201 const { store } = slowStore(40);
202 const svc = startSnapshots(app, store, { version: "t", nightly: false });
203 try {
204 expect(svc.progress()).toBeUndefined();
205 const taking = svc.now();
206 let seen: { objects: number; bytes: number } | undefined;
207 let done = false;
208 void taking.then(() => (done = true));
209 while (!done && (seen?.objects ?? 0) === 0) {
210 seen = svc.progress();
211 await Bun.sleep(10); // real-timer: polling a background snapshot until its first object lands
212 }
213 expect(seen).toBeDefined();
214 expect(seen!.objects).toBeGreaterThan(0);
215 expect(done).toBe(false);
216 await taking;
217 expect(svc.progress()).toBeUndefined();
218 } finally {
219 await svc.stop();
220 }
221 }, 15_000);
222
223 test("the service says how long the snapshot in flight has run, for the release keeper to wait on (#9653)", async () => {
224 newLogs();
225 const { store } = slowStore(40);
226 const svc = startSnapshots(app, store, { version: "t", nightly: false });
227 try {
228 expect(svc.takingMs()).toBeUndefined();
229 const taking = svc.now();
230 await Bun.sleep(30); // real-timer: lets the snapshot run long enough to report elapsed time
231 const ms = svc.takingMs();
232 expect(ms).toBeGreaterThanOrEqual(0);
233 await taking;
234 expect(svc.takingMs()).toBeUndefined();
235 } finally {
236 await svc.stop();
237 }
238 }, 15_000);
239
240 test("until the boot's look at the store is done, the service counts as taking, so no release lands in that gap", async () => {
241 const { store } = slowStore(0);
242 const svc = startSnapshots(app, store, { version: "t", snapshotCatchUpMs: 30, snapshotCheckMs: 60_000 });
243 try {
244 expect(svc.takingMs()).toBeGreaterThanOrEqual(0);
245 await Bun.sleep(400); // real-timer: the catch-up loop must get its turn
246 await svc.now().catch(() => {});
247 await Bun.sleep(50); // real-timer: lets the failed snapshot settle
248 expect(svc.takingMs()).toBeUndefined();
249 } finally {
250 await svc.stop();
251 }
252 }, 15_000);
253 });
test/snapshot.test.ts +2 −237
@@ -6,13 +6,12 @@
66 * data and can still open the variable. Then the store is pruned and the
77 * restore refuses what it must.
88 */
9 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
9 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
1010 import {
1111 existsSync,
1212 mkdirSync,
1313 mkdtempSync,
1414 readFileSync,
15 utimesSync,
1615 writeFileSync,
1716 } from "node:fs";
1817 import { rm } from "node:fs/promises";
@@ -30,20 +29,17 @@ import { createUser, getUserByName, type User } from "../src/core/users.ts";
3029 import { commitLog } from "../src/git/read.ts";
3130 import { repoDir } from "../src/git/repos.ts";
3231 import { readWhole, restoreSnapshot } from "../src/snapshot/restore.ts";
33 import { chunkKey } from "../src/snapshot/chunks.ts";
3432 import { startSnapshots, untilNightly } from "../src/snapshot/service.ts";
3533 import {
36 IN_FLIGHT_BYTES,
3734 isRunBundle,
3835 listSnapshots,
3936 pruneSnapshots,
4037 snapshotName,
41 packRuns,
4238 takeSnapshot,
4339 unpackRuns,
4440 } from "../src/snapshot/snapshot.ts";
4541 import { unwrapKey, wrapKey } from "../src/snapshot/wrap.ts";
46 import { dirStore, type Store } from "../src/snapshot/store.ts";
42 import { dirStore, } from "../src/snapshot/store.ts";
4743
4844 const PASS = "correct horse battery staple";
4945
@@ -429,234 +425,3 @@ describe("a forge back from a snapshot", () => {
429425 }
430426 });
431427 });
432
433 describe("a snapshot's uploads", () => {
434 // Each test gets a forge of its own, so no test reads the logs another wrote.
435 let root: string;
436 let app: App;
437 beforeEach(() => {
438 root = mkdtempSync(join(tmpdir(), "forge-snapshot-queue-"));
439 app = createApp({ dataDir: join(root, "live"), port: 0 });
440 mkdirSync(join(app.config.dataDir, "runs"), { recursive: true });
441 });
442 afterEach(async () => {
443 app.close();
444 await rm(root, { recursive: true, force: true });
445 });
446 const forge = (extra: object = {}) => ({
447 db: app.db,
448 dataDir: app.config.dataDir,
449 reposRoot: app.reposRoot,
450 version: "t",
451 ...extra,
452 });
453 /**
454 * A store that keeps names and, of chunks, no bodies; it counts, and answers after `ms`. `failOn`
455 * is the put (from 1) it refuses.
456 */
457 const slowStore = (ms: number, failOn?: number) => {
458 const seen = { inFlight: 0, peak: 0, bytes: 0, peakBytes: 0, keys: [] as string[], started: [] as string[] };
459 const manifests = new Map<string, Uint8Array>();
460 const store: Store = {
461 ...dirStore(join(root, "unused")),
462 where: "fake",
463 async put(key, body) {
464 const size = typeof body === "string" ? body.length : body.byteLength;
465 seen.started.push(key);
466 seen.inFlight++;
467 seen.bytes += size;
468 seen.peak = Math.max(seen.peak, seen.inFlight);
469 seen.peakBytes = Math.max(seen.peakBytes, seen.bytes);
470 await Bun.sleep(ms);
471 seen.inFlight--;
472 seen.bytes -= size;
473 if (failOn !== undefined && seen.started.length >= failOn && seen.started[failOn - 1] === key)
474 throw new Error("refused");
475 seen.keys.push(key);
476 if (!key.startsWith("chunks/")) manifests.set(key, typeof body === "string" ? new TextEncoder().encode(body) : body);
477 },
478 get: async (key) => manifests.get(key),
479 list: async (prefix) => seen.keys.filter((k) => k.startsWith(prefix)),
480 };
481 return { store, seen };
482 };
483 const logsDir = () => join(app.config.dataDir, "runs");
484 /**
485 * Forty run logs, new-00.log to new-39.log, each last written on a day of its own: forty bundles,
486 * forty chunks.
487 */
488 const newLogs = () => {
489 for (let i = 0; i < 40; i++)
490 old(`new-${String(i).padStart(2, "0")}.log`, `x${i}`, new Date(Date.UTC(2026, 7, 1 + i)).toISOString().slice(0, 10));
491 };
492 const old = (name: string, body: string, day: string) => {
493 const file = join(logsDir(), name);
494 writeFileSync(file, body);
495 const when = new Date(`${day}T12:00:00Z`);
496 utimesSync(file, when, when);
497 };
498
499 test("at most eight puts are in flight, and the manifest keeps its order", async () => {
500 newLogs();
501 const { store, seen } = slowStore(15);
502 const manifest = await takeSnapshot(forge(), store);
503 expect(seen.peak).toBeGreaterThan(1);
504 expect(seen.peak).toBeLessThanOrEqual(8);
505 // Handed over in the manifest's order, whatever order they landed in: each chunk once, the
506 // manifest last.
507 const chunks = [...new Set(manifest.files.flatMap((f) => f.chunks ?? []))];
508 expect(seen.started.slice(0, -1)).toEqual(chunks.map(chunkKey));
509 expect(seen.started.at(-1)).toBe(`snapshots/${manifest.name}/manifest.json`);
510 expect(manifest.files.filter((f) => isRunBundle(f.path)).length).toBe(40);
511 expect(seen.keys.length).toBe(chunks.length + 1);
512 expect(manifest.sent).toEqual({ chunks: chunks.length, bytes: expect.any(Number), held: expect.any(Number) });
513 });
514
515 test("held bytes are bounded too: what is in flight never passes the cap", async () => {
516 // Bytes that do not compress: three logs of 9 MiB, each a chunk of 8 MiB and the rest.
517 for (let i = 0; i < 3; i++) {
518 const big = new Uint8Array(9 * 1024 * 1024);
519 for (let at = 0; at < big.byteLength; at += 65536) crypto.getRandomValues(big.subarray(at, at + 65536));
520 writeFileSync(join(logsDir(), `big-${i}.log`), big);
521 }
522 const { store, seen } = slowStore(30);
523 const manifest = await takeSnapshot(forge(), store);
524 expect(manifest.files.filter((f) => isRunBundle(f.path)).flatMap((f) => f.chunks ?? []).length).toBe(6);
525 expect(seen.peak).toBeLessThanOrEqual(8);
526 expect(seen.peakBytes).toBeGreaterThan(1024 * 1024);
527 expect(seen.peakBytes).toBeLessThanOrEqual(IN_FLIGHT_BYTES);
528 }, 30_000);
529
530 test("a failed put fails the snapshot and names its key", async () => {
531 newLogs();
532 const { store } = slowStore(1, 7);
533 await expect(takeSnapshot(forge(), store)).rejects.toThrow(
534 /could not put chunks\/[0-9a-f]{64}\.chunk: /,
535 );
536 // No manifest: the store lists no snapshot.
537 expect((await store.list("snapshots/")).length).toBe(0);
538 }, 15_000);
539
540 test("a slow store gets a progress line every so many objects", async () => {
541 newLogs();
542 const { store } = slowStore(10);
543 const lines: { objects: number; bytes: number }[] = [];
544 await takeSnapshot(
545 forge({
546 onProgress: (p: { objects: number; bytes: number }) => lines.push(p),
547 progressEvery: { objects: 10, ms: 60_000 },
548 }),
549 store,
550 );
551 expect(lines.length).toBeGreaterThanOrEqual(4);
552 expect(lines[0]!.objects).toBeGreaterThanOrEqual(10);
553 expect(lines.at(-1)!.bytes).toBeGreaterThan(0);
554 // And by the clock when objects are slow to land.
555 const ticks: number[] = [];
556 await takeSnapshot(
557 forge({
558 onProgress: (p: { objects: number }) => ticks.push(p.objects),
559 progressEvery: { objects: 100_000, ms: 20 },
560 }),
561 slowStore(30).store,
562 );
563 expect(ticks.length).toBeGreaterThan(0);
564 });
565
566 test("run logs go up a day to a bundle and restore as the logs they were", async () => {
567 newLogs();
568 old("a.log", "alpha\n", "2026-10-10");
569 old("b.log", "beta\n", "2026-10-10");
570 old("c.log", "gamma\n", "2026-10-11");
571 const store = dirStore(join(root, "bundled"));
572 const manifest = await takeSnapshot(forge(), store);
573 const paths = manifest.files.map((f) => f.path);
574 expect(paths).toContain("runs/2026-10-10.ndjson");
575 expect(paths).toContain("runs/2026-10-11.ndjson");
576 expect(paths).not.toContain("runs/a.log");
577 const target = join(root, "restored");
578 await restoreSnapshot(store, manifest.name, target);
579 expect(readFileSync(join(target, "runs", "a.log"), "utf8")).toBe("alpha\n");
580 expect(readFileSync(join(target, "runs", "b.log"), "utf8")).toBe("beta\n");
581 expect(readFileSync(join(target, "runs", "c.log"), "utf8")).toBe("gamma\n");
582 expect(readFileSync(join(target, "runs", "new-03.log"), "utf8")).toBe("x3");
583 });
584
585 test("a bundle that names a path outside runs is refused", () => {
586 const bad = packRuns([{ path: "../evil", body: new Uint8Array([1]) }]);
587 expect(() => unpackRuns(bad)).toThrow(/names/);
588 });
589
590 test("a bundle is cut before it grows past the cap, and every log still restores", async () => {
591 const big = "z".repeat(1_500_000);
592 for (const n of [1, 2, 3, 4]) old(`big-${n}.log`, big + n, "2026-10-09");
593 const store = dirStore(join(root, "capped"));
594 const manifest = await takeSnapshot(forge(), store);
595 const bundles = manifest.files
596 .map((f) => f.path)
597 .filter((p) => p.startsWith("runs/2026-10-09"));
598 expect(bundles.length).toBeGreaterThan(1);
599 expect(bundles.every(isRunBundle)).toBe(true);
600 const target = join(root, "capped-restored");
601 await restoreSnapshot(store, manifest.name, target);
602 for (const n of [1, 2, 3, 4])
603 expect(readFileSync(join(target, "runs", `big-${n}.log`), "utf8")).toBe(
604 big + n,
605 );
606 }, 15_000);
607
608 test("the service shows a running snapshot's progress while it runs", async () => {
609 newLogs();
610 const { store } = slowStore(40);
611 const svc = startSnapshots(app, store, { version: "t", nightly: false });
612 try {
613 expect(svc.progress()).toBeUndefined();
614 const taking = svc.now();
615 let seen: { objects: number; bytes: number } | undefined;
616 let done = false;
617 void taking.then(() => (done = true));
618 while (!done && (seen?.objects ?? 0) === 0) {
619 seen = svc.progress();
620 await Bun.sleep(10);
621 }
622 expect(seen).toBeDefined();
623 expect(seen!.objects).toBeGreaterThan(0);
624 expect(done).toBe(false);
625 await taking;
626 expect(svc.progress()).toBeUndefined();
627 } finally {
628 await svc.stop();
629 }
630 }, 15_000);
631
632 test("the service says how long the snapshot in flight has run, for the release keeper to wait on (#9653)", async () => {
633 newLogs();
634 const { store } = slowStore(40);
635 const svc = startSnapshots(app, store, { version: "t", nightly: false });
636 try {
637 expect(svc.takingMs()).toBeUndefined();
638 const taking = svc.now();
639 await Bun.sleep(30);
640 const ms = svc.takingMs();
641 expect(ms).toBeGreaterThanOrEqual(0);
642 await taking;
643 expect(svc.takingMs()).toBeUndefined();
644 } finally {
645 await svc.stop();
646 }
647 }, 15_000);
648
649 test("until the boot's look at the store is done, the service counts as taking, so no release lands in that gap", async () => {
650 const { store } = slowStore(0);
651 const svc = startSnapshots(app, store, { version: "t", snapshotCatchUpMs: 30, snapshotCheckMs: 60_000 });
652 try {
653 expect(svc.takingMs()).toBeGreaterThanOrEqual(0);
654 await Bun.sleep(400);
655 await svc.now().catch(() => {});
656 await Bun.sleep(50);
657 expect(svc.takingMs()).toBeUndefined();
658 } finally {
659 await svc.stop();
660 }
661 }, 15_000);
662 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs