Public: anyone can read this; the crew can change it.
Full suite red: test/snapshot.test.ts #12763
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12746-full-suite-red-test-snapshot-test-ts → main 2 commits fork
2 files
changed +255
−237
test/snapshot-uploads.test.ts added +253
| @@ -0,0 +1,253 @@ | ||
| 1 | /** | |
| 2 | * A snapshot's uploads: how many puts are in flight, how run logs are | |
| 3 | * bundled, and what the service says while one runs. Split from | |
| 4 | * snapshot.test.ts so neither file passes the file budget. | |
| 5 | */ | |
| 6 | import { afterEach, beforeEach, describe, expect, test } from "bun:test"; | |
| 7 | import { mkdirSync, mkdtempSync, readFileSync, utimesSync, writeFileSync } from "node:fs"; | |
| 8 | import { rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { chunkKey } from "../src/snapshot/chunks.ts"; | |
| 13 | import { restoreSnapshot } from "../src/snapshot/restore.ts"; | |
| 14 | import { startSnapshots } from "../src/snapshot/service.ts"; | |
| 15 | import { | |
| 16 | IN_FLIGHT_BYTES, | |
| 17 | isRunBundle, | |
| 18 | packRuns, | |
| 19 | takeSnapshot, | |
| 20 | unpackRuns, | |
| 21 | } from "../src/snapshot/snapshot.ts"; | |
| 22 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 23 | ||
| 24 | describe("a snapshot's uploads", () => { | |
| 25 | // Each test gets a forge of its own, so no test reads the logs another wrote. | |
| 26 | let root: string; | |
| 27 | let app: App; | |
| 28 | beforeEach(() => { | |
| 29 | root = mkdtempSync(join(tmpdir(), "forge-snapshot-queue-")); | |
| 30 | app = createApp({ dataDir: join(root, "live"), port: 0 }); | |
| 31 | mkdirSync(join(app.config.dataDir, "runs"), { recursive: true }); | |
| 32 | }); | |
| 33 | afterEach(async () => { | |
| 34 | app.close(); | |
| 35 | await rm(root, { recursive: true, force: true }); | |
| 36 | }); | |
| 37 | const forge = (extra: object = {}) => ({ | |
| 38 | db: app.db, | |
| 39 | dataDir: app.config.dataDir, | |
| 40 | reposRoot: app.reposRoot, | |
| 41 | version: "t", | |
| 42 | ...extra, | |
| 43 | }); | |
| 44 | /** | |
| 45 | * A store that keeps names and, of chunks, no bodies; it counts, and answers after `ms`. `failOn` | |
| 46 | * is the put (from 1) it refuses. | |
| 47 | */ | |
| 48 | const slowStore = (ms: number, failOn?: number) => { | |
| 49 | const seen = { inFlight: 0, peak: 0, bytes: 0, peakBytes: 0, keys: [] as string[], started: [] as string[] }; | |
| 50 | const manifests = new Map<string, Uint8Array>(); | |
| 51 | const store: Store = { | |
| 52 | ...dirStore(join(root, "unused")), | |
| 53 | where: "fake", | |
| 54 | async put(key, body) { | |
| 55 | const size = typeof body === "string" ? body.length : body.byteLength; | |
| 56 | seen.started.push(key); | |
| 57 | seen.inFlight++; | |
| 58 | seen.bytes += size; | |
| 59 | seen.peak = Math.max(seen.peak, seen.inFlight); | |
| 60 | seen.peakBytes = Math.max(seen.peakBytes, seen.bytes); | |
| 61 | await Bun.sleep(ms); // real-timer: a held put stands in for a slow store so uploads overlap | |
| 62 | seen.inFlight--; | |
| 63 | seen.bytes -= size; | |
| 64 | if (failOn !== undefined && seen.started.length >= failOn && seen.started[failOn - 1] === key) | |
| 65 | throw new Error("refused"); | |
| 66 | seen.keys.push(key); | |
| 67 | if (!key.startsWith("chunks/")) manifests.set(key, typeof body === "string" ? new TextEncoder().encode(body) : body); | |
| 68 | }, | |
| 69 | get: async (key) => manifests.get(key), | |
| 70 | list: async (prefix) => seen.keys.filter((k) => k.startsWith(prefix)), | |
| 71 | }; | |
| 72 | return { store, seen }; | |
| 73 | }; | |
| 74 | const logsDir = () => join(app.config.dataDir, "runs"); | |
| 75 | /** | |
| 76 | * Forty run logs, new-00.log to new-39.log, each last written on a day of its own: forty bundles, | |
| 77 | * forty chunks. | |
| 78 | */ | |
| 79 | const newLogs = () => { | |
| 80 | for (let i = 0; i < 40; i++) | |
| 81 | old(`new-${String(i).padStart(2, "0")}.log`, `x${i}`, new Date(Date.UTC(2026, 7, 1 + i)).toISOString().slice(0, 10)); | |
| 82 | }; | |
| 83 | const old = (name: string, body: string, day: string) => { | |
| 84 | const file = join(logsDir(), name); | |
| 85 | writeFileSync(file, body); | |
| 86 | const when = new Date(`${day}T12:00:00Z`); | |
| 87 | utimesSync(file, when, when); | |
| 88 | }; | |
| 89 | ||
| 90 | test("at most eight puts are in flight, and the manifest keeps its order", async () => { | |
| 91 | newLogs(); | |
| 92 | const { store, seen } = slowStore(15); | |
| 93 | const manifest = await takeSnapshot(forge(), store); | |
| 94 | expect(seen.peak).toBeGreaterThan(1); | |
| 95 | expect(seen.peak).toBeLessThanOrEqual(8); | |
| 96 | // Handed over in the manifest's order, whatever order they landed in: each chunk once, the | |
| 97 | // manifest last. | |
| 98 | const chunks = [...new Set(manifest.files.flatMap((f) => f.chunks ?? []))]; | |
| 99 | expect(seen.started.slice(0, -1)).toEqual(chunks.map(chunkKey)); | |
| 100 | expect(seen.started.at(-1)).toBe(`snapshots/${manifest.name}/manifest.json`); | |
| 101 | expect(manifest.files.filter((f) => isRunBundle(f.path)).length).toBe(40); | |
| 102 | expect(seen.keys.length).toBe(chunks.length + 1); | |
| 103 | expect(manifest.sent).toEqual({ chunks: chunks.length, bytes: expect.any(Number), held: expect.any(Number) }); | |
| 104 | }); | |
| 105 | ||
| 106 | test("held bytes are bounded too: what is in flight never passes the cap", async () => { | |
| 107 | // Bytes that do not compress: three logs of 9 MiB, each a chunk of 8 MiB and the rest. | |
| 108 | for (let i = 0; i < 3; i++) { | |
| 109 | const big = new Uint8Array(9 * 1024 * 1024); | |
| 110 | for (let at = 0; at < big.byteLength; at += 65536) crypto.getRandomValues(big.subarray(at, at + 65536)); | |
| 111 | writeFileSync(join(logsDir(), `big-${i}.log`), big); | |
| 112 | } | |
| 113 | const { store, seen } = slowStore(30); | |
| 114 | const manifest = await takeSnapshot(forge(), store); | |
| 115 | expect(manifest.files.filter((f) => isRunBundle(f.path)).flatMap((f) => f.chunks ?? []).length).toBe(6); | |
| 116 | expect(seen.peak).toBeLessThanOrEqual(8); | |
| 117 | expect(seen.peakBytes).toBeGreaterThan(1024 * 1024); | |
| 118 | expect(seen.peakBytes).toBeLessThanOrEqual(IN_FLIGHT_BYTES); | |
| 119 | }, 30_000); | |
| 120 | ||
| 121 | test("a failed put fails the snapshot and names its key", async () => { | |
| 122 | newLogs(); | |
| 123 | const { store } = slowStore(1, 7); | |
| 124 | await expect(takeSnapshot(forge(), store)).rejects.toThrow( | |
| 125 | /could not put chunks\/[0-9a-f]{64}\.chunk: /, | |
| 126 | ); | |
| 127 | // No manifest: the store lists no snapshot. | |
| 128 | expect((await store.list("snapshots/")).length).toBe(0); | |
| 129 | }, 15_000); | |
| 130 | ||
| 131 | test("a slow store gets a progress line every so many objects", async () => { | |
| 132 | newLogs(); | |
| 133 | const { store } = slowStore(10); | |
| 134 | const lines: { objects: number; bytes: number }[] = []; | |
| 135 | await takeSnapshot( | |
| 136 | forge({ | |
| 137 | onProgress: (p: { objects: number; bytes: number }) => lines.push(p), | |
| 138 | progressEvery: { objects: 10, ms: 60_000 }, | |
| 139 | }), | |
| 140 | store, | |
| 141 | ); | |
| 142 | expect(lines.length).toBeGreaterThanOrEqual(4); | |
| 143 | expect(lines[0]!.objects).toBeGreaterThanOrEqual(10); | |
| 144 | expect(lines.at(-1)!.bytes).toBeGreaterThan(0); | |
| 145 | // And by the clock when objects are slow to land. | |
| 146 | const ticks: number[] = []; | |
| 147 | await takeSnapshot( | |
| 148 | forge({ | |
| 149 | onProgress: (p: { objects: number }) => ticks.push(p.objects), | |
| 150 | progressEvery: { objects: 100_000, ms: 20 }, | |
| 151 | }), | |
| 152 | slowStore(30).store, | |
| 153 | ); | |
| 154 | expect(ticks.length).toBeGreaterThan(0); | |
| 155 | }); | |
| 156 | ||
| 157 | test("run logs go up a day to a bundle and restore as the logs they were", async () => { | |
| 158 | newLogs(); | |
| 159 | old("a.log", "alpha\n", "2026-10-10"); | |
| 160 | old("b.log", "beta\n", "2026-10-10"); | |
| 161 | old("c.log", "gamma\n", "2026-10-11"); | |
| 162 | const store = dirStore(join(root, "bundled")); | |
| 163 | const manifest = await takeSnapshot(forge(), store); | |
| 164 | const paths = manifest.files.map((f) => f.path); | |
| 165 | expect(paths).toContain("runs/2026-10-10.ndjson"); | |
| 166 | expect(paths).toContain("runs/2026-10-11.ndjson"); | |
| 167 | expect(paths).not.toContain("runs/a.log"); | |
| 168 | const target = join(root, "restored"); | |
| 169 | await restoreSnapshot(store, manifest.name, target); | |
| 170 | expect(readFileSync(join(target, "runs", "a.log"), "utf8")).toBe("alpha\n"); | |
| 171 | expect(readFileSync(join(target, "runs", "b.log"), "utf8")).toBe("beta\n"); | |
| 172 | expect(readFileSync(join(target, "runs", "c.log"), "utf8")).toBe("gamma\n"); | |
| 173 | expect(readFileSync(join(target, "runs", "new-03.log"), "utf8")).toBe("x3"); | |
| 174 | }); | |
| 175 | ||
| 176 | test("a bundle that names a path outside runs is refused", () => { | |
| 177 | const bad = packRuns([{ path: "../evil", body: new Uint8Array([1]) }]); | |
| 178 | expect(() => unpackRuns(bad)).toThrow(/names/); | |
| 179 | }); | |
| 180 | ||
| 181 | test("a bundle is cut before it grows past the cap, and every log still restores", async () => { | |
| 182 | const big = "z".repeat(1_500_000); | |
| 183 | for (const n of [1, 2, 3, 4]) old(`big-${n}.log`, big + n, "2026-10-09"); | |
| 184 | const store = dirStore(join(root, "capped")); | |
| 185 | const manifest = await takeSnapshot(forge(), store); | |
| 186 | const bundles = manifest.files | |
| 187 | .map((f) => f.path) | |
| 188 | .filter((p) => p.startsWith("runs/2026-10-09")); | |
| 189 | expect(bundles.length).toBeGreaterThan(1); | |
| 190 | expect(bundles.every(isRunBundle)).toBe(true); | |
| 191 | const target = join(root, "capped-restored"); | |
| 192 | await restoreSnapshot(store, manifest.name, target); | |
| 193 | for (const n of [1, 2, 3, 4]) | |
| 194 | expect(readFileSync(join(target, "runs", `big-${n}.log`), "utf8")).toBe( | |
| 195 | big + n, | |
| 196 | ); | |
| 197 | }, 15_000); | |
| 198 | ||
| 199 | test("the service shows a running snapshot's progress while it runs", async () => { | |
| 200 | newLogs(); | |
| 201 | const { store } = slowStore(40); | |
| 202 | const svc = startSnapshots(app, store, { version: "t", nightly: false }); | |
| 203 | try { | |
| 204 | expect(svc.progress()).toBeUndefined(); | |
| 205 | const taking = svc.now(); | |
| 206 | let seen: { objects: number; bytes: number } | undefined; | |
| 207 | let done = false; | |
| 208 | void taking.then(() => (done = true)); | |
| 209 | while (!done && (seen?.objects ?? 0) === 0) { | |
| 210 | seen = svc.progress(); | |
| 211 | await Bun.sleep(10); // real-timer: polling a background snapshot until its first object lands | |
| 212 | } | |
| 213 | expect(seen).toBeDefined(); | |
| 214 | expect(seen!.objects).toBeGreaterThan(0); | |
| 215 | expect(done).toBe(false); | |
| 216 | await taking; | |
| 217 | expect(svc.progress()).toBeUndefined(); | |
| 218 | } finally { | |
| 219 | await svc.stop(); | |
| 220 | } | |
| 221 | }, 15_000); | |
| 222 | ||
| 223 | test("the service says how long the snapshot in flight has run, for the release keeper to wait on (#9653)", async () => { | |
| 224 | newLogs(); | |
| 225 | const { store } = slowStore(40); | |
| 226 | const svc = startSnapshots(app, store, { version: "t", nightly: false }); | |
| 227 | try { | |
| 228 | expect(svc.takingMs()).toBeUndefined(); | |
| 229 | const taking = svc.now(); | |
| 230 | await Bun.sleep(30); // real-timer: lets the snapshot run long enough to report elapsed time | |
| 231 | const ms = svc.takingMs(); | |
| 232 | expect(ms).toBeGreaterThanOrEqual(0); | |
| 233 | await taking; | |
| 234 | expect(svc.takingMs()).toBeUndefined(); | |
| 235 | } finally { | |
| 236 | await svc.stop(); | |
| 237 | } | |
| 238 | }, 15_000); | |
| 239 | ||
| 240 | test("until the boot's look at the store is done, the service counts as taking, so no release lands in that gap", async () => { | |
| 241 | const { store } = slowStore(0); | |
| 242 | const svc = startSnapshots(app, store, { version: "t", snapshotCatchUpMs: 30, snapshotCheckMs: 60_000 }); | |
| 243 | try { | |
| 244 | expect(svc.takingMs()).toBeGreaterThanOrEqual(0); | |
| 245 | await Bun.sleep(400); // real-timer: the catch-up loop must get its turn | |
| 246 | await svc.now().catch(() => {}); | |
| 247 | await Bun.sleep(50); // real-timer: lets the failed snapshot settle | |
| 248 | expect(svc.takingMs()).toBeUndefined(); | |
| 249 | } finally { | |
| 250 | await svc.stop(); | |
| 251 | } | |
| 252 | }, 15_000); | |
| 253 | }); |
test/snapshot.test.ts +2 −237
| @@ -6,13 +6,12 @@ | ||
| 6 | 6 | * data and can still open the variable. Then the store is pruned and the |
| 7 | 7 | * restore refuses what it must. |
| 8 | 8 | */ |
| 9 | import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; | |
| 9 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 10 | 10 | import { |
| 11 | 11 | existsSync, |
| 12 | 12 | mkdirSync, |
| 13 | 13 | mkdtempSync, |
| 14 | 14 | readFileSync, |
| 15 | utimesSync, | |
| 16 | 15 | writeFileSync, |
| 17 | 16 | } from "node:fs"; |
| 18 | 17 | import { rm } from "node:fs/promises"; |
| @@ -30,20 +29,17 @@ import { createUser, getUserByName, type User } from "../src/core/users.ts"; | ||
| 30 | 29 | import { commitLog } from "../src/git/read.ts"; |
| 31 | 30 | import { repoDir } from "../src/git/repos.ts"; |
| 32 | 31 | import { readWhole, restoreSnapshot } from "../src/snapshot/restore.ts"; |
| 33 | import { chunkKey } from "../src/snapshot/chunks.ts"; | |
| 34 | 32 | import { startSnapshots, untilNightly } from "../src/snapshot/service.ts"; |
| 35 | 33 | import { |
| 36 | IN_FLIGHT_BYTES, | |
| 37 | 34 | isRunBundle, |
| 38 | 35 | listSnapshots, |
| 39 | 36 | pruneSnapshots, |
| 40 | 37 | snapshotName, |
| 41 | packRuns, | |
| 42 | 38 | takeSnapshot, |
| 43 | 39 | unpackRuns, |
| 44 | 40 | } from "../src/snapshot/snapshot.ts"; |
| 45 | 41 | import { unwrapKey, wrapKey } from "../src/snapshot/wrap.ts"; |
| 46 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 42 | import { dirStore, } from "../src/snapshot/store.ts"; | |
| 47 | 43 | |
| 48 | 44 | const PASS = "correct horse battery staple"; |
| 49 | 45 | |
| @@ -429,234 +425,3 @@ describe("a forge back from a snapshot", () => { | ||
| 429 | 425 | } |
| 430 | 426 | }); |
| 431 | 427 | }); |
| 432 | ||
| 433 | describe("a snapshot's uploads", () => { | |
| 434 | // Each test gets a forge of its own, so no test reads the logs another wrote. | |
| 435 | let root: string; | |
| 436 | let app: App; | |
| 437 | beforeEach(() => { | |
| 438 | root = mkdtempSync(join(tmpdir(), "forge-snapshot-queue-")); | |
| 439 | app = createApp({ dataDir: join(root, "live"), port: 0 }); | |
| 440 | mkdirSync(join(app.config.dataDir, "runs"), { recursive: true }); | |
| 441 | }); | |
| 442 | afterEach(async () => { | |
| 443 | app.close(); | |
| 444 | await rm(root, { recursive: true, force: true }); | |
| 445 | }); | |
| 446 | const forge = (extra: object = {}) => ({ | |
| 447 | db: app.db, | |
| 448 | dataDir: app.config.dataDir, | |
| 449 | reposRoot: app.reposRoot, | |
| 450 | version: "t", | |
| 451 | ...extra, | |
| 452 | }); | |
| 453 | /** | |
| 454 | * A store that keeps names and, of chunks, no bodies; it counts, and answers after `ms`. `failOn` | |
| 455 | * is the put (from 1) it refuses. | |
| 456 | */ | |
| 457 | const slowStore = (ms: number, failOn?: number) => { | |
| 458 | const seen = { inFlight: 0, peak: 0, bytes: 0, peakBytes: 0, keys: [] as string[], started: [] as string[] }; | |
| 459 | const manifests = new Map<string, Uint8Array>(); | |
| 460 | const store: Store = { | |
| 461 | ...dirStore(join(root, "unused")), | |
| 462 | where: "fake", | |
| 463 | async put(key, body) { | |
| 464 | const size = typeof body === "string" ? body.length : body.byteLength; | |
| 465 | seen.started.push(key); | |
| 466 | seen.inFlight++; | |
| 467 | seen.bytes += size; | |
| 468 | seen.peak = Math.max(seen.peak, seen.inFlight); | |
| 469 | seen.peakBytes = Math.max(seen.peakBytes, seen.bytes); | |
| 470 | await Bun.sleep(ms); | |
| 471 | seen.inFlight--; | |
| 472 | seen.bytes -= size; | |
| 473 | if (failOn !== undefined && seen.started.length >= failOn && seen.started[failOn - 1] === key) | |
| 474 | throw new Error("refused"); | |
| 475 | seen.keys.push(key); | |
| 476 | if (!key.startsWith("chunks/")) manifests.set(key, typeof body === "string" ? new TextEncoder().encode(body) : body); | |
| 477 | }, | |
| 478 | get: async (key) => manifests.get(key), | |
| 479 | list: async (prefix) => seen.keys.filter((k) => k.startsWith(prefix)), | |
| 480 | }; | |
| 481 | return { store, seen }; | |
| 482 | }; | |
| 483 | const logsDir = () => join(app.config.dataDir, "runs"); | |
| 484 | /** | |
| 485 | * Forty run logs, new-00.log to new-39.log, each last written on a day of its own: forty bundles, | |
| 486 | * forty chunks. | |
| 487 | */ | |
| 488 | const newLogs = () => { | |
| 489 | for (let i = 0; i < 40; i++) | |
| 490 | old(`new-${String(i).padStart(2, "0")}.log`, `x${i}`, new Date(Date.UTC(2026, 7, 1 + i)).toISOString().slice(0, 10)); | |
| 491 | }; | |
| 492 | const old = (name: string, body: string, day: string) => { | |
| 493 | const file = join(logsDir(), name); | |
| 494 | writeFileSync(file, body); | |
| 495 | const when = new Date(`${day}T12:00:00Z`); | |
| 496 | utimesSync(file, when, when); | |
| 497 | }; | |
| 498 | ||
| 499 | test("at most eight puts are in flight, and the manifest keeps its order", async () => { | |
| 500 | newLogs(); | |
| 501 | const { store, seen } = slowStore(15); | |
| 502 | const manifest = await takeSnapshot(forge(), store); | |
| 503 | expect(seen.peak).toBeGreaterThan(1); | |
| 504 | expect(seen.peak).toBeLessThanOrEqual(8); | |
| 505 | // Handed over in the manifest's order, whatever order they landed in: each chunk once, the | |
| 506 | // manifest last. | |
| 507 | const chunks = [...new Set(manifest.files.flatMap((f) => f.chunks ?? []))]; | |
| 508 | expect(seen.started.slice(0, -1)).toEqual(chunks.map(chunkKey)); | |
| 509 | expect(seen.started.at(-1)).toBe(`snapshots/${manifest.name}/manifest.json`); | |
| 510 | expect(manifest.files.filter((f) => isRunBundle(f.path)).length).toBe(40); | |
| 511 | expect(seen.keys.length).toBe(chunks.length + 1); | |
| 512 | expect(manifest.sent).toEqual({ chunks: chunks.length, bytes: expect.any(Number), held: expect.any(Number) }); | |
| 513 | }); | |
| 514 | ||
| 515 | test("held bytes are bounded too: what is in flight never passes the cap", async () => { | |
| 516 | // Bytes that do not compress: three logs of 9 MiB, each a chunk of 8 MiB and the rest. | |
| 517 | for (let i = 0; i < 3; i++) { | |
| 518 | const big = new Uint8Array(9 * 1024 * 1024); | |
| 519 | for (let at = 0; at < big.byteLength; at += 65536) crypto.getRandomValues(big.subarray(at, at + 65536)); | |
| 520 | writeFileSync(join(logsDir(), `big-${i}.log`), big); | |
| 521 | } | |
| 522 | const { store, seen } = slowStore(30); | |
| 523 | const manifest = await takeSnapshot(forge(), store); | |
| 524 | expect(manifest.files.filter((f) => isRunBundle(f.path)).flatMap((f) => f.chunks ?? []).length).toBe(6); | |
| 525 | expect(seen.peak).toBeLessThanOrEqual(8); | |
| 526 | expect(seen.peakBytes).toBeGreaterThan(1024 * 1024); | |
| 527 | expect(seen.peakBytes).toBeLessThanOrEqual(IN_FLIGHT_BYTES); | |
| 528 | }, 30_000); | |
| 529 | ||
| 530 | test("a failed put fails the snapshot and names its key", async () => { | |
| 531 | newLogs(); | |
| 532 | const { store } = slowStore(1, 7); | |
| 533 | await expect(takeSnapshot(forge(), store)).rejects.toThrow( | |
| 534 | /could not put chunks\/[0-9a-f]{64}\.chunk: /, | |
| 535 | ); | |
| 536 | // No manifest: the store lists no snapshot. | |
| 537 | expect((await store.list("snapshots/")).length).toBe(0); | |
| 538 | }, 15_000); | |
| 539 | ||
| 540 | test("a slow store gets a progress line every so many objects", async () => { | |
| 541 | newLogs(); | |
| 542 | const { store } = slowStore(10); | |
| 543 | const lines: { objects: number; bytes: number }[] = []; | |
| 544 | await takeSnapshot( | |
| 545 | forge({ | |
| 546 | onProgress: (p: { objects: number; bytes: number }) => lines.push(p), | |
| 547 | progressEvery: { objects: 10, ms: 60_000 }, | |
| 548 | }), | |
| 549 | store, | |
| 550 | ); | |
| 551 | expect(lines.length).toBeGreaterThanOrEqual(4); | |
| 552 | expect(lines[0]!.objects).toBeGreaterThanOrEqual(10); | |
| 553 | expect(lines.at(-1)!.bytes).toBeGreaterThan(0); | |
| 554 | // And by the clock when objects are slow to land. | |
| 555 | const ticks: number[] = []; | |
| 556 | await takeSnapshot( | |
| 557 | forge({ | |
| 558 | onProgress: (p: { objects: number }) => ticks.push(p.objects), | |
| 559 | progressEvery: { objects: 100_000, ms: 20 }, | |
| 560 | }), | |
| 561 | slowStore(30).store, | |
| 562 | ); | |
| 563 | expect(ticks.length).toBeGreaterThan(0); | |
| 564 | }); | |
| 565 | ||
| 566 | test("run logs go up a day to a bundle and restore as the logs they were", async () => { | |
| 567 | newLogs(); | |
| 568 | old("a.log", "alpha\n", "2026-10-10"); | |
| 569 | old("b.log", "beta\n", "2026-10-10"); | |
| 570 | old("c.log", "gamma\n", "2026-10-11"); | |
| 571 | const store = dirStore(join(root, "bundled")); | |
| 572 | const manifest = await takeSnapshot(forge(), store); | |
| 573 | const paths = manifest.files.map((f) => f.path); | |
| 574 | expect(paths).toContain("runs/2026-10-10.ndjson"); | |
| 575 | expect(paths).toContain("runs/2026-10-11.ndjson"); | |
| 576 | expect(paths).not.toContain("runs/a.log"); | |
| 577 | const target = join(root, "restored"); | |
| 578 | await restoreSnapshot(store, manifest.name, target); | |
| 579 | expect(readFileSync(join(target, "runs", "a.log"), "utf8")).toBe("alpha\n"); | |
| 580 | expect(readFileSync(join(target, "runs", "b.log"), "utf8")).toBe("beta\n"); | |
| 581 | expect(readFileSync(join(target, "runs", "c.log"), "utf8")).toBe("gamma\n"); | |
| 582 | expect(readFileSync(join(target, "runs", "new-03.log"), "utf8")).toBe("x3"); | |
| 583 | }); | |
| 584 | ||
| 585 | test("a bundle that names a path outside runs is refused", () => { | |
| 586 | const bad = packRuns([{ path: "../evil", body: new Uint8Array([1]) }]); | |
| 587 | expect(() => unpackRuns(bad)).toThrow(/names/); | |
| 588 | }); | |
| 589 | ||
| 590 | test("a bundle is cut before it grows past the cap, and every log still restores", async () => { | |
| 591 | const big = "z".repeat(1_500_000); | |
| 592 | for (const n of [1, 2, 3, 4]) old(`big-${n}.log`, big + n, "2026-10-09"); | |
| 593 | const store = dirStore(join(root, "capped")); | |
| 594 | const manifest = await takeSnapshot(forge(), store); | |
| 595 | const bundles = manifest.files | |
| 596 | .map((f) => f.path) | |
| 597 | .filter((p) => p.startsWith("runs/2026-10-09")); | |
| 598 | expect(bundles.length).toBeGreaterThan(1); | |
| 599 | expect(bundles.every(isRunBundle)).toBe(true); | |
| 600 | const target = join(root, "capped-restored"); | |
| 601 | await restoreSnapshot(store, manifest.name, target); | |
| 602 | for (const n of [1, 2, 3, 4]) | |
| 603 | expect(readFileSync(join(target, "runs", `big-${n}.log`), "utf8")).toBe( | |
| 604 | big + n, | |
| 605 | ); | |
| 606 | }, 15_000); | |
| 607 | ||
| 608 | test("the service shows a running snapshot's progress while it runs", async () => { | |
| 609 | newLogs(); | |
| 610 | const { store } = slowStore(40); | |
| 611 | const svc = startSnapshots(app, store, { version: "t", nightly: false }); | |
| 612 | try { | |
| 613 | expect(svc.progress()).toBeUndefined(); | |
| 614 | const taking = svc.now(); | |
| 615 | let seen: { objects: number; bytes: number } | undefined; | |
| 616 | let done = false; | |
| 617 | void taking.then(() => (done = true)); | |
| 618 | while (!done && (seen?.objects ?? 0) === 0) { | |
| 619 | seen = svc.progress(); | |
| 620 | await Bun.sleep(10); | |
| 621 | } | |
| 622 | expect(seen).toBeDefined(); | |
| 623 | expect(seen!.objects).toBeGreaterThan(0); | |
| 624 | expect(done).toBe(false); | |
| 625 | await taking; | |
| 626 | expect(svc.progress()).toBeUndefined(); | |
| 627 | } finally { | |
| 628 | await svc.stop(); | |
| 629 | } | |
| 630 | }, 15_000); | |
| 631 | ||
| 632 | test("the service says how long the snapshot in flight has run, for the release keeper to wait on (#9653)", async () => { | |
| 633 | newLogs(); | |
| 634 | const { store } = slowStore(40); | |
| 635 | const svc = startSnapshots(app, store, { version: "t", nightly: false }); | |
| 636 | try { | |
| 637 | expect(svc.takingMs()).toBeUndefined(); | |
| 638 | const taking = svc.now(); | |
| 639 | await Bun.sleep(30); | |
| 640 | const ms = svc.takingMs(); | |
| 641 | expect(ms).toBeGreaterThanOrEqual(0); | |
| 642 | await taking; | |
| 643 | expect(svc.takingMs()).toBeUndefined(); | |
| 644 | } finally { | |
| 645 | await svc.stop(); | |
| 646 | } | |
| 647 | }, 15_000); | |
| 648 | ||
| 649 | test("until the boot's look at the store is done, the service counts as taking, so no release lands in that gap", async () => { | |
| 650 | const { store } = slowStore(0); | |
| 651 | const svc = startSnapshots(app, store, { version: "t", snapshotCatchUpMs: 30, snapshotCheckMs: 60_000 }); | |
| 652 | try { | |
| 653 | expect(svc.takingMs()).toBeGreaterThanOrEqual(0); | |
| 654 | await Bun.sleep(400); | |
| 655 | await svc.now().catch(() => {}); | |
| 656 | await Bun.sleep(50); | |
| 657 | expect(svc.takingMs()).toBeUndefined(); | |
| 658 | } finally { | |
| 659 | await svc.stop(); | |
| 660 | } | |
| 661 | }, 15_000); | |
| 662 | }); |
new above
offline