db2e423 · 909ca51 · 08dccc3
@@ -0,0 +1,170 @@
/**
 * A 400 MB snapshot through a store throttled to the forge's 150 Mbit/s
 * downlink restores, whole and for real, inside the drill's size-scaled
 * budget (#2813): streamed into decompression (#2743), each phase timed
 * (#2742), the budget grown with the size (#2773).
 *
 * Two proofs of the same rule. The full one is 400 MB at 150 Mbit/s (21 s
 * of wire, a 16 min 40 s budget) and writes the 400 MB database to disk, so
 * it runs only where the scratch disk has room (`ROOM`), and is skipped by
 * name where it does not: a check's scratch disk ran out (ENOSPC) at about
 * 320 MB. The scaled one is 1/4 of both, 100 MB at 37.5 Mbit/s (the same
 * 21 s of wire, an 11 min 40 s budget), and always runs, so the streaming,
 * the phase times and the budget are proven everywhere; only the disk
 * write of the full size is not.
 *
 * Both snapshots are generated here, a real `forge.db` followed by
 * incompressible padding, never kept in the repo. Neither has a large
 * repository, so the `bundles` phase is ~0 ms in both: clone time at this
 * size is not proven here.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync, statfsSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { drillBudgetMs, runDrill } from "../src/snapshot/drill.ts";
import { PART_BYTES, partPath, partPaths, unpack } from "../src/snapshot/parts.ts";
import { listSnapshots, takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts";
import { dirStore, type Store } from "../src/snapshot/store.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
const MB = 1_000_000;
/** Free bytes the full run wants on the scratch disk: the 400 MB restored, the rest as margin. */
const ROOM = 1_500_000_000;
const free = (): number => {
  const s = statfsSync(tmpdir());
  return s.bavail * s.bsize;
};
const roomy = free() >= ROOM;
/** Fill `out` (a multiple of 4 bytes) with pseudo-random words, so zstd can only store it. */
function fillNoise(out: Uint8Array): void {
  const words = new Uint32Array(out.buffer, out.byteOffset, out.byteLength / 4);
  let x = 2463534242;
  for (let i = 0; i < words.length; i++) {
    x ^= x << 13;
    x >>>= 0;
    x ^= x >>> 17;
    x ^= x << 5;
    x >>>= 0;
    words[i] = x;
  }
}
describe("a 400 MB snapshot through a 150 Mbit/s store", () => {
  let root = "";
  let app: App;
  let real: Store;
  let base: Manifest;
  let database: Uint8Array;
  beforeAll(async () => {
    root = mkdtempSync(join(tmpdir(), "forge-drill-400-"));
    app = createApp({ dataDir: join(root, "live"), port: 0 });
    const u = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!u.ok) throw new Error(u.error);
    real = dirStore(join(root, "store"));
    await takeSnapshot(
      { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
      real,
      new Date("2026-10-18T08:00:00Z"),
    );
    base = (await listSnapshots(real))[0]!;
    // The database is the snapshot's own copy, the live file being in its WAL.
    const own = base.packed!.find((p) => p.path === "forge.db")!;
    const prefix = `snapshots/${base.name}/`;
    database = await unpack(own, await Promise.all(partPaths(own).map(async (k) => (await real.get(prefix + k))!)));
  });
  afterAll(() => {
    app.close();
    rmSync(root, { recursive: true, force: true });
  });
  /**
   * Drill a snapshot of `wireBytes` served at `mbit` Mbit/s: forge.db then
   * padding to that size on the wire, a database whose tail SQLite never
   * reads, in one zstd frame as the real packer writes it.
   */
  async function prove(wireBytes: number, mbit: number): Promise<void> {
    const prefix = `snapshots/${base.name}/`;
    const wirePerMs = (mbit * MB) / 8 / 1000;
    const db = database;
    const whole = new Uint8Array(db.byteLength + wireBytes);
    whole.set(db);
    fillNoise(whole.subarray(db.byteLength));
    const raw = whole.byteLength;
    const zst = new Uint8Array(await Bun.zstdCompress(whole));
    const parts = Math.ceil(zst.byteLength / PART_BYTES);
    const partFiles = Array.from({ length: parts }, (_, n) => ({
      path: partPath("forge.db", n),
      bytes: zst.subarray(n * PART_BYTES, (n + 1) * PART_BYTES).byteLength,
    }));
    const others = base.files.filter((f) => !f.path.startsWith("forge.db.zst."));
    const files = [...others, ...partFiles];
    const manifest: Manifest = {
      ...base,
      files,
      bytes: files.reduce((n, f) => n + f.bytes, 0),
      packed: [
        ...(base.packed ?? []).filter((p) => p.path !== "forge.db"),
        { path: "forge.db", bytes: raw, packedBytes: zst.byteLength, parts },
      ],
    };
    expect(manifest.bytes).toBeGreaterThanOrEqual(wireBytes);
    // One link for every get: each waits its turn behind the bytes before it.
    let busy = 0;
    const link: Store = {
      ...real,
      where: `throttled ${mbit} Mbit/s`,
      async get(key) {
        if (key === `${prefix}manifest.json`) return new TextEncoder().encode(JSON.stringify(manifest));
        const m = key.startsWith(`${prefix}forge.db.zst.`) ? Number(key.slice(-4)) : -1;
        const body = m >= 0 ? zst.subarray(m * PART_BYTES, (m + 1) * PART_BYTES) : await real.get(key);
        if (body === undefined) return undefined;
        busy = Math.max(performance.now(), busy) + body.byteLength / wirePerMs;
        await Bun.sleep(Math.max(0, busy - performance.now()));
        return body;
      },
    };
    const t = performance.now();
    const r = await runDrill({
      store: link,
      dataDir: join(root, `drill-data-${mbit}`),
      boot: async () => ({ ok: true, reason: "" }),
    });
    const took = performance.now() - t;
    console.log(`drill of ${Math.round(manifest.bytes / MB)} MB at ${mbit} Mbit/s: ${Math.round(took)} ms; ${r.phases.map((p) => `${p.phase} ${p.ms} ms`).join(", ")}`);
    expect(r.reason).toBeNull();
    expect(r.verdict).toBe("green");
    expect(r.budgetMs).toBe(drillBudgetMs(manifest.bytes));
    expect(r.budgetMs).toBeGreaterThanOrEqual(10 * 60_000 + (wireBytes / MB) * 1_000);
    expect(r.ms).toBeLessThan(r.budgetMs);
    expect(r.phases.map((p) => p.phase)).toEqual(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]);
    // The wire is the floor of the download; every phase has a time.
    const download = r.phases.find((p) => p.phase === "files")!.ms;
    expect(download).toBeGreaterThanOrEqual(manifest.bytes / wirePerMs - 500);
    for (const p of r.phases) expect(p.ms).toBeGreaterThanOrEqual(0);
  }
  test.skipIf(!roomy)("400 MB at 150 Mbit/s restores green inside its 16 min 40 s budget, each phase timed", async () => {
    await prove(400 * MB, 150);
  }, 90_000);
  test("scaled by 4, 100 MB at 37.5 Mbit/s restores green inside its 11 min 40 s budget, each phase timed", async () => {
    await prove(100 * MB, 37.5);
  }, 90_000);
});
		@@ -0,0 +1,170 @@
	1	/**
	2	* A 400 MB snapshot through a store throttled to the forge's 150 Mbit/s
	3	* downlink restores, whole and for real, inside the drill's size-scaled
	4	* budget (#2813): streamed into decompression (#2743), each phase timed
	5	* (#2742), the budget grown with the size (#2773).
	6	*
	7	* Two proofs of the same rule. The full one is 400 MB at 150 Mbit/s (21 s
	8	* of wire, a 16 min 40 s budget) and writes the 400 MB database to disk, so
	9	* it runs only where the scratch disk has room (`ROOM`), and is skipped by
	10	* name where it does not: a check's scratch disk ran out (ENOSPC) at about
	11	* 320 MB. The scaled one is 1/4 of both, 100 MB at 37.5 Mbit/s (the same
	12	* 21 s of wire, an 11 min 40 s budget), and always runs, so the streaming,
	13	* the phase times and the budget are proven everywhere; only the disk
	14	* write of the full size is not.
	15	*
	16	* Both snapshots are generated here, a real `forge.db` followed by
	17	* incompressible padding, never kept in the repo. Neither has a large
	18	* repository, so the `bundles` phase is ~0 ms in both: clone time at this
	19	* size is not proven here.
	20	*/
	21	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	22	import { mkdtempSync, rmSync, statfsSync } from "node:fs";
	23	import { tmpdir } from "node:os";
	24	import { join } from "node:path";
	25	import { createApp, type App } from "../src/app.ts";
	26	import { createUser } from "../src/core/users.ts";
	27	import { drillBudgetMs, runDrill } from "../src/snapshot/drill.ts";
	28	import { PART_BYTES, partPath, partPaths, unpack } from "../src/snapshot/parts.ts";
	29	import { listSnapshots, takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts";
	30	import { dirStore, type Store } from "../src/snapshot/store.ts";
	31	import { quietLogs } from "./helpers/globals.ts";
	32
	33	quietLogs();
	34
	35	const MB = 1_000_000;
	36	/** Free bytes the full run wants on the scratch disk: the 400 MB restored, the rest as margin. */
	37	const ROOM = 1_500_000_000;
	38	const free = (): number => {
	39	const s = statfsSync(tmpdir());
	40	return s.bavail * s.bsize;
	41	};
	42	const roomy = free() >= ROOM;
	43
	44	/** Fill `out` (a multiple of 4 bytes) with pseudo-random words, so zstd can only store it. */
	45	function fillNoise(out: Uint8Array): void {
	46	const words = new Uint32Array(out.buffer, out.byteOffset, out.byteLength / 4);
	47	let x = 2463534242;
	48	for (let i = 0; i < words.length; i++) {
	49	x ^= x << 13;
	50	x >>>= 0;
	51	x ^= x >>> 17;
	52	x ^= x << 5;
	53	x >>>= 0;
	54	words[i] = x;
	55	}
	56	}
	57
	58	describe("a 400 MB snapshot through a 150 Mbit/s store", () => {
	59	let root = "";
	60	let app: App;
	61	let real: Store;
	62	let base: Manifest;
	63	let database: Uint8Array;
	64
	65	beforeAll(async () => {
	66	root = mkdtempSync(join(tmpdir(), "forge-drill-400-"));
	67	app = createApp({ dataDir: join(root, "live"), port: 0 });
	68	const u = await createUser(app.db, {
	69	origin: "seed",
	70	trusted: true,
	71	username: "alice",
	72	email: "[email protected]",
	73	password: "a-good-passphrase",
	74	adminIfFirst: true,
	75	});
	76	if (!u.ok) throw new Error(u.error);
	77	real = dirStore(join(root, "store"));
	78	await takeSnapshot(
	79	{ db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
	80	real,
	81	new Date("2026-10-18T08:00:00Z"),
	82	);
	83	base = (await listSnapshots(real))[0]!;
	84	// The database is the snapshot's own copy, the live file being in its WAL.
	85	const own = base.packed!.find((p) => p.path === "forge.db")!;
	86	const prefix = `snapshots/${base.name}/`;
	87	database = await unpack(own, await Promise.all(partPaths(own).map(async (k) => (await real.get(prefix + k))!)));
	88	});
	89	afterAll(() => {
	90	app.close();
	91	rmSync(root, { recursive: true, force: true });
	92	});
	93
	94	/**
	95	* Drill a snapshot of `wireBytes` served at `mbit` Mbit/s: forge.db then
	96	* padding to that size on the wire, a database whose tail SQLite never
	97	* reads, in one zstd frame as the real packer writes it.
	98	*/
	99	async function prove(wireBytes: number, mbit: number): Promise<void> {
	100	const prefix = `snapshots/${base.name}/`;
	101	const wirePerMs = (mbit * MB) / 8 / 1000;
	102	const db = database;
	103	const whole = new Uint8Array(db.byteLength + wireBytes);
	104	whole.set(db);
	105	fillNoise(whole.subarray(db.byteLength));
	106	const raw = whole.byteLength;
	107	const zst = new Uint8Array(await Bun.zstdCompress(whole));
	108	const parts = Math.ceil(zst.byteLength / PART_BYTES);
	109	const partFiles = Array.from({ length: parts }, (_, n) => ({
	110	path: partPath("forge.db", n),
	111	bytes: zst.subarray(n * PART_BYTES, (n + 1) * PART_BYTES).byteLength,
	112	}));
	113	const others = base.files.filter((f) => !f.path.startsWith("forge.db.zst."));
	114	const files = [...others, ...partFiles];
	115	const manifest: Manifest = {
	116	...base,
	117	files,
	118	bytes: files.reduce((n, f) => n + f.bytes, 0),
	119	packed: [
	120	...(base.packed ?? []).filter((p) => p.path !== "forge.db"),
	121	{ path: "forge.db", bytes: raw, packedBytes: zst.byteLength, parts },
	122	],
	123	};
	124	expect(manifest.bytes).toBeGreaterThanOrEqual(wireBytes);
	125
	126	// One link for every get: each waits its turn behind the bytes before it.
	127	let busy = 0;
	128	const link: Store = {
	129	...real,
	130	where: `throttled ${mbit} Mbit/s`,
	131	async get(key) {
	132	if (key === `${prefix}manifest.json`) return new TextEncoder().encode(JSON.stringify(manifest));
	133	const m = key.startsWith(`${prefix}forge.db.zst.`) ? Number(key.slice(-4)) : -1;
	134	const body = m >= 0 ? zst.subarray(m * PART_BYTES, (m + 1) * PART_BYTES) : await real.get(key);
	135	if (body === undefined) return undefined;
	136	busy = Math.max(performance.now(), busy) + body.byteLength / wirePerMs;
	137	await Bun.sleep(Math.max(0, busy - performance.now()));
	138	return body;
	139	},
	140	};
	141
	142	const t = performance.now();
	143	const r = await runDrill({
	144	store: link,
	145	dataDir: join(root, `drill-data-${mbit}`),
	146	boot: async () => ({ ok: true, reason: "" }),
	147	});
	148	const took = performance.now() - t;
	149	console.log(`drill of ${Math.round(manifest.bytes / MB)} MB at ${mbit} Mbit/s: ${Math.round(took)} ms; ${r.phases.map((p) => `${p.phase} ${p.ms} ms`).join(", ")}`);
	150
	151	expect(r.reason).toBeNull();
	152	expect(r.verdict).toBe("green");
	153	expect(r.budgetMs).toBe(drillBudgetMs(manifest.bytes));
	154	expect(r.budgetMs).toBeGreaterThanOrEqual(10 * 60_000 + (wireBytes / MB) * 1_000);
	155	expect(r.ms).toBeLessThan(r.budgetMs);
	156	expect(r.phases.map((p) => p.phase)).toEqual(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]);
	157	// The wire is the floor of the download; every phase has a time.
	158	const download = r.phases.find((p) => p.phase === "files")!.ms;
	159	expect(download).toBeGreaterThanOrEqual(manifest.bytes / wirePerMs - 500);
	160	for (const p of r.phases) expect(p.ms).toBeGreaterThanOrEqual(0);
	161	}
	162
	163	test.skipIf(!roomy)("400 MB at 150 Mbit/s restores green inside its 16 min 40 s budget, each phase timed", async () => {
	164	await prove(400 * MB, 150);
	165	}, 90_000);
	166
	167	test("scaled by 4, 100 MB at 37.5 Mbit/s restores green inside its 11 min 40 s budget, each phase timed", async () => {
	168	await prove(100 * MB, 37.5);
	169	}, 90_000);
	170	});