Public: anyone can read this; the crew can change it.

The boot-beside-peer, rehearsal, release-handover and drill-400mb tests finish under the test budget #12443

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12397-the-boot-beside-peer-rehearsal-release-h → main 14 commits fork 1 thread · 1 unresolved

  • Merged. scout/12397-the-boot-beside-peer-rehearsal-release-h was merged into main as bb5a2ad.

Review

Sign in to review this change.
16 files changed +752 −497
scripts/gate.ts −4
@@ -68,7 +68,6 @@ export const SLOW: readonly Exclusion[] = [
6868 { file: "test/big-clone.test.ts", reason: "clones a big repository over smart http through a throttled link; 4.5 s a test on the gate box", issue: 1242 },
6969 { file: "test/bindings.test.ts", reason: "generates forges and renders every page for every viewer to compare counts; 3.4 s a test on the gate box", issue: 1242 },
7070 { file: "test/body-limits.test.ts", reason: "streams request bodies up to each cap against a real server; 2.6 s a test on the gate box", issue: 1242 },
71 { file: "test/boot-beside-peer.test.ts", reason: "boots a real forge beside a 10 s peer, probing healthz until a second after it exits (#12381); was 32.6 s, about 15 s a test", issue: 1242 },
7271 { file: "test/boot-no-stall.test.ts", reason: "replays a week of findings through the boot sweeps while sampling the event loop; 9.0 s a test on the gate box", issue: 1242 },
7372 { file: "test/boundary-tokens-gates.test.ts", reason: "asks every registered route and verb with a token bound to another repository; 6.1 s a test on the gate box", issue: 1242 },
7473 { file: "test/branches-deep.test.ts", reason: "builds 200 git branches to compare the 30-row and 200-row pages; 2.2 s a test on the gate box", issue: 1242 },
@@ -98,7 +97,6 @@ export const SLOW: readonly Exclusion[] = [
9897 { file: "test/data-door.test.ts", reason: "runs SQL through the data door until the per-person limit holds; 2.1 s a test on the gate box", issue: 1242 },
9998 { file: "test/data-home.test.ts", reason: "starts main.ts as a real process to see it refuse this checkout; 2.4 s a test on the gate box", issue: 1242 },
10099 { file: "test/domain-dns.test.ts", reason: "drives a real terminal session through publishing and cutting over DNS records; 2.2 s a test on the gate box", issue: 1242 },
101 { file: "test/drill-400mb.test.ts", reason: "packs and restores a 400 MB snapshot, its wire time on an injected clock, not slept (#12381); was 26.6 s, about 7 s a test", issue: 1242 },
102100 { file: "test/drill-run.test.ts", reason: "spawns the drill script with piped stdin and waits for its JSON; 2.4 s a test on the gate box", issue: 1242 },
outdated

drill-400mb is still on SLOW, so box 3 is not met. At 17 s alone the file is too close to the 20 s budget. Run the 400 MB and 100 MB drills as separate files, or drop the 400 MB one to a nightly path.

103101 { file: "test/drill-verdict.test.ts", reason: "boots a restored directory as a real process on a side port; 2.2 s a test on the gate box", issue: 1242 },
104102 { file: "test/engine-split-soak.test.ts", reason: "soaks 48 write-then-read round trips beside a second engine process writing flat out (1000 nightly); its engine starts in a worker; 3-5 s a test", issue: 1242 },
@@ -137,9 +135,7 @@ export const SLOW: readonly Exclusion[] = [
137135 { file: "test/pull-variants.test.ts", reason: "runs two stand-in agent variants that each clone, push and open a pull; 10.7 s a test on the gate box", issue: 1242 },
138136 { file: "test/pulls-merge-budget.test.ts", reason: "seeds 50 pulls with git and a deliberately slow merge check; 5.5 s a test on the gate box", issue: 1242 },
139137 { file: "test/quotas-running.test.ts", reason: "waits for a queued stand-in agent run to start after the first one ends; 10.5 s a test on the gate box", issue: 1242 },
140 { file: "test/rehearsal-real.test.ts", reason: "boots two real forge processes on one data directory and rolls, ending 1 s after the old exits (#12381); was 34.6 s, about 15 s a test", issue: 1242 },
141138 { file: "test/release-boot.test.ts", reason: "copies src/ as the image ships it and boots that tree as a real process; 3.0 s a test on the gate box", issue: 1242 },
142 { file: "test/release-handover.test.ts", reason: "rolls a release between two forges with a git push trickled over 8 s, not 30 (#12381); was 28.5 s, about 12 s a test", issue: 1242 },
143139 { file: "test/release-launch.test.ts", reason: "waits out the readiness grace of a version that never becomes ready; 2.1 s a test on the gate box", issue: 1242 },
144140 { file: "test/release-load-run.test.ts", reason: "load-tests a forge stand-in answering in 150 ms to measure p95; 5.1 s a test on the gate box", issue: 1242 },
145141 { file: "test/release-preflight.test.ts", reason: "runs the release preflight on a copy of a seeded live database; 4.8 s a test on the gate box", issue: 1242 },
scripts/rehearsal.ts +5
@@ -27,6 +27,11 @@ export interface RehearsalArgs {
2727 * `seconds` (#12381): a roll that is over is not waited out. `seconds` stays the ceiling.
2828 */
2929 readonly settleMs?: number | undefined;
30 /**
31 * How long the old forge takes traffic alone before the new one is
32 * started (#12397); unset, a quarter of `seconds`, at most 2 s.
33 */
34 readonly leadMs?: number | undefined;
3035 /** How often rotation asks each forge's `/readyz` (the chart: 5 s). */
3136 readonly probeMs: number;
3237 /** The old forge's FORGE_DRAIN_LINGER_S. */
scripts/rehearse-roll.ts +1 −1
@@ -222,7 +222,7 @@ export async function rehearse(
222222 // The platform's side: probe every forge each period, start the new
223223 // one beside the old, and signal the old one once the new is ready.
224224 const roll = async (): Promise<void> => {
225 await Bun.sleep(Math.min(2000, (args.seconds * 1000) / 4));
225 await Bun.sleep(args.leadMs ?? Math.min(2000, (args.seconds * 1000) / 4));
226226 let next = await start("new", resolve(args.next), dataDir, args.lingerS, logs);
227227 forges.push(next);
228228 say(`new forge starting at ${next.url}`);
src/runs/peer.ts +5 −1
@@ -23,7 +23,11 @@ import { HEARTBEAT_MS, heldRuns } from "./lease.ts";
2323 export const PEER_FRESH_MS = 2 * HEARTBEAT_MS;
2424 /** The pod's grace (charts/app, 720 s): an old pod is gone by then. A drain is 10 s (`src/drain.ts`); this is the ceiling for a run that ignores it. */
2525 export const PEER_WAIT_MAX_MS = 720_000;
26 const POLL_MS = 5_000;
26 /**
27 * One indexed read a second while waiting: work starts within a second of the
28 * peer letting go (#12397; was 5 s).
29 */
30 const POLL_MS = 1_000;
2731
2832 /** Running runs whose lease another process renewed within `PEER_FRESH_MS` of `now`. */
2933 export function peerRuns(db: Database, now = Date.now()): number {
test/boot-beside-peer.test.ts +54 −22
@@ -10,20 +10,26 @@
1010 * for most of every second, then finishes and lets go.
1111 */
1212
13 import { expect, test } from "bun:test";
13 import { beforeAll, expect, test } from "bun:test";
1414 import { mkdtemp, rm, writeFile } from "node:fs/promises";
1515 import { tmpdir } from "node:os";
1616 import { join } from "node:path";
1717 import { createApp } from "../src/app.ts";
1818
19 // Long enough that the forge boots behind the lock and is probed while it is
20 // still held (counted below); the probes stop a second after the peer exits
21 // rather than PEER_MS after the forge is up (#12381).
19 // The peer's ceiling: it lets go sooner, once the forge is up and has
20 // answered HELD_PROBES probes while the lock is still held (#12397), so a slow
21 // boot still boots behind the lock and a fast one is not waited out. The
22 // probes stop a second after the peer exits (#12381). A hold is 1.5 s, over
23 // the probe's 1 s timeout, so a forge that waited on the lock to answer shows.
2224 const PEER_MS = 10_000;
23 const HOLD_MS = 2_500;
25 const HELD_PROBES = 5;
26 const HOLD_MS = 1_500;
2427 const GAP_MS = 150;
2528
26 /** Holds the write lock `HOLD_MS` of every `HOLD_MS + GAP_MS`, renewing the 8 leases each time; then lands them and exits. */
29 /**
30 * Holds the write lock `HOLD_MS` of every `HOLD_MS + GAP_MS`, renewing the 8 leases each time,
31 * until told on stdin or `PEER_MS`; then lands them and exits.
32 */
2733 const peerScript = `
2834 const { Database } = require("bun:sqlite");
2935 const db = new Database(process.argv[2]);
@@ -31,21 +37,32 @@ db.run("PRAGMA busy_timeout = 15000");
3137 const ids = ${JSON.stringify(Array.from({ length: 8 }, (_, n) => `gate-${n}`))};
3238 const until = Date.now() + ${PEER_MS};
3339 const spin = (ms) => { const end = Date.now() + ms; while (Date.now() < end) {} };
40 let told = false;
41 process.stdin.on("data", () => (told = true));
3442 console.log("peer up");
43 const land = () => {
44 db.run("UPDATE run SET state = 'ok' WHERE state = 'running'");
45 console.log("peer done");
46 process.exit(0);
47 };
3548 const tick = () => {
49 // Told during the last hold: stdin is read in the gap, so it is seen here.
50 if (told) return land();
3651 db.run("BEGIN IMMEDIATE");
3752 db.run("UPDATE run SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id IN (" + ids.map((i) => "'" + i + "'").join(",") + ")");
3853 spin(${HOLD_MS});
3954 db.run("COMMIT");
4055 if (Date.now() < until) return void setTimeout(tick, ${GAP_MS});
41 db.run("UPDATE run SET state = 'ok' WHERE state = 'running'");
42 console.log("peer done");
43 process.exit(0);
56 land();
4457 };
4558 tick();
4659 `;
4760
48 test("healthz answers within 1 s throughout a boot beside a draining forge's batch of 8, and work starts once it lets go", async () => {
61 /**
62 * The boot beside the peer, as plain data: whether it came up, the probes that were slow, how
63 * many were asked while the lock was held, and what the forge said.
64 */
65 async function bootBeside() {
4966 const dataDir = await mkdtemp(join(tmpdir(), "forge-bootpeer-"));
5067 const seed = createApp({ dataDir, port: 0 });
5168 seed.db.run("PRAGMA foreign_keys = OFF");
@@ -58,7 +75,7 @@ test("healthz answers within 1 s throughout a boot beside a draining forge's bat
5875 const script = join(dataDir, "..", `${dataDir.split("/").pop()}-peer.js`);
5976 await writeFile(script, peerScript);
6077
61 const peer = Bun.spawn(["bun", script, join(dataDir, "forge.db")], { stdout: "pipe", stderr: "inherit" });
78 const peer = Bun.spawn(["bun", script, join(dataDir, "forge.db")], { stdin: "pipe", stdout: "pipe", stderr: "inherit" });
6279 await peer.stdout.getReader().read();
6380
6481 const web = 20000 + Math.floor(Math.random() * 20000);
@@ -87,7 +104,7 @@ test("healthz answers within 1 s throughout a boot beside a draining forge's bat
87104 await Bun.sleep(100);
88105 }
89106 }
90 expect(up).toBe(true);
107 if (!up) return { up, slow: [], held: 0, out };
91108 // Then every probe answers within a second while the peer holds on, and after.
92109 const slow: number[] = [];
93110 let held = 0;
@@ -95,31 +112,46 @@ test("healthz answers within 1 s throughout a boot beside a draining forge's bat
95112 void peer.exited.then(() => (end = performance.now() + 1_000));
96113 const ceiling = performance.now() + PEER_MS + 5_000;
97114 while (performance.now() < Math.min(end, ceiling)) {
98 if (peer.exitCode === null) held++;
99115 const t = performance.now();
100116 try {
101117 const res = await fetch(`http://127.0.0.1:${web}/healthz`, { signal: AbortSignal.timeout(1_000) });
102118 if (!res.ok) slow.push(-res.status);
119 // Answered while the peer still held on: counted only once it came back.
120 else if (peer.exitCode === null && ++held === HELD_PROBES) {
121 peer.stdin.write("let go\n");
122 void peer.stdin.flush();
123 }
103124 } catch {
104125 slow.push(Math.round(performance.now() - t));
105126 }
106127 await Bun.sleep(200);
107128 }
108 // A probe that timed out: the forge's own log names what held it.
109 if (slow.length > 0) console.log(out);
110 expect(slow).toEqual([]);
111 // The forge was asked while the peer still held the lock, not only after.
112 expect(held).toBeGreaterThan(0);
113 expect(out).toContain("another forge still works this database");
114129 await peer.exited;
115130 for (let i = 0; i < 100 && !out.includes("the other forge let go of this database"); i++) await Bun.sleep(100);
116 expect(out).toContain("the other forge let go of this database");
117 expect(out).not.toContain("event loop stalled");
131 return { up, slow, held, out };
118132 } finally {
119133 peer.kill();
120 forge.kill();
134 // Its shutdown is not what is proven here, and a drain takes seconds (#12397).
135 forge.kill("SIGKILL");
121136 await Promise.all([peer.exited, forge.exited]);
122137 await rm(dataDir, { recursive: true, force: true });
123138 await rm(script, { force: true });
124139 }
140 }
141
142 let boot: Awaited<ReturnType<typeof bootBeside>>;
143 beforeAll(async () => {
144 boot = await bootBeside();
125145 }, 60_000);
146
147 test("healthz answers within 1 s throughout a boot beside a draining forge's batch of 8, and work starts once it lets go", () => {
148 expect(boot.up).toBe(true);
149 // A probe that timed out: the forge's own log names what held it.
150 if (boot.slow.length > 0) console.log(boot.out);
151 expect(boot.slow).toEqual([]);
152 // The forge was asked while the peer still held the lock, not only after.
153 expect(boot.held).toBeGreaterThanOrEqual(HELD_PROBES);
154 expect(boot.out).toContain("another forge still works this database");
155 expect(boot.out).toContain("the other forge let go of this database");
156 expect(boot.out).not.toContain("event loop stalled");
157 });
test/drill-100mb.test.ts added +24
@@ -0,0 +1,24 @@
1 /**
2 * The 400 MB drill of `drill-400mb.test.ts` scaled by 4 (#2813): 100 MB at
3 * 37.5 Mbit/s, the same 21 s of wire on the drill's injected clock and an
4 * 11 min 40 s budget. It needs no large scratch disk, so it always runs: the
5 * streaming, the phase times and the budget are proven everywhere; only the
6 * disk write of the full size is not.
7 *
8 * The drill (`test/helpers/throttled-drill.ts`) runs in `beforeAll`; the file
9 * budget times the whole file, hooks included (#12397).
10 */
11 import { beforeAll, test } from "bun:test";
12 import { quietLogs } from "./helpers/globals.ts";
13 import { judge, MB, throttledDrill, type Drilled } from "./helpers/throttled-drill.ts";
14
15 quietLogs();
16
17 let scaled: Drilled;
18 beforeAll(async () => {
19 scaled = await throttledDrill(100 * MB, 37.5);
20 }, 60_000);
21
22 test("scaled by 4, 100 MB at 37.5 Mbit/s restores green inside its 11 min 40 s budget, each phase timed", () => {
23 judge(scaled, 100 * MB);
24 });
test/drill-400mb.test.ts +16 −165
@@ -4,176 +4,27 @@
44 * budget (#2813): streamed into decompression (#2743), each phase timed
55 * (#2742), the budget grown with the size (#2773).
66 *
7 * Two proofs of the same rule. The full one is 400 MB at 150 Mbit/s (21 s
8 * of wire, a 16 min 40 s budget) and writes the 400 MB database to disk, so
9 * it runs only where the scratch disk has room (`ROOM`), and is skipped by
10 * name where it does not: a check's scratch disk ran out (ENOSPC) at about
11 * 320 MB. The scaled one is 1/4 of both, 100 MB at 37.5 Mbit/s (the same
12 * 21 s of wire, an 11 min 40 s budget), and always runs, so the streaming,
13 * the phase times and the budget are proven everywhere; only the disk
14 * write of the full size is not.
7 * 400 MB at 150 Mbit/s is 21 s of wire, on the drill's injected clock, and a
8 * 16 min 40 s budget. It writes the 400 MB database to disk, so it runs only
9 * where the scratch disk has room, and is skipped by name where it does not;
10 * the same rule scaled by 4, which always runs, is `drill-100mb.test.ts`.
1511 *
16 * Both snapshots are generated here, a real `forge.db` followed by
17 * incompressible padding, never kept in the repo. Neither has a large
18 * repository, so the `bundles` phase is ~0 ms in both: clone time at this
19 * size is not proven here.
12 * The drill (`test/helpers/throttled-drill.ts`) runs in `beforeAll`; the file
13 * budget times the whole file, hooks included (#12397).
2014 */
21 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
22 import { mkdtempSync, rmSync, statfsSync } from "node:fs";
23 import { tmpdir } from "node:os";
24 import { join } from "node:path";
25 import { createApp, type App } from "../src/app.ts";
26 import { createUser } from "../src/core/users.ts";
27 import { drillBudgetMs, runDrill } from "../src/snapshot/drill.ts";
28 import { chunkId, chunkKey, packChunk } from "../src/snapshot/chunks.ts";
29 import { readWhole } from "../src/snapshot/restore.ts";
30 import { listSnapshots, takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts";
31 import { dirStore, type Store } from "../src/snapshot/store.ts";
15 import { beforeAll, expect, test } from "bun:test";
3216 import { quietLogs } from "./helpers/globals.ts";
33 // The wire's 21 s is on the drill's injected clock (`ticks`), not slept.
17 import { judge, MB, roomy, throttledDrill, type Drilled } from "./helpers/throttled-drill.ts";
3418
3519 quietLogs();
3620
37 const MB = 1_000_000;
38 /** Free bytes the full run wants on the scratch disk: the 400 MB restored, the rest as margin. */
39 const ROOM = 1_500_000_000;
40 const free = (): number => {
41 const s = statfsSync(tmpdir());
42 return s.bavail * s.bsize;
43 };
44 const roomy = free() >= ROOM;
21 const room = roomy();
22 let full: Drilled;
23 beforeAll(async () => {
24 if (room) full = await throttledDrill(400 * MB, 150);
25 }, 60_000);
4526
46 /** Fill `out` (a multiple of 4 bytes) with pseudo-random words, so zstd can only store it. */
47 function fillNoise(out: Uint8Array): void {
48 const words = new Uint32Array(out.buffer, out.byteOffset, out.byteLength / 4);
49 let x = 2463534242;
50 for (let i = 0; i < words.length; i++) {
51 x ^= x << 13;
52 x >>>= 0;
53 x ^= x >>> 17;
54 x ^= x << 5;
55 x >>>= 0;
56 words[i] = x;
57 }
58 }
59
60 describe("a 400 MB snapshot through a 150 Mbit/s store", () => {
61 let root = "";
62 let app: App;
63 let real: Store;
64 let base: Manifest;
65 let database: Uint8Array;
66
67 beforeAll(async () => {
68 root = mkdtempSync(join(tmpdir(), "forge-drill-400-"));
69 app = createApp({ dataDir: join(root, "live"), port: 0 });
70 const u = await createUser(app.db, {
71 origin: "seed",
72 trusted: true,
73 username: "alice",
74 email: "[email protected]",
75 password: "a-good-passphrase",
76 adminIfFirst: true,
77 });
78 if (!u.ok) throw new Error(u.error);
79 real = dirStore(join(root, "store"));
80 await takeSnapshot(
81 { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
82 real,
83 new Date("2026-10-18T08:00:00Z"),
84 );
85 base = (await listSnapshots(real))[0]!;
86 // The database is the snapshot's own copy, the live file being in its WAL.
87 database = await readWhole(real, base.name, base.files.find((f) => f.path === "forge.db")!);
88 });
89 afterAll(() => {
90 app.close();
91 rmSync(root, { recursive: true, force: true });
92 });
93
94 /**
95 * Drill a snapshot of `wireBytes` served at `mbit` Mbit/s: forge.db then
96 * padding to that size on the wire, a database whose tail SQLite never
97 * reads, in chunks of 4 MiB that zstd can only store.
98 */
99 async function prove(wireBytes: number, mbit: number): Promise<void> {
100 const prefix = `snapshots/${base.name}/`;
101 const wirePerMs = (mbit * MB) / 8 / 1000;
102 const db = database;
103 const whole = new Uint8Array(db.byteLength + wireBytes);
104 whole.set(db);
105 fillNoise(whole.subarray(db.byteLength));
106 const CHUNK = 4 * 1024 * 1024;
107 // Each chunk is a view of `whole`, served packed when it is asked for: nothing of this size is
108 // written to the store.
109 const served = new Map<string, Uint8Array>();
110 const chunks: string[] = [];
111 for (let at = 0; at < whole.byteLength; at += CHUNK) {
112 const plain = whole.subarray(at, at + CHUNK);
113 const id = chunkId(plain);
114 served.set(chunkKey(id), plain);
115 chunks.push(id);
116 }
117 const others = base.files.filter((f) => f.path !== "forge.db");
118 const files = [...others, { path: "forge.db", bytes: whole.byteLength, sha256: chunkId(whole), chunks }];
119 const manifest: Manifest = {
120 ...base,
121 files,
122 bytes: files.reduce((n, f) => n + f.bytes, 0),
123 };
124 expect(manifest.bytes).toBeGreaterThanOrEqual(wireBytes);
125
126 // One link for every get: each waits its turn behind the bytes before it.
127 // The wait is on the drill's clock, moved forward by the wire time rather
128 // than slept (#12381), so the phases see the full wire and the run does not.
129 let skew = 0;
130 const ticks = (): number => performance.now() + skew;
131 let busy = 0;
132 // What crossed the wire: the padding as it is, the forge's own files compressed.
133 let wire = 0;
134 const link: Store = {
135 ...real,
136 where: `throttled ${mbit} Mbit/s`,
137 async get(key) {
138 if (key === `${prefix}manifest.json`) return new TextEncoder().encode(JSON.stringify(manifest));
139 const plain = served.get(key);
140 const body = plain !== undefined ? await packChunk(plain) : await real.get(key);
141 if (body === undefined) return undefined;
142 wire += body.byteLength;
143 busy = Math.max(ticks(), busy) + body.byteLength / wirePerMs;
144 skew += Math.max(0, busy - ticks());
145 return body;
146 },
147 };
148
149 const t = ticks();
150 const r = await runDrill({
151 ticks,
152 store: link,
153 dataDir: join(root, `drill-data-${mbit}`),
154 boot: async () => ({ ok: true, reason: "" }),
155 });
156 const took = ticks() - t;
157 console.log(`drill of ${Math.round(manifest.bytes / MB)} MB at ${mbit} Mbit/s: ${Math.round(took)} ms; ${r.phases.map((p) => `${p.phase} ${p.ms} ms`).join(", ")}`);
158
159 expect(r.reason).toBeNull();
160 expect(r.verdict).toBe("green");
161 expect(r.budgetMs).toBe(drillBudgetMs(manifest.bytes));
162 expect(r.budgetMs).toBeGreaterThanOrEqual(10 * 60_000 + (wireBytes / MB) * 1_000);
163 expect(r.ms).toBeLessThan(r.budgetMs);
164 expect(r.phases.map((p) => p.phase)).toEqual(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]);
165 // The wire is the floor of the download; every phase has a time.
166 const download = r.phases.find((p) => p.phase === "files")!.ms;
167 expect(wire).toBeGreaterThanOrEqual(wireBytes);
168 expect(download).toBeGreaterThanOrEqual(wire / wirePerMs - 500);
169 for (const p of r.phases) expect(p.ms).toBeGreaterThanOrEqual(0);
170 }
171
172 test.skipIf(!roomy)("400 MB at 150 Mbit/s restores green inside its 16 min 40 s budget, each phase timed", async () => {
173 await prove(400 * MB, 150);
174 }, 90_000);
175
176 test("scaled by 4, 100 MB at 37.5 Mbit/s restores green inside its 11 min 40 s budget, each phase timed", async () => {
177 await prove(100 * MB, 37.5);
178 }, 90_000);
27 test.skipIf(!room)("400 MB at 150 Mbit/s restores green inside its 16 min 40 s budget, each phase timed", () => {
28 judge(full, 400 * MB);
29 expect(full.bytes).toBeGreaterThanOrEqual(400 * MB);
17930 });
test/helpers/handover-roll.ts added +282
@@ -0,0 +1,282 @@
1 /**
2 * The two rolls of a release swap (#2040), for `release-handover.test.ts`
3 * and `release-handover-drain.test.ts`: two forges on one data directory
4 * share the port (SO_REUSEPORT), and a push that started on the old one
5 * trickles in for eight seconds while the new one comes up and the old one
6 * drains. One roll a file (#12397), so neither file waits out two pushes.
7 */
8
9 import { readFileSync } from "node:fs";
10 import { mkdtemp, rm } from "node:fs/promises";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp } from "../../src/app.ts";
14 import { commitToBranch, createRepo } from "../../src/actions/repos.ts";
15 import { createToken } from "../../src/core/tokens.ts";
16 import { createUser } from "../../src/core/users.ts";
17 import { repoDir } from "../../src/git/repos.ts";
18 import { drainOnHandover, handover } from "../../src/release/swap.ts";
19 import { serve } from "../../src/http/server.ts";
20 import { gitClientEnv } from "./git-client.ts";
21
22 // Long enough to span the new forge's start and the old one's drain start
23 // (both well under a second in process) and over a hundred probes; it was 30 s
24 // and proved no more (#12381).
25 export const PUSH_MS = 8_000;
26
27 const dirs: string[] = [];
28 export async function cleanup(): Promise<void> {
29 for (const d of dirs) await rm(d, { recursive: true, force: true });
30 }
31
32 // Without it, what sat in the old listener's accept queue is reset when it
33 // closes — at drain start now, at the end before; either way the kernel's.
34 export const migrates = (() => {
35 try {
36 return readFileSync("/proc/sys/net/ipv4/tcp_migrate_req", "utf8").trim() === "1";
37 } catch {
38 return false;
39 }
40 })();
41
42 async function git(args: string[], cwd: string, stdin?: string): Promise<Uint8Array> {
43 const out = Bun.spawnSync(["git", ...args], {
44 cwd,
45 env: gitClientEnv(),
46 ...(stdin === undefined ? {} : { stdin: new TextEncoder().encode(stdin) }),
47 });
48 if (out.exitCode !== 0) throw new Error(`git ${args.join(" ")}: ${out.stderr.toString()}`);
49 return new Uint8Array(out.stdout);
50 }
51
52 const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes).trim();
53
54 function pktLine(line: string): Uint8Array {
55 const body = new TextEncoder().encode(line);
56 const head = (body.length + 4).toString(16).padStart(4, "0");
57 return new Uint8Array([...new TextEncoder().encode(head), ...body]);
58 }
59
60 /** `bytes` sent in pieces spread over `ms`, so the request stays in flight that long. */
61 function trickle(bytes: Uint8Array, ms: number): ReadableStream<Uint8Array> {
62 const pieces = 30;
63 const size = Math.ceil(bytes.length / pieces);
64 let i = 0;
65 return new ReadableStream({
66 async pull(controller) {
67 if (i > 0) await Bun.sleep(ms / pieces);
68 const piece = bytes.subarray(i * size, (i + 1) * size);
69 i++;
70 if (piece.length > 0) controller.enqueue(piece);
71 if (i >= pieces) controller.close();
72 },
73 });
74 }
75
76 /**
77 * The old forge on a port of its own, alice's repository, and `push(ms)`: a
78 * receive-pack of one new commit on main, trickled in over `ms`.
79 */
80 async function forgeWithPush() {
81 const dataDir = await mkdtemp(join(tmpdir(), "forge-handover-"));
82 const work = await mkdtemp(join(tmpdir(), "forge-handover-work-"));
83 dirs.push(dataDir, work);
84 // The old forge picks the port; the release binds the one it got.
85 const oldApp = createApp({ dataDir, port: 0, selfSwap: true });
86 const oldServers = await serve(oldApp);
87 const port = oldServers.port;
88 const made = await createUser(oldApp.db, {
89 origin: "seed",
90 trusted: true,
91 username: "alice",
92 email: "[email protected]",
93 password: "a-good-passphrase",
94 adminIfFirst: true,
95 });
96 if (!made.ok) throw new Error("user");
97 const alice = made.value;
98 const repo = await createRepo(oldApp, alice, { name: "garden", visibility: "public" });
99 if (!repo.ok) throw new Error("repo");
100 await commitToBranch(oldApp, alice, repo.value, {
101 message: "seed",
102 files: [{ path: "a.txt", content: "a" }],
103 });
104 const { token } = createToken(oldApp.db, alice.id, "push", "rw");
105
106 // The push: one commit on main, as git sends it to receive-pack.
107 const bare = repoDir(oldApp.reposRoot, "alice", "garden");
108 const clone = join(work, "garden");
109 await git(["clone", "-q", bare, clone], work);
110 await Bun.write(join(clone, "b.txt"), "b");
111 await git(["add", "b.txt"], clone);
112 await git(["commit", "-q", "-m", "b"], clone);
113 const before = text(await git(["rev-parse", "origin/main"], clone));
114 const after = text(await git(["rev-parse", "HEAD"], clone));
115 const pack = await git(
116 ["pack-objects", "--stdout", "--revs", "--quiet"],
117 clone,
118 `${after}\n^${before}\n`,
119 );
120 const command = pktLine(`${before} ${after} refs/heads/main\0report-status\n`);
121 const flush = new TextEncoder().encode("0000");
122 const body = new Uint8Array(command.length + flush.length + pack.length);
123 body.set(command, 0);
124 body.set(flush, command.length);
125 body.set(pack, command.length + flush.length);
126
127 const url = `http://127.0.0.1:${port}`;
128 const push = (ms: number) =>
129 fetch(`${url}/alice/garden.git/git-receive-pack`, {
130 method: "POST",
131 headers: {
132 authorization: `Basic ${btoa(`alice:${token}`)}`,
133 "content-type": "application/x-git-receive-pack-request",
134 },
135 body: trickle(body, ms),
136 });
137 return { oldApp, oldServers, port, url, bare, after, push };
138 }
139
140 /**
141 * The roll of the first test, as plain data: what the push, the prober and the orphan probe saw.
142 */
143 export async function rollAcross() {
144 const { oldApp, oldServers, port, url, bare, after, push: pushOver } = await forgeWithPush();
145 const dataDir = oldApp.config.dataDir;
146 const started = Date.now();
147 const push = pushOver(PUSH_MS);
148 // The push is the old forge's before the release exists.
149 await Bun.sleep(500);
150
151 const newApp = createApp({ dataDir, port, selfSwap: true });
152 const newServers = await serve(newApp);
153
154 let looping = true;
155 let asked = 0;
156 let refused = 0;
157 const client = (async () => {
158 while (looping) {
159 asked++;
160 try {
161 const res = await fetch(`${url}/api/v1/health`, {
162 headers: { connection: "close" },
163 signal: AbortSignal.timeout(3_000),
164 });
165 await res.arrayBuffer();
166 } catch {
167 refused++;
168 }
169 await Bun.sleep(20);
170 }
171 })();
172
173 try {
174 // Drain start: the old forge stops listening, keeping what it holds.
175 const drained = oldServers.stopAccepting(PUSH_MS * 2);
176
177 const answer = await push;
178 const status = answer.status;
179 const report = await answer.text();
180 const tookMs = Date.now() - started;
181 await drained;
182
183 looping = false;
184 await client;
185 const head = text(await git(["rev-parse", "refs/heads/main"], bare));
186
187 // Nobody else listens: the old forge let go of the port at drain start.
188 // `stop` closes the listener asynchronously, so ask until it is gone.
189 newServers.stop();
190 let orphan = "answered";
191 for (const until = Date.now() + 10_000; Date.now() < until; ) {
192 orphan = await fetch(`${url}/api/v1/health`, {
193 headers: { connection: "close" },
194 signal: AbortSignal.timeout(2_000),
195 }).then(
196 async (res) => {
197 await res.arrayBuffer();
198 return "answered";
199 },
200 () => "refused",
201 );
202 if (orphan === "refused") break;
203 await Bun.sleep(100); // real-timer: polls a real orphaned port until a real process lets it go
204 }
205 return { status, report, tookMs, asked, refused, head, after, orphan };
206 } finally {
207 looping = false;
208 await client;
209 newServers.stop();
210 oldServers.stop();
211 newApp.close();
212 oldApp.close();
213 }
214 }
215
216 /**
217 * The drain of the second test, as plain data: when the port went, the rest finished, the push
218 * was answered and the close came.
219 */
220 export async function drainAcross() {
221 const { oldApp, oldServers, url, bare, after, push: pushOver } = await forgeWithPush();
222 const HELD_MS = 4_000;
223 const REST_MS = 2_000;
224 let answeredAt = 0;
225 let restDoneAt = 0;
226 let closedAt = 0;
227 const push = pushOver(HELD_MS).then(async (res) => {
228 const report = await res.text();
229 answeredAt = Date.now();
230 return { status: res.status, report };
231 });
232 // Wired as `main.ts` wires it; `rest` stands in for the shutdown's other
233 // steps (crew, apps, checks), `close` for its last.
234 const handOver = drainOnHandover(
235 oldServers,
236 async () => {
237 await Bun.sleep(REST_MS);
238 restDoneAt = Date.now();
239 },
240 () => {
241 closedAt = Date.now();
242 oldServers.stop();
243 },
244 );
245 handover.drain = () => void handOver();
246 try {
247 await Bun.sleep(500);
248 handover.drain();
249
250 // Nobody else listens here, so a refused connection is this listener gone.
251 let refusedAt = 0;
252 while (restDoneAt === 0) {
253 const gone = await fetch(`${url}/api/v1/health`, {
254 headers: { connection: "close" },
255 signal: AbortSignal.timeout(1_000),
256 }).then(
257 async (res) => {
258 await res.arrayBuffer();
259 return false;
260 },
261 () => true,
262 );
263 if (gone) {
264 refusedAt = Date.now();
265 break;
266 }
267 await Bun.sleep(50);
268 }
269 const restDoneAtRefusal = restDoneAt;
270
271 const answered = await push;
272 for (const until = Date.now() + 10_000; closedAt === 0 && Date.now() < until; ) {
273 await Bun.sleep(50); // real-timer: waits on a real socket close
274 }
275 const head = text(await git(["rev-parse", "refs/heads/main"], bare));
276 return { refusedAt, restDoneAtRefusal, answered, answeredAt, restDoneAt, closedAt, head, after };
277 } finally {
278 delete handover.drain;
279 oldServers.stop();
280 oldApp.close();
281 }
282 }
test/helpers/real-roll.ts added +40
@@ -0,0 +1,40 @@
1 /**
2 * One short real roll (#873) of two forge processes on one data directory,
3 * for `rehearsal-real.test.ts` and `rehearsal-abrupt.test.ts`: one roll a
4 * file, so neither file runs two heavy scenarios at once (#12397).
5 *
6 * The data directory is migrated once in-process before the roll, so the
7 * old forge does not spend its boot migrating: what is left is two boots,
8 * the linger and a second of settling.
9 */
10
11 import { mkdtempSync, rmSync } from "node:fs";
12 import { tmpdir } from "node:os";
13 import { join } from "node:path";
14 import { createApp } from "../../src/app.ts";
15 import { rehearse } from "../../scripts/rehearse-roll.ts";
16 import { parseArgs } from "../../scripts/rehearsal.ts";
17
18 export async function realRoll(lingerS: number): Promise<Awaited<ReturnType<typeof rehearse>>> {
19 const seeded = join(mkdtempSync(join(tmpdir(), "forge-rehearsal-seed-")), "data");
20 try {
21 createApp({ dataDir: seeded, port: 0 }).close();
22 return await rehearse({
23 ...parseArgs([], `${import.meta.dir}/../..`),
24 from: seeded,
25 // Room for a slow boot and a long linger on a busy machine (#2020).
26 seconds: 30,
27 // The roll ends a second (over three probe periods) after the old forge exits, not at
28 // 30 s (#12381, #12397): the new forge takes traffic alone and the tallies settle.
29 settleMs: 1_000,
30 // The old forge takes traffic alone for a probe period before the new one starts, and
31 // alone again for the seconds the new one boots; the 2 s default adds nothing to that.
32 leadMs: 300,
33 probeMs: 300,
34 lingerS,
35 concurrency: 2,
36 });
37 } finally {
38 rmSync(join(seeded, ".."), { recursive: true, force: true });
39 }
40 }
test/helpers/throttled-drill.ts added +183
@@ -0,0 +1,183 @@
1 /**
2 * One drill (#2813) of a generated snapshot through a store throttled to a
3 * given downlink, for `drill-400mb.test.ts` (400 MB at 150 Mbit/s) and
4 * `drill-100mb.test.ts` (the same rule scaled by 4): one drill a file
5 * (#12397), so neither file waits out the other's.
6 *
7 * The snapshot is a real `forge.db` followed by incompressible padding,
8 * never kept in the repo. Neither has a large repository, so the `bundles`
9 * phase is ~0 ms: clone time at this size is not proven here. The wire's
10 * time is on the drill's injected clock (`ticks`), not slept.
11 */
12 import { expect } from "bun:test";
13 import { mkdtempSync, rmSync, statfsSync } from "node:fs";
14 import { tmpdir } from "node:os";
15 import { join } from "node:path";
16 import { createApp } from "../../src/app.ts";
17 import { createUser } from "../../src/core/users.ts";
18 import { drillBudgetMs, runDrill } from "../../src/snapshot/drill.ts";
19 import { chunkId, chunkKey, packChunk } from "../../src/snapshot/chunks.ts";
20 import { readWhole } from "../../src/snapshot/restore.ts";
21 import { listSnapshots, takeSnapshot, type Manifest } from "../../src/snapshot/snapshot.ts";
22 import { dirStore, type Store } from "../../src/snapshot/store.ts";
23
24 export const MB = 1_000_000;
25 /** Free bytes the full run wants on the scratch disk: the 400 MB restored, the rest as margin. */
26 const ROOM = 1_500_000_000;
27 /**
28 * Whether the scratch disk has room for the 400 MB drill: a check's ran out
29 * (ENOSPC) at about 320 MB.
30 */
31 export const roomy = (): boolean => {
32 const s = statfsSync(tmpdir());
33 return s.bavail * s.bsize >= ROOM;
34 };
35
36 export type Drilled = { r: Awaited<ReturnType<typeof runDrill>>; bytes: number; wire: number; wirePerMs: number };
37 /** 4 MiB of pseudo-random words, made once: zstd can only store them. */
38 const NOISE = (() => {
39 const words = new Uint32Array(1 << 20);
40 let x = 2463534242;
41 for (let i = 0; i < words.length; i++) {
42 x ^= x << 13;
43 x >>>= 0;
44 x ^= x >>> 17;
45 x ^= x << 5;
46 x >>>= 0;
47 words[i] = x;
48 }
49 return new Uint8Array(words.buffer);
50 })();
51
52 /**
53 * Fill `out` with `NOISE` over and over, each copy's first word stamped with
54 * its number: no two 4 MiB chunks are the same bytes, and none compresses.
55 */
56 function fillNoise(out: Uint8Array): void {
57 for (let at = 0, n = 0; at < out.byteLength; at += NOISE.byteLength, n++) {
58 const copy = NOISE.subarray(0, Math.min(NOISE.byteLength, out.byteLength - at));
59 out.set(copy, at);
60 for (let k = 0; k < 4 && at + k < out.byteLength; k++) out[at + k] = (NOISE[k]! ^ (n >>> (8 * k))) & 255;
61 }
62 }
63
64 /**
65 * Drill a snapshot of `wireBytes` served at `mbit` Mbit/s: a forge with one
66 * person is snapshotted, then its forge.db is served padded to that size on
67 * the wire, a database whose tail SQLite never reads, in chunks of 4 MiB
68 * that zstd can only store. Everything it wrote is removed before it returns.
69 */
70 export async function throttledDrill(wireBytes: number, mbit: number): Promise<Drilled> {
71 const root = mkdtempSync(join(tmpdir(), "forge-drill-"));
72 const app = createApp({ dataDir: join(root, "live"), port: 0 });
73 try {
74 const u = await createUser(app.db, {
75 origin: "seed",
76 trusted: true,
77 username: "alice",
78 email: "[email protected]",
79 password: "a-good-passphrase",
80 adminIfFirst: true,
81 });
82 if (!u.ok) throw new Error(u.error);
83 const real = dirStore(join(root, "store"));
84 await takeSnapshot(
85 { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
86 real,
87 new Date("2026-10-18T08:00:00Z"),
88 );
89 const base = (await listSnapshots(real))[0]!;
90 // The database is the snapshot's own copy, the live file being in its WAL.
91 const database = await readWhole(real, base.name, base.files.find((f) => f.path === "forge.db")!);
92 return await drillOf(root, real, base, database, wireBytes, mbit);
93 } finally {
94 app.close();
95 rmSync(root, { recursive: true, force: true });
96 }
97 }
98
99 async function drillOf(
100 root: string,
101 real: Store,
102 base: Manifest,
103 database: Uint8Array,
104 wireBytes: number,
105 mbit: number,
106 ): Promise<Drilled> {
107 const prefix = `snapshots/${base.name}/`;
108 const wirePerMs = (mbit * MB) / 8 / 1000;
109 const db = database;
110 const whole = new Uint8Array(db.byteLength + wireBytes);
111 whole.set(db);
112 fillNoise(whole.subarray(db.byteLength));
113 const CHUNK = 4 * 1024 * 1024;
114 // Each chunk is a view of `whole`, served packed when it is asked for: nothing of this size is
115 // written to the store.
116 // The ids are sha256 like `chunkId`'s, taken on the thread pool at once: one after the other,
117 // hashing 400 MB twice was a third of the file's time (#12397).
118 const hex = (b: ArrayBuffer): string => Buffer.from(b).toString("hex");
119 const sha = (b: Uint8Array<ArrayBuffer>): Promise<string> => crypto.subtle.digest("SHA-256", b).then(hex);
120 const plains: Uint8Array<ArrayBuffer>[] = [];
121 for (let at = 0; at < whole.byteLength; at += CHUNK) plains.push(whole.subarray(at, at + CHUNK));
122 const [wholeId, ...chunks] = await Promise.all([sha(whole), ...plains.map(sha)]);
123 const served = new Map<string, Uint8Array>(plains.map((plain, i) => [chunkKey(chunks[i]!), plain]));
124 if (chunkId(plains[0]!) !== chunks[0] || served.size !== plains.length) throw new Error("the chunks are not named as the store names them, or two are the same");
125 const others = base.files.filter((f) => f.path !== "forge.db");
126 const files = [...others, { path: "forge.db", bytes: whole.byteLength, sha256: wholeId!, chunks }];
127 const manifest: Manifest = {
128 ...base,
129 files,
130 bytes: files.reduce((n, f) => n + f.bytes, 0),
131 };
132 if (manifest.bytes < wireBytes) throw new Error(`the snapshot is ${manifest.bytes} bytes, under ${wireBytes}`);
133
134 // One link for every get: each waits its turn behind the bytes before it.
135 // The wait is on the drill's clock, moved forward by the wire time rather
136 // than slept (#12381), so the phases see the full wire and the run does not.
137 let skew = 0;
138 const ticks = (): number => performance.now() + skew;
139 let busy = 0;
140 // What crossed the wire: the padding as it is, the forge's own files compressed.
141 let wire = 0;
142 const link: Store = {
143 ...real,
144 where: `throttled ${mbit} Mbit/s`,
145 async get(key) {
146 if (key === `${prefix}manifest.json`) return new TextEncoder().encode(JSON.stringify(manifest));
147 const plain = served.get(key);
148 const body = plain !== undefined ? await packChunk(plain) : await real.get(key);
149 if (body === undefined) return undefined;
150 wire += body.byteLength;
151 busy = Math.max(ticks(), busy) + body.byteLength / wirePerMs;
152 skew += Math.max(0, busy - ticks());
153 return body;
154 },
155 };
156
157 const t = ticks();
158 const r = await runDrill({
159 ticks,
160 store: link,
161 dataDir: join(root, `drill-data-${mbit}`),
162 boot: async () => ({ ok: true, reason: "" }),
163 });
164 const took = ticks() - t;
165 console.log(`drill of ${Math.round(manifest.bytes / MB)} MB at ${mbit} Mbit/s: ${Math.round(took)} ms; ${r.phases.map((p) => `${p.phase} ${p.ms} ms`).join(", ")}`);
166 return { r, bytes: manifest.bytes, wire, wirePerMs };
167 }
168
169 /** What a drill left, judged by its test. */
170 export function judge({ r, bytes, wire, wirePerMs }: Drilled, wireBytes: number): void {
171 expect(r.reason).toBeNull();
172 expect(r.verdict).toBe("green");
173 expect(bytes).toBeGreaterThanOrEqual(wireBytes);
174 expect(r.budgetMs).toBe(drillBudgetMs(bytes));
175 expect(r.budgetMs).toBeGreaterThanOrEqual(10 * 60_000 + (wireBytes / MB) * 1_000);
176 expect(r.ms).toBeLessThan(r.budgetMs);
177 expect(r.phases.map((p) => p.phase)).toEqual(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]);
178 // The wire is the floor of the download; every phase has a time.
179 const download = r.phases.find((p) => p.phase === "files")!.ms;
180 expect(wire).toBeGreaterThanOrEqual(wireBytes);
181 expect(download).toBeGreaterThanOrEqual(wire / wirePerMs - 500);
182 for (const p of r.phases) expect(p.ms).toBeGreaterThanOrEqual(0);
183 }
test/rehearsal-abrupt.test.ts added +23
@@ -0,0 +1,23 @@
1 /**
2 * The roll rehearsal (#873), for real: an old forge that exits the moment it
3 * is signalled refuses the requests routers still send it. This is what makes
4 * `rehearsal-real.test.ts`'s "refuses nobody" mean something: the same roll,
5 * linger 0, is seen refusing.
6 *
7 * The roll runs in `beforeAll` (`test/helpers/real-roll.ts`); the file budget
8 * times the whole file, hooks included (#12397).
9 */
10
11 import { beforeAll, expect, test } from "bun:test";
12 import { realRoll } from "./helpers/real-roll.ts";
13
14 let abrupt: Awaited<ReturnType<typeof realRoll>>;
15 beforeAll(async () => {
16 abrupt = await realRoll(0);
17 }, 60_000);
18
19 test("an old forge that exits on SIGTERM refuses what routers still send it", () => {
20 expect(abrupt.problem, abrupt.problem ?? "").toBeNull();
21 expect(abrupt.refused).toBeGreaterThan(0);
22 expect(abrupt.tally.byOutcome["refused"]).toBeGreaterThan(0);
23 });
test/rehearsal-real.test.ts +27 −29
@@ -1,46 +1,44 @@
11 /**
2 * The roll rehearsal (#873), for real: two short rolls of two forge
3 * processes on one data directory. A forge that exits the moment it is
4 * signalled refuses the requests routers still send it; one that lingers
5 * past a probe period refuses none.
2 * The roll rehearsal (#873), for real: an old forge that lingers past a
3 * probe period, rolled onto a new one on the same data directory, refuses
4 * none of the requests routers still send it. Its counterpart, an old forge
5 * that exits the moment it is signalled, is `rehearsal-abrupt.test.ts`.
6 *
7 * The roll runs in `beforeAll` (`test/helpers/real-roll.ts`) and the tests
8 * judge what it left: the file budget times the whole file, hooks included,
9 * so its cost is counted there, and a hook that throws fails every test here.
610 *
711 * In the gate (#11098); `scripts/flake-hunt.ts admit` is how it is proven.
812 */
913
10 import { describe, expect, test } from "bun:test";
11 import { rehearse } from "../scripts/rehearse-roll.ts";
12 import { isRefusal, missLine, parseArgs, type RehearsalArgs } from "../scripts/rehearsal.ts";
14 import { beforeAll, describe, expect, test } from "bun:test";
15 import { isRefusal, missLine } from "../scripts/rehearsal.ts";
16 import { realRoll } from "./helpers/real-roll.ts";
17
18 // Six probe periods: the old forge is seen draining, and leaves rotation,
19 // long before it stops answering.
20 const LINGER_S = 2;
1321
1422 describe("a real roll of two forge processes on one data directory", () => {
15 const short = (lingerS: number): RehearsalArgs => ({
16 ...parseArgs([], `${import.meta.dir}/..`),
17 // Room for a slow boot and a long linger on a busy machine (#2020).
18 seconds: 30,
19 // The roll ends a second after the old forge exits, not at 30 s (#12381):
20 // enough for the new forge to take traffic alone and the tallies to settle.
21 settleMs: 1000,
22 probeMs: 300,
23 lingerS,
24 concurrency: 2,
25 });
26
27 test("an old forge that exits on SIGTERM refuses what routers still send it", async () => {
28 const v = await rehearse(short(0));
29 expect(v.problem, v.problem ?? "").toBeNull();
30 expect(v.refused).toBeGreaterThan(0);
31 expect(v.tally.byOutcome["refused"]).toBeGreaterThan(0);
23 let v: Awaited<ReturnType<typeof realRoll>>;
24 beforeAll(async () => {
25 v = await realRoll(LINGER_S);
3226 }, 60_000);
3327
34 test("an old forge that lingers past a probe period refuses nobody", async () => {
35 const v = await rehearse(short(5));
28 test("an old forge that lingers past a probe period refuses nobody", () => {
3629 expect(v.problem, v.problem ?? "").toBeNull();
3730 // Each refusal says when, whom, what came back and where both processes were.
3831 expect(v.tally.misses.filter((m) => isRefusal(m.outcome)).map(missLine)).toEqual([]);
3932 expect(v.refused).toBe(0);
33 });
34
35 test("both forges served, the new one after the old was signalled", () => {
4036 expect(v.tally.byForge["old"]).toBeGreaterThan(0);
4137 expect(v.tally.byForge["new"]).toBeGreaterThan(0);
42 // It lingered: the old forge left rotation before it stopped answering.
43 expect(v.timeline.oldExitMs).toBeGreaterThanOrEqual(5000);
38 });
39
40 test("it lingered: the old forge left rotation before it stopped answering", () => {
41 expect(v.timeline.oldExitMs).toBeGreaterThanOrEqual(LINGER_S * 1000);
4442 expect(v.timeline.oldOutOfRotationMs).toBeLessThan(v.timeline.oldExitMs ?? 0);
45 }, 60_000);
43 });
4644 });
test/rehearsal.test.ts +24 −1
@@ -1,6 +1,6 @@
11 /**
22 * The roll rehearsal (#873): its plain parts. The two short real rolls of
3 * two forge processes on one data directory are `rehearsal-real.test.ts`.
3 * two forge processes on one data directory are `rehearsal-real.test.ts` and `rehearsal-abrupt.test.ts`.
44 */
55
66 import { describe, expect, test } from "bun:test";
@@ -260,3 +260,26 @@ describe("rehearse: a forge that dies at boot", () => {
260260 expect(message).toMatch(/Is port \d+ in use\?$/);
261261 }, 30_000);
262262 });
263
264 describe("rehearse: the old forge's lead before the new one starts", () => {
265 // A stand-in forge that is ready at once and leaves on SIGTERM: only the roll's clock is real.
266 const stub = 'Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response("ok") }); process.on("SIGTERM", () => process.exit(0));';
267 const rollWith = async (leadMs: number | undefined) => {
268 const checkout = mkdtempSync(join(tmpdir(), "forge-rehearsal-lead-"));
269 mkdirSync(join(checkout, "src"));
270 writeFileSync(join(checkout, "src", "main.ts"), stub);
271 try {
272 return await rehearse({ ...parseArgs([], checkout), seconds: 4, settleMs: 100, leadMs, probeMs: 50, lingerS: 0, concurrency: 1 });
273 } finally {
274 rmSync(checkout, { recursive: true, force: true });
275 }
276 };
277
278 test("leadMs is how long the old forge serves alone; unset, a quarter of the roll (#12397)", async () => {
279 const [short, unset] = await Promise.all([rollWith(100), rollWith(undefined)]);
280 expect(short.timeline.newInRotationMs).toBeGreaterThanOrEqual(100);
281 expect(short.timeline.newInRotationMs).toBeLessThan(800);
282 // A quarter of 4 s.
283 expect(unset.timeline.newInRotationMs).toBeGreaterThanOrEqual(1_000);
284 }, 30_000);
285 });
test/release-handover-drain.test.ts added +33
@@ -0,0 +1,33 @@
1 /**
2 * `handover.drain` (#2040) lets go of the port first, so new connections
3 * are the new forge's, then finishes the rest of the drain, and closes only
4 * once the push in flight is answered. Its counterpart, the push across the
5 * whole roll, is `release-handover.test.ts`.
6 *
7 * The drain (`test/helpers/handover-roll.ts`) runs in `beforeAll`; the file
8 * budget times the whole file, hooks included (#12397).
9 */
10
11 import { afterAll, beforeAll, expect, test } from "bun:test";
12 import { cleanup, drainAcross, PUSH_MS } from "./helpers/handover-roll.ts";
13
14 afterAll(cleanup);
15
16 let drain: Awaited<ReturnType<typeof drainAcross>>;
17 beforeAll(async () => {
18 drain = await drainAcross();
19 }, PUSH_MS * 3);
20
21 test("handover.drain lets go of the port before the rest of the drain, and closes only after the push is answered", () => {
22 expect(drain.refusedAt).toBeGreaterThan(0);
23 expect(drain.restDoneAtRefusal).toBe(0);
24 expect(drain.answered.status).toBe(200);
25 expect(drain.answered.report).toContain("ok refs/heads/main");
26 expect(drain.closedAt).toBeGreaterThan(0);
27 // The push answered in full (200, "ok refs/heads/main") is the proof the
28 // close waited for it: the client stamps `answeredAt` after reading the
29 // body, which can trail the server's close by a millisecond.
30 expect(drain.answeredAt).toBeGreaterThan(0);
31 expect(drain.closedAt).toBeGreaterThanOrEqual(drain.restDoneAt);
32 expect(drain.head).toBe(drain.after);
33 });
test/release-handover.test.ts +24 −274
@@ -1,281 +1,31 @@
11 /**
2 * A roll never cuts an in-flight push (#2040). Two forges on one data
3 * directory share the port (SO_REUSEPORT), as during a release swap: a push
4 * that started on the old one trickles in for eight seconds while the new
5 * one comes up and the old one drains. The old stops accepting at drain
6 * start — new connections are the new forge's — and answers the push to the
7 * end before it goes.
2 * A roll never cuts an in-flight push (#2040): a push that started on the
3 * old forge trickles in for eight seconds while the new one comes up on the
4 * same port and the old one drains, and it completes, with no connection
5 * refused. What `handover.drain` does in what order is
6 * `release-handover-drain.test.ts`.
7 *
8 * The roll (`test/helpers/handover-roll.ts`) runs in `beforeAll`; the file
9 * budget times the whole file, hooks included (#12397).
810 */
911
10 import { afterAll, expect, test } from "bun:test";
11 import { readFileSync } from "node:fs";
12 import { mkdtemp, rm } from "node:fs/promises";
13 import { tmpdir } from "node:os";
14 import { join } from "node:path";
15 import { createApp } from "../src/app.ts";
16 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
17 import { createToken } from "../src/core/tokens.ts";
18 import { createUser } from "../src/core/users.ts";
19 import { repoDir } from "../src/git/repos.ts";
20 import { drainOnHandover, handover } from "../src/release/swap.ts";
21 import { serve } from "../src/http/server.ts";
22 import { gitClientEnv } from "./helpers/git-client.ts";
12 import { afterAll, beforeAll, expect, test } from "bun:test";
13 import { cleanup, migrates, PUSH_MS, rollAcross } from "./helpers/handover-roll.ts";
2314
24 // Long enough to span the new forge's start and the old one's drain start
25 // (both well under a second in process) and over a hundred probes; it was 30 s
26 // and proved no more (#12381).
27 const PUSH_MS = 8_000;
15 afterAll(cleanup);
2816
29 const dirs: string[] = [];
30 afterAll(async () => {
31 for (const d of dirs) await rm(d, { recursive: true, force: true });
32 });
33
34 // Without it, what sat in the old listener's accept queue is reset when it
35 // closes — at drain start now, at the end before; either way the kernel's.
36 const migrates = (() => {
37 try {
38 return readFileSync("/proc/sys/net/ipv4/tcp_migrate_req", "utf8").trim() === "1";
39 } catch {
40 return false;
41 }
42 })();
43
44 async function git(args: string[], cwd: string, stdin?: string): Promise<Uint8Array> {
45 const out = Bun.spawnSync(["git", ...args], {
46 cwd,
47 env: gitClientEnv(),
48 ...(stdin === undefined ? {} : { stdin: new TextEncoder().encode(stdin) }),
49 });
50 if (out.exitCode !== 0) throw new Error(`git ${args.join(" ")}: ${out.stderr.toString()}`);
51 return new Uint8Array(out.stdout);
52 }
53
54 const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes).trim();
55
56 function pktLine(line: string): Uint8Array {
57 const body = new TextEncoder().encode(line);
58 const head = (body.length + 4).toString(16).padStart(4, "0");
59 return new Uint8Array([...new TextEncoder().encode(head), ...body]);
60 }
61
62 /** `bytes` sent in pieces spread over `ms`, so the request stays in flight that long. */
63 function trickle(bytes: Uint8Array, ms: number): ReadableStream<Uint8Array> {
64 const pieces = 30;
65 const size = Math.ceil(bytes.length / pieces);
66 let i = 0;
67 return new ReadableStream({
68 async pull(controller) {
69 if (i > 0) await Bun.sleep(ms / pieces);
70 const piece = bytes.subarray(i * size, (i + 1) * size);
71 i++;
72 if (piece.length > 0) controller.enqueue(piece);
73 if (i >= pieces) controller.close();
74 },
75 });
76 }
77
78 /**
79 * The old forge on a port of its own, alice's repository, and `push(ms)`: a
80 * receive-pack of one new commit on main, trickled in over `ms`.
81 */
82 async function forgeWithPush() {
83 const dataDir = await mkdtemp(join(tmpdir(), "forge-handover-"));
84 const work = await mkdtemp(join(tmpdir(), "forge-handover-work-"));
85 dirs.push(dataDir, work);
86 // The old forge picks the port; the release binds the one it got.
87 const oldApp = createApp({ dataDir, port: 0, selfSwap: true });
88 const oldServers = await serve(oldApp);
89 const port = oldServers.port;
90 const made = await createUser(oldApp.db, {
91 origin: "seed",
92 trusted: true,
93 username: "alice",
94 email: "[email protected]",
95 password: "a-good-passphrase",
96 adminIfFirst: true,
97 });
98 if (!made.ok) throw new Error("user");
99 const alice = made.value;
100 const repo = await createRepo(oldApp, alice, { name: "garden", visibility: "public" });
101 if (!repo.ok) throw new Error("repo");
102 await commitToBranch(oldApp, alice, repo.value, {
103 message: "seed",
104 files: [{ path: "a.txt", content: "a" }],
105 });
106 const { token } = createToken(oldApp.db, alice.id, "push", "rw");
107
108 // The push: one commit on main, as git sends it to receive-pack.
109 const bare = repoDir(oldApp.reposRoot, "alice", "garden");
110 const clone = join(work, "garden");
111 await git(["clone", "-q", bare, clone], work);
112 await Bun.write(join(clone, "b.txt"), "b");
113 await git(["add", "b.txt"], clone);
114 await git(["commit", "-q", "-m", "b"], clone);
115 const before = text(await git(["rev-parse", "origin/main"], clone));
116 const after = text(await git(["rev-parse", "HEAD"], clone));
117 const pack = await git(
118 ["pack-objects", "--stdout", "--revs", "--quiet"],
119 clone,
120 `${after}\n^${before}\n`,
121 );
122 const command = pktLine(`${before} ${after} refs/heads/main\0report-status\n`);
123 const flush = new TextEncoder().encode("0000");
124 const body = new Uint8Array(command.length + flush.length + pack.length);
125 body.set(command, 0);
126 body.set(flush, command.length);
127 body.set(pack, command.length + flush.length);
128
129 const url = `http://127.0.0.1:${port}`;
130 const push = (ms: number) =>
131 fetch(`${url}/alice/garden.git/git-receive-pack`, {
132 method: "POST",
133 headers: {
134 authorization: `Basic ${btoa(`alice:${token}`)}`,
135 "content-type": "application/x-git-receive-pack-request",
136 },
137 body: trickle(body, ms),
138 });
139 return { oldApp, oldServers, port, url, bare, after, push };
140 }
141
142 test("a push in flight on the old forge completes across the roll, and no connection is refused", async () => {
143 const { oldApp, oldServers, port, url, bare, after, push: pushOver } = await forgeWithPush();
144 const dataDir = oldApp.config.dataDir;
145 const started = Date.now();
146 const push = pushOver(PUSH_MS);
147 // The push is the old forge's before the release exists.
148 await Bun.sleep(500);
149
150 const newApp = createApp({ dataDir, port, selfSwap: true });
151 const newServers = await serve(newApp);
152
153 let looping = true;
154 let asked = 0;
155 let refused = 0;
156 const client = (async () => {
157 while (looping) {
158 asked++;
159 try {
160 const res = await fetch(`${url}/api/v1/health`, {
161 headers: { connection: "close" },
162 signal: AbortSignal.timeout(3_000),
163 });
164 await res.arrayBuffer();
165 } catch {
166 refused++;
167 }
168 await Bun.sleep(20);
169 }
170 })();
171
172 // Drain start: the old forge stops listening, keeping what it holds.
173 const drained = oldServers.stopAccepting(PUSH_MS * 2);
174
175 const answer = await push;
176 const report = await answer.text();
177 expect(answer.status).toBe(200);
178 expect(report).toContain("unpack ok");
179 expect(report).toContain("ok refs/heads/main");
180 expect(Date.now() - started).toBeGreaterThanOrEqual(PUSH_MS * 0.9);
181 await drained;
182
183 looping = false;
184 await client;
185 expect(asked).toBeGreaterThan(100);
186 if (migrates) expect(refused).toBe(0);
187 expect(text(await git(["rev-parse", "refs/heads/main"], bare))).toBe(after);
188
189 // Nobody else listens: the old forge let go of the port at drain start.
190 // `stop` closes the listener asynchronously, so ask until it is gone.
191 newServers.stop();
192 let orphan = "answered";
193 for (const until = Date.now() + 10_000; Date.now() < until; ) {
194 orphan = await fetch(`${url}/api/v1/health`, {
195 headers: { connection: "close" },
196 signal: AbortSignal.timeout(2_000),
197 }).then(
198 async (res) => {
199 await res.arrayBuffer();
200 return "answered";
201 },
202 () => "refused",
203 );
204 if (orphan === "refused") break;
205 await Bun.sleep(100);
206 }
207 expect(orphan).toBe("refused");
208
209 oldServers.stop();
210 newApp.close();
211 oldApp.close();
17 let across: Awaited<ReturnType<typeof rollAcross>>;
18 beforeAll(async () => {
19 across = await rollAcross();
21220 }, PUSH_MS * 3);
21321
214 test("handover.drain lets go of the port before the rest of the drain, and closes only after the push is answered", async () => {
215 const { oldApp, oldServers, url, bare, after, push: pushOver } = await forgeWithPush();
216 const HELD_MS = 4_000;
217 const REST_MS = 2_000;
218 const push = pushOver(HELD_MS).then(async (res) => {
219 const report = await res.text();
220 answeredAt = Date.now();
221 return { status: res.status, report };
222 });
223 let answeredAt = 0;
224 let restDoneAt = 0;
225 let closedAt = 0;
226 // Wired as `main.ts` wires it; `rest` stands in for the shutdown's other
227 // steps (crew, apps, checks), `close` for its last.
228 const handOver = drainOnHandover(
229 oldServers,
230 async () => {
231 await Bun.sleep(REST_MS);
232 restDoneAt = Date.now();
233 },
234 () => {
235 closedAt = Date.now();
236 oldServers.stop();
237 },
238 );
239 handover.drain = () => void handOver();
240 try {
241 await Bun.sleep(500);
242 handover.drain();
243
244 // Nobody else listens here, so a refused connection is this listener gone.
245 let refusedAt = 0;
246 while (restDoneAt === 0) {
247 const gone = await fetch(`${url}/api/v1/health`, {
248 headers: { connection: "close" },
249 signal: AbortSignal.timeout(1_000),
250 }).then(
251 async (res) => {
252 await res.arrayBuffer();
253 return false;
254 },
255 () => true,
256 );
257 if (gone) {
258 refusedAt = Date.now();
259 break;
260 }
261 await Bun.sleep(50);
262 }
263 expect(refusedAt).toBeGreaterThan(0);
264 expect(restDoneAt).toBe(0);
265
266 const answered = await push;
267 expect(answered.status).toBe(200);
268 expect(answered.report).toContain("ok refs/heads/main");
269 while (closedAt === 0) await Bun.sleep(50);
270 // The push answered in full (200, "ok refs/heads/main") is the proof the
271 // close waited for it: the client stamps `answeredAt` after reading the
272 // body, which can trail the server's close by a millisecond.
273 expect(answeredAt).toBeGreaterThan(0);
274 expect(closedAt).toBeGreaterThanOrEqual(restDoneAt);
275 expect(text(await git(["rev-parse", "refs/heads/main"], bare))).toBe(after);
276 } finally {
277 delete handover.drain;
278 oldServers.stop();
279 oldApp.close();
280 }
281 }, 30_000);
22 test("a push in flight on the old forge completes across the roll, and no connection is refused", () => {
23 expect(across.status).toBe(200);
24 expect(across.report).toContain("unpack ok");
25 expect(across.report).toContain("ok refs/heads/main");
26 expect(across.tookMs).toBeGreaterThanOrEqual(PUSH_MS * 0.9);
27 expect(across.asked).toBeGreaterThan(100);
28 if (migrates) expect(across.refused).toBe(0);
29 expect(across.head).toBe(across.after);
30 expect(across.orphan).toBe("refused");
31 });
test/test-budget.test.ts +11
@@ -64,6 +64,17 @@ describe("breaches", () => {
6464 expect(breaches([{ file: "test/a.test.ts", seconds: 99 }], tests, new Set(["test/a.test.ts"]))).toEqual([]);
6565 });
6666
67 // A file whose work runs in beforeAll (boot-beside-peer, the real rolls, the drill,
68 // the handover: #12397) has tests of about 0 s; the file's time is its whole run,
69 // hooks included (`scripts/test-shards.ts` times the lone file's process), so the
70 // hook's cost is still held to the file budget, and a hook that throws fails the file.
71 test("hook time counts: tests of 0 s in a file over its budget is still a breach", () => {
72 const hooked = [{ file: "test/h.test.ts", test: "judges what the hook left", seconds: 0.001 }];
73 expect(breaches([{ file: "test/h.test.ts", seconds: FILE_BUDGET_SECONDS + 0.5 }], hooked, new Set())).toEqual([
74 { file: "test/h.test.ts", seconds: FILE_BUDGET_SECONDS + 0.5, budget: FILE_BUDGET_SECONDS },
75 ]);
76 });
77
6778 test("exactly at the budget is within it", () => {
6879 expect(breaches([{ file: "test/a.test.ts", seconds: FILE_BUDGET_SECONDS }], [], new Set())).toEqual([]);
6980 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs