Public: anyone can read this; the crew can change it.
The boot-beside-peer, rehearsal, release-handover and drill-400mb tests finish under the test budget #12443
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12397-the-boot-beside-peer-rehearsal-release-h → main 14 commits fork 1 thread · 1 unresolved
16 files
changed +752
−497
scripts/gate.ts −4
| @@ -68,7 +68,6 @@ export const SLOW: readonly Exclusion[] = [ | ||
| 68 | 68 | { file: "test/big-clone.test.ts", reason: "clones a big repository over smart http through a throttled link; 4.5 s a test on the gate box", issue: 1242 }, |
| 69 | 69 | { file: "test/bindings.test.ts", reason: "generates forges and renders every page for every viewer to compare counts; 3.4 s a test on the gate box", issue: 1242 }, |
| 70 | 70 | { file: "test/body-limits.test.ts", reason: "streams request bodies up to each cap against a real server; 2.6 s a test on the gate box", issue: 1242 }, |
| 71 | { file: "test/boot-beside-peer.test.ts", reason: "boots a real forge beside a 10 s peer, probing healthz until a second after it exits (#12381); was 32.6 s, about 15 s a test", issue: 1242 }, | |
| 72 | 71 | { file: "test/boot-no-stall.test.ts", reason: "replays a week of findings through the boot sweeps while sampling the event loop; 9.0 s a test on the gate box", issue: 1242 }, |
| 73 | 72 | { file: "test/boundary-tokens-gates.test.ts", reason: "asks every registered route and verb with a token bound to another repository; 6.1 s a test on the gate box", issue: 1242 }, |
| 74 | 73 | { file: "test/branches-deep.test.ts", reason: "builds 200 git branches to compare the 30-row and 200-row pages; 2.2 s a test on the gate box", issue: 1242 }, |
| @@ -98,7 +97,6 @@ export const SLOW: readonly Exclusion[] = [ | ||
| 98 | 97 | { file: "test/data-door.test.ts", reason: "runs SQL through the data door until the per-person limit holds; 2.1 s a test on the gate box", issue: 1242 }, |
| 99 | 98 | { file: "test/data-home.test.ts", reason: "starts main.ts as a real process to see it refuse this checkout; 2.4 s a test on the gate box", issue: 1242 }, |
| 100 | 99 | { file: "test/domain-dns.test.ts", reason: "drives a real terminal session through publishing and cutting over DNS records; 2.2 s a test on the gate box", issue: 1242 }, |
| 101 | { file: "test/drill-400mb.test.ts", reason: "packs and restores a 400 MB snapshot, its wire time on an injected clock, not slept (#12381); was 26.6 s, about 7 s a test", issue: 1242 }, | |
| 102 | 100 | { file: "test/drill-run.test.ts", reason: "spawns the drill script with piped stdin and waits for its JSON; 2.4 s a test on the gate box", issue: 1242 }, |
| 103 | 101 | { file: "test/drill-verdict.test.ts", reason: "boots a restored directory as a real process on a side port; 2.2 s a test on the gate box", issue: 1242 }, |
| 104 | 102 | { file: "test/engine-split-soak.test.ts", reason: "soaks 48 write-then-read round trips beside a second engine process writing flat out (1000 nightly); its engine starts in a worker; 3-5 s a test", issue: 1242 }, |
| @@ -137,9 +135,7 @@ export const SLOW: readonly Exclusion[] = [ | ||
| 137 | 135 | { file: "test/pull-variants.test.ts", reason: "runs two stand-in agent variants that each clone, push and open a pull; 10.7 s a test on the gate box", issue: 1242 }, |
| 138 | 136 | { file: "test/pulls-merge-budget.test.ts", reason: "seeds 50 pulls with git and a deliberately slow merge check; 5.5 s a test on the gate box", issue: 1242 }, |
| 139 | 137 | { file: "test/quotas-running.test.ts", reason: "waits for a queued stand-in agent run to start after the first one ends; 10.5 s a test on the gate box", issue: 1242 }, |
| 140 | { file: "test/rehearsal-real.test.ts", reason: "boots two real forge processes on one data directory and rolls, ending 1 s after the old exits (#12381); was 34.6 s, about 15 s a test", issue: 1242 }, | |
| 141 | 138 | { file: "test/release-boot.test.ts", reason: "copies src/ as the image ships it and boots that tree as a real process; 3.0 s a test on the gate box", issue: 1242 }, |
| 142 | { file: "test/release-handover.test.ts", reason: "rolls a release between two forges with a git push trickled over 8 s, not 30 (#12381); was 28.5 s, about 12 s a test", issue: 1242 }, | |
| 143 | 139 | { file: "test/release-launch.test.ts", reason: "waits out the readiness grace of a version that never becomes ready; 2.1 s a test on the gate box", issue: 1242 }, |
| 144 | 140 | { file: "test/release-load-run.test.ts", reason: "load-tests a forge stand-in answering in 150 ms to measure p95; 5.1 s a test on the gate box", issue: 1242 }, |
| 145 | 141 | { file: "test/release-preflight.test.ts", reason: "runs the release preflight on a copy of a seeded live database; 4.8 s a test on the gate box", issue: 1242 }, |
scripts/rehearsal.ts +5
| @@ -27,6 +27,11 @@ export interface RehearsalArgs { | ||
| 27 | 27 | * `seconds` (#12381): a roll that is over is not waited out. `seconds` stays the ceiling. |
| 28 | 28 | */ |
| 29 | 29 | readonly settleMs?: number | undefined; |
| 30 | /** | |
| 31 | * How long the old forge takes traffic alone before the new one is | |
| 32 | * started (#12397); unset, a quarter of `seconds`, at most 2 s. | |
| 33 | */ | |
| 34 | readonly leadMs?: number | undefined; | |
| 30 | 35 | /** How often rotation asks each forge's `/readyz` (the chart: 5 s). */ |
| 31 | 36 | readonly probeMs: number; |
| 32 | 37 | /** The old forge's FORGE_DRAIN_LINGER_S. */ |
scripts/rehearse-roll.ts +1 −1
| @@ -222,7 +222,7 @@ export async function rehearse( | ||
| 222 | 222 | // The platform's side: probe every forge each period, start the new |
| 223 | 223 | // one beside the old, and signal the old one once the new is ready. |
| 224 | 224 | const roll = async (): Promise<void> => { |
| 225 | await Bun.sleep(Math.min(2000, (args.seconds * 1000) / 4)); | |
| 225 | await Bun.sleep(args.leadMs ?? Math.min(2000, (args.seconds * 1000) / 4)); | |
| 226 | 226 | let next = await start("new", resolve(args.next), dataDir, args.lingerS, logs); |
| 227 | 227 | forges.push(next); |
| 228 | 228 | say(`new forge starting at ${next.url}`); |
src/runs/peer.ts +5 −1
| @@ -23,7 +23,11 @@ import { HEARTBEAT_MS, heldRuns } from "./lease.ts"; | ||
| 23 | 23 | export const PEER_FRESH_MS = 2 * HEARTBEAT_MS; |
| 24 | 24 | /** The pod's grace (charts/app, 720 s): an old pod is gone by then. A drain is 10 s (`src/drain.ts`); this is the ceiling for a run that ignores it. */ |
| 25 | 25 | export const PEER_WAIT_MAX_MS = 720_000; |
| 26 | const POLL_MS = 5_000; | |
| 26 | /** | |
| 27 | * One indexed read a second while waiting: work starts within a second of the | |
| 28 | * peer letting go (#12397; was 5 s). | |
| 29 | */ | |
| 30 | const POLL_MS = 1_000; | |
| 27 | 31 | |
| 28 | 32 | /** Running runs whose lease another process renewed within `PEER_FRESH_MS` of `now`. */ |
| 29 | 33 | export function peerRuns(db: Database, now = Date.now()): number { |
test/boot-beside-peer.test.ts +54 −22
| @@ -10,20 +10,26 @@ | ||
| 10 | 10 | * for most of every second, then finishes and lets go. |
| 11 | 11 | */ |
| 12 | 12 | |
| 13 | import { expect, test } from "bun:test"; | |
| 13 | import { beforeAll, expect, test } from "bun:test"; | |
| 14 | 14 | import { mkdtemp, rm, writeFile } from "node:fs/promises"; |
| 15 | 15 | import { tmpdir } from "node:os"; |
| 16 | 16 | import { join } from "node:path"; |
| 17 | 17 | import { createApp } from "../src/app.ts"; |
| 18 | 18 | |
| 19 | // Long enough that the forge boots behind the lock and is probed while it is | |
| 20 | // still held (counted below); the probes stop a second after the peer exits | |
| 21 | // rather than PEER_MS after the forge is up (#12381). | |
| 19 | // The peer's ceiling: it lets go sooner, once the forge is up and has | |
| 20 | // answered HELD_PROBES probes while the lock is still held (#12397), so a slow | |
| 21 | // boot still boots behind the lock and a fast one is not waited out. The | |
| 22 | // probes stop a second after the peer exits (#12381). A hold is 1.5 s, over | |
| 23 | // the probe's 1 s timeout, so a forge that waited on the lock to answer shows. | |
| 22 | 24 | const PEER_MS = 10_000; |
| 23 | const HOLD_MS = 2_500; | |
| 25 | const HELD_PROBES = 5; | |
| 26 | const HOLD_MS = 1_500; | |
| 24 | 27 | const GAP_MS = 150; |
| 25 | 28 | |
| 26 | /** Holds the write lock `HOLD_MS` of every `HOLD_MS + GAP_MS`, renewing the 8 leases each time; then lands them and exits. */ | |
| 29 | /** | |
| 30 | * Holds the write lock `HOLD_MS` of every `HOLD_MS + GAP_MS`, renewing the 8 leases each time, | |
| 31 | * until told on stdin or `PEER_MS`; then lands them and exits. | |
| 32 | */ | |
| 27 | 33 | const peerScript = ` |
| 28 | 34 | const { Database } = require("bun:sqlite"); |
| 29 | 35 | const db = new Database(process.argv[2]); |
| @@ -31,21 +37,32 @@ db.run("PRAGMA busy_timeout = 15000"); | ||
| 31 | 37 | const ids = ${JSON.stringify(Array.from({ length: 8 }, (_, n) => `gate-${n}`))}; |
| 32 | 38 | const until = Date.now() + ${PEER_MS}; |
| 33 | 39 | const spin = (ms) => { const end = Date.now() + ms; while (Date.now() < end) {} }; |
| 40 | let told = false; | |
| 41 | process.stdin.on("data", () => (told = true)); | |
| 34 | 42 | console.log("peer up"); |
| 43 | const land = () => { | |
| 44 | db.run("UPDATE run SET state = 'ok' WHERE state = 'running'"); | |
| 45 | console.log("peer done"); | |
| 46 | process.exit(0); | |
| 47 | }; | |
| 35 | 48 | const tick = () => { |
| 49 | // Told during the last hold: stdin is read in the gap, so it is seen here. | |
| 50 | if (told) return land(); | |
| 36 | 51 | db.run("BEGIN IMMEDIATE"); |
| 37 | 52 | db.run("UPDATE run SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id IN (" + ids.map((i) => "'" + i + "'").join(",") + ")"); |
| 38 | 53 | spin(${HOLD_MS}); |
| 39 | 54 | db.run("COMMIT"); |
| 40 | 55 | if (Date.now() < until) return void setTimeout(tick, ${GAP_MS}); |
| 41 | db.run("UPDATE run SET state = 'ok' WHERE state = 'running'"); | |
| 42 | console.log("peer done"); | |
| 43 | process.exit(0); | |
| 56 | land(); | |
| 44 | 57 | }; |
| 45 | 58 | tick(); |
| 46 | 59 | `; |
| 47 | 60 | |
| 48 | test("healthz answers within 1 s throughout a boot beside a draining forge's batch of 8, and work starts once it lets go", async () => { | |
| 61 | /** | |
| 62 | * The boot beside the peer, as plain data: whether it came up, the probes that were slow, how | |
| 63 | * many were asked while the lock was held, and what the forge said. | |
| 64 | */ | |
| 65 | async function bootBeside() { | |
| 49 | 66 | const dataDir = await mkdtemp(join(tmpdir(), "forge-bootpeer-")); |
| 50 | 67 | const seed = createApp({ dataDir, port: 0 }); |
| 51 | 68 | seed.db.run("PRAGMA foreign_keys = OFF"); |
| @@ -58,7 +75,7 @@ test("healthz answers within 1 s throughout a boot beside a draining forge's bat | ||
| 58 | 75 | const script = join(dataDir, "..", `${dataDir.split("/").pop()}-peer.js`); |
| 59 | 76 | await writeFile(script, peerScript); |
| 60 | 77 | |
| 61 | const peer = Bun.spawn(["bun", script, join(dataDir, "forge.db")], { stdout: "pipe", stderr: "inherit" }); | |
| 78 | const peer = Bun.spawn(["bun", script, join(dataDir, "forge.db")], { stdin: "pipe", stdout: "pipe", stderr: "inherit" }); | |
| 62 | 79 | await peer.stdout.getReader().read(); |
| 63 | 80 | |
| 64 | 81 | const web = 20000 + Math.floor(Math.random() * 20000); |
| @@ -87,7 +104,7 @@ test("healthz answers within 1 s throughout a boot beside a draining forge's bat | ||
| 87 | 104 | await Bun.sleep(100); |
| 88 | 105 | } |
| 89 | 106 | } |
| 90 | expect(up).toBe(true); | |
| 107 | if (!up) return { up, slow: [], held: 0, out }; | |
| 91 | 108 | // Then every probe answers within a second while the peer holds on, and after. |
| 92 | 109 | const slow: number[] = []; |
| 93 | 110 | let held = 0; |
| @@ -95,31 +112,46 @@ test("healthz answers within 1 s throughout a boot beside a draining forge's bat | ||
| 95 | 112 | void peer.exited.then(() => (end = performance.now() + 1_000)); |
| 96 | 113 | const ceiling = performance.now() + PEER_MS + 5_000; |
| 97 | 114 | while (performance.now() < Math.min(end, ceiling)) { |
| 98 | if (peer.exitCode === null) held++; | |
| 99 | 115 | const t = performance.now(); |
| 100 | 116 | try { |
| 101 | 117 | const res = await fetch(`http://127.0.0.1:${web}/healthz`, { signal: AbortSignal.timeout(1_000) }); |
| 102 | 118 | if (!res.ok) slow.push(-res.status); |
| 119 | // Answered while the peer still held on: counted only once it came back. | |
| 120 | else if (peer.exitCode === null && ++held === HELD_PROBES) { | |
| 121 | peer.stdin.write("let go\n"); | |
| 122 | void peer.stdin.flush(); | |
| 123 | } | |
| 103 | 124 | } catch { |
| 104 | 125 | slow.push(Math.round(performance.now() - t)); |
| 105 | 126 | } |
| 106 | 127 | await Bun.sleep(200); |
| 107 | 128 | } |
| 108 | // A probe that timed out: the forge's own log names what held it. | |
| 109 | if (slow.length > 0) console.log(out); | |
| 110 | expect(slow).toEqual([]); | |
| 111 | // The forge was asked while the peer still held the lock, not only after. | |
| 112 | expect(held).toBeGreaterThan(0); | |
| 113 | expect(out).toContain("another forge still works this database"); | |
| 114 | 129 | await peer.exited; |
| 115 | 130 | for (let i = 0; i < 100 && !out.includes("the other forge let go of this database"); i++) await Bun.sleep(100); |
| 116 | expect(out).toContain("the other forge let go of this database"); | |
| 117 | expect(out).not.toContain("event loop stalled"); | |
| 131 | return { up, slow, held, out }; | |
| 118 | 132 | } finally { |
| 119 | 133 | peer.kill(); |
| 120 | forge.kill(); | |
| 134 | // Its shutdown is not what is proven here, and a drain takes seconds (#12397). | |
| 135 | forge.kill("SIGKILL"); | |
| 121 | 136 | await Promise.all([peer.exited, forge.exited]); |
| 122 | 137 | await rm(dataDir, { recursive: true, force: true }); |
| 123 | 138 | await rm(script, { force: true }); |
| 124 | 139 | } |
| 140 | } | |
| 141 | ||
| 142 | let boot: Awaited<ReturnType<typeof bootBeside>>; | |
| 143 | beforeAll(async () => { | |
| 144 | boot = await bootBeside(); | |
| 125 | 145 | }, 60_000); |
| 146 | ||
| 147 | test("healthz answers within 1 s throughout a boot beside a draining forge's batch of 8, and work starts once it lets go", () => { | |
| 148 | expect(boot.up).toBe(true); | |
| 149 | // A probe that timed out: the forge's own log names what held it. | |
| 150 | if (boot.slow.length > 0) console.log(boot.out); | |
| 151 | expect(boot.slow).toEqual([]); | |
| 152 | // The forge was asked while the peer still held the lock, not only after. | |
| 153 | expect(boot.held).toBeGreaterThanOrEqual(HELD_PROBES); | |
| 154 | expect(boot.out).toContain("another forge still works this database"); | |
| 155 | expect(boot.out).toContain("the other forge let go of this database"); | |
| 156 | expect(boot.out).not.toContain("event loop stalled"); | |
| 157 | }); |
test/drill-100mb.test.ts added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | /** | |
| 2 | * The 400 MB drill of `drill-400mb.test.ts` scaled by 4 (#2813): 100 MB at | |
| 3 | * 37.5 Mbit/s, the same 21 s of wire on the drill's injected clock and an | |
| 4 | * 11 min 40 s budget. It needs no large scratch disk, so it always runs: the | |
| 5 | * streaming, the phase times and the budget are proven everywhere; only the | |
| 6 | * disk write of the full size is not. | |
| 7 | * | |
| 8 | * The drill (`test/helpers/throttled-drill.ts`) runs in `beforeAll`; the file | |
| 9 | * budget times the whole file, hooks included (#12397). | |
| 10 | */ | |
| 11 | import { beforeAll, test } from "bun:test"; | |
| 12 | import { quietLogs } from "./helpers/globals.ts"; | |
| 13 | import { judge, MB, throttledDrill, type Drilled } from "./helpers/throttled-drill.ts"; | |
| 14 | ||
| 15 | quietLogs(); | |
| 16 | ||
| 17 | let scaled: Drilled; | |
| 18 | beforeAll(async () => { | |
| 19 | scaled = await throttledDrill(100 * MB, 37.5); | |
| 20 | }, 60_000); | |
| 21 | ||
| 22 | test("scaled by 4, 100 MB at 37.5 Mbit/s restores green inside its 11 min 40 s budget, each phase timed", () => { | |
| 23 | judge(scaled, 100 * MB); | |
| 24 | }); |
test/drill-400mb.test.ts +16 −165
| @@ -4,176 +4,27 @@ | ||
| 4 | 4 | * budget (#2813): streamed into decompression (#2743), each phase timed |
| 5 | 5 | * (#2742), the budget grown with the size (#2773). |
| 6 | 6 | * |
| 7 | * Two proofs of the same rule. The full one is 400 MB at 150 Mbit/s (21 s | |
| 8 | * of wire, a 16 min 40 s budget) and writes the 400 MB database to disk, so | |
| 9 | * it runs only where the scratch disk has room (`ROOM`), and is skipped by | |
| 10 | * name where it does not: a check's scratch disk ran out (ENOSPC) at about | |
| 11 | * 320 MB. The scaled one is 1/4 of both, 100 MB at 37.5 Mbit/s (the same | |
| 12 | * 21 s of wire, an 11 min 40 s budget), and always runs, so the streaming, | |
| 13 | * the phase times and the budget are proven everywhere; only the disk | |
| 14 | * write of the full size is not. | |
| 7 | * 400 MB at 150 Mbit/s is 21 s of wire, on the drill's injected clock, and a | |
| 8 | * 16 min 40 s budget. It writes the 400 MB database to disk, so it runs only | |
| 9 | * where the scratch disk has room, and is skipped by name where it does not; | |
| 10 | * the same rule scaled by 4, which always runs, is `drill-100mb.test.ts`. | |
| 15 | 11 | * |
| 16 | * Both snapshots are generated here, a real `forge.db` followed by | |
| 17 | * incompressible padding, never kept in the repo. Neither has a large | |
| 18 | * repository, so the `bundles` phase is ~0 ms in both: clone time at this | |
| 19 | * size is not proven here. | |
| 12 | * The drill (`test/helpers/throttled-drill.ts`) runs in `beforeAll`; the file | |
| 13 | * budget times the whole file, hooks included (#12397). | |
| 20 | 14 | */ |
| 21 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 22 | import { mkdtempSync, rmSync, statfsSync } from "node:fs"; | |
| 23 | import { tmpdir } from "node:os"; | |
| 24 | import { join } from "node:path"; | |
| 25 | import { createApp, type App } from "../src/app.ts"; | |
| 26 | import { createUser } from "../src/core/users.ts"; | |
| 27 | import { drillBudgetMs, runDrill } from "../src/snapshot/drill.ts"; | |
| 28 | import { chunkId, chunkKey, packChunk } from "../src/snapshot/chunks.ts"; | |
| 29 | import { readWhole } from "../src/snapshot/restore.ts"; | |
| 30 | import { listSnapshots, takeSnapshot, type Manifest } from "../src/snapshot/snapshot.ts"; | |
| 31 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 15 | import { beforeAll, expect, test } from "bun:test"; | |
| 32 | 16 | import { quietLogs } from "./helpers/globals.ts"; |
| 33 | // The wire's 21 s is on the drill's injected clock (`ticks`), not slept. | |
| 17 | import { judge, MB, roomy, throttledDrill, type Drilled } from "./helpers/throttled-drill.ts"; | |
| 34 | 18 | |
| 35 | 19 | quietLogs(); |
| 36 | 20 | |
| 37 | const MB = 1_000_000; | |
| 38 | /** Free bytes the full run wants on the scratch disk: the 400 MB restored, the rest as margin. */ | |
| 39 | const ROOM = 1_500_000_000; | |
| 40 | const free = (): number => { | |
| 41 | const s = statfsSync(tmpdir()); | |
| 42 | return s.bavail * s.bsize; | |
| 43 | }; | |
| 44 | const roomy = free() >= ROOM; | |
| 21 | const room = roomy(); | |
| 22 | let full: Drilled; | |
| 23 | beforeAll(async () => { | |
| 24 | if (room) full = await throttledDrill(400 * MB, 150); | |
| 25 | }, 60_000); | |
| 45 | 26 | |
| 46 | /** Fill `out` (a multiple of 4 bytes) with pseudo-random words, so zstd can only store it. */ | |
| 47 | function fillNoise(out: Uint8Array): void { | |
| 48 | const words = new Uint32Array(out.buffer, out.byteOffset, out.byteLength / 4); | |
| 49 | let x = 2463534242; | |
| 50 | for (let i = 0; i < words.length; i++) { | |
| 51 | x ^= x << 13; | |
| 52 | x >>>= 0; | |
| 53 | x ^= x >>> 17; | |
| 54 | x ^= x << 5; | |
| 55 | x >>>= 0; | |
| 56 | words[i] = x; | |
| 57 | } | |
| 58 | } | |
| 59 | ||
| 60 | describe("a 400 MB snapshot through a 150 Mbit/s store", () => { | |
| 61 | let root = ""; | |
| 62 | let app: App; | |
| 63 | let real: Store; | |
| 64 | let base: Manifest; | |
| 65 | let database: Uint8Array; | |
| 66 | ||
| 67 | beforeAll(async () => { | |
| 68 | root = mkdtempSync(join(tmpdir(), "forge-drill-400-")); | |
| 69 | app = createApp({ dataDir: join(root, "live"), port: 0 }); | |
| 70 | const u = await createUser(app.db, { | |
| 71 | origin: "seed", | |
| 72 | trusted: true, | |
| 73 | username: "alice", | |
| 74 | email: "[email protected]", | |
| 75 | password: "a-good-passphrase", | |
| 76 | adminIfFirst: true, | |
| 77 | }); | |
| 78 | if (!u.ok) throw new Error(u.error); | |
| 79 | real = dirStore(join(root, "store")); | |
| 80 | await takeSnapshot( | |
| 81 | { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" }, | |
| 82 | real, | |
| 83 | new Date("2026-10-18T08:00:00Z"), | |
| 84 | ); | |
| 85 | base = (await listSnapshots(real))[0]!; | |
| 86 | // The database is the snapshot's own copy, the live file being in its WAL. | |
| 87 | database = await readWhole(real, base.name, base.files.find((f) => f.path === "forge.db")!); | |
| 88 | }); | |
| 89 | afterAll(() => { | |
| 90 | app.close(); | |
| 91 | rmSync(root, { recursive: true, force: true }); | |
| 92 | }); | |
| 93 | ||
| 94 | /** | |
| 95 | * Drill a snapshot of `wireBytes` served at `mbit` Mbit/s: forge.db then | |
| 96 | * padding to that size on the wire, a database whose tail SQLite never | |
| 97 | * reads, in chunks of 4 MiB that zstd can only store. | |
| 98 | */ | |
| 99 | async function prove(wireBytes: number, mbit: number): Promise<void> { | |
| 100 | const prefix = `snapshots/${base.name}/`; | |
| 101 | const wirePerMs = (mbit * MB) / 8 / 1000; | |
| 102 | const db = database; | |
| 103 | const whole = new Uint8Array(db.byteLength + wireBytes); | |
| 104 | whole.set(db); | |
| 105 | fillNoise(whole.subarray(db.byteLength)); | |
| 106 | const CHUNK = 4 * 1024 * 1024; | |
| 107 | // Each chunk is a view of `whole`, served packed when it is asked for: nothing of this size is | |
| 108 | // written to the store. | |
| 109 | const served = new Map<string, Uint8Array>(); | |
| 110 | const chunks: string[] = []; | |
| 111 | for (let at = 0; at < whole.byteLength; at += CHUNK) { | |
| 112 | const plain = whole.subarray(at, at + CHUNK); | |
| 113 | const id = chunkId(plain); | |
| 114 | served.set(chunkKey(id), plain); | |
| 115 | chunks.push(id); | |
| 116 | } | |
| 117 | const others = base.files.filter((f) => f.path !== "forge.db"); | |
| 118 | const files = [...others, { path: "forge.db", bytes: whole.byteLength, sha256: chunkId(whole), chunks }]; | |
| 119 | const manifest: Manifest = { | |
| 120 | ...base, | |
| 121 | files, | |
| 122 | bytes: files.reduce((n, f) => n + f.bytes, 0), | |
| 123 | }; | |
| 124 | expect(manifest.bytes).toBeGreaterThanOrEqual(wireBytes); | |
| 125 | ||
| 126 | // One link for every get: each waits its turn behind the bytes before it. | |
| 127 | // The wait is on the drill's clock, moved forward by the wire time rather | |
| 128 | // than slept (#12381), so the phases see the full wire and the run does not. | |
| 129 | let skew = 0; | |
| 130 | const ticks = (): number => performance.now() + skew; | |
| 131 | let busy = 0; | |
| 132 | // What crossed the wire: the padding as it is, the forge's own files compressed. | |
| 133 | let wire = 0; | |
| 134 | const link: Store = { | |
| 135 | ...real, | |
| 136 | where: `throttled ${mbit} Mbit/s`, | |
| 137 | async get(key) { | |
| 138 | if (key === `${prefix}manifest.json`) return new TextEncoder().encode(JSON.stringify(manifest)); | |
| 139 | const plain = served.get(key); | |
| 140 | const body = plain !== undefined ? await packChunk(plain) : await real.get(key); | |
| 141 | if (body === undefined) return undefined; | |
| 142 | wire += body.byteLength; | |
| 143 | busy = Math.max(ticks(), busy) + body.byteLength / wirePerMs; | |
| 144 | skew += Math.max(0, busy - ticks()); | |
| 145 | return body; | |
| 146 | }, | |
| 147 | }; | |
| 148 | ||
| 149 | const t = ticks(); | |
| 150 | const r = await runDrill({ | |
| 151 | ticks, | |
| 152 | store: link, | |
| 153 | dataDir: join(root, `drill-data-${mbit}`), | |
| 154 | boot: async () => ({ ok: true, reason: "" }), | |
| 155 | }); | |
| 156 | const took = ticks() - t; | |
| 157 | console.log(`drill of ${Math.round(manifest.bytes / MB)} MB at ${mbit} Mbit/s: ${Math.round(took)} ms; ${r.phases.map((p) => `${p.phase} ${p.ms} ms`).join(", ")}`); | |
| 158 | ||
| 159 | expect(r.reason).toBeNull(); | |
| 160 | expect(r.verdict).toBe("green"); | |
| 161 | expect(r.budgetMs).toBe(drillBudgetMs(manifest.bytes)); | |
| 162 | expect(r.budgetMs).toBeGreaterThanOrEqual(10 * 60_000 + (wireBytes / MB) * 1_000); | |
| 163 | expect(r.ms).toBeLessThan(r.budgetMs); | |
| 164 | expect(r.phases.map((p) => p.phase)).toEqual(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]); | |
| 165 | // The wire is the floor of the download; every phase has a time. | |
| 166 | const download = r.phases.find((p) => p.phase === "files")!.ms; | |
| 167 | expect(wire).toBeGreaterThanOrEqual(wireBytes); | |
| 168 | expect(download).toBeGreaterThanOrEqual(wire / wirePerMs - 500); | |
| 169 | for (const p of r.phases) expect(p.ms).toBeGreaterThanOrEqual(0); | |
| 170 | } | |
| 171 | ||
| 172 | test.skipIf(!roomy)("400 MB at 150 Mbit/s restores green inside its 16 min 40 s budget, each phase timed", async () => { | |
| 173 | await prove(400 * MB, 150); | |
| 174 | }, 90_000); | |
| 175 | ||
| 176 | test("scaled by 4, 100 MB at 37.5 Mbit/s restores green inside its 11 min 40 s budget, each phase timed", async () => { | |
| 177 | await prove(100 * MB, 37.5); | |
| 178 | }, 90_000); | |
| 27 | test.skipIf(!room)("400 MB at 150 Mbit/s restores green inside its 16 min 40 s budget, each phase timed", () => { | |
| 28 | judge(full, 400 * MB); | |
| 29 | expect(full.bytes).toBeGreaterThanOrEqual(400 * MB); | |
| 179 | 30 | }); |
test/helpers/handover-roll.ts added +282
| @@ -0,0 +1,282 @@ | ||
| 1 | /** | |
| 2 | * The two rolls of a release swap (#2040), for `release-handover.test.ts` | |
| 3 | * and `release-handover-drain.test.ts`: two forges on one data directory | |
| 4 | * share the port (SO_REUSEPORT), and a push that started on the old one | |
| 5 | * trickles in for eight seconds while the new one comes up and the old one | |
| 6 | * drains. One roll a file (#12397), so neither file waits out two pushes. | |
| 7 | */ | |
| 8 | ||
| 9 | import { readFileSync } from "node:fs"; | |
| 10 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { createApp } from "../../src/app.ts"; | |
| 14 | import { commitToBranch, createRepo } from "../../src/actions/repos.ts"; | |
| 15 | import { createToken } from "../../src/core/tokens.ts"; | |
| 16 | import { createUser } from "../../src/core/users.ts"; | |
| 17 | import { repoDir } from "../../src/git/repos.ts"; | |
| 18 | import { drainOnHandover, handover } from "../../src/release/swap.ts"; | |
| 19 | import { serve } from "../../src/http/server.ts"; | |
| 20 | import { gitClientEnv } from "./git-client.ts"; | |
| 21 | ||
| 22 | // Long enough to span the new forge's start and the old one's drain start | |
| 23 | // (both well under a second in process) and over a hundred probes; it was 30 s | |
| 24 | // and proved no more (#12381). | |
| 25 | export const PUSH_MS = 8_000; | |
| 26 | ||
| 27 | const dirs: string[] = []; | |
| 28 | export async function cleanup(): Promise<void> { | |
| 29 | for (const d of dirs) await rm(d, { recursive: true, force: true }); | |
| 30 | } | |
| 31 | ||
| 32 | // Without it, what sat in the old listener's accept queue is reset when it | |
| 33 | // closes — at drain start now, at the end before; either way the kernel's. | |
| 34 | export const migrates = (() => { | |
| 35 | try { | |
| 36 | return readFileSync("/proc/sys/net/ipv4/tcp_migrate_req", "utf8").trim() === "1"; | |
| 37 | } catch { | |
| 38 | return false; | |
| 39 | } | |
| 40 | })(); | |
| 41 | ||
| 42 | async function git(args: string[], cwd: string, stdin?: string): Promise<Uint8Array> { | |
| 43 | const out = Bun.spawnSync(["git", ...args], { | |
| 44 | cwd, | |
| 45 | env: gitClientEnv(), | |
| 46 | ...(stdin === undefined ? {} : { stdin: new TextEncoder().encode(stdin) }), | |
| 47 | }); | |
| 48 | if (out.exitCode !== 0) throw new Error(`git ${args.join(" ")}: ${out.stderr.toString()}`); | |
| 49 | return new Uint8Array(out.stdout); | |
| 50 | } | |
| 51 | ||
| 52 | const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes).trim(); | |
| 53 | ||
| 54 | function pktLine(line: string): Uint8Array { | |
| 55 | const body = new TextEncoder().encode(line); | |
| 56 | const head = (body.length + 4).toString(16).padStart(4, "0"); | |
| 57 | return new Uint8Array([...new TextEncoder().encode(head), ...body]); | |
| 58 | } | |
| 59 | ||
| 60 | /** `bytes` sent in pieces spread over `ms`, so the request stays in flight that long. */ | |
| 61 | function trickle(bytes: Uint8Array, ms: number): ReadableStream<Uint8Array> { | |
| 62 | const pieces = 30; | |
| 63 | const size = Math.ceil(bytes.length / pieces); | |
| 64 | let i = 0; | |
| 65 | return new ReadableStream({ | |
| 66 | async pull(controller) { | |
| 67 | if (i > 0) await Bun.sleep(ms / pieces); | |
| 68 | const piece = bytes.subarray(i * size, (i + 1) * size); | |
| 69 | i++; | |
| 70 | if (piece.length > 0) controller.enqueue(piece); | |
| 71 | if (i >= pieces) controller.close(); | |
| 72 | }, | |
| 73 | }); | |
| 74 | } | |
| 75 | ||
| 76 | /** | |
| 77 | * The old forge on a port of its own, alice's repository, and `push(ms)`: a | |
| 78 | * receive-pack of one new commit on main, trickled in over `ms`. | |
| 79 | */ | |
| 80 | async function forgeWithPush() { | |
| 81 | const dataDir = await mkdtemp(join(tmpdir(), "forge-handover-")); | |
| 82 | const work = await mkdtemp(join(tmpdir(), "forge-handover-work-")); | |
| 83 | dirs.push(dataDir, work); | |
| 84 | // The old forge picks the port; the release binds the one it got. | |
| 85 | const oldApp = createApp({ dataDir, port: 0, selfSwap: true }); | |
| 86 | const oldServers = await serve(oldApp); | |
| 87 | const port = oldServers.port; | |
| 88 | const made = await createUser(oldApp.db, { | |
| 89 | origin: "seed", | |
| 90 | trusted: true, | |
| 91 | username: "alice", | |
| 92 | email: "[email protected]", | |
| 93 | password: "a-good-passphrase", | |
| 94 | adminIfFirst: true, | |
| 95 | }); | |
| 96 | if (!made.ok) throw new Error("user"); | |
| 97 | const alice = made.value; | |
| 98 | const repo = await createRepo(oldApp, alice, { name: "garden", visibility: "public" }); | |
| 99 | if (!repo.ok) throw new Error("repo"); | |
| 100 | await commitToBranch(oldApp, alice, repo.value, { | |
| 101 | message: "seed", | |
| 102 | files: [{ path: "a.txt", content: "a" }], | |
| 103 | }); | |
| 104 | const { token } = createToken(oldApp.db, alice.id, "push", "rw"); | |
| 105 | ||
| 106 | // The push: one commit on main, as git sends it to receive-pack. | |
| 107 | const bare = repoDir(oldApp.reposRoot, "alice", "garden"); | |
| 108 | const clone = join(work, "garden"); | |
| 109 | await git(["clone", "-q", bare, clone], work); | |
| 110 | await Bun.write(join(clone, "b.txt"), "b"); | |
| 111 | await git(["add", "b.txt"], clone); | |
| 112 | await git(["commit", "-q", "-m", "b"], clone); | |
| 113 | const before = text(await git(["rev-parse", "origin/main"], clone)); | |
| 114 | const after = text(await git(["rev-parse", "HEAD"], clone)); | |
| 115 | const pack = await git( | |
| 116 | ["pack-objects", "--stdout", "--revs", "--quiet"], | |
| 117 | clone, | |
| 118 | `${after}\n^${before}\n`, | |
| 119 | ); | |
| 120 | const command = pktLine(`${before} ${after} refs/heads/main\0report-status\n`); | |
| 121 | const flush = new TextEncoder().encode("0000"); | |
| 122 | const body = new Uint8Array(command.length + flush.length + pack.length); | |
| 123 | body.set(command, 0); | |
| 124 | body.set(flush, command.length); | |
| 125 | body.set(pack, command.length + flush.length); | |
| 126 | ||
| 127 | const url = `http://127.0.0.1:${port}`; | |
| 128 | const push = (ms: number) => | |
| 129 | fetch(`${url}/alice/garden.git/git-receive-pack`, { | |
| 130 | method: "POST", | |
| 131 | headers: { | |
| 132 | authorization: `Basic ${btoa(`alice:${token}`)}`, | |
| 133 | "content-type": "application/x-git-receive-pack-request", | |
| 134 | }, | |
| 135 | body: trickle(body, ms), | |
| 136 | }); | |
| 137 | return { oldApp, oldServers, port, url, bare, after, push }; | |
| 138 | } | |
| 139 | ||
| 140 | /** | |
| 141 | * The roll of the first test, as plain data: what the push, the prober and the orphan probe saw. | |
| 142 | */ | |
| 143 | export async function rollAcross() { | |
| 144 | const { oldApp, oldServers, port, url, bare, after, push: pushOver } = await forgeWithPush(); | |
| 145 | const dataDir = oldApp.config.dataDir; | |
| 146 | const started = Date.now(); | |
| 147 | const push = pushOver(PUSH_MS); | |
| 148 | // The push is the old forge's before the release exists. | |
| 149 | await Bun.sleep(500); | |
| 150 | ||
| 151 | const newApp = createApp({ dataDir, port, selfSwap: true }); | |
| 152 | const newServers = await serve(newApp); | |
| 153 | ||
| 154 | let looping = true; | |
| 155 | let asked = 0; | |
| 156 | let refused = 0; | |
| 157 | const client = (async () => { | |
| 158 | while (looping) { | |
| 159 | asked++; | |
| 160 | try { | |
| 161 | const res = await fetch(`${url}/api/v1/health`, { | |
| 162 | headers: { connection: "close" }, | |
| 163 | signal: AbortSignal.timeout(3_000), | |
| 164 | }); | |
| 165 | await res.arrayBuffer(); | |
| 166 | } catch { | |
| 167 | refused++; | |
| 168 | } | |
| 169 | await Bun.sleep(20); | |
| 170 | } | |
| 171 | })(); | |
| 172 | ||
| 173 | try { | |
| 174 | // Drain start: the old forge stops listening, keeping what it holds. | |
| 175 | const drained = oldServers.stopAccepting(PUSH_MS * 2); | |
| 176 | ||
| 177 | const answer = await push; | |
| 178 | const status = answer.status; | |
| 179 | const report = await answer.text(); | |
| 180 | const tookMs = Date.now() - started; | |
| 181 | await drained; | |
| 182 | ||
| 183 | looping = false; | |
| 184 | await client; | |
| 185 | const head = text(await git(["rev-parse", "refs/heads/main"], bare)); | |
| 186 | ||
| 187 | // Nobody else listens: the old forge let go of the port at drain start. | |
| 188 | // `stop` closes the listener asynchronously, so ask until it is gone. | |
| 189 | newServers.stop(); | |
| 190 | let orphan = "answered"; | |
| 191 | for (const until = Date.now() + 10_000; Date.now() < until; ) { | |
| 192 | orphan = await fetch(`${url}/api/v1/health`, { | |
| 193 | headers: { connection: "close" }, | |
| 194 | signal: AbortSignal.timeout(2_000), | |
| 195 | }).then( | |
| 196 | async (res) => { | |
| 197 | await res.arrayBuffer(); | |
| 198 | return "answered"; | |
| 199 | }, | |
| 200 | () => "refused", | |
| 201 | ); | |
| 202 | if (orphan === "refused") break; | |
| 203 | await Bun.sleep(100); // real-timer: polls a real orphaned port until a real process lets it go | |
| 204 | } | |
| 205 | return { status, report, tookMs, asked, refused, head, after, orphan }; | |
| 206 | } finally { | |
| 207 | looping = false; | |
| 208 | await client; | |
| 209 | newServers.stop(); | |
| 210 | oldServers.stop(); | |
| 211 | newApp.close(); | |
| 212 | oldApp.close(); | |
| 213 | } | |
| 214 | } | |
| 215 | ||
| 216 | /** | |
| 217 | * The drain of the second test, as plain data: when the port went, the rest finished, the push | |
| 218 | * was answered and the close came. | |
| 219 | */ | |
| 220 | export async function drainAcross() { | |
| 221 | const { oldApp, oldServers, url, bare, after, push: pushOver } = await forgeWithPush(); | |
| 222 | const HELD_MS = 4_000; | |
| 223 | const REST_MS = 2_000; | |
| 224 | let answeredAt = 0; | |
| 225 | let restDoneAt = 0; | |
| 226 | let closedAt = 0; | |
| 227 | const push = pushOver(HELD_MS).then(async (res) => { | |
| 228 | const report = await res.text(); | |
| 229 | answeredAt = Date.now(); | |
| 230 | return { status: res.status, report }; | |
| 231 | }); | |
| 232 | // Wired as `main.ts` wires it; `rest` stands in for the shutdown's other | |
| 233 | // steps (crew, apps, checks), `close` for its last. | |
| 234 | const handOver = drainOnHandover( | |
| 235 | oldServers, | |
| 236 | async () => { | |
| 237 | await Bun.sleep(REST_MS); | |
| 238 | restDoneAt = Date.now(); | |
| 239 | }, | |
| 240 | () => { | |
| 241 | closedAt = Date.now(); | |
| 242 | oldServers.stop(); | |
| 243 | }, | |
| 244 | ); | |
| 245 | handover.drain = () => void handOver(); | |
| 246 | try { | |
| 247 | await Bun.sleep(500); | |
| 248 | handover.drain(); | |
| 249 | ||
| 250 | // Nobody else listens here, so a refused connection is this listener gone. | |
| 251 | let refusedAt = 0; | |
| 252 | while (restDoneAt === 0) { | |
| 253 | const gone = await fetch(`${url}/api/v1/health`, { | |
| 254 | headers: { connection: "close" }, | |
| 255 | signal: AbortSignal.timeout(1_000), | |
| 256 | }).then( | |
| 257 | async (res) => { | |
| 258 | await res.arrayBuffer(); | |
| 259 | return false; | |
| 260 | }, | |
| 261 | () => true, | |
| 262 | ); | |
| 263 | if (gone) { | |
| 264 | refusedAt = Date.now(); | |
| 265 | break; | |
| 266 | } | |
| 267 | await Bun.sleep(50); | |
| 268 | } | |
| 269 | const restDoneAtRefusal = restDoneAt; | |
| 270 | ||
| 271 | const answered = await push; | |
| 272 | for (const until = Date.now() + 10_000; closedAt === 0 && Date.now() < until; ) { | |
| 273 | await Bun.sleep(50); // real-timer: waits on a real socket close | |
| 274 | } | |
| 275 | const head = text(await git(["rev-parse", "refs/heads/main"], bare)); | |
| 276 | return { refusedAt, restDoneAtRefusal, answered, answeredAt, restDoneAt, closedAt, head, after }; | |
| 277 | } finally { | |
| 278 | delete handover.drain; | |
| 279 | oldServers.stop(); | |
| 280 | oldApp.close(); | |
| 281 | } | |
| 282 | } |
test/helpers/real-roll.ts added +40
| @@ -0,0 +1,40 @@ | ||
| 1 | /** | |
| 2 | * One short real roll (#873) of two forge processes on one data directory, | |
| 3 | * for `rehearsal-real.test.ts` and `rehearsal-abrupt.test.ts`: one roll a | |
| 4 | * file, so neither file runs two heavy scenarios at once (#12397). | |
| 5 | * | |
| 6 | * The data directory is migrated once in-process before the roll, so the | |
| 7 | * old forge does not spend its boot migrating: what is left is two boots, | |
| 8 | * the linger and a second of settling. | |
| 9 | */ | |
| 10 | ||
| 11 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 12 | import { tmpdir } from "node:os"; | |
| 13 | import { join } from "node:path"; | |
| 14 | import { createApp } from "../../src/app.ts"; | |
| 15 | import { rehearse } from "../../scripts/rehearse-roll.ts"; | |
| 16 | import { parseArgs } from "../../scripts/rehearsal.ts"; | |
| 17 | ||
| 18 | export async function realRoll(lingerS: number): Promise<Awaited<ReturnType<typeof rehearse>>> { | |
| 19 | const seeded = join(mkdtempSync(join(tmpdir(), "forge-rehearsal-seed-")), "data"); | |
| 20 | try { | |
| 21 | createApp({ dataDir: seeded, port: 0 }).close(); | |
| 22 | return await rehearse({ | |
| 23 | ...parseArgs([], `${import.meta.dir}/../..`), | |
| 24 | from: seeded, | |
| 25 | // Room for a slow boot and a long linger on a busy machine (#2020). | |
| 26 | seconds: 30, | |
| 27 | // The roll ends a second (over three probe periods) after the old forge exits, not at | |
| 28 | // 30 s (#12381, #12397): the new forge takes traffic alone and the tallies settle. | |
| 29 | settleMs: 1_000, | |
| 30 | // The old forge takes traffic alone for a probe period before the new one starts, and | |
| 31 | // alone again for the seconds the new one boots; the 2 s default adds nothing to that. | |
| 32 | leadMs: 300, | |
| 33 | probeMs: 300, | |
| 34 | lingerS, | |
| 35 | concurrency: 2, | |
| 36 | }); | |
| 37 | } finally { | |
| 38 | rmSync(join(seeded, ".."), { recursive: true, force: true }); | |
| 39 | } | |
| 40 | } |
test/helpers/throttled-drill.ts added +183
| @@ -0,0 +1,183 @@ | ||
| 1 | /** | |
| 2 | * One drill (#2813) of a generated snapshot through a store throttled to a | |
| 3 | * given downlink, for `drill-400mb.test.ts` (400 MB at 150 Mbit/s) and | |
| 4 | * `drill-100mb.test.ts` (the same rule scaled by 4): one drill a file | |
| 5 | * (#12397), so neither file waits out the other's. | |
| 6 | * | |
| 7 | * The snapshot is a real `forge.db` followed by incompressible padding, | |
| 8 | * never kept in the repo. Neither has a large repository, so the `bundles` | |
| 9 | * phase is ~0 ms: clone time at this size is not proven here. The wire's | |
| 10 | * time is on the drill's injected clock (`ticks`), not slept. | |
| 11 | */ | |
| 12 | import { expect } from "bun:test"; | |
| 13 | import { mkdtempSync, rmSync, statfsSync } from "node:fs"; | |
| 14 | import { tmpdir } from "node:os"; | |
| 15 | import { join } from "node:path"; | |
| 16 | import { createApp } from "../../src/app.ts"; | |
| 17 | import { createUser } from "../../src/core/users.ts"; | |
| 18 | import { drillBudgetMs, runDrill } from "../../src/snapshot/drill.ts"; | |
| 19 | import { chunkId, chunkKey, packChunk } from "../../src/snapshot/chunks.ts"; | |
| 20 | import { readWhole } from "../../src/snapshot/restore.ts"; | |
| 21 | import { listSnapshots, takeSnapshot, type Manifest } from "../../src/snapshot/snapshot.ts"; | |
| 22 | import { dirStore, type Store } from "../../src/snapshot/store.ts"; | |
| 23 | ||
| 24 | export const MB = 1_000_000; | |
| 25 | /** Free bytes the full run wants on the scratch disk: the 400 MB restored, the rest as margin. */ | |
| 26 | const ROOM = 1_500_000_000; | |
| 27 | /** | |
| 28 | * Whether the scratch disk has room for the 400 MB drill: a check's ran out | |
| 29 | * (ENOSPC) at about 320 MB. | |
| 30 | */ | |
| 31 | export const roomy = (): boolean => { | |
| 32 | const s = statfsSync(tmpdir()); | |
| 33 | return s.bavail * s.bsize >= ROOM; | |
| 34 | }; | |
| 35 | ||
| 36 | export type Drilled = { r: Awaited<ReturnType<typeof runDrill>>; bytes: number; wire: number; wirePerMs: number }; | |
| 37 | /** 4 MiB of pseudo-random words, made once: zstd can only store them. */ | |
| 38 | const NOISE = (() => { | |
| 39 | const words = new Uint32Array(1 << 20); | |
| 40 | let x = 2463534242; | |
| 41 | for (let i = 0; i < words.length; i++) { | |
| 42 | x ^= x << 13; | |
| 43 | x >>>= 0; | |
| 44 | x ^= x >>> 17; | |
| 45 | x ^= x << 5; | |
| 46 | x >>>= 0; | |
| 47 | words[i] = x; | |
| 48 | } | |
| 49 | return new Uint8Array(words.buffer); | |
| 50 | })(); | |
| 51 | ||
| 52 | /** | |
| 53 | * Fill `out` with `NOISE` over and over, each copy's first word stamped with | |
| 54 | * its number: no two 4 MiB chunks are the same bytes, and none compresses. | |
| 55 | */ | |
| 56 | function fillNoise(out: Uint8Array): void { | |
| 57 | for (let at = 0, n = 0; at < out.byteLength; at += NOISE.byteLength, n++) { | |
| 58 | const copy = NOISE.subarray(0, Math.min(NOISE.byteLength, out.byteLength - at)); | |
| 59 | out.set(copy, at); | |
| 60 | for (let k = 0; k < 4 && at + k < out.byteLength; k++) out[at + k] = (NOISE[k]! ^ (n >>> (8 * k))) & 255; | |
| 61 | } | |
| 62 | } | |
| 63 | ||
| 64 | /** | |
| 65 | * Drill a snapshot of `wireBytes` served at `mbit` Mbit/s: a forge with one | |
| 66 | * person is snapshotted, then its forge.db is served padded to that size on | |
| 67 | * the wire, a database whose tail SQLite never reads, in chunks of 4 MiB | |
| 68 | * that zstd can only store. Everything it wrote is removed before it returns. | |
| 69 | */ | |
| 70 | export async function throttledDrill(wireBytes: number, mbit: number): Promise<Drilled> { | |
| 71 | const root = mkdtempSync(join(tmpdir(), "forge-drill-")); | |
| 72 | const app = createApp({ dataDir: join(root, "live"), port: 0 }); | |
| 73 | try { | |
| 74 | const u = await createUser(app.db, { | |
| 75 | origin: "seed", | |
| 76 | trusted: true, | |
| 77 | username: "alice", | |
| 78 | email: "[email protected]", | |
| 79 | password: "a-good-passphrase", | |
| 80 | adminIfFirst: true, | |
| 81 | }); | |
| 82 | if (!u.ok) throw new Error(u.error); | |
| 83 | const real = dirStore(join(root, "store")); | |
| 84 | await takeSnapshot( | |
| 85 | { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" }, | |
| 86 | real, | |
| 87 | new Date("2026-10-18T08:00:00Z"), | |
| 88 | ); | |
| 89 | const base = (await listSnapshots(real))[0]!; | |
| 90 | // The database is the snapshot's own copy, the live file being in its WAL. | |
| 91 | const database = await readWhole(real, base.name, base.files.find((f) => f.path === "forge.db")!); | |
| 92 | return await drillOf(root, real, base, database, wireBytes, mbit); | |
| 93 | } finally { | |
| 94 | app.close(); | |
| 95 | rmSync(root, { recursive: true, force: true }); | |
| 96 | } | |
| 97 | } | |
| 98 | ||
| 99 | async function drillOf( | |
| 100 | root: string, | |
| 101 | real: Store, | |
| 102 | base: Manifest, | |
| 103 | database: Uint8Array, | |
| 104 | wireBytes: number, | |
| 105 | mbit: number, | |
| 106 | ): Promise<Drilled> { | |
| 107 | const prefix = `snapshots/${base.name}/`; | |
| 108 | const wirePerMs = (mbit * MB) / 8 / 1000; | |
| 109 | const db = database; | |
| 110 | const whole = new Uint8Array(db.byteLength + wireBytes); | |
| 111 | whole.set(db); | |
| 112 | fillNoise(whole.subarray(db.byteLength)); | |
| 113 | const CHUNK = 4 * 1024 * 1024; | |
| 114 | // Each chunk is a view of `whole`, served packed when it is asked for: nothing of this size is | |
| 115 | // written to the store. | |
| 116 | // The ids are sha256 like `chunkId`'s, taken on the thread pool at once: one after the other, | |
| 117 | // hashing 400 MB twice was a third of the file's time (#12397). | |
| 118 | const hex = (b: ArrayBuffer): string => Buffer.from(b).toString("hex"); | |
| 119 | const sha = (b: Uint8Array<ArrayBuffer>): Promise<string> => crypto.subtle.digest("SHA-256", b).then(hex); | |
| 120 | const plains: Uint8Array<ArrayBuffer>[] = []; | |
| 121 | for (let at = 0; at < whole.byteLength; at += CHUNK) plains.push(whole.subarray(at, at + CHUNK)); | |
| 122 | const [wholeId, ...chunks] = await Promise.all([sha(whole), ...plains.map(sha)]); | |
| 123 | const served = new Map<string, Uint8Array>(plains.map((plain, i) => [chunkKey(chunks[i]!), plain])); | |
| 124 | if (chunkId(plains[0]!) !== chunks[0] || served.size !== plains.length) throw new Error("the chunks are not named as the store names them, or two are the same"); | |
| 125 | const others = base.files.filter((f) => f.path !== "forge.db"); | |
| 126 | const files = [...others, { path: "forge.db", bytes: whole.byteLength, sha256: wholeId!, chunks }]; | |
| 127 | const manifest: Manifest = { | |
| 128 | ...base, | |
| 129 | files, | |
| 130 | bytes: files.reduce((n, f) => n + f.bytes, 0), | |
| 131 | }; | |
| 132 | if (manifest.bytes < wireBytes) throw new Error(`the snapshot is ${manifest.bytes} bytes, under ${wireBytes}`); | |
| 133 | ||
| 134 | // One link for every get: each waits its turn behind the bytes before it. | |
| 135 | // The wait is on the drill's clock, moved forward by the wire time rather | |
| 136 | // than slept (#12381), so the phases see the full wire and the run does not. | |
| 137 | let skew = 0; | |
| 138 | const ticks = (): number => performance.now() + skew; | |
| 139 | let busy = 0; | |
| 140 | // What crossed the wire: the padding as it is, the forge's own files compressed. | |
| 141 | let wire = 0; | |
| 142 | const link: Store = { | |
| 143 | ...real, | |
| 144 | where: `throttled ${mbit} Mbit/s`, | |
| 145 | async get(key) { | |
| 146 | if (key === `${prefix}manifest.json`) return new TextEncoder().encode(JSON.stringify(manifest)); | |
| 147 | const plain = served.get(key); | |
| 148 | const body = plain !== undefined ? await packChunk(plain) : await real.get(key); | |
| 149 | if (body === undefined) return undefined; | |
| 150 | wire += body.byteLength; | |
| 151 | busy = Math.max(ticks(), busy) + body.byteLength / wirePerMs; | |
| 152 | skew += Math.max(0, busy - ticks()); | |
| 153 | return body; | |
| 154 | }, | |
| 155 | }; | |
| 156 | ||
| 157 | const t = ticks(); | |
| 158 | const r = await runDrill({ | |
| 159 | ticks, | |
| 160 | store: link, | |
| 161 | dataDir: join(root, `drill-data-${mbit}`), | |
| 162 | boot: async () => ({ ok: true, reason: "" }), | |
| 163 | }); | |
| 164 | const took = ticks() - t; | |
| 165 | console.log(`drill of ${Math.round(manifest.bytes / MB)} MB at ${mbit} Mbit/s: ${Math.round(took)} ms; ${r.phases.map((p) => `${p.phase} ${p.ms} ms`).join(", ")}`); | |
| 166 | return { r, bytes: manifest.bytes, wire, wirePerMs }; | |
| 167 | } | |
| 168 | ||
| 169 | /** What a drill left, judged by its test. */ | |
| 170 | export function judge({ r, bytes, wire, wirePerMs }: Drilled, wireBytes: number): void { | |
| 171 | expect(r.reason).toBeNull(); | |
| 172 | expect(r.verdict).toBe("green"); | |
| 173 | expect(bytes).toBeGreaterThanOrEqual(wireBytes); | |
| 174 | expect(r.budgetMs).toBe(drillBudgetMs(bytes)); | |
| 175 | expect(r.budgetMs).toBeGreaterThanOrEqual(10 * 60_000 + (wireBytes / MB) * 1_000); | |
| 176 | expect(r.ms).toBeLessThan(r.budgetMs); | |
| 177 | expect(r.phases.map((p) => p.phase)).toEqual(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]); | |
| 178 | // The wire is the floor of the download; every phase has a time. | |
| 179 | const download = r.phases.find((p) => p.phase === "files")!.ms; | |
| 180 | expect(wire).toBeGreaterThanOrEqual(wireBytes); | |
| 181 | expect(download).toBeGreaterThanOrEqual(wire / wirePerMs - 500); | |
| 182 | for (const p of r.phases) expect(p.ms).toBeGreaterThanOrEqual(0); | |
| 183 | } |
test/rehearsal-abrupt.test.ts added +23
| @@ -0,0 +1,23 @@ | ||
| 1 | /** | |
| 2 | * The roll rehearsal (#873), for real: an old forge that exits the moment it | |
| 3 | * is signalled refuses the requests routers still send it. This is what makes | |
| 4 | * `rehearsal-real.test.ts`'s "refuses nobody" mean something: the same roll, | |
| 5 | * linger 0, is seen refusing. | |
| 6 | * | |
| 7 | * The roll runs in `beforeAll` (`test/helpers/real-roll.ts`); the file budget | |
| 8 | * times the whole file, hooks included (#12397). | |
| 9 | */ | |
| 10 | ||
| 11 | import { beforeAll, expect, test } from "bun:test"; | |
| 12 | import { realRoll } from "./helpers/real-roll.ts"; | |
| 13 | ||
| 14 | let abrupt: Awaited<ReturnType<typeof realRoll>>; | |
| 15 | beforeAll(async () => { | |
| 16 | abrupt = await realRoll(0); | |
| 17 | }, 60_000); | |
| 18 | ||
| 19 | test("an old forge that exits on SIGTERM refuses what routers still send it", () => { | |
| 20 | expect(abrupt.problem, abrupt.problem ?? "").toBeNull(); | |
| 21 | expect(abrupt.refused).toBeGreaterThan(0); | |
| 22 | expect(abrupt.tally.byOutcome["refused"]).toBeGreaterThan(0); | |
| 23 | }); |
test/rehearsal-real.test.ts +27 −29
| @@ -1,46 +1,44 @@ | ||
| 1 | 1 | /** |
| 2 | * The roll rehearsal (#873), for real: two short rolls of two forge | |
| 3 | * processes on one data directory. A forge that exits the moment it is | |
| 4 | * signalled refuses the requests routers still send it; one that lingers | |
| 5 | * past a probe period refuses none. | |
| 2 | * The roll rehearsal (#873), for real: an old forge that lingers past a | |
| 3 | * probe period, rolled onto a new one on the same data directory, refuses | |
| 4 | * none of the requests routers still send it. Its counterpart, an old forge | |
| 5 | * that exits the moment it is signalled, is `rehearsal-abrupt.test.ts`. | |
| 6 | * | |
| 7 | * The roll runs in `beforeAll` (`test/helpers/real-roll.ts`) and the tests | |
| 8 | * judge what it left: the file budget times the whole file, hooks included, | |
| 9 | * so its cost is counted there, and a hook that throws fails every test here. | |
| 6 | 10 | * |
| 7 | 11 | * In the gate (#11098); `scripts/flake-hunt.ts admit` is how it is proven. |
| 8 | 12 | */ |
| 9 | 13 | |
| 10 | import { describe, expect, test } from "bun:test"; | |
| 11 | import { rehearse } from "../scripts/rehearse-roll.ts"; | |
| 12 | import { isRefusal, missLine, parseArgs, type RehearsalArgs } from "../scripts/rehearsal.ts"; | |
| 14 | import { beforeAll, describe, expect, test } from "bun:test"; | |
| 15 | import { isRefusal, missLine } from "../scripts/rehearsal.ts"; | |
| 16 | import { realRoll } from "./helpers/real-roll.ts"; | |
| 17 | ||
| 18 | // Six probe periods: the old forge is seen draining, and leaves rotation, | |
| 19 | // long before it stops answering. | |
| 20 | const LINGER_S = 2; | |
| 13 | 21 | |
| 14 | 22 | describe("a real roll of two forge processes on one data directory", () => { |
| 15 | const short = (lingerS: number): RehearsalArgs => ({ | |
| 16 | ...parseArgs([], `${import.meta.dir}/..`), | |
| 17 | // Room for a slow boot and a long linger on a busy machine (#2020). | |
| 18 | seconds: 30, | |
| 19 | // The roll ends a second after the old forge exits, not at 30 s (#12381): | |
| 20 | // enough for the new forge to take traffic alone and the tallies to settle. | |
| 21 | settleMs: 1000, | |
| 22 | probeMs: 300, | |
| 23 | lingerS, | |
| 24 | concurrency: 2, | |
| 25 | }); | |
| 26 | ||
| 27 | test("an old forge that exits on SIGTERM refuses what routers still send it", async () => { | |
| 28 | const v = await rehearse(short(0)); | |
| 29 | expect(v.problem, v.problem ?? "").toBeNull(); | |
| 30 | expect(v.refused).toBeGreaterThan(0); | |
| 31 | expect(v.tally.byOutcome["refused"]).toBeGreaterThan(0); | |
| 23 | let v: Awaited<ReturnType<typeof realRoll>>; | |
| 24 | beforeAll(async () => { | |
| 25 | v = await realRoll(LINGER_S); | |
| 32 | 26 | }, 60_000); |
| 33 | 27 | |
| 34 | test("an old forge that lingers past a probe period refuses nobody", async () => { | |
| 35 | const v = await rehearse(short(5)); | |
| 28 | test("an old forge that lingers past a probe period refuses nobody", () => { | |
| 36 | 29 | expect(v.problem, v.problem ?? "").toBeNull(); |
| 37 | 30 | // Each refusal says when, whom, what came back and where both processes were. |
| 38 | 31 | expect(v.tally.misses.filter((m) => isRefusal(m.outcome)).map(missLine)).toEqual([]); |
| 39 | 32 | expect(v.refused).toBe(0); |
| 33 | }); | |
| 34 | ||
| 35 | test("both forges served, the new one after the old was signalled", () => { | |
| 40 | 36 | expect(v.tally.byForge["old"]).toBeGreaterThan(0); |
| 41 | 37 | expect(v.tally.byForge["new"]).toBeGreaterThan(0); |
| 42 | // It lingered: the old forge left rotation before it stopped answering. | |
| 43 | expect(v.timeline.oldExitMs).toBeGreaterThanOrEqual(5000); | |
| 38 | }); | |
| 39 | ||
| 40 | test("it lingered: the old forge left rotation before it stopped answering", () => { | |
| 41 | expect(v.timeline.oldExitMs).toBeGreaterThanOrEqual(LINGER_S * 1000); | |
| 44 | 42 | expect(v.timeline.oldOutOfRotationMs).toBeLessThan(v.timeline.oldExitMs ?? 0); |
| 45 | }, 60_000); | |
| 43 | }); | |
| 46 | 44 | }); |
test/rehearsal.test.ts +24 −1
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The roll rehearsal (#873): its plain parts. The two short real rolls of |
| 3 | * two forge processes on one data directory are `rehearsal-real.test.ts`. | |
| 3 | * two forge processes on one data directory are `rehearsal-real.test.ts` and `rehearsal-abrupt.test.ts`. | |
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | import { describe, expect, test } from "bun:test"; |
| @@ -260,3 +260,26 @@ describe("rehearse: a forge that dies at boot", () => { | ||
| 260 | 260 | expect(message).toMatch(/Is port \d+ in use\?$/); |
| 261 | 261 | }, 30_000); |
| 262 | 262 | }); |
| 263 | ||
| 264 | describe("rehearse: the old forge's lead before the new one starts", () => { | |
| 265 | // A stand-in forge that is ready at once and leaves on SIGTERM: only the roll's clock is real. | |
| 266 | const stub = 'Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response("ok") }); process.on("SIGTERM", () => process.exit(0));'; | |
| 267 | const rollWith = async (leadMs: number | undefined) => { | |
| 268 | const checkout = mkdtempSync(join(tmpdir(), "forge-rehearsal-lead-")); | |
| 269 | mkdirSync(join(checkout, "src")); | |
| 270 | writeFileSync(join(checkout, "src", "main.ts"), stub); | |
| 271 | try { | |
| 272 | return await rehearse({ ...parseArgs([], checkout), seconds: 4, settleMs: 100, leadMs, probeMs: 50, lingerS: 0, concurrency: 1 }); | |
| 273 | } finally { | |
| 274 | rmSync(checkout, { recursive: true, force: true }); | |
| 275 | } | |
| 276 | }; | |
| 277 | ||
| 278 | test("leadMs is how long the old forge serves alone; unset, a quarter of the roll (#12397)", async () => { | |
| 279 | const [short, unset] = await Promise.all([rollWith(100), rollWith(undefined)]); | |
| 280 | expect(short.timeline.newInRotationMs).toBeGreaterThanOrEqual(100); | |
| 281 | expect(short.timeline.newInRotationMs).toBeLessThan(800); | |
| 282 | // A quarter of 4 s. | |
| 283 | expect(unset.timeline.newInRotationMs).toBeGreaterThanOrEqual(1_000); | |
| 284 | }, 30_000); | |
| 285 | }); |
test/release-handover-drain.test.ts added +33
| @@ -0,0 +1,33 @@ | ||
| 1 | /** | |
| 2 | * `handover.drain` (#2040) lets go of the port first, so new connections | |
| 3 | * are the new forge's, then finishes the rest of the drain, and closes only | |
| 4 | * once the push in flight is answered. Its counterpart, the push across the | |
| 5 | * whole roll, is `release-handover.test.ts`. | |
| 6 | * | |
| 7 | * The drain (`test/helpers/handover-roll.ts`) runs in `beforeAll`; the file | |
| 8 | * budget times the whole file, hooks included (#12397). | |
| 9 | */ | |
| 10 | ||
| 11 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 12 | import { cleanup, drainAcross, PUSH_MS } from "./helpers/handover-roll.ts"; | |
| 13 | ||
| 14 | afterAll(cleanup); | |
| 15 | ||
| 16 | let drain: Awaited<ReturnType<typeof drainAcross>>; | |
| 17 | beforeAll(async () => { | |
| 18 | drain = await drainAcross(); | |
| 19 | }, PUSH_MS * 3); | |
| 20 | ||
| 21 | test("handover.drain lets go of the port before the rest of the drain, and closes only after the push is answered", () => { | |
| 22 | expect(drain.refusedAt).toBeGreaterThan(0); | |
| 23 | expect(drain.restDoneAtRefusal).toBe(0); | |
| 24 | expect(drain.answered.status).toBe(200); | |
| 25 | expect(drain.answered.report).toContain("ok refs/heads/main"); | |
| 26 | expect(drain.closedAt).toBeGreaterThan(0); | |
| 27 | // The push answered in full (200, "ok refs/heads/main") is the proof the | |
| 28 | // close waited for it: the client stamps `answeredAt` after reading the | |
| 29 | // body, which can trail the server's close by a millisecond. | |
| 30 | expect(drain.answeredAt).toBeGreaterThan(0); | |
| 31 | expect(drain.closedAt).toBeGreaterThanOrEqual(drain.restDoneAt); | |
| 32 | expect(drain.head).toBe(drain.after); | |
| 33 | }); |
test/release-handover.test.ts +24 −274
| @@ -1,281 +1,31 @@ | ||
| 1 | 1 | /** |
| 2 | * A roll never cuts an in-flight push (#2040). Two forges on one data | |
| 3 | * directory share the port (SO_REUSEPORT), as during a release swap: a push | |
| 4 | * that started on the old one trickles in for eight seconds while the new | |
| 5 | * one comes up and the old one drains. The old stops accepting at drain | |
| 6 | * start — new connections are the new forge's — and answers the push to the | |
| 7 | * end before it goes. | |
| 2 | * A roll never cuts an in-flight push (#2040): a push that started on the | |
| 3 | * old forge trickles in for eight seconds while the new one comes up on the | |
| 4 | * same port and the old one drains, and it completes, with no connection | |
| 5 | * refused. What `handover.drain` does in what order is | |
| 6 | * `release-handover-drain.test.ts`. | |
| 7 | * | |
| 8 | * The roll (`test/helpers/handover-roll.ts`) runs in `beforeAll`; the file | |
| 9 | * budget times the whole file, hooks included (#12397). | |
| 8 | 10 | */ |
| 9 | 11 | |
| 10 | import { afterAll, expect, test } from "bun:test"; | |
| 11 | import { readFileSync } from "node:fs"; | |
| 12 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 13 | import { tmpdir } from "node:os"; | |
| 14 | import { join } from "node:path"; | |
| 15 | import { createApp } from "../src/app.ts"; | |
| 16 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 17 | import { createToken } from "../src/core/tokens.ts"; | |
| 18 | import { createUser } from "../src/core/users.ts"; | |
| 19 | import { repoDir } from "../src/git/repos.ts"; | |
| 20 | import { drainOnHandover, handover } from "../src/release/swap.ts"; | |
| 21 | import { serve } from "../src/http/server.ts"; | |
| 22 | import { gitClientEnv } from "./helpers/git-client.ts"; | |
| 12 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 13 | import { cleanup, migrates, PUSH_MS, rollAcross } from "./helpers/handover-roll.ts"; | |
| 23 | 14 | |
| 24 | // Long enough to span the new forge's start and the old one's drain start | |
| 25 | // (both well under a second in process) and over a hundred probes; it was 30 s | |
| 26 | // and proved no more (#12381). | |
| 27 | const PUSH_MS = 8_000; | |
| 15 | afterAll(cleanup); | |
| 28 | 16 | |
| 29 | const dirs: string[] = []; | |
| 30 | afterAll(async () => { | |
| 31 | for (const d of dirs) await rm(d, { recursive: true, force: true }); | |
| 32 | }); | |
| 33 | ||
| 34 | // Without it, what sat in the old listener's accept queue is reset when it | |
| 35 | // closes — at drain start now, at the end before; either way the kernel's. | |
| 36 | const migrates = (() => { | |
| 37 | try { | |
| 38 | return readFileSync("/proc/sys/net/ipv4/tcp_migrate_req", "utf8").trim() === "1"; | |
| 39 | } catch { | |
| 40 | return false; | |
| 41 | } | |
| 42 | })(); | |
| 43 | ||
| 44 | async function git(args: string[], cwd: string, stdin?: string): Promise<Uint8Array> { | |
| 45 | const out = Bun.spawnSync(["git", ...args], { | |
| 46 | cwd, | |
| 47 | env: gitClientEnv(), | |
| 48 | ...(stdin === undefined ? {} : { stdin: new TextEncoder().encode(stdin) }), | |
| 49 | }); | |
| 50 | if (out.exitCode !== 0) throw new Error(`git ${args.join(" ")}: ${out.stderr.toString()}`); | |
| 51 | return new Uint8Array(out.stdout); | |
| 52 | } | |
| 53 | ||
| 54 | const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes).trim(); | |
| 55 | ||
| 56 | function pktLine(line: string): Uint8Array { | |
| 57 | const body = new TextEncoder().encode(line); | |
| 58 | const head = (body.length + 4).toString(16).padStart(4, "0"); | |
| 59 | return new Uint8Array([...new TextEncoder().encode(head), ...body]); | |
| 60 | } | |
| 61 | ||
| 62 | /** `bytes` sent in pieces spread over `ms`, so the request stays in flight that long. */ | |
| 63 | function trickle(bytes: Uint8Array, ms: number): ReadableStream<Uint8Array> { | |
| 64 | const pieces = 30; | |
| 65 | const size = Math.ceil(bytes.length / pieces); | |
| 66 | let i = 0; | |
| 67 | return new ReadableStream({ | |
| 68 | async pull(controller) { | |
| 69 | if (i > 0) await Bun.sleep(ms / pieces); | |
| 70 | const piece = bytes.subarray(i * size, (i + 1) * size); | |
| 71 | i++; | |
| 72 | if (piece.length > 0) controller.enqueue(piece); | |
| 73 | if (i >= pieces) controller.close(); | |
| 74 | }, | |
| 75 | }); | |
| 76 | } | |
| 77 | ||
| 78 | /** | |
| 79 | * The old forge on a port of its own, alice's repository, and `push(ms)`: a | |
| 80 | * receive-pack of one new commit on main, trickled in over `ms`. | |
| 81 | */ | |
| 82 | async function forgeWithPush() { | |
| 83 | const dataDir = await mkdtemp(join(tmpdir(), "forge-handover-")); | |
| 84 | const work = await mkdtemp(join(tmpdir(), "forge-handover-work-")); | |
| 85 | dirs.push(dataDir, work); | |
| 86 | // The old forge picks the port; the release binds the one it got. | |
| 87 | const oldApp = createApp({ dataDir, port: 0, selfSwap: true }); | |
| 88 | const oldServers = await serve(oldApp); | |
| 89 | const port = oldServers.port; | |
| 90 | const made = await createUser(oldApp.db, { | |
| 91 | origin: "seed", | |
| 92 | trusted: true, | |
| 93 | username: "alice", | |
| 94 | email: "[email protected]", | |
| 95 | password: "a-good-passphrase", | |
| 96 | adminIfFirst: true, | |
| 97 | }); | |
| 98 | if (!made.ok) throw new Error("user"); | |
| 99 | const alice = made.value; | |
| 100 | const repo = await createRepo(oldApp, alice, { name: "garden", visibility: "public" }); | |
| 101 | if (!repo.ok) throw new Error("repo"); | |
| 102 | await commitToBranch(oldApp, alice, repo.value, { | |
| 103 | message: "seed", | |
| 104 | files: [{ path: "a.txt", content: "a" }], | |
| 105 | }); | |
| 106 | const { token } = createToken(oldApp.db, alice.id, "push", "rw"); | |
| 107 | ||
| 108 | // The push: one commit on main, as git sends it to receive-pack. | |
| 109 | const bare = repoDir(oldApp.reposRoot, "alice", "garden"); | |
| 110 | const clone = join(work, "garden"); | |
| 111 | await git(["clone", "-q", bare, clone], work); | |
| 112 | await Bun.write(join(clone, "b.txt"), "b"); | |
| 113 | await git(["add", "b.txt"], clone); | |
| 114 | await git(["commit", "-q", "-m", "b"], clone); | |
| 115 | const before = text(await git(["rev-parse", "origin/main"], clone)); | |
| 116 | const after = text(await git(["rev-parse", "HEAD"], clone)); | |
| 117 | const pack = await git( | |
| 118 | ["pack-objects", "--stdout", "--revs", "--quiet"], | |
| 119 | clone, | |
| 120 | `${after}\n^${before}\n`, | |
| 121 | ); | |
| 122 | const command = pktLine(`${before} ${after} refs/heads/main\0report-status\n`); | |
| 123 | const flush = new TextEncoder().encode("0000"); | |
| 124 | const body = new Uint8Array(command.length + flush.length + pack.length); | |
| 125 | body.set(command, 0); | |
| 126 | body.set(flush, command.length); | |
| 127 | body.set(pack, command.length + flush.length); | |
| 128 | ||
| 129 | const url = `http://127.0.0.1:${port}`; | |
| 130 | const push = (ms: number) => | |
| 131 | fetch(`${url}/alice/garden.git/git-receive-pack`, { | |
| 132 | method: "POST", | |
| 133 | headers: { | |
| 134 | authorization: `Basic ${btoa(`alice:${token}`)}`, | |
| 135 | "content-type": "application/x-git-receive-pack-request", | |
| 136 | }, | |
| 137 | body: trickle(body, ms), | |
| 138 | }); | |
| 139 | return { oldApp, oldServers, port, url, bare, after, push }; | |
| 140 | } | |
| 141 | ||
| 142 | test("a push in flight on the old forge completes across the roll, and no connection is refused", async () => { | |
| 143 | const { oldApp, oldServers, port, url, bare, after, push: pushOver } = await forgeWithPush(); | |
| 144 | const dataDir = oldApp.config.dataDir; | |
| 145 | const started = Date.now(); | |
| 146 | const push = pushOver(PUSH_MS); | |
| 147 | // The push is the old forge's before the release exists. | |
| 148 | await Bun.sleep(500); | |
| 149 | ||
| 150 | const newApp = createApp({ dataDir, port, selfSwap: true }); | |
| 151 | const newServers = await serve(newApp); | |
| 152 | ||
| 153 | let looping = true; | |
| 154 | let asked = 0; | |
| 155 | let refused = 0; | |
| 156 | const client = (async () => { | |
| 157 | while (looping) { | |
| 158 | asked++; | |
| 159 | try { | |
| 160 | const res = await fetch(`${url}/api/v1/health`, { | |
| 161 | headers: { connection: "close" }, | |
| 162 | signal: AbortSignal.timeout(3_000), | |
| 163 | }); | |
| 164 | await res.arrayBuffer(); | |
| 165 | } catch { | |
| 166 | refused++; | |
| 167 | } | |
| 168 | await Bun.sleep(20); | |
| 169 | } | |
| 170 | })(); | |
| 171 | ||
| 172 | // Drain start: the old forge stops listening, keeping what it holds. | |
| 173 | const drained = oldServers.stopAccepting(PUSH_MS * 2); | |
| 174 | ||
| 175 | const answer = await push; | |
| 176 | const report = await answer.text(); | |
| 177 | expect(answer.status).toBe(200); | |
| 178 | expect(report).toContain("unpack ok"); | |
| 179 | expect(report).toContain("ok refs/heads/main"); | |
| 180 | expect(Date.now() - started).toBeGreaterThanOrEqual(PUSH_MS * 0.9); | |
| 181 | await drained; | |
| 182 | ||
| 183 | looping = false; | |
| 184 | await client; | |
| 185 | expect(asked).toBeGreaterThan(100); | |
| 186 | if (migrates) expect(refused).toBe(0); | |
| 187 | expect(text(await git(["rev-parse", "refs/heads/main"], bare))).toBe(after); | |
| 188 | ||
| 189 | // Nobody else listens: the old forge let go of the port at drain start. | |
| 190 | // `stop` closes the listener asynchronously, so ask until it is gone. | |
| 191 | newServers.stop(); | |
| 192 | let orphan = "answered"; | |
| 193 | for (const until = Date.now() + 10_000; Date.now() < until; ) { | |
| 194 | orphan = await fetch(`${url}/api/v1/health`, { | |
| 195 | headers: { connection: "close" }, | |
| 196 | signal: AbortSignal.timeout(2_000), | |
| 197 | }).then( | |
| 198 | async (res) => { | |
| 199 | await res.arrayBuffer(); | |
| 200 | return "answered"; | |
| 201 | }, | |
| 202 | () => "refused", | |
| 203 | ); | |
| 204 | if (orphan === "refused") break; | |
| 205 | await Bun.sleep(100); | |
| 206 | } | |
| 207 | expect(orphan).toBe("refused"); | |
| 208 | ||
| 209 | oldServers.stop(); | |
| 210 | newApp.close(); | |
| 211 | oldApp.close(); | |
| 17 | let across: Awaited<ReturnType<typeof rollAcross>>; | |
| 18 | beforeAll(async () => { | |
| 19 | across = await rollAcross(); | |
| 212 | 20 | }, PUSH_MS * 3); |
| 213 | 21 | |
| 214 | test("handover.drain lets go of the port before the rest of the drain, and closes only after the push is answered", async () => { | |
| 215 | const { oldApp, oldServers, url, bare, after, push: pushOver } = await forgeWithPush(); | |
| 216 | const HELD_MS = 4_000; | |
| 217 | const REST_MS = 2_000; | |
| 218 | const push = pushOver(HELD_MS).then(async (res) => { | |
| 219 | const report = await res.text(); | |
| 220 | answeredAt = Date.now(); | |
| 221 | return { status: res.status, report }; | |
| 222 | }); | |
| 223 | let answeredAt = 0; | |
| 224 | let restDoneAt = 0; | |
| 225 | let closedAt = 0; | |
| 226 | // Wired as `main.ts` wires it; `rest` stands in for the shutdown's other | |
| 227 | // steps (crew, apps, checks), `close` for its last. | |
| 228 | const handOver = drainOnHandover( | |
| 229 | oldServers, | |
| 230 | async () => { | |
| 231 | await Bun.sleep(REST_MS); | |
| 232 | restDoneAt = Date.now(); | |
| 233 | }, | |
| 234 | () => { | |
| 235 | closedAt = Date.now(); | |
| 236 | oldServers.stop(); | |
| 237 | }, | |
| 238 | ); | |
| 239 | handover.drain = () => void handOver(); | |
| 240 | try { | |
| 241 | await Bun.sleep(500); | |
| 242 | handover.drain(); | |
| 243 | ||
| 244 | // Nobody else listens here, so a refused connection is this listener gone. | |
| 245 | let refusedAt = 0; | |
| 246 | while (restDoneAt === 0) { | |
| 247 | const gone = await fetch(`${url}/api/v1/health`, { | |
| 248 | headers: { connection: "close" }, | |
| 249 | signal: AbortSignal.timeout(1_000), | |
| 250 | }).then( | |
| 251 | async (res) => { | |
| 252 | await res.arrayBuffer(); | |
| 253 | return false; | |
| 254 | }, | |
| 255 | () => true, | |
| 256 | ); | |
| 257 | if (gone) { | |
| 258 | refusedAt = Date.now(); | |
| 259 | break; | |
| 260 | } | |
| 261 | await Bun.sleep(50); | |
| 262 | } | |
| 263 | expect(refusedAt).toBeGreaterThan(0); | |
| 264 | expect(restDoneAt).toBe(0); | |
| 265 | ||
| 266 | const answered = await push; | |
| 267 | expect(answered.status).toBe(200); | |
| 268 | expect(answered.report).toContain("ok refs/heads/main"); | |
| 269 | while (closedAt === 0) await Bun.sleep(50); | |
| 270 | // The push answered in full (200, "ok refs/heads/main") is the proof the | |
| 271 | // close waited for it: the client stamps `answeredAt` after reading the | |
| 272 | // body, which can trail the server's close by a millisecond. | |
| 273 | expect(answeredAt).toBeGreaterThan(0); | |
| 274 | expect(closedAt).toBeGreaterThanOrEqual(restDoneAt); | |
| 275 | expect(text(await git(["rev-parse", "refs/heads/main"], bare))).toBe(after); | |
| 276 | } finally { | |
| 277 | delete handover.drain; | |
| 278 | oldServers.stop(); | |
| 279 | oldApp.close(); | |
| 280 | } | |
| 281 | }, 30_000); | |
| 22 | test("a push in flight on the old forge completes across the roll, and no connection is refused", () => { | |
| 23 | expect(across.status).toBe(200); | |
| 24 | expect(across.report).toContain("unpack ok"); | |
| 25 | expect(across.report).toContain("ok refs/heads/main"); | |
| 26 | expect(across.tookMs).toBeGreaterThanOrEqual(PUSH_MS * 0.9); | |
| 27 | expect(across.asked).toBeGreaterThan(100); | |
| 28 | if (migrates) expect(across.refused).toBe(0); | |
| 29 | expect(across.head).toBe(across.after); | |
| 30 | expect(across.orphan).toBe("refused"); | |
| 31 | }); |
test/test-budget.test.ts +11
| @@ -64,6 +64,17 @@ describe("breaches", () => { | ||
| 64 | 64 | expect(breaches([{ file: "test/a.test.ts", seconds: 99 }], tests, new Set(["test/a.test.ts"]))).toEqual([]); |
| 65 | 65 | }); |
| 66 | 66 | |
| 67 | // A file whose work runs in beforeAll (boot-beside-peer, the real rolls, the drill, | |
| 68 | // the handover: #12397) has tests of about 0 s; the file's time is its whole run, | |
| 69 | // hooks included (`scripts/test-shards.ts` times the lone file's process), so the | |
| 70 | // hook's cost is still held to the file budget, and a hook that throws fails the file. | |
| 71 | test("hook time counts: tests of 0 s in a file over its budget is still a breach", () => { | |
| 72 | const hooked = [{ file: "test/h.test.ts", test: "judges what the hook left", seconds: 0.001 }]; | |
| 73 | expect(breaches([{ file: "test/h.test.ts", seconds: FILE_BUDGET_SECONDS + 0.5 }], hooked, new Set())).toEqual([ | |
| 74 | { file: "test/h.test.ts", seconds: FILE_BUDGET_SECONDS + 0.5, budget: FILE_BUDGET_SECONDS }, | |
| 75 | ]); | |
| 76 | }); | |
| 77 | ||
| 67 | 78 | test("exactly at the budget is within it", () => { |
| 68 | 79 | expect(breaches([{ file: "test/a.test.ts", seconds: FILE_BUDGET_SECONDS }], [], new Set())).toEqual([]); |
| 69 | 80 | }); |
new above
offline
drill-400mb is still on SLOW, so box 3 is not met. At 17 s alone the file is too close to the 20 s budget. Run the 400 MB and 100 MB drills as separate files, or drop the 400 MB one to a nightly path.