Public: anyone can read this; the crew can change it.
Fast test/chaos.test.ts: the seeded run of every fault under 1 s #1443
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1439-fast-test-chaos-test-ts-the-seeded-run-o → main 3 commits fork 3 threads · 3 unresolved
4 files
changed +104
−28
src/chaos/invariants.ts +3
| @@ -15,6 +15,8 @@ import { SEALED_VALUE, type World } from "./world.ts"; | ||
| 15 | 15 | |
| 16 | 16 | export interface Invariant { |
| 17 | 17 | readonly name: string; |
| 18 | /** Costly: checked once at the end of a night, not after every fault. */ | |
| 19 | readonly final?: true; | |
| 18 | 20 | check(world: World): Promise<string[]>; |
| 19 | 21 | } |
| 20 | 22 | |
| @@ -71,6 +73,7 @@ const pullsKeepHeads: Invariant = { | ||
| 71 | 73 | /** A snapshot taken now restores into an empty directory with the same runs. */ |
| 72 | 74 | const snapshotRestores: Invariant = { |
| 73 | 75 | name: "a snapshot taken after restores", |
| 76 | final: true, | |
| 74 | 77 | async check(world) { |
| 75 | 78 | const root = mkdtempSync(join(tmpdir(), "forge-chaos-snap-")); |
| 76 | 79 | try { |
src/chaos/run.ts +21 −9
| @@ -12,9 +12,9 @@ import { mkdtempSync, rmSync } from "node:fs"; | ||
| 12 | 12 | import { tmpdir } from "node:os"; |
| 13 | 13 | import { join } from "node:path"; |
| 14 | 14 | import { FAULTS, type Fault } from "./faults.ts"; |
| 15 | import { INVARIANTS } from "./invariants.ts"; | |
| 15 | import { INVARIANTS, type Invariant } from "./invariants.ts"; | |
| 16 | 16 | import { fileReport, type Finding } from "./report.ts"; |
| 17 | import { bootWorld } from "./world.ts"; | |
| 17 | import { bootWorld, type World } from "./world.ts"; | |
| 18 | 18 | |
| 19 | 19 | /** mulberry32: a seed in, the same sequence out. */ |
| 20 | 20 | export function rng(seed: number): () => number { |
| @@ -35,18 +35,33 @@ export interface Night { | ||
| 35 | 35 | readonly log: readonly string[]; |
| 36 | 36 | } |
| 37 | 37 | |
| 38 | export interface Options { | |
| 39 | /** A world already booted (the caller closes it); default is a fresh one per night. */ | |
| 40 | readonly world?: World; | |
| 41 | readonly invariants?: readonly Invariant[]; | |
| 42 | } | |
| 43 | ||
| 38 | 44 | export async function chaos( |
| 39 | 45 | seed: number, |
| 40 | 46 | rounds = FAULTS.length, |
| 41 | 47 | faults: readonly Fault[] = FAULTS, |
| 48 | options: Options = {}, | |
| 42 | 49 | ): Promise<Night> { |
| 50 | const invariants = options.invariants ?? INVARIANTS; | |
| 43 | 51 | const rand = rng(seed); |
| 44 | 52 | const root = mkdtempSync(join(tmpdir(), "forge-chaos-")); |
| 45 | 53 | const log: string[] = []; |
| 46 | 54 | const findings: Finding[] = []; |
| 47 | 55 | const order: string[] = []; |
| 48 | 56 | try { |
| 49 | const world = await bootWorld(join(root, "data")); | |
| 57 | const world = options.world ?? (await bootWorld(join(root, "data"))); | |
| 58 | const assertAll = async (which: readonly Invariant[], fault: string): Promise<void> => { | |
| 59 | for (const invariant of which) { | |
| 60 | const broken = await invariant.check(world).catch((e) => [`check threw: ${String(e)}`]); | |
| 61 | log.push(` ${broken.length === 0 ? "holds" : "BROKEN"}: ${invariant.name}`); | |
| 62 | if (broken.length > 0) findings.push({ fault, broken: broken.map((b) => `${invariant.name}: ${b}`) }); | |
| 63 | } | |
| 64 | }; | |
| 50 | 65 | try { |
| 51 | 66 | for (let i = 0; i < rounds; i++) { |
| 52 | 67 | const fault = faults[Math.floor(rand() * faults.length)]!; |
| @@ -59,14 +74,11 @@ export async function chaos( | ||
| 59 | 74 | log.push(` failed: ${String(error)}`); |
| 60 | 75 | continue; |
| 61 | 76 | } |
| 62 | for (const invariant of INVARIANTS) { | |
| 63 | const broken = await invariant.check(world).catch((e) => [`check threw: ${String(e)}`]); | |
| 64 | log.push(` ${broken.length === 0 ? "holds" : "BROKEN"}: ${invariant.name}`); | |
| 65 | if (broken.length > 0) findings.push({ fault: fault.name, broken: broken.map((b) => `${invariant.name}: ${b}`) }); | |
| 66 | } | |
| 77 | await assertAll(invariants.filter((i) => i.final !== true), fault.name); | |
Final invariants are skipped when order is empty (every fault threw before being recorded). Say so in a comment, or run them regardless. |
||
| 67 | 78 | } |
| 79 | if (order.length > 0) await assertAll(invariants.filter((i) => i.final === true), order[order.length - 1]!); | |
| 68 | 80 | } finally { |
| 69 | world.app.close(); | |
| 81 | if (options.world === undefined) world.app.close(); | |
| 70 | 82 | } |
| 71 | 83 | } finally { |
| 72 | 84 | rmSync(root, { recursive: true, force: true }); |
src/chaos/world.ts −1
| @@ -29,7 +29,6 @@ export async function bootWorld(dataDir: string): Promise<World> { | ||
| 29 | 29 | trusted: true, |
| 30 | 30 | username: "chaos", |
| 31 | 31 | email: "[email protected]", |
| 32 | password: "a-good-passphrase", | |
| 33 | 32 | adminIfFirst: true, |
| 34 | 33 | }); |
| 35 | 34 | if (!made.ok) throw new Error(made.error); |
test/chaos.test.ts +80 −18
| @@ -2,18 +2,40 @@ | ||
| 2 | 2 | * The small chaos in the gate: every fault once, on a side instance in a |
| 3 | 3 | * temporary directory, with every invariant asserted after each. |
| 4 | 4 | */ |
| 5 | import { describe, expect, test } from "bun:test"; | |
| 6 | import { FAULTS, slowProxy } from "../src/chaos/faults.ts"; | |
| 7 | import { INVARIANTS } from "../src/chaos/invariants.ts"; | |
| 5 | import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; | |
| 6 | import { FAULTS, slowProxy, type Fault } from "../src/chaos/faults.ts"; | |
| 7 | import { INVARIANTS, type Invariant } from "../src/chaos/invariants.ts"; | |
| 8 | 8 | import { issueFor, quietLine } from "../src/chaos/report.ts"; |
| 9 | 9 | import { chaos, rng } from "../src/chaos/run.ts"; |
| 10 | import { bootWorld, SEALED_VALUE } from "../src/chaos/world.ts"; | |
| 10 | import { bootWorld, SEALED_VALUE, type World } from "../src/chaos/world.ts"; | |
| 11 | 11 | import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 12 | 12 | import { tmpdir } from "node:os"; |
| 13 | 13 | import { join } from "node:path"; |
| 14 | 14 | |
| 15 | 15 | const SEED = Number(process.env["CHAOS_SEED"] ?? 1244); |
| 16 | 16 | |
| 17 | let root: string; | |
| 18 | let world: World; | |
| 19 | ||
| 20 | // The per-fault invariants; the snapshot one is costly and runs once per night. | |
| 21 | const quick = INVARIANTS.filter((i) => i.final !== true); | |
| 22 | ||
| 23 | // One world for the file. Faults share it, so after every test the cheap | |
| 24 | // invariants are asserted: a test that leaves damage behind fails itself, | |
| 25 | // not whichever test runs next. | |
| 26 | afterEach(async () => { | |
| 27 | const broken = (await Promise.all(quick.map((i) => i.check(world)))).flat(); | |
| 28 | expect(broken).toEqual([]); | |
| 29 | }); | |
| 30 | beforeAll(async () => { | |
| 31 | root = mkdtempSync(join(tmpdir(), "forge-chaos-")); | |
| 32 | world = await bootWorld(join(root, "data")); | |
| 33 | }); | |
| 34 | afterAll(() => { | |
| 35 | world.app.close(); | |
| 36 | rmSync(root, { recursive: true, force: true }); | |
| 37 | }); | |
| 38 | ||
| 17 | 39 | describe("chaos", () => { |
| 18 | 40 | test("the rng repeats from a seed", () => { |
| 19 | 41 | const a = rng(7); |
| @@ -21,18 +43,58 @@ describe("chaos", () => { | ||
| 21 | 43 | expect([a(), a(), a()]).toEqual([b(), b(), b()]); |
| 22 | 44 | }); |
| 23 | 45 | |
| 46 | const stub = (name: string): Fault => ({ name, inject: async () => {} }); | |
| 47 | const stubs = [stub("a"), stub("b"), stub("c")]; | |
| 48 | ||
| 49 | test("the same seed picks the same fault order", async () => { | |
| 50 | const a = await chaos(SEED, 5, stubs, { world, invariants: [] }); | |
outdated
Stubs with invariants: [] mean this no longer exercises the faults seeded together; it duplicates the rng test. Keep one real multi-fault night, or drop the test. |
||
| 51 | const b = await chaos(SEED, 5, stubs, { world, invariants: [] }); | |
| 52 | expect(a.faults).toHaveLength(5); | |
| 53 | expect(a.faults).toEqual(b.faults); | |
| 54 | }); | |
| 55 | ||
| 56 | test("a final invariant is checked once after the last fault, the others every round", async () => { | |
| 57 | const events: string[] = []; | |
| 58 | const spy = (name: string, final: boolean): Invariant => ({ | |
| 59 | name, | |
| 60 | ...(final ? { final: true as const } : {}), | |
| 61 | check: async () => { | |
| 62 | events.push(name); | |
| 63 | return []; | |
| 64 | }, | |
| 65 | }); | |
| 66 | const faults = stubs.map((f) => ({ | |
| 67 | name: f.name, | |
| 68 | inject: async () => { | |
| 69 | events.push(`fault ${f.name}`); | |
| 70 | }, | |
| 71 | })); | |
| 72 | const night = await chaos(SEED, 3, faults, { world, invariants: [spy("each", false), spy("last", true)] }); | |
| 73 | expect(events.filter((e) => e === "last")).toHaveLength(1); | |
| 74 | expect(events.filter((e) => e === "each")).toHaveLength(3); | |
| 75 | expect(events.at(-1)).toBe("last"); | |
| 76 | expect(events.lastIndexOf("each")).toBeLessThan(events.indexOf("last")); | |
| 77 | expect(night.findings).toEqual([]); | |
| 78 | }); | |
| 79 | ||
| 80 | test("a broken final invariant is attributed to the last fault of the night", async () => { | |
| 81 | const broken: Invariant = { name: "snap", final: true, check: async () => ["gone"] }; | |
| 82 | const night = await chaos(SEED, 4, stubs, { world, invariants: [broken] }); | |
| 83 | expect(night.findings).toEqual([{ fault: night.faults[3]!, broken: ["snap: gone"] }]); | |
| 84 | }); | |
| 85 | ||
| 24 | 86 | for (const fault of FAULTS) { |
| 25 | 87 | test(`after "${fault.name}" every invariant holds (seed ${SEED})`, async () => { |
| 26 | const night = await chaos(SEED, 1, [fault]); | |
| 88 | const night = await chaos(SEED, 1, [fault], { world, invariants: quick }); | |
| 27 | 89 | expect(night.findings, `reproduce: CHAOS_SEED=${SEED}\n${night.log.join("\n")}`).toEqual([]); |
| 28 | }, 60_000); | |
| 90 | }); | |
| 29 | 91 | } |
| 30 | 92 | |
| 31 | 93 | test("all faults in a seeded order, one after another", async () => { |
| 32 | const night = await chaos(SEED, 6); | |
| 33 | expect(night.findings, night.log.join("\n")).toEqual([]); | |
| 34 | expect(night.faults).toEqual((await chaos(SEED, 6)).faults); | |
| 35 | }, 120_000); | |
| 94 | const night = await chaos(SEED, FAULTS.length, FAULTS, { world }); | |
| 95 | expect(night.faults).toHaveLength(FAULTS.length); | |
| 96 | expect(night.findings, `reproduce: CHAOS_SEED=${SEED}\n${night.log.join("\n")}`).toEqual([]); | |
| 97 | }); | |
| 36 | 98 | |
| 37 | 99 | test("the slow proxy delays but delivers", async () => { |
| 38 | 100 | const backend = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("ok") }); |
| @@ -49,19 +111,19 @@ describe("chaos", () => { | ||
| 49 | 111 | }); |
| 50 | 112 | |
| 51 | 113 | test("an invariant catches what it guards", async () => { |
| 52 | const root = mkdtempSync(join(tmpdir(), "forge-chaos-neg-")); | |
| 114 | const dir = mkdtempSync(join(tmpdir(), "forge-chaos-neg-")); | |
| 115 | const side = await bootWorld(join(dir, "data")); | |
| 53 | 116 | try { |
| 54 | const world = await bootWorld(join(root, "data")); | |
| 55 | world.app.db.query("DELETE FROM run WHERE id = ?1").run([...world.runIds][1]!); | |
| 56 | writeFileSync(join(world.dataDir, "leak.txt"), SEALED_VALUE); | |
| 57 | const broken = (await Promise.all(INVARIANTS.map((i) => i.check(world)))).flat(); | |
| 117 | side.app.db.query("DELETE FROM run WHERE id = ?1").run([...side.runIds][1]!); | |
| 118 | writeFileSync(join(side.dataDir, "leak.txt"), SEALED_VALUE); | |
| 119 | const broken = (await Promise.all(INVARIANTS.map((i) => i.check(side)))).flat(); | |
| 58 | 120 | expect(broken.some((b) => b.includes("vanished"))).toBe(true); |
| 59 | 121 | expect(broken.some((b) => b.includes("sealed value is readable"))).toBe(true); |
| 60 | world.app.close(); | |
| 61 | 122 | } finally { |
| 62 | rmSync(root, { recursive: true, force: true }); | |
| 123 | side.app.close(); | |
| 124 | rmSync(dir, { recursive: true, force: true }); | |
| 63 | 125 | } |
| 64 | }, 60_000); | |
| 126 | }); | |
| 65 | 127 | |
| 66 | 128 | test("a broken night makes an issue with fault, seed and log; a quiet one a line", () => { |
| 67 | 129 | const night = { |
new above
offline
The root = "" sentinel as a flag for 'nothing to clean' is awkward; use an undefined/nullable root or split the owned-world path out.