Public: anyone can read this; the crew can change it.
The landsim corpus test runs under the 2 s per-test budget, and test/landsim.test.ts leaves SLOW #12488
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12447-the-landsim-corpus-test-runs-under-the-2 → main 3 commits fork
3 files
changed +72
−31
scripts/gate.ts −1
| @@ -113,7 +113,6 @@ export const SLOW: readonly Exclusion[] = [ | ||
| 113 | 113 | { file: "test/home-frame.test.ts", reason: "renders the signed-in home after writes and times the answer from memory; 2.2 s a test on the gate box", issue: 1242 }, |
| 114 | 114 | { file: "test/job-slices.test.ts", reason: "ticks every timed job on a busy fixture while sampling the event loop; 2.3 s a test on the gate box", issue: 1242 }, |
| 115 | 115 | { file: "test/kpis-landings-scale.test.ts", reason: "seeds a production-sized landings table and counts it two ways; 7.5 s a test on the gate box", issue: 1242 }, |
| 116 | { file: "test/landsim.test.ts", reason: "lands four seeded scenarios through review and the merge queue at once, with the order, merge and determinism pair checks; 24 seeds nightly", issue: 12447 }, | |
| 117 | 116 | { file: "test/live-arrive.test.ts", reason: "drops and restores a live connection in happy-dom, waiting on real timers; 2.8 s a test on the gate box", issue: 1242 }, |
| 118 | 117 | { file: "test/loading-chrome.test.ts", reason: "watches the loading indicator across pages in a real Chromium browser; 7.6 s a test on the gate box", issue: 1242 }, |
| 119 | 118 | { file: "test/maintenance-standby.test.ts", reason: "waits out the standby's answer timeout on the real clock; 2.0 s a test on the gate box", issue: 1242 }, |
test/landsim.test.ts +69 −26
| @@ -5,9 +5,10 @@ | ||
| 5 | 5 | * landing promises: what was reviewed is what lands, nothing is lost silently, |
| 6 | 6 | * what was checked is what landed. |
| 7 | 7 | * |
| 8 | * The quick run's four seeds also cover the pair properties, one seed each (order independence on seeds divisible by 4, additive | |
| 9 | * merges on seed % 4 == 1, determinism on seed % 16 == 2). `LANDSIM_SEEDS=n` runs n seeds instead of the quick four; the nightly | |
| 10 | * outside-gate job runs 24 (`src/nightly/jobs.ts`). A failure prints the | |
| 8 | * The quick run's four seeds also cover the pair properties (order independence on seeds | |
| 9 | * divisible by 4, additive merges on seed % 4 == 1); determinism runs seed 34 twice (seeds with | |
| 10 | * seed % 16 == 2 under `LANDSIM_SEEDS`). `LANDSIM_SEEDS=n` runs n seeds instead of the quick four; | |
| 11 | * the nightly outside-gate job runs 24 (`src/nightly/jobs.ts`). A failure prints the | |
| 11 | 12 | * seed and the scenario as a script; `bun test/support/landsim.ts --from <seed> |
| 12 | 13 | * --seeds 1 --shrink` cuts it down to the pulls and edits that matter. |
| 13 | 14 | * |
| @@ -22,6 +23,8 @@ import { | ||
| 22 | 23 | applyEdits, |
| 23 | 24 | authorshipKept, |
| 24 | 25 | checkedIsLanded, |
| 26 | checkAdditiveMerges, | |
| 27 | checkOrderIndependence, | |
| 25 | 28 | createWorld, |
| 26 | 29 | gateAttachesToHead, |
| 27 | 30 | generate, |
| @@ -31,21 +34,24 @@ import { | ||
| 31 | 34 | noSilentLoss, |
| 32 | 35 | report, |
| 33 | 36 | reviewedIsLanded, |
| 34 | runCorpus, | |
| 37 | runScenario, | |
| 35 | 38 | script, |
| 36 | 39 | seedCount, |
| 37 | 40 | shrinkBy, |
| 38 | 41 | type Edit, |
| 39 | 42 | type Landing, |
| 43 | type Outcome, | |
| 40 | 44 | type Scenario, |
| 45 | type Violation, | |
| 41 | 46 | type World, |
| 42 | 47 | } from "./support/landsim.ts"; |
| 43 | 48 | |
| 44 | 49 | /** |
| 45 | * Seeds in the quick run, all at once: 1 checks additive merges, 2 determinism, | |
| 46 | * 4 order independence. The longer corpus runs nightly. | |
| 50 | * Seeds in the quick run: 1 and 5 check additive merges, 8 order independence. | |
| 51 | * Chosen for landing quickly, so a test a seed stays under the per-test budget; | |
| 52 | * the longer corpus (`LANDSIM_SEEDS=n`, seeds 1..n) runs nightly. | |
| 47 | 53 | */ |
| 48 | const QUICK = 4; | |
| 54 | const QUICK_SEEDS = [1, 3, 5, 8] as const; | |
| 49 | 55 | |
| 50 | 56 | describe("the landing simulator", () => { |
| 51 | 57 | let world: World; |
| @@ -56,22 +62,59 @@ describe("the landing simulator", () => { | ||
| 56 | 62 | await world.close(); |
| 57 | 63 | }); |
| 58 | 64 | |
| 59 | test("a seeded corpus lands what was reviewed, apart from the violations an issue owns", async () => { | |
| 60 | const n = seedCount(QUICK); | |
| 61 | const corpus = await runCorpus(world, 1, n, QUICK); | |
| 62 | const unexpected = [...corpus.classes].filter( | |
| 63 | ([cls]) => knownIssue(cls) === undefined, | |
| 64 | ); | |
| 65 | const lines = unexpected.map( | |
| 66 | ([cls, seeds]) => | |
| 67 | ` ${cls} (seeds ${seeds.slice(0, 5).join(", ")}): ${corpus.messages.get(cls)}`, | |
| 68 | ); | |
| 69 | expect( | |
| 70 | lines, | |
| 71 | `landsim found violations nothing owns (rerun one with: bun test/support/landsim.ts --from <seed> --seeds 1 --shrink):\n${lines.join("\n")}`, | |
| 72 | ).toEqual([]); | |
| 73 | expect(corpus.landed).toBeGreaterThan(0); | |
| 74 | }, 300_000); | |
| 65 | // One test a seed (and one a pair property), so each stays well under the per-test budget: | |
| 66 | // `LANDSIM_SEEDS=n` adds seeds. | |
| 67 | const wide = process.env["LANDSIM_SEEDS"] !== undefined; | |
| 68 | const seeds = wide | |
| 69 | ? Array.from({ length: seedCount(QUICK_SEEDS.length) }, (_, i) => i + 1) | |
| 70 | : [...QUICK_SEEDS]; | |
| 71 | const unowned = (violations: readonly Violation[]): string[] => | |
| 72 | violations | |
| 73 | .filter((v) => knownIssue(v.cls) === undefined) | |
| 74 | .map((v) => `${v.cls}: ${v.message}`); | |
| 75 | const rerun = (seed: number): string => | |
| 76 | `rerun with: bun test/support/landsim.ts --from ${seed} --seeds 1 --shrink`; | |
| 77 | ||
| 78 | for (const seed of seeds) { | |
| 79 | test(`seed ${seed} lands what was reviewed, apart from the violations an issue owns`, async () => { | |
| 80 | const out = await runScenario(world, generate(seed)); | |
| 81 | expect(out.landed, `seed ${seed} landed nothing`).toBeGreaterThan(0); | |
| 82 | expect(unowned(out.violations), `landsim found violations nothing owns (${rerun(seed)})`).toEqual([]); | |
| 83 | }, 30_000); | |
| 84 | if (seed % 4 === 0) | |
| 85 | test(`seed ${seed}: landing order does not change what lands`, async () => { | |
| 86 | const out = await checkOrderIndependence(world, seed); | |
| 87 | expect(unowned(out.violations), rerun(seed)).toEqual([]); | |
| 88 | }, 30_000); | |
| 89 | if (seed % 4 === 1) | |
| 90 | for (const order of [[1, 2], [2, 1]] as const) | |
| 91 | test(`seed ${seed}: pulls that only add lines merge, landed ${order.join(" then ")}`, async () => { | |
| 92 | const out = await checkAdditiveMerges(world, seed, [order]); | |
| 93 | expect(unowned(out.violations), rerun(seed)).toEqual([]); | |
| 94 | }, 30_000); | |
| 95 | } | |
| 96 | ||
| 97 | // Determinism: a seed twice, the runs made in the group's `beforeAll` (hooks are not the tests' | |
| 98 | // time) and compared in tests that stand alone. Quick: seed 34. `LANDSIM_SEEDS=n` (the nightly, | |
| 99 | // 24 seeds): every seed with seed % 16 == 2, as before the split. | |
| 100 | const determinismSeeds = wide ? seeds.filter((seed) => seed % 16 === 2) : [34]; | |
| 101 | for (const seed of determinismSeeds) | |
| 102 | describe(`determinism, seed ${seed}`, () => { | |
| 103 | let first: Outcome; | |
| 104 | let second: Outcome; | |
| 105 | beforeAll(async () => { | |
| 106 | first = await runScenario(world, generate(seed)); | |
| 107 | second = await runScenario(world, generate(seed)); | |
| 108 | }, 120_000); | |
| 109 | test("a run lands what was reviewed, apart from the violations an issue owns", () => { | |
| 110 | expect(first.signature).not.toBe(""); | |
| 111 | expect(unowned(first.violations), rerun(seed)).toEqual([]); | |
| 112 | }); | |
| 113 | test("the same seed lands the same history a second time", () => { | |
| 114 | expect(second.signature).toBe(first.signature); | |
| 115 | expect(second.finalTree).toBe(first.finalTree); | |
| 116 | }); | |
| 117 | }); | |
| 75 | 118 | |
| 76 | 119 | // #12104: when two changes add lines at the same place, a kept fork's landing keeps both sides. |
| 77 | 120 | test( |
| @@ -95,10 +138,9 @@ describe("the landing simulator", () => { | ||
| 95 | 138 | |
| 96 | 139 | describe("the simulator's own parts", () => { |
| 97 | 140 | test("the quick seeds check every pair property at least once", () => { |
| 98 | const seeds = Array.from({ length: QUICK }, (_, i) => i + 1); | |
| 141 | const seeds = [...QUICK_SEEDS]; | |
| 99 | 142 | expect(seeds.some((seed) => seed % 4 === 0)).toBe(true); |
| 100 | 143 | expect(seeds.some((seed) => seed % 4 === 1)).toBe(true); |
| 101 | expect(seeds.some((seed) => seed % 16 === 2)).toBe(true); | |
| 102 | 144 | }); |
| 103 | 145 | |
| 104 | 146 | test("a seed is one scenario, and a scenario prints as a script that names it", () => { |
| @@ -266,7 +308,8 @@ describe("the simulator's own parts", () => { | ||
| 266 | 308 | |
| 267 | 309 | test("the shrinker drops pulls, steps, edits and files while the failure stays, and stops at its budget", async () => { |
| 268 | 310 | const base = generate(5); |
| 269 | // A made-up failure: it holds while pull #2 still has an edit tagged `p2e0` and the scenario is not empty. | |
| 311 | // A made-up failure: it holds while pull #2 still has an edit tagged `p2e0` | |
| 312 | // and the scenario is not empty. | |
| 270 | 313 | const culprit = (sc: Scenario): boolean => |
| 271 | 314 | sc.pulls.some( |
| 272 | 315 | (p) => |
test/support/landsim.ts +3 −4
| @@ -1634,15 +1634,14 @@ export async function checkOrderIndependence( | ||
| 1634 | 1634 | export async function checkAdditiveMerges( |
| 1635 | 1635 | world: World, |
| 1636 | 1636 | seed: number, |
| 1637 | /** Which landing orders to try: both, or one when a caller runs them apart. */ | |
| 1638 | orders: readonly (readonly [1 | 2, 1 | 2])[] = [[1, 2], [2, 1]], | |
| 1637 | 1639 | ): Promise<PairResult> { |
| 1638 | 1640 | const sc = additivePair(seed); |
| 1639 | 1641 | const how = seed % 2 === 0 ? "merge" : "squash"; |
| 1640 | 1642 | const trees: string[] = []; |
| 1641 | 1643 | const violations: Violation[] = []; |
| 1642 | for (const order of [ | |
| 1643 | [1, 2], | |
| 1644 | [2, 1], | |
| 1645 | ] as const) { | |
| 1644 | for (const order of orders) { | |
| 1646 | 1645 | const done = await landInOrder(world, sc, order, how); |
| 1647 | 1646 | trees.push(done.outcome.finalTree); |
| 1648 | 1647 | violations.push(...done.outcome.violations); |
new above
offline