Public: anyone can read this; the crew can change it.

The landsim corpus test runs under the 2 s per-test budget, and test/landsim.test.ts leaves SLOW #12488

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12447-the-landsim-corpus-test-runs-under-the-2 → main 3 commits fork

  • Merged. scout/12447-the-landsim-corpus-test-runs-under-the-2 was merged into main as 91fb68b.

Review

Sign in to review this change.
3 files changed +72 −31
scripts/gate.ts −1
@@ -113,7 +113,6 @@ export const SLOW: readonly Exclusion[] = [
113113 { file: "test/home-frame.test.ts", reason: "renders the signed-in home after writes and times the answer from memory; 2.2 s a test on the gate box", issue: 1242 },
114114 { file: "test/job-slices.test.ts", reason: "ticks every timed job on a busy fixture while sampling the event loop; 2.3 s a test on the gate box", issue: 1242 },
115115 { file: "test/kpis-landings-scale.test.ts", reason: "seeds a production-sized landings table and counts it two ways; 7.5 s a test on the gate box", issue: 1242 },
116 { file: "test/landsim.test.ts", reason: "lands four seeded scenarios through review and the merge queue at once, with the order, merge and determinism pair checks; 24 seeds nightly", issue: 12447 },
117116 { file: "test/live-arrive.test.ts", reason: "drops and restores a live connection in happy-dom, waiting on real timers; 2.8 s a test on the gate box", issue: 1242 },
118117 { file: "test/loading-chrome.test.ts", reason: "watches the loading indicator across pages in a real Chromium browser; 7.6 s a test on the gate box", issue: 1242 },
119118 { file: "test/maintenance-standby.test.ts", reason: "waits out the standby's answer timeout on the real clock; 2.0 s a test on the gate box", issue: 1242 },
test/landsim.test.ts +69 −26
@@ -5,9 +5,10 @@
55 * landing promises: what was reviewed is what lands, nothing is lost silently,
66 * what was checked is what landed.
77 *
8 * The quick run's four seeds also cover the pair properties, one seed each (order independence on seeds divisible by 4, additive
9 * merges on seed % 4 == 1, determinism on seed % 16 == 2). `LANDSIM_SEEDS=n` runs n seeds instead of the quick four; the nightly
10 * outside-gate job runs 24 (`src/nightly/jobs.ts`). A failure prints the
8 * The quick run's four seeds also cover the pair properties (order independence on seeds
9 * divisible by 4, additive merges on seed % 4 == 1); determinism runs seed 34 twice (seeds with
10 * seed % 16 == 2 under `LANDSIM_SEEDS`). `LANDSIM_SEEDS=n` runs n seeds instead of the quick four;
11 * the nightly outside-gate job runs 24 (`src/nightly/jobs.ts`). A failure prints the
1112 * seed and the scenario as a script; `bun test/support/landsim.ts --from <seed>
1213 * --seeds 1 --shrink` cuts it down to the pulls and edits that matter.
1314 *
@@ -22,6 +23,8 @@ import {
2223 applyEdits,
2324 authorshipKept,
2425 checkedIsLanded,
26 checkAdditiveMerges,
27 checkOrderIndependence,
2528 createWorld,
2629 gateAttachesToHead,
2730 generate,
@@ -31,21 +34,24 @@ import {
3134 noSilentLoss,
3235 report,
3336 reviewedIsLanded,
34 runCorpus,
37 runScenario,
3538 script,
3639 seedCount,
3740 shrinkBy,
3841 type Edit,
3942 type Landing,
43 type Outcome,
4044 type Scenario,
45 type Violation,
4146 type World,
4247 } from "./support/landsim.ts";
4348
4449 /**
45 * Seeds in the quick run, all at once: 1 checks additive merges, 2 determinism,
46 * 4 order independence. The longer corpus runs nightly.
50 * Seeds in the quick run: 1 and 5 check additive merges, 8 order independence.
51 * Chosen for landing quickly, so a test a seed stays under the per-test budget;
52 * the longer corpus (`LANDSIM_SEEDS=n`, seeds 1..n) runs nightly.
4753 */
48 const QUICK = 4;
54 const QUICK_SEEDS = [1, 3, 5, 8] as const;
4955
5056 describe("the landing simulator", () => {
5157 let world: World;
@@ -56,22 +62,59 @@ describe("the landing simulator", () => {
5662 await world.close();
5763 });
5864
59 test("a seeded corpus lands what was reviewed, apart from the violations an issue owns", async () => {
60 const n = seedCount(QUICK);
61 const corpus = await runCorpus(world, 1, n, QUICK);
62 const unexpected = [...corpus.classes].filter(
63 ([cls]) => knownIssue(cls) === undefined,
64 );
65 const lines = unexpected.map(
66 ([cls, seeds]) =>
67 ` ${cls} (seeds ${seeds.slice(0, 5).join(", ")}): ${corpus.messages.get(cls)}`,
68 );
69 expect(
70 lines,
71 `landsim found violations nothing owns (rerun one with: bun test/support/landsim.ts --from <seed> --seeds 1 --shrink):\n${lines.join("\n")}`,
72 ).toEqual([]);
73 expect(corpus.landed).toBeGreaterThan(0);
74 }, 300_000);
65 // One test a seed (and one a pair property), so each stays well under the per-test budget:
66 // `LANDSIM_SEEDS=n` adds seeds.
67 const wide = process.env["LANDSIM_SEEDS"] !== undefined;
68 const seeds = wide
69 ? Array.from({ length: seedCount(QUICK_SEEDS.length) }, (_, i) => i + 1)
70 : [...QUICK_SEEDS];
71 const unowned = (violations: readonly Violation[]): string[] =>
72 violations
73 .filter((v) => knownIssue(v.cls) === undefined)
74 .map((v) => `${v.cls}: ${v.message}`);
75 const rerun = (seed: number): string =>
76 `rerun with: bun test/support/landsim.ts --from ${seed} --seeds 1 --shrink`;
77
78 for (const seed of seeds) {
79 test(`seed ${seed} lands what was reviewed, apart from the violations an issue owns`, async () => {
80 const out = await runScenario(world, generate(seed));
81 expect(out.landed, `seed ${seed} landed nothing`).toBeGreaterThan(0);
82 expect(unowned(out.violations), `landsim found violations nothing owns (${rerun(seed)})`).toEqual([]);
83 }, 30_000);
84 if (seed % 4 === 0)
85 test(`seed ${seed}: landing order does not change what lands`, async () => {
86 const out = await checkOrderIndependence(world, seed);
87 expect(unowned(out.violations), rerun(seed)).toEqual([]);
88 }, 30_000);
89 if (seed % 4 === 1)
90 for (const order of [[1, 2], [2, 1]] as const)
91 test(`seed ${seed}: pulls that only add lines merge, landed ${order.join(" then ")}`, async () => {
92 const out = await checkAdditiveMerges(world, seed, [order]);
93 expect(unowned(out.violations), rerun(seed)).toEqual([]);
94 }, 30_000);
95 }
96
97 // Determinism: a seed twice, the runs made in the group's `beforeAll` (hooks are not the tests'
98 // time) and compared in tests that stand alone. Quick: seed 34. `LANDSIM_SEEDS=n` (the nightly,
99 // 24 seeds): every seed with seed % 16 == 2, as before the split.
100 const determinismSeeds = wide ? seeds.filter((seed) => seed % 16 === 2) : [34];
101 for (const seed of determinismSeeds)
102 describe(`determinism, seed ${seed}`, () => {
103 let first: Outcome;
104 let second: Outcome;
105 beforeAll(async () => {
106 first = await runScenario(world, generate(seed));
107 second = await runScenario(world, generate(seed));
108 }, 120_000);
109 test("a run lands what was reviewed, apart from the violations an issue owns", () => {
110 expect(first.signature).not.toBe("");
111 expect(unowned(first.violations), rerun(seed)).toEqual([]);
112 });
113 test("the same seed lands the same history a second time", () => {
114 expect(second.signature).toBe(first.signature);
115 expect(second.finalTree).toBe(first.finalTree);
116 });
117 });
75118
76119 // #12104: when two changes add lines at the same place, a kept fork's landing keeps both sides.
77120 test(
@@ -95,10 +138,9 @@ describe("the landing simulator", () => {
95138
96139 describe("the simulator's own parts", () => {
97140 test("the quick seeds check every pair property at least once", () => {
98 const seeds = Array.from({ length: QUICK }, (_, i) => i + 1);
141 const seeds = [...QUICK_SEEDS];
99142 expect(seeds.some((seed) => seed % 4 === 0)).toBe(true);
100143 expect(seeds.some((seed) => seed % 4 === 1)).toBe(true);
101 expect(seeds.some((seed) => seed % 16 === 2)).toBe(true);
102144 });
103145
104146 test("a seed is one scenario, and a scenario prints as a script that names it", () => {
@@ -266,7 +308,8 @@ describe("the simulator's own parts", () => {
266308
267309 test("the shrinker drops pulls, steps, edits and files while the failure stays, and stops at its budget", async () => {
268310 const base = generate(5);
269 // A made-up failure: it holds while pull #2 still has an edit tagged `p2e0` and the scenario is not empty.
311 // A made-up failure: it holds while pull #2 still has an edit tagged `p2e0`
312 // and the scenario is not empty.
270313 const culprit = (sc: Scenario): boolean =>
271314 sc.pulls.some(
272315 (p) =>
test/support/landsim.ts +3 −4
@@ -1634,15 +1634,14 @@ export async function checkOrderIndependence(
16341634 export async function checkAdditiveMerges(
16351635 world: World,
16361636 seed: number,
1637 /** Which landing orders to try: both, or one when a caller runs them apart. */
1638 orders: readonly (readonly [1 | 2, 1 | 2])[] = [[1, 2], [2, 1]],
16371639 ): Promise<PairResult> {
16381640 const sc = additivePair(seed);
16391641 const how = seed % 2 === 0 ? "merge" : "squash";
16401642 const trees: string[] = [];
16411643 const violations: Violation[] = [];
1642 for (const order of [
1643 [1, 2],
1644 [2, 1],
1645 ] as const) {
1644 for (const order of orders) {
16461645 const done = await landInOrder(world, sc, order, how);
16471646 trees.push(done.outcome.finalTree);
16481647 violations.push(...done.outcome.violations);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs