Public: anyone can read this; the crew can change it.

A test refuses new wall-clock assertions under 50 ms in test/, so timing flakes stop landing #6387

Closed opened this
Closed without merging

scout/forge:scout/6378-a-test-refuses-new-wall-clock-assertions → main 3 commits fork

Held

holds this, : steward tick: red on its own merge with main (the queue gated it alone and it failed); held so the queue lands the rest - rebase, fix, push, then the hold comes off

No rule sends it to review or lands it until it is released. A person can still ship it.

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
1 file changed +83 −0
test/no-wallclock.test.ts added +83
@@ -0,0 +1,83 @@
1 /**
2 * No new wall-clock assertions under 50 ms: a `toBeLessThan(N)` or
3 * `toBeLessThanOrEqual(N)` with N under 50, after a `performance.now()` or
4 * `Date.now()` in the same test, flakes under load. Take `now` as an
5 * argument and assert on what it returns, or bound at 200 ms or more.
6 *
7 * The match is textual: it sees numeric literals only, so a bound held in a
8 * variable (`toBeLessThan(BUDGET)`) is not caught, and a count asserted after
9 * a clock read in the same test is a false positive that the list names.
10 */
11
12 import { expect, test } from "bun:test";
13 import { readdirSync, readFileSync } from "node:fs";
14
15 const LIMIT_MS = 50;
16
17 /** Today's offenders: "file :: test name" and one word on why. */
18 const ALLOWED: Record<string, string> = {
19 "explore-rank.test.ts :: a warm rank of 500 public repositories is a fraction of a cold one and within its budget": "timing",
20 "explore-rank.test.ts :: a cold rank runs many statements, a warm one only the write clock": "count",
21 "explore-speed.test.ts :: a warm rank is under 10 ms, even after a write the ranking does not read": "timing",
22 "explore-speed.test.ts :: a meter write between two requests does not rank again (#6219)": "count",
23 "flake.test.ts :: the detector sees a measured time bounded below the floor": "fixture",
24 "invites.test.ts :: mint, list, revoke; the token shows once and is never stored": "days",
25 "issues-words.test.ts :: a page of fifty is worded in under 5 ms": "timing",
26 "share.test.ts :: a sender who is opened rises, one who is not sinks": "score",
27 };
28
29 const CLOCK = /\b(?:performance|Date)\.now\(\)/;
30 const BOUND = /\.toBeLessThan(?:OrEqual)?\(\s*(\d[\d_]*(?:\.\d+)?)\s*\)/g;
31 const TEST_START = /^\s*(?:test|it)(?:\.\w+)*\(\s*(["'`])((?:(?!\1).)*)\1/;
32
33 /** Names of the tests in `source` that compare a clock against a bound under 50. */
34 export function wallClockTests(source: string): string[] {
35 const bodies: { name: string; lines: string[] }[] = [];
36 for (const line of source.split("\n")) {
37 const start = TEST_START.exec(line);
38 if (start) bodies.push({ name: start[2]!, lines: [] });
39 bodies.at(-1)?.lines.push(line);
40 }
41 const flagged: string[] = [];
42 for (const { name, lines } of bodies) {
43 const text = lines.join("\n");
44 const clock = text.search(CLOCK);
45 if (clock < 0) continue;
46 const tight = [...text.matchAll(BOUND)].some(
47 (m) => m.index! > clock && Number(m[1]!.replaceAll("_", "")) < LIMIT_MS,
48 );
49 if (tight) flagged.push(name);
50 }
51 return flagged;
52 }
53
54 test("no test asserts a wall-clock time under 50 ms unless allowlisted", () => {
55 const dir = new URL(".", import.meta.url).pathname;
56 const found: string[] = [];
57 for (const file of readdirSync(dir).filter((f) => f.endsWith(".test.ts")).sort()) {
58 if (file === "no-wallclock.test.ts") continue;
59 for (const name of wallClockTests(readFileSync(dir + file, "utf8"))) found.push(`${file} :: ${name}`);
60 }
61 expect(found.filter((key) => !(key in ALLOWED))).toEqual([]);
62 expect(Object.keys(ALLOWED).filter((key) => !found.includes(key))).toEqual([]);
63 });
64
65 test("the detector flags a clock difference under 50 ms and passes a clean test", () => {
66 const flagged = `
67 test("fast", () => {
68 const t0 = performance.now();
69 work();
70 expect(performance.now() - t0).toBeLessThan(10);
71 });`;
72 const clean = `
73 test("slow bound", () => {
74 const t0 = Date.now();
75 work();
76 expect(Date.now() - t0).toBeLessThan(2_000);
77 });
78 test("count", () => {
79 expect(items.length).toBeLessThanOrEqual(3);
80 });`;
81 expect(wallClockTests(flagged)).toEqual(["fast"]);
82 expect(wallClockTests(clean)).toEqual([]);
83 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs