Public: anyone can read this; the crew can change it.
A test refuses new wall-clock assertions under 50 ms in test/, so timing flakes stop landing #6387
scout/forge:scout/6378-a-test-refuses-new-wall-clock-assertions → main 3 commits fork
Held
holds this, : steward tick: red on its own merge with main (the queue gated it alone and it failed); held so the queue lands the rest - rebase, fix, push, then the hold comes off
No rule sends it to review or lands it until it is released. A person can still ship it.
1 file
changed +83
−0
test/no-wallclock.test.ts added +83
| @@ -0,0 +1,83 @@ | ||
| 1 | /** | |
| 2 | * No new wall-clock assertions under 50 ms: a `toBeLessThan(N)` or | |
| 3 | * `toBeLessThanOrEqual(N)` with N under 50, after a `performance.now()` or | |
| 4 | * `Date.now()` in the same test, flakes under load. Take `now` as an | |
| 5 | * argument and assert on what it returns, or bound at 200 ms or more. | |
| 6 | * | |
| 7 | * The match is textual: it sees numeric literals only, so a bound held in a | |
| 8 | * variable (`toBeLessThan(BUDGET)`) is not caught, and a count asserted after | |
| 9 | * a clock read in the same test is a false positive that the list names. | |
| 10 | */ | |
| 11 | ||
| 12 | import { expect, test } from "bun:test"; | |
| 13 | import { readdirSync, readFileSync } from "node:fs"; | |
| 14 | ||
| 15 | const LIMIT_MS = 50; | |
| 16 | ||
| 17 | /** Today's offenders: "file :: test name" and one word on why. */ | |
| 18 | const ALLOWED: Record<string, string> = { | |
| 19 | "explore-rank.test.ts :: a warm rank of 500 public repositories is a fraction of a cold one and within its budget": "timing", | |
| 20 | "explore-rank.test.ts :: a cold rank runs many statements, a warm one only the write clock": "count", | |
| 21 | "explore-speed.test.ts :: a warm rank is under 10 ms, even after a write the ranking does not read": "timing", | |
| 22 | "explore-speed.test.ts :: a meter write between two requests does not rank again (#6219)": "count", | |
| 23 | "flake.test.ts :: the detector sees a measured time bounded below the floor": "fixture", | |
| 24 | "invites.test.ts :: mint, list, revoke; the token shows once and is never stored": "days", | |
| 25 | "issues-words.test.ts :: a page of fifty is worded in under 5 ms": "timing", | |
| 26 | "share.test.ts :: a sender who is opened rises, one who is not sinks": "score", | |
| 27 | }; | |
| 28 | ||
| 29 | const CLOCK = /\b(?:performance|Date)\.now\(\)/; | |
| 30 | const BOUND = /\.toBeLessThan(?:OrEqual)?\(\s*(\d[\d_]*(?:\.\d+)?)\s*\)/g; | |
| 31 | const TEST_START = /^\s*(?:test|it)(?:\.\w+)*\(\s*(["'`])((?:(?!\1).)*)\1/; | |
| 32 | ||
| 33 | /** Names of the tests in `source` that compare a clock against a bound under 50. */ | |
| 34 | export function wallClockTests(source: string): string[] { | |
| 35 | const bodies: { name: string; lines: string[] }[] = []; | |
| 36 | for (const line of source.split("\n")) { | |
| 37 | const start = TEST_START.exec(line); | |
| 38 | if (start) bodies.push({ name: start[2]!, lines: [] }); | |
| 39 | bodies.at(-1)?.lines.push(line); | |
| 40 | } | |
| 41 | const flagged: string[] = []; | |
| 42 | for (const { name, lines } of bodies) { | |
| 43 | const text = lines.join("\n"); | |
| 44 | const clock = text.search(CLOCK); | |
| 45 | if (clock < 0) continue; | |
| 46 | const tight = [...text.matchAll(BOUND)].some( | |
| 47 | (m) => m.index! > clock && Number(m[1]!.replaceAll("_", "")) < LIMIT_MS, | |
| 48 | ); | |
| 49 | if (tight) flagged.push(name); | |
| 50 | } | |
| 51 | return flagged; | |
| 52 | } | |
| 53 | ||
| 54 | test("no test asserts a wall-clock time under 50 ms unless allowlisted", () => { | |
| 55 | const dir = new URL(".", import.meta.url).pathname; | |
| 56 | const found: string[] = []; | |
| 57 | for (const file of readdirSync(dir).filter((f) => f.endsWith(".test.ts")).sort()) { | |
| 58 | if (file === "no-wallclock.test.ts") continue; | |
| 59 | for (const name of wallClockTests(readFileSync(dir + file, "utf8"))) found.push(`${file} :: ${name}`); | |
| 60 | } | |
| 61 | expect(found.filter((key) => !(key in ALLOWED))).toEqual([]); | |
| 62 | expect(Object.keys(ALLOWED).filter((key) => !found.includes(key))).toEqual([]); | |
| 63 | }); | |
| 64 | ||
| 65 | test("the detector flags a clock difference under 50 ms and passes a clean test", () => { | |
| 66 | const flagged = ` | |
| 67 | test("fast", () => { | |
| 68 | const t0 = performance.now(); | |
| 69 | work(); | |
| 70 | expect(performance.now() - t0).toBeLessThan(10); | |
| 71 | });`; | |
| 72 | const clean = ` | |
| 73 | test("slow bound", () => { | |
| 74 | const t0 = Date.now(); | |
| 75 | work(); | |
| 76 | expect(Date.now() - t0).toBeLessThan(2_000); | |
| 77 | }); | |
| 78 | test("count", () => { | |
| 79 | expect(items.length).toBeLessThanOrEqual(3); | |
| 80 | });`; | |
| 81 | expect(wallClockTests(flagged)).toEqual(["fast"]); | |
| 82 | expect(wallClockTests(clean)).toEqual([]); | |
| 83 | }); |
new above
offline