Public: anyone can read this; the crew can change it.
Automate a review finding as a lint or test: Two things to fix, first one first. #9301
scout/forge:scout/9268-automate-a-review-finding-as-a-lint-or-t → main 9 commits fork
2 files
changed +139
−33
scripts/review-rules.ts added +108
| @@ -0,0 +1,108 @@ | ||
| 1 | /** | |
| 2 | * Review findings that repeated, kept as a check (#9268, #8716). | |
| 3 | * | |
| 4 | * Reviewers asked again and again for the same two things on a test: do not | |
| 5 | * wait a fixed time for something to happen (#7080: "Fixed 300 ms sleep is a | |
| 6 | * flake source on a loaded runner. Poll … with a deadline instead"), and do | |
| 7 | * not assert a wall-clock margin that scheduler jitter can break. | |
| 8 | * | |
| 9 | * Rule `fixed-sleep`: a test file does not `sleep` a literal of 250 ms or | |
| 10 | * more. Poll for the condition with a deadline instead. Files that already | |
| 11 | * did are pinned in `BASELINE` at their current count: the count may fall, | |
| 12 | * never rise, so a new fixed sleep is named in the pull that adds it. | |
| 13 | */ | |
| 14 | ||
| 15 | import { readdirSync, readFileSync } from "node:fs"; | |
| 16 | import { join } from "node:path"; | |
| 17 | ||
| 18 | export const RULE = "fixed-sleep"; | |
| 19 | /** A sleep at or over this many ms, written as a number in the test. */ | |
| 20 | export const SLEEP_MS = 250; | |
| 21 | ||
| 22 | const SLEEP = /\b(?:Bun\.sleep|sleep)\(\s*([0-9][0-9_]*)\s*\)/g; | |
| 23 | ||
| 24 | /** Lines of `source` that sleep a literal of `SLEEP_MS` or more. */ | |
| 25 | export function fixedSleeps(source: string): number[] { | |
| 26 | const lines: number[] = []; | |
| 27 | source.split("\n").forEach((line, i) => { | |
| 28 | for (const m of line.matchAll(SLEEP)) { | |
| 29 | if (Number((m[1] ?? "0").replaceAll("_", "")) >= SLEEP_MS) lines.push(i + 1); | |
| 30 | } | |
| 31 | }); | |
| 32 | return lines; | |
| 33 | } | |
| 34 | ||
| 35 | /** Existing fixed sleeps per file, to burn down; never to raise. */ | |
| 36 | export const BASELINE: Readonly<Record<string, number>> = { | |
| 37 | "test/release-keeper-swap.test.ts": 3, | |
| 38 | "test/home-frame.test.ts": 1, | |
| 39 | "test/gate-interrupted.test.ts": 1, | |
| 40 | "test/apps-adversarial.test.ts": 1, | |
| 41 | "test/pull-pipe.test.ts": 2, | |
| 42 | "test/apps-sleep.test.ts": 2, | |
| 43 | "test/big-clone.test.ts": 3, | |
| 44 | "test/drill-run.test.ts": 2, | |
| 45 | "test/snapshot-outcome.test.ts": 1, | |
| 46 | "test/standing-every-process.test.ts": 1, | |
| 47 | "test/release-later-script.test.ts": 1, | |
| 48 | "test/shards-run.test.ts": 1, | |
| 49 | "test/serving-priority.test.ts": 1, | |
| 50 | "test/overlap.test.ts": 1, | |
| 51 | "test/conversation-files.test.ts": 1, | |
| 52 | "test/ssh-version-exchange.test.ts": 1, | |
| 53 | "test/cgroup.test.ts": 1, | |
| 54 | "test/crew-early.test.ts": 1, | |
| 55 | "test/fork-pull-approval.test.ts": 1, | |
| 56 | "test/accounts.test.ts": 2, | |
| 57 | "test/drill-weekly.test.ts": 1, | |
| 58 | "test/backup-truth.test.ts": 1, | |
| 59 | "test/ssh-app.test.ts": 1, | |
| 60 | "test/apps-toolbar.test.ts": 1, | |
| 61 | "test/check-tiers.test.ts": 1, | |
| 62 | "test/composer-draft.test.ts": 1, | |
| 63 | "test/apps-token.test.ts": 2, | |
| 64 | "test/snapshot-restore-streamed.test.ts": 1, | |
| 65 | "test/checks.test.ts": 4, | |
| 66 | "test/apps-rollout.test.ts": 1, | |
| 67 | "test/spawn-chain.test.ts": 1, | |
| 68 | "test/upstream-forge.test.ts": 1, | |
| 69 | "test/proposal-run.test.ts": 1, | |
| 70 | "test/release-handover.test.ts": 2, | |
| 71 | "test/event-audience.test.ts": 2, | |
| 72 | "test/signins.test.ts": 5, | |
| 73 | "test/crew-closed.test.ts": 1, | |
| 74 | "test/release-swap.test.ts": 3, | |
| 75 | "test/apps-idle.test.ts": 3, | |
| 76 | "test/check-memory-warning.test.ts": 1, | |
| 77 | "test/approve-here.test.ts": 1, | |
| 78 | "test/crew-feedback.test.ts": 2, | |
| 79 | "test/fork-follow-mode.test.ts": 1, | |
| 80 | "test/ssh.test.ts": 1, | |
| 81 | "test/forge-signin.test.ts": 1, | |
| 82 | "test/contain.test.ts": 3, | |
| 83 | "test/conflicts-tui.test.ts": 1, | |
| 84 | "test/apps-ssh.test.ts": 2, | |
| 85 | }; | |
| 86 | ||
| 87 | /** One line per file that sleeps more fixed times than it is allowed. */ | |
| 88 | export function audit(files: Map<string, string>): string[] { | |
| 89 | const out: string[] = []; | |
| 90 | for (const [file, source] of [...files].sort(([a], [b]) => a.localeCompare(b))) { | |
| 91 | const lines = fixedSleeps(source); | |
| 92 | const allowed = BASELINE[file] ?? 0; | |
| 93 | if (lines.length > allowed) { | |
| 94 | out.push( | |
| 95 | `review rule ${RULE}: ${file}:${lines.join(",")} sleeps a fixed ${SLEEP_MS} ms or more (${lines.length}, allowed ${allowed}); poll for the condition with a deadline instead (#7080)`, | |
| 96 | ); | |
| 97 | } | |
| 98 | } | |
| 99 | return out; | |
| 100 | } | |
| 101 | ||
| 102 | export function readTests(root: string): Map<string, string> { | |
| 103 | const files = new Map<string, string>(); | |
| 104 | for (const name of readdirSync(join(root, "test")).filter((n) => n.endsWith(".test.ts"))) { | |
| 105 | files.set(`test/${name}`, readFileSync(join(root, "test", name), "utf8")); | |
| 106 | } | |
| 107 | return files; | |
| 108 | } |
test/review-rules.test.ts +31 −33
| @@ -1,36 +1,34 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { readdirSync, readFileSync } from "node:fs"; | |
| 3 | import { reviewProblems } from "../src/kernel/review_rules.ts"; | |
| 4 | import { reviewRefusal } from "../scripts/gate.ts"; | |
| 1 | /** | |
| 2 | * A finding reviewers repeated is a check (#9268): no new fixed sleep of | |
| 3 | * 250 ms or more in a test; poll with a deadline instead. The failure names | |
| 4 | * the rule, the file and the lines. | |
| 5 | */ | |
| 5 | 6 | |
| 6 | // Built apart so this file does not itself import a test file. | |
| 7 | const T = ".test"; | |
| 7 | import { describe, expect, test } from "bun:test"; | |
| 8 | import { audit, BASELINE, fixedSleeps, RULE, readTests } from "../scripts/review-rules.ts"; | |
| 8 | 9 | |
| 9 | test("a test that imports another test file is refused, naming the rule, file and line", () => { | |
| 10 | const [p, ...rest] = reviewProblems([ | |
| 11 | { path: "test/a.test.ts", text: `import { x } from "bun:test";\nimport { boot } from "./b${T}.ts";\n` }, | |
| 12 | ]); | |
| 13 | expect(rest).toEqual([]); | |
| 14 | expect(p).toContain('rule "a test file does not import another test file"'); | |
| 15 | expect(p).toContain("test/a.test.ts:2"); | |
| 16 | expect(p).toContain("test/helpers/"); | |
| 17 | }); | |
| 18 | ||
| 19 | test("dynamic and extensionless imports of a test file are caught too", () => { | |
| 20 | expect(reviewProblems([{ path: "test/a.test.ts", text: `const m = await import("../test/b${T}");` }])).toHaveLength(1); | |
| 21 | }); | |
| 22 | ||
| 23 | test("a helper import, a source import and a non-test file pass", () => { | |
| 24 | expect( | |
| 25 | reviewProblems([ | |
| 26 | { path: "test/a.test.ts", text: 'import { boot } from "./helpers/toolbar-boot.ts";\nimport { f } from "../src/x.ts";' }, | |
| 27 | { path: "test/helpers/h.ts", text: `import "./b${T}.ts";` }, | |
| 28 | ]), | |
| 29 | ).toEqual([]); | |
| 30 | }); | |
| 31 | ||
| 32 | test("the repository's own tests pass the rules", () => { | |
| 33 | expect(reviewRefusal()).toEqual([]); | |
| 34 | expect(readdirSync("test").length).toBeGreaterThan(0); | |
| 35 | expect(readFileSync("test/toolbar-camera.test.ts", "utf8")).toContain("helpers/toolbar-boot.ts"); | |
| 10 | const root = new URL("..", import.meta.url).pathname; | |
| 11 | ||
| 12 | describe("review rule fixed-sleep", () => { | |
| 13 | test("the tests carry no fixed sleep beyond the pinned baseline", () => { | |
| 14 | expect(audit(readTests(root))).toEqual([]); | |
| 15 | }); | |
| 16 | ||
| 17 | test("a planted fixed sleep is named with the rule and its line", () => { | |
| 18 | const planted = new Map([["test/zz.test.ts", ["await 1;", `await Bun.sleep(${300});`, ""].join("\n")]]); | |
| 19 | const out = audit(planted); | |
| 20 | expect(out).toHaveLength(1); | |
| 21 | expect(out[0]).toContain(`review rule ${RULE}`); | |
| 22 | expect(out[0]).toContain("test/zz.test.ts:2"); | |
| 23 | }); | |
| 24 | ||
| 25 | test("short sleeps and polling pass, and underscores count", () => { | |
| 26 | expect(fixedSleeps("await Bun.sleep(100);\nawait sleep(5)")).toEqual([]); | |
| 27 | expect(fixedSleeps(`await sleep(${"1_000"})`)).toEqual([1]); | |
| 28 | }); | |
| 29 | ||
| 30 | test("the baseline names only files that exist", () => { | |
| 31 | const present = readTests(root); | |
| 32 | for (const f of Object.keys(BASELINE)) expect(present.has(f)).toBe(true); | |
| 33 | }); | |
| 36 | 34 | }); |
new above
offline