Public: anyone can read this; the crew can change it.

Automate a review finding as a lint or test: Two things to fix, first one first. #9301

Parked opened this
Closed without merging

scout/forge:scout/9268-automate-a-review-finding-as-a-lint-or-t → main 9 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
2 files changed +139 −33
scripts/review-rules.ts added +108
@@ -0,0 +1,108 @@
1 /**
2 * Review findings that repeated, kept as a check (#9268, #8716).
3 *
4 * Reviewers asked again and again for the same two things on a test: do not
5 * wait a fixed time for something to happen (#7080: "Fixed 300 ms sleep is a
6 * flake source on a loaded runner. Poll … with a deadline instead"), and do
7 * not assert a wall-clock margin that scheduler jitter can break.
8 *
9 * Rule `fixed-sleep`: a test file does not `sleep` a literal of 250 ms or
10 * more. Poll for the condition with a deadline instead. Files that already
11 * did are pinned in `BASELINE` at their current count: the count may fall,
12 * never rise, so a new fixed sleep is named in the pull that adds it.
13 */
14
15 import { readdirSync, readFileSync } from "node:fs";
16 import { join } from "node:path";
17
18 export const RULE = "fixed-sleep";
19 /** A sleep at or over this many ms, written as a number in the test. */
20 export const SLEEP_MS = 250;
21
22 const SLEEP = /\b(?:Bun\.sleep|sleep)\(\s*([0-9][0-9_]*)\s*\)/g;
23
24 /** Lines of `source` that sleep a literal of `SLEEP_MS` or more. */
25 export function fixedSleeps(source: string): number[] {
26 const lines: number[] = [];
27 source.split("\n").forEach((line, i) => {
28 for (const m of line.matchAll(SLEEP)) {
29 if (Number((m[1] ?? "0").replaceAll("_", "")) >= SLEEP_MS) lines.push(i + 1);
30 }
31 });
32 return lines;
33 }
34
35 /** Existing fixed sleeps per file, to burn down; never to raise. */
36 export const BASELINE: Readonly<Record<string, number>> = {
37 "test/release-keeper-swap.test.ts": 3,
38 "test/home-frame.test.ts": 1,
39 "test/gate-interrupted.test.ts": 1,
40 "test/apps-adversarial.test.ts": 1,
41 "test/pull-pipe.test.ts": 2,
42 "test/apps-sleep.test.ts": 2,
43 "test/big-clone.test.ts": 3,
44 "test/drill-run.test.ts": 2,
45 "test/snapshot-outcome.test.ts": 1,
46 "test/standing-every-process.test.ts": 1,
47 "test/release-later-script.test.ts": 1,
48 "test/shards-run.test.ts": 1,
49 "test/serving-priority.test.ts": 1,
50 "test/overlap.test.ts": 1,
51 "test/conversation-files.test.ts": 1,
52 "test/ssh-version-exchange.test.ts": 1,
53 "test/cgroup.test.ts": 1,
54 "test/crew-early.test.ts": 1,
55 "test/fork-pull-approval.test.ts": 1,
56 "test/accounts.test.ts": 2,
57 "test/drill-weekly.test.ts": 1,
58 "test/backup-truth.test.ts": 1,
59 "test/ssh-app.test.ts": 1,
60 "test/apps-toolbar.test.ts": 1,
61 "test/check-tiers.test.ts": 1,
62 "test/composer-draft.test.ts": 1,
63 "test/apps-token.test.ts": 2,
64 "test/snapshot-restore-streamed.test.ts": 1,
65 "test/checks.test.ts": 4,
66 "test/apps-rollout.test.ts": 1,
67 "test/spawn-chain.test.ts": 1,
68 "test/upstream-forge.test.ts": 1,
69 "test/proposal-run.test.ts": 1,
70 "test/release-handover.test.ts": 2,
71 "test/event-audience.test.ts": 2,
72 "test/signins.test.ts": 5,
73 "test/crew-closed.test.ts": 1,
74 "test/release-swap.test.ts": 3,
75 "test/apps-idle.test.ts": 3,
76 "test/check-memory-warning.test.ts": 1,
77 "test/approve-here.test.ts": 1,
78 "test/crew-feedback.test.ts": 2,
79 "test/fork-follow-mode.test.ts": 1,
80 "test/ssh.test.ts": 1,
81 "test/forge-signin.test.ts": 1,
82 "test/contain.test.ts": 3,
83 "test/conflicts-tui.test.ts": 1,
84 "test/apps-ssh.test.ts": 2,
85 };
86
87 /** One line per file that sleeps more fixed times than it is allowed. */
88 export function audit(files: Map<string, string>): string[] {
89 const out: string[] = [];
90 for (const [file, source] of [...files].sort(([a], [b]) => a.localeCompare(b))) {
91 const lines = fixedSleeps(source);
92 const allowed = BASELINE[file] ?? 0;
93 if (lines.length > allowed) {
94 out.push(
95 `review rule ${RULE}: ${file}:${lines.join(",")} sleeps a fixed ${SLEEP_MS} ms or more (${lines.length}, allowed ${allowed}); poll for the condition with a deadline instead (#7080)`,
96 );
97 }
98 }
99 return out;
100 }
101
102 export function readTests(root: string): Map<string, string> {
103 const files = new Map<string, string>();
104 for (const name of readdirSync(join(root, "test")).filter((n) => n.endsWith(".test.ts"))) {
105 files.set(`test/${name}`, readFileSync(join(root, "test", name), "utf8"));
106 }
107 return files;
108 }
test/review-rules.test.ts +31 −33
@@ -1,36 +1,34 @@
1 import { expect, test } from "bun:test";
2 import { readdirSync, readFileSync } from "node:fs";
3 import { reviewProblems } from "../src/kernel/review_rules.ts";
4 import { reviewRefusal } from "../scripts/gate.ts";
1 /**
2 * A finding reviewers repeated is a check (#9268): no new fixed sleep of
3 * 250 ms or more in a test; poll with a deadline instead. The failure names
4 * the rule, the file and the lines.
5 */
56
6 // Built apart so this file does not itself import a test file.
7 const T = ".test";
7 import { describe, expect, test } from "bun:test";
8 import { audit, BASELINE, fixedSleeps, RULE, readTests } from "../scripts/review-rules.ts";
89
9 test("a test that imports another test file is refused, naming the rule, file and line", () => {
10 const [p, ...rest] = reviewProblems([
11 { path: "test/a.test.ts", text: `import { x } from "bun:test";\nimport { boot } from "./b${T}.ts";\n` },
12 ]);
13 expect(rest).toEqual([]);
14 expect(p).toContain('rule "a test file does not import another test file"');
15 expect(p).toContain("test/a.test.ts:2");
16 expect(p).toContain("test/helpers/");
17 });
18
19 test("dynamic and extensionless imports of a test file are caught too", () => {
20 expect(reviewProblems([{ path: "test/a.test.ts", text: `const m = await import("../test/b${T}");` }])).toHaveLength(1);
21 });
22
23 test("a helper import, a source import and a non-test file pass", () => {
24 expect(
25 reviewProblems([
26 { path: "test/a.test.ts", text: 'import { boot } from "./helpers/toolbar-boot.ts";\nimport { f } from "../src/x.ts";' },
27 { path: "test/helpers/h.ts", text: `import "./b${T}.ts";` },
28 ]),
29 ).toEqual([]);
30 });
31
32 test("the repository's own tests pass the rules", () => {
33 expect(reviewRefusal()).toEqual([]);
34 expect(readdirSync("test").length).toBeGreaterThan(0);
35 expect(readFileSync("test/toolbar-camera.test.ts", "utf8")).toContain("helpers/toolbar-boot.ts");
10 const root = new URL("..", import.meta.url).pathname;
11
12 describe("review rule fixed-sleep", () => {
13 test("the tests carry no fixed sleep beyond the pinned baseline", () => {
14 expect(audit(readTests(root))).toEqual([]);
15 });
16
17 test("a planted fixed sleep is named with the rule and its line", () => {
18 const planted = new Map([["test/zz.test.ts", ["await 1;", `await Bun.sleep(${300});`, ""].join("\n")]]);
19 const out = audit(planted);
20 expect(out).toHaveLength(1);
21 expect(out[0]).toContain(`review rule ${RULE}`);
22 expect(out[0]).toContain("test/zz.test.ts:2");
23 });
24
25 test("short sleeps and polling pass, and underscores count", () => {
26 expect(fixedSleeps("await Bun.sleep(100);\nawait sleep(5)")).toEqual([]);
27 expect(fixedSleeps(`await sleep(${"1_000"})`)).toEqual([1]);
28 });
29
30 test("the baseline names only files that exist", () => {
31 const present = readTests(root);
32 for (const f of Object.keys(BASELINE)) expect(present.has(f)).toBe(true);
33 });
3634 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs