Public: anyone can read this; the crew can change it.

Automate a review finding as a lint or test: Mostly right and well tested, but two things to fix in one round. #15132

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/15130-automate-a-review-finding-as-a-lint-or-t → main 4 commits fork 4 threads · 4 unresolved

  • Merged. scout/15130-automate-a-review-finding-as-a-lint-or-t was merged into main as 9259081.

Review

Sign in to review this change.
2 files changed +118 −0
src/kernel/review_rules.ts +55
@@ -25,6 +25,20 @@
2525 * `setTimeout` promise makes the test slow and time-dependent. Inject the
2626 * clock and step it; where a real wait is the point, end the line with
2727 * `// real-timer: <why>`.
28 * - "a test restores the environment it changes" (#14846, juniper: a failed
29 * `expect()` left `FORGE_CHECK_SHARDS=5` set for the tests after it): a test
30 * that adds a write to the environment (`process.env.X = ...`, `["X"]`,
31 * `||=`, `+=` and the other compound forms, `process.env = ...`,
32 * `Object.assign(process.env, ...)`, `delete process.env.X`) also adds the
33 * restore: a `finally {` block (not the promise method `.finally(`), or an
34 * `afterEach(`/`afterAll(` whose body, read until its parentheses close and
35 * at most HOOK_REACH lines on, holds a restore: a `delete process.env...`
36 * or an assignment from a saved value (a name, not a literal), so a hook
37 * that only sets is no restore. A line cannot show what a block holds, so
38 * that is the whole check. Limit: only added lines are seen, so a pull that
39 * adds one env line to a file whose existing hook already restores it is
40 * refused; end the write with `// env-restored: <where>` to say so. Every
41 * unrestored write is named, so one push shows them all.
2842 *
2943 * A comment's width is not here: it is formatting, decided and fixed by
3044 * `src/kernel/comment_wrap.ts` (`bun scripts/gate.ts --fix`, #12968).
@@ -90,15 +104,56 @@ export function parseAdded(diff: string): Added[] {
90104
91105 const AS_NEVER = /\bas never\b/;
92106 const REAL_TIMER = /\bBun\.sleep(?:Sync)?\s*\(|\bsetTimeout\s*\(\s*(?:r|res|resolve)\b/;
outdated

Any added afterEach(/afterAll( marks the file restored, even if it never touches env, and a restore already in the file is not seen. Tie the hook to env, or accept existing restores. Say which in the header comment and test it.

107 const ENV_TARGET = String.raw`\bprocess\.env(?:\.[A-Za-z_$][\w$]*|\[[^\]]+\])?\s*(?:\|\||&&|\?\?|\*\*|<<|>>>?|[-+*/%&|^])?=(?![=>])`;
108 const ENV_WRITE = new RegExp(`${ENV_TARGET}|\\bObject\\.assign\\(\\s*process\\.env\\b|\\bdelete\\s+process\\.env\\b`);
109 // A restore: a delete, or putting back a saved value (a name or a spread of
110 // one, never a literal), by assignment or by Object.assign.
111 const SAVED = String.raw`\s*(?:\{\s*\.\.\.\s*)?(?!(?:true|false|null|undefined)\b)[A-Za-z_$]`;
112 const ENV_RESTORE = new RegExp(
113 String.raw`\bdelete\s+process\.env\b|\bprocess\.env(?:\.[A-Za-z_$][\w$]*|\[[^\]]+\])?\s*(?:\?\?)?=(?![=>])${SAVED}|\bObject\.assign\(\s*process\.env\s*,${SAVED}`,
114 );
115 const FINALLY_BLOCK = /(?:^|[^.\w])finally\s*\{/;
116 const HOOK = /\bafter(?:Each|All)\s*\(/;
117 const HOOK_REACH = 6;
93118 const CODE_FILE = /^(?:src|test)\/(?!.*\/vendor\/).+\.ts$/;
outdated

The window counts any process.env mention, including the write being judged or a set inside the hook. Require a restore (delete or assignment from a saved value) inside the hook, and test that a hook which only sets does not pass.

94119
120 /** A line as code: string literals emptied, a trailing comment dropped. */
121 const code = (text: string): string => text.replace(/(["'`])(?:\\.|(?!\1).)*\1/g, '""').replace(/\/\/.*$/, "");
122
123 /**
124 * Whether the hook that `a` opens restores the environment: its own text,
125 * from `afterEach(` on through the added lines after it, until its
126 * parentheses close or HOOK_REACH lines are read.
127 */
128 function hookRestores(added: readonly Added[], a: Added): boolean {
129 const m = HOOK.exec(a.text);
130 if (!m) return false;
131 let depth = 0;
132 for (let n = 0; n <= HOOK_REACH; n++) {
133 const row = n === 0 ? a : added.find((b) => b.path === a.path && b.line === a.line + n);
134 if (!row) return false;
135 const part = n === 0 ? row.text.slice(m.index) : row.text;
136 if (ENV_RESTORE.test(code(part))) return true;
137 for (const ch of code(part)) {
138 if (ch === "(") depth++;
139 else if (ch === ")" && --depth === 0) return false;
140 }
141 }
142 return false;
143 }
144
95145 /** One line per problem among the lines a pull adds, each naming its rule. */
96146 export function addedProblems(added: readonly Added[]): string[] {
97147 const out: string[] = [];
148 const restores = (path: string): boolean =>
149 added.some((a) => a.path === path && (FINALLY_BLOCK.test(code(a.text)) || hookRestores(added, a)));
98150 for (const { path, line, text } of added) {
99151 if (!CODE_FILE.test(path)) continue;
100152 const at = `${path}:${line}`;
101153 if (TEST_FILE.test(path)) {
154 if (ENV_WRITE.test(code(text)) && !text.includes("// env-restored:") && !restores(path)) {
155 out.push(`rule "a test restores the environment it changes": ${at} sets process.env; put it back in a \`finally {\` block or an afterEach that restores it, or end the line with \`// env-restored: <where>\`; so a failed expect() leaves nothing set for the next test`);
156 }
102157 if (AS_NEVER.test(text))
103158 out.push(`rule "a test builds the real input, not \`as never\`": ${at} casts; build the proper record so the type checks`);
104159 if (REAL_TIMER.test(text) && !text.includes("// real-timer:"))
test/review-rules.test.ts +63
@@ -74,6 +74,69 @@ test("an added real-timer wait in a test is refused unless it says why", () => {
7474 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", `${SLEEP} // real-timer: the socket needs it`)))).toEqual([]);
7575 });
7676
77 test("an added process.env write in a test is refused unless the pull restores it in a finally", () => {
78 const SET = ' process.env.FORGE_CHECK_SHARDS = "5";';
79 const bad = addedProblems(parseAdded(diffOf("test/a.test.ts", SET, " delete process.env.OTHER;")));
80 expect(bad).toHaveLength(2);
81 expect(bad[0]).toContain('rule "a test restores the environment it changes"');
82 expect(bad[0]).toContain("test/a.test.ts:4");
83 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "} finally {")))).toEqual([]);
84 // A promise `.finally(` or the word in a comment restores nothing.
85 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "p.finally(() => {});")))).toHaveLength(1);
86 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "// finally { later")))).toHaveLength(1);
87 // A hook counts only when it touches the environment; afterAll counts like afterEach.
88 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterEach(() => {})")))).toHaveLength(1);
89 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterAll(() => {", " delete process.env.FORGE_CHECK_SHARDS;", "});")))).toEqual([]);
90 // The bracket form is a write; the escape says an existing hook restores it.
91 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", 'process.env["X"] = "1";')))).toHaveLength(1);
92 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", `${SET} // env-restored: the afterEach above`)))).toEqual([]);
93 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "expect(process.env.X === 'a').toBe(true);")))).toEqual([]);
94 expect(addedProblems(parseAdded(diffOf("src/a.ts", SET)))).toEqual([]);
95 });
96
97 test("every common way to write the environment is a write", () => {
98 for (const write of [
99 "process.env = { ...process.env, X: '1' };",
100 "Object.assign(process.env, { X: '1' });",
101 "process.env.X ||= '1';",
102 "process.env.PATH += ':/bin';",
103 "process.env['X'] ??= '1';",
104 ])
105 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", write)))).toHaveLength(1);
106 // Reading and comparing are not writes; a write inside a string is not code.
107 for (const read of ["if (process.env.X !== 'a') {}", "const ok = process.env.X <= 'b';", "const s = 'process.env.X = 1';"])
108 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", read)))).toEqual([]);
109 });
110
111 test("a hook counts only when its own body restores, not when it sets or sits near a set", () => {
112 const SET = ' process.env.X = "1";';
113 // A hook that only sets is no restore, and the write cannot vouch for itself.
114 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", 'afterEach(() => { process.env.X = "1"; });')))).toHaveLength(1);
115 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "afterEach(() => {", SET, "});")))).toHaveLength(1);
116 // An empty hook next to a beforeEach that sets does not cover it.
117 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "afterEach(() => {});", "beforeEach(() => {", SET, "});")))).toHaveLength(1);
118 // Putting back a saved value, by assignment or Object.assign, is a restore.
119 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "const saved = process.env.X;", SET, "afterEach(() => {", " process.env.X = saved;", "});")))).toEqual([]);
120 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "const saved = { ...process.env };", SET, "afterAll(() => {", " process.env = { ...saved };", "});")))).toEqual([]);
121 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterAll(() => Object.assign(process.env, saved));")))).toEqual([]);
122 });
123
124 test("a hook's restore counts up to HOOK_REACH (6) lines on, not 7", () => {
125 const SET = ' process.env.X = "1";';
126 const fill = (n: number) => Array.from({ length: n }, () => " const a = 1;");
127 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterEach(() => {", ...fill(5), " delete process.env.X;", "});")))).toEqual([]);
128 expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterEach(() => {", ...fill(6), " delete process.env.X;", "});")))).toContain(
129 'rule "a test restores the environment it changes": test/a.test.ts:4 sets process.env; put it back in a `finally {` block or an afterEach that restores it, or end the line with `// env-restored: <where>`; so a failed expect() leaves nothing set for the next test',
130 );
131 });
132
133 test("every unrestored write in a file is named, not only the first", () => {
134 const bad = addedProblems(parseAdded(diffOf("test/a.test.ts", 'process.env.A = "1";', "const x = 1;", 'process.env.B = "2";')));
135 expect(bad).toHaveLength(2);
136 expect(bad[0]).toContain("test/a.test.ts:4");
137 expect(bad[1]).toContain("test/a.test.ts:6");
138 });
139
77140 test("a `++ x` line added is read as a line, not as a file header", () => {
78141 const diff = diffOf("test/a.test.ts", "++ " + CAST);
79142 expect(parseAdded(diff)).toEqual([{ path: "test/a.test.ts", line: 4, text: "++ " + CAST }]);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs