Public: anyone can read this; the crew can change it.
Automate a review finding as a lint or test: Mostly right and well tested, but two things to fix in one round. #15132
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/15130-automate-a-review-finding-as-a-lint-or-t → main 4 commits fork 4 threads · 4 unresolved
2 files
changed +118
−0
src/kernel/review_rules.ts +55
| @@ -25,6 +25,20 @@ | ||
| 25 | 25 | * `setTimeout` promise makes the test slow and time-dependent. Inject the |
| 26 | 26 | * clock and step it; where a real wait is the point, end the line with |
| 27 | 27 | * `// real-timer: <why>`. |
| 28 | * - "a test restores the environment it changes" (#14846, juniper: a failed | |
| 29 | * `expect()` left `FORGE_CHECK_SHARDS=5` set for the tests after it): a test | |
| 30 | * that adds a write to the environment (`process.env.X = ...`, `["X"]`, | |
| 31 | * `||=`, `+=` and the other compound forms, `process.env = ...`, | |
| 32 | * `Object.assign(process.env, ...)`, `delete process.env.X`) also adds the | |
| 33 | * restore: a `finally {` block (not the promise method `.finally(`), or an | |
| 34 | * `afterEach(`/`afterAll(` whose body, read until its parentheses close and | |
| 35 | * at most HOOK_REACH lines on, holds a restore: a `delete process.env...` | |
| 36 | * or an assignment from a saved value (a name, not a literal), so a hook | |
| 37 | * that only sets is no restore. A line cannot show what a block holds, so | |
| 38 | * that is the whole check. Limit: only added lines are seen, so a pull that | |
| 39 | * adds one env line to a file whose existing hook already restores it is | |
| 40 | * refused; end the write with `// env-restored: <where>` to say so. Every | |
| 41 | * unrestored write is named, so one push shows them all. | |
| 28 | 42 | * |
| 29 | 43 | * A comment's width is not here: it is formatting, decided and fixed by |
| 30 | 44 | * `src/kernel/comment_wrap.ts` (`bun scripts/gate.ts --fix`, #12968). |
| @@ -90,15 +104,56 @@ export function parseAdded(diff: string): Added[] { | ||
| 90 | 104 | |
| 91 | 105 | const AS_NEVER = /\bas never\b/; |
| 92 | 106 | const REAL_TIMER = /\bBun\.sleep(?:Sync)?\s*\(|\bsetTimeout\s*\(\s*(?:r|res|resolve)\b/; |
| 107 | const ENV_TARGET = String.raw`\bprocess\.env(?:\.[A-Za-z_$][\w$]*|\[[^\]]+\])?\s*(?:\|\||&&|\?\?|\*\*|<<|>>>?|[-+*/%&|^])?=(?![=>])`; | |
| 108 | const ENV_WRITE = new RegExp(`${ENV_TARGET}|\\bObject\\.assign\\(\\s*process\\.env\\b|\\bdelete\\s+process\\.env\\b`); | |
| 109 | // A restore: a delete, or putting back a saved value (a name or a spread of | |
| 110 | // one, never a literal), by assignment or by Object.assign. | |
| 111 | const SAVED = String.raw`\s*(?:\{\s*\.\.\.\s*)?(?!(?:true|false|null|undefined)\b)[A-Za-z_$]`; | |
| 112 | const ENV_RESTORE = new RegExp( | |
| 113 | String.raw`\bdelete\s+process\.env\b|\bprocess\.env(?:\.[A-Za-z_$][\w$]*|\[[^\]]+\])?\s*(?:\?\?)?=(?![=>])${SAVED}|\bObject\.assign\(\s*process\.env\s*,${SAVED}`, | |
| 114 | ); | |
| 115 | const FINALLY_BLOCK = /(?:^|[^.\w])finally\s*\{/; | |
| 116 | const HOOK = /\bafter(?:Each|All)\s*\(/; | |
| 117 | const HOOK_REACH = 6; | |
| 93 | 118 | const CODE_FILE = /^(?:src|test)\/(?!.*\/vendor\/).+\.ts$/; |
outdated
The window counts any process.env mention, including the write being judged or a set inside the hook. Require a restore (delete or assignment from a saved value) inside the hook, and test that a hook which only sets does not pass. |
||
| 94 | 119 | |
| 120 | /** A line as code: string literals emptied, a trailing comment dropped. */ | |
| 121 | const code = (text: string): string => text.replace(/(["'`])(?:\\.|(?!\1).)*\1/g, '""').replace(/\/\/.*$/, ""); | |
| 122 | ||
| 123 | /** | |
| 124 | * Whether the hook that `a` opens restores the environment: its own text, | |
| 125 | * from `afterEach(` on through the added lines after it, until its | |
| 126 | * parentheses close or HOOK_REACH lines are read. | |
| 127 | */ | |
| 128 | function hookRestores(added: readonly Added[], a: Added): boolean { | |
| 129 | const m = HOOK.exec(a.text); | |
| 130 | if (!m) return false; | |
| 131 | let depth = 0; | |
| 132 | for (let n = 0; n <= HOOK_REACH; n++) { | |
| 133 | const row = n === 0 ? a : added.find((b) => b.path === a.path && b.line === a.line + n); | |
| 134 | if (!row) return false; | |
| 135 | const part = n === 0 ? row.text.slice(m.index) : row.text; | |
| 136 | if (ENV_RESTORE.test(code(part))) return true; | |
| 137 | for (const ch of code(part)) { | |
| 138 | if (ch === "(") depth++; | |
| 139 | else if (ch === ")" && --depth === 0) return false; | |
| 140 | } | |
| 141 | } | |
| 142 | return false; | |
| 143 | } | |
| 144 | ||
| 95 | 145 | /** One line per problem among the lines a pull adds, each naming its rule. */ |
| 96 | 146 | export function addedProblems(added: readonly Added[]): string[] { |
| 97 | 147 | const out: string[] = []; |
| 148 | const restores = (path: string): boolean => | |
| 149 | added.some((a) => a.path === path && (FINALLY_BLOCK.test(code(a.text)) || hookRestores(added, a))); | |
| 98 | 150 | for (const { path, line, text } of added) { |
| 99 | 151 | if (!CODE_FILE.test(path)) continue; |
| 100 | 152 | const at = `${path}:${line}`; |
| 101 | 153 | if (TEST_FILE.test(path)) { |
| 154 | if (ENV_WRITE.test(code(text)) && !text.includes("// env-restored:") && !restores(path)) { | |
| 155 | out.push(`rule "a test restores the environment it changes": ${at} sets process.env; put it back in a \`finally {\` block or an afterEach that restores it, or end the line with \`// env-restored: <where>\`; so a failed expect() leaves nothing set for the next test`); | |
| 156 | } | |
| 102 | 157 | if (AS_NEVER.test(text)) |
| 103 | 158 | out.push(`rule "a test builds the real input, not \`as never\`": ${at} casts; build the proper record so the type checks`); |
| 104 | 159 | if (REAL_TIMER.test(text) && !text.includes("// real-timer:")) |
test/review-rules.test.ts +63
| @@ -74,6 +74,69 @@ test("an added real-timer wait in a test is refused unless it says why", () => { | ||
| 74 | 74 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", `${SLEEP} // real-timer: the socket needs it`)))).toEqual([]); |
| 75 | 75 | }); |
| 76 | 76 | |
| 77 | test("an added process.env write in a test is refused unless the pull restores it in a finally", () => { | |
| 78 | const SET = ' process.env.FORGE_CHECK_SHARDS = "5";'; | |
| 79 | const bad = addedProblems(parseAdded(diffOf("test/a.test.ts", SET, " delete process.env.OTHER;"))); | |
| 80 | expect(bad).toHaveLength(2); | |
| 81 | expect(bad[0]).toContain('rule "a test restores the environment it changes"'); | |
| 82 | expect(bad[0]).toContain("test/a.test.ts:4"); | |
| 83 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "} finally {")))).toEqual([]); | |
| 84 | // A promise `.finally(` or the word in a comment restores nothing. | |
| 85 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "p.finally(() => {});")))).toHaveLength(1); | |
| 86 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "// finally { later")))).toHaveLength(1); | |
| 87 | // A hook counts only when it touches the environment; afterAll counts like afterEach. | |
| 88 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterEach(() => {})")))).toHaveLength(1); | |
| 89 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterAll(() => {", " delete process.env.FORGE_CHECK_SHARDS;", "});")))).toEqual([]); | |
| 90 | // The bracket form is a write; the escape says an existing hook restores it. | |
| 91 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", 'process.env["X"] = "1";')))).toHaveLength(1); | |
| 92 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", `${SET} // env-restored: the afterEach above`)))).toEqual([]); | |
| 93 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "expect(process.env.X === 'a').toBe(true);")))).toEqual([]); | |
| 94 | expect(addedProblems(parseAdded(diffOf("src/a.ts", SET)))).toEqual([]); | |
| 95 | }); | |
| 96 | ||
| 97 | test("every common way to write the environment is a write", () => { | |
| 98 | for (const write of [ | |
| 99 | "process.env = { ...process.env, X: '1' };", | |
| 100 | "Object.assign(process.env, { X: '1' });", | |
| 101 | "process.env.X ||= '1';", | |
| 102 | "process.env.PATH += ':/bin';", | |
| 103 | "process.env['X'] ??= '1';", | |
| 104 | ]) | |
| 105 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", write)))).toHaveLength(1); | |
| 106 | // Reading and comparing are not writes; a write inside a string is not code. | |
| 107 | for (const read of ["if (process.env.X !== 'a') {}", "const ok = process.env.X <= 'b';", "const s = 'process.env.X = 1';"]) | |
| 108 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", read)))).toEqual([]); | |
| 109 | }); | |
| 110 | ||
| 111 | test("a hook counts only when its own body restores, not when it sets or sits near a set", () => { | |
| 112 | const SET = ' process.env.X = "1";'; | |
| 113 | // A hook that only sets is no restore, and the write cannot vouch for itself. | |
| 114 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", 'afterEach(() => { process.env.X = "1"; });')))).toHaveLength(1); | |
| 115 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "afterEach(() => {", SET, "});")))).toHaveLength(1); | |
| 116 | // An empty hook next to a beforeEach that sets does not cover it. | |
| 117 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "afterEach(() => {});", "beforeEach(() => {", SET, "});")))).toHaveLength(1); | |
| 118 | // Putting back a saved value, by assignment or Object.assign, is a restore. | |
| 119 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "const saved = process.env.X;", SET, "afterEach(() => {", " process.env.X = saved;", "});")))).toEqual([]); | |
| 120 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", "const saved = { ...process.env };", SET, "afterAll(() => {", " process.env = { ...saved };", "});")))).toEqual([]); | |
| 121 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterAll(() => Object.assign(process.env, saved));")))).toEqual([]); | |
| 122 | }); | |
| 123 | ||
| 124 | test("a hook's restore counts up to HOOK_REACH (6) lines on, not 7", () => { | |
| 125 | const SET = ' process.env.X = "1";'; | |
| 126 | const fill = (n: number) => Array.from({ length: n }, () => " const a = 1;"); | |
| 127 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterEach(() => {", ...fill(5), " delete process.env.X;", "});")))).toEqual([]); | |
| 128 | expect(addedProblems(parseAdded(diffOf("test/a.test.ts", SET, "afterEach(() => {", ...fill(6), " delete process.env.X;", "});")))).toContain( | |
| 129 | 'rule "a test restores the environment it changes": test/a.test.ts:4 sets process.env; put it back in a `finally {` block or an afterEach that restores it, or end the line with `// env-restored: <where>`; so a failed expect() leaves nothing set for the next test', | |
| 130 | ); | |
| 131 | }); | |
| 132 | ||
| 133 | test("every unrestored write in a file is named, not only the first", () => { | |
| 134 | const bad = addedProblems(parseAdded(diffOf("test/a.test.ts", 'process.env.A = "1";', "const x = 1;", 'process.env.B = "2";'))); | |
| 135 | expect(bad).toHaveLength(2); | |
| 136 | expect(bad[0]).toContain("test/a.test.ts:4"); | |
| 137 | expect(bad[1]).toContain("test/a.test.ts:6"); | |
| 138 | }); | |
| 139 | ||
| 77 | 140 | test("a `++ x` line added is read as a line, not as a file header", () => { |
| 78 | 141 | const diff = diffOf("test/a.test.ts", "++ " + CAST); |
| 79 | 142 | expect(parseAdded(diff)).toEqual([{ path: "test/a.test.ts", line: 4, text: "++ " + CAST }]); |
new above
offline
Any added
afterEach(/afterAll(marks the file restored, even if it never touches env, and a restore already in the file is not seen. Tie the hook to env, or accept existing restores. Say which in the header comment and test it.