Public: anyone can read this; the crew can change it.

Automate a review finding as a lint or test: **Tests (blocking).** Several behaviours this pull adds have no test that would #14237

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14235-automate-a-review-finding-as-a-lint-or-t → main 2 commits fork

  • Merged. scout/14235-automate-a-review-finding-as-a-lint-or-t was merged into main as 973c224.

Review

Sign in to review this change.
5 files changed +99 −1
AGENTS.md +2
@@ -61,3 +61,5 @@ An app token's requests go behind people's and repeat reads are 304s (`src/http/
6161 `src/web/studio/waiting_on_me.ts`: `waitingRows`, each viewer's issues and pulls that wait for them, held 10 seconds per viewer (at most 500 viewers, gone on restart) so the board's live lanes share one `decisionsFor` read; the terminal door is `src/tui/screens/waiting_for_me.ts` (palette entry for any person).
6262 `src/crew/beat.ts`: `lastBeat`, when each repository's beat last ran, and `declared`, the `beat` last read from each default branch with the commit it came from (#13709); at most one entry each per repository, gone on restart (a restart beats once at once; the declaration is a file in git).
6363 `src/perf/job_alloc.ts`: `counted`, called by `during` around each job's synchronous slice, adds the JS heap's growth to the job's name (at most 128 names); the minute memory line reads and empties it (`memory allocated by jobs:`, #13924), so a timer that allocates with no request running is named. `bench/idle.ts` is the idle budget on a seeded forge (heap under 20 MiB a minute, no stall over 100 ms). Gone on restart.
64
65 A pull that changes code under `src/` adds or edits a `test/*.test.ts` too, or says `No test: <why>` in a commit line; `bun scripts/gate.ts` refuses it otherwise ("a behaviour a pull adds has a test that fails without it", #14235).
scripts/gate.ts +16
@@ -692,6 +692,21 @@ export async function bothDoorsRefusal(cwd: string = process.cwd()): Promise<str
692692 return bothDoorsProblems(changes, log.out.split("\0").map((m) => m.trim()).filter(Boolean), imported);
693693 }
694694
695 /** A pull that changes `src/` changes a test too (#14235). */
696 export async function testsWithChangeRefusal(cwd: string = process.cwd()): Promise<string[]> {
697 const { parseNameStatus } = await import("../src/kernel/pin_edits.ts");
698 const { testsWithChangeProblems } = await import("../src/kernel/tests_with_change.ts");
699 const base =
700 process.env.FORGE_CHECK_BASE ||
701 git(["merge-base", "origin/main", "HEAD"], cwd).out ||
702 git(["merge-base", "main", "HEAD"], cwd).out;
703 if (!base) return [];
704 const diff = git(["diff", "--name-status", "-M", `${base}..${judgedHead(cwd)}`], cwd);
705 const log = git(["log", "--format=%B%x00", `${base}..${judgedHead(cwd)}`], cwd);
706 if (!diff.ok || !log.ok) return [];
707 return testsWithChangeProblems(parseNameStatus(diff.out), log.out.split("\0").map((m) => m.trim()).filter(Boolean));
708 }
709
695710 /**
696711 * A pull that changes a screen: the pages the forge pictures for it, never a refusal (#12362,
697712 * #13362).
@@ -813,6 +828,7 @@ export async function gateVerdict(cwd: string = process.cwd()): Promise<{ notes:
813828 ...(await pinRefusal(cwd)),
814829 ...(await tryItRefusal(cwd)),
815830 ...(await bothDoorsRefusal(cwd)),
831 ...(await testsWithChangeRefusal(cwd)),
816832 ...(await directionRefusal(cwd)),
817833 ...strayTestRefusal(cwd),
818834 ...reviewRefusal(cwd),
src/kernel/tests_with_change.ts added +27
@@ -0,0 +1,27 @@
1 /**
2 * Four reviews in a week said "the behaviour this pull adds has no test that
3 * would fail without it" (#14235: #11389, #12484, #14210, #12456). A pull that
4 * adds or edits code under `src/` therefore adds or edits a test under `test/`
5 * too, or says in a commit message `No test: <why>` (a rename, a comment, a
6 * change no behaviour shows). Judged by files, not by coverage: it cannot tell
7 * that the test fails without the line, but it names the pull that has no test
8 * at all, which was the commonest case, and the reviewer keeps the rest.
9 */
10
11 import type { Change } from "./pin_edits.ts";
12
13 const SOURCE = /^src\/(?!.*\/vendor\/).+\.ts$/;
14 const TEST = /^test\/.+\.test\.ts$/;
15 const WAIVER = /^No test:\s*\S[^\n]{9,}/im;
16
17 export function testsWithChangeProblems(changes: readonly Change[], messages: readonly string[]): string[] {
18 const live = changes.filter((c) => !c.status.startsWith("D"));
19 const code = live.filter((c) => SOURCE.test(c.path)).map((c) => c.path);
20 if (code.length === 0) return [];
21 if (live.some((c) => TEST.test(c.path))) return [];
22 if (messages.some((m) => WAIVER.test(m))) return [];
23 const shown = code.slice(0, 3).join(", ") + (code.length > 3 ? ` and ${code.length - 3} more` : "");
24 return [
25 `rule "a behaviour a pull adds has a test that fails without it": this pull changes ${shown} but adds or edits no test/*.test.ts; add a test that fails without the change, or put a "No test: <why>" line (10+ characters) in a commit message when no behaviour changes`,
26 ];
27 }
test/pin-refusal.test.ts +1 −1
@@ -189,7 +189,7 @@ test("a migration added without its pin is named with the one-step fix, last, as
189189 put(dir, "src/core/migrations/0004-d.ts", "d");
190190 put(dir, "src/core/migration-pins/0004-d.pin", "x");
191191 run(dir, "add", "-A");
192 run(dir, "commit", "-qm", "add");
192 run(dir, "commit", "-qm", "add", "-m", "No test: fixture repo for the pin rule");
193193 const out = await withBase(base, () => pinRefusal(dir));
194194 const named =
195195 "this pull adds migration 0003-c without its pin: run bun scripts/pin-migrations.ts and commit src/core/migration-pins/0003-c.pin";
test/tests-with-change.test.ts added +53
@@ -0,0 +1,53 @@
1 import { expect, test } from "bun:test";
2 import { testsWithChangeProblems } from "../src/kernel/tests_with_change.ts";
3 import { testsWithChangeRefusal } from "../scripts/gate.ts";
4 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7
8 const A = (path: string) => ({ status: "A", path });
9 const M = (path: string) => ({ status: "M", path });
10
11 test("a src change with no test is refused, naming the rule and the file", () => {
12 const out = testsWithChangeProblems([M("src/http/api.ts")], ["Add a 403"]);
13 expect(out).toHaveLength(1);
14 expect(out[0]).toContain('rule "a behaviour a pull adds has a test that fails without it"');
15 expect(out[0]).toContain("src/http/api.ts");
16 });
17
18 test("an added or edited test, or a No test line, passes", () => {
19 expect(testsWithChangeProblems([M("src/a.ts"), A("test/a.test.ts")], [])).toEqual([]);
20 expect(testsWithChangeProblems([M("src/a.ts"), M("test/a.test.ts")], [])).toEqual([]);
21 expect(testsWithChangeProblems([M("src/a.ts")], ["x\n\nNo test: a rename, no behaviour moves"])).toEqual([]);
22 expect(testsWithChangeProblems([M("src/a.ts")], ["No test: short"])).toHaveLength(1);
23 });
24
25 test("a deleted test does not count; vendor, docs and test-only changes are not judged", () => {
26 expect(testsWithChangeProblems([M("src/a.ts"), { status: "D", path: "test/a.test.ts" }], [])).toHaveLength(1);
27 expect(testsWithChangeProblems([M("src/vendor/x.ts"), M("docs/a.md"), M("test/a.test.ts")], [])).toEqual([]);
28 });
29
30 test("the gate judges a real commit range from FORGE_CHECK_BASE", async () => {
31 const dir = mkdtempSync(join(tmpdir(), "forge-twc-"));
32 const saved = process.env.FORGE_CHECK_BASE;
33 try {
34 const sh = (...a: string[]) => {
35 const r = Bun.spawnSync(["git", "-c", "user.email=t@t", "-c", "user.name=t", "-c", "commit.gpgsign=false", ...a], { cwd: dir });
36 if (r.exitCode !== 0) throw new Error(r.stderr.toString());
37 return r.stdout.toString().trim();
38 };
39 sh("init", "-q", "-b", "trunk");
40 mkdirSync(join(dir, "src"));
41 writeFileSync(join(dir, "src/a.ts"), "export const a = 1;\n");
42 sh("add", "-A");
43 sh("commit", "-qm", "base");
44 process.env.FORGE_CHECK_BASE = sh("rev-parse", "HEAD");
45 writeFileSync(join(dir, "src/a.ts"), "export const a = 2;\n");
46 sh("commit", "-qam", "change");
47 expect((await testsWithChangeRefusal(dir))[0]).toContain("a behaviour a pull adds has a test");
48 } finally {
49 if (saved === undefined) delete process.env.FORGE_CHECK_BASE;
50 else process.env.FORGE_CHECK_BASE = saved;
51 rmSync(dir, { recursive: true, force: true });
52 }
53 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs