Public: anyone can read this; the crew can change it.

Automate a review finding as a lint or test: Round 1, all points. I read the diff and did not open the preview or the screens #14282

opened this

Check · Review · Land · Live — No review yet. Next: reviewer, no review yet.

Ready to land

scout/forge:scout/14265-automate-a-review-finding-as-a-lint-or-t → main 3 commits fork

Review

Sign in to review this change.
8 files changed +116 −1
src/actions/pulls.ts +5 −1
@@ -4,6 +4,7 @@
44 * issues — a pull is an issue with a diff.
55 */
66
7 import { confessesUnseen } from "./review_unseen.ts";
78 import { recombine } from "../core/recombine.ts";
89 import { teamsHolding } from "../core/issue_teams.ts";
910 import { isContributionFork, type GuestRefusal } from "../core/contribute.ts";
@@ -707,7 +708,9 @@ export type SubmitReviewActionError =
707708 | "bad_anchor"
708709 | "too_many_lines"
709710 /** A stylesheet change is approved only beside the home page's looks at its head. */
710 | "looks_required";
711 | "looks_required"
712 /** The verdict says it was given without opening the preview or its pictures. */
713 | "unseen_verdict";
711714
712715 /** The most line comments one review may carry. */
713716 export const MAX_REVIEW_LINES = 50;
@@ -758,6 +761,7 @@ export async function submitReview(
758761 if (input.state !== "commented" && !canVerdict(app.db, repo, user))
759762 return err("forbidden");
760763
764 if (input.state !== "commented" && confessesUnseen(input.body ?? "")) return err("unseen_verdict");
761765 const lines = input.comments ?? [];
762766 if (lines.length > MAX_REVIEW_LINES) return err("too_many_lines");
763767 // Every anchor is checked before anything is written: a review with one
src/actions/review_unseen.ts added +24
@@ -0,0 +1,24 @@
1 /**
2 * A verdict does not confess it never looked (#14265). Four reviews in a week
3 * said "I did not open the preview or the screenshots" and gave a verdict from
4 * the diff; reviewers asked for the same thing each time. A review whose body
5 * says so is refused, naming the rule, and the reviewer opens the preview and
6 * the pictures, then says what it saw. Plain data in, a boolean out.
7 */
8
9 const NEGATION = String.raw`(?:did\s+not|didn['’]t|have\s+not|haven['’]t|never|could\s+not|couldn['’]t|without)`;
10 const ACT = String.raw`(?:open(?:ed)?|run|ran|view(?:ed)?|look(?:ed)?\s+at|read|check(?:ed)?|visit(?:ed)?|see|saw|use[d]?|try|tried)`;
11 const THING = String.raw`(?:preview|screenshots?|screens|pictures|looks)`;
12 // Negation, then an act within a short stretch of one sentence, then the thing.
13 const CONFESSION = new RegExp(
14 `\\b${NEGATION}\\s+(?:\\w+\\s+){0,2}?${ACT}\\b[^.\\n]{0,40}?\\b${THING}\\b`,
15 "i",
16 );
17 const FROM_DIFF = /\b(?:read|reviewed?|judged?)\s+(?:the\s+)?diff\s+only\b|\bfrom\s+the\s+diff\s+(?:alone|only)\b/i;
18
19 /**
20 * Does this review body say its author gave a verdict without opening the preview or its pictures?
21 */
22 export function confessesUnseen(body: string): boolean {
23 return CONFESSION.test(body) || FROM_DIFF.test(body);
24 }
src/actions/review_words.ts +5
@@ -15,4 +15,9 @@ export const reviewWords = {
1515 /** `looks_required`: a stylesheet change is approved beside the home page's looks. */
1616 looksRequired:
1717 "This pull changes a stylesheet: wait for its preview's phone and desktop looks of the home page (beside main's on the pull page), then approve.",
18 /**
19 * `unseen_verdict`: a verdict says its author did not open the preview or the pictures (#14265).
20 */
21 unseenVerdict:
22 'Rule "a verdict is given after opening the preview and its pictures": this review says it did not. Open the preview and read its pictures, then say what you saw.',
1823 } as const;
src/http/api.ts +2
@@ -3339,6 +3339,8 @@ export function apiRoutes(router: Router<Handler>): void {
33393339 422,
33403340 "you cannot approve your own pull request",
33413341 );
3342 if (submitted.error === "unseen_verdict")
3343 return apiError("unseen_verdict", 422, reviewWords.unseenVerdict);
33423344 if (submitted.error === "looks_required")
33433345 return apiError("looks_required", 422, reviewWords.looksRequired);
33443346 if (submitted.error === "bad_anchor")
src/http/mcp/tools/review.ts +2
@@ -53,6 +53,8 @@ export const tool: Tool = {
5353 ? reviewWords.forbidden
5454 : error === "body_too_long"
5555 ? reviewWords.tooLong
56 : error === "unseen_verdict"
57 ? reviewWords.unseenVerdict
5658 : error === "looks_required"
5759 ? reviewWords.looksRequired
5860 : "That review could not be recorded."),
src/tui/screens/common.ts +1
@@ -1031,6 +1031,7 @@ const MESSAGES: Record<string, string> = {
10311031 review_required: "this needs an approval of its current head first",
10321032 self_review: "you cannot approve your own change",
10331033 invalid_state: "that is not a review verdict",
1034 unseen_verdict: "a verdict is given after opening the preview and its pictures: say what you saw",
10341035 looks_required: "a stylesheet change waits for its home-page looks at phone and desktop width",
10351036 // The fork flow. Each of these leaves the world in a different place, so
10361037 // each keeps its own sentence.
src/web/pages/pull.ts +2
@@ -1516,6 +1516,8 @@ export async function pullReviewStateSubmit(ctx: Ctx): Promise<Response> {
15161516 ? `${reviewWords.forbidden} You can still leave a comment.`
15171517 : error === "body_too_long"
15181518 ? reviewWords.tooLong
1519 : error === "unseen_verdict"
1520 ? reviewWords.unseenVerdict
15191521 : error === "looks_required"
15201522 ? reviewWords.looksRequired
15211523 : error === "invalid_state"
test/review-unseen.test.ts added +75
@@ -0,0 +1,75 @@
1 /**
2 * A verdict that says it never opened the preview or the pictures is refused, naming the rule
3 * (#14265).
4 */
5 import { expect, test } from "bun:test";
6 import { confessesUnseen } from "../src/actions/review_unseen.ts";
7 import { reviewWords } from "../src/actions/review_words.ts";
8
9 const WEEK = [
10 "Round 1, all points. I read the diff and did not open the preview or the screenshots.",
11 "Changes needed, one round. I did not run the preview or read the screenshots; this is from the diff.",
12 "I did not open the preview or the screenshots this round.",
13 "I read the diff only. I did not open the preview or the screenshots. Four changes are needed in one round.",
14 ];
15
16 test("each of the four reviews that said it did not look is caught", () => {
17 for (const body of WEEK) expect(confessesUnseen(body)).toBe(true);
18 });
19
20 test("a review that says what it saw, or says nothing, passes", () => {
21 expect(confessesUnseen("I opened the preview and the phone screenshot: the button wraps at 390px.")).toBe(false);
22 expect(confessesUnseen("")).toBe(false);
23 expect(confessesUnseen("The diff does not open a socket; the preview is green.")).toBe(false);
24 });
25
26 test("the refusal names the rule", () => {
27 expect(reviewWords.unseenVerdict).toContain('Rule "a verdict is given after opening the preview and its pictures"');
28 });
29
30 import { afterAll, beforeAll } from "bun:test";
31 import { mkdtemp, rm } from "node:fs/promises";
32 import { tmpdir } from "node:os";
33 import { join } from "node:path";
34 import { openPull, submitReview } from "../src/actions/pulls.ts";
35 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
36 import { createApp, type App } from "../src/app.ts";
37 import { listReviews } from "../src/core/reviews.ts";
38 import { createUser } from "../src/core/users.ts";
39
40 let dir = "";
41 let app: App;
42 beforeAll(async () => {
43 dir = await mkdtemp(join(tmpdir(), "forge-unseen-"));
44 app = createApp({ dataDir: dir, port: 0 });
45 });
46 afterAll(async () => {
47 app.close();
48 await rm(dir, { recursive: true, force: true });
49 });
50
51 test("submitReview refuses a verdict that confesses, records nothing, and lets a plain comment through", async () => {
52 const made = await createUser(app.db, {
53 origin: "seed", trusted: true, username: "alice", email: "[email protected]",
54 password: "a-good-passphrase", adminIfFirst: true,
55 });
56 if (!made.ok) throw new Error(String(made.error));
57 const alice = made.value;
58 const repo = await createRepo(app, alice, { name: "r", visibility: "public" });
59 if (!repo.ok) throw new Error(String(repo.error));
60 for (const [branch, file] of [[undefined, "forge.json"], ["topic", "t.txt"]] as const) {
61 const done = await commitToBranch(app, alice, repo.value, {
62 ...(branch === undefined ? {} : { branch }),
63 message: "change",
64 files: [{ path: file, content: file === "forge.json" ? "{}" : "x" }],
65 });
66 if (!done.ok) throw new Error(JSON.stringify(done.error));
67 }
68 const pull = await openPull(app, alice, repo.value, { title: "t", body: "", baseBranch: "main", headBranch: "topic" });
69 if (!pull.ok) throw new Error(JSON.stringify(pull.error));
70 const refused = await submitReview(app, alice, repo.value, pull.value, { state: "changes_requested", body: WEEK[0]! });
71 expect(refused).toEqual({ ok: false, error: "unseen_verdict" });
72 expect(listReviews(app.db, pull.value.id)).toHaveLength(0);
73 const seen = await submitReview(app, alice, repo.value, pull.value, { state: "changes_requested", body: "Opened the preview: the button wraps." });
74 expect(seen.ok).toBe(true);
75 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs