Public: anyone can read this; the crew can change it.
Automate a review finding as a lint or test: Round 1, all points. I read the diff and did not open the preview or the screens #14282
Check · Review · Land · Live — No review yet.
scout/forge:scout/14265-automate-a-review-finding-as-a-lint-or-t → main 3 commits fork
8 files
changed +116
−1
src/actions/pulls.ts +5 −1
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | * issues — a pull is an issue with a diff. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { confessesUnseen } from "./review_unseen.ts"; | |
| 7 | 8 | import { recombine } from "../core/recombine.ts"; |
| 8 | 9 | import { teamsHolding } from "../core/issue_teams.ts"; |
| 9 | 10 | import { isContributionFork, type GuestRefusal } from "../core/contribute.ts"; |
| @@ -707,7 +708,9 @@ export type SubmitReviewActionError = | ||
| 707 | 708 | | "bad_anchor" |
| 708 | 709 | | "too_many_lines" |
| 709 | 710 | /** A stylesheet change is approved only beside the home page's looks at its head. */ |
| 710 | | "looks_required"; | |
| 711 | | "looks_required" | |
| 712 | /** The verdict says it was given without opening the preview or its pictures. */ | |
| 713 | | "unseen_verdict"; | |
| 711 | 714 | |
| 712 | 715 | /** The most line comments one review may carry. */ |
| 713 | 716 | export const MAX_REVIEW_LINES = 50; |
| @@ -758,6 +761,7 @@ export async function submitReview( | ||
| 758 | 761 | if (input.state !== "commented" && !canVerdict(app.db, repo, user)) |
| 759 | 762 | return err("forbidden"); |
| 760 | 763 | |
| 764 | if (input.state !== "commented" && confessesUnseen(input.body ?? "")) return err("unseen_verdict"); | |
| 761 | 765 | const lines = input.comments ?? []; |
| 762 | 766 | if (lines.length > MAX_REVIEW_LINES) return err("too_many_lines"); |
| 763 | 767 | // Every anchor is checked before anything is written: a review with one |
src/actions/review_unseen.ts added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | /** | |
| 2 | * A verdict does not confess it never looked (#14265). Four reviews in a week | |
| 3 | * said "I did not open the preview or the screenshots" and gave a verdict from | |
| 4 | * the diff; reviewers asked for the same thing each time. A review whose body | |
| 5 | * says so is refused, naming the rule, and the reviewer opens the preview and | |
| 6 | * the pictures, then says what it saw. Plain data in, a boolean out. | |
| 7 | */ | |
| 8 | ||
| 9 | const NEGATION = String.raw`(?:did\s+not|didn['’]t|have\s+not|haven['’]t|never|could\s+not|couldn['’]t|without)`; | |
| 10 | const ACT = String.raw`(?:open(?:ed)?|run|ran|view(?:ed)?|look(?:ed)?\s+at|read|check(?:ed)?|visit(?:ed)?|see|saw|use[d]?|try|tried)`; | |
| 11 | const THING = String.raw`(?:preview|screenshots?|screens|pictures|looks)`; | |
| 12 | // Negation, then an act within a short stretch of one sentence, then the thing. | |
| 13 | const CONFESSION = new RegExp( | |
| 14 | `\\b${NEGATION}\\s+(?:\\w+\\s+){0,2}?${ACT}\\b[^.\\n]{0,40}?\\b${THING}\\b`, | |
| 15 | "i", | |
| 16 | ); | |
| 17 | const FROM_DIFF = /\b(?:read|reviewed?|judged?)\s+(?:the\s+)?diff\s+only\b|\bfrom\s+the\s+diff\s+(?:alone|only)\b/i; | |
| 18 | ||
| 19 | /** | |
| 20 | * Does this review body say its author gave a verdict without opening the preview or its pictures? | |
| 21 | */ | |
| 22 | export function confessesUnseen(body: string): boolean { | |
| 23 | return CONFESSION.test(body) || FROM_DIFF.test(body); | |
| 24 | } |
src/actions/review_words.ts +5
| @@ -15,4 +15,9 @@ export const reviewWords = { | ||
| 15 | 15 | /** `looks_required`: a stylesheet change is approved beside the home page's looks. */ |
| 16 | 16 | looksRequired: |
| 17 | 17 | "This pull changes a stylesheet: wait for its preview's phone and desktop looks of the home page (beside main's on the pull page), then approve.", |
| 18 | /** | |
| 19 | * `unseen_verdict`: a verdict says its author did not open the preview or the pictures (#14265). | |
| 20 | */ | |
| 21 | unseenVerdict: | |
| 22 | 'Rule "a verdict is given after opening the preview and its pictures": this review says it did not. Open the preview and read its pictures, then say what you saw.', | |
| 18 | 23 | } as const; |
src/http/api.ts +2
| @@ -3339,6 +3339,8 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 3339 | 3339 | 422, |
| 3340 | 3340 | "you cannot approve your own pull request", |
| 3341 | 3341 | ); |
| 3342 | if (submitted.error === "unseen_verdict") | |
| 3343 | return apiError("unseen_verdict", 422, reviewWords.unseenVerdict); | |
| 3342 | 3344 | if (submitted.error === "looks_required") |
| 3343 | 3345 | return apiError("looks_required", 422, reviewWords.looksRequired); |
| 3344 | 3346 | if (submitted.error === "bad_anchor") |
src/http/mcp/tools/review.ts +2
| @@ -53,6 +53,8 @@ export const tool: Tool = { | ||
| 53 | 53 | ? reviewWords.forbidden |
| 54 | 54 | : error === "body_too_long" |
| 55 | 55 | ? reviewWords.tooLong |
| 56 | : error === "unseen_verdict" | |
| 57 | ? reviewWords.unseenVerdict | |
| 56 | 58 | : error === "looks_required" |
| 57 | 59 | ? reviewWords.looksRequired |
| 58 | 60 | : "That review could not be recorded."), |
src/tui/screens/common.ts +1
| @@ -1031,6 +1031,7 @@ const MESSAGES: Record<string, string> = { | ||
| 1031 | 1031 | review_required: "this needs an approval of its current head first", |
| 1032 | 1032 | self_review: "you cannot approve your own change", |
| 1033 | 1033 | invalid_state: "that is not a review verdict", |
| 1034 | unseen_verdict: "a verdict is given after opening the preview and its pictures: say what you saw", | |
| 1034 | 1035 | looks_required: "a stylesheet change waits for its home-page looks at phone and desktop width", |
| 1035 | 1036 | // The fork flow. Each of these leaves the world in a different place, so |
| 1036 | 1037 | // each keeps its own sentence. |
src/web/pages/pull.ts +2
| @@ -1516,6 +1516,8 @@ export async function pullReviewStateSubmit(ctx: Ctx): Promise<Response> { | ||
| 1516 | 1516 | ? `${reviewWords.forbidden} You can still leave a comment.` |
| 1517 | 1517 | : error === "body_too_long" |
| 1518 | 1518 | ? reviewWords.tooLong |
| 1519 | : error === "unseen_verdict" | |
| 1520 | ? reviewWords.unseenVerdict | |
| 1519 | 1521 | : error === "looks_required" |
| 1520 | 1522 | ? reviewWords.looksRequired |
| 1521 | 1523 | : error === "invalid_state" |
test/review-unseen.test.ts added +75
| @@ -0,0 +1,75 @@ | ||
| 1 | /** | |
| 2 | * A verdict that says it never opened the preview or the pictures is refused, naming the rule | |
| 3 | * (#14265). | |
| 4 | */ | |
| 5 | import { expect, test } from "bun:test"; | |
| 6 | import { confessesUnseen } from "../src/actions/review_unseen.ts"; | |
| 7 | import { reviewWords } from "../src/actions/review_words.ts"; | |
| 8 | ||
| 9 | const WEEK = [ | |
| 10 | "Round 1, all points. I read the diff and did not open the preview or the screenshots.", | |
| 11 | "Changes needed, one round. I did not run the preview or read the screenshots; this is from the diff.", | |
| 12 | "I did not open the preview or the screenshots this round.", | |
| 13 | "I read the diff only. I did not open the preview or the screenshots. Four changes are needed in one round.", | |
| 14 | ]; | |
| 15 | ||
| 16 | test("each of the four reviews that said it did not look is caught", () => { | |
| 17 | for (const body of WEEK) expect(confessesUnseen(body)).toBe(true); | |
| 18 | }); | |
| 19 | ||
| 20 | test("a review that says what it saw, or says nothing, passes", () => { | |
| 21 | expect(confessesUnseen("I opened the preview and the phone screenshot: the button wraps at 390px.")).toBe(false); | |
| 22 | expect(confessesUnseen("")).toBe(false); | |
| 23 | expect(confessesUnseen("The diff does not open a socket; the preview is green.")).toBe(false); | |
| 24 | }); | |
| 25 | ||
| 26 | test("the refusal names the rule", () => { | |
| 27 | expect(reviewWords.unseenVerdict).toContain('Rule "a verdict is given after opening the preview and its pictures"'); | |
| 28 | }); | |
| 29 | ||
| 30 | import { afterAll, beforeAll } from "bun:test"; | |
| 31 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 32 | import { tmpdir } from "node:os"; | |
| 33 | import { join } from "node:path"; | |
| 34 | import { openPull, submitReview } from "../src/actions/pulls.ts"; | |
| 35 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 36 | import { createApp, type App } from "../src/app.ts"; | |
| 37 | import { listReviews } from "../src/core/reviews.ts"; | |
| 38 | import { createUser } from "../src/core/users.ts"; | |
| 39 | ||
| 40 | let dir = ""; | |
| 41 | let app: App; | |
| 42 | beforeAll(async () => { | |
| 43 | dir = await mkdtemp(join(tmpdir(), "forge-unseen-")); | |
| 44 | app = createApp({ dataDir: dir, port: 0 }); | |
| 45 | }); | |
| 46 | afterAll(async () => { | |
| 47 | app.close(); | |
| 48 | await rm(dir, { recursive: true, force: true }); | |
| 49 | }); | |
| 50 | ||
| 51 | test("submitReview refuses a verdict that confesses, records nothing, and lets a plain comment through", async () => { | |
| 52 | const made = await createUser(app.db, { | |
| 53 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 54 | password: "a-good-passphrase", adminIfFirst: true, | |
| 55 | }); | |
| 56 | if (!made.ok) throw new Error(String(made.error)); | |
| 57 | const alice = made.value; | |
| 58 | const repo = await createRepo(app, alice, { name: "r", visibility: "public" }); | |
| 59 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 60 | for (const [branch, file] of [[undefined, "forge.json"], ["topic", "t.txt"]] as const) { | |
| 61 | const done = await commitToBranch(app, alice, repo.value, { | |
| 62 | ...(branch === undefined ? {} : { branch }), | |
| 63 | message: "change", | |
| 64 | files: [{ path: file, content: file === "forge.json" ? "{}" : "x" }], | |
| 65 | }); | |
| 66 | if (!done.ok) throw new Error(JSON.stringify(done.error)); | |
| 67 | } | |
| 68 | const pull = await openPull(app, alice, repo.value, { title: "t", body: "", baseBranch: "main", headBranch: "topic" }); | |
| 69 | if (!pull.ok) throw new Error(JSON.stringify(pull.error)); | |
| 70 | const refused = await submitReview(app, alice, repo.value, pull.value, { state: "changes_requested", body: WEEK[0]! }); | |
| 71 | expect(refused).toEqual({ ok: false, error: "unseen_verdict" }); | |
| 72 | expect(listReviews(app.db, pull.value.id)).toHaveLength(0); | |
| 73 | const seen = await submitReview(app, alice, repo.value, pull.value, { state: "changes_requested", body: "Opened the preview: the button wraps." }); | |
| 74 | expect(seen.ok).toBe(true); | |
| 75 | }); |
new above
offline