Public: anyone can read this; the crew can change it.
A review that says it could not read the diff is refused, and the second time the pull waits for a person #15012
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/15007-a-review-that-says-it-could-not-read-the → main 6 commits fork 2 threads · 2 unresolved
9 files
changed +238
−9
docs/features/review-unread.md added +11
| @@ -0,0 +1,11 @@ | ||
| 1 | --- | |
| 2 | topic: When something is wrong | |
| 3 | row: A verdict that says it could not read the diff is refused | |
| 4 | --- | |
| 5 | # A verdict that says it could not read the diff is refused (#15007) | |
| 6 | ||
| 7 | A review saying "I could not read the diff", "the head branch is not fetchable" or "this rests on the pull description" is not a verdict. An approve or a request for changes whose body says so is refused (API 422 `unread_diff`, the same sentence in the MCP tool, pull page and terminal) and nothing is recorded, so the pull stays unreviewed. | |
| 8 | ||
| 9 | - **Twice, then a person.** The run records no review, review sends the pull once more, and after the second `reviewStage` is `lapsed`: the pull waits for a person (`waitsForPerson`). No new machinery; the refusal feeds the existing two-tries rule. | |
| 10 | - **Where.** `submitReview` calls `confessesUnread` (`src/actions/review_unread.ts`), a phrase matcher beside `review_unseen.ts`. A plain comment may say it. | |
| 11 | - **Left out.** Phrases only; a review worded another way passes. The first take's `refusedUnread` shared with `recheckedHead` (`src/checks/outside_diff.ts`) is not here: that branch was not reachable and nothing in this change counts runs by reason. |
src/actions/pulls.ts +6 −1
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | * issues — a pull is an issue with a diff. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { confessesUnread } from "./review_unread.ts"; | |
| 7 | 8 | import { confessesUnseen } from "./review_unseen.ts"; |
| 8 | 9 | import { recombine } from "../core/recombine.ts"; |
| 9 | 10 | import { teamsHolding } from "../core/issue_teams.ts"; |
| @@ -710,7 +711,9 @@ export type SubmitReviewActionError = | ||
| 710 | 711 | /** A stylesheet change is approved only beside the home page's looks at its head. */ |
| 711 | 712 | | "looks_required" |
| 712 | 713 | /** The verdict says it was given without opening the preview or its pictures. */ |
| 713 | | "unseen_verdict"; | |
| 714 | | "unseen_verdict" | |
| 715 | /** The verdict says its author could not read the diff (#15007). */ | |
| 716 | | "unread_diff"; | |
| 714 | 717 | |
| 715 | 718 | /** The most line comments one review may carry. */ |
| 716 | 719 | export const MAX_REVIEW_LINES = 50; |
| @@ -763,6 +766,8 @@ export async function submitReview( | ||
| 763 | 766 | |
| 764 | 767 | if (input.state !== "commented" && confessesUnseen(input.body ?? "")) |
| 765 | 768 | return err("unseen_verdict"); |
| 769 | if (input.state !== "commented" && confessesUnread(input.body ?? "")) | |
| 770 | return err("unread_diff"); | |
| 766 | 771 | const lines = input.comments ?? []; |
| 767 | 772 | if (lines.length > MAX_REVIEW_LINES) return err("too_many_lines"); |
| 768 | 773 | // Every anchor is checked before anything is written: a review with one |
src/actions/review_unread.ts added +46
| @@ -0,0 +1,46 @@ | ||
| 1 | /** | |
| 2 | * A verdict does not confess it never read the diff (#15007). A review that says | |
| 3 | * "I could not read the diff", "the head branch is not fetchable" or "this rests | |
| 4 | * on the pull description" judges nothing: it is refused (API 422 `unread_diff`), | |
| 5 | * the run ends without a review, and review sends the pull again. After the | |
| 6 | * second the pull waits for a person (`reviewStage`, `waitsForPerson`). | |
| 7 | * Plain data in, a boolean out; a phrase matcher like `review_unseen.ts`, and | |
| 8 | * only approve and request-changes are held to it. | |
| 9 | */ | |
| 10 | ||
| 11 | // The object follows the verb directly, after at most a determiner: "could not | |
| 12 | // read the diff". A gap would also refuse an honest "could not find any issue in | |
| 13 | // the code", so there is none, and bare `code`, `changes` and `branch` are not | |
| 14 | // objects (#15007). | |
| 15 | const DIFF = String.raw`(?:diff|patch|head\s+(?:branch|commit))`; | |
| 16 | const DET = String.raw`(?:(?:the|this|its|any|a|that)\s+)?`; | |
| 17 | const NOT = String.raw`(?:did\s+not|didn['’]t|could\s+not|couldn['’]t|can\s*not|can['’]t` + | |
| 18 | String.raw`|unable\s+to|was\s+not\s+able\s+to|wasn['’]t\s+able\s+to|failed\s+to)`; | |
| 19 | const ACT = String.raw`(?:read|view|open|fetch|access|load|inspect|retrieve|obtain)`; | |
| 20 | const BAD = String.raw`(?:not|n['’]t|never)\s+(?:\w+\s+)?` + | |
| 21 | String.raw`(?:fetchable|readable|available|accessible|reachable|visible)`; | |
| 22 | ||
| 23 | const CONFESSIONS: readonly RegExp[] = [ | |
| 24 | // "could not read the diff", "unable to fetch the head branch" | |
| 25 | new RegExp(`\\b${NOT}\\s+(?:\\w+ly\\s+)?${ACT}\\s+${DET}${DIFF}\\b`, "i"), | |
| 26 | // "the head branch is not fetchable", "the diff was not available" | |
| 27 | new RegExp(`\\b${DET}${DIFF}\\s+(?:is|was|are|were)\\s+${BAD}\\b`, "i"), | |
| 28 | new RegExp(`\\b${DET}${DIFF}\\s+(?:is|was)\\s+(?:unreadable|unfetchable|unavailable)\\b`, "i"), | |
| 29 | // "no diff was available", "without access to the diff", "without reading the diff" | |
| 30 | new RegExp(`\\bno\\s+${DIFF}\\s+(?:was\\s+|is\\s+)?(?:available|accessible|readable)\\b`, "i"), | |
| 31 | new RegExp( | |
| 32 | `\\bwithout\\s+(?:access\\s+to|reading|seeing|viewing|fetching|opening)\\s+${DET}${DIFF}\\b`, | |
| 33 | "i", | |
| 34 | ), | |
| 35 | // "rests on the pull description", "based only on the description" | |
| 36 | new RegExp( | |
| 37 | String.raw`\b(?:rests?|resting|based|relies|relying|judged?|going)\s+(?:only\s+)?(?:on|upon|from)\s+` + | |
| 38 | String.raw`(?:only\s+)?the\s+(?:pull(?:\s+request)?\s+|pr\s+)?(?:description|title|summary)\b`, | |
| 39 | "i", | |
| 40 | ), | |
| 41 | ]; | |
| 42 | ||
| 43 | /** Does this review body say its author could not read the diff? */ | |
| 44 | export function confessesUnread(body: string): boolean { | |
| 45 | return CONFESSIONS.some((re) => re.test(body)); | |
| 46 | } |
src/actions/review_words.ts +5
| @@ -20,4 +20,9 @@ export const reviewWords = { | ||
| 20 | 20 | */ |
| 21 | 21 | unseenVerdict: |
| 22 | 22 | 'Rule "a verdict is given after opening the preview and its pictures": this review says it did not. Open the preview and read its pictures, then say what you saw.', |
| 23 | /** | |
| 24 | * `unread_diff`: a verdict says its author could not read the diff (#15007). | |
| 25 | */ | |
| 26 | unreadDiff: | |
| 27 | "Rule \"a verdict is given on the diff you read\": this review says it could not read the diff, so it is no verdict and nothing was recorded. Fetch the head branch and read the diff, then review it; if you cannot, say so in a comment, not a verdict.", | |
| 23 | 28 | } as const; |
src/http/api.ts +2
| @@ -3347,6 +3347,8 @@ export function apiRoutes(router: Router<Handler>, isPage: (path: string) => boo | ||
| 3347 | 3347 | ); |
| 3348 | 3348 | if (submitted.error === "unseen_verdict") |
| 3349 | 3349 | return apiError("unseen_verdict", 422, reviewWords.unseenVerdict); |
| 3350 | if (submitted.error === "unread_diff") | |
| 3351 | return apiError("unread_diff", 422, reviewWords.unreadDiff); | |
| 3350 | 3352 | if (submitted.error === "looks_required") |
| 3351 | 3353 | return apiError("looks_required", 422, reviewWords.looksRequired); |
| 3352 | 3354 | if (submitted.error === "bad_anchor") |
src/http/mcp/tools/review.ts +5 −3
| @@ -55,9 +55,11 @@ export const tool: Tool = { | ||
| 55 | 55 | ? reviewWords.tooLong |
| 56 | 56 | : error === "unseen_verdict" |
| 57 | 57 | ? reviewWords.unseenVerdict |
| 58 | : error === "looks_required" | |
| 59 | ? reviewWords.looksRequired | |
| 60 | : "That review could not be recorded."), | |
| 58 | : error === "unread_diff" | |
| 59 | ? reviewWords.unreadDiff | |
| 60 | : error === "looks_required" | |
| 61 | ? reviewWords.looksRequired | |
| 62 | : "That review could not be recorded."), | |
| 61 | 63 | ); |
| 62 | 64 | } |
| 63 | 65 | return good({ |
src/tui/screens/common.ts +1
| @@ -1032,6 +1032,7 @@ const MESSAGES: Record<string, string> = { | ||
| 1032 | 1032 | self_review: "you cannot approve your own change", |
| 1033 | 1033 | invalid_state: "that is not a review verdict", |
| 1034 | 1034 | unseen_verdict: "a verdict is given after opening the preview and its pictures: say what you saw", |
| 1035 | unread_diff: "a verdict is given on the diff you read: this one says it could not", | |
| 1035 | 1036 | looks_required: "a stylesheet change waits for its home-page looks at phone and desktop width", |
| 1036 | 1037 | // The fork flow. Each of these leaves the world in a different place, so |
| 1037 | 1038 | // each keeps its own sentence. |
src/web/pages/pull.ts +7 −5
| @@ -1518,11 +1518,13 @@ export async function pullReviewStateSubmit(ctx: Ctx): Promise<Response> { | ||
| 1518 | 1518 | ? reviewWords.tooLong |
| 1519 | 1519 | : error === "unseen_verdict" |
| 1520 | 1520 | ? reviewWords.unseenVerdict |
| 1521 | : error === "looks_required" | |
| 1522 | ? reviewWords.looksRequired | |
| 1523 | : error === "invalid_state" | |
| 1524 | ? "That is not a verdict this forge knows." | |
| 1525 | : "That review could not be recorded."), | |
| 1521 | : error === "unread_diff" | |
| 1522 | ? reviewWords.unreadDiff | |
| 1523 | : error === "looks_required" | |
| 1524 | ? reviewWords.looksRequired | |
| 1525 | : error === "invalid_state" | |
| 1526 | ? "That is not a verdict this forge knows." | |
| 1527 | : "That review could not be recorded."), | |
| 1526 | 1528 | }); |
| 1527 | 1529 | } |
| 1528 | 1530 | return flashTo(ctx, `${dest}#conversation`, { |
test/review-unread.test.ts added +155
| @@ -0,0 +1,155 @@ | ||
| 1 | /** | |
| 2 | * A verdict that says it could not read the diff is refused (#15007), and two such | |
| 3 | * reviews leave the pull waiting for a person. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { confessesUnread } from "../src/actions/review_unread.ts"; | |
| 10 | import { reviewWords } from "../src/actions/review_words.ts"; | |
| 11 | import { openPull, submitReview } from "../src/actions/pulls.ts"; | |
| 12 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 13 | import { createApp, type App } from "../src/app.ts"; | |
| 14 | import { listReviews } from "../src/core/reviews.ts"; | |
| 15 | import { createToken } from "../src/core/tokens.ts"; | |
| 16 | import { createUser } from "../src/core/users.ts"; | |
| 17 | import { reviewStage } from "../src/crew/review.ts"; | |
| 18 | import { errorText } from "../src/tui/screens/common.ts"; | |
| 19 | import { serve } from "../src/http/server.ts"; | |
| 20 | import { quietLogs } from "./helpers/globals.ts"; | |
| 21 | ||
| 22 | quietLogs(); | |
| 23 | ||
| 24 | const UNREAD = [ | |
| 25 | "I could not read the diff, so this rests on the pull description. Looks fine.", | |
| 26 | "The head branch is not fetchable from here; approving from the description.", | |
| 27 | "Unable to fetch the head branch. Changes needed.", | |
| 28 | "I couldn't view the diff.", | |
| 29 | "Reviewed without access to the diff.", | |
| 30 | "This approval rests on the pull description.", | |
| 31 | "The diff was not available.", | |
| 32 | ]; | |
| 33 | ||
| 34 | const HONEST = [ | |
| 35 | "I read the diff: the loop in a.ts never ends.", | |
| 36 | "", | |
| 37 | "The diff does not read the config twice.", | |
| 38 | "Fetched the head branch; the test is missing a case.", | |
outdated
Add honest sentences that must pass: "I could not find any issue in the code.", "I did not see a problem in the changes." These fail today. |
||
| 39 | "Do not fetch the branch inside the loop; it is slow.", | |
| 40 | "The description matches the diff.", | |
| 41 | "I could not find any issue in the code.", | |
| 42 | "I did not see a problem in the changes.", | |
| 43 | "I could not find a bug in the code.", | |
| 44 | "Did not check out the branch locally, but the diff is clean.", | |
| 45 | "I could not get this to fail; the patch is fine.", | |
| 46 | ]; | |
| 47 | ||
| 48 | test("the ways of saying the diff was unreadable are caught", () => { | |
| 49 | for (const b of UNREAD) expect([b, confessesUnread(b)]).toEqual([b, true]); | |
| 50 | }); | |
| 51 | ||
| 52 | test("a review that read the diff, or says nothing, passes", () => { | |
| 53 | for (const b of HONEST) expect([b, confessesUnread(b)]).toEqual([b, false]); | |
| 54 | }); | |
| 55 | ||
| 56 | test("two reviews refused leave the pull waiting: one is sent again, then it lapses", () => { | |
| 57 | const green = "2026-10-13T03:00:00.000Z"; | |
| 58 | const one = [{ createdAt: "2026-10-13T03:01:00.000Z", active: false }]; | |
| 59 | const two = [...one, { createdAt: "2026-10-13T03:10:00.000Z", active: false }]; | |
| 60 | const now = new Date("2026-10-13T03:20:00.000Z"); | |
| 61 | expect(reviewStage(green, one, now)).toBe("send"); | |
| 62 | expect(reviewStage(green, two, now)).toBe("lapsed"); | |
| 63 | }); | |
| 64 | ||
| 65 | let dir = ""; | |
| 66 | let app: App; | |
| 67 | beforeAll(async () => { | |
| 68 | dir = await mkdtemp(join(tmpdir(), "forge-unread-")); | |
| 69 | app = createApp({ dataDir: dir, port: 0 }); | |
| 70 | }); | |
| 71 | afterAll(async () => { | |
| 72 | app.close(); | |
| 73 | await rm(dir, { recursive: true, force: true }); | |
| 74 | }); | |
| 75 | ||
| 76 | test("submitReview and the API refuse it, a comment may say it, nothing is recorded", async () => { | |
| 77 | const made = await createUser(app.db, { | |
| 78 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 79 | password: "a-good-passphrase", adminIfFirst: true, | |
| 80 | }); | |
| 81 | if (!made.ok) throw new Error(String(made.error)); | |
| 82 | const alice = made.value; | |
| 83 | const repo = await createRepo(app, alice, { name: "r", visibility: "public" }); | |
| 84 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 85 | for (const [branch, file] of [[undefined, "forge.json"], ["topic", "t.txt"]] as const) { | |
| 86 | const done = await commitToBranch(app, alice, repo.value, { | |
| 87 | ...(branch === undefined ? {} : { branch }), | |
| 88 | message: "change", | |
| 89 | files: [{ path: file, content: file === "forge.json" ? "{}" : "x" }], | |
| 90 | }); | |
| 91 | if (!done.ok) throw new Error(JSON.stringify(done.error)); | |
| 92 | } | |
| 93 | const opened = await openPull(app, alice, repo.value, { title: "t", body: "", baseBranch: "main", headBranch: "topic" }); | |
| 94 | if (!opened.ok) throw new Error(JSON.stringify(opened.error)); | |
| 95 | const pull = opened.value; | |
| 96 | ||
| 97 | expect(await submitReview(app, alice, repo.value, pull, { state: "approved", body: UNREAD[0]! })).toEqual({ | |
| 98 | ok: false, | |
| 99 | error: "unread_diff", | |
| 100 | }); | |
| 101 | expect(listReviews(app.db, pull.id)).toHaveLength(0); | |
| 102 | ||
| 103 | const servers = await serve(app); | |
| 104 | try { | |
| 105 | const token = createToken(app.db, alice.id, "t", "admin").token; | |
| 106 | const res = await fetch( | |
| 107 | `${servers.url}/api/v1/repos/${repo.value.ownerName}/${repo.value.name}/pulls/${pull.number}/review-states`, | |
| 108 | { | |
| 109 | method: "POST", | |
| 110 | headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, | |
| 111 | body: JSON.stringify({ state: "changes_requested", body: UNREAD[1] }), | |
| 112 | }, | |
| 113 | ); | |
| 114 | expect(res.status).toBe(422); | |
| 115 | expect(((await res.json()) as { error: string }).error).toBe("unread_diff"); | |
| 116 | } finally { | |
| 117 | servers.stop(); | |
| 118 | } | |
| 119 | expect(reviewWords.unreadDiff).toContain("could not read the diff"); | |
| 120 | expect(listReviews(app.db, pull.id)).toHaveLength(0); | |
| 121 | ||
| 122 | const servers2 = await serve(app); | |
| 123 | try { | |
| 124 | const token = createToken(app.db, alice.id, "t2", "admin").token; | |
| 125 | const auth = { authorization: `Bearer ${token}` }; | |
| 126 | const mcp = await fetch(`${servers2.url}/mcp`, { | |
| 127 | method: "POST", | |
| 128 | headers: { ...auth, "content-type": "application/json", accept: "application/json, text/event-stream" }, | |
| 129 | body: JSON.stringify({ | |
| 130 | jsonrpc: "2.0", id: 1, method: "tools/call", | |
| 131 | params: { name: "review", arguments: { repo: `${repo.value.ownerName}/${repo.value.name}`, number: pull.number, verdict: "changes", reason: UNREAD[1] } }, | |
| 132 | }), | |
| 133 | }); | |
| 134 | const mcpBody = (await mcp.json()) as { result: { isError: boolean; content: { text: string }[] } }; | |
| 135 | expect(mcpBody.result.isError).toBe(true); | |
| 136 | expect(mcpBody.result.content[0]!.text).toContain(reviewWords.unreadDiff); | |
| 137 | ||
| 138 | const form = await fetch(`${servers2.url}/${repo.value.ownerName}/${repo.value.name}/pulls/${pull.number}/review-state`, { | |
| 139 | method: "POST", | |
| 140 | redirect: "manual", | |
| 141 | headers: { ...auth, "content-type": "application/x-www-form-urlencoded" }, | |
| 142 | body: new URLSearchParams({ state: "changes_requested", body: UNREAD[1]! }), | |
| 143 | }); | |
| 144 | expect(form.status).toBe(303); | |
| 145 | const cookies = decodeURIComponent(form.headers.getSetCookie().join(";")).replace(/\+/g, " "); | |
| 146 | expect(cookies).toContain("could not read the diff"); | |
| 147 | } finally { | |
| 148 | servers2.stop(); | |
| 149 | } | |
| 150 | expect(errorText("unread_diff")).toContain("could not"); | |
| 151 | expect(listReviews(app.db, pull.id)).toHaveLength(0); | |
| 152 | ||
| 153 | const note = await submitReview(app, alice, repo.value, pull, { state: "commented", body: UNREAD[0]! }); | |
| 154 | expect(note.ok).toBe(true); | |
| 155 | }); |
new above
offline
False positive: "could not find a bug in the code" matches (ACT includes find/see, GAP allows 40 chars, DIFF includes bare
code/changes). Narrow the verbs and nouns and drop the gap, as in the review body.