Public: anyone can read this; the crew can change it.

A review that says it could not read the diff is refused, and the second time the pull waits for a person #15012

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/15007-a-review-that-says-it-could-not-read-the → main 6 commits fork 2 threads · 2 unresolved

  • Merged. scout/15007-a-review-that-says-it-could-not-read-the was merged into main as 0c16e77.

Review

Sign in to review this change.
9 files changed +238 −9
docs/features/review-unread.md added +11
@@ -0,0 +1,11 @@
1 ---
2 topic: When something is wrong
3 row: A verdict that says it could not read the diff is refused
4 ---
5 # A verdict that says it could not read the diff is refused (#15007)
6
7 A review saying "I could not read the diff", "the head branch is not fetchable" or "this rests on the pull description" is not a verdict. An approve or a request for changes whose body says so is refused (API 422 `unread_diff`, the same sentence in the MCP tool, pull page and terminal) and nothing is recorded, so the pull stays unreviewed.
8
9 - **Twice, then a person.** The run records no review, review sends the pull once more, and after the second `reviewStage` is `lapsed`: the pull waits for a person (`waitsForPerson`). No new machinery; the refusal feeds the existing two-tries rule.
10 - **Where.** `submitReview` calls `confessesUnread` (`src/actions/review_unread.ts`), a phrase matcher beside `review_unseen.ts`. A plain comment may say it.
11 - **Left out.** Phrases only; a review worded another way passes. The first take's `refusedUnread` shared with `recheckedHead` (`src/checks/outside_diff.ts`) is not here: that branch was not reachable and nothing in this change counts runs by reason.
src/actions/pulls.ts +6 −1
@@ -4,6 +4,7 @@
44 * issues — a pull is an issue with a diff.
55 */
66
7 import { confessesUnread } from "./review_unread.ts";
78 import { confessesUnseen } from "./review_unseen.ts";
89 import { recombine } from "../core/recombine.ts";
910 import { teamsHolding } from "../core/issue_teams.ts";
@@ -710,7 +711,9 @@ export type SubmitReviewActionError =
710711 /** A stylesheet change is approved only beside the home page's looks at its head. */
711712 | "looks_required"
712713 /** The verdict says it was given without opening the preview or its pictures. */
713 | "unseen_verdict";
714 | "unseen_verdict"
715 /** The verdict says its author could not read the diff (#15007). */
716 | "unread_diff";
714717
715718 /** The most line comments one review may carry. */
716719 export const MAX_REVIEW_LINES = 50;
@@ -763,6 +766,8 @@ export async function submitReview(
763766
764767 if (input.state !== "commented" && confessesUnseen(input.body ?? ""))
765768 return err("unseen_verdict");
769 if (input.state !== "commented" && confessesUnread(input.body ?? ""))
770 return err("unread_diff");
766771 const lines = input.comments ?? [];
767772 if (lines.length > MAX_REVIEW_LINES) return err("too_many_lines");
768773 // Every anchor is checked before anything is written: a review with one
src/actions/review_unread.ts added +46
@@ -0,0 +1,46 @@
1 /**
2 * A verdict does not confess it never read the diff (#15007). A review that says
3 * "I could not read the diff", "the head branch is not fetchable" or "this rests
4 * on the pull description" judges nothing: it is refused (API 422 `unread_diff`),
5 * the run ends without a review, and review sends the pull again. After the
6 * second the pull waits for a person (`reviewStage`, `waitsForPerson`).
7 * Plain data in, a boolean out; a phrase matcher like `review_unseen.ts`, and
8 * only approve and request-changes are held to it.
9 */
10
11 // The object follows the verb directly, after at most a determiner: "could not
12 // read the diff". A gap would also refuse an honest "could not find any issue in
13 // the code", so there is none, and bare `code`, `changes` and `branch` are not
14 // objects (#15007).
15 const DIFF = String.raw`(?:diff|patch|head\s+(?:branch|commit))`;
16 const DET = String.raw`(?:(?:the|this|its|any|a|that)\s+)?`;
17 const NOT = String.raw`(?:did\s+not|didn['’]t|could\s+not|couldn['’]t|can\s*not|can['’]t` +
18 String.raw`|unable\s+to|was\s+not\s+able\s+to|wasn['’]t\s+able\s+to|failed\s+to)`;
outdated

False positive: "could not find a bug in the code" matches (ACT includes find/see, GAP allows 40 chars, DIFF includes bare code/changes). Narrow the verbs and nouns and drop the gap, as in the review body.

19 const ACT = String.raw`(?:read|view|open|fetch|access|load|inspect|retrieve|obtain)`;
20 const BAD = String.raw`(?:not|n['’]t|never)\s+(?:\w+\s+)?` +
21 String.raw`(?:fetchable|readable|available|accessible|reachable|visible)`;
22
23 const CONFESSIONS: readonly RegExp[] = [
24 // "could not read the diff", "unable to fetch the head branch"
25 new RegExp(`\\b${NOT}\\s+(?:\\w+ly\\s+)?${ACT}\\s+${DET}${DIFF}\\b`, "i"),
26 // "the head branch is not fetchable", "the diff was not available"
27 new RegExp(`\\b${DET}${DIFF}\\s+(?:is|was|are|were)\\s+${BAD}\\b`, "i"),
28 new RegExp(`\\b${DET}${DIFF}\\s+(?:is|was)\\s+(?:unreadable|unfetchable|unavailable)\\b`, "i"),
29 // "no diff was available", "without access to the diff", "without reading the diff"
30 new RegExp(`\\bno\\s+${DIFF}\\s+(?:was\\s+|is\\s+)?(?:available|accessible|readable)\\b`, "i"),
31 new RegExp(
32 `\\bwithout\\s+(?:access\\s+to|reading|seeing|viewing|fetching|opening)\\s+${DET}${DIFF}\\b`,
33 "i",
34 ),
35 // "rests on the pull description", "based only on the description"
36 new RegExp(
37 String.raw`\b(?:rests?|resting|based|relies|relying|judged?|going)\s+(?:only\s+)?(?:on|upon|from)\s+` +
38 String.raw`(?:only\s+)?the\s+(?:pull(?:\s+request)?\s+|pr\s+)?(?:description|title|summary)\b`,
39 "i",
40 ),
41 ];
42
43 /** Does this review body say its author could not read the diff? */
44 export function confessesUnread(body: string): boolean {
45 return CONFESSIONS.some((re) => re.test(body));
46 }
src/actions/review_words.ts +5
@@ -20,4 +20,9 @@ export const reviewWords = {
2020 */
2121 unseenVerdict:
2222 'Rule "a verdict is given after opening the preview and its pictures": this review says it did not. Open the preview and read its pictures, then say what you saw.',
23 /**
24 * `unread_diff`: a verdict says its author could not read the diff (#15007).
25 */
26 unreadDiff:
27 "Rule \"a verdict is given on the diff you read\": this review says it could not read the diff, so it is no verdict and nothing was recorded. Fetch the head branch and read the diff, then review it; if you cannot, say so in a comment, not a verdict.",
2328 } as const;
src/http/api.ts +2
@@ -3347,6 +3347,8 @@ export function apiRoutes(router: Router<Handler>, isPage: (path: string) => boo
33473347 );
33483348 if (submitted.error === "unseen_verdict")
33493349 return apiError("unseen_verdict", 422, reviewWords.unseenVerdict);
3350 if (submitted.error === "unread_diff")
3351 return apiError("unread_diff", 422, reviewWords.unreadDiff);
33503352 if (submitted.error === "looks_required")
33513353 return apiError("looks_required", 422, reviewWords.looksRequired);
33523354 if (submitted.error === "bad_anchor")
src/http/mcp/tools/review.ts +5 −3
@@ -55,9 +55,11 @@ export const tool: Tool = {
5555 ? reviewWords.tooLong
5656 : error === "unseen_verdict"
5757 ? reviewWords.unseenVerdict
58 : error === "looks_required"
59 ? reviewWords.looksRequired
60 : "That review could not be recorded."),
58 : error === "unread_diff"
59 ? reviewWords.unreadDiff
60 : error === "looks_required"
61 ? reviewWords.looksRequired
62 : "That review could not be recorded."),
6163 );
6264 }
6365 return good({
src/tui/screens/common.ts +1
@@ -1032,6 +1032,7 @@ const MESSAGES: Record<string, string> = {
10321032 self_review: "you cannot approve your own change",
10331033 invalid_state: "that is not a review verdict",
10341034 unseen_verdict: "a verdict is given after opening the preview and its pictures: say what you saw",
1035 unread_diff: "a verdict is given on the diff you read: this one says it could not",
10351036 looks_required: "a stylesheet change waits for its home-page looks at phone and desktop width",
10361037 // The fork flow. Each of these leaves the world in a different place, so
10371038 // each keeps its own sentence.
src/web/pages/pull.ts +7 −5
@@ -1518,11 +1518,13 @@ export async function pullReviewStateSubmit(ctx: Ctx): Promise<Response> {
15181518 ? reviewWords.tooLong
15191519 : error === "unseen_verdict"
15201520 ? reviewWords.unseenVerdict
1521 : error === "looks_required"
1522 ? reviewWords.looksRequired
1523 : error === "invalid_state"
1524 ? "That is not a verdict this forge knows."
1525 : "That review could not be recorded."),
1521 : error === "unread_diff"
1522 ? reviewWords.unreadDiff
1523 : error === "looks_required"
1524 ? reviewWords.looksRequired
1525 : error === "invalid_state"
1526 ? "That is not a verdict this forge knows."
1527 : "That review could not be recorded."),
15261528 });
15271529 }
15281530 return flashTo(ctx, `${dest}#conversation`, {
test/review-unread.test.ts added +155
@@ -0,0 +1,155 @@
1 /**
2 * A verdict that says it could not read the diff is refused (#15007), and two such
3 * reviews leave the pull waiting for a person.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { confessesUnread } from "../src/actions/review_unread.ts";
10 import { reviewWords } from "../src/actions/review_words.ts";
11 import { openPull, submitReview } from "../src/actions/pulls.ts";
12 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
13 import { createApp, type App } from "../src/app.ts";
14 import { listReviews } from "../src/core/reviews.ts";
15 import { createToken } from "../src/core/tokens.ts";
16 import { createUser } from "../src/core/users.ts";
17 import { reviewStage } from "../src/crew/review.ts";
18 import { errorText } from "../src/tui/screens/common.ts";
19 import { serve } from "../src/http/server.ts";
20 import { quietLogs } from "./helpers/globals.ts";
21
22 quietLogs();
23
24 const UNREAD = [
25 "I could not read the diff, so this rests on the pull description. Looks fine.",
26 "The head branch is not fetchable from here; approving from the description.",
27 "Unable to fetch the head branch. Changes needed.",
28 "I couldn't view the diff.",
29 "Reviewed without access to the diff.",
30 "This approval rests on the pull description.",
31 "The diff was not available.",
32 ];
33
34 const HONEST = [
35 "I read the diff: the loop in a.ts never ends.",
36 "",
37 "The diff does not read the config twice.",
38 "Fetched the head branch; the test is missing a case.",
outdated

Add honest sentences that must pass: "I could not find any issue in the code.", "I did not see a problem in the changes." These fail today.

39 "Do not fetch the branch inside the loop; it is slow.",
40 "The description matches the diff.",
41 "I could not find any issue in the code.",
42 "I did not see a problem in the changes.",
43 "I could not find a bug in the code.",
44 "Did not check out the branch locally, but the diff is clean.",
45 "I could not get this to fail; the patch is fine.",
46 ];
47
48 test("the ways of saying the diff was unreadable are caught", () => {
49 for (const b of UNREAD) expect([b, confessesUnread(b)]).toEqual([b, true]);
50 });
51
52 test("a review that read the diff, or says nothing, passes", () => {
53 for (const b of HONEST) expect([b, confessesUnread(b)]).toEqual([b, false]);
54 });
55
56 test("two reviews refused leave the pull waiting: one is sent again, then it lapses", () => {
57 const green = "2026-10-13T03:00:00.000Z";
58 const one = [{ createdAt: "2026-10-13T03:01:00.000Z", active: false }];
59 const two = [...one, { createdAt: "2026-10-13T03:10:00.000Z", active: false }];
60 const now = new Date("2026-10-13T03:20:00.000Z");
61 expect(reviewStage(green, one, now)).toBe("send");
62 expect(reviewStage(green, two, now)).toBe("lapsed");
63 });
64
65 let dir = "";
66 let app: App;
67 beforeAll(async () => {
68 dir = await mkdtemp(join(tmpdir(), "forge-unread-"));
69 app = createApp({ dataDir: dir, port: 0 });
70 });
71 afterAll(async () => {
72 app.close();
73 await rm(dir, { recursive: true, force: true });
74 });
75
76 test("submitReview and the API refuse it, a comment may say it, nothing is recorded", async () => {
77 const made = await createUser(app.db, {
78 origin: "seed", trusted: true, username: "alice", email: "[email protected]",
79 password: "a-good-passphrase", adminIfFirst: true,
80 });
81 if (!made.ok) throw new Error(String(made.error));
82 const alice = made.value;
83 const repo = await createRepo(app, alice, { name: "r", visibility: "public" });
84 if (!repo.ok) throw new Error(String(repo.error));
85 for (const [branch, file] of [[undefined, "forge.json"], ["topic", "t.txt"]] as const) {
86 const done = await commitToBranch(app, alice, repo.value, {
87 ...(branch === undefined ? {} : { branch }),
88 message: "change",
89 files: [{ path: file, content: file === "forge.json" ? "{}" : "x" }],
90 });
91 if (!done.ok) throw new Error(JSON.stringify(done.error));
92 }
93 const opened = await openPull(app, alice, repo.value, { title: "t", body: "", baseBranch: "main", headBranch: "topic" });
94 if (!opened.ok) throw new Error(JSON.stringify(opened.error));
95 const pull = opened.value;
96
97 expect(await submitReview(app, alice, repo.value, pull, { state: "approved", body: UNREAD[0]! })).toEqual({
98 ok: false,
99 error: "unread_diff",
100 });
101 expect(listReviews(app.db, pull.id)).toHaveLength(0);
102
103 const servers = await serve(app);
104 try {
105 const token = createToken(app.db, alice.id, "t", "admin").token;
106 const res = await fetch(
107 `${servers.url}/api/v1/repos/${repo.value.ownerName}/${repo.value.name}/pulls/${pull.number}/review-states`,
108 {
109 method: "POST",
110 headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
111 body: JSON.stringify({ state: "changes_requested", body: UNREAD[1] }),
112 },
113 );
114 expect(res.status).toBe(422);
115 expect(((await res.json()) as { error: string }).error).toBe("unread_diff");
116 } finally {
117 servers.stop();
118 }
119 expect(reviewWords.unreadDiff).toContain("could not read the diff");
120 expect(listReviews(app.db, pull.id)).toHaveLength(0);
121
122 const servers2 = await serve(app);
123 try {
124 const token = createToken(app.db, alice.id, "t2", "admin").token;
125 const auth = { authorization: `Bearer ${token}` };
126 const mcp = await fetch(`${servers2.url}/mcp`, {
127 method: "POST",
128 headers: { ...auth, "content-type": "application/json", accept: "application/json, text/event-stream" },
129 body: JSON.stringify({
130 jsonrpc: "2.0", id: 1, method: "tools/call",
131 params: { name: "review", arguments: { repo: `${repo.value.ownerName}/${repo.value.name}`, number: pull.number, verdict: "changes", reason: UNREAD[1] } },
132 }),
133 });
134 const mcpBody = (await mcp.json()) as { result: { isError: boolean; content: { text: string }[] } };
135 expect(mcpBody.result.isError).toBe(true);
136 expect(mcpBody.result.content[0]!.text).toContain(reviewWords.unreadDiff);
137
138 const form = await fetch(`${servers2.url}/${repo.value.ownerName}/${repo.value.name}/pulls/${pull.number}/review-state`, {
139 method: "POST",
140 redirect: "manual",
141 headers: { ...auth, "content-type": "application/x-www-form-urlencoded" },
142 body: new URLSearchParams({ state: "changes_requested", body: UNREAD[1]! }),
143 });
144 expect(form.status).toBe(303);
145 const cookies = decodeURIComponent(form.headers.getSetCookie().join(";")).replace(/\+/g, " ");
146 expect(cookies).toContain("could not read the diff");
147 } finally {
148 servers2.stop();
149 }
150 expect(errorText("unread_diff")).toContain("could not");
151 expect(listReviews(app.db, pull.id)).toHaveLength(0);
152
153 const note = await submitReview(app, alice, repo.value, pull, { state: "commented", body: UNREAD[0]! });
154 expect(note.ok).toBe(true);
155 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs