Public: anyone can read this; the crew can change it.

A changes-requested pull whose follow-up run ended without a push gets one more run with the reviewer's words, then goes to a person #2862

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/2855-a-changes-requested-pull-whose-follow-up → main 3 commits fork 2 threads · 2 unresolved

  • Merged. scout/2855-a-changes-requested-pull-whose-follow-up was merged into main as 98e1563.

Review

Sign in to review this change.
7 files changed +229 −22
src/crew/brief.ts +25
@@ -201,6 +201,8 @@ export interface TaskInput extends BriefInput {
201201 readonly retry?: boolean | undefined;
202202 /** For `review`: pictures of the preview at this head, phone and desktop. */
203203 readonly looks?: readonly string[] | undefined;
204 /** For a feedback run asked again: why the last try did not land (`requested.ts`). */
205 readonly lastTry?: LastTry | undefined;
204206 /** For a planner: the issues the crew parked twice, with why. */
205207 readonly parkedTwice?: readonly HandedOver[] | undefined;
206208 /** For a planner: the limbs a person handed over to re-plan as fewer, larger pieces. */
@@ -213,6 +215,13 @@ export interface TaskInput extends BriefInput {
213215 readonly owed?: readonly Owed[] | undefined;
214216 }
215217
218 /** The reviewer's latest verdict, and the final answer of the run that did not push for it. */
219 export interface LastTry {
220 readonly reviewer: string;
221 readonly verdict: string;
222 readonly answer: string;
223 }
224
216225 /** Ready issues no builder can start, as the planner is told of them (#2244). */
217226 export interface Stuck {
218227 /** The count per filter, as the tick line says it: `open_pull=10 stalled=3`. */
@@ -506,9 +515,25 @@ function taskBody(input: TaskInput): string {
506515 for (const c of news.comments) out.push("", quote(c.authorName, c.body));
507516 // A hand-over is the news, in words, whether or not anyone wrote.
508517 if (handed.length > 0) out.push("", ...handed);
518 const tried = input.lastTry;
519 if (tried !== undefined) {
520 out.push(
521 "",
522 "Your last try on this did not land: the run ended without a push, and the review still asks for changes. This is the last one before a person takes over.",
523 "",
524 `${said(tried.reviewer)}'s latest verdict asked for changes${tried.verdict.trim() === "" ? "." : ":"}`,
525 );
526 if (tried.verdict.trim() !== "") out.push(indent(tried.verdict));
527 out.push(
528 "",
529 "Your last run's final answer:",
530 indent(tried.answer.trim() === "" ? "(none)" : tried.answer),
531 );
532 }
509533 if (
510534 news.comments.length + news.reviews.length + news.lines.length === 0 &&
511535 handed.length === 0 &&
536 tried === undefined &&
512537 input.paused !== true
513538 ) {
514539 out.push(
src/crew/dispatch.ts +4
@@ -65,6 +65,7 @@ import {
6565 capNoted,
6666 capReached,
6767 HANDOVER_NOTE,
68 RETRY_SUMMARY,
6869 handoverNote,
6970 handoverOf,
7071 type Handover,
@@ -288,6 +289,7 @@ export function queueAgentRun(
288289 actorId?: string,
289290 again?: { notBefore: string; attempt: number; summary: string },
290291 variantGroup?: string,
292 retry = false,
291293 ): Run | undefined {
292294 // A suspended or removed owner's repositories run nothing, whoever asks.
293295 if (!accountActive(app.db, repo.ownerId)) return undefined;
@@ -348,6 +350,7 @@ export function queueAgentRun(
348350 // queued run asks again as it starts (`runner.ts`).
349351 if (
350352 reason === "feedback" &&
353 !retry &&
351354 again === undefined &&
352355 active === undefined &&
353356 skipsFeedback(app.db, repo, issue, agent)
@@ -369,6 +372,7 @@ export function queueAgentRun(
369372 issueId: issue.id,
370373 actorId: agent.user.id,
371374 reason,
375 ...(retry ? { summary: RETRY_SUMMARY } : {}),
372376 ...again,
373377 // A run queued again (a restart, a pause, the sweep) stays in the
374378 // group its issue was started in.
src/crew/guard.ts +7
@@ -54,6 +54,13 @@ export const CAP_NOTE =
5454 /** What a skipped follow-up records, and what its run says when one was already queued. */
5555 export const SKIP_SUMMARY = "skipped: nothing new to act on since the last run";
5656
57 /**
58 * What the second follow-up on a verdict carries as its summary while it
59 * waits: the run that asks again because the last try pushed nothing
60 * (`requested.ts`). It is never skipped for having nothing new.
61 */
62 export const RETRY_SUMMARY = "retry: the last follow-up ended without a push";
63
5764 /** A follow-up for `agent` on `issue` that would find nothing new: skip it. */
5865 export function skipsFeedback(
5966 db: Database,
src/crew/requested.ts +49 −16
@@ -12,16 +12,19 @@
1212 * started since the verdict: a feedback run is queued to the pull's
1313 * builder, one being assigned when the pull has none. One run per pull,
1414 * and each takes a builder slot — `free` is how many the tick has.
15 * When that run ended without a push, one more is queued carrying the
16 * verdict and the run's final answer; when that one too ended without
17 * one, the pull is handed to a person and the pass stops.
1518 *
1619 * Every pull it passes over gets one line saying why, said once until the
1720 * reason changes, so a stall is never silent and never a flood.
1821 */
1922
2023 import type { App } from "../app.ts";
21 import { assignIssue } from "../actions/issues.ts";
24 import { addComment, assignIssue } from "../actions/issues.ts";
2225 import { getAgent, listCrewRoles, type Agent } from "../core/agents.ts";
2326 import { getHold } from "../core/holds.ts";
24 import { getIssueById } from "../core/issues.ts";
27 import { getIssueById, listComments, type Issue } from "../core/issues.ts";
2528 import { getPull } from "../core/pulls.ts";
2629 import type { Repo } from "../core/repos.ts";
2730 import { requestedPulls, verdictIsStale, type RequestedPull } from "../core/requested.ts";
@@ -30,12 +33,15 @@ import { activeRunFor } from "../core/runs.ts";
3033 import { getUserById } from "../core/users.ts";
3134 import { createLog } from "../kernel/log.ts";
3235 import { queueAgentRun } from "./dispatch.ts";
33 import { handoverOf } from "./guard.ts";
36 import { HANDOVER_NOTE, handoverOf } from "./guard.ts";
3437 import { greenHead, sendToReview } from "./review.ts";
3538 import { pickBuilder } from "./route.ts";
3639
3740 const log = createLog("crew");
3841
42 /** The runs a verdict gets from this pass before a person has it: the first, and one more with its words. */
43 const FOLLOW_UPS = 2;
44
3945 /** What the tick did with one pull: moved it, or why not. */
4046 export type RequestedMove =
4147 | { readonly pull: number; readonly did: "asked reviewer" }
@@ -170,8 +176,10 @@ function followUp(
170176 return notMoved(requested, "past the review rounds; the verdict is advice");
171177 if (activeRunFor(app.db, "agent", pull.id) !== undefined)
172178 return notMoved(requested, "a run is in flight");
173 if (startedSince(app, pull.id, requested.at))
174 return notMoved(requested, "a run since the verdict ended without a push");
179 // A push ends the verdict's turn (the reviewer reads the new head), so every
180 // run counted here ended without one: one is asked again, two go to a person.
181 const tried = startedSince(app, pull.id, requested.at);
182 if (tried >= FOLLOW_UPS) return handOverPushless(app, repo, requested, issue);
175183 if (builders.length === 0) return notMoved(requested, "no builder on the crew");
176184 const assignee = issue.assigneeId;
177185 let builder = builders.find((b) => b.user.id === assignee);
@@ -194,7 +202,7 @@ function followUp(
194202 const current = getIssueById(app.db, issue.id) ?? issue;
195203 // The assignment may already have queued the builder's run; either is the one.
196204 const run =
197 queueAgentRun(app, repo, current, builder, "feedback") ??
205 queueAgentRun(app, repo, current, builder, "feedback", undefined, undefined, undefined, tried > 0) ??
198206 activeRunFor(app.db, "agent", pull.id, builder.user.id);
199207 if (run === undefined)
200208 return notMoved(requested, `${builder.user.username}'s run was not queued (ceiling, cap or breaker)`);
@@ -206,17 +214,42 @@ function followUp(
206214 };
207215 }
208216
209 /** Whether a run other than a review started on the pull after the verdict. */
210 function startedSince(app: App, pullId: string, at: string): boolean {
211 return (
212 app.db
213 .query<{ n: number }, [string, string]>(
214 `SELECT COUNT(*) AS n FROM run WHERE issue_id = ?1 AND kind = 'agent'
215 AND reason != 'review' AND created_at > ?2
216 AND summary NOT LIKE 'skipped: %'`,
217 )
218 .get(pullId, at)!.n > 0
217 /** How many runs other than a review started on the pull after the verdict. */
218 function startedSince(app: App, pullId: string, at: string): number {
219 return app.db
220 .query<{ n: number }, [string, string]>(
221 `SELECT COUNT(*) AS n FROM run WHERE issue_id = ?1 AND kind = 'agent'
222 AND reason != 'review' AND created_at > ?2
223 AND summary NOT LIKE 'skipped: %'`,
224 )
225 .get(pullId, at)!.n;
226 }
227
228 /** The builder has had its follow-ups and pushed nothing: one note, from the reviewer, and no more runs. */
229 function handOverPushless(
230 app: App,
231 repo: Repo,
232 requested: RequestedPull,
233 issue: Issue,
234 ): RequestedMove {
235 const said = listComments(app.db, issue.id).some(
236 (c) => c.body.startsWith(HANDOVER_NOTE) && c.createdAt >= requested.at,
219237 );
238 const reviewer = getUserById(app.db, requested.reviewerId);
239 if (!said && reviewer !== undefined) {
240 addComment(
241 app,
242 reviewer,
243 repo,
244 issue,
245 `${HANDOVER_NOTE} two follow-up runs on the review's changes ended without a push, so no more start here until you say what to try. (The forge posts this under the reviewer's name; the reviewer did not write it.)`,
246 );
247 log.info("handed to a person after two follow-ups without a push", {
248 repo: `${repo.ownerName}/${repo.name}`,
249 pull: issue.number,

Posting as the reviewer, even with the disclosure, puts words in their mouth in the pull history. Prefer a forge/system author that does not wake the builder.

250 });
251 }
252 return notMoved(requested, "handed to a person after two follow-ups ended without a push");
220253 }
221254
222255 /** A move is always said; a why-not once, until it changes. */
src/crew/runner.ts +26 −1
@@ -90,6 +90,7 @@ import {
9090 systemPrompt,
9191 taskPrompt,
9292 wrapUpPrompt,
93 type LastTry,
9394 type RunReason,
9495 } from "./brief.ts";
9596 import { policyBrief } from "../contract/audit.ts";
@@ -139,7 +140,7 @@ import { historyHolds, scrubTree } from "./leak.ts";
139140 import { workdirFor } from "./workdir.ts";
140141 import { readersAmong, voiceOf } from "./direct.ts";
141142 import { feedbackSince } from "./news.ts";
142 import { handedOverPulls, SKIP_SUMMARY, skipsFeedback } from "./guard.ts";
143 import { handedOverPulls, RETRY_SUMMARY, SKIP_SUMMARY, skipsFeedback } from "./guard.ts";
143144 import { passportEnv } from "./passport_env.ts";
144145
145146 const log = createLog("crew");
@@ -296,6 +297,7 @@ export async function executeAgentRun(
296297 run.reason === "feedback" &&
297298 run.attempt === 0 &&
298299 run.startedAt === null &&
300 run.summary !== RETRY_SUMMARY &&

This guard is what keeps the retry from being cancelled as a skip. No test fails if it is removed. Cover it with a runner-level test.

299301 skipsQueued(app, run)
300302 ) {
301303 const skipped = cancelRun(app.db, run.id, SKIP_SUMMARY);
@@ -449,6 +451,25 @@ async function pullHeadOf(
449451 );
450452 }
451453
454 /** What a retried follow-up is told: the latest verdict asking for changes, and the last builder run's final answer. */
455 function lastTryOf(
456 app: App,
457 pullId: string,
458 run: Run,
459 reviews: readonly { state: string; authorName: string; body: string }[],
460 ): LastTry | undefined {
461 const verdict = reviews.findLast((r) => r.state === "changes_requested");
462 if (verdict === undefined) return undefined;
463 const answer = app.db
464 .query<{ summary: string }, [string, string]>(
465 `SELECT summary FROM run WHERE issue_id = ?1 AND kind = 'agent' AND reason != 'review'
466 AND id <> ?2 AND summary NOT LIKE 'skipped: %'
467 ORDER BY created_at DESC, id DESC LIMIT 1`,
468 )
469 .get(pullId, run.id);
470 return { reviewer: verdict.authorName, verdict: verdict.body, answer: answer?.summary ?? "" };
471 }
472
452473 /** Whether a queued follow-up would find nothing new (`news.ts`): recorded as a skip. */
453474 function skipsQueued(app: App, run: Run): boolean {
454475 const repo = getRepoById(app.db, run.repoId);
@@ -942,6 +963,10 @@ async function work(
942963 ),
943964 ),
944965 since,
966 lastTry:
967 reason === "feedback" && run.summary === RETRY_SUMMARY
968 ? lastTryOf(app, issue.id, run, reviews)
969 : undefined,
945970 checkSummary: failedCheck?.run?.summary,
946971 resolution:
947972 reason === "resolve" && pull
test/crew-retry.test.ts added +76
@@ -0,0 +1,76 @@
1 /**
2 * A follow-up asked again because the last try pushed nothing (#2855) is
3 * not skipped as having nothing new as it starts, and its prompt carries
4 * the reviewer's verdict and the previous run's final answer.
5 */
6
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { readFileSync } from "node:fs";
9 import type { App } from "../src/app.ts";
10 import { assignIssue, openIssue } from "../src/actions/issues.ts";
11 import { getAgent } from "../src/core/agents.ts";
12 import { listPulls } from "../src/core/pulls.ts";
13 import type { Repo } from "../src/core/repos.ts";
14 import { submitReview } from "../src/core/reviews.ts";
15 import { claimRun, createRun, finishRun } from "../src/core/runs.ts";
16 import type { User } from "../src/core/users.ts";
17 import { queueAgentRun } from "../src/crew/dispatch.ts";
18 import { RETRY_SUMMARY, SKIP_SUMMARY } from "../src/crew/guard.ts";
19 import {
20 agentRunSettled,
21 startStandInCrew,
22 stopStandInCrew,
23 type StandInCrew,
24 } from "./helpers/crew-standin.ts";
25
26 let standIn: StandInCrew;
27 let app: App;
28 let alice: User;
29 let bot: User;
30 let repo: Repo;
31
32 beforeAll(async () => {
33 standIn = await startStandInCrew();
34 ({ app, alice, bot, repo } = standIn);
35 const issue = openIssue(app, alice, repo, { title: "Plant tulips", body: "Along the fence." });
36 if (!issue.ok) throw new Error("issue not opened");
37 assignIssue(app, alice, repo, issue.value, bot);
38 const run = await agentRunSettled(app, repo, (r) => r.reason === "assigned");
39 expect(run.state).toBe("ok");
40 }, 60_000);
41
42 afterAll(() => stopStandInCrew(standIn));
43
44 test("the retry runs, and is told the verdict and the last run's answer", async () => {
45 const pull = listPulls(app.db, repo.id, { state: "open" })[0]!;
46 const verdict = submitReview(app.db, {
47 issueId: pull.id,
48 authorId: alice.id,
49 state: "changes_requested",
50 body: "Name the fence.",
51 commitOid: pull.headOid,
52 });
53 if (!verdict.ok) throw new Error(verdict.error);
54 await Bun.sleep(20);
55 // The first follow-up began after the verdict and pushed nothing.
56 const first = createRun(app.db, {
57 repoId: repo.id,
58 kind: "agent",
59 issueId: pull.id,
60 actorId: bot.id,
61 reason: "feedback",
62 });
63 claimRun(app.db, first.id);
64 finishRun(app.db, first.id, { state: "ok", summary: "I could not reproduce the fence." });
65
66 const queued = queueAgentRun(
67 app, repo, pull, getAgent(app.db, bot.id)!, "feedback", undefined, undefined, undefined, true,
68 );
69 expect(queued?.summary).toBe(RETRY_SUMMARY);
70 const run = await agentRunSettled(app, repo, (r) => r.id === queued!.id);
71 expect(run.summary).not.toBe(SKIP_SUMMARY);
72 expect(run.startedAt).not.toBeNull();
73 const prompt = readFileSync(standIn.promptFile, "utf8");
74 expect(prompt).toContain("Name the fence.");
75 expect(prompt).toContain("I could not reproduce the fence.");
76 }, 60_000);
test/requested.test.ts +42 −5
@@ -15,13 +15,14 @@ import { openPull } from "../src/actions/pulls.ts";
1515 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
1616 import { createApp, type App } from "../src/app.ts";
1717 import { setCrewRole } from "../src/core/agents.ts";
18 import { getIssueById } from "../src/core/issues.ts";
18 import { getIssueById, listComments } from "../src/core/issues.ts";
1919 import { getPull, type Pull } from "../src/core/pulls.ts";
2020 import { getRepoById, type Repo } from "../src/core/repos.ts";
2121 import { submitReview } from "../src/core/reviews.ts";
2222 import { claimRun, createRun, finishRun, listRuns } from "../src/core/runs.ts";
2323 import { createUser, getUserByName, type User } from "../src/core/users.ts";
2424 import { autonomyTick } from "../src/crew/autonomy.ts";
25 import { HANDOVER_NOTE } from "../src/crew/guard.ts";
2526 import { moveRequested, tallyRequested, tallyWords } from "../src/crew/requested.ts";
2627 import { requestedSays, requestedWaits } from "../src/flow/requested.ts";
2728 import { serve, type Servers } from "../src/http/server.ts";
@@ -215,11 +216,47 @@ describe("a pull with changes requested always moves", () => {
215216 did: "not moved",
216217 why: "a run is in flight",
217218 });
218 // A run that answered nothing is not asked again.
219 clearRuns();
220 const after = await moveRequested(app, fresh(), 1);
221 expect(after.find((m) => m.pull === pull.number)?.did).toBe("not moved");
219 app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(pull.id);
220 });
221
222 test("a run that ended without a push is asked once more with the verdict and its answer, then a person has it", async () => {
223 const pull = await currentShape();
224 await moveRequested(app, fresh(), 1);
222225 expect(runs(pull, "feedback")).toHaveLength(1);
226 // The first try ends without a push, saying why.
227 app.db
228 .query("UPDATE run SET state = 'ok', started_at = created_at, finished_at = created_at, summary = 'I could not reproduce it.' WHERE issue_id = ?1 AND reason = 'feedback'")
229 .run(pull.id);
230
231 // Exactly one new run within one tick; a second tick starts no other.
232 const second = await moveRequested(app, fresh(), 1);
233 expect(second).toContainEqual({
234 pull: pull.number,
235 did: "queued feedback",
236 builder: "mason",
237 assigned: false,
238 });
239 expect(runs(pull, "feedback")).toHaveLength(2);
240 const again = await moveRequested(app, fresh(), 1);
241 expect(again.find((m) => m.pull === pull.number)).toEqual({
242 pull: pull.number,
243 did: "not moved",
244 why: "a run is in flight",
245 });
246 expect(runs(pull, "feedback")).toHaveLength(2);
247 // After the second also ends without a push: handed over, once, and no third run.
248 clearRuns();
249 for (let i = 0; i < 2; i++) {
250 const over = await moveRequested(app, fresh(), 1);
251 expect(over.find((m) => m.pull === pull.number)).toEqual({
252 pull: pull.number,
253 did: "not moved",
254 why: "handed to a person after two follow-ups ended without a push",
255 });
256 }
257 expect(runs(pull, "feedback")).toHaveLength(2);
258 const notes = listComments(app.db, pull.id).filter((c) => c.body.startsWith(HANDOVER_NOTE));
259 expect(notes).toHaveLength(1);
223260 app.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(pull.id);
224261 });
225262

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs