Public: anyone can read this; the crew can change it.

The merge queue re-checks an approved pull whose check went stale, instead of waiting forever #3057

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/3055-the-merge-queue-re-checks-an-approved-pu → main 1 commit fork

  • Merged. mason/3055-the-merge-queue-re-checks-an-approved-pu was merged into main as 5a6eeea.

Review

Sign in to review this change.
4 files changed +280 −23
README.md +6
@@ -1025,6 +1025,12 @@ that turn green while a candidate is gating wait for the next one. Each
10251025 queue's numbers — how many pulls are queued, how many are gating, and how
10261026 many landed in the last hour — show in three places: the flow panel on `/boards`, the
10271027 terminal's Flow command, and the MCP `flow` tool's `queue` field.
1028 When a repository requires checks and an approved pull is green only against
1029 an older main, the merge would refuse that green. The sweep therefore queues
1030 the pull's check again (reason `stale`) on main as it is now, and the fresh
1031 green lands the pull on a later sweep, with no person re-queuing it. The
1032 autonomy tick line counts the approved pulls the last sweep saw and why each
1033 waits (`approved="3 approved: queued=2 stale=1"`, from `src/crew/waits.ts`).
10281034
10291035 An autonomous app's planner is **held** while nothing is landing or the plate
10301036 is full: no pull has merged in 48 h while one was closed unmerged, or more than
src/crew/autonomy.ts +60 −23
@@ -23,6 +23,8 @@ import {
2323 import type { AccessError } from "../actions/access.ts";
2424 import { gateCandidate, releaseOnLand } from "../checks/landing.ts";
2525 import { addLearned } from "../checks/learned.ts";
26 import { checksFor } from "../checks/service.ts";
27 import { checkStale } from "../checks/stale.ts";
2628 import { enqueue, type GateCandidate, type Queued } from "../checks/queue.ts";
2729 import { mergeable, resolveOid, treeOf } from "../git/merge.ts";
2830 import { repoDir } from "../git/repos.ts";
@@ -74,6 +76,7 @@ import {
7476 activeRunFor,
7577 automaticRunsSince,
7678 listActive,
79 checkSummary,
7780 type Run,
7881 verdictForHead,
7982 } from "../core/runs.ts";
@@ -105,6 +108,7 @@ import {
105108 parkReasons,
106109 } from "./parks.ts";
107110 import { createLog } from "../kernel/log.ts";
111 import { err, ok, type Result } from "../kernel/result.ts";
108112 import {
109113 directorsAmong,
110114 landsOnItsOwn,
@@ -113,6 +117,7 @@ import {
113117 import { crewOffInPreview } from "./preview.ts";
114118 import { moveRequested, tallyRequested, tallyWords } from "./requested.ts";
115119 import { reviewersOf, sendToReview, sweepReviews } from "./review.ts";
120 import { lastWaits, noteWaits, tallyWaits, type Wait, waitWords } from "./waits.ts";
116121
117122 const log = createLog("autonomy");
118123
@@ -1035,10 +1040,16 @@ export async function sweepGreen(app: App, repo: Repo): Promise<number> {
10351040 // one sweep are one candidate and one gate (#2036).
10361041 const open = listPulls(app.db, repo.id, { sealed: true, state: "open", limit: 50 });
10371042 const entries: Queued[] = [];
1043 // Why each approved pull waits, for the tick line (#3055).
1044 const waits: Wait[] = [];
10381045 for (const pull of open.sort((a, b) => a.number - b.number)) {
1039 const entry = await admit(app, repo, pull);
1040 if (entry !== undefined) entries.push(entry);
1046 const admitted = await admit(app, repo, pull);
1047 if (admitted.ok) entries.push(admitted.value);
1048 const seen = reviewSummary(app.db, pull.id, pull.headOid);
1049 if (seen.approvals > 0 && seen.changesRequested === 0)
1050 waits.push(admitted.ok ? "queued" : admitted.error);
10411051 }
1052 noteWaits(repo.id, tallyWaits(waits));
10421053 if (entries.length === 0) return 0;
10431054 const landed = await enqueue(repo.id, entries, gateFor(app, repo));
10441055 return landed.filter(Boolean).length;
@@ -1145,6 +1156,7 @@ export async function autonomyTick(
11451156 // Every tick says what it did and why, so an app that sits still is
11461157 // never a mystery: the numbers behind "idle" are on the line.
11471158 const status = autonomyStatus(app, current, now);
1159 const approved = waitWords(lastWaits(current.id));
11481160 const ready = readyNow(repoGraph(app.db, current.id, { sealed: true }));
11491161 const held = heldBy(
11501162 app,
@@ -1168,6 +1180,8 @@ export async function autonomyTick(
11681180 : {
11691181 requested: tallyWords(tallyRequested(requested)),
11701182 }),
1183 // The approved pulls the last green sweep saw, and why each waits (#3055).
1184 ...(approved === undefined ? {} : { approved }),
11711185 spent: `${status.spentTodayUsd.toFixed(2)}/${status.ceilingUsd}`,
11721186 ...(status.paused === null ? {} : { paused: status.paused }),
11731187 });
@@ -1601,9 +1615,9 @@ export async function landIfGreen(
16011615 repo: Repo,
16021616 pull: Pull,
16031617 ): Promise<boolean> {
1604 const entry = await admit(app, repo, pull);
1605 if (entry === undefined) return false;
1606 const [landed] = await enqueue(repo.id, [entry], gateFor(app, repo));
1618 const admitted = await admit(app, repo, pull);
1619 if (!admitted.ok) return false;
1620 const [landed] = await enqueue(repo.id, [admitted.value], gateFor(app, repo));
16071621 return landed === true;
16081622 }
16091623
@@ -1618,36 +1632,36 @@ async function admit(
16181632 app: App,
16191633 repo: Repo,
16201634 pull: Pull,
1621 ): Promise<Queued | undefined> {
1622 if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) return undefined;
1623 if (pull.state !== "open" || pull.mergedAt !== null) return undefined;
1635 ): Promise<Result<Queued, Wait>> {
1636 if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) return err("off");
1637 if (pull.state !== "open" || pull.mergedAt !== null) return err("closed");
16241638 // The caller's copy may predate a landing: the row says whether it is
16251639 // still open (#2017).
16261640 const row = getPull(app.db, repo.id, pull.number);
16271641 if (row === undefined || row.state !== "open" || row.mergedAt !== null)
1628 return undefined;
1642 return err("closed");
16291643 // A person said "not this one yet": nothing below happens until it is
16301644 // released — no landing, no review sent, no run to bring the branch up
16311645 // to date (`core/holds.ts`).
1632 if (getHold(app.db, pull.id) !== undefined) return undefined;
1646 if (getHold(app.db, pull.id) !== undefined) return err("held");
16331647 // Only work the crew wrote, or a writer's, lands on green by itself: a
16341648 // reader's pull — from a fork, say — waits for a person, outsiders
16351649 // switch or not, and is never sent to the crew's reviewer (#619).
1636 if (!landsOnItsOwn(app.db, repo, pull)) return undefined;
1650 if (!landsOnItsOwn(app.db, repo, pull)) return err("outsider");
16371651 const by = getUserById(app.db, repo.autonomyBy);
1638 if (!by) return undefined;
1652 if (!by) return err("off");
16391653 // The newest run of each kind is what the head says: a red that a re-run
16401654 // on the same commit has since turned green is history.
16411655 const checks = verdictForHead(app.db, pull.id, pull.headOid);
1642 if (checks.length === 0) return undefined;
1643 if (checks.some((c) => c.state !== "ok")) return undefined;
1656 if (checks.length === 0) return err("unchecked");
1657 if (checks.some((c) => c.state !== "ok")) return err("not-green");
16441658 // A stop the crew took on a red check ends with the green after it; a
16451659 // person's stop waits for a person.
16461660 const greenAt = checks
16471661 .map((c) => c.finishedAt ?? "")
16481662 .reduce((a, b) => (a > b ? a : b), "");
16491663 if (stopStands(latestDecision(app.db, pull.id, "retry"), greenAt))
1650 return undefined;
1664 return err("stopped");
16511665 // Conflicts first: a pull that no longer merges is the crew's to bring
16521666 // up to date, before anyone is asked whether it should land. A conflict
16531667 // is remembered by the two tips it was between: until one of them moves
@@ -1657,7 +1671,7 @@ async function admit(
16571671 const baseTip =
16581672 (await resolveOid(dir, `refs/heads/${pull.baseBranch}`)) ?? "";
16591673 const between = `${pull.headOid}:${baseTip}`;
1660 if (conflicted.get(pull.id) === between) return undefined;
1674 if (conflicted.get(pull.id) === between) return err("conflict");
16611675 if (baseTip !== "") {
16621676 const clean = await mergeable(dir, baseTip, pull.headOid);
16631677 if (!clean.ok) {
@@ -1665,7 +1679,7 @@ async function admit(
16651679 conflicted.set(pull.id, between);
16661680 bringUpToDate(app, repo, pull, clean.error.files);
16671681 }
1668 return undefined;
1682 return err("conflict");
16691683 }
16701684 // Nothing the base does not already hold: no gate and no review, for
16711685 // there is nothing to land. The merge closes the pull as merged into
@@ -1677,7 +1691,7 @@ async function admit(
16771691 pull: pull.number,
16781692 closed: closed.ok,
16791693 });
1680 return undefined;
1694 return err("carried");
16811695 }
16821696 }
16831697 // A pull that loosens the contract is a change of policy, which is a
@@ -1719,7 +1733,7 @@ async function admit(
17191733 cells: cells.join("; "),
17201734 });
17211735 }
1722 return undefined;
1736 return err("wall");
17231737 }
17241738 // One gate for every pull, whoever wrote it: when the crew has a
17251739 // reviewer, a green head is queued for review and waits for an approval
@@ -1729,7 +1743,7 @@ async function admit(
17291743 // (#597). Only a person's own pull lands without a reviewer, or after a
17301744 // review lapsed.
17311745 const review = await reviewed(app, repo, pull);
1732 if (review === "hold") return undefined;
1746 if (review === "hold") return err("review");
17331747 // The full tier, once, on the queue's candidate (#1198, #2036). Red is
17341748 // told to the pull once per head and base, and the pull waits for a push.
17351749 const told = latestDecision(app.db, pull.id, "land");
@@ -1740,8 +1754,31 @@ async function admit(
17401754 told.state !== null &&
17411755 (told.state as { at?: unknown }).at === gatedAt
17421756 )
1743 return undefined;
1744 return {
1757 return err("red");
1758 // Green only against a base that has moved since: the merge would hold
1759 // it (`checksHold`), so the queue asks for the check again on the base
1760 // as it is now rather than wait for a person to (#3055). The fresh green
1761 // admits it on a later sweep; one already queued is not asked twice.
1762 if (
1763 repo.requireChecks &&
1764 checkStale(checkSummary(app.db, pull.id, pull.headOid).run, baseTip)
1765 ) {
1766 const again = await checksFor(app)?.request(
1767 repo,
1768 `pull-${pull.number}`,
1769 pull.headOid,
1770 "stale",
1771 pull.id,
1772 );
1773 log.info("green on an older base, checked again", {
1774 repo: `${repo.ownerName}/${repo.name}`,
1775 pull: pull.number,
1776 base: baseTip.slice(0, 7),
1777 queued: again !== undefined,
1778 });
1779 return err("stale");
1780 }
1781 return ok({
17451782 pull,
17461783 dir,
17471784 land: (expect) => landGated(app, repo, by, pull, review, expect),
@@ -1771,7 +1808,7 @@ async function admit(
17711808 addLearned(app.config.dataDir, pull.id, red.learn);
17721809 addComment(app, by, repo, getIssueById(app.db, pull.id) ?? pull, red.comment);
17731810 },
1774 };
1811 });
17751812 }
17761813
17771814 /** The queue's gate: the full tier on the candidate's tip, asked as its first pull. */
src/crew/waits.ts added +53
@@ -0,0 +1,53 @@
1 /**
2 * Why an approved pull has not landed (#3055): the reason the green sweep's
3 * `admit` gave each one, counted per repository, so the tick line says how
4 * many approved pulls wait and on what. An approved pull that sits still
5 * is then never a mystery: "stale" is a check the queue has asked for
6 * again on the moved base, "queued" is a pull in the merge queue's line.
7 */
8
9 /** Why a pull did not join the merge queue on this sweep, or that it did. */
10 export type Wait =
11 | "queued"
12 | "off"
13 | "closed"
14 | "held"
15 | "outsider"
16 | "unchecked"
17 | "not-green"
18 | "stopped"
19 | "conflict"
20 | "carried"
21 | "wall"
22 | "review"
23 | "red"
24 | "stale";
25
26 /** How many approved pulls waited on each reason. */
27 export type Waits = Partial<Record<Wait, number>>;
28
29 export function tallyWaits(waits: readonly Wait[]): Waits {
30 const out: Waits = {};
31 for (const w of waits) out[w] = (out[w] ?? 0) + 1;
32 return out;
33 }
34
35 /** `2 approved: queued=1 stale=1`, most first; undefined when none waited. */
36 export function waitWords(waits: Waits): string | undefined {
37 const said = (Object.entries(waits) as [Wait, number][])
38 .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
39 if (said.length === 0) return undefined;
40 const total = said.reduce((n, [, k]) => n + k, 0);
41 return `${total} approved: ${said.map(([w, n]) => `${w}=${n}`).join(" ")}`;
42 }
43
44 /** The last finished sweep's tally, by repository id: the sweep runs beside the tick. */
45 const last = new Map<string, Waits>();
46
47 export function noteWaits(repoId: string, waits: Waits): void {
48 last.set(repoId, waits);
49 }
50
51 export function lastWaits(repoId: string): Waits {
52 return last.get(repoId) ?? {};
53 }
test/stale-recheck.test.ts added +161
@@ -0,0 +1,161 @@
1 /**
2 * An approved pull whose green went stale lands on its own (#3055). Its
3 * check passed against an older base; main moved; a repository that
4 * requires checks will not merge it on that green (`checksHold`). The
5 * green sweep asks for the check again on the base as it is now, and the
6 * fresh green lands it — no person re-queues anything. The tick line's
7 * words say how many approved pulls wait, and why.
8 */
9 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
10 import { mkdtemp, rm } from "node:fs/promises";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp, type App } from "../src/app.ts";
14 import { openPull } from "../src/actions/pulls.ts";
15 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
16 import { provideChecksForTest, type ChecksHandle } from "../src/checks/service.ts";
17 import { getPull, type Pull } from "../src/core/pulls.ts";
18 import { getRepoById, type Repo } from "../src/core/repos.ts";
19 import { submitReview } from "../src/core/reviews.ts";
20 import { claimRun, createRun, finishRun, setRunBase } from "../src/core/runs.ts";
21 import { createUser, type User } from "../src/core/users.ts";
22 import { sweepGreen } from "../src/crew/autonomy.ts";
23 import { lastWaits, tallyWaits, waitWords } from "../src/crew/waits.ts";
24 import { repoDir } from "../src/git/repos.ts";
25
26 describe("a stale green is checked again by the queue", () => {
27 let dataDir: string;
28 let app: App;
29 let alice: User;
30 let bob: User;
31 let repo: Repo;
32 /** Every check the queue asked for: the stand-in runs it green on main's tip. */
33 const asked: { commit: string; reason: string }[] = [];
34 const fresh = () => getRepoById(app.db, repo.id)!;
35 const mainTip = () =>
36 Bun.spawnSync(["git", "rev-parse", "refs/heads/main"], {
37 cwd: repoDir(app.reposRoot, repo.ownerName, repo.name),
38 })
39 .stdout.toString()
40 .trim();
41
42 function passCheck(pull: Pull, commit: string, reason: string) {
43 const run = createRun(app.db, {
44 repoId: repo.id,
45 kind: "check",
46 issueId: pull.id,
47 env: `pull-${pull.number}`,
48 commitOid: commit,
49 reason,
50 });
51 claimRun(app.db, run.id);
52 setRunBase(app.db, run.id, mainTip());
53 return finishRun(app.db, run.id, { state: "ok", summary: "check passed" });
54 }
55
56 beforeAll(async () => {
57 dataDir = await mkdtemp(join(tmpdir(), "forge-stale-recheck-"));
58 app = createApp({ dataDir, port: 0 });
59 const made = async (username: string) => {
60 const u = await createUser(app.db, {
61 origin: "seed",
62 trusted: true,
63 username,
64 email: `${username}@forge.local`,
65 password: "a-good-passphrase",
66 adminIfFirst: true,
67 });
68 if (!u.ok) throw new Error(u.error);
69 return u.value;
70 };
71 alice = await made("alice");
72 bob = await made("bob");
73 const created = await createRepo(app, alice, { name: "engine", visibility: "private" });
74 if (!created.ok) throw new Error(String(created.error));
75 repo = created.value;
76 const seeded = await commitToBranch(app, alice, repo, {
77 message: "Start",
78 files: [{ path: "README.md", content: "# engine\n" }],
79 });
80 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
81 app.db
82 .query(
83 "UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?2, require_checks = 1 WHERE id = ?1",
84 )
85 .run(repo.id, alice.id);
86 provideChecksForTest(app, {
87 gate: async () => ({ ok: true, error: "", lines: [] }),
88 request: async (_repo: Repo, _env: string, commit: string, reason: string, issueId?: string) => {
89 asked.push({ commit, reason });
90 const pull = [...app.db
91 .query<{ number: number }, [string]>("SELECT number FROM issue WHERE id = ?1")
92 .all(issueId ?? "")][0];
93 return pull === undefined
94 ? undefined
95 : passCheck(getPull(app.db, repo.id, pull.number)!, commit, reason);
96 },
97 } as unknown as ChecksHandle);
98 });
99
100 afterAll(async () => {
101 await app.close?.();
102 await rm(dataDir, { recursive: true, force: true });
103 });
104
105 test("approve a green pull, move main, and it lands with nobody acting", async () => {
106 const pushed = await commitToBranch(app, alice, repo, {
107 branch: "feature/a",
108 base: "main",
109 message: "a",
110 files: [{ path: "a.ts", content: "export const a = 1;\n" }],
111 });
112 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
113 const opened = await openPull(app, alice, repo, {
114 title: "a",
115 baseBranch: "main",
116 headBranch: "feature/a",
117 });
118 if (!opened.ok) throw new Error(String(opened.error));
119 const pull = getPull(app.db, repo.id, opened.value.number)!;
120 passCheck(pull, pull.headOid, "pull");
121 submitReview(app.db, {
122 issueId: pull.id,
123 authorId: bob.id,
124 state: "approved",
125 commitOid: pull.headOid,
126 });
127
128 // Main moves past the base the green was checked on.
129 const moved = await commitToBranch(app, alice, repo, {
130 message: "elsewhere",
131 files: [{ path: "b.ts", content: "export const b = 1;\n" }],
132 });
133 if (!moved.ok) throw new Error(JSON.stringify(moved.error));
134 const before = mainTip();
135
136 // The first sweep does not land on the stale green: it asks again.
137 expect(await sweepGreen(app, fresh())).toBe(0);
138 expect(asked).toEqual([{ commit: pull.headOid, reason: "stale" }]);
139 expect(lastWaits(repo.id)).toEqual({ stale: 1 });
140 expect(mainTip()).toBe(before);
141
142 // The fresh green, on main as it is now, lands on the next sweep.
143 expect(await sweepGreen(app, fresh())).toBe(1);
144 expect(asked).toHaveLength(1);
145 const landed = getPull(app.db, repo.id, pull.number)!;
146 expect(landed.mergedAt).not.toBeNull();
147 expect(mainTip()).not.toBe(before);
148 });
149 });
150
151 describe("the tick's words for approved pulls that wait", () => {
152 test("say how many, and why, most first", () => {
153 expect(waitWords(tallyWaits(["stale", "queued", "stale", "review"]))).toBe(
154 "4 approved: stale=2 queued=1 review=1",
155 );
156 });
157
158 test("say nothing when no approved pull waits", () => {
159 expect(waitWords(tallyWaits([]))).toBeUndefined();
160 });
161 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs