Public: anyone can read this; the crew can change it.

Approved-but-conflicted pulls drain one at a time: give rebases priority in the check lane, spread them across idle builders, and re-cut split files mechanically #6173

Closed opened this in answer to
Closed without merging

mason/forge:mason/6140-approved-but-conflicted-pulls-drain-one → main 1 commit fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
12 files changed +559 −50
docs/FLOW.md +19
@@ -27,6 +27,25 @@ bottleneck; the knobs stay on Settings → Capacity. The same sentence
2727 (`verdict`) is the terminal's admin-only Flow command and the MCP `flow` tool,
2828 which also returns the stages; anyone but a forge admin is refused.
2929
30 ## Rebases
31
32 Approved work lands before new work is built, in the check lane too (#6140).
33 A check on a pull whose last builder's run brought it up to date (a
34 `conflict` run; a review after it does not count) goes before any other
35 queued check (`checks/finishing_first.ts`). So the room for more rebases
36 (`crew/conflict_room.ts`) is the check slots minus the rebases' checks and the
37 bring-up-to-date runs in flight; a new build's check does not stand in front
38 of a rebase and takes no room. When the pull's own builder already has a run
39 in flight, the finishing pass (`crew/finishing.ts`) lends the update to an
40 idle builder, round-robin by name; the run's agent is who rebased, and the log
41 line names whose pull it was.
42
43 The tick line's `lane` field, the `Rebases:` line on `/flow` and in the
44 terminal's Flow, and `rebases` in the MCP `flow` tool all read
45 `conflicted N, rebasing M, room R`: the conflicted pulls (with the approved
46 ones still waiting for room, which have no run yet), the bring-up-to-date runs
47 and their checks in flight, and the runs the lane has room to start now.
48
3049 ## Throughput
3150
3251 `src/flow/throughput.ts` counts what each pipeline stage let out over the last
src/checks/finishing_first.ts added +52
@@ -0,0 +1,52 @@
1 /**
2 * Land before you build, in the check lane (#6140).
3 *
4 * A pull brought up to date (a `conflict` run: main moved, the crew merged
5 * it in and pushed) is nearly done work; its check is the last thing
6 * between it and landing. When a landing conflicts dozens of approved
7 * pulls at once, new builds' checks kept the lane full, and the rebases
8 * drained one at a time behind them. So a rebase's check goes before any
9 * other in the lane, and the conflict room (`crew/conflict_room.ts`)
10 * counts only these checks as standing in front of the next rebase.
11 *
12 * A check is a rebase's when the last builder's run on its pull — any
13 * agent run but a review, which comes after the push — was a `conflict`
14 * run. Read from the runs alone, so every forge sharing the database
15 * agrees on the order.
16 */
17
18 import type { Database } from "bun:sqlite";
19
20 /** The pulls (issue ids), among `issueIds`, whose last builder's run was a bring-up-to-date run. */
21 export function rebasedPulls(db: Database, issueIds: readonly string[]): Set<string> {
22 const ids = [...new Set(issueIds)];
23 if (ids.length === 0) return new Set();
24 const marks = ids.map((_, i) => `?${i + 1}`).join(",");
25 const rows = db
26 .query<{ issue_id: string }, string[]>(
27 `SELECT a.issue_id FROM run a
28 WHERE a.kind = 'agent' AND a.reason = 'conflict' AND a.issue_id IN (${marks})
29 AND NOT EXISTS (
30 SELECT 1 FROM run b
31 WHERE b.issue_id = a.issue_id AND b.kind = 'agent' AND b.reason != 'review'
32 AND (b.created_at > a.created_at OR (b.created_at = a.created_at AND b.id > a.id)))`,
33 )
34 .all(...ids);
35 return new Set(rows.map((r) => r.issue_id));
36 }
37
38 /** The runs that are rebases' checks, and the rest: each keeps its order. */
39 export function rebasesFirst<R extends { readonly issueId: string | null }>(
40 db: Database,
41 queued: readonly R[],
42 ): { readonly rebases: R[]; readonly rest: R[] } {
43 const rebased = rebasedPulls(
44 db,
45 queued.flatMap((r) => (r.issueId === null ? [] : [r.issueId])),
46 );
47 const rebases: R[] = [];
48 const rest: R[] = [];
49 for (const run of queued)
50 (run.issueId !== null && rebased.has(run.issueId) ? rebases : rest).push(run);
51 return { rebases, rest };
52 }
src/checks/service.ts +17 −8
@@ -55,6 +55,7 @@ import { keepCache, restoreCache } from "./cache.ts";
5555 import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts";
5656 import { withDisk } from "./disk_red.ts";
5757 import { ASKED, redMain, unheld } from "./red_main.ts";
58 import { rebasesFirst } from "./finishing_first.ts";
5859 import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
5960 import { getIssueById } from "../core/issues.ts";
6061 import { BUDGETS, budgetSummary } from "../kernel/budgets.ts";
@@ -434,6 +435,18 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
434435 * opens up. Claiming is what decides between two forges, so calling this
435436 * from both never runs the same check twice.
436437 */
438 /** `pickNext` among `queued`, passing over any whose account is at its limit. */
439 function startable(queued: readonly Run[]): Run | undefined {
440 let rest = queued;
441 let next = pickNext(rest);
442 while (next !== undefined && atLimit(next)) {
443 const held = next;
444 rest = rest.filter((run) => run !== held);
445 next = pickNext(rest);
446 }
447 return next;
448 }
449
437450 function schedule(): void {
438451 if (stopped) return;
439452 const limit = capacity();
@@ -453,14 +466,10 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
453466 );
454467 // The limit is asked at the point of starting, one pick at a time:
455468 // a started check is in flight before the next is asked about, so a
456 // burst of one account's checks cannot pass together.
457 let rest = queued;
458 let next = pickNext(rest);
459 while (next !== undefined && atLimit(next)) {
460 const held = next;
461 rest = rest.filter((run) => run !== held);
462 next = pickNext(rest);
463 }
469 // burst of one account's checks cannot pass together. A rebase's
470 // check goes before any other (`finishing_first.ts`, #6140).
471 const { rebases, rest: others } = rebasesFirst(app.db, queued);
472 const next = startable(rebases) ?? startable(others);
464473 if (!next) return;
465474 const repo = getRepoById(app.db, next.repoId);
466475 if (!repo) return;
src/crew/autonomy.ts +4 −3
@@ -27,7 +27,7 @@ import { addLearned } from "../checks/learned.ts";
2727 import { enqueue, type GateCandidate, type Queued, type QueueOptions, queueClashes, queueWords } from "../checks/queue.ts";
2828 import { checksFor } from "../checks/service.ts";
2929 import { queueDepth } from "../core/capacity.ts";
30 import { readConflictRoom } from "./conflict_room.ts";
30 import { readConflictLane, readConflictRoom } from "./conflict_room.ts";
3131 import { mergeable, resolveOid, treeOf } from "../git/merge.ts";
3232 import { CLOSED_NOTE } from "../core/done_on_main.ts";
3333 import { mergePull } from "../actions/pulls.ts";
@@ -1285,8 +1285,9 @@ export async function autonomyTick(
12851285 unstuck,
12861286 active: status.active,
12871287 slots: status.slots,
1288 // Approved pulls being brought up to date, and those still waiting (#3682).
1289 ...finishFields(finishing),
1288 // Approved pulls being brought up to date, those still waiting (#3682),
1289 // and the lane for rebases: conflicted N, rebasing M, room R (#6140).
1290 ...finishFields(finishing, readConflictLane(app.db)),
12901291 ...(requested.length === 0
12911292 ? {}
12921293 : {
src/crew/conflict_room.ts +32 −15
@@ -20,15 +20,23 @@
2020 * pulls: it queues before the sweep, and the sweep counts its runs as in
2121 * flight, so between them a tick never starts more than the lane holds
2222 * (#4784).
23 *
24 * The checks counted are the rebases' own, not every check (#6140): a
25 * rebase's check goes before any other in the lane (`checks/
26 * finishing_first.ts`), so a new build's check does not stand in front of
27 * it. Counting those let new builds keep the room at one, and 64 approved
28 * pulls drained one at a time.
2329 */
2430
2531 import type { Database } from "bun:sqlite";
32 import { rebasesFirst } from "../checks/finishing_first.ts";
2633 import { capacityValue } from "../core/capacity.ts";
34 import { listActive } from "../core/runs.ts";
2735
2836 export function conflictRoom(
2937 /** Checks the forge runs at once (`checks.concurrency`). */
3038 slots: number,
31 /** Check runs queued or running. */
39 /** Rebases' check runs queued or running: the ones ahead of the next. */
3240 checks: number,
3341 /** Bring-up-to-date runs queued or running. */
3442 inFlight: number,
@@ -37,20 +45,29 @@ export function conflictRoom(
3745 return Math.max(room, inFlight === 0 ? 1 : 0);
3846 }
3947
48 /** What the lane holds for rebases now, and the room left: the tick line's and /flow's words (#6140). */
49 export interface ConflictLane {
50 /** Bring-up-to-date runs queued or running. */
51 readonly runs: number;
52 /** Rebases' checks queued or running. */
53 readonly checks: number;
54 readonly room: number;
55 }
56
57 /** The lane from what the forge holds now. */
58 export function readConflictLane(db: Database): ConflictLane {
59 const runs = db
60 .query<{ n: number }, []>(
61 `SELECT COUNT(*) AS n FROM run
62 WHERE kind = 'agent' AND reason = 'conflict' AND state IN ('queued','running')`,
63 )
64 .get()!.n;
65 const checks = rebasesFirst(db, listActive(db, "check")).rebases.length;
66 const slots = Number(capacityValue(db, "checks.concurrency"));
67 return { runs, checks, room: conflictRoom(Number.isInteger(slots) && slots > 0 ? slots : 1, checks, runs) };
68 }
69
4070 /** `conflictRoom` from what the forge holds now. */
4171 export function readConflictRoom(db: Database): number {
42 const row = db
43 .query<{ checks: number; conflicts: number }, []>(
44 `SELECT
45 COALESCE(SUM(kind = 'check'), 0) AS checks,
46 COALESCE(SUM(kind = 'agent' AND reason = 'conflict'), 0) AS conflicts
47 FROM run WHERE state IN ('queued','running')`,
48 )
49 .get();
50 const slots = Number(capacityValue(db, "checks.concurrency"));
51 return conflictRoom(
52 Number.isInteger(slots) && slots > 0 ? slots : 1,
53 row?.checks ?? 0,
54 row?.conflicts ?? 0,
55 );
72 return readConflictLane(db).room;
5673 }
src/crew/finishing.ts +101 −16
@@ -11,7 +11,11 @@
1111 * - With a slot free, a conflict run (the #2039 resolver brief: merge the
1212 * base, resolve keeping both intents, re-run the check) is queued for
1313 * its builder — the agent on it, else its author, else any builder —
14 * and it takes that slot.
14 * and it takes that slot. When that builder already has a run in
15 * flight, the run goes to an idle builder instead, round-robin, so one
16 * busy agent holding many pulls does not bring them up to date one at
17 * a time (#6140); the run's agent is who rebased, and the log names
18 * whose pull it was.
1519 * - Its push is a check, so it also waits for room in the check lane, as
1620 * the conflict sweep's runs do (`conflict_room.ts`): a landing that
1721 * conflicts ten approved pulls at once must not put ten checks in the
@@ -36,11 +40,12 @@ import { getIssueById, type Issue } from "../core/issues.ts";
3640 import { listPulls, pullDir, type Pull } from "../core/pulls.ts";
3741 import type { Repo } from "../core/repos.ts";
3842 import { reviewSummary } from "../core/reviews.ts";
39 import { activeRunFor } from "../core/runs.ts";
43 import { activeRunFor, listActive } from "../core/runs.ts";
4044 import { getUserById } from "../core/users.ts";
4145 import { mergeable, resolveOid } from "../git/merge.ts";
4246 import { createLog } from "../kernel/log.ts";
43 import { readConflictRoom } from "./conflict_room.ts";
47 import { noteWaitingToFinish, rebaseLaneSays } from "../flow/conflicted.ts";
48 import { type ConflictLane, readConflictRoom } from "./conflict_room.ts";
4449 import { CHECK_FAILED_LIMIT, lastHumanWord, queueAgentRun } from "./dispatch.ts";
4550 import { HANDOVER_NOTE } from "./guard.ts";
4651 import { pickBuilder } from "./route.ts";
@@ -50,7 +55,13 @@ const log = createLog("crew");
5055
5156 /** What the pass did with one approved, conflicted pull. */
5257 export type FinishMove =
53 | { readonly pull: number; readonly did: "queued update"; readonly builder: string }
58 | {
59 readonly pull: number;
60 readonly did: "queued update";
61 readonly builder: string;
62 /** The pull's own builder, when it was busy and another took the update. */
63 readonly for?: string;
64 }
5465 | { readonly pull: number; readonly did: "in flight" }
5566 | { readonly pull: number; readonly did: "handed to a person"; readonly to: string }
5667 | { readonly pull: number; readonly did: "not moved"; readonly why: string };
@@ -61,13 +72,26 @@ export function approvedOnHead(db: Database, pull: Pull): boolean {
6172 return seen.approvals > 0 && seen.changesRequested === 0;
6273 }
6374
64 /** The tick line's two numbers: pulls being brought up to date, and the rest still waiting. */
65 export function finishFields(moves: readonly FinishMove[]): { finishing?: number; conflict?: string } {
75 /**
76 * The tick line's numbers: pulls being brought up to date, the rest still
77 * waiting, and, given the lane, `conflicted N, rebasing M, room R` — this
78 * repository's approved conflicted pulls against the forge-wide lane
79 * (#6140) — whenever any is conflicted or rebasing.
80 */
81 export function finishFields(
82 moves: readonly FinishMove[],
83 lane?: ConflictLane,
84 ): { finishing?: number; conflict?: string; lane?: string } {
6685 const finishing = moves.filter((m) => m.did === "queued update" || m.did === "in flight").length;
6786 const waiting = moves.length - finishing;
87 const words =
88 lane === undefined
89 ? null
90 : rebaseLaneSays({ conflicted: moves.length, rebasing: lane.runs + lane.checks, room: lane.room });
6891 return {
6992 ...(finishing === 0 ? {} : { finishing }),
7093 ...(waiting === 0 ? {} : { conflict: `${waiting} waiting` }),
94 ...(words === null ? {} : { lane: words }),
7195 };
7296 }
7397
@@ -98,7 +122,10 @@ async function conflictOf(app: App, repo: Repo, pull: Pull): Promise<string[] |
98122 * pull first.
99123 */
100124 export async function finishApproved(app: App, repo: Repo, free: number): Promise<FinishMove[]> {
101 if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) return [];
125 if (repo.autonomy !== "autonomous" || repo.autonomyBy === null) {
126 noteWaitingToFinish(repo.id, []);
127 return [];
128 }
102129 const moves: FinishMove[] = [];
103130 const builders = listCrewRoles(app.db, repo.id)
104131 .filter((c) => c.role === "builder")
@@ -108,6 +135,9 @@ export async function finishApproved(app: App, repo: Repo, free: number): Promis
108135 for (const id of read.keys()) if (!ids.has(id)) read.delete(id);
109136 let slots = free;
110137 let room = readConflictRoom(app.db);
138 const load = new Map<string, number>();
139 for (const run of listActive(app.db, "agent"))
140 if (run.actorId !== null) load.set(run.actorId, (load.get(run.actorId) ?? 0) + 1);
111141 for (const pull of open.sort((a, b) => a.number - b.number)) {
112142 if (!pull.authorIsAgent || pull.mergedAt !== null) continue;
113143 if (getHold(app.db, pull.id) !== undefined) continue;
@@ -116,14 +146,26 @@ export async function finishApproved(app: App, repo: Repo, free: number): Promis
116146 if (files === undefined) continue;
117147 const issue = getIssueById(app.db, pull.id);
118148 if (!issue || issue.state !== "open") continue;
119 const move = finishOne(app, repo, pull, issue, files, builders, slots, room);
149 const move = finishOne(app, repo, pull, issue, files, builders, load, slots, room);
120150 if (move.did === "queued update") {
121151 slots--;
122152 room--;
153 const by = builders.find((b) => b.user.username === move.builder)!.user.id;
154 load.set(by, (load.get(by) ?? 0) + 1);
123155 }
124156 moves.push(move);
125157 say(repo, pull, move);
126158 }
159 const since = new Date().toISOString();
160 noteWaitingToFinish(
161 repo.id,
162 moves.map((m) => ({
163 repo: `${repo.ownerName}/${repo.name}`,
164 number: m.pull,
165 since,
166 public: repo.visibility === "public",
167 })),
168 );
127169 return moves;
128170 }
129171
@@ -134,6 +176,7 @@ function finishOne(
134176 issue: Issue,
135177 files: readonly string[],
136178 builders: readonly Agent[],
179 load: ReadonlyMap<string, number>,
137180 slots: number,
138181 room: number,
139182 ): FinishMove {
@@ -142,12 +185,11 @@ function finishOne(
142185 // A person's name on it is a person holding it — after a hand-over too.
143186 if (issue.assigneeId !== null && getAgent(app.db, issue.assigneeId) === undefined)
144187 return not("a person has it");
145 const builder =
146 builders.find((b) => b.user.id === issue.assigneeId) ??
147 builders.find((b) => b.user.id === issue.authorId) ??
148 pickBuilder(issue, builders, new Map());
149 if (builder === undefined) return not(NO_BUILDER_ON_CREW);
150 if (failedUpdates(app, issue, builder) >= CHECK_FAILED_LIMIT) return handOver(app, repo, pull, issue, files);
188 const own = ownBuilder(issue, builders, load);
189 if (own === undefined) return not(NO_BUILDER_ON_CREW);
190 // Two failed updates are counted against the pull's own builder, whoever ran them.
191 const builder = (slots > 0 && room > 0 ? lentBuilder(own, builders, load) : undefined) ?? own;
192 if (failedUpdates(app, issue, own) >= CHECK_FAILED_LIMIT) return handOver(app, repo, pull, issue, files);
151193 if (slots <= 0) return not(WAITS_FOR_BUILDER_SLOT);
152194 if (room <= 0) return not("waits for room in the check lane");
153195 const run = queueAgentRun(app, repo, issue, builder, "conflict", undefined, {
@@ -156,7 +198,46 @@ function finishOne(
156198 summary: `main moved: ${files.join(", ")}`,
157199 });
158200 if (run === undefined) return not(`${builder.user.username}'s run was not queued (ceiling or breaker)`);
159 return { pull: pull.number, did: "queued update", builder: builder.user.username };
201 return {
202 pull: pull.number,
203 did: "queued update",
204 builder: builder.user.username,
205 ...(builder === own ? {} : { for: own.user.username }),
206 };
207 }
208
209 /** The pull's own builder: the agent on it, else its author, else the builder with least in flight. */
210 export function ownBuilder(
211 issue: Issue,
212 builders: readonly Agent[],
213 load: ReadonlyMap<string, number>,
214 ): Agent | undefined {
215 return (
216 builders.find((b) => b.user.id === issue.assigneeId) ??
217 builders.find((b) => b.user.id === issue.authorId) ??
218 pickBuilder(issue, builders, load)
219 );
220 }
221
222 /** Whose turn it is among the idle builders, kept across ticks so lent updates go round. */
223 let turn = 0;
224
225 /**
226 * The builder who takes an update when `own` already has a run in flight:
227 * the next idle builder, round-robin by name (#6140). Undefined when `own`
228 * is idle or every builder is busy: then the update is `own`'s.
229 */
230 export function lentBuilder(
231 own: Agent,
232 builders: readonly Agent[],
233 load: ReadonlyMap<string, number>,
234 ): Agent | undefined {
235 if ((load.get(own.user.id) ?? 0) === 0) return undefined;
236 const idle = builders
237 .filter((b) => (load.get(b.user.id) ?? 0) === 0)
238 .sort((a, b) => a.user.username.localeCompare(b.user.username));
239 if (idle.length === 0) return undefined;
240 return idle[turn++ % idle.length];
160241 }
161242
162243 /**
@@ -203,7 +284,11 @@ function say(repo: Repo, pull: Pull, move: FinishMove): void {
203284 const key = `${repo.id}:${pull.number}`;
204285 if (move.did === "queued update") {
205286 said.delete(key);
206 log.info("approved pull conflicts: brought up to date first", { ...where, builder: move.builder });
287 log.info("approved pull conflicts: brought up to date first", {
288 ...where,
289 builder: move.builder,
290 ...(move.for === undefined ? {} : { for: move.for }),
291 });
207292 return;
208293 }
209294 if (move.did !== "not moved") return;
src/flow/conflicted.ts +49 −1
@@ -2,10 +2,13 @@
22 * Pulls that main moved out from under (#2039): open pulls with a
33 * bring-up-to-date run queued, running or failed, by how long since the
44 * crew was first asked to bring each one up to date. Read from the runs
5 * alone, so /flow, the terminal and MCP say the same without asking git.
5 * alone, so /flow, the terminal and MCP say the same without asking git —
6 * and the approved pulls the finishing pass last found conflicted and
7 * still waiting for room, which have no run yet (#6140).
68 */
79
810 import type { Database } from "bun:sqlite";
11 import { readConflictLane } from "../crew/conflict_room.ts";
912 import { span } from "./pipeline.ts";
1013
1114 export interface ConflictedPull {
@@ -15,8 +18,33 @@ export interface ConflictedPull {
1518 readonly since: string;
1619 }
1720
21 /** An approved pull the finishing pass found conflicted, with no run on it yet. */
22 export interface WaitingToFinish extends ConflictedPull {
23 readonly public: boolean;
24 }
25
26 /** Repository id → its approved, conflicted pulls on the finishing pass's last tick. */
27 const waiting = new Map<string, readonly WaitingToFinish[]>();
28
29 /** What the finishing pass found this tick in one repository; a pull seen before keeps its first `since`. */
30 export function noteWaitingToFinish(repoId: string, pulls: readonly WaitingToFinish[]): void {
31 const before = new Map((waiting.get(repoId) ?? []).map((p) => [p.number, p.since]));
32 if (pulls.length === 0) waiting.delete(repoId);
33 else waiting.set(repoId, pulls.map((p) => ({ ...p, since: before.get(p.number) ?? p.since })));
34 }
35
1836 /** Oldest first; `publicOnly` leaves out private repositories, for a reader who must not learn their names. */
1937 export function conflictedPulls(db: Database, publicOnly = false): ConflictedPull[] {
38 const ran = conflictRuns(db, publicOnly);
39 const seen = new Set(ran.map((p) => `${p.repo}#${p.number}`));
40 const noted = [...waiting.values()]
41 .flat()
42 .filter((p) => (!publicOnly || p.public) && !seen.has(`${p.repo}#${p.number}`))
43 .map(({ repo, number, since }) => ({ repo, number, since }));
44 return noted.length === 0 ? ran : [...ran, ...noted].sort((a, b) => a.since.localeCompare(b.since));
45 }
46
47 function conflictRuns(db: Database, publicOnly: boolean): ConflictedPull[] {
2048 return db
2149 .query<ConflictedPull, [number]>(
2250 `SELECT o.username || '/' || r.name AS repo, i.number AS number,
@@ -46,3 +74,23 @@ export function conflictedSays(
4674 const more = pulls.length > 5 ? `, and ${pulls.length - 5} more` : "";
4775 return `${pulls.length} conflicted ${pulls.length === 1 ? "pull" : "pulls"}, oldest first: ${list}${more}.`;
4876 }
77
78 /** The lane for rebases: conflicted pulls, how many are being brought up to date, and the room for more (#6140). */
79 export interface RebaseLane {
80 readonly conflicted: number;
81 /** Bring-up-to-date runs and their checks, queued or running, forge-wide. */
82 readonly rebasing: number;
83 /** Bring-up-to-date runs the lane has room to start now (`crew/conflict_room.ts`). */
84 readonly room: number;
85 }
86
87 export function rebaseLane(db: Database, pulls: readonly ConflictedPull[]): RebaseLane {
88 const lane = readConflictLane(db);
89 return { conflicted: pulls.length, rebasing: lane.runs + lane.checks, room: lane.room };
90 }
91
92 /** `conflicted N, rebasing M, room R`, so a stall reads in a minute; null when nothing is conflicted or rebasing. */
93 export function rebaseLaneSays(lane: RebaseLane): string | null {
94 if (lane.conflicted === 0 && lane.rebasing === 0) return null;
95 return `conflicted ${lane.conflicted}, rebasing ${lane.rebasing}, room ${lane.room}`;
96 }
src/system/telemetry.ts +4 −1
@@ -7,7 +7,7 @@
77 */
88 import type { App } from "../app.ts";
99 import { SOURCES } from "../boards/sources.ts";
10 import { conflictedPulls, conflictedSays } from "../flow/conflicted.ts";
10 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../flow/conflicted.ts";
1111 import { readPipeline, verdict } from "../flow/pipeline.ts";
1212 import { type QueueReading, readQueues } from "../flow/queue.ts";
1313 import { reviewersBusy, reviewingSays } from "../flow/reviewing.ts";
@@ -27,6 +27,7 @@ export function flowFor(app: App, reader: Reader, now = new Date()) {
2727 const flow = readThroughput(app.db, stages.map((s) => s.key), now);
2828 const dup = systemFor(app, reader).duplicates;
2929 const conflicted = conflictedPulls(app.db, reader.kind === "agent");
30 const lane = rebaseLane(app.db, conflicted);
3031 const requested = requestedWaits(app.db, reader.kind === "agent");
3132 const busy = reviewersBusy(app.db, reader.kind === "agent");
3233 return {
@@ -34,6 +35,8 @@ export function flowFor(app: App, reader: Reader, now = new Date()) {
3435 // Review runs in flight per reviewer, cut to public repositories for an agent (#4733).
3536 reviewing: { says: reviewingSays(busy), reviewers: busy },
3637 conflicted: { says: conflictedSays(conflicted, now.getTime()), pulls: conflicted },
38 // conflicted N, rebasing M, room R: a stall in the lane reads in a minute (#6140).
39 rebases: { ...lane, says: rebaseLaneSays(lane) },
3740 // Pulls with changes requested, by whom they wait on (#2172).
3841 requested: {
3942 builder: { says: requestedSays(requested.builder, "builder", now.getTime()), pulls: requested.builder },
src/tui/screens/flow.ts +5 −2
@@ -16,7 +16,7 @@
1616 import type { Database } from "bun:sqlite";
1717 import { HOLDS } from "../../crew/held.ts";
1818 import type { HeldRow } from "../../flow/held.ts";
19 import { conflictedPulls, conflictedSays } from "../../flow/conflicted.ts";
19 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts";
2020 import { readPipeline, STAGE_LABEL, verdict } from "../../flow/pipeline.ts";
2121 import { readQueues, redBatchWords } from "../../flow/queue.ts";
2222 import { reviewersBusy, reviewingSays } from "../../flow/reviewing.ts";
@@ -47,7 +47,9 @@ export function flowLines(
4747 const queues = agent?.queues ?? readQueues(db, now, { batches: true });
4848 // The row is full at 58 columns: the red-batch rate (#4235) is a line under it.
4949 const red = queues.filter((q) => (q.batches?.batches ?? 0) > 0);
50 const conflicted = conflictedSays(conflictedPulls(db, agent !== undefined), now.getTime());
50 const conflictedList = conflictedPulls(db, agent !== undefined);
51 const conflicted = conflictedSays(conflictedList, now.getTime());
52 const lane = rebaseLaneSays(rebaseLane(db, conflictedList));
5153 const requested = requestedWaits(db, agent !== undefined);
5254 const waits = [
5355 requestedSays(requested.builder, "builder", now.getTime()),
@@ -75,6 +77,7 @@ export function flowLines(
7577 ...(red.length === 0
7678 ? []
7779 : wrap(`Red batches, last hour: ${red.map((q) => `${q.repo} ${redBatchWords(q)}`).join(", ")}.`, inner)),
80 ...(lane === null ? [] : ["", ...wrap(`Rebases: ${lane}.`, inner)]),
7881 ...(conflicted === null ? [] : ["", ...wrap(conflicted, inner)]),
7982 ...waits,
8083 ...(reviewing === null ? [] : ["", ...wrap(reviewing, inner)]),
src/web/pages/flow-panel.ts +5 −2
@@ -9,7 +9,7 @@
99
1010 import type { Reader } from "../../actions/telemetry.ts";
1111 import { queuesFor, systemFor } from "../../system/telemetry.ts";
12 import { conflictedPulls, conflictedSays } from "../../flow/conflicted.ts";
12 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts";
1313 import { reviewersBusy, reviewingSays } from "../../flow/reviewing.ts";
1414 import { requestedSays, requestedWaits } from "../../flow/requested.ts";
1515 import {
@@ -90,7 +90,9 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm
9090 const out = new Map(flow.stages.map((t) => [t.key, t]));
9191 const landed = out.get("approved");
9292 const dups = systemFor(app, reader).duplicates;
93 const conflicted = conflictedSays(conflictedPulls(app.db, reader.kind === "agent"), now);
93 const conflictedList = conflictedPulls(app.db, reader.kind === "agent");
94 const conflicted = conflictedSays(conflictedList, now);
95 const lane = rebaseLaneSays(rebaseLane(app.db, conflictedList));
9496 const requested = requestedWaits(app.db, reader.kind === "agent");
9597 const onBuilder = requestedSays(requested.builder, "builder", now);
9698 const onReviewer = requestedSays(requested.reviewer, "reviewer", now);
@@ -101,6 +103,7 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm
101103 html`<p id="bottleneck" data-stage="${worst?.key ?? ""}">
102104 <strong>${verdict(stages, now)}</strong>
103105 </p>
106 ${lane === null ? "" : html`<p id="rebase-lane">Rebases: ${lane}.</p>`}
104107 ${conflicted === null ? "" : html`<p id="conflicted">${conflicted}</p>`}
105108 ${onBuilder === null ? "" : html`<p id="requested-builder">${onBuilder}</p>`}
106109 ${onReviewer === null ? "" : html`<p id="requested-reviewer">${onReviewer}</p>`}
test/conflict_sweep.test.ts +11 −2
@@ -372,8 +372,15 @@ test("one landing that conflicts many pulls starts only as many runs as the chec
372372 });
373373 if (!shared.ok) throw new Error(JSON.stringify(shared.error));
374374 setCapacity(app.db, "checks.concurrency", "3", alice.id);
375 // A check already waits: two slots are left for the pushes to come.
376 const waiting = createRun(app.db, { repoId: repo.id, kind: "check", env: "main" });
375 // A rebase's check already waits: two slots are left for the pushes to
376 // come. Another kind of check does not stand in front of them (#6140).
377 const mainCheck = createRun(app.db, { repoId: repo.id, kind: "check", env: "main" });
378 const rebased = openIssue(app, alice, repo, { title: "Rebased elsewhere" });
379 if (!rebased.ok) throw new Error(String(rebased.error));
380 const pushedBy = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: rebased.value.id, reason: "conflict" });
381 claimRun(app.db, pushedBy.id);
382 finishRun(app.db, pushedBy.id, { state: "ok", summary: "pushed" });
383 const waiting = createRun(app.db, { repoId: repo.id, kind: "check", issueId: rebased.value.id });
377384 try {
378385 expect(await sweepConflicts(app, fresh())).toBe(2);
379386 expect(pulls.map((p) => p.runs().length)).toEqual([1, 1, 0]);
@@ -389,6 +396,8 @@ test("one landing that conflicts many pulls starts only as many runs as the chec
389396 claimRun(app.db, run.id);
390397 finishRun(app.db, run.id, { state: "ok", summary: "" });
391398 }
399 claimRun(app.db, mainCheck.id);
400 finishRun(app.db, mainCheck.id, { state: "ok", summary: "" });
392401 setCapacity(app.db, "checks.concurrency", null, alice.id);
393402 }
394403 });
test/rebase-lane.test.ts added +260
@@ -0,0 +1,260 @@
1 /**
2 * Approved, conflicted pulls do not drain one at a time (#6140): a rebase's
3 * check goes before a new build's in the lane, the room for more rebases
4 * counts only the checks that stand in front of them, a busy builder's
5 * pulls go to idle builders round-robin, and the lane reads as
6 * `conflicted N, rebasing M, room R` on the tick line and /flow.
7 */
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
9 import { writeFileSync } from "node:fs";
10 import { mkdtemp, rm } from "node:fs/promises";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp, type App } from "../src/app.ts";
14 import { openIssue } from "../src/actions/issues.ts";
15 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
16 import { rebasedPulls, rebasesFirst } from "../src/checks/finishing_first.ts";
17 import { startChecks } from "../src/checks/service.ts";
18 import type { Agent } from "../src/core/agents.ts";
19 import { setCapacity } from "../src/core/capacity.ts";
20 import type { Issue } from "../src/core/issues.ts";
21 import type { Repo } from "../src/core/repos.ts";
22 import { cancelRun, createRun, getRun, listActive } from "../src/core/runs.ts";
23 import { createUser, type User } from "../src/core/users.ts";
24 import { readConflictLane } from "../src/crew/conflict_room.ts";
25 import { finishFields, lentBuilder, ownBuilder } from "../src/crew/finishing.ts";
26 import { serve } from "../src/http/server.ts";
27 import {
28 conflictedPulls,
29 noteWaitingToFinish,
30 rebaseLane,
31 rebaseLaneSays,
32 } from "../src/flow/conflicted.ts";
33
34 let dataDir: string;
35 let app: App;
36 let alice: User;
37 let repo: Repo;
38
39 async function issue(title: string): Promise<Issue> {
40 const opened = openIssue(app, alice, repo, { title });
41 if (!opened.ok) throw new Error(String(opened.error));
42 return opened.value;
43 }
44
45 function agentRun(on: Issue, reason: string) {
46 return createRun(app.db, { repoId: repo.id, kind: "agent", issueId: on.id, reason });
47 }
48
49 function checkRun(on: Issue) {
50 return createRun(app.db, { repoId: repo.id, kind: "check", issueId: on.id, reason: "push" });
51 }
52
53 /** Settle every queued or running run, so each test starts from an empty lane. */
54 function settleAll() {
55 for (const kind of ["agent", "check"] as const)
56 for (const run of listActive(app.db, kind)) cancelRun(app.db, run.id, "test");
57 }
58
59 beforeAll(async () => {
60 dataDir = await mkdtemp(join(tmpdir(), "forge-rebase-lane-"));
61 app = createApp({ dataDir, port: 0 });
62 const a = await createUser(app.db, {
63 origin: "seed",
64 trusted: true,
65 username: "alice",
66 email: "[email protected]",
67 password: "a-good-passphrase",
68 adminIfFirst: true,
69 });
70 if (!a.ok) throw new Error(a.error);
71 alice = a.value;
72 const made = await createRepo(app, alice, { name: "engine", visibility: "private" });
73 if (!made.ok) throw new Error(String(made.error));
74 repo = made.value;
75 });
76
77 afterAll(async () => {
78 app.db.close();
79 await rm(dataDir, { recursive: true, force: true });
80 });
81
82 describe("a rebase's check goes first in the lane", () => {
83 test("a pull whose last builder's run was a conflict run is a rebase; a review after it does not change that", async () => {
84 const rebased = await issue("rebased");
85 const reviewed = await issue("rebased, then reviewed");
86 const built = await issue("a new build");
87 const rebuilt = await issue("rebased, then sent back to its builder");
88 agentRun(rebased, "conflict");
89 agentRun(reviewed, "conflict");
90 agentRun(reviewed, "review");
91 agentRun(built, "assigned");
92 agentRun(rebuilt, "conflict");
93 agentRun(rebuilt, "feedback");
94 expect(rebasedPulls(app.db, [rebased.id, reviewed.id, built.id, rebuilt.id])).toEqual(
95 new Set([rebased.id, reviewed.id]),
96 );
97 expect(rebasedPulls(app.db, [])).toEqual(new Set());
98 settleAll();
99 });
100
101 test("rebasesFirst splits queued checks, keeping each side's order", async () => {
102 const rebased = await issue("rebased pull");
103 const built = await issue("built pull");
104 agentRun(rebased, "conflict");
105 agentRun(built, "assigned");
106 const newBuild = checkRun(built);
107 const rebase = checkRun(rebased);
108 const main = createRun(app.db, { repoId: repo.id, kind: "check", env: "main" });
109 const split = rebasesFirst(app.db, [newBuild, rebase, main]);
110 expect(split.rebases.map((r) => r.id)).toEqual([rebase.id]);
111 expect(split.rest.map((r) => r.id)).toEqual([newBuild.id, main.id]);
112 settleAll();
113 });
114 });
115
116 describe("the room counts only the checks in front of a rebase", () => {
117 test("new builds' checks take no room; rebases' checks and conflict runs do", async () => {
118 setCapacity(app.db, "checks.concurrency", "4", alice.id);
119 for (let i = 0; i < 6; i++) {
120 const built = await issue(`build ${i}`);
121 agentRun(built, "assigned");
122 checkRun(built);
123 }
124 // Six new builds' checks in a lane of four: before #6140 the room was one.
125 expect(readConflictLane(app.db)).toEqual({ runs: 0, checks: 0, room: 4 });
126 const rebased = await issue("rebased, check queued");
127 // Its conflict run ended with a push; the push's check waits.
128 cancelRun(app.db, agentRun(rebased, "conflict").id, "pushed");
129 checkRun(rebased);
130 const rebasing = await issue("being rebased");
131 agentRun(rebasing, "conflict");
132 expect(readConflictLane(app.db)).toEqual({ runs: 1, checks: 1, room: 2 });
133 settleAll();
134 });
135 });
136
137 describe("a busy builder's rebases go to idle builders", () => {
138 const builder = (id: string): Agent => ({ user: { id, username: id } }) as unknown as Agent;
139 const [ada, bo, cy, di] = ["ada", "bo", "cy", "di"].map(builder) as [Agent, Agent, Agent, Agent];
140 const crew = [di, bo, ada, cy];
141
142 test("the pull's own builder is the agent on it, else its author", () => {
143 const on = { assigneeId: "cy", authorId: "bo" } as Issue;
144 expect(ownBuilder(on, crew, new Map())?.user.id).toBe("cy");
145 expect(ownBuilder({ ...on, assigneeId: null } as Issue, crew, new Map())?.user.id).toBe("bo");
146 });
147
148 test("an idle own builder keeps its pull; a busy one lends it round-robin among the idle", () => {
149 expect(lentBuilder(ada, crew, new Map())).toBeUndefined();
150 const load = new Map([["ada", 1], ["di", 2]]);
151 const lent = [1, 2, 3, 4].map(() => lentBuilder(ada, crew, load)?.user.id);
152 // Only bo and cy are idle; the turns alternate between them.
153 expect(new Set(lent)).toEqual(new Set(["bo", "cy"]));
154 expect(lent[0]).not.toBe(lent[1]);
155 expect(lent[1]).not.toBe(lent[2]);
156 });
157
158 test("with every builder busy, the update stays with its own builder", () => {
159 const load = new Map(crew.map((b) => [b.user.id, 1]));
160 expect(lentBuilder(ada, crew, load)).toBeUndefined();
161 });
162 });
163
164 describe("the lane reads as conflicted N, rebasing M, room R", () => {
165 test("words, and nothing said when nothing is conflicted or rebasing", () => {
166 expect(rebaseLaneSays({ conflicted: 64, rebasing: 1, room: 1 })).toBe("conflicted 64, rebasing 1, room 1");
167 expect(rebaseLaneSays({ conflicted: 0, rebasing: 0, room: 7 })).toBeNull();
168 });
169
170 test("the tick line carries the lane beside the finishing counts", () => {
171 const fields = finishFields(
172 [
173 { pull: 1, did: "queued update", builder: "bo", for: "ada" },
174 { pull: 2, did: "not moved", why: "waits for room in the check lane" },
175 ],
176 { runs: 1, checks: 2, room: 0 },
177 );
178 expect(fields).toEqual({ finishing: 1, conflict: "1 waiting", lane: "conflicted 2, rebasing 3, room 0" });
179 expect(finishFields([])).toEqual({});
180 });
181
182 test("approved pulls waiting for room count as conflicted, oldest first, and a private one only for the owner", () => {
183 // Other suites' ticks note their own repositories in the same process: read only this one.
184 const mine = (publicOnly = false) =>
185 conflictedPulls(app.db, publicOnly).filter((p) => p.repo === "zed/noted");
186 noteWaitingToFinish("zed-noted", [
187 { repo: "zed/noted", number: 9, since: "2026-10-26T10:00:00.000Z", public: false },
188 { repo: "zed/noted", number: 4, since: "2026-10-26T09:00:00.000Z", public: true },
189 ]);
190 // Seen again later: each keeps when it was first seen.
191 noteWaitingToFinish("zed-noted", [
192 { repo: "zed/noted", number: 9, since: "2026-10-26T11:00:00.000Z", public: false },
193 { repo: "zed/noted", number: 4, since: "2026-10-26T11:00:00.000Z", public: true },
194 ]);
195 expect(mine().map((p) => [p.number, p.since])).toEqual([
196 [4, "2026-10-26T09:00:00.000Z"],
197 [9, "2026-10-26T10:00:00.000Z"],
198 ]);
199 expect(mine(true).map((p) => p.number)).toEqual([4]);
200 expect(rebaseLane(app.db, mine()).conflicted).toBe(2);
201 noteWaitingToFinish("zed-noted", []);
202 expect(mine()).toEqual([]);
203 });
204 });
205
206 describe("the checks service starts a rebase's check before a newer build's", () => {
207 test("with one slot, the older rebase's check runs first though the newest would otherwise win", async () => {
208 const dir = await mkdtemp(join(tmpdir(), "forge-rebase-first-"));
209 const own = createApp({ dataDir: dir, port: 0 });
210 const servers = await serve(own);
211 const go = join(dir, "go");
212 let checks: ReturnType<typeof startChecks> | undefined;
213 try {
214 const made = await createUser(own.db, {
215 origin: "seed",
216 trusted: true,
217 username: "alice",
218 email: "[email protected]",
219 password: "a-good-passphrase",
220 adminIfFirst: true,
221 });
222 if (!made.ok) throw new Error(made.error);
223 const created = await createRepo(own, made.value, { name: "lane", visibility: "public" });
224 if (!created.ok) throw new Error(String(created.error));
225 const lane = created.value;
226 const pushed = await commitToBranch(own, made.value, lane, {
227 message: "Check me",
228 files: [{ path: "forge.json", content: JSON.stringify({ check: { run: `while [ ! -f ${go} ]; do sleep 0.05; done` } }) }],
229 });
230 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
231 const ask = (title: string) => {
232 const opened = openIssue(own, made.value, lane, { title });
233 if (!opened.ok) throw new Error(String(opened.error));
234 return opened.value;
235 };
236 const rebased = ask("rebased");
237 const built = ask("built");
238 createRun(own.db, { repoId: lane.id, kind: "agent", issueId: rebased.id, reason: "conflict" });
239 createRun(own.db, { repoId: lane.id, kind: "agent", issueId: built.id, reason: "assigned" });
240 for (const run of listActive(own.db, "agent")) cancelRun(own.db, run.id, "pushed");
241 const check = (on: Issue) =>
242 createRun(own.db, { repoId: lane.id, kind: "check", env: "production", issueId: on.id, commitOid: pushed.value.commitOid, reason: "push" });
243 const rebase = check(rebased);
244 const build = check(built);
245 setCapacity(own.db, "checks.concurrency", "1", made.value.id);
246 checks = startChecks(own, { forgeUrl: servers.url, concurrency: 1 });
247 const state = (id: string) => getRun(own.db, id)?.state;
248 const deadline = Date.now() + 60_000;
249 while (state(rebase.id) === "queued" && state(build.id) === "queued" && Date.now() < deadline) await Bun.sleep(50);
250 expect(state(rebase.id)).toBe("running");
251 expect(state(build.id)).toBe("queued");
252 } finally {
253 writeFileSync(go, "");
254 await checks?.stop();
255 servers.stop();
256 own.close();
257 await rm(dir, { recursive: true, force: true });
258 }
259 }, 120_000);
260 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs