Public: anyone can read this; the crew can change it.

A pull whose diff spans hundreds of files no longer holds every build: the hot-file hold skips a pull wider than 60 files and names it #6236

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/6235-a-pull-whose-diff-spans-hundreds-of-file → main 2 commits fork 3 threads · 3 unresolved

  • Merged. scout/6235-a-pull-whose-diff-spans-hundreds-of-file was merged into main as 22a226d.

Review

Sign in to review this change.
5 files changed +135 −13
docs/CREW.md +12
@@ -56,3 +56,15 @@ strip shows the last hunt and the count of open findings.
5656 Left to a person on purpose: ingress routes at the edge are not in the
5757 database, and the sentinel opens no pull itself — a finding with a mechanical
5858 fix says what it is, and a run of the crew picks the issue up.
59
60 ## Hot files (#5934, #6235)
61
62 A build is not started while a file its issue names is touched by 5 or more
63 open pulls (`HOT_PULLS`). A pull whose diff against the base tip touches more
64 than 60 files (`WIDE_PULL`) counts toward no file's heat: such a diff is
65 almost always the base's history on a conflicted pull, not its own work, and
66 five of them would make hundreds of files hot and hold the whole map. It is
67 named instead, with its file count, on the feed post that says what waits
68 (`too wide to count: #4840 (1,081 files)`), so a person can close or rebase
69 it. A real sweeping change, such as a rename across the tree, goes uncounted
70 too; it conflicts with everything anyway, and it is named.
docs/FLOW.md +3 −1
@@ -92,4 +92,6 @@ the repository paths its title and body name (`src/http/mcp.ts`,
9292 before the picker sorts. The tick line names the hot files (`hot=src/http/mcp.ts=61`),
9393 and each repository's writers see one post on the feed — on the web and
9494 through the MCP `feed` tool — saying which build waits on which file, gone
95 when the hold lifts.
95 when the hold lifts. A pull whose diff touches more than 60 files counts
96 toward no file's heat and is named on the feed (`too wide to count: #4840
97 (1,081 files)`), even when no build is held.
src/crew/hot_files.ts +50 −6
@@ -23,6 +23,13 @@ import { runGitCapture } from "../git/env.ts";
2323 /** A file this many open pulls touch is hot: no new build starts on it. */
2424 export const HOT_PULLS = 5;
2525
26 /**
27 * A pull whose diff touches more than this many files counts toward no file's
28 * heat (#6235): such a diff carries the base's history (a conflicted pull
29 * never rebased), not its own work, and would make every file hot.
30 */
31 export const WIDE_PULL = 60;
32
2633 /** The most paths read from one issue: a list longer than this is a manifest, not a plan. */
2734 const MAX_EXPECTED = 40;
2835
@@ -52,7 +59,7 @@ export interface HotFile {
5259 }
5360
5461 /** Per repository, per open pull: the head and base it was read at, and the files. */
55 const touched = new Map<string, Map<string, { at: string; files: readonly string[] }>>();
62 const touched = new Map<string, Map<string, { at: string; number: number; files: readonly string[] }>>();
5663
5764 const OID = /^[0-9a-f]{40,64}$/;
5865
@@ -71,7 +78,7 @@ export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
7178 .map((r) => r.id);
7279 const pulls = joinMany(app.db, [...getIssuesByIds(app.db, ids).values()]);
7380 const before = touched.get(repo.id) ?? new Map();
74 const after = new Map<string, { at: string; files: readonly string[] }>();
81 const after = new Map<string, { at: string; number: number; files: readonly string[] }>();
7582 for (const pull of pulls) {
7683 if (!OID.test(pull.headOid)) continue;
7784 const dir = await pullDir(app.db, app.reposRoot, repo, pull);
@@ -89,7 +96,7 @@ export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
8996 }
9097 const out = await runGitCapture(["diff", "--name-only", "-z", at], { cwd: dir });
9198 if (out.code !== 0) continue;
92 after.set(pull.id, { at, files: out.stdout.split("\0").filter((f) => f !== "") });
99 after.set(pull.id, { at, number: pull.number, files: out.stdout.split("\0").filter((f) => f !== "") });
93100 }
94101 touched.set(repo.id, after);
95102 }
@@ -97,12 +104,35 @@ export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
97104 /** Every file of `repo` that `HOT_PULLS` or more open pulls touch, with how many. */
98105 export function hotFiles(repoId: string): Map<string, number> {
99106 const counts = new Map<string, number>();
100 for (const { files } of touched.get(repoId)?.values() ?? [])
107 for (const { files } of touched.get(repoId)?.values() ?? []) {
108 if (files.length > WIDE_PULL) continue;
101109 for (const file of files) counts.set(file, (counts.get(file) ?? 0) + 1);
110 }
102111 for (const [file, n] of counts) if (n < HOT_PULLS) counts.delete(file);
103112 return counts;
104113 }
105114
115 /** An open pull too wide to count toward heat, with the files its diff touches. */
116 export interface WidePull {
117 readonly number: number;
118 readonly files: number;
119 }
120
121 /** The open pulls of `repoId` wider than `WIDE_PULL`, widest first. */
122 export function widePulls(repoId: string): WidePull[] {
123 return [...(touched.get(repoId)?.values() ?? [])]
124 .filter((p) => p.files.length > WIDE_PULL)
125 .map((p) => ({ number: p.number, files: p.files.length }))
126 .sort((a, b) => b.files - a.files || a.number - b.number);
127 }
128
129 /** `too wide to count: #4840 (1,081 files), #5460 (898 files)`, or "" when none. */
130 export function wideSays(wide: readonly WidePull[]): string {
131 if (wide.length === 0) return "";
132 const list = wide.map((w) => `#${w.number} (${w.files.toLocaleString("en-US")} files)`).join(", ");
133 return `too wide to count: ${list}`;
134 }
135
106136 /** The hot files `issue` expects to touch, hottest first. */
107137 export function hotOverlap(
108138 issue: Pick<Issue, "title" | "body">,
@@ -124,11 +154,22 @@ export interface HotHold {
124154 readonly files: readonly HotFile[];
125155 }
126156
157 /** The pulls too wide to count as the picker last saw them, per repository, held builds or not. */
158 const standingWideBy = new Map<string, { at: string; wide: readonly WidePull[] }>();
159
160 /** The pulls too wide to count in `repoId` as the picker last saw them, if any. */
161 export function standingWide(repoId: string): { at: string; wide: readonly WidePull[] } | undefined {
162 return standingWideBy.get(repoId);
163 }
164
127165 /** What the picker last held on hot files, per repository, and when. */
128 const standing = new Map<string, { at: string; holds: readonly HotHold[] }>();
166 const standing = new Map<string, { at: string; holds: readonly HotHold[]; wide: readonly WidePull[] }>();

wide here duplicates standingWideBy; nothing reads it. Remove the field and keep standingHot returning its old shape.

129167
130168 /** Remember the builds the picker holds on hot files in `repoId` now; none clears it. */
131169 export function holdHot(repoId: string, issues: readonly Issue[], at = new Date()): void {
170 const wide = widePulls(repoId);
171 if (wide.length === 0) standingWideBy.delete(repoId);
172 else standingWideBy.set(repoId, { at: at.toISOString(), wide });
132173 if (issues.length === 0) {
133174 standing.delete(repoId);
134175 return;
@@ -137,11 +178,14 @@ export function holdHot(repoId: string, issues: readonly Issue[], at = new Date(
137178 standing.set(repoId, {
138179 at: at.toISOString(),
139180 holds: issues.map((i) => ({ number: i.number, title: i.title, files: hotOverlap(i, hot) })),
181 wide,
140182 });
141183 }
142184
143185 /** The builds held on hot files in `repoId` as the picker last said, if any. */
144 export function standingHot(repoId: string): { at: string; holds: readonly HotHold[] } | undefined {
186 export function standingHot(
187 repoId: string,
188 ): { at: string; holds: readonly HotHold[]; wide: readonly WidePull[] } | undefined {
145189 return standing.get(repoId);
146190 }
147191
src/feed/items.ts +14 −6
@@ -33,7 +33,7 @@ import { chooseCandidates } from "./choose.ts";
3333 import { writeStamp } from "../core/write_clock.ts";
3434 import { Stamped } from "../kernel/stamped.ts";
3535 import { QUIET_WINDOW_MS, quietRunEventIds } from "./quiet.ts";
36 import { hotSays, standingHot } from "../crew/hot_files.ts";
36 import { hotSays, standingHot, standingWide, wideSays } from "../crew/hot_files.ts";
3737 import { ago } from "./rank.ts";
3838 import { askLinesFor } from "../web/studio/asks.ts";
3939
@@ -278,10 +278,18 @@ function hotCandidates(writable: readonly Repo[]): Candidate[] {
278278 const out: Candidate[] = [];
279279 for (const repo of writable) {
280280 const standing = standingHot(repo.id);
281 if (standing === undefined) continue;
282 const n = standing.holds.length;
283 const first = standing.holds[0]!;
284 const what = `${n === 1 ? "1 build waits" : `${n} builds wait`} on hot files: ${hotSays(first)}${n > 1 ? ` And ${n - 1} more.` : ""}`;
281 const wide = standingWide(repo.id);
282 if (standing === undefined && wide === undefined) continue;
283 const wideSuffix = wide === undefined ? "" : wideSays(wide.wide);
284 let what: string;
outdated

This line is only reached when standing exists, so wide pulls are named only while some build is held. Emit the wide post when nothing is held as well. Also, this line is now long and nested; pull the wide suffix out into a variable.

285 if (standing === undefined) {
286 what = `${wideSuffix} - close or rebase.`;
287 } else {
288 const n = standing.holds.length;
289 const more = n > 1 ? ` And ${n - 1} more.` : "";
290 const held = `${n === 1 ? "1 build waits" : `${n} builds wait`} on hot files: ${hotSays(standing.holds[0]!)}${more}`;
291 what = wideSuffix === "" ? held : `${held} ${wideSuffix}.`;

The held text plus the wide suffix goes through clip, which can drop the wide part on a long title. Put the wide part where clipping cannot remove it, and add a test for the held case.

292 }
285293 out.push({
286294 id: `hot:${repo.id}`,
287295 kind: "map",
@@ -289,7 +297,7 @@ function hotCandidates(writable: readonly Repo[]): Candidate[] {
289297 app: slug(repo),
290298 actorId: null,
291299 actor: null,
292 at: standing.at,
300 at: (standing ?? wide!).at,
293301 link: `${base(repo)}/map`,
294302 what: clip(what),
295303 urgency: 0,
test/crew-hot-files.test.ts +56
@@ -26,9 +26,13 @@ import {
2626 expectedFiles,
2727 HOT_PULLS,
2828 hotFiles,
29 holdHot,
2930 hotOverlap,
3031 learnPullFiles,
3132 standingHot,
33 WIDE_PULL,
34 widePulls,
35 wideSays,
3236 } from "../src/crew/hot_files.ts";
3337 import { readFeed } from "../src/feed/feed.ts";
3438 import { serve, type Servers } from "../src/http/server.ts";
@@ -192,4 +196,56 @@ describe("the picker holds a build on a hot file", () => {
192196 expect(standingHot(repo.id)).toBeUndefined();
193197 expect(readFeed(app.db, alice, { limit: 50 }).items.some((i) => i.id === `hot:${repo.id}`)).toBe(false);
194198 });
199
200 test("a pull wider than WIDE_PULL counts toward no file's heat and is named", async () => {
201 const made = await createRepo(app, alice, { name: "wide", visibility: "private" });
202 if (!made.ok) throw new Error(String(made.error));
203 const repo = fresh(made.value);
204 const seeded = await commitToBranch(app, alice, repo, {
205 message: "Start",
206 files: [{ path: "src/hot.ts", content: "export const n = 0;\n" }],
207 });
208 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
209 const wide = async (n: number): Promise<Pull> => {
210 const files = Array.from({ length: WIDE_PULL + 1 }, (_, k) => ({
211 path: k === 0 ? "src/hot.ts" : `gen/${n}/f${k}.ts`,
212 content: `export const n = ${n};\n`,
213 }));
214 const branch = await commitToBranch(app, alice, fresh(repo), {
215 branch: `wide/${n}`,
216 base: "main",
217 message: `Wide ${n}`,
218 files,
219 });
220 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
221 const pull = await openPull(app, alice, fresh(repo), {
222 title: `Wide ${n}`,
223 body: "",
224 baseBranch: "main",
225 headBranch: `wide/${n}`,
226 });
227 if (!pull.ok) throw new Error(String(pull.error));
228 return pull.value;
229 };
230 const wides: Pull[] = [];
231 for (let n = 1; n <= HOT_PULLS; n++) wides.push(await wide(n));
232 wides.push(await pullOn(repo, 90, "src/hot.ts"));
233 await learnPullFiles(app, fresh(repo));
234 expect(hotFiles(repo.id).size).toBe(0);
235 expect(widePulls(repo.id)).toHaveLength(HOT_PULLS);
236 // No build is held, yet the feed still names them.
237 const work = issue(repo, "Add a flag to src/hot.ts");
238 expect(heldNow(app, fresh(repo)).by.hot_file).toEqual([]);
239 holdHot(repo.id, []);
240 const said = widePulls(repo.id).map((w) => `#${w.number} (${w.files} files)`).join(", ");
241 const post = readFeed(app.db, alice, { limit: 50 }).items.find((i) => i.id === `hot:${repo.id}`);
242 expect(post?.what).toBe(`too wide to count: ${said} - close or rebase.`);
243 expect(wideSays([{ number: 4840, files: 1081 }])).toBe("too wide to count: #4840 (1,081 files)");
244
245 // Four ordinary pulls plus one more on the same file still hold it.
246 for (let n = 91; n <= 94; n++) await pullOn(repo, n, "src/hot.ts");
247 await learnPullFiles(app, fresh(repo));
248 expect(hotFiles(repo.id)).toEqual(new Map([["src/hot.ts", 5]]));
249 expect(heldNow(app, fresh(repo)).by.hot_file.map((i) => i.number)).toEqual([work.number]);
250 });
195251 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs