Public: anyone can read this; the crew can change it.

Land the warming-files line from branch scout/6547-a-pure-read-names-the-files-that-gained, with its history tested and each line naming its repository #6598

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/6583-land-the-warming-files-line-from-branch → main 5 commits fork

  • Merged. scout/6583-land-the-warming-files-line-from-branch was merged into main as 7745a7e.

Review

Sign in to review this change.
10 files changed +285 −8
README.md +1
@@ -681,6 +681,7 @@ This page is the front door. Everything else is a file of its own under
681681 - [`docs/CREW.md`](docs/CREW.md) — the crew: agents, how a run works, review, live triage, the merge queue, spend, roles and invites, the sandbox, passports, and the direction a vision pull will write.
682682 - [`docs/SENTINEL.md`](docs/SENTINEL.md) — the forge hunting its own faults; [`docs/HUNT.md`](docs/HUNT.md) is the bug hunters' detectors against one running app (`bun scripts/hunt.ts <origin>`).
683683 - [`docs/FLOW.md`](docs/FLOW.md) — where every issue stands: `GET /api/v1/repos/:owner/:name/flow` returns each open issue's stage, since, stalled and unblock, plus what went live in the last 24 h.
684 - The flow panel's Hot files card and the terminal Flow dialog also name files that gained open pulls over the last hour ("warming: src/a.ts 2 → 4 pulls", at most 5; `src/flow/hot_trend.ts`). The hour-old counts are samples kept in memory by `src/crew/hot_history.ts`, so after a restart nothing is named until a half-hour-old sample exists; an agent sees public repositories only. The MCP `flow` tool does not carry it yet.
684685 - The flow panel on `/boards` and the terminal Flow dialog say how the open crew pulls sort into piles (red, conflicted, changes asked, approved and waiting, awaiting review), each pull counted once in the first pile it fits (`src/flow/pile.ts`).
685686 - [`docs/PLANNING.md`](docs/PLANNING.md) — how issues depend on issues: the edge, the map, and what "ready now" means to a person and to the crew.
686687 - [`docs/FLEET.md`](docs/FLEET.md) — one change across many repositories, runs or issues: a query, a dry-run plan of every row before → after, a receipt, and undo.
src/crew/hot_files.ts +15 −6
@@ -21,9 +21,10 @@ import { getIssuesByIds, type Issue } from "../core/issues.ts";
2121 import { joinMany, pullDir } from "../core/pulls.ts";
2222 import type { Repo } from "../core/repos.ts";
2323 import { runGitCapture } from "../git/env.ts";
24 import { HOT_PULLS } from "../flow/hot_trend.ts";
25 import { sampleHeat } from "./hot_history.ts";
2426
25 /** A file this many open pulls touch is hot: no new build starts on it. */
26 export const HOT_PULLS = 5;
27 export { HOT_PULLS };
2728
2829 /**
2930 * A pull whose diff touches more than this many files counts toward no file's
@@ -69,9 +70,10 @@ const OID = /^[0-9a-f]{40,64}$/;
6970 /**
7071 * Read the files each open pull of `repo` touches, for any pull whose head
7172 * moved since it was last read, and forget the pulls no longer open. A pull
72 * git cannot answer for touches nothing until it can.
73 * git cannot answer for touches nothing until it can. Each pass samples the
74 * counts at `now` (`hot_history.ts`).
7375 */
74 export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
76 export async function learnPullFiles(app: App, repo: Repo, now: number = Date.now()): Promise<void> {
7577 const ids = app.db
7678 .query<{ id: string }, [string]>(
7779 `SELECT p.issue_id AS id FROM pull p JOIN issue i ON i.id = p.issue_id
@@ -102,15 +104,22 @@ export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
102104 after.set(pull.id, { at, number: pull.number, files: out.stdout.split("\0").filter((f) => f !== "") });
103105 }
104106 touched.set(repo.id, after);
107 sampleHeat(repo.id, pullCounts(repo.id), now);
105108 }
106109
107 /** Every file of `repo` that `HOT_PULLS` or more open pulls touch, with how many. */
108 export function hotFiles(repoId: string): Map<string, number> {
110 /** Every file of `repoId` with how many open pulls touch it, narrow pulls only. */
111 export function pullCounts(repoId: string): Map<string, number> {
109112 const counts = new Map<string, number>();
110113 for (const { files } of touched.get(repoId)?.values() ?? []) {
111114 if (files.length > WIDE_PULL) continue;
112115 for (const file of files) counts.set(file, (counts.get(file) ?? 0) + 1);
113116 }
117 return counts;
118 }
119
120 /** Every file of `repo` that `HOT_PULLS` or more open pulls touch, with how many. */
121 export function hotFiles(repoId: string): Map<string, number> {
122 const counts = pullCounts(repoId);
114123 for (const [file, n] of counts) if (n < HOT_PULLS) counts.delete(file);
115124 return counts;
116125 }
src/crew/hot_history.ts added +83
@@ -0,0 +1,83 @@
1 /**
2 * The hot-file counts an hour ago (#6547), so a file can be named while it
3 * warms. Each `learnPullFiles` pass leaves a sample of every file's count of
4 * open pulls per repository; the baseline is the sample nearest an hour old.
5 * Memory only, like the cache it samples: a restart starts the history over,
6 * and nothing is said until a sample at least `MIN_AGE_MS` old exists.
7 */
8
9 import type { Database } from "bun:sqlite";
10 import { getRepoById } from "../core/repos.ts";
11 import { byRise, hotTrend, hotTrendSays, TREND_NAMED, type Warming } from "../flow/hot_trend.ts";
12
13 const HOUR_MS = 3_600_000;
14 /** A baseline younger than this is not "the last hour" yet. */
15 export const MIN_AGE_MS = HOUR_MS / 2;
16 /** Samples are kept this long, so one is always near an hour old. */
17 const KEEP_MS = HOUR_MS + HOUR_MS / 2;
18 /** A sample closer than this to the last one is not kept. */
19 const EVERY_MS = 5 * 60_000;
20
21 const samples = new Map<string, { at: number; counts: ReadonlyMap<string, number> }[]>();
22
23 /** Keep `counts` for `repoId` as read at `at` (epoch ms), and drop samples too old to matter. */
24 export function sampleHeat(repoId: string, counts: ReadonlyMap<string, number>, at: number): void {
25 const kept = (samples.get(repoId) ?? []).filter((s) => at - s.at <= KEEP_MS);
26 const last = kept[kept.length - 1];
27 if (last === undefined || at - last.at >= EVERY_MS) kept.push({ at, counts: new Map(counts) });
28 samples.set(repoId, kept);
29 }
30
31 /** When each kept sample of `repoId` was read, oldest first. */
32 export function sampleTimes(repoId: string): number[] {
33 return (samples.get(repoId) ?? []).map((s) => s.at);
34 }
35
36 /** Forget every sample; for tests. */
37 export function forgetHeat(): void {
38 samples.clear();
39 }
40
41 export interface WarmingFile extends Warming {
42 readonly repoId: string;
43 }
44
45 /** The sample of `repoId` nearest an hour before `now`, if one is at least `MIN_AGE_MS` old. */
46 function baseline(repoId: string, now: number): ReadonlyMap<string, number> | undefined {
47 let best: { at: number; counts: ReadonlyMap<string, number> } | undefined;
48 for (const s of samples.get(repoId) ?? []) {
49 if (now - s.at < MIN_AGE_MS) continue;
50 if (best === undefined || Math.abs(now - s.at - HOUR_MS) < Math.abs(now - best.at - HOUR_MS)) best = s;
51 }
52 return best?.counts;
53 }
54
55 /** The files warming in the repositories `allow` lets through, largest rise first, at most five; each against its own hour-old sample. */
56 export function warmingFiles(
57 now: number,
58 current: (repoId: string) => ReadonlyMap<string, number>,
59 allow: (repoId: string) => boolean = () => true,
60 ): WarmingFile[] {
61 const out: WarmingFile[] = [];
62 for (const repoId of samples.keys()) {
63 const before = allow(repoId) ? baseline(repoId, now) : undefined;
64 if (before === undefined) continue;
65 for (const w of hotTrend(before, current(repoId))) out.push({ ...w, repoId });
66 }
67 return out.sort((a, b) => byRise(a, b) || a.repoId.localeCompare(b.repoId)).slice(0, TREND_NAMED);
68 }
69
70 /** The warming files a reader may see (every repository, or only the public ones), one line each naming its repository; "" when none. A repository that no longer exists loses its samples. */
71 export function warmingLine(db: Database, now: number, current: (repoId: string) => ReadonlyMap<string, number>, publicOnly: boolean): string {
72 const names = new Map<string, string>();
73 const allow = (id: string): boolean => {
74 const repo = getRepoById(db, id);
75 if (repo == null) {
76 samples.delete(id);
77 return false;
78 }
79 names.set(id, `${repo.ownerName}/${repo.name}`);
80 return !publicOnly || repo.visibility === "public";
81 };
82 return hotTrendSays(warmingFiles(now, current, allow), (w) => names.get(w.repoId) ?? w.repoId);
83 }
src/flow/hot_trend.ts added +39
@@ -0,0 +1,39 @@
1 /**
2 * Files warming up (#6547): the paths whose count of open pulls touching them
3 * rose over the last hour, named before they reach the hot-file limit. A pure
4 * read of two plain maps `path → open pulls`, an hour apart. A file already at
5 * the limit is the hot-file list's to name, and one pull more (0 → 1) is not
6 * a trend.
7 */
8
9 export const TREND_NAMED = 5;
10
11 /** A file this many open pulls touch is hot: no new build starts on it. */
12 export const HOT_PULLS = 5;
13
14 export interface Warming {
15 readonly path: string;
16 readonly from: number;
17 readonly to: number;
18 }
19
20 /** Largest rise first, ties by path. */
21 export function byRise(a: Warming, b: Warming): number {
22 return b.to - b.from - (a.to - a.from) || a.path.localeCompare(b.path);
23 }
24
25 /** Files whose count rose by at least two from `before` to `now` and is still under `HOT_PULLS`, largest rise first (ties by path), at most five; a file absent from `before` counts from 0. */
26 export function hotTrend(before: ReadonlyMap<string, number>, now: ReadonlyMap<string, number>): Warming[] {
27 const rows: Warming[] = [];
28 for (const [path, to] of now) {
29 const from = before.get(path) ?? 0;
30 if (to - from >= 2 && to < HOT_PULLS) rows.push({ path, from, to });
31 }
32 rows.sort(byRise);
33 return rows.slice(0, TREND_NAMED);
34 }
35
36 /** "warming: <path> 2 → 4 pulls", with " in <where>" when a caller names the repository; one line per file; empty when nothing rose. */
37 export function hotTrendSays<R extends Warming>(rows: readonly R[], where?: (row: R) => string): string {
38 return rows.map((r) => `warming: ${r.path} ${r.from} → ${r.to} pulls${where ? ` in ${where(r)}` : ""}`).join("\n");
39 }
src/tui/screens/flow.ts +4
@@ -15,6 +15,8 @@
1515
1616 import type { Database } from "bun:sqlite";
1717 import { HOLDS } from "../../crew/held.ts";
18 import { pullCounts } from "../../crew/hot_files.ts";
19 import { warmingLine } from "../../crew/hot_history.ts";
1820 import type { HeldRow } from "../../flow/held.ts";
1921 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts";
2022 import { pileSays, readPile } from "../../flow/pile.ts";
@@ -60,6 +62,7 @@ export function flowLines(
6062 requestedSays(requested.builder, "builder", now.getTime()),
6163 requestedSays(requested.reviewer, "reviewer", now.getTime()),
6264 ].flatMap((says) => (says === null ? [] : ["", ...wrap(says, inner)]));
65 const warming = warmingLine(db, now.getTime(), pullCounts, agent !== undefined);
6366 const reviewing = reviewingSays(reviewersBusy(db, agent !== undefined));
6467 return [
6568 ...wrap(verdict(stages, now.getTime()), inner),
@@ -91,6 +94,7 @@ export function flowLines(
9194 ...(conflicted === null ? [] : ["", ...wrap(conflicted, inner)]),
9295 ...waits,
9396 ...(reviewing === null ? [] : ["", ...wrap(reviewing, inner)]),
97 ...(warming === "" ? [] : ["", ...warming.split("\n").flatMap((l) => wrap(l, inner))]),
9498 ...(held === undefined ? [] : ["", ...heldLines(held, inner)]),
9599 "",
96100 ...duplicateLines(dups, inner),
src/web/pages/flow-panel.ts +5 −2
@@ -33,7 +33,8 @@ import {
3333 type StageThroughput,
3434 } from "../../flow/throughput.ts";
3535 import { type QueueRow, redBatchWords } from "../../flow/queue.ts";
36 import { HOT_PULLS, warmFiles } from "../../crew/hot_files.ts";
36 import { HOT_PULLS, pullCounts, warmFiles } from "../../crew/hot_files.ts";
37 import { warmingLine } from "../../crew/hot_history.ts";
3738 import { getRepoById } from "../../core/repos.ts";
3839 import { readHeld } from "../../flow/held.ts";
3940 import type { App } from "../../app.ts";
@@ -262,6 +263,7 @@ function hotFilesCard(app: App, reader: Reader): Html {
262263 }
263264 rows.sort((a, b) => b.pulls - a.pulls || a.path.localeCompare(b.path) || a.repo.localeCompare(b.repo));
264265 const top = rows.slice(0, HOT_LISTED);
266 const warming = warmingLine(app.db, Date.now(), pullCounts, reader.kind !== "owner").split("\n").filter((l) => l !== "");
265267 const max = Math.max(1, ...top.map((r) => r.pulls));
266268 return card(
267269 html`<h2>Hot files</h2>
@@ -277,7 +279,8 @@ function hotFilesCard(app: App, reader: Reader): Html {
277279 ${meter((r.pulls / max) * 100, `${r.pulls} open pulls touch ${r.path}`)}
278280 </li>`,
279281 )}
280 </ul>`}`,
282 </ul>`}
283 ${warming.length === 0 ? null : html`<h3>Warming</h3><ul data-warming aria-label="Files warming up">${warming.map((l) => html`<li class="muted small">${l}</li>`)}</ul>`}`,
281284 { id: "hot-files" },
282285 );
283286 }
test/crew-hot-files.test.ts +22
@@ -20,6 +20,7 @@ import type { Pull } from "../src/core/pulls.ts";
2020 import { getRepoById, type Repo } from "../src/core/repos.ts";
2121 import { createToken } from "../src/core/tokens.ts";
2222 import { createUser, getUserByName, type User } from "../src/core/users.ts";
23 import { forgetHeat, warmingLine } from "../src/crew/hot_history.ts";
2324 import { driveRepo } from "../src/crew/autonomy.ts";
2425 import { heldNow, heldWords } from "../src/crew/held.ts";
2526 import {
@@ -29,6 +30,7 @@ import {
2930 holdHot,
3031 hotOverlap,
3132 learnPullFiles,
33 pullCounts,
3234 standingHot,
3335 WIDE_PULL,
3436 widePulls,
@@ -197,6 +199,26 @@ describe("the picker holds a build on a hot file", () => {
197199 expect(readFeed(app.db, alice, { limit: 50 }).items.some((i) => i.id === `hot:${repo.id}`)).toBe(false);
198200 });
199201
202 test("two passes an hour apart name the file that gained pulls, in its repository", async () => {
203 const made = await createRepo(app, alice, { name: "warm", visibility: "private" });
204 if (!made.ok) throw new Error(String(made.error));
205 const repo = fresh(made.value);
206 const seeded = await commitToBranch(app, alice, repo, {
207 message: "Start",
208 files: [{ path: "src/warm.ts", content: "export const n = 0;\n" }],
209 });
210 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
211 const t0 = Date.now();
212 await pullOn(repo, 201, "src/warm.ts");
213 await learnPullFiles(app, fresh(repo), t0);
214 await pullOn(repo, 202, "src/warm.ts");
215 await pullOn(repo, 203, "src/warm.ts");
216 await learnPullFiles(app, fresh(repo), t0 + 3_600_000);
217 const line = warmingLine(app.db, t0 + 3_600_000, pullCounts, false);
218 forgetHeat();
219 expect(line).toBe("warming: src/warm.ts 1 → 3 pulls in alice/warm");
220 });
221
200222 test("a pull wider than WIDE_PULL counts toward no file's heat and is named", async () => {
201223 const made = await createRepo(app, alice, { name: "wide", visibility: "private" });
202224 if (!made.ok) throw new Error(String(made.error));
test/crew-hot-history.test.ts added +29
@@ -0,0 +1,29 @@
1 /** The hourly samples behind the warming lines (#6547): what is kept, pruned, and taken as the baseline. */
2 import { afterEach, expect, test } from "bun:test";
3 import { forgetHeat, sampleHeat, sampleTimes, warmingFiles } from "../src/crew/hot_history.ts";
4
5 const MIN = 60_000;
6 const NOW = 1_000_000_000_000;
7 const counts = (n: number) => new Map([["src/a.ts", n]]);
8
9 afterEach(forgetHeat);
10
11 test("two samples one minute apart keep one", () => {
12 sampleHeat("r", counts(1), NOW);
13 sampleHeat("r", counts(2), NOW + MIN);
14 expect(sampleTimes("r")).toEqual([NOW]);
15 });
16
17 test("a sample two hours old is pruned", () => {
18 sampleHeat("r", counts(1), NOW - 120 * MIN);
19 sampleHeat("r", counts(1), NOW);
20 expect(sampleTimes("r")).toEqual([NOW]);
21 });
22
23 test("among samples 40, 60 and 85 minutes old, the 60-minute one is the baseline", () => {
24 sampleHeat("r", counts(0), NOW - 85 * MIN);
25 sampleHeat("r", counts(1), NOW - 60 * MIN);
26 sampleHeat("r", counts(2), NOW - 40 * MIN);
27 expect(sampleTimes("r")).toHaveLength(3);
28 expect(warmingFiles(NOW, () => counts(4))).toEqual([{ path: "src/a.ts", from: 1, to: 4, repoId: "r" }]);
29 });
test/flow-hot-files.test.ts +45
@@ -8,6 +8,8 @@ import { createApp, type App } from "../src/app.ts";
88 import { createRepo } from "../src/actions/repos.ts";
99 import { createUser } from "../src/core/users.ts";
1010 import { rememberPullFiles } from "../src/crew/hot_files.ts";
11 import { forgetHeat, sampleHeat } from "../src/crew/hot_history.ts";
12 import { flowLines } from "../src/tui/screens/flow.ts";
1113 import { flowPanel } from "../src/web/pages/flow-panel.ts";
1214
1315 describe("flow panel hot files", () => {
@@ -71,4 +73,47 @@ describe("flow panel hot files", () => {
7173 app.db.query("UPDATE repo SET visibility = 'public' WHERE id = ?1").run(repoId);
7274 expect(draw("agent")).toContain("secret/plan.ts");
7375 });
76
77 describe("warming files (#6547)", () => {
78 const HOUR = 3_600_000;
79 const seed = () => {
80 sampleHeat(repoId, new Map([["src/a.ts", 1]]), Date.now() - HOUR);
81 rememberPullFiles(repoId, [1, 2, 3].map((number) => ({ number, files: ["src/a.ts"] })));
82 };
83 afterEach(forgetHeat);
84
85 test("the page and the terminal name a file that gained pulls in the hour", () => {
86 seed();
87 expect(draw("owner")).toContain("warming: src/a.ts 1 → 3 pulls in alice/shop");
88 expect(flowLines(app.db).join("\n")).toContain("warming: src/a.ts 1 → 3 pulls in alice/shop");
89 });
90
91 test("two repositories sharing a path are told apart", async () => {
92 const u = app.db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").get()!;
93 const other = await createRepo(app, { id: u.id } as never, { name: "blog" });
94 if (!other.ok) throw new Error(String(other.error));
95 seed();
96 sampleHeat(other.value.id, new Map([["src/a.ts", 0]]), Date.now() - HOUR);
97 rememberPullFiles(other.value.id, [1, 2].map((number) => ({ number, files: ["src/a.ts"] })));
98 const page = draw("owner");
99 expect(page).toContain("src/a.ts 1 → 3 pulls in alice/shop");
100 expect(page).toContain("src/a.ts 0 → 2 pulls in alice/blog");
101 rememberPullFiles(other.value.id, []);
102 });
103
104 test("without an hour of history nothing is said", () => {
105 sampleHeat(repoId, new Map(), Date.now() - 60_000);
106 rememberPullFiles(repoId, [1, 2, 3].map((number) => ({ number, files: ["src/a.ts"] })));
107 expect(draw("owner")).not.toContain("warming:");
108 expect(flowLines(app.db).join("\n")).not.toContain("warming:");
109 });
110
111 test("a private repository's warming file is for the owner only", () => {
112 seed();
113 app.db.query("UPDATE repo SET visibility = 'private' WHERE id = ?1").run(repoId);
114 expect(draw("owner")).toContain("warming: src/a.ts");
115 expect(draw("agent")).not.toContain("warming:");
116 expect(flowLines(app.db, new Date(), undefined, { queues: [] }).join("\n")).not.toContain("warming:");
117 });
118 });
74119 });
test/flow-hot-trend.test.ts added +42
@@ -0,0 +1,42 @@
1 import { expect, test } from "bun:test";
2 import { HOT_PULLS, hotTrend, hotTrendSays } from "../src/flow/hot_trend.ts";
3
4 const m = (o: Record<string, number>) => new Map(Object.entries(o));
5
6 test("rises sorted by size, falls and unchanged left out", () => {
7 const rows = hotTrend(m({ a: 1, b: 1, c: 3, d: 4 }), m({ a: 3, b: 4, c: 1, d: 4 }));
8 expect(rows).toEqual([
9 { path: "b", from: 1, to: 4 },
10 { path: "a", from: 1, to: 3 },
11 ]);
12 });
13
14 test("equal rises are ordered by path", () => {
15 const rows = hotTrend(m({}), m({ "z.ts": 2, "a.ts": 2, "m.ts": 2 }));
16 expect(rows.map((r) => r.path)).toEqual(["a.ts", "m.ts", "z.ts"]);
17 });
18
19 test("a new file counts from 0", () => {
20 expect(hotTrend(m({}), m({ x: 2 }))).toEqual([{ path: "x", from: 0, to: 2 }]);
21 });
22
23 test("a file at the hot-file limit is left out", () => {
24 expect(hotTrend(m({ a: 1, b: 1 }), m({ a: HOT_PULLS, b: HOT_PULLS - 1 }))).toEqual([{ path: "b", from: 1, to: HOT_PULLS - 1 }]);
25 });
26
27 test("a rise of one pull is not a trend", () => {
28 expect(hotTrend(m({}), m({ x: 1 }))).toEqual([]);
29 expect(hotTrend(m({ x: 2 }), m({ x: 3 }))).toEqual([]);
30 });
31
32 test("at most 5", () => {
33 const now = m(Object.fromEntries("abcdefg".split("").map((p, i) => [p, 2 + (i % 3)])));
34 const rows = hotTrend(m({}), now);
35 expect(rows.length).toBe(5);
36 expect(rows.map((r) => r.path)).toEqual(["c", "f", "b", "e", "a"]);
37 });
38
39 test("words it, empty when nothing rose", () => {
40 expect(hotTrendSays([{ path: "src/a.ts", from: 2, to: 4 }])).toBe("warming: src/a.ts 2 → 4 pulls");
41 expect(hotTrendSays(hotTrend(m({ a: 2 }), m({ a: 2 })))).toBe("");
42 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs