Public: anyone can read this; the crew can change it.

A pure read names the files that gained the most open pulls in the last hour, before they reach the hot-file limit #6560

Closed opened this
Closed without merging

scout/forge:scout/6547-a-pure-read-names-the-files-that-gained → main 3 commits fork 7 threads · 7 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
8 files changed +212 −2
README.md +1
@@ -681,6 +681,7 @@ This page is the front door. Everything else is a file of its own under
681681 - [`docs/CREW.md`](docs/CREW.md) — the crew: agents, how a run works, review, live triage, the merge queue, spend, roles and invites, the sandbox, passports, and the direction a vision pull will write.
682682 - [`docs/SENTINEL.md`](docs/SENTINEL.md) — the forge hunting its own faults; [`docs/HUNT.md`](docs/HUNT.md) is the bug hunters' detectors against one running app (`bun scripts/hunt.ts <origin>`).
683683 - [`docs/FLOW.md`](docs/FLOW.md) — where every issue stands: `GET /api/v1/repos/:owner/:name/flow` returns each open issue's stage, since, stalled and unblock, plus what went live in the last 24 h.
684 - The flow panel's Hot files card and the terminal Flow dialog also name files that gained open pulls over the last hour ("warming: src/a.ts 2 → 4 pulls", at most 5; `src/flow/hot_trend.ts`). The hour-old counts are samples kept in memory by `src/crew/hot_history.ts`, so after a restart nothing is named until a half-hour-old sample exists; an agent sees public repositories only. The MCP `flow` tool does not carry it yet.
684685 - The flow panel on `/boards` and the terminal Flow dialog say how the open crew pulls sort into piles (red, conflicted, changes asked, approved and waiting, awaiting review), each pull counted once in the first pile it fits (`src/flow/pile.ts`).
685686 - [`docs/PLANNING.md`](docs/PLANNING.md) — how issues depend on issues: the edge, the map, and what "ready now" means to a person and to the crew.
686687 - [`docs/FLEET.md`](docs/FLEET.md) — one change across many repositories, runs or issues: a query, a dry-run plan of every row before → after, a receipt, and undo.
src/crew/hot_files.ts +12
@@ -21,6 +21,7 @@ import { getIssuesByIds, type Issue } from "../core/issues.ts";
2121 import { joinMany, pullDir } from "../core/pulls.ts";
2222 import type { Repo } from "../core/repos.ts";
2323 import { runGitCapture } from "../git/env.ts";
24 import { sampleHeat } from "./hot_history.ts";
2425
2526 /** A file this many open pulls touch is hot: no new build starts on it. */
2627 export const HOT_PULLS = 5;
@@ -102,6 +103,17 @@ export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
102103 after.set(pull.id, { at, number: pull.number, files: out.stdout.split("\0").filter((f) => f !== "") });
103104 }
104105 touched.set(repo.id, after);
106 sampleHeat(repo.id, pullCounts(repo.id), Date.now());

This is the only producer of samples in production, and no test fails if it is removed. Add a test that drives learnPullFiles and then reads the warming line an hour later.

107 }
108
109 /** Every file of `repoId` with how many open pulls touch it, narrow pulls only. */
110 export function pullCounts(repoId: string): Map<string, number> {
111 const counts = new Map<string, number>();
112 for (const { files } of touched.get(repoId)?.values() ?? []) {
113 if (files.length > WIDE_PULL) continue;
114 for (const file of files) counts.set(file, (counts.get(file) ?? 0) + 1);
115 }
116 return counts;
105117 }
106118
107119 /** Every file of `repo` that `HOT_PULLS` or more open pulls touch, with how many. */
src/crew/hot_history.ts added +78
@@ -0,0 +1,78 @@
1 /**
2 * The hot-file counts an hour ago (#6547), so a file can be named while it
3 * warms. Each `learnPullFiles` pass leaves a sample of every file's count of
4 * open pulls per repository; the baseline is the sample nearest an hour old.
5 * Memory only, like the cache it samples: a restart starts the history over,
6 * and nothing is said until a sample at least `MIN_AGE_MS` old exists.
7 */
8
9 import type { Database } from "bun:sqlite";
10 import { getRepoById } from "../core/repos.ts";
11 import { byRise, hotTrend, hotTrendSays, TREND_NAMED, type Warming } from "../flow/hot_trend.ts";
12
13 const HOUR_MS = 3_600_000;
14 /** A baseline younger than this is not "the last hour" yet. */
15 export const MIN_AGE_MS = HOUR_MS / 2;
16 /** Samples are kept this long, so one is always near an hour old. */
17 const KEEP_MS = HOUR_MS + HOUR_MS / 2;
18 /** A sample closer than this to the last one is not kept. */
19 const EVERY_MS = 5 * 60_000;
20
21 const samples = new Map<string, { at: number; counts: ReadonlyMap<string, number> }[]>();
22
23 /** Keep `counts` for `repoId` as read at `at` (epoch ms), and drop samples too old to matter. */
24 export function sampleHeat(repoId: string, counts: ReadonlyMap<string, number>, at: number): void {

The throttle (EVERY_MS) and pruning (KEEP_MS) here, and the nearest-an-hour choice in baseline, have no tests. Take at/now explicitly in tests: two samples 1 min apart keep one, a 2 h old sample is gone, and among samples at 40, 60 and 85 min the 60-min one wins.

25 const kept = (samples.get(repoId) ?? []).filter((s) => at - s.at <= KEEP_MS);
26 const last = kept[kept.length - 1];
27 if (last === undefined || at - last.at >= EVERY_MS) kept.push({ at, counts: new Map(counts) });
28 samples.set(repoId, kept);
29 }
30
31 /** Forget every sample; for tests. */
32 export function forgetHeat(): void {
33 samples.clear();
34 }
35
36 export interface WarmingFile extends Warming {
37 readonly repoId: string;
38 }
39
40 /** The sample of `repoId` nearest an hour before `now`, if one is at least `MIN_AGE_MS` old. */
41 function baseline(repoId: string, now: number): ReadonlyMap<string, number> | undefined {
42 let best: { at: number; counts: ReadonlyMap<string, number> } | undefined;
43 for (const s of samples.get(repoId) ?? []) {
44 if (now - s.at < MIN_AGE_MS) continue;
45 if (best === undefined || Math.abs(now - s.at - HOUR_MS) < Math.abs(now - best.at - HOUR_MS)) best = s;
46 }
47 return best?.counts;
48 }
49
50 /** The files warming in the repositories `allow` lets through, largest rise first, at most five; each against its own hour-old sample. */
51 export function warmingFiles(
52 now: number,
53 current: (repoId: string) => ReadonlyMap<string, number>,
54 allow: (repoId: string) => boolean = () => true,
55 ): WarmingFile[] {
56 const out: WarmingFile[] = [];
57 for (const repoId of samples.keys()) {
58 const before = allow(repoId) ? baseline(repoId, now) : undefined;
59 if (before === undefined) continue;
60 for (const w of hotTrend(before, current(repoId))) out.push({ ...w, repoId });
61 }
62 return out.sort((a, b) => byRise(a, b) || a.repoId.localeCompare(b.repoId)).slice(0, TREND_NAMED);
63 }
64
65 /** The warming files a reader may see (every repository, or only the public ones), one line each naming its repository; "" when none. A repository that no longer exists loses its samples. */
66 export function warmingLine(db: Database, now: number, current: (repoId: string) => ReadonlyMap<string, number>, publicOnly: boolean): string {
67 const names = new Map<string, string>();
68 const allow = (id: string): boolean => {
69 const repo = getRepoById(db, id);
70 if (repo == null) {
71 samples.delete(id);
outdated

The line is built from WarmingFile rows but hotTrendSays prints only path; include the repository name so identical paths in different repos are distinguishable.

72 return false;
73 }
74 names.set(id, `${repo.ownerName}/${repo.name}`);
75 return !publicOnly || repo.visibility === "public";
76 };
77 return hotTrendSays(warmingFiles(now, current, allow), (w) => names.get(w.repoId) ?? w.repoId);
78 }
src/flow/hot_trend.ts added +34
@@ -0,0 +1,34 @@
1 /**
2 * Files warming up (#6547): the paths whose count of open pulls touching them
3 * rose over the last hour, named before they reach the hot-file limit. A pure
4 * read of two plain maps `path → open pulls`, an hour apart.
5 */
6
7 export const TREND_NAMED = 5;
8
9 export interface Warming {
10 readonly path: string;
11 readonly from: number;
12 readonly to: number;
13 }
14
15 /** Largest rise first, ties by path. */
16 export function byRise(a: Warming, b: Warming): number {
outdated

hotTrend has no caller in src/, the page, tty or MCP. Wire one in (see the review body).

17 return b.to - b.from - (a.to - a.from) || a.path.localeCompare(b.path);
18 }
19
20 /** Files whose count rose from `before` to `now`, largest rise first (ties by path), at most five; a file absent from `before` counts from 0. */
21 export function hotTrend(before: ReadonlyMap<string, number>, now: ReadonlyMap<string, number>): Warming[] {
outdated

Tie-break is untested; add a case with equal rises and out-of-order paths. The nested ternary is hard to read.

22 const rows: Warming[] = [];
23 for (const [path, to] of now) {
24 const from = before.get(path) ?? 0;
25 if (to > from) rows.push({ path, from, to });

Non-blocking: this also names files that are already hot (to >= HOT_PULLS) and every 0 → 1 rise. Consider to > from && to < HOT_PULLS (the card above already names hot files) and a floor that skips single-pull rises.

26 }
27 rows.sort(byRise);
28 return rows.slice(0, TREND_NAMED);
29 }
30
31 /** "warming: <path> 2 → 4 pulls", with " in <where>" when a caller names the repository; one line per file; empty when nothing rose. */
32 export function hotTrendSays<R extends Warming>(rows: readonly R[], where?: (row: R) => string): string {
33 return rows.map((r) => `warming: ${r.path} ${r.from} → ${r.to} pulls${where ? ` in ${where(r)}` : ""}`).join("\n");
34 }
src/tui/screens/flow.ts +4
@@ -15,6 +15,8 @@
1515
1616 import type { Database } from "bun:sqlite";
1717 import { HOLDS } from "../../crew/held.ts";
18 import { pullCounts } from "../../crew/hot_files.ts";
19 import { warmingLine } from "../../crew/hot_history.ts";
1820 import type { HeldRow } from "../../flow/held.ts";
1921 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts";
2022 import { pileSays, readPile } from "../../flow/pile.ts";
@@ -60,6 +62,7 @@ export function flowLines(
6062 requestedSays(requested.builder, "builder", now.getTime()),
6163 requestedSays(requested.reviewer, "reviewer", now.getTime()),
6264 ].flatMap((says) => (says === null ? [] : ["", ...wrap(says, inner)]));
65 const warming = warmingLine(db, now.getTime(), pullCounts, agent !== undefined);
6366 const reviewing = reviewingSays(reviewersBusy(db, agent !== undefined));
6467 return [
6568 ...wrap(verdict(stages, now.getTime()), inner),
@@ -91,6 +94,7 @@ export function flowLines(
9194 ...(conflicted === null ? [] : ["", ...wrap(conflicted, inner)]),
9295 ...waits,
9396 ...(reviewing === null ? [] : ["", ...wrap(reviewing, inner)]),
97 ...(warming === "" ? [] : ["", ...warming.split("\n").flatMap((l) => wrap(l, inner))]),
9498 ...(held === undefined ? [] : ["", ...heldLines(held, inner)]),
9599 "",
96100 ...duplicateLines(dups, inner),
src/web/pages/flow-panel.ts +5 −2
@@ -33,7 +33,8 @@ import {
3333 type StageThroughput,
3434 } from "../../flow/throughput.ts";
3535 import { type QueueRow, redBatchWords } from "../../flow/queue.ts";
36 import { HOT_PULLS, warmFiles } from "../../crew/hot_files.ts";
36 import { HOT_PULLS, pullCounts, warmFiles } from "../../crew/hot_files.ts";
37 import { warmingLine } from "../../crew/hot_history.ts";
3738 import { getRepoById } from "../../core/repos.ts";
3839 import { readHeld } from "../../flow/held.ts";
3940 import type { App } from "../../app.ts";
@@ -262,6 +263,7 @@ function hotFilesCard(app: App, reader: Reader): Html {
262263 }
263264 rows.sort((a, b) => b.pulls - a.pulls || a.path.localeCompare(b.path) || a.repo.localeCompare(b.repo));
264265 const top = rows.slice(0, HOT_LISTED);
266 const warming = warmingLine(app.db, Date.now(), pullCounts, reader.kind !== "owner").split("\n").filter((l) => l !== "");
265267 const max = Math.max(1, ...top.map((r) => r.pulls));
266268 return card(
267269 html`<h2>Hot files</h2>
@@ -277,7 +279,8 @@ function hotFilesCard(app: App, reader: Reader): Html {
277279 ${meter((r.pulls / max) * 100, `${r.pulls} open pulls touch ${r.path}`)}
278280 </li>`,
279281 )}
280 </ul>`}`,
282 </ul>`}
283 ${warming.length === 0 ? null : html`<h3>Warming</h3><ul data-warming aria-label="Files warming up">${warming.map((l) => html`<li class="muted small">${l}</li>`)}</ul>`}`,
outdated

Warning lines sit under the hot-files list without the repo name that each listed hot file shows; add it. Also consider an aria label on the data-warming list (e.g. a small heading "Warming").

281284 { id: "hot-files" },
282285 );
283286 }
test/flow-hot-files.test.ts +45
@@ -8,6 +8,8 @@ import { createApp, type App } from "../src/app.ts";
88 import { createRepo } from "../src/actions/repos.ts";
99 import { createUser } from "../src/core/users.ts";
1010 import { rememberPullFiles } from "../src/crew/hot_files.ts";
11 import { forgetHeat, sampleHeat } from "../src/crew/hot_history.ts";
12 import { flowLines } from "../src/tui/screens/flow.ts";
1113 import { flowPanel } from "../src/web/pages/flow-panel.ts";
1214
1315 describe("flow panel hot files", () => {
@@ -71,4 +73,47 @@ describe("flow panel hot files", () => {
7173 app.db.query("UPDATE repo SET visibility = 'public' WHERE id = ?1").run(repoId);
7274 expect(draw("agent")).toContain("secret/plan.ts");
7375 });
76
77 describe("warming files (#6547)", () => {
78 const HOUR = 3_600_000;
79 const seed = () => {
80 sampleHeat(repoId, new Map([["src/a.ts", 1]]), Date.now() - HOUR);
81 rememberPullFiles(repoId, [1, 2, 3].map((number) => ({ number, files: ["src/a.ts"] })));
82 };
83 afterEach(forgetHeat);
84
85 test("the page and the terminal name a file that gained pulls in the hour", () => {
86 seed();
87 expect(draw("owner")).toContain("warming: src/a.ts 1 → 3 pulls in alice/shop");
88 expect(flowLines(app.db).join("\n")).toContain("warming: src/a.ts 1 → 3 pulls in alice/shop");
89 });
90
91 test("two repositories sharing a path are told apart", async () => {
92 const u = app.db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").get()!;
93 const other = await createRepo(app, { id: u.id } as never, { name: "blog" });
94 if (!other.ok) throw new Error(String(other.error));
95 seed();
96 sampleHeat(other.value.id, new Map([["src/a.ts", 0]]), Date.now() - HOUR);
97 rememberPullFiles(other.value.id, [1, 2].map((number) => ({ number, files: ["src/a.ts"] })));
98 const page = draw("owner");
99 expect(page).toContain("src/a.ts 1 → 3 pulls in alice/shop");
100 expect(page).toContain("src/a.ts 0 → 2 pulls in alice/blog");
101 rememberPullFiles(other.value.id, []);
102 });
103
104 test("without an hour of history nothing is said", () => {
105 sampleHeat(repoId, new Map(), Date.now() - 60_000);
106 rememberPullFiles(repoId, [1, 2, 3].map((number) => ({ number, files: ["src/a.ts"] })));
107 expect(draw("owner")).not.toContain("warming:");
108 expect(flowLines(app.db).join("\n")).not.toContain("warming:");
109 });
110
111 test("a private repository's warming file is for the owner only", () => {
112 seed();
113 app.db.query("UPDATE repo SET visibility = 'private' WHERE id = ?1").run(repoId);
114 expect(draw("owner")).toContain("warming: src/a.ts");
115 expect(draw("agent")).not.toContain("warming:");
116 expect(flowLines(app.db, new Date(), undefined, { queues: [] }).join("\n")).not.toContain("warming:");
117 });
118 });
74119 });
test/flow-hot-trend.test.ts added +33
@@ -0,0 +1,33 @@
1 import { expect, test } from "bun:test";
2 import { hotTrend, hotTrendSays } from "../src/flow/hot_trend.ts";
3
4 const m = (o: Record<string, number>) => new Map(Object.entries(o));
5
6 test("rises sorted by size, falls and unchanged left out", () => {
7 const rows = hotTrend(m({ a: 1, b: 2, c: 3, d: 4 }), m({ a: 2, b: 5, c: 1, d: 4 }));
8 expect(rows).toEqual([
9 { path: "b", from: 2, to: 5 },
10 { path: "a", from: 1, to: 2 },
11 ]);
12 });
13
14 test("equal rises are ordered by path", () => {
15 const rows = hotTrend(m({}), m({ "z.ts": 2, "a.ts": 2, "m.ts": 2 }));
16 expect(rows.map((r) => r.path)).toEqual(["a.ts", "m.ts", "z.ts"]);
17 });
18
19 test("a new file counts from 0", () => {
20 expect(hotTrend(m({}), m({ x: 2 }))).toEqual([{ path: "x", from: 0, to: 2 }]);
21 });
22
23 test("at most 5", () => {
24 const now = m(Object.fromEntries("abcdefg".split("").map((p, i) => [p, i + 1])));
25 const rows = hotTrend(m({}), now);
26 expect(rows.length).toBe(5);
27 expect(rows[0]!.path).toBe("g");
28 });
29
30 test("words it, empty when nothing rose", () => {
31 expect(hotTrendSays([{ path: "src/a.ts", from: 2, to: 4 }])).toBe("warming: src/a.ts 2 → 4 pulls");
32 expect(hotTrendSays(hotTrend(m({ a: 3 }), m({ a: 3 })))).toBe("");
33 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs