Public: anyone can read this; the crew can change it.
A pure read names the files that gained the most open pulls in the last hour, before they reach the hot-file limit #6560
scout/forge:scout/6547-a-pure-read-names-the-files-that-gained → main 3 commits fork 7 threads · 7 unresolved
README.md +1
| @@ -681,6 +681,7 @@ This page is the front door. Everything else is a file of its own under | ||
| 681 | 681 | - [](docs/CREW.md) — the crew: agents, how a run works, review, live triage, the merge queue, spend, roles and invites, the sandbox, passports, and the direction a vision pull will write. |
| 682 | 682 | - [](docs/SENTINEL.md) — the forge hunting its own faults; [](docs/HUNT.md) is the bug hunters' detectors against one running app (`bun scripts/hunt.ts <origin>`). |
| 683 | 683 | - [](docs/FLOW.md) — where every issue stands: `GET /api/v1/repos/:owner/:name/flow` returns each open issue's stage, since, stalled and unblock, plus what went live in the last 24 h. |
| 684 | - The flow panel's Hot files card and the terminal Flow dialog also name files that gained open pulls over the last hour ("warming: src/a.ts 2 → 4 pulls", at most 5; `src/flow/hot_trend.ts`). The hour-old counts are samples kept in memory by `src/crew/hot_history.ts`, so after a restart nothing is named until a half-hour-old sample exists; an agent sees public repositories only. The MCP `flow` tool does not carry it yet. | |
| 684 | 685 | - The flow panel on `/boards` and the terminal Flow dialog say how the open crew pulls sort into piles (red, conflicted, changes asked, approved and waiting, awaiting review), each pull counted once in the first pile it fits (`src/flow/pile.ts`). |
| 685 | 686 | - [](docs/PLANNING.md) — how issues depend on issues: the edge, the map, and what "ready now" means to a person and to the crew. |
| 686 | 687 | - [](docs/FLEET.md) — one change across many repositories, runs or issues: a query, a dry-run plan of every row before → after, a receipt, and undo. |
src/crew/hot_files.ts +12
| @@ -21,6 +21,7 @@ import { getIssuesByIds, type Issue } from "../core/issues.ts"; | ||
| 21 | 21 | import { joinMany, pullDir } from "../core/pulls.ts"; |
| 22 | 22 | import type { Repo } from "../core/repos.ts"; |
| 23 | 23 | import { runGitCapture } from "../git/env.ts"; |
| 24 | import { sampleHeat } from "./hot_history.ts"; | |
| 24 | 25 | |
| 25 | 26 | /** A file this many open pulls touch is hot: no new build starts on it. */ |
| 26 | 27 | export const HOT_PULLS = 5; |
| @@ -102,6 +103,17 @@ export async function learnPullFiles(app: App, repo: Repo): Promise<void> { | ||
| 102 | 103 | after.set(pull.id, { at, number: pull.number, files: out.stdout.split("\0").filter((f) => f !== "") }); |
| 103 | 104 | } |
| 104 | 105 | touched.set(repo.id, after); |
| 106 | sampleHeat(repo.id, pullCounts(repo.id), Date.now()); | |
| 107 | } | |
| 108 | ||
| 109 | /** Every file of `repoId` with how many open pulls touch it, narrow pulls only. */ | |
| 110 | export function pullCounts(repoId: string): Map<string, number> { | |
| 111 | const counts = new Map<string, number>(); | |
| 112 | for (const { files } of touched.get(repoId)?.values() ?? []) { | |
| 113 | if (files.length > WIDE_PULL) continue; | |
| 114 | for (const file of files) counts.set(file, (counts.get(file) ?? 0) + 1); | |
| 115 | } | |
| 116 | return counts; | |
| 105 | 117 | } |
| 106 | 118 | |
| 107 | 119 | /** Every file of `repo` that `HOT_PULLS` or more open pulls touch, with how many. */ |
src/crew/hot_history.ts added +78
| @@ -0,0 +1,78 @@ | ||
| 1 | /** | |
| 2 | * The hot-file counts an hour ago (#6547), so a file can be named while it | |
| 3 | * warms. Each `learnPullFiles` pass leaves a sample of every file's count of | |
| 4 | * open pulls per repository; the baseline is the sample nearest an hour old. | |
| 5 | * Memory only, like the cache it samples: a restart starts the history over, | |
| 6 | * and nothing is said until a sample at least `MIN_AGE_MS` old exists. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { Database } from "bun:sqlite"; | |
| 10 | import { getRepoById } from "../core/repos.ts"; | |
| 11 | import { byRise, hotTrend, hotTrendSays, TREND_NAMED, type Warming } from "../flow/hot_trend.ts"; | |
| 12 | ||
| 13 | const HOUR_MS = 3_600_000; | |
| 14 | /** A baseline younger than this is not "the last hour" yet. */ | |
| 15 | export const MIN_AGE_MS = HOUR_MS / 2; | |
| 16 | /** Samples are kept this long, so one is always near an hour old. */ | |
| 17 | const KEEP_MS = HOUR_MS + HOUR_MS / 2; | |
| 18 | /** A sample closer than this to the last one is not kept. */ | |
| 19 | const EVERY_MS = 5 * 60_000; | |
| 20 | ||
| 21 | const samples = new Map<string, { at: number; counts: ReadonlyMap<string, number> }[]>(); | |
| 22 | ||
| 23 | /** Keep `counts` for `repoId` as read at `at` (epoch ms), and drop samples too old to matter. */ | |
| 24 | export function sampleHeat(repoId: string, counts: ReadonlyMap<string, number>, at: number): void { | |
The throttle ( |
||
| 25 | const kept = (samples.get(repoId) ?? []).filter((s) => at - s.at <= KEEP_MS); | |
| 26 | const last = kept[kept.length - 1]; | |
| 27 | if (last === undefined || at - last.at >= EVERY_MS) kept.push({ at, counts: new Map(counts) }); | |
| 28 | samples.set(repoId, kept); | |
| 29 | } | |
| 30 | ||
| 31 | /** Forget every sample; for tests. */ | |
| 32 | export function forgetHeat(): void { | |
| 33 | samples.clear(); | |
| 34 | } | |
| 35 | ||
| 36 | export interface WarmingFile extends Warming { | |
| 37 | readonly repoId: string; | |
| 38 | } | |
| 39 | ||
| 40 | /** The sample of `repoId` nearest an hour before `now`, if one is at least `MIN_AGE_MS` old. */ | |
| 41 | function baseline(repoId: string, now: number): ReadonlyMap<string, number> | undefined { | |
| 42 | let best: { at: number; counts: ReadonlyMap<string, number> } | undefined; | |
| 43 | for (const s of samples.get(repoId) ?? []) { | |
| 44 | if (now - s.at < MIN_AGE_MS) continue; | |
| 45 | if (best === undefined || Math.abs(now - s.at - HOUR_MS) < Math.abs(now - best.at - HOUR_MS)) best = s; | |
| 46 | } | |
| 47 | return best?.counts; | |
| 48 | } | |
| 49 | ||
| 50 | /** The files warming in the repositories `allow` lets through, largest rise first, at most five; each against its own hour-old sample. */ | |
| 51 | export function warmingFiles( | |
| 52 | now: number, | |
| 53 | current: (repoId: string) => ReadonlyMap<string, number>, | |
| 54 | allow: (repoId: string) => boolean = () => true, | |
| 55 | ): WarmingFile[] { | |
| 56 | const out: WarmingFile[] = []; | |
| 57 | for (const repoId of samples.keys()) { | |
| 58 | const before = allow(repoId) ? baseline(repoId, now) : undefined; | |
| 59 | if (before === undefined) continue; | |
| 60 | for (const w of hotTrend(before, current(repoId))) out.push({ ...w, repoId }); | |
| 61 | } | |
| 62 | return out.sort((a, b) => byRise(a, b) || a.repoId.localeCompare(b.repoId)).slice(0, TREND_NAMED); | |
| 63 | } | |
| 64 | ||
| 65 | /** The warming files a reader may see (every repository, or only the public ones), one line each naming its repository; "" when none. A repository that no longer exists loses its samples. */ | |
| 66 | export function warmingLine(db: Database, now: number, current: (repoId: string) => ReadonlyMap<string, number>, publicOnly: boolean): string { | |
| 67 | const names = new Map<string, string>(); | |
| 68 | const allow = (id: string): boolean => { | |
| 69 | const repo = getRepoById(db, id); | |
| 70 | if (repo == null) { | |
| 71 | samples.delete(id); | |
outdated
The line is built from WarmingFile rows but hotTrendSays prints only path; include the repository name so identical paths in different repos are distinguishable. |
||
| 72 | return false; | |
| 73 | } | |
| 74 | names.set(id, `${repo.ownerName}/${repo.name}`); | |
| 75 | return !publicOnly || repo.visibility === "public"; | |
| 76 | }; | |
| 77 | return hotTrendSays(warmingFiles(now, current, allow), (w) => names.get(w.repoId) ?? w.repoId); | |
| 78 | } |
src/flow/hot_trend.ts added +34
| @@ -0,0 +1,34 @@ | ||
| 1 | /** | |
| 2 | * Files warming up (#6547): the paths whose count of open pulls touching them | |
| 3 | * rose over the last hour, named before they reach the hot-file limit. A pure | |
| 4 | * read of two plain maps `path → open pulls`, an hour apart. | |
| 5 | */ | |
| 6 | ||
| 7 | export const TREND_NAMED = 5; | |
| 8 | ||
| 9 | export interface Warming { | |
| 10 | readonly path: string; | |
| 11 | readonly from: number; | |
| 12 | readonly to: number; | |
| 13 | } | |
| 14 | ||
| 15 | /** Largest rise first, ties by path. */ | |
| 16 | export function byRise(a: Warming, b: Warming): number { | |
outdated
|
||
| 17 | return b.to - b.from - (a.to - a.from) || a.path.localeCompare(b.path); | |
| 18 | } | |
| 19 | ||
| 20 | /** Files whose count rose from `before` to `now`, largest rise first (ties by path), at most five; a file absent from `before` counts from 0. */ | |
| 21 | export function hotTrend(before: ReadonlyMap<string, number>, now: ReadonlyMap<string, number>): Warming[] { | |
outdated
Tie-break is untested; add a case with equal rises and out-of-order paths. The nested ternary is hard to read. |
||
| 22 | const rows: Warming[] = []; | |
| 23 | for (const [path, to] of now) { | |
| 24 | const from = before.get(path) ?? 0; | |
| 25 | if (to > from) rows.push({ path, from, to }); | |
Non-blocking: this also names files that are already hot (to >= HOT_PULLS) and every 0 → 1 rise. Consider |
||
| 26 | } | |
| 27 | rows.sort(byRise); | |
| 28 | return rows.slice(0, TREND_NAMED); | |
| 29 | } | |
| 30 | ||
| 31 | /** "warming: <path> 2 → 4 pulls", with " in <where>" when a caller names the repository; one line per file; empty when nothing rose. */ | |
| 32 | export function hotTrendSays<R extends Warming>(rows: readonly R[], where?: (row: R) => string): string { | |
| 33 | return rows.map((r) => `warming: ${r.path} ${r.from} → ${r.to} pulls${where ? ` in ${where(r)}` : ""}`).join("\n"); | |
| 34 | } |
src/tui/screens/flow.ts +4
| @@ -15,6 +15,8 @@ | ||
| 15 | 15 | |
| 16 | 16 | import type { Database } from "bun:sqlite"; |
| 17 | 17 | import { HOLDS } from "../../crew/held.ts"; |
| 18 | import { pullCounts } from "../../crew/hot_files.ts"; | |
| 19 | import { warmingLine } from "../../crew/hot_history.ts"; | |
| 18 | 20 | import type { HeldRow } from "../../flow/held.ts"; |
| 19 | 21 | import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../../flow/conflicted.ts"; |
| 20 | 22 | import { pileSays, readPile } from "../../flow/pile.ts"; |
| @@ -60,6 +62,7 @@ export function flowLines( | ||
| 60 | 62 | requestedSays(requested.builder, "builder", now.getTime()), |
| 61 | 63 | requestedSays(requested.reviewer, "reviewer", now.getTime()), |
| 62 | 64 | ].flatMap((says) => (says === null ? [] : ["", ...wrap(says, inner)])); |
| 65 | const warming = warmingLine(db, now.getTime(), pullCounts, agent !== undefined); | |
| 63 | 66 | const reviewing = reviewingSays(reviewersBusy(db, agent !== undefined)); |
| 64 | 67 | return [ |
| 65 | 68 | ...wrap(verdict(stages, now.getTime()), inner), |
| @@ -91,6 +94,7 @@ export function flowLines( | ||
| 91 | 94 | ...(conflicted === null ? [] : ["", ...wrap(conflicted, inner)]), |
| 92 | 95 | ...waits, |
| 93 | 96 | ...(reviewing === null ? [] : ["", ...wrap(reviewing, inner)]), |
| 97 | ...(warming === "" ? [] : ["", ...warming.split("\n").flatMap((l) => wrap(l, inner))]), | |
| 94 | 98 | ...(held === undefined ? [] : ["", ...heldLines(held, inner)]), |
| 95 | 99 | "", |
| 96 | 100 | ...duplicateLines(dups, inner), |
src/web/pages/flow-panel.ts +5 −2
| @@ -33,7 +33,8 @@ import { | ||
| 33 | 33 | type StageThroughput, |
| 34 | 34 | } from "../../flow/throughput.ts"; |
| 35 | 35 | import { type QueueRow, redBatchWords } from "../../flow/queue.ts"; |
| 36 | import { HOT_PULLS, warmFiles } from "../../crew/hot_files.ts"; | |
| 36 | import { HOT_PULLS, pullCounts, warmFiles } from "../../crew/hot_files.ts"; | |
| 37 | import { warmingLine } from "../../crew/hot_history.ts"; | |
| 37 | 38 | import { getRepoById } from "../../core/repos.ts"; |
| 38 | 39 | import { readHeld } from "../../flow/held.ts"; |
| 39 | 40 | import type { App } from "../../app.ts"; |
| @@ -262,6 +263,7 @@ function hotFilesCard(app: App, reader: Reader): Html { | ||
| 262 | 263 | } |
| 263 | 264 | rows.sort((a, b) => b.pulls - a.pulls || a.path.localeCompare(b.path) || a.repo.localeCompare(b.repo)); |
| 264 | 265 | const top = rows.slice(0, HOT_LISTED); |
| 266 | const warming = warmingLine(app.db, Date.now(), pullCounts, reader.kind !== "owner").split("\n").filter((l) => l !== ""); | |
| 265 | 267 | const max = Math.max(1, ...top.map((r) => r.pulls)); |
| 266 | 268 | return card( |
| 267 | 269 | html`<h2>Hot files</h2> |
| @@ -277,7 +279,8 @@ function hotFilesCard(app: App, reader: Reader): Html { | ||
| 277 | 279 | ${meter((r.pulls / max) * 100, `${r.pulls} open pulls touch ${r.path}`)} |
| 278 | 280 | </li>`, |
| 279 | 281 | )} |
| 280 | </ul>`}`, | |
| 282 | </ul>`} | |
| 283 | ${warming.length === 0 ? null : html`<h3>Warming</h3><ul data-warming aria-label="Files warming up">${warming.map((l) => html`<li class="muted small">${l}</li>`)}</ul>`}`, | |
outdated
Warning lines sit under the hot-files list without the repo name that each listed hot file shows; add it. Also consider an aria label on the data-warming list (e.g. a small heading "Warming"). |
||
| 281 | 284 | { id: "hot-files" }, |
| 282 | 285 | ); |
| 283 | 286 | } |
test/flow-hot-files.test.ts +45
| @@ -8,6 +8,8 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 8 | 8 | import { createRepo } from "../src/actions/repos.ts"; |
| 9 | 9 | import { createUser } from "../src/core/users.ts"; |
| 10 | 10 | import { rememberPullFiles } from "../src/crew/hot_files.ts"; |
| 11 | import { forgetHeat, sampleHeat } from "../src/crew/hot_history.ts"; | |
| 12 | import { flowLines } from "../src/tui/screens/flow.ts"; | |
| 11 | 13 | import { flowPanel } from "../src/web/pages/flow-panel.ts"; |
| 12 | 14 | |
| 13 | 15 | describe("flow panel hot files", () => { |
| @@ -71,4 +73,47 @@ describe("flow panel hot files", () => { | ||
| 71 | 73 | app.db.query("UPDATE repo SET visibility = 'public' WHERE id = ?1").run(repoId); |
| 72 | 74 | expect(draw("agent")).toContain("secret/plan.ts"); |
| 73 | 75 | }); |
| 76 | ||
| 77 | describe("warming files (#6547)", () => { | |
| 78 | const HOUR = 3_600_000; | |
| 79 | const seed = () => { | |
| 80 | sampleHeat(repoId, new Map([["src/a.ts", 1]]), Date.now() - HOUR); | |
| 81 | rememberPullFiles(repoId, [1, 2, 3].map((number) => ({ number, files: ["src/a.ts"] }))); | |
| 82 | }; | |
| 83 | afterEach(forgetHeat); | |
| 84 | ||
| 85 | test("the page and the terminal name a file that gained pulls in the hour", () => { | |
| 86 | seed(); | |
| 87 | expect(draw("owner")).toContain("warming: src/a.ts 1 → 3 pulls in alice/shop"); | |
| 88 | expect(flowLines(app.db).join("\n")).toContain("warming: src/a.ts 1 → 3 pulls in alice/shop"); | |
| 89 | }); | |
| 90 | ||
| 91 | test("two repositories sharing a path are told apart", async () => { | |
| 92 | const u = app.db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").get()!; | |
| 93 | const other = await createRepo(app, { id: u.id } as never, { name: "blog" }); | |
| 94 | if (!other.ok) throw new Error(String(other.error)); | |
| 95 | seed(); | |
| 96 | sampleHeat(other.value.id, new Map([["src/a.ts", 0]]), Date.now() - HOUR); | |
| 97 | rememberPullFiles(other.value.id, [1, 2].map((number) => ({ number, files: ["src/a.ts"] }))); | |
| 98 | const page = draw("owner"); | |
| 99 | expect(page).toContain("src/a.ts 1 → 3 pulls in alice/shop"); | |
| 100 | expect(page).toContain("src/a.ts 0 → 2 pulls in alice/blog"); | |
| 101 | rememberPullFiles(other.value.id, []); | |
| 102 | }); | |
| 103 | ||
| 104 | test("without an hour of history nothing is said", () => { | |
| 105 | sampleHeat(repoId, new Map(), Date.now() - 60_000); | |
| 106 | rememberPullFiles(repoId, [1, 2, 3].map((number) => ({ number, files: ["src/a.ts"] }))); | |
| 107 | expect(draw("owner")).not.toContain("warming:"); | |
| 108 | expect(flowLines(app.db).join("\n")).not.toContain("warming:"); | |
| 109 | }); | |
| 110 | ||
| 111 | test("a private repository's warming file is for the owner only", () => { | |
| 112 | seed(); | |
| 113 | app.db.query("UPDATE repo SET visibility = 'private' WHERE id = ?1").run(repoId); | |
| 114 | expect(draw("owner")).toContain("warming: src/a.ts"); | |
| 115 | expect(draw("agent")).not.toContain("warming:"); | |
| 116 | expect(flowLines(app.db, new Date(), undefined, { queues: [] }).join("\n")).not.toContain("warming:"); | |
| 117 | }); | |
| 118 | }); | |
| 74 | 119 | }); |
test/flow-hot-trend.test.ts added +33
| @@ -0,0 +1,33 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { hotTrend, hotTrendSays } from "../src/flow/hot_trend.ts"; | |
| 3 | ||
| 4 | const m = (o: Record<string, number>) => new Map(Object.entries(o)); | |
| 5 | ||
| 6 | test("rises sorted by size, falls and unchanged left out", () => { | |
| 7 | const rows = hotTrend(m({ a: 1, b: 2, c: 3, d: 4 }), m({ a: 2, b: 5, c: 1, d: 4 })); | |
| 8 | expect(rows).toEqual([ | |
| 9 | { path: "b", from: 2, to: 5 }, | |
| 10 | { path: "a", from: 1, to: 2 }, | |
| 11 | ]); | |
| 12 | }); | |
| 13 | ||
| 14 | test("equal rises are ordered by path", () => { | |
| 15 | const rows = hotTrend(m({}), m({ "z.ts": 2, "a.ts": 2, "m.ts": 2 })); | |
| 16 | expect(rows.map((r) => r.path)).toEqual(["a.ts", "m.ts", "z.ts"]); | |
| 17 | }); | |
| 18 | ||
| 19 | test("a new file counts from 0", () => { | |
| 20 | expect(hotTrend(m({}), m({ x: 2 }))).toEqual([{ path: "x", from: 0, to: 2 }]); | |
| 21 | }); | |
| 22 | ||
| 23 | test("at most 5", () => { | |
| 24 | const now = m(Object.fromEntries("abcdefg".split("").map((p, i) => [p, i + 1]))); | |
| 25 | const rows = hotTrend(m({}), now); | |
| 26 | expect(rows.length).toBe(5); | |
| 27 | expect(rows[0]!.path).toBe("g"); | |
| 28 | }); | |
| 29 | ||
| 30 | test("words it, empty when nothing rose", () => { | |
| 31 | expect(hotTrendSays([{ path: "src/a.ts", from: 2, to: 4 }])).toBe("warming: src/a.ts 2 → 4 pulls"); | |
| 32 | expect(hotTrendSays(hotTrend(m({ a: 3 }), m({ a: 3 })))).toBe(""); | |
| 33 | }); |
This is the only producer of samples in production, and no test fails if it is removed. Add a test that drives
learnPullFilesand then reads the warming line an hour later.