e1a963a · f8f951c
@@ -63,3 +63,14 @@
tick logs as `held=`, read through `flow/held.ts`, which keeps the rows 5 s.
The terminal's Flow command prints the same rows, one line per repository
(`alice/engine startable=0 stalled=4 run_cap=1`). A granted agent sees
public repositories only, on both.
One of those filters is the hot-file WIP limit (#5934, `crew/hot_files.ts`):
a ready issue that names a file 5 or more open pulls already touch is held as
`hot_file` until that pile drains under 5, so a sixth pull never joins a
file every landing would make the rest rebase. An issue's expected files are
the repository paths its title and body name (`src/http/mcp.ts`,
`` `README.md` ``); an open pull's files are read from git once per head
before the picker sorts. The tick line names the hot files (`hot=src/http/mcp.ts=61`),
and each repository's writers see one post on the feed — on the web and
through the MCP `feed` tool — saying which build waits on which file, gone
when the hold lifts.
@@ -99,6 +99,7 @@
import { BREAKER_NOTE, breakerTripped } from "./breaker.ts";
import { given, pendingReplans } from "../core/limb_replan.ts";
import { handedOverPulls, SKIP_SUMMARY, skipsReplan } from "./guard.ts";
import { atRunCap, type Held, heldBy, heldWords, replannable } from "./held.ts";
import { holdHot, hotFiles, hotWords, learnPullFiles } from "./hot_files.ts";
import { NO_PULL, shunnedBy, stallReason, unstickOwed } from "./stalls.ts";
import { BY_CREW, DONE_LIKE } from "../core/done_on_main.ts";
import { ALREADY_NOTE, alreadyDoneOids } from "./already.ts";
@@ -365,9 +366,18 @@
export async function driveRepo(
  // operator's; at a wall nothing new starts until the reset.
  const paced = repoPace(app, repo, now);
  if (paced.slots === 0 && slotted.length === 0) return "waiting";
  // The files the open pulls touch, read before the sort: a build that
  // names a hot one waits for its pile to drain (#5934).
  await learnPullFiles(app, repo).catch((error) =>
    log.warn("could not read the open pulls' files", {
      repo: `${repo.ownerName}/${repo.name}`,
      error: String(error),
    }),
  );
  const sorted = () => heldBy(app, repo, readyNow(repoGraph(app.db, repo.id, { sealed: true })), crew, active);
  let held = sorted();
  if (freeParked(app, repo, by, held) > 0) held = sorted();
  holdHot(repo.id, held.by.hot_file, now);
      
        junipercrew
        19d ago
        
      
      holdHot runs only past the early "waiting" return above, so a repo that stays paced out keeps its last feed post. Clear the standing entry on that path too.
    
  const ready = held.startable;
  if (slotted.length >= paced.slots) {
    // The builders are busy; the planner can still refill the map, when
@@ -1261,6 +1271,8 @@
export async function autonomyTick(
        // The standing reserve the planner keeps: startable/target (#3237).
        reserve: reserveWords(reserveOf(held.startable.length, status.slots)),
        held: heldWords(held),
        // The files 5 or more open pulls touch, which no new build starts on (#5934).
        ...(held.by.hot_file.length === 0 ? {} : { hot: hotWords(hotFiles(current.id)) }),
        // Stalled issues this tick parked and freed for another builder (#4073).
        unstuck,
        active: status.active,
@@ -20,6 +20,8 @@
 * - `run_cap`: it has used its runs for the day (`guard.ts`).
 * - `parked_twice`: the crew parked it twice; it is the planner's (`parks.ts`).
 * - `red_fix`: a fix for a red main already covers it (`red_fix.ts`).
 * - `hot_file`: a file it names is touched by `HOT_PULLS` or more open
 *   pulls; it starts when that pile drains under the limit (`hot_files.ts`).
 *
 * The direction issue is the planner's standing issue and never a
 * builder's; it is left out, not counted.
@@ -39,6 +41,7 @@
import { proposedWork, speaksForCrew } from "./direct.ts";
import { capReached, runCapOf } from "./guard.ts";
import { interruptionOf } from "./interrupted.ts";
import { PARK_STRIKES, parkedCount } from "./parks.ts";
import { hotFiles, hotOverlap } from "./hot_files.ts";
import { coveredByRedFix } from "./red_fix.ts";
export const HOLDS = [
@@ -51,6 +54,7 @@
export const HOLDS = [
  "run_cap",
  "parked_twice",
  "red_fix",
  "hot_file",
] as const;
export type HoldName = (typeof HOLDS)[number];
@@ -120,6 +124,7 @@
export function heldBy(
    crew.filter((c) => c.role !== "builder").map((c) => c.agent.user.id),
  );
  const handed = withPlanner(app.db, repo);
  const hot = hotFiles(repo.id);
  const holdOf = (issue: Issue): HoldName | null => {
    if (busyIssues.has(issue.id)) return "running";
    if (proposed.has(issue.number)) return "open_pull";
@@ -130,6 +135,7 @@
export function heldBy(
    if (atRunCap(app, repo, issue)) return "run_cap";
    if (parkedCount(app.db, repo, issue.id) >= PARK_STRIKES) return "parked_twice";
    if (coveredByRedFix(app.db, repo.id, issue)) return "red_fix";
    if (hotOverlap(issue, hot).length > 0) return "hot_file";
    return null;
  };
  const by = Object.fromEntries(HOLDS.map((h) => [h, [] as Issue[]])) as Record<
@@ -0,0 +1,157 @@
/**
 * The hot-file WIP limit (#5934, #5924 step 3): a build is not started
 * while a file it expects to touch is already touched by `HOT_PULLS` or
 * more open pulls. One more pull on that file is one more rebase for each
 * of the others when any of them lands; the build waits until the pile
 * drains under the limit.
 *
 * - An issue's expected files are the repository paths its title and body
 *   name (`expectedFiles`): `src/http/mcp.ts`, `docs/MCP.md`.
 * - The files an open pull touches are read from git once per head
 *   (`learnPullFiles`, asked by the picker before it sorts), and kept in
 *   memory: a cache git rebuilds, never state of its own.
 * - The holds the picker last said, per repository (`standingHot`), are what
 *   the feed shows while they stand.
 */
import type { App } from "../app.ts";
import { getIssuesByIds, type Issue } from "../core/issues.ts";
import { joinMany, pullDir } from "../core/pulls.ts";
import type { Repo } from "../core/repos.ts";
import { runGitCapture } from "../git/env.ts";
/** A file this many open pulls touch is hot: no new build starts on it. */
export const HOT_PULLS = 5;
      
        junipercrew
        19d ago
        
      
      Path extraction from free prose is a heuristic; a mention of a hot file that the build will not touch holds it indefinitely. Consider limiting to a fenced "Files:" line or the title, or surfacing the matched path so a person can fix the issue text.
    
/** The most paths read from one issue: a list longer than this is a manifest, not a plan. */
const MAX_EXPECTED = 40;
/** A path with at least one directory and a file extension: `src/http/mcp.ts`. */
const PATH = /(?<![\w./:-])((?:[\w-][\w.-]*\/)+[\w-][\w.-]*\.[A-Za-z0-9]+)(?![\w/])/g;
/** A bare file name in backticks: `README.md`, `forge.json`. */
const QUOTED = /`([\w-][\w.-]*\.[A-Za-z0-9]+)`/g;
/** The repository paths an issue names, in the order it names them, each once. */
export function expectedFiles(issue: Pick<Issue, "title" | "body">): string[] {
  const text = `${issue.title}\n${issue.body}`;
  const found = new Set<string>();
  for (const pattern of [PATH, QUOTED])
    for (const match of text.matchAll(pattern)) {
      const path = match[1]!.replace(/\.+$/, "");
      if (path.split("/").some((part) => part === "." || part === "..")) continue;
      found.add(path);
      if (found.size >= MAX_EXPECTED) return [...found];
    }
  return [...found];
}
export interface HotFile {
  readonly path: string;
  /** Open pulls that touch it. */
  readonly pulls: number;
}
/** Per repository, per open pull: the head and base it was read at, and the files. */
const touched = new Map<string, Map<string, { at: string; files: readonly string[] }>>();
const OID = /^[0-9a-f]{40,64}$/;
/**
 * Read the files each open pull of `repo` touches, for any pull whose head
 * moved since it was last read, and forget the pulls no longer open. A pull
 * git cannot answer for touches nothing until it can.
 */
export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
  const ids = app.db
    .query<{ id: string }, [string]>(
      `SELECT p.issue_id AS id FROM pull p JOIN issue i ON i.id = p.issue_id
        WHERE i.repo_id = ?1 AND i.state = 'open' AND p.merged_at IS NULL`,
    )
    .all(repo.id)
    .map((r) => r.id);
  const pulls = joinMany(app.db, [...getIssuesByIds(app.db, ids).values()]);
  const before = touched.get(repo.id) ?? new Map();
  const after = new Map<string, { at: string; files: readonly string[] }>();
  for (const pull of pulls) {
    const at = `${pull.baseOid}...${pull.headOid}`;
    const known = before.get(pull.id);
    if (known !== undefined && known.at === at) {
      after.set(pull.id, known);
      continue;
    }
    if (!OID.test(pull.baseOid) || !OID.test(pull.headOid)) continue;
    const dir = await pullDir(app.db, app.reposRoot, repo, pull);
    const out = await runGitCapture(["diff", "--name-only", "-z", at], { cwd: dir });
    if (out.code !== 0) continue;
    after.set(pull.id, { at, files: out.stdout.split("\0").filter((f) => f !== "") });
  }
  touched.set(repo.id, after);
}
/** Every file of `repo` that `HOT_PULLS` or more open pulls touch, with how many. */
export function hotFiles(repoId: string): Map<string, number> {
  const counts = new Map<string, number>();
  for (const { files } of touched.get(repoId)?.values() ?? [])
    for (const file of files) counts.set(file, (counts.get(file) ?? 0) + 1);
  for (const [file, n] of counts) if (n < HOT_PULLS) counts.delete(file);
  return counts;
}
/** The hot files `issue` expects to touch, hottest first. */
export function hotOverlap(
  issue: Pick<Issue, "title" | "body">,
  hot: ReadonlyMap<string, number>,
): HotFile[] {
  if (hot.size === 0) return [];
  return expectedFiles(issue)
    .flatMap((path) => {
      const pulls = hot.get(path);
      return pulls === undefined ? [] : [{ path, pulls }];
    })
    .sort((a, b) => b.pulls - a.pulls || a.path.localeCompare(b.path));
}
/** A held build as the feed says it. */
export interface HotHold {
  readonly number: number;
  readonly title: string;
  readonly files: readonly HotFile[];
}
/** What the picker last held on hot files, per repository, and when. */
const standing = new Map<string, { at: string; holds: readonly HotHold[] }>();
/** Remember the builds the picker holds on hot files in `repoId` now; none clears it. */
export function holdHot(repoId: string, issues: readonly Issue[], at = new Date()): void {
  if (issues.length === 0) {
    standing.delete(repoId);
    return;
  }
  const hot = hotFiles(repoId);
  standing.set(repoId, {
    at: at.toISOString(),
    holds: issues.map((i) => ({ number: i.number, title: i.title, files: hotOverlap(i, hot) })),
  });
}
/** The builds held on hot files in `repoId` as the picker last said, if any. */
export function standingHot(repoId: string): { at: string; holds: readonly HotHold[] } | undefined {
  return standing.get(repoId);
}
/** The hot files as the tick line says them: `src/http/mcp.ts=61 docs/MCP.md=54`, hottest first. */
export function hotWords(hot: ReadonlyMap<string, number>): string {
  return [...hot]
    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
    .map(([path, n]) => `${path}=${n}`)
    .join(" ");
}
/** One sentence for a held build: `#12 waits: src/http/mcp.ts is in 6 open pulls (hot at 5).` */
export function hotSays(hold: HotHold): string {
  const files = hold.files
    .slice(0, 3)
    .map((f) => `${f.path} is in ${f.pulls} open pulls`)
    .join(", ");
  return `#${hold.number} waits: ${files === "" ? "a file it names is hot" : files} (hot at ${HOT_PULLS}).`;
}
@@ -27,6 +27,7 @@
import {
import { getUserById, type User } from "../core/users.ts";
import { watchedBy } from "../core/watches.ts";
import { chooseCandidates } from "./choose.ts";
import { hotSays, standingHot } from "../crew/hot_files.ts";
import { ago } from "./rank.ts";
import { askLinesFor } from "../web/studio/asks.ts";
@@ -197,6 +198,35 @@
function frozenCandidates(
  return out;
}
/**
 * Builds the picker holds because a file they name is in 5 or more open
 * pulls (#5934): one post per repository the viewer writes to, read from
 * what the picker said last, so it goes when the pile drains.
 */
function hotCandidates(writable: readonly Repo[]): Candidate[] {
  const out: Candidate[] = [];
  for (const repo of writable) {
    const standing = standingHot(repo.id);
    if (standing === undefined) continue;
    const n = standing.holds.length;
    const first = standing.holds[0]!;
    const what = `${n === 1 ? "1 build waits" : `${n} builds wait`} on hot files: ${hotSays(first)}${n > 1 ? ` And ${n - 1} more.` : ""}`;
    out.push({
      id: `hot:${repo.id}`,
      kind: "map",
      repoId: repo.id,
      app: slug(repo),
      actorId: null,
      actor: null,
      at: standing.at,
      link: `${base(repo)}/map`,
      what: clip(what),
      urgency: 0,
    });
  }
  return out;
}
export function candidatesFor(
  db: Database,
  viewer: User,
@@ -297,6 +327,7 @@
export function candidatesFor(
  out.push(...watchedCandidates(db, viewer, now));
  out.push(...frozenCandidates(db, viewer, now));
  out.push(...chooseCandidates(db, writable));
  out.push(...hotCandidates(writable));
  for (const d of decided) {
    const repo = repos.get(d.repoId);
    if (!repo) continue;
@@ -32,7 +32,7 @@
export function heldCard(rows: readonly HeldRow[]): Html {
      <p class="muted small">
        Each ready issue a builder could start, or the first filter that holds
        it: a run on it, an open pull, the planner, stalled, its run cap,
        parked twice, a red-main fix. When nothing can
        parked twice, a red-main fix, a file 5 or more open pulls touch. When nothing can
        start and builders are free, the planner is asked to re-plan what is
        stalled or capped.
      </p>
@@ -0,0 +1,195 @@
/**
 * The hot-file WIP limit (#5934): with five open pulls on one file, a sixth
 * build naming that file waits while a build on a cold file starts, and
 * the hold is said where the picker says why it skipped — the held counts,
 * the feed, and the feed through the MCP. The hold lifts when the pile
 * drains under five.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { addToCrew, createAgent } from "../src/actions/crew.ts";
import { openIssue, setIssueState } from "../src/actions/issues.ts";
import { openPull } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo, updateRepoSettings } from "../src/actions/repos.ts";
import { setCrewRole } from "../src/core/agents.ts";
import { getIssueById, type Issue } from "../src/core/issues.ts";
import type { Pull } from "../src/core/pulls.ts";
import { getRepoById, type Repo } from "../src/core/repos.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser, getUserByName, type User } from "../src/core/users.ts";
import { driveRepo } from "../src/crew/autonomy.ts";
import { heldNow, heldWords } from "../src/crew/held.ts";
import {
  expectedFiles,
  HOT_PULLS,
  hotFiles,
  hotOverlap,
  learnPullFiles,
  standingHot,
} from "../src/crew/hot_files.ts";
import { readFeed } from "../src/feed/feed.ts";
import { serve, type Servers } from "../src/http/server.ts";
describe("expected files", () => {
  test("are the repository paths an issue names, each once", () => {
    expect(
      expectedFiles({
        title: "Split src/http/mcp.ts into one file per tool",
        body: "Touches `docs/MCP.md`, `README.md` and src/http/mcp.ts again; see https://example.com/a/b.html and ../up/x.ts.",
      }),
    ).toEqual(["src/http/mcp.ts", "docs/MCP.md", "README.md"]);
    expect(expectedFiles({ title: "Make the home page faster", body: "" })).toEqual([]);
  });
  test("overlap a hot file only when it is named", () => {
    const hot = new Map([["src/a.ts", 7], ["src/b.ts", 5]]);
    expect(hotOverlap({ title: "src/b.ts and src/a.ts", body: "" }, hot)).toEqual([
      { path: "src/a.ts", pulls: 7 },
      { path: "src/b.ts", pulls: 5 },
    ]);
    expect(hotOverlap({ title: "src/c.ts", body: "" }, hot)).toEqual([]);
  });
});
describe("the picker holds a build on a hot file", () => {
  let dataDir: string;
  let app: App;
  let alice: User;
  let servers: Servers;
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-hot-"));
    app = createApp({ dataDir, port: 0 });
    const a = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!a.ok) throw new Error(a.error);
    alice = a.value;
    servers = await serve(app);
  });
  afterAll(async () => {
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  const fresh = (repo: Repo) => getRepoById(app.db, repo.id)!;
  function issue(repo: Repo, title: string): Issue {
    const opened = openIssue(app, alice, fresh(repo), { title, body: "" });
    if (!opened.ok) throw new Error(String(opened.error));
    return opened.value;
  }
  async function pullOn(repo: Repo, n: number, path: string): Promise<Pull> {
    const branch = await commitToBranch(app, alice, fresh(repo), {
      branch: `feature/${n}`,
      base: "main",
      message: `Change ${n}`,
      files: [{ path, content: `export const n = ${n};\n` }],
    });
    if (!branch.ok) throw new Error(JSON.stringify(branch.error));
    const pull = await openPull(app, alice, fresh(repo), {
      title: `Change ${n}`,
      body: "",
      baseBranch: "main",
      headBranch: `feature/${n}`,
    });
    if (!pull.ok) throw new Error(String(pull.error));
    return pull.value;
  }
  test("a sixth build on a hot file waits while a build on a cold file starts", async () => {
    const made = await createRepo(app, alice, { name: "engine", visibility: "private" });
    if (!made.ok) throw new Error(String(made.error));
    for (const agent of ["atlas", "mason"]) {
      const created = await createAgent(app, alice, { username: agent });
      if (!created.ok) throw new Error(String(created.error));
      expect(addToCrew(app, alice, made.value, agent).ok).toBe(true);
    }
    setCrewRole(app.db, made.value.id, getUserByName(app.db, "atlas")!.id, "planner");
    const set = await updateRepoSettings(app, alice, made.value, {
      autonomy: "autonomous",
      autonomyDirection: "Keep the engine small.",
      autonomyCeilingUsd: 100,
      autonomySlots: 1,
      autonomyBy: alice.id,
    });
    if (!set.ok) throw new Error(JSON.stringify(set.error));
    const repo = fresh(made.value);
    const seeded = await commitToBranch(app, alice, repo, {
      message: "Start",
      files: [
        { path: "src/hot.ts", content: "export const n = 0;\n" },
        { path: "src/cold.ts", content: "export const n = 0;\n" },
      ],
    });
    if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
    const pulls: Pull[] = [];
    for (let n = 1; n <= HOT_PULLS; n++) pulls.push(await pullOn(repo, n, "src/hot.ts"));
    // First on the map, so only the hold keeps it from being picked.
    const hotWork = issue(repo, "Add a flag to src/hot.ts");
    const coldWork = issue(repo, "Tidy src/cold.ts");
    expect(await driveRepo(app, fresh(repo))).toBe("picked");
    const mason = getUserByName(app.db, "mason")!;
    expect(getIssueById(app.db, coldWork.id)!.assigneeId).toBe(mason.id);
    expect(getIssueById(app.db, hotWork.id)!.assigneeId).toBeNull();
    expect(hotFiles(repo.id)).toEqual(new Map([["src/hot.ts", HOT_PULLS]]));
    // Said where the picker says why it skipped: the held counts the tick
    // line and /flow read…
    const held = heldNow(app, fresh(repo));
    expect(held.by.hot_file.map((i) => i.number)).toEqual([hotWork.number]);
    expect(heldWords(held)).toContain("hot_file=1");
    expect(standingHot(repo.id)?.holds).toEqual([
      { number: hotWork.number, title: hotWork.title, files: [{ path: "src/hot.ts", pulls: HOT_PULLS }] },
    ]);
    // …the feed…
    const said = `#${hotWork.number} waits: src/hot.ts is in ${HOT_PULLS} open pulls (hot at ${HOT_PULLS}).`;
    const feed = readFeed(app.db, alice, { limit: 50 });
    const post = feed.items.find((i) => i.id === `hot:${repo.id}`);
    expect(post?.what).toBe(`1 build waits on hot files: ${said}`);
    expect(post?.link).toBe(`/alice/engine/map`);
    // …and the feed through the MCP.
    const token = createToken(app.db, alice.id, "t", "ro").token;
    const res = await fetch(`${servers.url}/mcp`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${token}`,
        "content-type": "application/json",
        accept: "application/json, text/event-stream",
      },
      body: JSON.stringify({
        jsonrpc: "2.0",
        id: 1,
        method: "tools/call",
        params: { name: "feed", arguments: { limit: 50 } },
      }),
    });
    const body = (await res.json()) as { result: { content: { text: string }[] } };
    expect(body.result.content[0]!.text).toContain(said);
    // The pile drains under the limit: the hold lifts, and the post goes.
    const closed = setIssueState(app, alice, fresh(repo), getIssueById(app.db, pulls[0]!.id)!, "closed");
    expect(closed.ok).toBe(true);
    await learnPullFiles(app, fresh(repo));
    expect(hotFiles(repo.id).size).toBe(0);
    expect(heldNow(app, fresh(repo)).by.hot_file).toEqual([]);
    await driveRepo(app, fresh(repo));
    expect(standingHot(repo.id)).toBeUndefined();
    expect(readFeed(app.db, alice, { limit: 50 }).items.some((i) => i.id === `hot:${repo.id}`)).toBe(false);
  });
});
		@@ -63,3 +63,14 @@ tick logs as `held=`, read through `flow/held.ts`, which keeps the rows 5 s.
63	63	The terminal's Flow command prints the same rows, one line per repository
64	64	(`alice/engine startable=0 stalled=4 run_cap=1`). A granted agent sees
65	65	public repositories only, on both.
	66
	67	One of those filters is the hot-file WIP limit (#5934, `crew/hot_files.ts`):
	68	a ready issue that names a file 5 or more open pulls already touch is held as
	69	`hot_file` until that pile drains under 5, so a sixth pull never joins a
	70	file every landing would make the rest rebase. An issue's expected files are
	71	the repository paths its title and body name (`src/http/mcp.ts`,
	72	`` `README.md` ``); an open pull's files are read from git once per head
	73	before the picker sorts. The tick line names the hot files (`hot=src/http/mcp.ts=61`),
	74	and each repository's writers see one post on the feed — on the web and
	75	through the MCP `feed` tool — saying which build waits on which file, gone
	76	when the hold lifts.
		@@ -99,6 +99,7 @@ import { BREAKER_NOTE, breakerTripped } from "./breaker.ts";
99	99	import { given, pendingReplans } from "../core/limb_replan.ts";
100	100	import { handedOverPulls, SKIP_SUMMARY, skipsReplan } from "./guard.ts";
101	101	import { atRunCap, type Held, heldBy, heldWords, replannable } from "./held.ts";
	102	import { holdHot, hotFiles, hotWords, learnPullFiles } from "./hot_files.ts";
102	103	import { NO_PULL, shunnedBy, stallReason, unstickOwed } from "./stalls.ts";
103	104	import { BY_CREW, DONE_LIKE } from "../core/done_on_main.ts";
104	105	import { ALREADY_NOTE, alreadyDoneOids } from "./already.ts";
		@@ -365,9 +366,18 @@ export async function driveRepo(
365	366	// operator's; at a wall nothing new starts until the reset.
366	367	const paced = repoPace(app, repo, now);
367	368	if (paced.slots === 0 && slotted.length === 0) return "waiting";
	369	// The files the open pulls touch, read before the sort: a build that
	370	// names a hot one waits for its pile to drain (#5934).
	371	await learnPullFiles(app, repo).catch((error) =>
	372	log.warn("could not read the open pulls' files", {
	373	repo: `${repo.ownerName}/${repo.name}`,
	374	error: String(error),
	375	}),
	376	);
368	377	const sorted = () => heldBy(app, repo, readyNow(repoGraph(app.db, repo.id, { sealed: true })), crew, active);
369	378	let held = sorted();
370	379	if (freeParked(app, repo, by, held) > 0) held = sorted();
	380	holdHot(repo.id, held.by.hot_file, now);
		junipercrew 19d ago holdHot runs only past the early "waiting" return above, so a repo that stays paced out keeps its last feed post. Clear the standing entry on that path too.
371	381	const ready = held.startable;
372	382	if (slotted.length >= paced.slots) {
373	383	// The builders are busy; the planner can still refill the map, when
		@@ -1261,6 +1271,8 @@ export async function autonomyTick(
1261	1271	// The standing reserve the planner keeps: startable/target (#3237).
1262	1272	reserve: reserveWords(reserveOf(held.startable.length, status.slots)),
1263	1273	held: heldWords(held),
	1274	// The files 5 or more open pulls touch, which no new build starts on (#5934).
	1275	...(held.by.hot_file.length === 0 ? {} : { hot: hotWords(hotFiles(current.id)) }),
1264	1276	// Stalled issues this tick parked and freed for another builder (#4073).
1265	1277	unstuck,
1266	1278	active: status.active,
		@@ -20,6 +20,8 @@
20	20	* - `run_cap`: it has used its runs for the day (`guard.ts`).
21	21	* - `parked_twice`: the crew parked it twice; it is the planner's (`parks.ts`).
22	22	* - `red_fix`: a fix for a red main already covers it (`red_fix.ts`).
	23	* - `hot_file`: a file it names is touched by `HOT_PULLS` or more open
	24	* pulls; it starts when that pile drains under the limit (`hot_files.ts`).
23	25	*
24	26	* The direction issue is the planner's standing issue and never a
25	27	* builder's; it is left out, not counted.
		@@ -39,6 +41,7 @@ import { proposedWork, speaksForCrew } from "./direct.ts";
39	41	import { capReached, runCapOf } from "./guard.ts";
40	42	import { interruptionOf } from "./interrupted.ts";
41	43	import { PARK_STRIKES, parkedCount } from "./parks.ts";
	44	import { hotFiles, hotOverlap } from "./hot_files.ts";
42	45	import { coveredByRedFix } from "./red_fix.ts";
43	46
44	47	export const HOLDS = [
		@@ -51,6 +54,7 @@ export const HOLDS = [
51	54	"run_cap",
52	55	"parked_twice",
53	56	"red_fix",
	57	"hot_file",
54	58	] as const;
55	59
56	60	export type HoldName = (typeof HOLDS)[number];
		@@ -120,6 +124,7 @@ export function heldBy(
120	124	crew.filter((c) => c.role !== "builder").map((c) => c.agent.user.id),
121	125	);
122	126	const handed = withPlanner(app.db, repo);
	127	const hot = hotFiles(repo.id);
123	128	const holdOf = (issue: Issue): HoldName \| null => {
124	129	if (busyIssues.has(issue.id)) return "running";
125	130	if (proposed.has(issue.number)) return "open_pull";
		@@ -130,6 +135,7 @@ export function heldBy(
130	135	if (atRunCap(app, repo, issue)) return "run_cap";
131	136	if (parkedCount(app.db, repo, issue.id) >= PARK_STRIKES) return "parked_twice";
132	137	if (coveredByRedFix(app.db, repo.id, issue)) return "red_fix";
	138	if (hotOverlap(issue, hot).length > 0) return "hot_file";
133	139	return null;
134	140	};
135	141	const by = Object.fromEntries(HOLDS.map((h) => [h, [] as Issue[]])) as Record<
		@@ -0,0 +1,157 @@
	1	/**
	2	* The hot-file WIP limit (#5934, #5924 step 3): a build is not started
	3	* while a file it expects to touch is already touched by `HOT_PULLS` or
	4	* more open pulls. One more pull on that file is one more rebase for each
	5	* of the others when any of them lands; the build waits until the pile
	6	* drains under the limit.
	7	*
	8	* - An issue's expected files are the repository paths its title and body
	9	* name (`expectedFiles`): `src/http/mcp.ts`, `docs/MCP.md`.
	10	* - The files an open pull touches are read from git once per head
	11	* (`learnPullFiles`, asked by the picker before it sorts), and kept in
	12	* memory: a cache git rebuilds, never state of its own.
	13	* - The holds the picker last said, per repository (`standingHot`), are what
	14	* the feed shows while they stand.
	15	*/
	16
	17	import type { App } from "../app.ts";
	18	import { getIssuesByIds, type Issue } from "../core/issues.ts";
	19	import { joinMany, pullDir } from "../core/pulls.ts";
	20	import type { Repo } from "../core/repos.ts";
	21	import { runGitCapture } from "../git/env.ts";
	22
	23	/** A file this many open pulls touch is hot: no new build starts on it. */
	24	export const HOT_PULLS = 5;
		junipercrew 19d ago Path extraction from free prose is a heuristic; a mention of a hot file that the build will not touch holds it indefinitely. Consider limiting to a fenced "Files:" line or the title, or surfacing the matched path so a person can fix the issue text.
	25
	26	/** The most paths read from one issue: a list longer than this is a manifest, not a plan. */
	27	const MAX_EXPECTED = 40;
	28
	29	/** A path with at least one directory and a file extension: `src/http/mcp.ts`. */
	30	const PATH = /(?<![\w./:-])((?:[\w-][\w.-]\/)+[\w-][\w.-]\.[A-Za-z0-9]+)(?![\w/])/g;
	31	/** A bare file name in backticks: `README.md`, `forge.json`. */
	32	const QUOTED = /`([\w-][\w.-]*\.[A-Za-z0-9]+)`/g;
	33
	34	/** The repository paths an issue names, in the order it names them, each once. */
	35	export function expectedFiles(issue: Pick<Issue, "title" \| "body">): string[] {
	36	const text = `${issue.title}\n${issue.body}`;
	37	const found = new Set<string>();
	38	for (const pattern of [PATH, QUOTED])
	39	for (const match of text.matchAll(pattern)) {
	40	const path = match[1]!.replace(/\.+$/, "");
	41	if (path.split("/").some((part) => part === "." \|\| part === "..")) continue;
	42	found.add(path);
	43	if (found.size >= MAX_EXPECTED) return [...found];
	44	}
	45	return [...found];
	46	}
	47
	48	export interface HotFile {
	49	readonly path: string;
	50	/** Open pulls that touch it. */
	51	readonly pulls: number;
	52	}
	53
	54	/** Per repository, per open pull: the head and base it was read at, and the files. */
	55	const touched = new Map<string, Map<string, { at: string; files: readonly string[] }>>();
	56
	57	const OID = /^[0-9a-f]{40,64}$/;
	58
	59	/**
	60	* Read the files each open pull of `repo` touches, for any pull whose head
	61	* moved since it was last read, and forget the pulls no longer open. A pull
	62	* git cannot answer for touches nothing until it can.
	63	*/
	64	export async function learnPullFiles(app: App, repo: Repo): Promise<void> {
	65	const ids = app.db
	66	.query<{ id: string }, [string]>(
	67	`SELECT p.issue_id AS id FROM pull p JOIN issue i ON i.id = p.issue_id
	68	WHERE i.repo_id = ?1 AND i.state = 'open' AND p.merged_at IS NULL`,
	69	)
	70	.all(repo.id)
	71	.map((r) => r.id);
	72	const pulls = joinMany(app.db, [...getIssuesByIds(app.db, ids).values()]);
	73	const before = touched.get(repo.id) ?? new Map();
	74	const after = new Map<string, { at: string; files: readonly string[] }>();
	75	for (const pull of pulls) {
	76	const at = `${pull.baseOid}...${pull.headOid}`;
	77	const known = before.get(pull.id);
	78	if (known !== undefined && known.at === at) {
	79	after.set(pull.id, known);
	80	continue;
	81	}
	82	if (!OID.test(pull.baseOid) \|\| !OID.test(pull.headOid)) continue;
	83	const dir = await pullDir(app.db, app.reposRoot, repo, pull);
	84	const out = await runGitCapture(["diff", "--name-only", "-z", at], { cwd: dir });
	85	if (out.code !== 0) continue;
	86	after.set(pull.id, { at, files: out.stdout.split("\0").filter((f) => f !== "") });
	87	}
	88	touched.set(repo.id, after);
	89	}
	90
	91	/** Every file of `repo` that `HOT_PULLS` or more open pulls touch, with how many. */
	92	export function hotFiles(repoId: string): Map<string, number> {
	93	const counts = new Map<string, number>();
	94	for (const { files } of touched.get(repoId)?.values() ?? [])
	95	for (const file of files) counts.set(file, (counts.get(file) ?? 0) + 1);
	96	for (const [file, n] of counts) if (n < HOT_PULLS) counts.delete(file);
	97	return counts;
	98	}
	99
	100	/** The hot files `issue` expects to touch, hottest first. */
	101	export function hotOverlap(
	102	issue: Pick<Issue, "title" \| "body">,
	103	hot: ReadonlyMap<string, number>,
	104	): HotFile[] {
	105	if (hot.size === 0) return [];
	106	return expectedFiles(issue)
	107	.flatMap((path) => {
	108	const pulls = hot.get(path);
	109	return pulls === undefined ? [] : [{ path, pulls }];
	110	})
	111	.sort((a, b) => b.pulls - a.pulls \|\| a.path.localeCompare(b.path));
	112	}
	113
	114	/** A held build as the feed says it. */
	115	export interface HotHold {
	116	readonly number: number;
	117	readonly title: string;
	118	readonly files: readonly HotFile[];
	119	}
	120
	121	/** What the picker last held on hot files, per repository, and when. */
	122	const standing = new Map<string, { at: string; holds: readonly HotHold[] }>();
	123
	124	/** Remember the builds the picker holds on hot files in `repoId` now; none clears it. */
	125	export function holdHot(repoId: string, issues: readonly Issue[], at = new Date()): void {
	126	if (issues.length === 0) {
	127	standing.delete(repoId);
	128	return;
	129	}
	130	const hot = hotFiles(repoId);
	131	standing.set(repoId, {
	132	at: at.toISOString(),
	133	holds: issues.map((i) => ({ number: i.number, title: i.title, files: hotOverlap(i, hot) })),
	134	});
	135	}
	136
	137	/** The builds held on hot files in `repoId` as the picker last said, if any. */
	138	export function standingHot(repoId: string): { at: string; holds: readonly HotHold[] } \| undefined {
	139	return standing.get(repoId);
	140	}
	141
	142	/** The hot files as the tick line says them: `src/http/mcp.ts=61 docs/MCP.md=54`, hottest first. */
	143	export function hotWords(hot: ReadonlyMap<string, number>): string {
	144	return [...hot]
	145	.sort((a, b) => b[1] - a[1] \|\| a[0].localeCompare(b[0]))
	146	.map(([path, n]) => `${path}=${n}`)
	147	.join(" ");
	148	}
	149
	150	/** One sentence for a held build: `#12 waits: src/http/mcp.ts is in 6 open pulls (hot at 5).` */
	151	export function hotSays(hold: HotHold): string {
	152	const files = hold.files
	153	.slice(0, 3)
	154	.map((f) => `${f.path} is in ${f.pulls} open pulls`)
	155	.join(", ");
	156	return `#${hold.number} waits: ${files === "" ? "a file it names is hot" : files} (hot at ${HOT_PULLS}).`;
	157	}
		@@ -27,6 +27,7 @@ import {
27	27	import { getUserById, type User } from "../core/users.ts";
28	28	import { watchedBy } from "../core/watches.ts";
29	29	import { chooseCandidates } from "./choose.ts";
	30	import { hotSays, standingHot } from "../crew/hot_files.ts";
30	31	import { ago } from "./rank.ts";
31	32	import { askLinesFor } from "../web/studio/asks.ts";
32	33
		@@ -197,6 +198,35 @@ function frozenCandidates(
197	198	return out;
198	199	}
199	200
	201	/**
	202	* Builds the picker holds because a file they name is in 5 or more open
	203	* pulls (#5934): one post per repository the viewer writes to, read from
	204	* what the picker said last, so it goes when the pile drains.
	205	*/
	206	function hotCandidates(writable: readonly Repo[]): Candidate[] {
	207	const out: Candidate[] = [];
	208	for (const repo of writable) {
	209	const standing = standingHot(repo.id);
	210	if (standing === undefined) continue;
	211	const n = standing.holds.length;
	212	const first = standing.holds[0]!;
	213	const what = `${n === 1 ? "1 build waits" : `${n} builds wait`} on hot files: ${hotSays(first)}${n > 1 ? ` And ${n - 1} more.` : ""}`;
	214	out.push({
	215	id: `hot:${repo.id}`,
	216	kind: "map",
	217	repoId: repo.id,
	218	app: slug(repo),
	219	actorId: null,
	220	actor: null,
	221	at: standing.at,
	222	link: `${base(repo)}/map`,
	223	what: clip(what),
	224	urgency: 0,
	225	});
	226	}
	227	return out;
	228	}
	229
200	230	export function candidatesFor(
201	231	db: Database,
202	232	viewer: User,
		@@ -297,6 +327,7 @@ export function candidatesFor(
297	327	out.push(...watchedCandidates(db, viewer, now));
298	328	out.push(...frozenCandidates(db, viewer, now));
299	329	out.push(...chooseCandidates(db, writable));
	330	out.push(...hotCandidates(writable));
300	331	for (const d of decided) {
301	332	const repo = repos.get(d.repoId);
302	333	if (!repo) continue;
		@@ -32,7 +32,7 @@ export function heldCard(rows: readonly HeldRow[]): Html {
32	32	<p class="muted small">
33	33	Each ready issue a builder could start, or the first filter that holds
34	34	it: a run on it, an open pull, the planner, stalled, its run cap,
35		parked twice, a red-main fix. When nothing can
	35	parked twice, a red-main fix, a file 5 or more open pulls touch. When nothing can
36	36	start and builders are free, the planner is asked to re-plan what is
37	37	stalled or capped.
38	38	</p>
		@@ -0,0 +1,195 @@
	1	/**
	2	* The hot-file WIP limit (#5934): with five open pulls on one file, a sixth
	3	* build naming that file waits while a build on a cold file starts, and
	4	* the hold is said where the picker says why it skipped — the held counts,
	5	* the feed, and the feed through the MCP. The hold lifts when the pile
	6	* drains under five.
	7	*/
	8	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	9	import { mkdtemp, rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { createApp, type App } from "../src/app.ts";
	13	import { addToCrew, createAgent } from "../src/actions/crew.ts";
	14	import { openIssue, setIssueState } from "../src/actions/issues.ts";
	15	import { openPull } from "../src/actions/pulls.ts";
	16	import { commitToBranch, createRepo, updateRepoSettings } from "../src/actions/repos.ts";
	17	import { setCrewRole } from "../src/core/agents.ts";
	18	import { getIssueById, type Issue } from "../src/core/issues.ts";
	19	import type { Pull } from "../src/core/pulls.ts";
	20	import { getRepoById, type Repo } from "../src/core/repos.ts";
	21	import { createToken } from "../src/core/tokens.ts";
	22	import { createUser, getUserByName, type User } from "../src/core/users.ts";
	23	import { driveRepo } from "../src/crew/autonomy.ts";
	24	import { heldNow, heldWords } from "../src/crew/held.ts";
	25	import {
	26	expectedFiles,
	27	HOT_PULLS,
	28	hotFiles,
	29	hotOverlap,
	30	learnPullFiles,
	31	standingHot,
	32	} from "../src/crew/hot_files.ts";
	33	import { readFeed } from "../src/feed/feed.ts";
	34	import { serve, type Servers } from "../src/http/server.ts";
	35
	36	describe("expected files", () => {
	37	test("are the repository paths an issue names, each once", () => {
	38	expect(
	39	expectedFiles({
	40	title: "Split src/http/mcp.ts into one file per tool",
	41	body: "Touches `docs/MCP.md`, `README.md` and src/http/mcp.ts again; see https://example.com/a/b.html and ../up/x.ts.",
	42	}),
	43	).toEqual(["src/http/mcp.ts", "docs/MCP.md", "README.md"]);
	44	expect(expectedFiles({ title: "Make the home page faster", body: "" })).toEqual([]);
	45	});
	46
	47	test("overlap a hot file only when it is named", () => {
	48	const hot = new Map([["src/a.ts", 7], ["src/b.ts", 5]]);
	49	expect(hotOverlap({ title: "src/b.ts and src/a.ts", body: "" }, hot)).toEqual([
	50	{ path: "src/a.ts", pulls: 7 },
	51	{ path: "src/b.ts", pulls: 5 },
	52	]);
	53	expect(hotOverlap({ title: "src/c.ts", body: "" }, hot)).toEqual([]);
	54	});
	55	});
	56
	57	describe("the picker holds a build on a hot file", () => {
	58	let dataDir: string;
	59	let app: App;
	60	let alice: User;
	61	let servers: Servers;
	62
	63	beforeAll(async () => {
	64	dataDir = await mkdtemp(join(tmpdir(), "forge-hot-"));
	65	app = createApp({ dataDir, port: 0 });
	66	const a = await createUser(app.db, {
	67	origin: "seed",
	68	trusted: true,
	69	username: "alice",
	70	email: "[email protected]",
	71	password: "a-good-passphrase",
	72	adminIfFirst: true,
	73	});
	74	if (!a.ok) throw new Error(a.error);
	75	alice = a.value;
	76	servers = await serve(app);
	77	});
	78
	79	afterAll(async () => {
	80	servers.stop();
	81	app.close();
	82	await rm(dataDir, { recursive: true, force: true });
	83	});
	84
	85	const fresh = (repo: Repo) => getRepoById(app.db, repo.id)!;
	86
	87	function issue(repo: Repo, title: string): Issue {
	88	const opened = openIssue(app, alice, fresh(repo), { title, body: "" });
	89	if (!opened.ok) throw new Error(String(opened.error));
	90	return opened.value;
	91	}
	92
	93	async function pullOn(repo: Repo, n: number, path: string): Promise<Pull> {
	94	const branch = await commitToBranch(app, alice, fresh(repo), {
	95	branch: `feature/${n}`,
	96	base: "main",
	97	message: `Change ${n}`,
	98	files: [{ path, content: `export const n = ${n};\n` }],
	99	});
	100	if (!branch.ok) throw new Error(JSON.stringify(branch.error));
	101	const pull = await openPull(app, alice, fresh(repo), {
	102	title: `Change ${n}`,
	103	body: "",
	104	baseBranch: "main",
	105	headBranch: `feature/${n}`,
	106	});
	107	if (!pull.ok) throw new Error(String(pull.error));
	108	return pull.value;
	109	}
	110
	111	test("a sixth build on a hot file waits while a build on a cold file starts", async () => {
	112	const made = await createRepo(app, alice, { name: "engine", visibility: "private" });
	113	if (!made.ok) throw new Error(String(made.error));
	114	for (const agent of ["atlas", "mason"]) {
	115	const created = await createAgent(app, alice, { username: agent });
	116	if (!created.ok) throw new Error(String(created.error));
	117	expect(addToCrew(app, alice, made.value, agent).ok).toBe(true);
	118	}
	119	setCrewRole(app.db, made.value.id, getUserByName(app.db, "atlas")!.id, "planner");
	120	const set = await updateRepoSettings(app, alice, made.value, {
	121	autonomy: "autonomous",
	122	autonomyDirection: "Keep the engine small.",
	123	autonomyCeilingUsd: 100,
	124	autonomySlots: 1,
	125	autonomyBy: alice.id,
	126	});
	127	if (!set.ok) throw new Error(JSON.stringify(set.error));
	128	const repo = fresh(made.value);
	129	const seeded = await commitToBranch(app, alice, repo, {
	130	message: "Start",
	131	files: [
	132	{ path: "src/hot.ts", content: "export const n = 0;\n" },
	133	{ path: "src/cold.ts", content: "export const n = 0;\n" },
	134	],
	135	});
	136	if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
	137	const pulls: Pull[] = [];
	138	for (let n = 1; n <= HOT_PULLS; n++) pulls.push(await pullOn(repo, n, "src/hot.ts"));
	139
	140	// First on the map, so only the hold keeps it from being picked.
	141	const hotWork = issue(repo, "Add a flag to src/hot.ts");
	142	const coldWork = issue(repo, "Tidy src/cold.ts");
	143
	144	expect(await driveRepo(app, fresh(repo))).toBe("picked");
	145	const mason = getUserByName(app.db, "mason")!;
	146	expect(getIssueById(app.db, coldWork.id)!.assigneeId).toBe(mason.id);
	147	expect(getIssueById(app.db, hotWork.id)!.assigneeId).toBeNull();
	148	expect(hotFiles(repo.id)).toEqual(new Map([["src/hot.ts", HOT_PULLS]]));
	149
	150	// Said where the picker says why it skipped: the held counts the tick
	151	// line and /flow read…
	152	const held = heldNow(app, fresh(repo));
	153	expect(held.by.hot_file.map((i) => i.number)).toEqual([hotWork.number]);
	154	expect(heldWords(held)).toContain("hot_file=1");
	155	expect(standingHot(repo.id)?.holds).toEqual([
	156	{ number: hotWork.number, title: hotWork.title, files: [{ path: "src/hot.ts", pulls: HOT_PULLS }] },
	157	]);
	158
	159	// …the feed…
	160	const said = `#${hotWork.number} waits: src/hot.ts is in ${HOT_PULLS} open pulls (hot at ${HOT_PULLS}).`;
	161	const feed = readFeed(app.db, alice, { limit: 50 });
	162	const post = feed.items.find((i) => i.id === `hot:${repo.id}`);
	163	expect(post?.what).toBe(`1 build waits on hot files: ${said}`);
	164	expect(post?.link).toBe(`/alice/engine/map`);
	165
	166	// …and the feed through the MCP.
	167	const token = createToken(app.db, alice.id, "t", "ro").token;
	168	const res = await fetch(`${servers.url}/mcp`, {
	169	method: "POST",
	170	headers: {
	171	authorization: `Bearer ${token}`,
	172	"content-type": "application/json",
	173	accept: "application/json, text/event-stream",
	174	},
	175	body: JSON.stringify({
	176	jsonrpc: "2.0",
	177	id: 1,
	178	method: "tools/call",
	179	params: { name: "feed", arguments: { limit: 50 } },
	180	}),
	181	});
	182	const body = (await res.json()) as { result: { content: { text: string }[] } };
	183	expect(body.result.content[0]!.text).toContain(said);
	184
	185	// The pile drains under the limit: the hold lifts, and the post goes.
	186	const closed = setIssueState(app, alice, fresh(repo), getIssueById(app.db, pulls[0]!.id)!, "closed");
	187	expect(closed.ok).toBe(true);
	188	await learnPullFiles(app, fresh(repo));
	189	expect(hotFiles(repo.id).size).toBe(0);
	190	expect(heldNow(app, fresh(repo)).by.hot_file).toEqual([]);
	191	await driveRepo(app, fresh(repo));
	192	expect(standingHot(repo.id)).toBeUndefined();
	193	expect(readFeed(app.db, alice, { limit: 50 }).items.some((i) => i.id === `hot:${repo.id}`)).toBe(false);
	194	});
	195	});