bca1ef3 · 69be77e · 70723f9
@@ -558,6 +558,13 @@
The web page has the same *Cost and landed* section on a milestone, and the
milestones API and MCP carry it as `spend` (`usd`, `landed`, `costliest`),
owner only.
The milestones page's `Server-Timing` names what a plan cost when it had to be
built: `plan-graph`, `plan-crew`, `plan-week`, `plan-spend` and `plan-limbs`. A
plan kept from an earlier request builds nothing, so those names appear on the
request that built it and not on the ones it served; `plan` is the whole phase,
which on a kept plan is the one stamp read that says nothing changed. That stamp
is itself kept while the forge's write clock has not moved.
Working on the UI itself? `bun run src/tui/main.ts --user alice` runs it over
stdio against your local data directory. That `--user` flag is a development
convenience and exists only there: over SSH, the key decides who you are and
@@ -287,6 +287,8 @@
export function repoGraph(
    readonly sealedBy?: string;
    /** The repository's pillars, when the caller has read them already. */
    readonly pillars?: Pillars;
    /** Issues come without their bodies, when the caller shows none (a plan). */
    readonly lean?: boolean;
  } = {},
): Graph {
  const edges = db
@@ -312,7 +314,7 @@
export function repoGraph(
    .all(repoId);
  // A big graph wants most of its repository: one read of it beats chunked
  // lookups, and the open issues are found in it.
  const whole = edges.length > GRAPH_MAX ? getIssuesOfRepo(db, repoId) : undefined;
  const whole = edges.length > GRAPH_MAX ? getIssuesOfRepo(db, repoId, "issue", options.lean) : undefined;
  const open =
    whole === undefined
      ? db
@@ -98,6 +98,14 @@
const SELECT = `
  FROM issue JOIN user ON user.id = issue.author_id
  LEFT JOIN user AS assignee ON assignee.id = issue.assignee_id`;
/** `SELECT` without the body: a plan reads titles and states, and a body is most of a row's bytes. */
const SELECT_LEAN = SELECT.replace(
  "issue.*,",
  `issue.id, issue.repo_id, issue.number, issue.kind, issue.title, '' AS body, issue.author_id,
         issue.state, issue.created_at, issue.updated_at, issue.closed_at, issue.close_reason,
         issue.parked, issue.origin_author, issue.origin_html_url, issue.assignee_id, issue.sealed,`,
);
function toIssue(row: IssueRow): Issue {
  return {
    id: row.id,
@@ -169,11 +177,20 @@
export function getIssuesByIds(
 * Every conversation of one repository in one read, by id: cheaper than
 * `getIssuesByIds` once most of a repository is wanted.
 */
export function getIssuesOfRepo(db: Database, repoId: string): Map<string, Issue> {
export function getIssuesOfRepo(
  db: Database,
  repoId: string,
  /** Only issues, or only pulls: a pull's body is the bulk of a repository's rows. */
  kind?: IssueKind,
  /** Leave the bodies empty: for a reader that never shows one. */
  lean = false,
): Map<string, Issue> {
  const out = new Map<string, Issue>();
  for (const row of db
    .query<IssueRow, [string]>(`${SELECT} WHERE issue.repo_id = ?1`)
    .all(repoId))
    .query<IssueRow, [string, string]>(
      `${lean ? SELECT_LEAN : SELECT} WHERE issue.repo_id = ?1 AND (?2 = '' OR issue.kind = ?2)`,
    )
    .all(repoId, kind ?? ""))
    out.set(row.id, toIssue(row));
  return out;
}
@@ -14,6 +14,7 @@
 */
import type { Database } from "bun:sqlite";
import { writeStamp } from "./write_clock.ts";
import { deferredIn, repoGraph, type Graph } from "./dependencies.ts";
import {
  limbTree,
@@ -181,7 +182,23 @@
const counted = (issue: Issue): boolean =>
 * inside a transaction: nothing is kept that a rollback could outlive.
 */
export function planStamp(db: Database, repo: Repo): string | undefined {
  if (db.inTransaction) return undefined;
  const clock = writeStamp(db);
  if (clock === undefined) return undefined;
  let kept = stamps.get(db);
  if (kept === undefined) stamps.set(db, (kept = new Map()));
  const known = kept.get(repo.id);
  if (known?.clock === clock) return known.stamp;
  const stamp = readStamp(db, repo);
  if (kept.size >= STAMPS_KEPT) kept.clear();
  kept.set(repo.id, { clock, stamp });
  return stamp;
}
/** The stamp last read for each repository, and the write clock it was read at: nothing written since means nothing to read again. */
      
        junipercrew
        19d ago
        
      
      STAMPS_KEPT clears the whole map when full, so a forge with more than 200 repos active will thrash. Evict one entry or size it from the repo count.
    
const stamps = new WeakMap<Database, Map<string, { clock: string; stamp: string }>>();
const STAMPS_KEPT = 200;
function readStamp(db: Database, repo: Repo): string {
  const one = (sql: string): string =>
    JSON.stringify(Object.values(db.query<Record<string, unknown>, [string]>(sql).get(repo.id) ?? {}));
  return [
@@ -200,18 +217,30 @@
export function planStamp(db: Database, repo: Repo): string | undefined {
export function readPlan(
  db: Database,
  repo: Repo,
  options: { readonly spend: boolean; readonly now?: Date } = { spend: false },
  options: {
    readonly spend: boolean;
    readonly now?: Date;
    /** Told how long each stretch took (`graph`, `crew`, `week`, `spend`, `limbs`), for Server-Timing. */
    readonly stretch?: (name: string, ms: number) => void;
  } = { spend: false },
): Plan {
  const now = options.now ?? new Date();
  const since = pillarWindowStart(now);
  let mark = performance.now();
  const lap = (name: string): void => {
    const at = performance.now();
    options.stretch?.(name, at - mark);
    mark = at;
  };
  const set: Pillars = pillarsOf(db, repo);
  const graph: Graph = repoGraph(db, repo.id, { pillars: set });
  const graph: Graph = repoGraph(db, repo.id, { pillars: set, lean: true });
  const byId = new Map(graph.issues.map((i) => [i.id, i]));
  const byNumber = new Map(graph.issues.map((i) => [i.number, i]));
  // What each issue waits for directly, furniture left out (`limb_order.ts`).
  const tree = limbTree(graph, set);
  const places = readPlaces(db, repo.id);
  lap("graph");
  const { waitsFor, waitedBy, waits, childrenOf } = tree;
  const memo = new Map<string, Set<string>>();
  const walking = new Set<string>();
@@ -262,6 +291,8 @@
export function readPlan(
    });
  }
  lap("crew");
  // The week's merged pulls, by the issues each closes.
  const merged: MergedPull[] = db
    .query<
@@ -281,12 +312,14 @@
export function readPlan(
      closes: closesFromBody(row.body),
    }));
  lap("week");
  // What each issue cost: its own runs, and the runs on the pulls for it.
  const cost = new Map<string, number>();
  if (options.spend) {
    const rows = db
      .query<{ id: string; kind: string; body: string; usd: number }, [string]>(
        `SELECT i.id, i.kind, i.body, SUM(r.cost_usd) AS usd
        `SELECT i.id, i.kind, CASE i.kind WHEN 'pull' THEN i.body ELSE '' END AS body, SUM(r.cost_usd) AS usd
           FROM run r JOIN issue i ON i.id = r.issue_id
          WHERE r.repo_id = ?1 AND r.kind = 'agent' AND r.cost_usd > 0
          GROUP BY i.id`,
@@ -305,16 +338,19 @@
export function readPlan(
  }
  // Every pull ever merged, by the issues it closes: what a limb landed.
  // One query that parses each merged pull's body; owner reads only.
  const everMerged: (readonly number[])[] = options.spend
    ? db
        .query<{ body: string }, [string]>(
          `SELECT i.body FROM pull p JOIN issue i ON i.id = p.issue_id
            WHERE i.repo_id = ?1 AND p.merged_at IS NOT NULL`,
        )
        .all(repo.id)
        .map((row) => closesFromBody(row.body))
    : [];
  // One query that parses each merged pull's body; owner reads only, and
  // only one milestone's page asks, so the list of limbs never pays for it.
  let ever: (readonly number[])[] | undefined;
  const everMerged = (): (readonly number[])[] =>
    (ever ??= db
      .query<{ body: string }, [string]>(
        `SELECT i.body FROM pull p JOIN issue i ON i.id = p.issue_id
          WHERE i.repo_id = ?1 AND p.merged_at IS NOT NULL`,
      )
      .all(repo.id)
      .map((row) => closesFromBody(row.body)));
  lap("spend");
  const stateOf = (issue: Issue): LeafState => {
    if (issue.state === "closed") return counted(issue) ? "done" : "gone";
@@ -391,9 +427,12 @@
export function readPlan(
    tree.top.map((i) => i.id),
  );
  const limbs = top.map(limbOf);
  lap("limbs");
  return {
    read: { graph, tree, places },
    limbs: top.map(limbOf),
    limbs,
    limb(number: number): { own: Issue[]; shared: Issue[] } | undefined {
      const issue = byNumber.get(number);
      if (issue === undefined) return undefined;
@@ -554,7 +593,7 @@
export function readPlan(
          ? {
              spend: {
                usd: limb.weight.spentUsd ?? 0,
                landed: everMerged.filter((closes) =>
                landed: everMerged().filter((closes) =>
                  closes.some((n) => numbers.has(n)),
                ).length,
                costliest: under
@@ -206,7 +206,11 @@
const planFor = (ctx: Ctx, repo: Repo, spend: boolean): Plan =>
      spend,
      now: new Date(),
    },
    () => readPlan(ctx.app.db, repo, { spend }),
    () =>
      readPlan(ctx.app.db, repo, {
        spend,
        stretch: (name, ms) => notePhase(ctx.request, `plan-${name}`, ms),
      }),
  );
/** GET /:owner/:name/milestones — the plan, collapsed to its limbs. */
@@ -0,0 +1,127 @@
/** A forge-shaped repository: pulls, runs, sealed issues and nested limbs, not only a flat fan of leaves. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { openIssue } from "../src/actions/issues.ts";
import { planStamp, readPlan } from "../src/core/milestones.ts";
import { getRepo } from "../src/core/repos.ts";
import { createUser, getUserByName } from "../src/core/users.ts";
import { expectBudget } from "./helpers/budget.ts";
import { quietLogs } from "./helpers/globals.ts";
import { proveAll } from "./helpers/verified.ts";
quietLogs();
const PILLARS = 10;
let root = "";
let app: App;
let uid = "";
beforeAll(async () => {
  root = mkdtempSync(join(tmpdir(), "forge-milestones-shape-"));
  app = createApp({ dataDir: root, port: 0, forgeRepo: "alice/forge" });
  const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
  if (!u.ok) throw new Error(u.error);
  uid = u.value.id;
  proveAll(app);
  const made = await createRepo(app, { id: uid, username: "alice" } as never, { name: "forge" });
  if (!made.ok) throw new Error(made.error);
  const repoId = getRepo(app.db, "alice", "forge")!.id;
  const issue = app.db.prepare(
    `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at, updated_at, closed_at, sealed)
     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?9, ?10, ?11)`,
  );
  const edge = app.db.prepare("INSERT INTO issue_dependency (issue_id, blocked_by_id) VALUES (?1, ?2)");
  const pull = app.db.prepare(
    `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) VALUES (?1, 'main', ?2, 'a', 'b', ?3)`,
  );
  const run = app.db.prepare(
    `INSERT INTO run (id, repo_id, kind, issue_id, state, cost_usd, actor_id, created_at, started_at) VALUES (?1, ?2, 'agent', ?3, ?4, 0.5, ?5, ?6, ?6)`,
  );
  const body = "Why this matters. ".repeat(70);
  const at = new Date(Date.now() - 3_600_000).toISOString();
  const old = new Date(Date.now() - 40 * 86_400_000).toISOString();
  app.db.transaction(() => {
    let n = 0;
    for (let p = 0; p < PILLARS; p++)
      issue.run(`u-${p}`, repoId, ++n, "issue", `P${p + 1} Pillar ${p + 1}: ask`, "", uid, "open", at, null, 0);
    const leaves = 2300;
    for (let k = 0; k < leaves; k++) {
      const closed = k < 2000;
      issue.run(`c-${k}`, repoId, ++n, "issue", `Step ${k}`, body, uid, closed ? "closed" : "open", at, closed ? at : null, k % 8 === 3 ? 1 : 0);
      // a mid-level limb every 20 leaves, nested under its pillar; leaves chain behind each other
      if (k % 20 === 0) edge.run(`u-${Math.floor(k / 20) % PILLARS}`, `c-${k}`);
      else edge.run(`c-${k - (k % 20)}`, `c-${k}`);
      if (k % 20 > 1) edge.run(`c-${k - 1}`, `c-${k}`);
    }
    for (let k = 0; k < 3000; k++) {
      const merged = k < 2800;
      // 400 landed this week, the rest long ago
      const when = k < 400 ? at : old;
      issue.run(`p-${k}`, repoId, ++n, "pull", `Pull ${k}`, `${body}\nCloses #${PILLARS + 1 + (k % leaves)}`, uid, merged ? "closed" : "open", when, merged ? when : null, 0);
      pull.run(`p-${k}`, `b-${k}`, merged ? when : null);
    }
    for (let k = 0; k < 1500; k++) run.run(`r-${k}`, repoId, `c-${k % leaves}`, k < 30 ? "running" : "ok", uid, at);
    app.db.run("UPDATE repo SET issue_seq = ?1 WHERE id = ?2", [n, repoId]);
  })();
});
afterAll(() => {
  app.close();
  rmSync(root, { recursive: true, force: true });
});
const weights = (spend: boolean) =>
  readPlan(app.db, getRepo(app.db, "alice", "forge")!, { spend }).limbs.map((l) => [
    l.name, l.weight.total, l.weight.done, l.weight.open, l.weight.merged, l.weight.building, l.weight.spentUsd,
  ]);
/** What each limb weighs on the code before the stamp and the lean read: the counts must not move. */
const BEFORE = [
  ["P1 Pillar 1", 204, 170, 34, 34, 0, 68],
  ["P2 Pillar 2", 216, 180, 36, 36, 0, 72],
  ["P3 Pillar 3", 204, 170, 34, 34, 0, 68],
  ["P4 Pillar 4", 216, 180, 36, 36, 0, 72],
  ["P5 Pillar 5", 204, 170, 34, 34, 0, 68],
  ["P6 Pillar 6", 198, 180, 18, 36, 0, 63],
  ["P7 Pillar 7", 187, 170, 17, 34, 0, 59.5],
  ["P8 Pillar 8", 198, 180, 18, 36, 0, 63],
  ["P9 Pillar 9", 187, 170, 17, 34, 0, 59.5],
  ["P10 Pillar 10", 198, 180, 18, 36, 0, 63],
];
test("a plan of 2,000 closed and 300 open issues, 3,000 pulls and 1,500 runs reads in under 50 ms with the same limbs", async () => {
  const against = 1;
  const repo = getRepo(app.db, "alice", "forge")!;
  readPlan(app.db, repo, { spend: true });
  const times: number[] = [];
  for (let n = 0; n < 7; n++) {
    const start = performance.now();
    readPlan(app.db, repo, { spend: true });
    times.push(performance.now() - start);
  }
  expectBudget(Math.min(...times), { ms: 50, against });
  expect(weights(true)).toEqual(BEFORE);
  expect(weights(false).map((w) => w.slice(0, 6))).toEqual(BEFORE.map((w) => w.slice(0, 6)));
});
test("a sealed issue is in no limb a viewer cannot see", () => {
  const repo = getRepo(app.db, "alice", "forge")!;
  const sealed = app.db.query<{ n: number }, [string]>("SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND sealed = 1").get(repo.id)!.n;
  expect(sealed).toBeGreaterThan(0);
  const plan = readPlan(app.db, repo, { spend: false });
  const seen = new Set(plan.read.graph.issues.map((i) => i.id));
  for (const row of app.db.query<{ id: string }, [string]>("SELECT id FROM issue WHERE repo_id = ?1 AND sealed = 1").all(repo.id))
    expect(seen.has(row.id)).toBe(false);
});
test("the stamp is kept while nothing is written, and moves when an issue is", async () => {
  const repo = getRepo(app.db, "alice", "forge")!;
  const first = planStamp(app.db, repo);
  const start = performance.now();
  for (let n = 0; n < 20; n++) expect(planStamp(app.db, repo)).toBe(first);
  expectBudget((performance.now() - start) / 20, { ms: 2, against: 1 });
  expect(openIssue(app, getUserByName(app.db, "alice")!, repo, { title: "Fresh" }).ok).toBe(true);
  expect(planStamp(app.db, repo)).not.toBe(first);
});
@@ -128,3 +128,22 @@
test("a milestone page names session, repo, plan and render", async () => {
  for (const name of ["session", "repo", "plan", "render"])
    expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
});
test("a cold plan names its stretches, stays under 50 ms, and keeps each limb's counts", async () => {
      
        junipercrew
        19d ago
        
      
      This test would pass on the code before the pull for the 50 ms budget and the counts, so it does not prove the issue. Build a fixture that is slow on the old code (trevato/forge-like shape), then assert the budget.
    
  const against = await baselineMs(servers.url);
  const stretches: string[] = [];
  const repo = getRepo(app.db, "alice", "forge")!;
  const start = performance.now();
  const plan = readPlan(app.db, repo, { spend: true, stretch: (name) => stretches.push(name) });
  expectBudget(performance.now() - start, { ms: 50, against });
  expect(stretches).toEqual(["graph", "crew", "week", "spend", "limbs"]);
  expect(plan.limbs).toHaveLength(PILLARS);
  for (const limb of plan.limbs)
    expect(limb.weight).toMatchObject({ total: (CLOSED + OPEN) / PILLARS, done: CLOSED / PILLARS, open: OPEN / PILLARS });
  clearMapCache();
  const res = await fetch(`${servers.url}/alice/forge/milestones`, { headers: { cookie } });
  await res.text();
  const timing = res.headers.get("server-timing") ?? "";
  for (const name of ["plan-graph", "plan-limbs"]) expect(timing).toContain(`${name};dur=`);
});
		@@ -558,6 +558,13 @@ The web page has the same Cost and landed section on a milestone, and the
558	558	milestones API and MCP carry it as `spend` (`usd`, `landed`, `costliest`),
559	559	owner only.
560	560
	561	The milestones page's `Server-Timing` names what a plan cost when it had to be
	562	built: `plan-graph`, `plan-crew`, `plan-week`, `plan-spend` and `plan-limbs`. A
	563	plan kept from an earlier request builds nothing, so those names appear on the
	564	request that built it and not on the ones it served; `plan` is the whole phase,
	565	which on a kept plan is the one stamp read that says nothing changed. That stamp
	566	is itself kept while the forge's write clock has not moved.
	567
561	568	Working on the UI itself? `bun run src/tui/main.ts --user alice` runs it over
562	569	stdio against your local data directory. That `--user` flag is a development
563	570	convenience and exists only there: over SSH, the key decides who you are and
		@@ -287,6 +287,8 @@ export function repoGraph(
287	287	readonly sealedBy?: string;
288	288	/** The repository's pillars, when the caller has read them already. */
289	289	readonly pillars?: Pillars;
	290	/** Issues come without their bodies, when the caller shows none (a plan). */
	291	readonly lean?: boolean;
290	292	} = {},
291	293	): Graph {
292	294	const edges = db
		@@ -312,7 +314,7 @@ export function repoGraph(
312	314	.all(repoId);
313	315	// A big graph wants most of its repository: one read of it beats chunked
314	316	// lookups, and the open issues are found in it.
315		const whole = edges.length > GRAPH_MAX ? getIssuesOfRepo(db, repoId) : undefined;
	317	const whole = edges.length > GRAPH_MAX ? getIssuesOfRepo(db, repoId, "issue", options.lean) : undefined;
316	318	const open =
317	319	whole === undefined
318	320	? db
		@@ -98,6 +98,14 @@ const SELECT = `
98	98	FROM issue JOIN user ON user.id = issue.author_id
99	99	LEFT JOIN user AS assignee ON assignee.id = issue.assignee_id`;
100	100
	101	/** `SELECT` without the body: a plan reads titles and states, and a body is most of a row's bytes. */
	102	const SELECT_LEAN = SELECT.replace(
	103	"issue.*,",
	104	`issue.id, issue.repo_id, issue.number, issue.kind, issue.title, '' AS body, issue.author_id,
	105	issue.state, issue.created_at, issue.updated_at, issue.closed_at, issue.close_reason,
	106	issue.parked, issue.origin_author, issue.origin_html_url, issue.assignee_id, issue.sealed,`,
	107	);
	108
101	109	function toIssue(row: IssueRow): Issue {
102	110	return {
103	111	id: row.id,
		@@ -169,11 +177,20 @@ export function getIssuesByIds(
169	177	* Every conversation of one repository in one read, by id: cheaper than
170	178	* `getIssuesByIds` once most of a repository is wanted.
171	179	*/
172		export function getIssuesOfRepo(db: Database, repoId: string): Map<string, Issue> {
	180	export function getIssuesOfRepo(
	181	db: Database,
	182	repoId: string,
	183	/** Only issues, or only pulls: a pull's body is the bulk of a repository's rows. */
	184	kind?: IssueKind,
	185	/** Leave the bodies empty: for a reader that never shows one. */
	186	lean = false,
	187	): Map<string, Issue> {
173	188	const out = new Map<string, Issue>();
174	189	for (const row of db
175		.query<IssueRow, [string]>(`${SELECT} WHERE issue.repo_id = ?1`)
176		.all(repoId))
	190	.query<IssueRow, [string, string]>(
	191	`${lean ? SELECT_LEAN : SELECT} WHERE issue.repo_id = ?1 AND (?2 = '' OR issue.kind = ?2)`,
	192	)
	193	.all(repoId, kind ?? ""))
177	194	out.set(row.id, toIssue(row));
178	195	return out;
179	196	}
		@@ -14,6 +14,7 @@
14	14	*/
15	15
16	16	import type { Database } from "bun:sqlite";
	17	import { writeStamp } from "./write_clock.ts";
17	18	import { deferredIn, repoGraph, type Graph } from "./dependencies.ts";
18	19	import {
19	20	limbTree,
		@@ -181,7 +182,23 @@ const counted = (issue: Issue): boolean =>
181	182	* inside a transaction: nothing is kept that a rollback could outlive.
182	183	*/
183	184	export function planStamp(db: Database, repo: Repo): string \| undefined {
184		if (db.inTransaction) return undefined;
	185	const clock = writeStamp(db);
	186	if (clock === undefined) return undefined;
	187	let kept = stamps.get(db);
	188	if (kept === undefined) stamps.set(db, (kept = new Map()));
	189	const known = kept.get(repo.id);
	190	if (known?.clock === clock) return known.stamp;
	191	const stamp = readStamp(db, repo);
	192	if (kept.size >= STAMPS_KEPT) kept.clear();
	193	kept.set(repo.id, { clock, stamp });
	194	return stamp;
	195	}
	196
	197	/** The stamp last read for each repository, and the write clock it was read at: nothing written since means nothing to read again. */
		junipercrew 19d ago STAMPS_KEPT clears the whole map when full, so a forge with more than 200 repos active will thrash. Evict one entry or size it from the repo count.
	198	const stamps = new WeakMap<Database, Map<string, { clock: string; stamp: string }>>();
	199	const STAMPS_KEPT = 200;
	200
	201	function readStamp(db: Database, repo: Repo): string {
185	202	const one = (sql: string): string =>
186	203	JSON.stringify(Object.values(db.query<Record<string, unknown>, [string]>(sql).get(repo.id) ?? {}));
187	204	return [
		@@ -200,18 +217,30 @@ export function planStamp(db: Database, repo: Repo): string \| undefined {
200	217	export function readPlan(
201	218	db: Database,
202	219	repo: Repo,
203		options: { readonly spend: boolean; readonly now?: Date } = { spend: false },
	220	options: {
	221	readonly spend: boolean;
	222	readonly now?: Date;
	223	/** Told how long each stretch took (`graph`, `crew`, `week`, `spend`, `limbs`), for Server-Timing. */
	224	readonly stretch?: (name: string, ms: number) => void;
	225	} = { spend: false },
204	226	): Plan {
205	227	const now = options.now ?? new Date();
206	228	const since = pillarWindowStart(now);
	229	let mark = performance.now();
	230	const lap = (name: string): void => {
	231	const at = performance.now();
	232	options.stretch?.(name, at - mark);
	233	mark = at;
	234	};
207	235	const set: Pillars = pillarsOf(db, repo);
208		const graph: Graph = repoGraph(db, repo.id, { pillars: set });
	236	const graph: Graph = repoGraph(db, repo.id, { pillars: set, lean: true });
209	237	const byId = new Map(graph.issues.map((i) => [i.id, i]));
210	238	const byNumber = new Map(graph.issues.map((i) => [i.number, i]));
211	239
212	240	// What each issue waits for directly, furniture left out (`limb_order.ts`).
213	241	const tree = limbTree(graph, set);
214	242	const places = readPlaces(db, repo.id);
	243	lap("graph");
215	244	const { waitsFor, waitedBy, waits, childrenOf } = tree;
216	245	const memo = new Map<string, Set<string>>();
217	246	const walking = new Set<string>();
		@@ -262,6 +291,8 @@ export function readPlan(
262	291	});
263	292	}
264	293
	294	lap("crew");
	295
265	296	// The week's merged pulls, by the issues each closes.
266	297	const merged: MergedPull[] = db
267	298	.query<
		@@ -281,12 +312,14 @@ export function readPlan(
281	312	closes: closesFromBody(row.body),
282	313	}));
283	314
	315	lap("week");
	316
284	317	// What each issue cost: its own runs, and the runs on the pulls for it.
285	318	const cost = new Map<string, number>();
286	319	if (options.spend) {
287	320	const rows = db
288	321	.query<{ id: string; kind: string; body: string; usd: number }, [string]>(
289		`SELECT i.id, i.kind, i.body, SUM(r.cost_usd) AS usd
	322	`SELECT i.id, i.kind, CASE i.kind WHEN 'pull' THEN i.body ELSE '' END AS body, SUM(r.cost_usd) AS usd
290	323	FROM run r JOIN issue i ON i.id = r.issue_id
291	324	WHERE r.repo_id = ?1 AND r.kind = 'agent' AND r.cost_usd > 0
292	325	GROUP BY i.id`,
		@@ -305,16 +338,19 @@ export function readPlan(
305	338	}
306	339
307	340	// Every pull ever merged, by the issues it closes: what a limb landed.
308		// One query that parses each merged pull's body; owner reads only.
309		const everMerged: (readonly number[])[] = options.spend
310		? db
311		.query<{ body: string }, [string]>(
312		`SELECT i.body FROM pull p JOIN issue i ON i.id = p.issue_id
313		WHERE i.repo_id = ?1 AND p.merged_at IS NOT NULL`,
314		)
315		.all(repo.id)
316		.map((row) => closesFromBody(row.body))
317		: [];
	341	// One query that parses each merged pull's body; owner reads only, and
	342	// only one milestone's page asks, so the list of limbs never pays for it.
	343	let ever: (readonly number[])[] \| undefined;
	344	const everMerged = (): (readonly number[])[] =>
	345	(ever ??= db
	346	.query<{ body: string }, [string]>(
	347	`SELECT i.body FROM pull p JOIN issue i ON i.id = p.issue_id
	348	WHERE i.repo_id = ?1 AND p.merged_at IS NOT NULL`,
	349	)
	350	.all(repo.id)
	351	.map((row) => closesFromBody(row.body)));
	352
	353	lap("spend");
318	354
319	355	const stateOf = (issue: Issue): LeafState => {
320	356	if (issue.state === "closed") return counted(issue) ? "done" : "gone";
		@@ -391,9 +427,12 @@ export function readPlan(
391	427	tree.top.map((i) => i.id),
392	428	);
393	429
	430	const limbs = top.map(limbOf);
	431	lap("limbs");
	432
394	433	return {
395	434	read: { graph, tree, places },
396		limbs: top.map(limbOf),
	435	limbs,
397	436	limb(number: number): { own: Issue[]; shared: Issue[] } \| undefined {
398	437	const issue = byNumber.get(number);
399	438	if (issue === undefined) return undefined;
		@@ -554,7 +593,7 @@ export function readPlan(
554	593	? {
555	594	spend: {
556	595	usd: limb.weight.spentUsd ?? 0,
557		landed: everMerged.filter((closes) =>
	596	landed: everMerged().filter((closes) =>
558	597	closes.some((n) => numbers.has(n)),
559	598	).length,
560	599	costliest: under
		@@ -206,7 +206,11 @@ const planFor = (ctx: Ctx, repo: Repo, spend: boolean): Plan =>
206	206	spend,
207	207	now: new Date(),
208	208	},
209		() => readPlan(ctx.app.db, repo, { spend }),
	209	() =>
	210	readPlan(ctx.app.db, repo, {
	211	spend,
	212	stretch: (name, ms) => notePhase(ctx.request, `plan-${name}`, ms),
	213	}),
210	214	);
211	215
212	216	/** GET /:owner/:name/milestones — the plan, collapsed to its limbs. */
		@@ -0,0 +1,127 @@
	1	/** A forge-shaped repository: pulls, runs, sealed issues and nested limbs, not only a flat fan of leaves. */
	2	import { afterAll, beforeAll, expect, test } from "bun:test";
	3	import { mkdtempSync, rmSync } from "node:fs";
	4	import { tmpdir } from "node:os";
	5	import { join } from "node:path";
	6	import { createRepo } from "../src/actions/repos.ts";
	7	import { createApp, type App } from "../src/app.ts";
	8	import { openIssue } from "../src/actions/issues.ts";
	9	import { planStamp, readPlan } from "../src/core/milestones.ts";
	10	import { getRepo } from "../src/core/repos.ts";
	11	import { createUser, getUserByName } from "../src/core/users.ts";
	12	import { expectBudget } from "./helpers/budget.ts";
	13	import { quietLogs } from "./helpers/globals.ts";
	14	import { proveAll } from "./helpers/verified.ts";
	15
	16	quietLogs();
	17	const PILLARS = 10;
	18	let root = "";
	19	let app: App;
	20	let uid = "";
	21
	22	beforeAll(async () => {
	23	root = mkdtempSync(join(tmpdir(), "forge-milestones-shape-"));
	24	app = createApp({ dataDir: root, port: 0, forgeRepo: "alice/forge" });
	25	const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
	26	if (!u.ok) throw new Error(u.error);
	27	uid = u.value.id;
	28	proveAll(app);
	29	const made = await createRepo(app, { id: uid, username: "alice" } as never, { name: "forge" });
	30	if (!made.ok) throw new Error(made.error);
	31	const repoId = getRepo(app.db, "alice", "forge")!.id;
	32	const issue = app.db.prepare(
	33	`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at, updated_at, closed_at, sealed)
	34	VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?9, ?10, ?11)`,
	35	);
	36	const edge = app.db.prepare("INSERT INTO issue_dependency (issue_id, blocked_by_id) VALUES (?1, ?2)");
	37	const pull = app.db.prepare(
	38	`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at) VALUES (?1, 'main', ?2, 'a', 'b', ?3)`,
	39	);
	40	const run = app.db.prepare(
	41	`INSERT INTO run (id, repo_id, kind, issue_id, state, cost_usd, actor_id, created_at, started_at) VALUES (?1, ?2, 'agent', ?3, ?4, 0.5, ?5, ?6, ?6)`,
	42	);
	43	const body = "Why this matters. ".repeat(70);
	44	const at = new Date(Date.now() - 3_600_000).toISOString();
	45	const old = new Date(Date.now() - 40 * 86_400_000).toISOString();
	46	app.db.transaction(() => {
	47	let n = 0;
	48	for (let p = 0; p < PILLARS; p++)
	49	issue.run(`u-${p}`, repoId, ++n, "issue", `P${p + 1} Pillar ${p + 1}: ask`, "", uid, "open", at, null, 0);
	50	const leaves = 2300;
	51	for (let k = 0; k < leaves; k++) {
	52	const closed = k < 2000;
	53	issue.run(`c-${k}`, repoId, ++n, "issue", `Step ${k}`, body, uid, closed ? "closed" : "open", at, closed ? at : null, k % 8 === 3 ? 1 : 0);
	54	// a mid-level limb every 20 leaves, nested under its pillar; leaves chain behind each other
	55	if (k % 20 === 0) edge.run(`u-${Math.floor(k / 20) % PILLARS}`, `c-${k}`);
	56	else edge.run(`c-${k - (k % 20)}`, `c-${k}`);
	57	if (k % 20 > 1) edge.run(`c-${k - 1}`, `c-${k}`);
	58	}
	59	for (let k = 0; k < 3000; k++) {
	60	const merged = k < 2800;
	61	// 400 landed this week, the rest long ago
	62	const when = k < 400 ? at : old;
	63	issue.run(`p-${k}`, repoId, ++n, "pull", `Pull ${k}`, `${body}\nCloses #${PILLARS + 1 + (k % leaves)}`, uid, merged ? "closed" : "open", when, merged ? when : null, 0);
	64	pull.run(`p-${k}`, `b-${k}`, merged ? when : null);
	65	}
	66	for (let k = 0; k < 1500; k++) run.run(`r-${k}`, repoId, `c-${k % leaves}`, k < 30 ? "running" : "ok", uid, at);
	67	app.db.run("UPDATE repo SET issue_seq = ?1 WHERE id = ?2", [n, repoId]);
	68	})();
	69	});
	70	afterAll(() => {
	71	app.close();
	72	rmSync(root, { recursive: true, force: true });
	73	});
	74
	75	const weights = (spend: boolean) =>
	76	readPlan(app.db, getRepo(app.db, "alice", "forge")!, { spend }).limbs.map((l) => [
	77	l.name, l.weight.total, l.weight.done, l.weight.open, l.weight.merged, l.weight.building, l.weight.spentUsd,
	78	]);
	79
	80	/** What each limb weighs on the code before the stamp and the lean read: the counts must not move. */
	81	const BEFORE = [
	82	["P1 Pillar 1", 204, 170, 34, 34, 0, 68],
	83	["P2 Pillar 2", 216, 180, 36, 36, 0, 72],
	84	["P3 Pillar 3", 204, 170, 34, 34, 0, 68],
	85	["P4 Pillar 4", 216, 180, 36, 36, 0, 72],
	86	["P5 Pillar 5", 204, 170, 34, 34, 0, 68],
	87	["P6 Pillar 6", 198, 180, 18, 36, 0, 63],
	88	["P7 Pillar 7", 187, 170, 17, 34, 0, 59.5],
	89	["P8 Pillar 8", 198, 180, 18, 36, 0, 63],
	90	["P9 Pillar 9", 187, 170, 17, 34, 0, 59.5],
	91	["P10 Pillar 10", 198, 180, 18, 36, 0, 63],
	92	];
	93
	94	test("a plan of 2,000 closed and 300 open issues, 3,000 pulls and 1,500 runs reads in under 50 ms with the same limbs", async () => {
	95	const against = 1;
	96	const repo = getRepo(app.db, "alice", "forge")!;
	97	readPlan(app.db, repo, { spend: true });
	98	const times: number[] = [];
	99	for (let n = 0; n < 7; n++) {
	100	const start = performance.now();
	101	readPlan(app.db, repo, { spend: true });
	102	times.push(performance.now() - start);
	103	}
	104	expectBudget(Math.min(...times), { ms: 50, against });
	105	expect(weights(true)).toEqual(BEFORE);
	106	expect(weights(false).map((w) => w.slice(0, 6))).toEqual(BEFORE.map((w) => w.slice(0, 6)));
	107	});
	108
	109	test("a sealed issue is in no limb a viewer cannot see", () => {
	110	const repo = getRepo(app.db, "alice", "forge")!;
	111	const sealed = app.db.query<{ n: number }, [string]>("SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND sealed = 1").get(repo.id)!.n;
	112	expect(sealed).toBeGreaterThan(0);
	113	const plan = readPlan(app.db, repo, { spend: false });
	114	const seen = new Set(plan.read.graph.issues.map((i) => i.id));
	115	for (const row of app.db.query<{ id: string }, [string]>("SELECT id FROM issue WHERE repo_id = ?1 AND sealed = 1").all(repo.id))
	116	expect(seen.has(row.id)).toBe(false);
	117	});
	118
	119	test("the stamp is kept while nothing is written, and moves when an issue is", async () => {
	120	const repo = getRepo(app.db, "alice", "forge")!;
	121	const first = planStamp(app.db, repo);
	122	const start = performance.now();
	123	for (let n = 0; n < 20; n++) expect(planStamp(app.db, repo)).toBe(first);
	124	expectBudget((performance.now() - start) / 20, { ms: 2, against: 1 });
	125	expect(openIssue(app, getUserByName(app.db, "alice")!, repo, { title: "Fresh" }).ok).toBe(true);
	126	expect(planStamp(app.db, repo)).not.toBe(first);
	127	});
		@@ -128,3 +128,22 @@ test("a milestone page names session, repo, plan and render", async () => {
128	128	for (const name of ["session", "repo", "plan", "render"])
129	129	expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
130	130	});
	131
	132	test("a cold plan names its stretches, stays under 50 ms, and keeps each limb's counts", async () => {
		junipercrew 19d ago This test would pass on the code before the pull for the 50 ms budget and the counts, so it does not prove the issue. Build a fixture that is slow on the old code (trevato/forge-like shape), then assert the budget.
	133	const against = await baselineMs(servers.url);
	134	const stretches: string[] = [];
	135	const repo = getRepo(app.db, "alice", "forge")!;
	136	const start = performance.now();
	137	const plan = readPlan(app.db, repo, { spend: true, stretch: (name) => stretches.push(name) });
	138	expectBudget(performance.now() - start, { ms: 50, against });
	139	expect(stretches).toEqual(["graph", "crew", "week", "spend", "limbs"]);
	140	expect(plan.limbs).toHaveLength(PILLARS);
	141	for (const limb of plan.limbs)
	142	expect(limb.weight).toMatchObject({ total: (CLOSED + OPEN) / PILLARS, done: CLOSED / PILLARS, open: OPEN / PILLARS });
	143
	144	clearMapCache();
	145	const res = await fetch(`${servers.url}/alice/forge/milestones`, { headers: { cookie } });
	146	await res.text();
	147	const timing = res.headers.get("server-timing") ?? "";
	148	for (const name of ["plan-graph", "plan-limbs"]) expect(timing).toContain(`${name};dur=`);
	149	});