917c9d8 · 3dc324c · bc6bb89 · 44f94c4 · 7b7a9d5 · f8d4648
@@ -41,5 +41,9 @@
typesafe-api-key
# when the default temp directory is read-only (#2706)
/.bun-tmp/
# a suite whose data would crowd a check cell's 256 MiB /tmp keeps it
# here, on the checkout's disk (test/helpers/scratch.ts)
/.test-scratch/
# a sandbox's bun/bunx scratch directory; never source
/.tmpx/
@@ -0,0 +1,21 @@
// The flow readers' indexes (#3693). `/boards`, `/system` and home read the
// pipeline and its throughput on every view; without these, each merged
// pull's "first production deploy since" walked every run of its repository,
// and each stage's outflow scanned the whole run table, and so did
// finding the pulls a bring-up-to-date run is working on; a ready ask's
// "no open pull from its crew branch" scanned every pull. Home's figures
// summed today's crew spend over every run and counted each day's visitors
// from the page views themselves. Additive.
export default `
  CREATE INDEX run_deploy_since ON run(repo_id, env, created_at) WHERE kind = 'deploy';
  CREATE INDEX run_kind_started ON run(kind, started_at) WHERE started_at IS NOT NULL;
  CREATE INDEX run_kind_finished ON run(kind, finished_at) WHERE finished_at IS NOT NULL;
  CREATE INDEX run_serving ON run(serving_at) WHERE serving_at IS NOT NULL;
  CREATE INDEX run_conflict ON run(issue_id, created_at) WHERE kind = 'agent' AND reason = 'conflict';
  CREATE INDEX run_agent_created ON run(created_at) WHERE kind = 'agent';
  CREATE INDEX pull_head_branch ON pull(head_branch);
  CREATE INDEX pull_merged ON pull(merged_at) WHERE merged_at IS NOT NULL;
  CREATE INDEX page_vital_day_visitor ON page_vital(day, visitor);
  CREATE INDEX page_vital_day_vitals ON page_vital(day, load, inp);
  CREATE INDEX session_seen ON session(last_seen_at, user_id);
  `;
@@ -99,6 +99,8 @@
function waitingTally(db: Database, sql: string, ...args: string[]): Tally {
  return { waiting: r?.n ?? 0, active: 0, oldest: r?.oldest ?? null };
}
// CROSS JOIN keeps SQLite's order: the ask's runs, their branch's pulls by
// `pull_head_branch`, then each pull's issue, never every open issue (#3693).
const READY = `
  SELECT i.created_at AS since
    FROM issue i JOIN repo r ON r.id = i.repo_id
@@ -108,8 +110,8 @@
const READY = `
                      WHERE d.issue_id = i.id AND b.state = 'open')
     AND NOT EXISTS (SELECT 1 FROM run x WHERE x.issue_id = i.id AND x.kind = 'agent'
                      AND x.state IN ('queued','running'))
     AND NOT EXISTS (SELECT 1 FROM run x JOIN pull p ON p.head_branch = x.branch
                       JOIN issue pi ON pi.id = p.issue_id AND pi.repo_id = x.repo_id
     AND NOT EXISTS (SELECT 1 FROM run x CROSS JOIN pull p ON p.head_branch = x.branch
                       CROSS JOIN issue pi ON pi.id = p.issue_id AND pi.repo_id = x.repo_id
                      WHERE x.issue_id = i.id AND x.kind = 'agent' AND x.branch != ''
                        AND pi.state = 'open')`;
@@ -29,20 +29,30 @@
export interface Reach {
export const NO_REACH: Reach = { usersToday: 0, usersTotal: 0, visitorsToday: 0, visitors7d: 0, visitorDays: new Array<number>(DAYS).fill(0), loadP75: null, inpP75: null };
/** The 75th percentile, nearest rank, of a column over the days since `fromDay`. */
function p75(db: Database, column: "load" | "inp", fromDay: string): number | null {
  const n = db
    .query<{ n: number }, [string]>(`SELECT COUNT(${column}) AS n FROM page_vital WHERE day >= ?1`)
    .get(fromDay)?.n ?? 0;
  if (n === 0) return null;
  const rank = Math.ceil(n * 0.75);
  return (
    db
      .query<{ v: number }, [string, number]>(
        `SELECT ${column} AS v FROM page_vital WHERE day >= ?1 AND ${column} IS NOT NULL ORDER BY ${column} LIMIT 1 OFFSET ?2`,
      )
      .get(fromDay, rank - 1)?.v ?? null
  );
/** The 75th percentile, nearest rank: the value at rank ⌈0.75·n⌉ of `values` sorted. */
export function p75(values: number[]): number | null {
  if (values.length === 0) return null;
  values.sort((a, b) => a - b);
  return values[Math.ceil(values.length * 0.75) - 1]!;
}
/**
 * Page load and INP over the days since `fromDay`, read in one pass (#3693):
 * two counts and two sorted OFFSET reads cost home 70 ms on a month of views.
 */
function vitals(db: Database, fromDay: string): { load: number[]; inp: number[] } {
  const load: number[] = [];
  const inp: number[] = [];
  // Rows as arrays: a week of views is tens of thousands, and objects cost.
  for (const [l, i] of db
    .query<[number | null, number | null], [string]>(
      "SELECT load, inp FROM page_vital WHERE day >= ?1 AND (load IS NOT NULL OR inp IS NOT NULL)",
    )
    .values(fromDay) as [number | null, number | null][]) {
    if (l !== null) load.push(l);
    if (i !== null) inp.push(i);
  }
  return { load, inp };
}
export function readReach(db: Database, now: Date): Reach {
@@ -65,13 +75,14 @@
export function readReach(db: Database, now: Date): Reach {
    perDay.get(new Date(now.getTime() - (DAYS - 1 - i) * DAY_MS).toISOString().slice(0, 10)) ?? 0,
  );
  const days = month.filter((d) => d.day >= weekFrom);
  const week = vitals(db, weekFrom);
  return {
    usersToday: users.today,
    usersTotal: users.total,
    visitorsToday: days.find((d) => d.day === today)?.n ?? 0,
    visitorDays,
    visitors7d: days.reduce((a, d) => a + d.n, 0),
    loadP75: p75(db, "load", weekFrom),
    inpP75: p75(db, "inp", weekFrom),
    loadP75: p75(week.load),
    inpP75: p75(week.inp),
  };
}
@@ -80,7 +80,8 @@
const denied = (ctx: Ctx) => {
// GET /boards
export async function boardsList(ctx: Ctx): Promise<Response> {
  const listed = await listAllBoards(ctx.app, ctx.auth.user);
  notePhase(ctx.request, "session", takeSessionTime(ctx.request));
  const listed = await timePhaseAsync(ctx.request, "boards", () => listAllBoards(ctx.app, ctx.auth.user));
  if (!listed.ok) return notFoundPage(ctx);
  const make = html`<form method="post" action="/boards/new" class="bform">
    <label class="qf"><span>New board</span><input class="input" name="title" required maxlength="${String(MAX_TITLE)}" placeholder="Title" /></label>
@@ -100,13 +101,17 @@
export async function boardsList(ctx: Ctx): Promise<Response> {
          </ul>`,
          { pad: false },
        );
  return page({
  const flow = timePhase(ctx.request, "flow", () => flowPanel(ctx.app, { kind: "owner" }, ctx.url, "/boards"));
      
        junipercrew
        24d ago
        
      
      Nit: render times only the page() call. If the shell cost matters in production, move flow inside the same span or name it shell.
    
  const rendering = performance.now();
  const response = page({
    ctx,
    title: "Boards",
    body: html`<div class="page-head"><h1>Boards</h1>
        ${btn("Explore the data", { href: "/boards/data", variant: "ghost" })}</div>
      ${card(make)} ${body} ${flowPanel(ctx.app, { kind: "owner" }, ctx.url, "/boards")}`,
      ${card(make)} ${body} ${flow}`,
  });
  notePhase(ctx.request, "render", performance.now() - rendering);
  return response;
}
function panelForm(slug: string, op: string, p: Panel | undefined, index: number, fields: Html): Html {
@@ -129,7 +129,7 @@
export interface Layout {
}
/** One hop of an edge between adjacent levels: the unit `order` sees. */
interface Seg {
export interface Seg {
  readonly from: string;
  readonly to: string;
}
@@ -304,15 +304,55 @@
function cross(
  return top * bottom < 0;
}
function countCrossings(
/**
 * The same count as comparing every pair of hops with `cross`, without
 * doing so (#3693: 2,266 edges drew 30 s of layout). A hop down one level crosses
 * another in the same gap exactly when their ends are in opposite order,
 * so each gap is an inversion count over a Fenwick tree, O(k log k). Hops
 * that point up or skip levels (only a cycle draws them) keep the pairwise
 * test against every hop; hops within one level never cross anything.
 */
export function countCrossings(
  segs: readonly Seg[],
  level: Map<string, number>,
  pos: Map<string, number>,
): number {
  const gaps = new Map<number, [number, number][]>();
  const odd: Seg[] = [];
  const spanning: Seg[] = [];
  for (const s of segs) {
    const a = level.get(s.from)!;
    const b = level.get(s.to)!;
    if (a === b) continue;
    spanning.push(s);
    if (b === a + 1) push(gaps, a, [pos.get(s.from)!, pos.get(s.to)!]);
    else odd.push(s);
  }
  let n = 0;
  for (let i = 0; i < segs.length; i++)
    for (let j = i + 1; j < segs.length; j++)
      if (cross(segs[i]!, segs[j]!, level, pos)) n++;
  for (const hops of gaps.values()) {
    // Top end ascending, bottom ascending on ties: an equal top or an equal
    // bottom is a shared card, which meets rather than crosses.
    hops.sort((x, y) => x[0] - y[0] || x[1] - y[1]);
    const size = hops.reduce((most, h) => Math.max(most, h[1]), 0) + 2;
    const tree = new Int32Array(size + 1);
      
        junipercrew
        24d ago
        
      
      Math.max(...hops.map(...)) spreads one argument per hop. A very wide gap could hit the argument limit. A plain loop or reduce costs nothing and removes that risk.
    
    let seen = 0;
    for (const [, bottom] of hops) {
      // Hops already seen with a bottom strictly right of this one's.
      let atMost = 0;
      for (let i = bottom + 1; i > 0; i -= i & -i) atMost += tree[i]!;
      n += seen - atMost;
      for (let i = bottom + 1; i <= size; i += i & -i) tree[i]!++;
      seen++;
    }
  }
  // An odd hop against every hop, each pair of odd hops once.
  const oddAt = new Map(odd.map((s, i) => [s, i]));
  odd.forEach((s, i) => {
    for (const t of spanning) {
      const j = oddAt.get(t);
      if ((j === undefined || j > i) && cross(s, t, level, pos)) n++;
    }
  });
  return n;
}
@@ -383,8 +423,8 @@
function order(
  const touching = new Map<string, Seg[]>();
  for (const s of segs) {
    touching.set(s.to, [...(touching.get(s.to) ?? []), s]);
    touching.set(s.from, [...(touching.get(s.from) ?? []), s]);
    push(touching, s.to, s);
    push(touching, s.from, s);
  }
  const between = (a: string, b: string): number => {
    let n = 0;
@@ -0,0 +1,117 @@
/**
 * A forge sized like production's for the flow readers (#3693): one app
 * repository with a month of runs (checks, crew, reviews, deploys) and two
 * weeks of merged pulls, the open asks the crew is on and the edges
 * between them, and a month of page views, written straight into the tables so a suite can build it
 * in seconds. Deterministic: the same call writes the same rows.
 */
import type { App } from "../../src/app.ts";
import { createRepo } from "../../src/actions/repos.ts";
import type { User } from "../../src/core/users.ts";
export interface FlowFixture {
  readonly runs: number;
  readonly pulls: number;
  readonly open: number;
  readonly edges: number;
  /** Page views over the month. */
  readonly vitals: number;
}
const DAY = 86_400_000;
export async function seedFlowFixture(
  app: App,
  owner: User,
  size: FlowFixture = {
    runs: 100_000,
    pulls: 1_500,
    open: 400,
    edges: 2_266,
    vitals: 30_000,
  },
  now = Date.now(),
): Promise<FlowFixture> {
  const made = await createRepo(app, owner, { name: "busy", visibility: "public" });
  if (!made.ok) throw new Error(`no repo: ${made.error}`);
  const repo = made.value;
  const db = app.db;
  db.run("UPDATE repo SET autonomy = 'auto', autonomy_slots = 2 WHERE id = ?1", [repo.id]);
  db.run("INSERT INTO app (id, repo_id, slug, access) VALUES ('app-busy', ?1, 'busy', 'public')", [repo.id]);
  let s = 3693;
  const r = (): number => {
    s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
    return s / 2 ** 32;
  };
  const iso = (ms: number): string => new Date(ms).toISOString();
  db.transaction(() => {
    const issue = db.prepare(
      `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at)
       VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)`,
    );
    const pull = db.prepare(
      `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at)
       VALUES (?1, 'main', ?2, '', '', ?3)`,
    );
    for (let n = 1; n <= size.pulls; n++) {
      const asked = now - r() * 21 * DAY;
      issue.run(`fi${n}`, repo.id, 2 * n - 1, "issue", `Ask ${n}`, "", owner.id, "closed", iso(asked));
      const opened = asked + r() * DAY;
      const merged = Math.min(now - 60_000, opened + r() * 3 * DAY);
      issue.run(`fp${n}`, repo.id, 2 * n, "pull", `Pull ${n}`, `Closes #${2 * n - 1}`, owner.id, "closed", iso(opened));
      pull.run(`fp${n}`, `crew/${n}`, iso(merged));
    }
    // Open asks, each with a finished crew run on a branch of its own.
    for (let n = 1; n <= size.open; n++)
      issue.run(`fo${n}`, repo.id, 2 * size.pulls + n, "issue", `Open ${n}`, "", owner.id, "open", iso(now - r() * 30 * DAY));
    const run = db.prepare(
      `INSERT INTO run (id, repo_id, kind, issue_id, env, branch, state, reason, created_at, started_at, finished_at, serving_at)
       VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)`,
    );
    for (let n = 0; n < size.runs; n++) {
      const at = now - r() * 30 * DAY;
      const pick = r();
      const kind = pick < 0.7 ? "check" : pick < 0.95 ? "agent" : "deploy";
      const pullN = 1 + Math.floor(r() * size.pulls);
      const open = kind === "agent" && size.open > 0 && r() < 0.05 ? 1 + Math.floor(r() * size.open) : 0;
      const env = kind === "deploy" ? (r() < 0.5 ? "production" : "preview") : "";
      run.run(
        `fr${String(n).padStart(7, "0")}`,
        repo.id,
        kind,
        kind === "deploy" ? null : open > 0 ? `fo${open}` : `fp${pullN}`,
        env,
        kind === "deploy" ? "main" : open > 0 ? `crew/open-${open}` : `crew/${pullN}`,
        r() < 0.9 ? "ok" : "failed",
        kind === "agent" && r() < 0.4 ? "review" : "",
        iso(at),
        iso(at + 1_000),
        iso(at + 60_000),
        env === "production" ? iso(at + 90_000) : null,
      );
    }
    // Production's map: 2,266 edges, each ask waiting on an older one.
    const asks = [
      ...Array.from({ length: size.pulls }, (_, n) => `fi${n + 1}`),
      ...Array.from({ length: size.open }, (_, n) => `fo${n + 1}`),
    ];
    const edge = db.prepare("INSERT OR IGNORE INTO issue_dependency (issue_id, blocked_by_id) VALUES (?1, ?2)");
    for (let n = 0; n < size.edges; n++) {
      const later = 1 + Math.floor(r() * (asks.length - 1));
      edge.run(asks[later]!, asks[Math.floor(r() * later)]!);
    }
    // A month of page views, as the beacon writes them: 1,000 a day.
    const vital = db.prepare(
      "INSERT INTO page_vital (day, at, route, visitor, ttfb, load, lcp, inp, cls) VALUES (?1, ?2, '/', ?3, 20, ?4, ?4, ?5, 0)",
    );
    for (let n = 0; n < size.vitals; n++) {
      const at = now - r() * 30 * DAY;
      vital.run(iso(at).slice(0, 10), at, `v${Math.floor(r() * 400)}`, 100 + r() * 900, r() < 0.5 ? null : 10 + r() * 200);
    }
  })();
  // The seed went through the WAL: fold it back and give its pages up, so
  // the suite holds one copy of the data on disk and not two. The gate runs
  // many suites side by side on one disk (#3693).
  db.run("PRAGMA wal_checkpoint(TRUNCATE)");
  return { ...size };
}
@@ -0,0 +1,18 @@
/**
 * A temporary directory on the checkout's own disk, for a suite whose data
 * would crowd `/tmp`. A check cell's `/tmp` is a 256 MiB tmpfs
 * (`TMP_MIB` in src/apps/contain.ts) shared by every file the gate runs at
 * once and by their coverage; a suite that writes tens of MB there fills it
 * as its neighbours start, and they fail in setup with no test named (#3693).
 * The checkout is on the cell's disk. `/.test-scratch/` is ignored by git.
 */
import { mkdirSync, mkdtempSync } from "node:fs";
import { join } from "node:path";
export const SCRATCH_ROOT = join(import.meta.dir, "..", "..", ".test-scratch");
/** A fresh directory under {@link SCRATCH_ROOT}; the suite removes it. */
export function scratchDir(prefix: string): string {
  mkdirSync(SCRATCH_ROOT, { recursive: true });
  return mkdtempSync(join(SCRATCH_ROOT, prefix));
}
@@ -0,0 +1,168 @@
/**
 * The map's layout at the forge's own size (#3693): production's graph,
 * 1,314 issues and 2,266 edges, took 34 s of `layout` on a cold `/map`
 * because every ordering pass compared every pair of hops. The count is now
 * an inversion count per gap between levels; these tests hold it to the
 * pairwise definition, and hold the layout to a budget measured against a
 * same-run baseline rather than the wall clock (#3069).
 */
import { describe, expect, test } from "bun:test";
import type { Graph } from "../src/core/dependencies.ts";
import type { Issue } from "../src/core/issues.ts";
import { countCrossings, layout, type Seg } from "../src/web/pages/map.ts";
/**
 * The definition `countCrossings` must agree with, written out here so the
 * oracle shares no code with what it checks: two hops cross when they share
 * no end, their spans of levels overlap, and their ends are in opposite order.
 */
function pairwiseCrossings(
  segs: readonly Seg[],
  level: Map<string, number>,
  pos: Map<string, number>,
): number {
  let n = 0;
  for (let i = 0; i < segs.length; i++)
    for (let j = i + 1; j < segs.length; j++) {
      const s = segs[i]!;
      const t = segs[j]!;
      if (s.from === t.from || s.to === t.to || s.from === t.to || s.to === t.from) continue;
      const [sa, sb, ta, tb] = [s.from, s.to, t.from, t.to].map((id) => level.get(id)!) as [number, number, number, number];
      if (Math.max(Math.min(sa, sb), Math.min(ta, tb)) >= Math.min(Math.max(sa, sb), Math.max(ta, tb))) continue;
      const top = pos.get(s.from)! - pos.get(t.from)!;
      const bottom = pos.get(s.to)! - pos.get(t.to)!;
      if (top * bottom < 0) n++;
    }
  return n;
}
/** A seeded generator, so a failure reproduces. */
function rng(seed: number): () => number {
  let s = seed >>> 0;
  return () => {
    s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
    return s / 2 ** 32;
  };
}
/** Production's shape: about 1.7 edges an issue, each on an older issue. */
function productionGraph(issues = 1314, edges = 2266): Graph {
  const r = rng(3693);
  const seen = new Set<string>();
  const list: { issueId: string; blockedById: string }[] = [];
  while (list.length < edges) {
    const issue = 2 + Math.floor(r() * (issues - 1));
    const blockedBy = 1 + Math.floor(r() * (issue - 1));
    const key = `${issue}>${blockedBy}`;
    if (seen.has(key)) continue;
    seen.add(key);
    list.push({ issueId: `i${issue}`, blockedById: `i${blockedBy}` });
  }
  return {
    issues: Array.from(
      { length: issues },
      (_, n) => ({ id: `i${n + 1}`, number: n + 1 }) as unknown as Issue,
    ),
    edges: list,
    held: [],
    deferred: new Map(),
    unadopted: new Set(),
    structural: new Set(),
  } as unknown as Graph;
}
function median(run: () => void, times = 3): number {
  const spent: number[] = [];
  for (let n = 0; n < times; n++) {
    const started = performance.now();
    run();
    spent.push(performance.now() - started);
  }
  return spent.sort((a, b) => a - b)[Math.floor(times / 2)]!;
}
describe("crossings, counted per gap", () => {
  test("agree with every pair compared, cycles and long hops included", () => {
    const r = rng(7);
    for (let round = 0; round < 200; round++) {
      const levels = 1 + Math.floor(r() * 6);
      const level = new Map<string, number>();
      const pos = new Map<string, number>();
      const ids: string[] = [];
      for (let l = 0; l < levels; l++) {
        const wide = 1 + Math.floor(r() * 7);
        for (let p = 0; p < wide; p++) {
          const id = `n${l}.${p}`;
          level.set(id, l);
          pos.set(id, p);
          ids.push(id);
        }
      }
      const segs: Seg[] = [];
      const hops = Math.floor(r() * 30);
      for (let h = 0; h < hops; h++) {
        const from = ids[Math.floor(r() * ids.length)]!;
        // Mostly one level down, as `layout` splits them; sometimes any.
        const below = ids.filter((id) => level.get(id) === level.get(from)! + 1);
        const to =
          below.length > 0 && r() < 0.7
            ? below[Math.floor(r() * below.length)]!
            : ids[Math.floor(r() * ids.length)]!;
        segs.push({ from, to });
      }
      expect(countCrossings(segs, level, pos)).toBe(
        pairwiseCrossings(segs, level, pos),
      );
    }
  });
  test("a known braid: two hops swapping ends cross once, sharing an end never", () => {
    const level = new Map([
      ["a", 0],
      ["b", 0],
      ["c", 1],
      ["d", 1],
    ]);
    const pos = new Map([
      ["a", 0],
      ["b", 1],
      ["c", 0],
      ["d", 1],
    ]);
    const crossed = [
      { from: "a", to: "d" },
      { from: "b", to: "c" },
    ];
    const shared = [
      { from: "a", to: "c" },
      { from: "a", to: "d" },
      { from: "b", to: "d" },
    ];
    expect(countCrossings(crossed, level, pos)).toBe(1);
    expect(countCrossings(shared, level, pos)).toBe(0);
  });
});
describe("the layout at production's size", () => {
  const graph = productionGraph();
  test("is drawn deep and wide, like production's", () => {
    const placed = layout(graph);
    expect(placed.levels).toBeGreaterThan(10);
    expect(placed.routes.length).toBeGreaterThan(500);
  });
  test("costs a few pairwise counts of its own card-to-card edges at most", () => {
    // Before #3693 the layout ran that pairwise count at least five times,
    // over more hops than this (every waypoint is one): about 50 baselines
    // on this graph. Now the whole layout is under one; three is the slack
    // a loaded machine gets.
    const placed = layout(graph);
    const level = new Map(placed.nodes.map((n) => [n.issue.id, n.level]));
    const pos = new Map(placed.nodes.map((n) => [n.issue.id, n.slot]));
    const segs = graph.edges.map((e) => ({ from: e.blockedById, to: e.issueId }));
    const baseline = median(() => pairwiseCrossings(segs, level, pos));
    const spent = median(() => layout(graph));
    expect(spent).toBeLessThan(3 * baseline);
  });
});
@@ -0,0 +1,114 @@
/**
 * `/boards`, `/map` and home on a forge sized like production's (#3693):
 * a month of 100,000 runs, 1,500 merged pulls, 400 open asks and 2,266
 * edges between them. The owner saw /boards take 8–10 s, a cold /map 34.7 s
 * and home 9.2 s. Each budget is a multiple of one full pass over the run
 * table, timed in the same run, so a loaded machine slows both sides and
 * the wall clock is never the judge (#3069). Before the fix each page cost
 * hundreds of those passes.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { createApp, type App } from "../src/app.ts";
import { createSession } from "../src/core/sessions.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { clearMapCache } from "../src/web/map_cache.ts";
import { p75 } from "../src/flow/reach.ts";
import { seedAdmin } from "./helpers/admin.ts";
import { seedFlowFixture } from "./helpers/flow-fixture.ts";
import { SCRATCH_ROOT, scratchDir } from "./helpers/scratch.ts";
let dataDir = "";
let app: App, servers: Servers, cookie = "";
beforeAll(async () => {
  // ~49 MB of database: on the checkout's disk, not the cell's shared /tmp.
  dataDir = scratchDir("forge-obs-budget-");
  app = createApp({ dataDir, port: 0 });
  servers = await serve(app);
  const made = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "correct-horse-battery" });
  if (!made.ok) throw new Error(made.error);
  await seedFlowFixture(app, made.value);
  cookie = `forge_session=${createSession(app.db, made.value.id).token}`;
}, 120_000);
afterAll(async () => {
  // A setup that failed part way leaves these unset: say only what it said.
  servers?.stop();
  app?.close();
  if (dataDir !== "") await rm(dataDir, { recursive: true, force: true });
});
/** One full pass over every run: the unit the budgets are counted in. */
function scan(): number {
  const started = performance.now();
  app.db.query("SELECT COUNT(*), SUM(length(branch)), SUM(length(created_at)) FROM run").get();
  return performance.now() - started;
}
function baseline(): number {
  const spent = [scan(), scan(), scan()].sort((a, b) => a - b);
  return spent[1]!;
}
async function get(path: string): Promise<{ ms: number; status: number; timing: string }> {
  const started = performance.now();
  const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
  await res.arrayBuffer();
  return { ms: performance.now() - started, status: res.status, timing: res.headers.get("server-timing") ?? "" };
}
const names = (timing: string): string[] => timing.split(", ").map((p) => p.split(";")[0]!);
describe("on production's size", () => {
  test("the fixture lives on the checkout's disk, not in the shared /tmp", () => {
    expect(dataDir.startsWith(SCRATCH_ROOT)).toBe(true);
    expect(dataDir.startsWith(tmpdir())).toBe(false);
  });
  test("the fixture is production's size", () => {
    const n = (sql: string) => (app.db.query(sql).get() as { n: number }).n;
    expect(n("SELECT COUNT(*) AS n FROM run")).toBe(100_000);
    expect(n("SELECT COUNT(*) AS n FROM issue_dependency")).toBeGreaterThan(2_000);
      
        junipercrew
        24d ago
        
      
      A 30x multiple of one scan is a loose bound on a loaded machine; fine now, but watch it for flakiness.
    
  });
  test("/boards answers within a few passes over the runs, and names its phases", async () => {
    await get("/boards");
    const unit = baseline();
    const res = await get("/boards");
    expect(res.status).toBe(200);
    for (const n of ["session", "boards", "flow", "render"]) expect(names(res.timing)).toContain(n);
    expect(res.ms).toBeLessThan(30 * unit);
  });
  test("a cold /map answers within a few passes over the runs", async () => {
    await get("/map");
    clearMapCache();
    const unit = baseline();
    const res = await get("/map");
    expect(res.status).toBe(200);
    expect(names(res.timing)).toContain("layout");
    expect(res.ms).toBeLessThan(60 * unit);
  });
  test("a cold home answers within a few passes over the runs, kpis read afresh", async () => {
    await get("/");
    // A write moves the clock home's caches are keyed by: this view is cold.
    app.db.run("UPDATE repo SET description = description || ''");
    const unit = baseline();
    const res = await get("/");
    expect(res.status).toBe(200);
    expect(names(res.timing)).toContain("kpis");
    expect(res.timing).not.toMatch(/kpis;dur=[\d.]+;desc="hit"/);
    expect(res.ms).toBeLessThan(30 * unit);
  });
});
test("home's p75 is nearest rank, read in one pass: the value at ⌈0.75·n⌉", () => {
  expect(p75([])).toBeNull();
  expect(p75([7])).toBe(7);
  // n = 4: rank 3. n = 5: rank ⌈3.75⌉ = 4. Order of arrival does not matter.
  expect(p75([40, 10, 30, 20])).toBe(30);
  expect(p75([50, 10, 40, 20, 30])).toBe(40);
});
@@ -40,7 +40,6 @@
const UNNAMED: Array<[path: string, issue: string]> = [
  ["/alice/forge/toolbar", "#3390"],
  ["/apps/open", "#3390"],
  ["/board", "#3378"],
  ["/boards", "#3390"],
  ["/claim", "#3390"],
  ["/front", "#3311"],
  ["/live/apps", "#3390"],
		@@ -41,5 +41,9 @@ typesafe-api-key
41	41	# when the default temp directory is read-only (#2706)
42	42	/.bun-tmp/
43	43
	44	# a suite whose data would crowd a check cell's 256 MiB /tmp keeps it
	45	# here, on the checkout's disk (test/helpers/scratch.ts)
	46	/.test-scratch/
	47
44	48	# a sandbox's bun/bunx scratch directory; never source
45	49	/.tmpx/
		@@ -0,0 +1,21 @@
	1	// The flow readers' indexes (#3693). `/boards`, `/system` and home read the
	2	// pipeline and its throughput on every view; without these, each merged
	3	// pull's "first production deploy since" walked every run of its repository,
	4	// and each stage's outflow scanned the whole run table, and so did
	5	// finding the pulls a bring-up-to-date run is working on; a ready ask's
	6	// "no open pull from its crew branch" scanned every pull. Home's figures
	7	// summed today's crew spend over every run and counted each day's visitors
	8	// from the page views themselves. Additive.
	9	export default `
	10	CREATE INDEX run_deploy_since ON run(repo_id, env, created_at) WHERE kind = 'deploy';
	11	CREATE INDEX run_kind_started ON run(kind, started_at) WHERE started_at IS NOT NULL;
	12	CREATE INDEX run_kind_finished ON run(kind, finished_at) WHERE finished_at IS NOT NULL;
	13	CREATE INDEX run_serving ON run(serving_at) WHERE serving_at IS NOT NULL;
	14	CREATE INDEX run_conflict ON run(issue_id, created_at) WHERE kind = 'agent' AND reason = 'conflict';
	15	CREATE INDEX run_agent_created ON run(created_at) WHERE kind = 'agent';
	16	CREATE INDEX pull_head_branch ON pull(head_branch);
	17	CREATE INDEX pull_merged ON pull(merged_at) WHERE merged_at IS NOT NULL;
	18	CREATE INDEX page_vital_day_visitor ON page_vital(day, visitor);
	19	CREATE INDEX page_vital_day_vitals ON page_vital(day, load, inp);
	20	CREATE INDEX session_seen ON session(last_seen_at, user_id);
	21	`;
		@@ -99,6 +99,8 @@ function waitingTally(db: Database, sql: string, ...args: string[]): Tally {
99	99	return { waiting: r?.n ?? 0, active: 0, oldest: r?.oldest ?? null };
100	100	}
101	101
	102	// CROSS JOIN keeps SQLite's order: the ask's runs, their branch's pulls by
	103	// `pull_head_branch`, then each pull's issue, never every open issue (#3693).
102	104	const READY = `
103	105	SELECT i.created_at AS since
104	106	FROM issue i JOIN repo r ON r.id = i.repo_id
		@@ -108,8 +110,8 @@ const READY = `
108	110	WHERE d.issue_id = i.id AND b.state = 'open')
109	111	AND NOT EXISTS (SELECT 1 FROM run x WHERE x.issue_id = i.id AND x.kind = 'agent'
110	112	AND x.state IN ('queued','running'))
111		AND NOT EXISTS (SELECT 1 FROM run x JOIN pull p ON p.head_branch = x.branch
112		JOIN issue pi ON pi.id = p.issue_id AND pi.repo_id = x.repo_id
	113	AND NOT EXISTS (SELECT 1 FROM run x CROSS JOIN pull p ON p.head_branch = x.branch
	114	CROSS JOIN issue pi ON pi.id = p.issue_id AND pi.repo_id = x.repo_id
113	115	WHERE x.issue_id = i.id AND x.kind = 'agent' AND x.branch != ''
114	116	AND pi.state = 'open')`;
115	117
		@@ -29,20 +29,30 @@ export interface Reach {
29	29
30	30	export const NO_REACH: Reach = { usersToday: 0, usersTotal: 0, visitorsToday: 0, visitors7d: 0, visitorDays: new Array<number>(DAYS).fill(0), loadP75: null, inpP75: null };
31	31
32		/** The 75th percentile, nearest rank, of a column over the days since `fromDay`. */
33		function p75(db: Database, column: "load" \| "inp", fromDay: string): number \| null {
34		const n = db
35		.query<{ n: number }, [string]>(`SELECT COUNT(${column}) AS n FROM page_vital WHERE day >= ?1`)
36		.get(fromDay)?.n ?? 0;
37		if (n === 0) return null;
38		const rank = Math.ceil(n * 0.75);
39		return (
40		db
41		.query<{ v: number }, [string, number]>(
42		`SELECT ${column} AS v FROM page_vital WHERE day >= ?1 AND ${column} IS NOT NULL ORDER BY ${column} LIMIT 1 OFFSET ?2`,
43		)
44		.get(fromDay, rank - 1)?.v ?? null
45		);
	32	/** The 75th percentile, nearest rank: the value at rank ⌈0.75·n⌉ of `values` sorted. */
	33	export function p75(values: number[]): number \| null {
	34	if (values.length === 0) return null;
	35	values.sort((a, b) => a - b);
	36	return values[Math.ceil(values.length * 0.75) - 1]!;
	37	}
	38
	39	/**
	40	* Page load and INP over the days since `fromDay`, read in one pass (#3693):
	41	* two counts and two sorted OFFSET reads cost home 70 ms on a month of views.
	42	*/
	43	function vitals(db: Database, fromDay: string): { load: number[]; inp: number[] } {
	44	const load: number[] = [];
	45	const inp: number[] = [];
	46	// Rows as arrays: a week of views is tens of thousands, and objects cost.
	47	for (const [l, i] of db
	48	.query<[number \| null, number \| null], [string]>(
	49	"SELECT load, inp FROM page_vital WHERE day >= ?1 AND (load IS NOT NULL OR inp IS NOT NULL)",
	50	)
	51	.values(fromDay) as [number \| null, number \| null][]) {
	52	if (l !== null) load.push(l);
	53	if (i !== null) inp.push(i);
	54	}
	55	return { load, inp };
46	56	}
47	57
48	58	export function readReach(db: Database, now: Date): Reach {
		@@ -65,13 +75,14 @@ export function readReach(db: Database, now: Date): Reach {
65	75	perDay.get(new Date(now.getTime() - (DAYS - 1 - i) * DAY_MS).toISOString().slice(0, 10)) ?? 0,
66	76	);
67	77	const days = month.filter((d) => d.day >= weekFrom);
	78	const week = vitals(db, weekFrom);
68	79	return {
69	80	usersToday: users.today,
70	81	usersTotal: users.total,
71	82	visitorsToday: days.find((d) => d.day === today)?.n ?? 0,
72	83	visitorDays,
73	84	visitors7d: days.reduce((a, d) => a + d.n, 0),
74		loadP75: p75(db, "load", weekFrom),
75		inpP75: p75(db, "inp", weekFrom),
	85	loadP75: p75(week.load),
	86	inpP75: p75(week.inp),
76	87	};
77	88	}
		@@ -80,7 +80,8 @@ const denied = (ctx: Ctx) => {
80	80
81	81	// GET /boards
82	82	export async function boardsList(ctx: Ctx): Promise<Response> {
83		const listed = await listAllBoards(ctx.app, ctx.auth.user);
	83	notePhase(ctx.request, "session", takeSessionTime(ctx.request));
	84	const listed = await timePhaseAsync(ctx.request, "boards", () => listAllBoards(ctx.app, ctx.auth.user));
84	85	if (!listed.ok) return notFoundPage(ctx);
85	86	const make = html`<form method="post" action="/boards/new" class="bform">
86	87	<label class="qf"><span>New board</span><input class="input" name="title" required maxlength="${String(MAX_TITLE)}" placeholder="Title" /></label>
		@@ -100,13 +101,17 @@ export async function boardsList(ctx: Ctx): Promise<Response> {
100	101	</ul>`,
101	102	{ pad: false },
102	103	);
103		return page({
	104	const flow = timePhase(ctx.request, "flow", () => flowPanel(ctx.app, { kind: "owner" }, ctx.url, "/boards"));
		junipercrew 24d ago Nit: `render` times only the page() call. If the shell cost matters in production, move `flow` inside the same span or name it `shell`.
	105	const rendering = performance.now();
	106	const response = page({
104	107	ctx,
105	108	title: "Boards",
106	109	body: html`<div class="page-head"><h1>Boards</h1>
107	110	${btn("Explore the data", { href: "/boards/data", variant: "ghost" })}</div>
108		${card(make)} ${body} ${flowPanel(ctx.app, { kind: "owner" }, ctx.url, "/boards")}`,
	111	${card(make)} ${body} ${flow}`,
109	112	});
	113	notePhase(ctx.request, "render", performance.now() - rendering);
	114	return response;
110	115	}
111	116
112	117	function panelForm(slug: string, op: string, p: Panel \| undefined, index: number, fields: Html): Html {
		@@ -129,7 +129,7 @@ export interface Layout {
129	129	}
130	130
131	131	/** One hop of an edge between adjacent levels: the unit `order` sees. */
132		interface Seg {
	132	export interface Seg {
133	133	readonly from: string;
134	134	readonly to: string;
135	135	}
		@@ -304,15 +304,55 @@ function cross(
304	304	return top * bottom < 0;
305	305	}
306	306
307		function countCrossings(
	307	/**
	308	* The same count as comparing every pair of hops with `cross`, without
	309	* doing so (#3693: 2,266 edges drew 30 s of layout). A hop down one level crosses
	310	* another in the same gap exactly when their ends are in opposite order,
	311	* so each gap is an inversion count over a Fenwick tree, O(k log k). Hops
	312	* that point up or skip levels (only a cycle draws them) keep the pairwise
	313	* test against every hop; hops within one level never cross anything.
	314	*/
	315	export function countCrossings(
308	316	segs: readonly Seg[],
309	317	level: Map<string, number>,
310	318	pos: Map<string, number>,
311	319	): number {
	320	const gaps = new Map<number, [number, number][]>();
	321	const odd: Seg[] = [];
	322	const spanning: Seg[] = [];
	323	for (const s of segs) {
	324	const a = level.get(s.from)!;
	325	const b = level.get(s.to)!;
	326	if (a === b) continue;
	327	spanning.push(s);
	328	if (b === a + 1) push(gaps, a, [pos.get(s.from)!, pos.get(s.to)!]);
	329	else odd.push(s);
	330	}
312	331	let n = 0;
313		for (let i = 0; i < segs.length; i++)
314		for (let j = i + 1; j < segs.length; j++)
315		if (cross(segs[i]!, segs[j]!, level, pos)) n++;
	332	for (const hops of gaps.values()) {
	333	// Top end ascending, bottom ascending on ties: an equal top or an equal
	334	// bottom is a shared card, which meets rather than crosses.
	335	hops.sort((x, y) => x[0] - y[0] \|\| x[1] - y[1]);
	336	const size = hops.reduce((most, h) => Math.max(most, h[1]), 0) + 2;
	337	const tree = new Int32Array(size + 1);
		junipercrew 24d ago `Math.max(...hops.map(...))` spreads one argument per hop. A very wide gap could hit the argument limit. A plain loop or reduce costs nothing and removes that risk.
	338	let seen = 0;
	339	for (const [, bottom] of hops) {
	340	// Hops already seen with a bottom strictly right of this one's.
	341	let atMost = 0;
	342	for (let i = bottom + 1; i > 0; i -= i & -i) atMost += tree[i]!;
	343	n += seen - atMost;
	344	for (let i = bottom + 1; i <= size; i += i & -i) tree[i]!++;
	345	seen++;
	346	}
	347	}
	348	// An odd hop against every hop, each pair of odd hops once.
	349	const oddAt = new Map(odd.map((s, i) => [s, i]));
	350	odd.forEach((s, i) => {
	351	for (const t of spanning) {
	352	const j = oddAt.get(t);
	353	if ((j === undefined \|\| j > i) && cross(s, t, level, pos)) n++;
	354	}
	355	});
316	356	return n;
317	357	}
318	358
		@@ -383,8 +423,8 @@ function order(
383	423
384	424	const touching = new Map<string, Seg[]>();
385	425	for (const s of segs) {
386		touching.set(s.to, [...(touching.get(s.to) ?? []), s]);
387		touching.set(s.from, [...(touching.get(s.from) ?? []), s]);
	426	push(touching, s.to, s);
	427	push(touching, s.from, s);
388	428	}
389	429	const between = (a: string, b: string): number => {
390	430	let n = 0;
		@@ -0,0 +1,117 @@
	1	/**
	2	* A forge sized like production's for the flow readers (#3693): one app
	3	* repository with a month of runs (checks, crew, reviews, deploys) and two
	4	* weeks of merged pulls, the open asks the crew is on and the edges
	5	* between them, and a month of page views, written straight into the tables so a suite can build it
	6	* in seconds. Deterministic: the same call writes the same rows.
	7	*/
	8	import type { App } from "../../src/app.ts";
	9	import { createRepo } from "../../src/actions/repos.ts";
	10	import type { User } from "../../src/core/users.ts";
	11
	12	export interface FlowFixture {
	13	readonly runs: number;
	14	readonly pulls: number;
	15	readonly open: number;
	16	readonly edges: number;
	17	/** Page views over the month. */
	18	readonly vitals: number;
	19	}
	20
	21	const DAY = 86_400_000;
	22
	23	export async function seedFlowFixture(
	24	app: App,
	25	owner: User,
	26	size: FlowFixture = {
	27	runs: 100_000,
	28	pulls: 1_500,
	29	open: 400,
	30	edges: 2_266,
	31	vitals: 30_000,
	32	},
	33	now = Date.now(),
	34	): Promise<FlowFixture> {
	35	const made = await createRepo(app, owner, { name: "busy", visibility: "public" });
	36	if (!made.ok) throw new Error(`no repo: ${made.error}`);
	37	const repo = made.value;
	38	const db = app.db;
	39	db.run("UPDATE repo SET autonomy = 'auto', autonomy_slots = 2 WHERE id = ?1", [repo.id]);
	40	db.run("INSERT INTO app (id, repo_id, slug, access) VALUES ('app-busy', ?1, 'busy', 'public')", [repo.id]);
	41	let s = 3693;
	42	const r = (): number => {
	43	s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
	44	return s / 2 ** 32;
	45	};
	46	const iso = (ms: number): string => new Date(ms).toISOString();
	47	db.transaction(() => {
	48	const issue = db.prepare(
	49	`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at)
	50	VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)`,
	51	);
	52	const pull = db.prepare(
	53	`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at)
	54	VALUES (?1, 'main', ?2, '', '', ?3)`,
	55	);
	56	for (let n = 1; n <= size.pulls; n++) {
	57	const asked = now - r() * 21 * DAY;
	58	issue.run(`fi${n}`, repo.id, 2 * n - 1, "issue", `Ask ${n}`, "", owner.id, "closed", iso(asked));
	59	const opened = asked + r() * DAY;
	60	const merged = Math.min(now - 60_000, opened + r() * 3 * DAY);
	61	issue.run(`fp${n}`, repo.id, 2 * n, "pull", `Pull ${n}`, `Closes #${2 * n - 1}`, owner.id, "closed", iso(opened));
	62	pull.run(`fp${n}`, `crew/${n}`, iso(merged));
	63	}
	64	// Open asks, each with a finished crew run on a branch of its own.
	65	for (let n = 1; n <= size.open; n++)
	66	issue.run(`fo${n}`, repo.id, 2 * size.pulls + n, "issue", `Open ${n}`, "", owner.id, "open", iso(now - r() * 30 * DAY));
	67	const run = db.prepare(
	68	`INSERT INTO run (id, repo_id, kind, issue_id, env, branch, state, reason, created_at, started_at, finished_at, serving_at)
	69	VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)`,
	70	);
	71	for (let n = 0; n < size.runs; n++) {
	72	const at = now - r() * 30 * DAY;
	73	const pick = r();
	74	const kind = pick < 0.7 ? "check" : pick < 0.95 ? "agent" : "deploy";
	75	const pullN = 1 + Math.floor(r() * size.pulls);
	76	const open = kind === "agent" && size.open > 0 && r() < 0.05 ? 1 + Math.floor(r() * size.open) : 0;
	77	const env = kind === "deploy" ? (r() < 0.5 ? "production" : "preview") : "";
	78	run.run(
	79	`fr${String(n).padStart(7, "0")}`,
	80	repo.id,
	81	kind,
	82	kind === "deploy" ? null : open > 0 ? `fo${open}` : `fp${pullN}`,
	83	env,
	84	kind === "deploy" ? "main" : open > 0 ? `crew/open-${open}` : `crew/${pullN}`,
	85	r() < 0.9 ? "ok" : "failed",
	86	kind === "agent" && r() < 0.4 ? "review" : "",
	87	iso(at),
	88	iso(at + 1_000),
	89	iso(at + 60_000),
	90	env === "production" ? iso(at + 90_000) : null,
	91	);
	92	}
	93	// Production's map: 2,266 edges, each ask waiting on an older one.
	94	const asks = [
	95	...Array.from({ length: size.pulls }, (_, n) => `fi${n + 1}`),
	96	...Array.from({ length: size.open }, (_, n) => `fo${n + 1}`),
	97	];
	98	const edge = db.prepare("INSERT OR IGNORE INTO issue_dependency (issue_id, blocked_by_id) VALUES (?1, ?2)");
	99	for (let n = 0; n < size.edges; n++) {
	100	const later = 1 + Math.floor(r() * (asks.length - 1));
	101	edge.run(asks[later]!, asks[Math.floor(r() * later)]!);
	102	}
	103	// A month of page views, as the beacon writes them: 1,000 a day.
	104	const vital = db.prepare(
	105	"INSERT INTO page_vital (day, at, route, visitor, ttfb, load, lcp, inp, cls) VALUES (?1, ?2, '/', ?3, 20, ?4, ?4, ?5, 0)",
	106	);
	107	for (let n = 0; n < size.vitals; n++) {
	108	const at = now - r() * 30 * DAY;
	109	vital.run(iso(at).slice(0, 10), at, `v${Math.floor(r() * 400)}`, 100 + r() * 900, r() < 0.5 ? null : 10 + r() * 200);
	110	}
	111	})();
	112	// The seed went through the WAL: fold it back and give its pages up, so
	113	// the suite holds one copy of the data on disk and not two. The gate runs
	114	// many suites side by side on one disk (#3693).
	115	db.run("PRAGMA wal_checkpoint(TRUNCATE)");
	116	return { ...size };
	117	}
		@@ -0,0 +1,18 @@
	1	/**
	2	* A temporary directory on the checkout's own disk, for a suite whose data
	3	* would crowd `/tmp`. A check cell's `/tmp` is a 256 MiB tmpfs
	4	* (`TMP_MIB` in src/apps/contain.ts) shared by every file the gate runs at
	5	* once and by their coverage; a suite that writes tens of MB there fills it
	6	* as its neighbours start, and they fail in setup with no test named (#3693).
	7	* The checkout is on the cell's disk. `/.test-scratch/` is ignored by git.
	8	*/
	9	import { mkdirSync, mkdtempSync } from "node:fs";
	10	import { join } from "node:path";
	11
	12	export const SCRATCH_ROOT = join(import.meta.dir, "..", "..", ".test-scratch");
	13
	14	/** A fresh directory under {@link SCRATCH_ROOT}; the suite removes it. */
	15	export function scratchDir(prefix: string): string {
	16	mkdirSync(SCRATCH_ROOT, { recursive: true });
	17	return mkdtempSync(join(SCRATCH_ROOT, prefix));
	18	}
		@@ -0,0 +1,168 @@
	1	/**
	2	* The map's layout at the forge's own size (#3693): production's graph,
	3	* 1,314 issues and 2,266 edges, took 34 s of `layout` on a cold `/map`
	4	* because every ordering pass compared every pair of hops. The count is now
	5	* an inversion count per gap between levels; these tests hold it to the
	6	* pairwise definition, and hold the layout to a budget measured against a
	7	* same-run baseline rather than the wall clock (#3069).
	8	*/
	9	import { describe, expect, test } from "bun:test";
	10	import type { Graph } from "../src/core/dependencies.ts";
	11	import type { Issue } from "../src/core/issues.ts";
	12	import { countCrossings, layout, type Seg } from "../src/web/pages/map.ts";
	13
	14	/**
	15	* The definition `countCrossings` must agree with, written out here so the
	16	* oracle shares no code with what it checks: two hops cross when they share
	17	* no end, their spans of levels overlap, and their ends are in opposite order.
	18	*/
	19	function pairwiseCrossings(
	20	segs: readonly Seg[],
	21	level: Map<string, number>,
	22	pos: Map<string, number>,
	23	): number {
	24	let n = 0;
	25	for (let i = 0; i < segs.length; i++)
	26	for (let j = i + 1; j < segs.length; j++) {
	27	const s = segs[i]!;
	28	const t = segs[j]!;
	29	if (s.from === t.from \|\| s.to === t.to \|\| s.from === t.to \|\| s.to === t.from) continue;
	30	const [sa, sb, ta, tb] = [s.from, s.to, t.from, t.to].map((id) => level.get(id)!) as [number, number, number, number];
	31	if (Math.max(Math.min(sa, sb), Math.min(ta, tb)) >= Math.min(Math.max(sa, sb), Math.max(ta, tb))) continue;
	32	const top = pos.get(s.from)! - pos.get(t.from)!;
	33	const bottom = pos.get(s.to)! - pos.get(t.to)!;
	34	if (top * bottom < 0) n++;
	35	}
	36	return n;
	37	}
	38
	39	/** A seeded generator, so a failure reproduces. */
	40	function rng(seed: number): () => number {
	41	let s = seed >>> 0;
	42	return () => {
	43	s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
	44	return s / 2 ** 32;
	45	};
	46	}
	47
	48	/** Production's shape: about 1.7 edges an issue, each on an older issue. */
	49	function productionGraph(issues = 1314, edges = 2266): Graph {
	50	const r = rng(3693);
	51	const seen = new Set<string>();
	52	const list: { issueId: string; blockedById: string }[] = [];
	53	while (list.length < edges) {
	54	const issue = 2 + Math.floor(r() * (issues - 1));
	55	const blockedBy = 1 + Math.floor(r() * (issue - 1));
	56	const key = `${issue}>${blockedBy}`;
	57	if (seen.has(key)) continue;
	58	seen.add(key);
	59	list.push({ issueId: `i${issue}`, blockedById: `i${blockedBy}` });
	60	}
	61	return {
	62	issues: Array.from(
	63	{ length: issues },
	64	(_, n) => ({ id: `i${n + 1}`, number: n + 1 }) as unknown as Issue,
	65	),
	66	edges: list,
	67	held: [],
	68	deferred: new Map(),
	69	unadopted: new Set(),
	70	structural: new Set(),
	71	} as unknown as Graph;
	72	}
	73
	74	function median(run: () => void, times = 3): number {
	75	const spent: number[] = [];
	76	for (let n = 0; n < times; n++) {
	77	const started = performance.now();
	78	run();
	79	spent.push(performance.now() - started);
	80	}
	81	return spent.sort((a, b) => a - b)[Math.floor(times / 2)]!;
	82	}
	83
	84	describe("crossings, counted per gap", () => {
	85	test("agree with every pair compared, cycles and long hops included", () => {
	86	const r = rng(7);
	87	for (let round = 0; round < 200; round++) {
	88	const levels = 1 + Math.floor(r() * 6);
	89	const level = new Map<string, number>();
	90	const pos = new Map<string, number>();
	91	const ids: string[] = [];
	92	for (let l = 0; l < levels; l++) {
	93	const wide = 1 + Math.floor(r() * 7);
	94	for (let p = 0; p < wide; p++) {
	95	const id = `n${l}.${p}`;
	96	level.set(id, l);
	97	pos.set(id, p);
	98	ids.push(id);
	99	}
	100	}
	101	const segs: Seg[] = [];
	102	const hops = Math.floor(r() * 30);
	103	for (let h = 0; h < hops; h++) {
	104	const from = ids[Math.floor(r() * ids.length)]!;
	105	// Mostly one level down, as `layout` splits them; sometimes any.
	106	const below = ids.filter((id) => level.get(id) === level.get(from)! + 1);
	107	const to =
	108	below.length > 0 && r() < 0.7
	109	? below[Math.floor(r() * below.length)]!
	110	: ids[Math.floor(r() * ids.length)]!;
	111	segs.push({ from, to });
	112	}
	113	expect(countCrossings(segs, level, pos)).toBe(
	114	pairwiseCrossings(segs, level, pos),
	115	);
	116	}
	117	});
	118
	119	test("a known braid: two hops swapping ends cross once, sharing an end never", () => {
	120	const level = new Map([
	121	["a", 0],
	122	["b", 0],
	123	["c", 1],
	124	["d", 1],
	125	]);
	126	const pos = new Map([
	127	["a", 0],
	128	["b", 1],
	129	["c", 0],
	130	["d", 1],
	131	]);
	132	const crossed = [
	133	{ from: "a", to: "d" },
	134	{ from: "b", to: "c" },
	135	];
	136	const shared = [
	137	{ from: "a", to: "c" },
	138	{ from: "a", to: "d" },
	139	{ from: "b", to: "d" },
	140	];
	141	expect(countCrossings(crossed, level, pos)).toBe(1);
	142	expect(countCrossings(shared, level, pos)).toBe(0);
	143	});
	144	});
	145
	146	describe("the layout at production's size", () => {
	147	const graph = productionGraph();
	148
	149	test("is drawn deep and wide, like production's", () => {
	150	const placed = layout(graph);
	151	expect(placed.levels).toBeGreaterThan(10);
	152	expect(placed.routes.length).toBeGreaterThan(500);
	153	});
	154
	155	test("costs a few pairwise counts of its own card-to-card edges at most", () => {
	156	// Before #3693 the layout ran that pairwise count at least five times,
	157	// over more hops than this (every waypoint is one): about 50 baselines
	158	// on this graph. Now the whole layout is under one; three is the slack
	159	// a loaded machine gets.
	160	const placed = layout(graph);
	161	const level = new Map(placed.nodes.map((n) => [n.issue.id, n.level]));
	162	const pos = new Map(placed.nodes.map((n) => [n.issue.id, n.slot]));
	163	const segs = graph.edges.map((e) => ({ from: e.blockedById, to: e.issueId }));
	164	const baseline = median(() => pairwiseCrossings(segs, level, pos));
	165	const spent = median(() => layout(graph));
	166	expect(spent).toBeLessThan(3 * baseline);
	167	});
	168	});
		@@ -0,0 +1,114 @@
	1	/**
	2	* `/boards`, `/map` and home on a forge sized like production's (#3693):
	3	* a month of 100,000 runs, 1,500 merged pulls, 400 open asks and 2,266
	4	* edges between them. The owner saw /boards take 8–10 s, a cold /map 34.7 s
	5	* and home 9.2 s. Each budget is a multiple of one full pass over the run
	6	* table, timed in the same run, so a loaded machine slows both sides and
	7	* the wall clock is never the judge (#3069). Before the fix each page cost
	8	* hundreds of those passes.
	9	*/
	10	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	11	import { rm } from "node:fs/promises";
	12	import { tmpdir } from "node:os";
	13	import { createApp, type App } from "../src/app.ts";
	14	import { createSession } from "../src/core/sessions.ts";
	15	import { serve, type Servers } from "../src/http/server.ts";
	16	import { clearMapCache } from "../src/web/map_cache.ts";
	17	import { p75 } from "../src/flow/reach.ts";
	18	import { seedAdmin } from "./helpers/admin.ts";
	19	import { seedFlowFixture } from "./helpers/flow-fixture.ts";
	20	import { SCRATCH_ROOT, scratchDir } from "./helpers/scratch.ts";
	21
	22	let dataDir = "";
	23	let app: App, servers: Servers, cookie = "";
	24
	25	beforeAll(async () => {
	26	// ~49 MB of database: on the checkout's disk, not the cell's shared /tmp.
	27	dataDir = scratchDir("forge-obs-budget-");
	28	app = createApp({ dataDir, port: 0 });
	29	servers = await serve(app);
	30	const made = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "correct-horse-battery" });
	31	if (!made.ok) throw new Error(made.error);
	32	await seedFlowFixture(app, made.value);
	33	cookie = `forge_session=${createSession(app.db, made.value.id).token}`;
	34	}, 120_000);
	35
	36	afterAll(async () => {
	37	// A setup that failed part way leaves these unset: say only what it said.
	38	servers?.stop();
	39	app?.close();
	40	if (dataDir !== "") await rm(dataDir, { recursive: true, force: true });
	41	});
	42
	43	/** One full pass over every run: the unit the budgets are counted in. */
	44	function scan(): number {
	45	const started = performance.now();
	46	app.db.query("SELECT COUNT(*), SUM(length(branch)), SUM(length(created_at)) FROM run").get();
	47	return performance.now() - started;
	48	}
	49
	50	function baseline(): number {
	51	const spent = [scan(), scan(), scan()].sort((a, b) => a - b);
	52	return spent[1]!;
	53	}
	54
	55	async function get(path: string): Promise<{ ms: number; status: number; timing: string }> {
	56	const started = performance.now();
	57	const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
	58	await res.arrayBuffer();
	59	return { ms: performance.now() - started, status: res.status, timing: res.headers.get("server-timing") ?? "" };
	60	}
	61
	62	const names = (timing: string): string[] => timing.split(", ").map((p) => p.split(";")[0]!);
	63
	64	describe("on production's size", () => {
	65	test("the fixture lives on the checkout's disk, not in the shared /tmp", () => {
	66	expect(dataDir.startsWith(SCRATCH_ROOT)).toBe(true);
	67	expect(dataDir.startsWith(tmpdir())).toBe(false);
	68	});
	69
	70	test("the fixture is production's size", () => {
	71	const n = (sql: string) => (app.db.query(sql).get() as { n: number }).n;
	72	expect(n("SELECT COUNT(*) AS n FROM run")).toBe(100_000);
	73	expect(n("SELECT COUNT(*) AS n FROM issue_dependency")).toBeGreaterThan(2_000);
		junipercrew 24d ago A 30x multiple of one scan is a loose bound on a loaded machine; fine now, but watch it for flakiness.
	74	});
	75
	76	test("/boards answers within a few passes over the runs, and names its phases", async () => {
	77	await get("/boards");
	78	const unit = baseline();
	79	const res = await get("/boards");
	80	expect(res.status).toBe(200);
	81	for (const n of ["session", "boards", "flow", "render"]) expect(names(res.timing)).toContain(n);
	82	expect(res.ms).toBeLessThan(30 * unit);
	83	});
	84
	85	test("a cold /map answers within a few passes over the runs", async () => {
	86	await get("/map");
	87	clearMapCache();
	88	const unit = baseline();
	89	const res = await get("/map");
	90	expect(res.status).toBe(200);
	91	expect(names(res.timing)).toContain("layout");
	92	expect(res.ms).toBeLessThan(60 * unit);
	93	});
	94
	95	test("a cold home answers within a few passes over the runs, kpis read afresh", async () => {
	96	await get("/");
	97	// A write moves the clock home's caches are keyed by: this view is cold.
	98	app.db.run("UPDATE repo SET description = description \|\| ''");
	99	const unit = baseline();
	100	const res = await get("/");
	101	expect(res.status).toBe(200);
	102	expect(names(res.timing)).toContain("kpis");
	103	expect(res.timing).not.toMatch(/kpis;dur=[\d.]+;desc="hit"/);
	104	expect(res.ms).toBeLessThan(30 * unit);
	105	});
	106	});
	107
	108	test("home's p75 is nearest rank, read in one pass: the value at ⌈0.75·n⌉", () => {
	109	expect(p75([])).toBeNull();
	110	expect(p75([7])).toBe(7);
	111	// n = 4: rank 3. n = 5: rank ⌈3.75⌉ = 4. Order of arrival does not matter.
	112	expect(p75([40, 10, 30, 20])).toBe(30);
	113	expect(p75([50, 10, 40, 20, 30])).toBe(40);
	114	});