9fdc64d · abadbb4
@@ -1,39 +0,0 @@
name: timing
# Nightly: the wall-clock budgets (#7789). No check the forge runs, a pull's or
# the merge queue's gate, enforces them: it only warns of a time
# over its budget (test/helpers/budget.ts); here, one test file at a time on an
# otherwise idle runner, each budget is held to its absolute ms. A newly red
# file opens one `Red: test/….test.ts` issue and a green one closes it, as
# outside-gate does (scripts/flake-hunt.ts timing). Needs secret FORGE_TOKEN and
# variable FORGE_URL.
on:
  schedule:
    - cron: "23 4 * * *"
  workflow_dispatch: {}
jobs:
  timing:
    runs-on: isolated
    timeout-minutes: 30
    env:
      REGISTRY: forgejo-http.forgejo.svc.cluster.local:3000
      FORGE_TEST_TIMING: "1"
      FORGE_TEST_QUIET: "1"
    steps:
      - name: clone main
        env:
          TOKEN: ${{ github.token }}
        run: |
          set -eu
          git clone -q "http://x-access-token:$TOKEN@$REGISTRY/$GITHUB_REPOSITORY.git" src
          cd src
          bun install --frozen-lockfile
      - name: budget tests, serially
        working-directory: src
        env:
          FORGE_API: ${{ vars.FORGE_URL }}/api/v1
          FORGE_REPO: ${{ github.repository }}
          FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
          FORGE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_number }}
        run: bun scripts/flake-hunt.ts timing --file-issues
@@ -106,14 +106,11 @@
bun run src/main.ts   # boots on :3000, data in ./data
  `EXCLUDED`, so never hand-edit that move. Never add `retry` — fix the
  cause: a shared port, a real clock, order dependence, a leaked server, a
  timeout near the real duration, a wall-clock bound under 200 ms on a
  request or a spawn (`test/flake.test.ts` refuses one). A speed is bounded only
  with `expectBudget` (`test/helpers/budget.ts`): relative to a same-run
  baseline. No check the forge runs fails on it, a pull's or the merge queue's
  gate: a time over budget is only warned. It is enforced only with
  `FORGE_TEST_TIMING=1`, by the nightly `timing` workflow (one file at a time,
  `FORGE_TEST_QUIET=1`, `Red:` issues). To fail a regression in a pull, count work
  (statements, bytes, phases) or use a fake clock (#7789);
  `test/flake.test.ts` refuses a raw `under N ms` title or bound.
  request or a spawn (`test/flake.test.ts` refuses one). A speed is never bounded by a clock: count work
  instead (SQL statements with `test/helpers/statements.ts`, spawns, bytes,
  phases), assert it with `expectWork` (`test/helpers/work.ts`), or use a fake
  clock (#7789, #7824); `test/flake.test.ts` refuses a raw `under N ms` title
  or bound.
- **Every non-trivial change ships with a test.** Prefer protocol-level tests
  that boot a real app (`createApp` + `serve`) over unit mocks. Adversarial
  tests (can this be bypassed?) are worth more than happy-path ones.
@@ -210,12 +210,8 @@
longer hollow turns the check red, so the list only shrinks.
`bun scripts/hollow.ts` prints the same lines on their own.
## Budget tests run last
## Shards
A test file that calls `expectBudget(` (a call in code, not a mention in a
comment or string) bounds a speed and only passes with the machine quiet. The
sharded gate (`scripts/test-shards.ts`) sets those files aside and runs them
one at a time after every other shard is green; a red shard skips them. The
rule is `callsBudget` in `scripts/shards.ts`. Each shard also gets its own
Each shard gets its own
`TMPDIR` and `FORGE_APP_DATA` (nested under the cell's, when it set one),
removed when the shard ends.
@@ -15,10 +15,6 @@
 *       `--file-issues` a newly red file opens one `Red: test/….test.ts`
 *       issue naming its first failing test, a file red on another test
 *       updates it, and a file green again closes it.
 *   bun scripts/flake-hunt.ts timing [--file-issues]
 *       The files that call `expectBudget`, one at a time, as `outside` does;
 *       the scheduled run sets FORGE_TEST_TIMING=1 FORGE_TEST_QUIET=1 so each
 *       budget is held to its absolute ms (#7789).
 *   bun scripts/flake-hunt.ts quarantine
 *       Puts every file an open `Flaky: test/….test.ts` issue the hunt filed
 *       (author is the token's user, not untrusted) names into
@@ -43,8 +39,6 @@
import {
  redActions,
  redFile,
  redIssueFor,
  type RedRun,
  timingFiles,
  type OpenRed,
  quarantineFlaky,
  readRun,
@@ -241,10 +235,10 @@
async function openRedIssues(): Promise<Map<string, OpenRed>> {
  return found;
}
/** The files outside the gate (or, for `timing`, the budget files), one at a time; red ones reported to the forge. */
async function scheduled(kind: RedRun = "outside"): Promise<number> {
/** The files outside the gate, one at a time; red ones reported to the forge. */
async function scheduled(): Promise<number> {
  const present = presentTests();
  const files = kind === "timing" ? timingFiles(present, (file) => readFileSync(file, "utf8")) : outsideFiles(present);
  const files = outsideFiles(present);
  const fileIssues = rest.includes("--file-issues");
  const open = fileIssues ? await openRedIssues() : new Map<string, OpenRed>();
  const runs: FileRun[] = [];
@@ -257,12 +251,12 @@
async function scheduled(kind: RedRun = "outside"): Promise<number> {
  for (const action of redActions(runs, open)) {
    if (!fileIssues) continue;
    if (action.kind === "open") {
      const res = await forgeJson("POST", "/issues", redIssueFor(action.run, where, kind));
      const res = await forgeJson("POST", "/issues", redIssueFor(action.run, where));
      const made = (await res.json().catch(() => ({}))) as { number?: unknown };
      if (!res.ok) fail(`filing the issue for ${action.run.file}: ${res.status}`);
      console.log(`  filed #${made.number} for ${action.run.file}`);
    } else if (action.kind === "update") {
      const { body } = redIssueFor(action.run, where, kind);
      const { body } = redIssueFor(action.run, where);
      const res = await forgeJson("PATCH", `/issues/${action.number}`, { body });
      if (!res.ok) {
        const note = await forgeJson("POST", `/issues/${action.number}/comments`, { body });
@@ -279,7 +273,7 @@
async function scheduled(kind: RedRun = "outside"): Promise<number> {
    }
  }
  const red = runs.filter((run) => !run.green).length;
  console.log(`${files.length} files ${kind === "timing" ? "holding wall-clock budgets" : "outside the gate"}, ${red} red`);
  console.log(`${files.length} files outside the gate, ${red} red`);
  return 0;
}
@@ -376,11 +370,10 @@
async function admit(): Promise<number> {
}
if (mode === "outside") process.exit(await scheduled());
else if (mode === "timing") process.exit(await scheduled("timing"));
else if (mode === "hunt") process.exit(await hunt());
else if (mode === "quarantine") process.exit(await quarantineAll());
else if (mode === "admit") process.exit(await admit());
else
  fail(
    "usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] | outside [--file-issues] | timing [--file-issues] | quarantine | admit test/x.test.ts [--runs N]",
    "usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] | outside [--file-issues] | quarantine | admit test/x.test.ts [--runs N]",
  );
@@ -295,7 +295,7 @@
const BUDGET_MEASURE =
/**
 * Every line of `source` that holds a speed to a raw millisecond number: a
 * title saying "under N ms", or a Server-Timing/elapsed time bounded by a
 * literal under the floor (a hang guard above it stays a plain expect). `test/helpers/budget.ts` is the one way (#4429).
 * literal under the floor (a hang guard above it stays a plain expect). `expectWork()` in `test/helpers/work.ts` counts work instead (#7824).
 */
export function rawBudgets(name: string, source: string): string[] {
  const found: string[] = [];
@@ -307,8 +307,8 @@
export function rawBudgets(name: string, source: string): string[] {
        Number(bound!.replaceAll("_", "")) < WALL_CLOCK_FLOOR_MS
      )
        raw = true;
    if (raw && !/expectBudget/.test(line))
      found.push(`${name}:${i + 1}: ${line.trim()} — bound speed with expectBudget() from test/helpers/budget.ts (#4429)`);
    if (raw)
      found.push(`${name}:${i + 1}: ${line.trim()} — count work (statements, spawns, bytes) with expectWork() from test/helpers/work.ts, not a clock (#7824)`);
  });
  return found;
}
@@ -357,52 +357,20 @@
export function firstFailing(run: FileRun): string {
    : "none named; the file failed outside a test";
}
/** Files that test the budget helper itself, not a speed: never in the timing run. */
export const TIMING_EXCLUDED: readonly string[] = ["test/budget.test.ts"];
/** How a file says it bounds a wall-clock speed: it imports the budget helper and calls it. */
const BUDGET_IMPORT = /from\s+["']\.\/helpers\/budget\.ts["']/;
const BUDGET_CALL = /\bexpectBudget\(/;
/**
 * The files the nightly timing run holds to their wall-clock budgets (#7789):
 * each present test that imports `test/helpers/budget.ts` and calls
 * `expectBudget`, less {@link TIMING_EXCLUDED}. A file that only mentions the
 * name, in a string or a comment, is not one.
 */
export function timingFiles(
  present: readonly string[],
  read: (file: string) => string,
): string[] {
  return present.filter((file) => {
    if (TIMING_EXCLUDED.includes(file)) return false;
    const text = read(file);
    return BUDGET_IMPORT.test(text) && BUDGET_CALL.test(text);
  });
}
/** Which scheduled run found a file red: the files outside the gate, or the wall-clock budgets. */
export type RedRun = "outside" | "timing";
/** The issue for a file the scheduled run found red. */
export function redIssueFor(
  run: FileRun,
  where: string,
  kind: RedRun = "outside",
): { title: string; body: string } {
  const fence = "```";
  return {
    title: `${RED_TITLE}${run.file}`,
    body: [
      kind === "timing"
        ? `\`${run.file}\` was over a wall-clock budget, or red, in the nightly timing run (one file at a time, absolute budgets). This issue closes itself when the file passes again.`
        : `\`${run.file}\` is outside the gate and was red in the scheduled run. This issue closes itself when the file passes again.`,
      `\`${run.file}\` is outside the gate and was red in the scheduled run. This issue closes itself when the file passes again.`,
      "",
      `- first failing test: ${firstFailing(run)}`,
      `- run: ${where || "(a local run)"}`,
      kind === "timing"
        ? `- reproduce: \`FORGE_TEST_TIMING=1 FORGE_TEST_QUIET=1 bun test --timeout 60000 ./${run.file}\``
        : `- reproduce: \`bun test --timeout 60000 ./${run.file}\``,
      `- reproduce: \`bun test --timeout 60000 ./${run.file}\``,
      "",
      fence,
      run.text || "(no failure text captured)",
@@ -188,32 +188,6 @@
export function splitCapped(
  );
}
/** Whether a test file's code (comments dropped) calls `expectBudget(`. */
export function callsBudget(text: string | undefined): boolean {
  if (text === undefined) return false;
  const code = text.replace(/\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, "");
  return /\bexpectBudget\s*\(/.test(code);
}
/**
 * Splits `files` into those that bound a speed (a call to `expectBudget(`, which only
 * passes with the cell quiet enough) and the rest. The runner runs the
 * budget files last, one at a time, after every other shard is done, so the
 * load of the gate's own shards cannot redden them (#7617). `read` returns
 * a file's text; one it cannot read counts as no budget. Each list keeps
 * the input's order.
 */
export function budgetLast(
  files: readonly string[],
  read: (file: string) => string | undefined,
): { rest: string[]; budget: string[] } {
  const rest: string[] = [];
  const budget: string[] = [];
  for (const file of files)
    (callsBudget(read(file)) ? budget : rest).push(file);
  return { rest, budget };
}
/** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
@@ -415,7 +389,7 @@
export function crashedOutside(result: Pick<ShardResult, "code" | "output">): bo
/**
 * The crashed run's output, kept in the log for whoever reads it, each line
 * behind `  │ ` so nothing that reads the log (the tally, the failing-test
 * list, the budget re-run) counts its reds beside the re-run's.
 * list) counts its reds beside the re-run's.
 */
export function quoteCrash(where: string, code: number, output: string): string {
  const lines = output.replace(/\n$/, "").split("\n");
@@ -74,7 +74,6 @@
import {
  tally,
  weigher,
  besideOthers,
  budgetLast,
  failingFiles,
  onlyTestFailures,
  crashedOutside,
@@ -414,23 +413,13 @@
const serialFiles = !record && count.value === 1;
// refreshes the time of every file it ran (#1797).
const timed = !record && !serialFiles && gate && tier === "full" && "none" in shardSpec;
const perFile = record || measure || serialFiles || timed;
// Files that bound a speed run last and alone, after every other shard,
// so the gate's own load cannot redden them (#7617).
const ordered = budgetLast(files, (file) => {
  try {
    return readFileSync(file, "utf8");
  } catch {
    return undefined;
  }
});
const slicesOf = (list: string[]): string[][] =>
  list.length === 0
    ? []
    : perFile
      ? heaviestFirst(list, weight).map((file) => [file])
      : splitCapped(list, count.value, weight);
const mainSlices = slicesOf(ordered.rest);
const slices = [...mainSlices, ...ordered.budget.map((file) => [file])];
const slices = slicesOf(files);
const work = files.reduce((sum, file) => sum + weight(file), 0);
const banner = record
@@ -447,8 +436,6 @@
const banner = record
        `expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`;
console.log(banner);
process.stderr.write(banner + "\n");
if (ordered.budget.length > 0)
  console.log(`test-shards: ${ordered.budget.length} budget files last, one at a time`);
const children = new Set<Bun.Subprocess>();
for (const signal of ["SIGINT", "SIGTERM"] as const)
@@ -683,12 +670,12 @@
const started = performance.now();
const results = [
  ...(await pool(
    // Never more side by side than the count: capped slices outnumber it.
    serialFiles ? 1 : Math.max(1, Math.min(count.value, mainSlices.length)),
    serialFiles ? 1 : Math.max(1, Math.min(count.value, slices.length)),
    0,
    mainSlices.length,
    slices.length,
  )),
  // Budget files, one at a time, once the rest are done and green.
  ...(await pool(1, mainSlices.length, slices.length)),
  ...(await pool(1, slices.length, slices.length)),
];
const wall = (performance.now() - started) / 1000;
const ending = tally(results, wall);
@@ -9,7 +9,8 @@
import { addTeamMember, createOrg, createTeam } from "../src/core/orgs.ts";
import { canAdmin, getRepoById } from "../src/core/repos.ts";
import { createUser, getUserById, type User } from "../src/core/users.ts";
import { id } from "../src/kernel/id.ts";
import { expectBudget } from "./helpers/budget.ts";
import { statements } from "./helpers/statements.ts";
import { expectWork } from "./helpers/work.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
@@ -124,21 +125,11 @@
test("it is exactly one statement", () => {
  expect(n).toBe(1);
});
test("on 2,000 repositories it costs about one scan of the repository table", () => {
test("on 2,000 repositories it is still one statement", async () => {
  app.db.transaction(() => {
    for (let i = 0; i < 2000; i++) repo(i % 3 === 0 ? pat.id : root.id, `bulk${i}`);
  })();
  const best = (run: () => unknown): number => {
    let least = Infinity;
    for (let n = 0; n < 5; n++) {
      const t = performance.now();
      run();
      least = Math.min(least, performance.now() - t);
    }
    return least;
  };
  // The baseline is a plain scan of the same table, taken in the same run.
  const against = best(() => app.db.query("SELECT id FROM repo").all());
  expectBudget(best(() => adminRepoIds(app.db, pat)), { ms: 5, against });
  // Counted, not timed: one statement however many repositories there are.
  expectWork(await statements(app.db, () => void adminRepoIds(app.db, pat)), { max: 1 });
  expect(adminRepoIds(app.db, pat).size).toBe(667 + 3);
});
@@ -10,7 +10,7 @@
import { createSession } from "../src/core/sessions.ts";
import type { Repo } from "../src/core/repos.ts";
import { createUser, type User } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
@@ -109,20 +109,15 @@
test("130 open pulls read their mergeability from the cache: the merge phase is
    });
    if (!pull.ok) throw new Error(JSON.stringify(pull.error));
  }
  const mergeMs = async (): Promise<number> => {
    const res = await fetch(`${servers.url}/api/v1/repos/alice/r/pulls?limit=200`, { headers: { cookie } });
    const items = (JSON.parse(await res.text()) as { items: unknown[] }).items;
    expect(items.length).toBeGreaterThanOrEqual(130);
    const m = /merge;dur=([\d.]+)/.exec(res.headers.get("server-timing") ?? "");
    return Number(m?.[1] ?? NaN);
  };
  const uncached = await mergeMs();
  const url = `${servers.url}/api/v1/repos/alice/r/pulls?limit=200`;
  const res = await fetch(url, { headers: { cookie } });
  expect((JSON.parse(await res.text()) as { items: unknown[] }).items.length).toBeGreaterThanOrEqual(130);
  const spawn = spyOn(Bun, "spawn");
  try {
    let best = Infinity;
    for (let k = 0; k < 5; k++) best = Math.min(best, await mergeMs());
    // Counted, not timed: one read per page, not one per pull (130 pulls, no N+1).
    const reads = await requestStatements(app.db, url, { headers: { cookie } });
    expect(spawn.mock.calls.filter((c) => (c[0] as string[]).includes("merge-tree")).length).toBe(0);
    expectBudget(best, { ms: 20, against: uncached / 100 });
    expectWork(reads, { max: 16 });
  } finally {
    spawn.mockRestore();
  }
@@ -12,7 +12,6 @@
import { getRepo } from "../src/core/repos.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { proveAll } from "./helpers/verified.ts";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { quietLogs } from "./helpers/globals.ts";
import { quiet } from "./helpers/statements.ts";
@@ -238,22 +237,7 @@
test("following stays cheap on a diverged fork (regression guard; this pull does
  expect(Math.abs((await costOf(path)) - small[path]!)).toBeLessThanOrEqual(SLACK);
});
async function phases(path: string): Promise<Record<string, number>> {
  const best: Record<string, number> = {};
  for (let i = 0; i < 6; i++) {
    const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
    expect(res.status).toBe(200);
    const body = await res.text();
    if (path.endsWith("-fork")) expect(body.replace(/\s+/g, " ")).toContain("needs a decision");
    for (const part of (res.headers.get("server-timing") ?? "").split(", ")) {
      const m = /^([a-z]+);dur=([\d.]+)/.exec(part);
      if (m) best[m[1]!] = Math.min(best[m[1]!] ?? Infinity, Number(m[2]));
    }
  }
  return best;
}
test("now stays within budget on the live shape: 300 open issues, 2,000 closed, 3,500 edges, 20 pillars", async () => {
test("now costs the same statements on the live shape: 300 open issues, 2,000 closed, 3,500 edges, 20 pillars", async () => {
  const repo = getRepo(app.db, "alice", "vault")!;
  const author = app.db.query<{ id: string }, []>("SELECT id FROM user WHERE username = 'alice'").get()!.id;
  app.db.transaction(() => {
@@ -264,10 +248,9 @@
test("now stays within budget on the live shape: 300 open issues, 2,000 closed,
      ).run(`${repo.id}i${i}`, repo.id, 5000 + i, `Tell ${i}`, author);
    seedLive(app.db, repo.id, author, 0);
  })();
  const against = await baselineMs(servers.url);
  const best = await phases("/alice/vault");
  expectBudget(best.now!, { ms: 5, against });
  expectBudget(best.shipped!, { ms: 5, against });
  // Counted, not timed: 2,000 closed issues and 3,500 edges cost four statements more than the small page
  // (measured), not one per row; a per-row read would add thousands.
  expect(Math.abs((await costOf("/alice/vault")) - small["/alice/vault"]!)).toBeLessThanOrEqual(4);
});
test("the lean deferred read says what the whole graph says, and something is deferred", () => {
@@ -1,6 +1,6 @@
/** `/:owner/:name/app` and its `answers`, `now` and `state` fragments name their Server-Timing phases and answer within budget (#4320). */
/** `/:owner/:name/app` and its `answers`, `now` and `state` fragments name their Server-Timing phases and cost a bounded number of statements (#4320). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -59,6 +59,14 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
/** The most statements a request may run: today's count with room for a read or two. */
const MAX_STATEMENTS: Record<string, number> = {
  "/app": 10,
  "/app/answers": 10,
  "/app/now": 26,
  "/app/state": 21,
};
/** Route → the reads it makes, between `session` and `render`. */
const ROUTES: [string, string[]][] = [
  ["/app", ["repo", "view"]],
@@ -80,24 +88,16 @@
const get = (path: string) =>
  fetch(`${servers.url}${path}`, { headers: { cookie }, redirect: "manual" });
for (const [route, reads] of ROUTES)
  test(`GET /alice/vault${route} names its phases and answers within its budget`, async () => {
    const against = await baselineMs(servers.url);
  test(`GET /alice/vault${route} names its phases and costs a bounded number of statements`, async () => {
    await (await get(`/alice/vault${route}`)).arrayBuffer(); // warm
    // Timers and GC jitter on a busy machine: judge the best of several requests.
    let total = Infinity;
    let left = Infinity;
    for (let i = 0; i < 5; i++) {
      const res = await get(`/alice/vault${route}`);
      await res.arrayBuffer();
      expect(res.status).toBeLessThan(400);
      const got = phases(res.headers.get("server-timing"));
      for (const name of ["session", ...reads, "render"]) expect(got.has(name)).toBe(true);
      const sum = ["session", ...reads, "render"].reduce((n, k) => n + got.get(k)!, 0);
      total = Math.min(total, got.get("app")!);
      left = Math.min(left, Math.abs(got.get("app")! - sum));
    }
    expectBudget(total, { ms: 50, against });
    expect(left).toBeLessThanOrEqual(1.5); // each is rounded to 0.1
    expectWork(await requestStatements(app.db, `${servers.url}/alice/vault${route}`, { headers: { cookie }, redirect: "manual" }), { max: MAX_STATEMENTS[route]! });
  });
for (const [route] of ROUTES)
@@ -9,7 +9,8 @@
import { closesFromBody } from "../src/core/pulls.ts";
import { sealedScope } from "../src/core/sealed.ts";
import type { Repo } from "../src/core/repos.ts";
import { askToLiveFor } from "../src/web/ask_to_live.ts";
import { expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
let root = "";
let app: App;
@@ -140,14 +141,7 @@
test("the answer is the per-pull answer, for a writer and for a stranger", () =>
  expect(sealedScope(app.db, repo, undefined)).toEqual({});
});
test(`${PULLS} merged pulls over ${PILLARS} pillars answer within the budget`, async () => {
test(`${PULLS} merged pulls over ${PILLARS} pillars answer in a bounded number of statements`, async () => {
  askToLiveFor(app, repo, view, undefined);
  const base = 5; // ms
  let best = Infinity;
  for (let i = 0; i < 5; i++) {
    const started = performance.now();
    askToLiveFor(app, repo, view, undefined);
    best = Math.min(best, performance.now() - started);
  }
  expectBudget(best, { ms: 50, against: base });
  expectWork(await statements(app.db, () => void askToLiveFor(app, repo, view, undefined)), { max: 5 });
});
@@ -1,6 +1,6 @@
/** `/asked` names session, each read and render in Server-Timing, and answers a person with 500 asks within budget (#3358). */
/** `/asked` names session, each read and render in Server-Timing, and costs a bounded number of statements for a person with 500 asks (#3358). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -72,12 +72,10 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
test("GET /asked names its phases and answers within its budget with 500 asks", async () => {
  const against = await baselineMs(servers.url);
test("GET /asked names its phases and costs a bounded number of statements with 500 asks", async () => {
  const get = () => fetch(`${servers.url}/asked?show=all`, { headers: { cookie } });
  await (await get()).text();
  // The best of three: a shared runner's stall hits one request, not the page's cost.
  let best = Infinity;
  for (let i = 0; i < 3; i++) {
    const res = await get();
    const body = await res.text();
@@ -86,7 +84,6 @@
test("GET /asked names its phases and answers within its budget with 500 asks",
    const parts = (res.headers.get("server-timing") ?? "").split(", ");
    for (const name of ["app", "session", "repos", "counts", "asks", "thoughts", "flags", "render"])
      expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
    best = Math.min(best, Number(/app;dur=([\d.]+)/.exec(parts.join(", "))![1]));
  }
  expectBudget(best, { ms: 50, against });
  expectWork(await requestStatements(app.db, `${servers.url}/asked?show=all`, { headers: { cookie } }), { max: 35 });
});
@@ -1,8 +1,8 @@
/**
 * A repository with 200 branches lists them in under 50 ms: ahead/behind and
 * A repository with 200 branches lists them with a fixed number of git processes: ahead/behind and
 * tips are one git call each, not one per branch.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -13,7 +13,7 @@
import { commitLog } from "../src/git/read.ts";
import { repoDir } from "../src/git/repos.ts";
import { gitClientEnv } from "./helpers/git-client.ts";
import { seedAdmin } from "./helpers/admin.ts";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
const env = gitClientEnv({
  GIT_AUTHOR_NAME: "A",
@@ -79,6 +79,17 @@
afterAll(async () => {
  await rm(work, { recursive: true, force: true });
});
/** How many processes `work` starts: counted, not timed (#7824). */
async function spawns(work: () => Promise<unknown>): Promise<number> {
  const spy = spyOn(Bun, "spawn");
  try {
    await work();
    return spy.mock.calls.length;
  } finally {
    spy.mockRestore();
  }
}
const get = async (path: string): Promise<{ text: string; timing: string }> => {
  const res = await fetch(`${servers.url}${path}`, { headers: { authorization: `Bearer ${bearer}` } });
  const text = await res.text();
@@ -86,13 +97,13 @@
const get = async (path: string): Promise<{ text: string; timing: string }> => {
  return { text, timing: res.headers.get("server-timing") ?? "" };
};
test("200 branches hold the 50 ms budget and name their phases", async () => {
test("200 branches start a bounded number of git processes and name their phases", async () => {
  await get("/alice/app/branches");
  const { text, timing } = await get("/alice/app/branches");
  let page = { text: "", timing: "" };
  // One batched read of the refs, not a process per branch.
  expectWork(await spawns(async () => void (page = await get("/alice/app/branches"))), { max: 4 });
  const { text, timing } = page;
  const dur = (name: string) => Number(new RegExp(`(?:^|, )${name};dur=([0-9.]+)`).exec(timing)?.[1]);
  // 50 ms absolute on a quiet machine (FORGE_TEST_QUIET=1); under load it is judged
  // against a /healthz baseline from the same run, capped at 4x the budget.
  expectBudget(dur("app"), { ms: 50, against: await baselineMs(servers.url) });
  for (const phase of ["session", "refs", "compare", "render"]) expect(dur(phase)).toBeGreaterThanOrEqual(0);
  expect(text).toContain("200 branches");
  expect(text).toContain("feat/b198");
@@ -115,15 +126,15 @@
test("the one-call counts equal the per-branch rev-list counts", async () => {
  expect(listed.value.branches[1]).toMatchObject({ name: "feat/b000", ahead: 1, behind: 1 });
}, 60_000);
test("the API returns the first 30 of 200 branches with the whole list's counts, and its git phase fits the budget", async () => {
test("the API returns the first 30 of 200 branches with the whole list's counts, and a page of it starts a bounded number of git processes", async () => {
  const dir = repoDir(app.reposRoot, "alice", "app");
  const full = await listBranches(dir, "main");
  if (!full.ok) throw new Error("listBranches failed");
  const path = "/api/v1/repos/alice/app/branches?limit=30";
  await get(path);
  const { text, timing } = await get(path);
  const dur = Number(/(?:^|, )git;dur=([0-9.]+)/.exec(timing)?.[1]);
  expectBudget(dur, { ms: 60, against: await baselineMs(servers.url) });
  let got = { text: "", timing: "" };
  expectWork(await spawns(async () => void (got = await get(path))), { max: 5 });
  const { text } = got;
  const page = JSON.parse(text) as { items: { name: string; ahead: number | null; behind: number | null }[]; next_cursor: string | null };
  // Rows, order and cursor are the whole list's first 30.
  expect(page.items.map((i) => [i.name, i.ahead, i.behind])).toEqual(
@@ -135,23 +146,19 @@
test("the API returns the first 30 of 200 branches with the whole list's counts,
  expect(next.items[0]).toMatchObject({ ahead: 1, behind: 1 });
}, 30_000);
test("the per-branch fallback gives the same rows, and the one-call path is faster", async () => {
test("the per-branch fallback gives the same rows, and the one-call path starts fewer processes", async () => {
  const dir = repoDir(app.reposRoot, "alice", "app");
  // Best of three each, so one stalled run on a loaded machine cannot flip it.
  const best = async (opts: { bulk?: boolean }) => {
    let fastest = Infinity, last = await listBranches(dir, "main", opts);
    for (let i = 0; i < 3; i++) {
      const t = performance.now();
      last = await listBranches(dir, "main", opts);
      fastest = Math.min(fastest, performance.now() - t);
    }
    return { last, fastest };
  const run = async (opts: { bulk?: boolean }) => {
    let last = await listBranches(dir, "main", opts);
    const started = await spawns(async () => void (last = await listBranches(dir, "main", opts)));
    return { last, started };
  };
  const { last: bulk, fastest: bulkMs } = await best({});
  const { last: slow, fastest: slowMs } = await best({ bulk: false });
  const { last: bulk, started: bulkSpawns } = await run({});
  const { last: slow, started: slowSpawns } = await run({ bulk: false });
  if (!bulk.ok || !slow.ok) throw new Error("listBranches failed");
  expect(slow.value).toEqual(bulk.value);
  expect(bulkMs).toBeLessThan(slowMs);
  expect(bulkSpawns).toBeLessThan(slowSpawns);
  expectWork(bulkSpawns, { max: 5 });
}, 60_000);
test("a malformed or unsupported ahead-behind output gives up so the slow path runs", () => {
@@ -1,50 +0,0 @@
/** `budgetLimit` is the one place a speed budget is decided (#4429): absolute when quiet, relative and capped when not. */
import { afterEach, expect, test } from "bun:test";
import { BASELINE_FACTOR, MAX_RELAXATION, budgetLimit, expectBudget } from "./helpers/budget.ts";
const savedTiming = process.env.FORGE_TEST_TIMING;
const saved = process.env.FORGE_TEST_QUIET;
afterEach(() => {
  if (saved === undefined) delete process.env.FORGE_TEST_QUIET;
  else process.env.FORGE_TEST_QUIET = saved;
  if (savedTiming === undefined) delete process.env.FORGE_TEST_TIMING;
  else process.env.FORGE_TEST_TIMING = savedTiming;
});
test("a quiet cell holds the absolute ms, whatever the baseline", () => {
  process.env.FORGE_TEST_QUIET = "1";
  expect(budgetLimit(50, 0.1)).toBe(50);
  expect(budgetLimit(50, 100)).toBe(50);
});
test("a loaded cell scales with the baseline, never below ms", () => {
  delete process.env.FORGE_TEST_QUIET;
  expect(budgetLimit(50, 0.1)).toBe(50);
  expect(budgetLimit(50, 6)).toBe(Math.min(50 * MAX_RELAXATION, 6 * BASELINE_FACTOR));
  expect(budgetLimit(50, 6)).toBe(60);
});
test("a loaded cell never relaxes a budget past MAX_RELAXATION times ms", () => {
  delete process.env.FORGE_TEST_QUIET;
  expect(budgetLimit(50, 1000)).toBe(50 * MAX_RELAXATION);
  expect(budgetLimit(15, 1000)).toBe(15 * MAX_RELAXATION);
});
test("only FORGE_TEST_QUIET=1 means quiet", () => {
  process.env.FORGE_TEST_QUIET = "true";
  expect(budgetLimit(50, 1000)).toBe(50 * MAX_RELAXATION);
});
test("expectBudget fails a page that regressed to 150 ms beside a 5 ms baseline, when timing is enforced", () => {
  delete process.env.FORGE_TEST_QUIET;
  process.env.FORGE_TEST_TIMING = "1";
  expect(() => expectBudget(150, { ms: 50, against: 5 })).toThrow();
  expect(() => expectBudget(40, { ms: 50, against: 5 })).not.toThrow();
});
test("without FORGE_TEST_TIMING a slow time never fails: load alone cannot turn a pull red (#7789)", () => {
  delete process.env.FORGE_TEST_TIMING;
  expect(() => expectBudget(5000, { ms: 50, against: 5 })).not.toThrow();
  // A missing phase is not a slow one: it still fails.
  expect(() => expectBudget(Number.NaN, { ms: 50, against: 5 })).toThrow();
});
@@ -559,46 +559,6 @@
describe("the tiers in the forge's own checks service", () => {
    expect(readRunLog(dataDir, run!.id).lines.join("\n")).toContain(`CELL-CPUS=${want}`);
  });
  test("no check the forge runs holds wall-clock budgets: a pull's check and the queue's gate alike (#7789)", async () => {
    // Set on the forge itself, so a leak through the check's environment shows.
    const saved = process.env["FORGE_TEST_TIMING"];
    process.env["FORGE_TEST_TIMING"] = "1";
    try {
      const probe = 'echo "TIMING=${FORGE_TEST_TIMING:-unset}"';
      const seeded = await commitToBranch(app, alice, repo, {
        message: "Probe the timing flag",
        files: [{ path: "forge.json", content: JSON.stringify({ check: { run: "echo WHOLE", fast: probe, full: probe } }) }],
      });
      if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
      const branched = await commitToBranch(app, alice, repo, {
        branch: "cpu-limit",
        message: "Touch e",
        files: [{ path: "e.txt", content: "e\n" }],
      });
      if (!branched.ok) throw new Error(JSON.stringify(branched.error));
      const pull = listPulls(app.db, repo.id, { state: "open", limit: 20 }).find((p) => p.headBranch === "cpu-limit")!;
      const deadline = Date.now() + 30_000;
      let run: Run | undefined;
      while (Date.now() < deadline) {
        run = listRuns(app.db, repo.id, { kind: "check", limit: 20 }).find(
          (r) => r.issueId === pull.id && r.commitOid === branched.value.commitOid,
        );
        if (run && (run.state === "ok" || run.state === "failed")) break;
        await Bun.sleep(50);
      }
      expect(run?.state).toBe("ok");
      const pullLog = readRunLog(dataDir, run!.id).lines.join("\n");
      expect(pullLog).toContain("TIMING=unset");
      const gated = await checks.gate(repo, pull, seeded.value.commitOid);
      expect(gated.ok).toBe(true);
      expect(gated.lines.join("\n")).toContain("TIMING=unset");
      expect(gated.lines.join("\n")).not.toContain("TIMING=1");
    } finally {
      if (saved === undefined) delete process.env["FORGE_TEST_TIMING"];
      else process.env["FORGE_TEST_TIMING"] = saved;
    }
  });
  test("a red full gate names its tests and the pull learns the files", async () => {
    const red = await commitToBranch(app, alice, repo, {
      message: "Full goes red",
@@ -1,6 +1,6 @@
/** `/settings/cloud/audit`, `/organizations/new` and `/apps/open` name their Server-Timing phases and answer within budget (#4322). */
/** `/settings/cloud/audit`, `/organizations/new` and `/apps/open` name their Server-Timing phases and cost a bounded number of statements (#4322). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -45,6 +45,13 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
/** The most statements a request may run: today's count with room for a read or two. */
const MAX_STATEMENTS: Record<string, number> = {
  "/settings/cloud/audit": 11,
  "/organizations/new": 10,
  "/apps/open": 10,
};
// /apps/open with no host is the answer a stranger gets: session, then render.
// `route` and `compress` are the server's own work around the handler: named,
// the accounting no longer hangs on how long a loaded machine takes outside it.
@@ -54,42 +61,21 @@
const PAGES: [string, string[]][] = [
  ["/apps/open", ["route", "session", "render", "compress"]],
];
/** The page's budget (ms), and the most a request may spend between its named phases (ms) when quiet. */
const BUDGET_MS = 50;
const ACCOUNTING_MS = 1.5;
/** Requests judged per page; each assertion takes its own best, so one stalled request cannot fail it. */
/** Requests checked for their named phases. */
const SAMPLES = 7;
for (const [path, names] of PAGES)
  test(`GET ${path} names its phases and answers within its budget`, async () => {
    const against = await baselineMs(servers.url);
  test(`GET ${path} names its phases and costs a bounded number of statements`, async () => {
    const get = () => fetch(`${servers.url}${path}`, { headers: { cookie }, redirect: "manual" });
    await (await get()).text(); // warm
    // Timers jitter on a busy machine: judge the best of several requests.
    let left = Infinity;
    let total = Infinity;
    let last = "";
    for (let i = 0; i < SAMPLES; i++) {
      const res = await get();
      await res.text();
      const parts = (res.headers.get("server-timing") ?? "").split(", ");
      const dur = (n: string) => Number(parts.find((p) => p.startsWith(`${n};`))!.split("dur=")[1]);
      last = parts.join(", ");
      for (const name of ["app", ...names])
        expect(parts.find((p) => p.startsWith(`${name};`)), `phase ${name} in ${last}`).toMatch(/^[a-z]+;dur=\d+\.\d$/);
      total = Math.min(total, dur("app"));
      // Phases name the whole request: what is left over is small.
      left = Math.min(left, Math.abs(dur("app") - names.reduce((sum, n) => sum + dur(n), 0)));
    }
    expectBudget(total, { ms: BUDGET_MS, against });
    // The unnamed stretch (routing, before `session`) is fixed work, ~0.9 ms quiet and ~1.7 ms in a loaded
    // shard (#7519): it stretches with the machine, so it is judged against the same-run baseline like speed.
    try {
      expectBudget(left, { ms: ACCOUNTING_MS, against });
    } catch (e) {
      // Name the phases, so a red says whether the machine stalled or a phase went missing.
      throw new Error(`${(e as Error).message}; best app ${total.toFixed(1)} ms; last request ${last}`);
        expect(parts.find((p) => p.startsWith(`${name};`)), `phase ${name} in ${parts.join(", ")}`).toMatch(/^[a-z]+;dur=\d+\.\d$/);
    }
    expectWork(await requestStatements(app.db, `${servers.url}${path}`, { headers: { cookie }, redirect: "manual" }), { max: MAX_STATEMENTS[path]! });
  });
// Signed out, the redirect to sign-in is still a named phase.
@@ -99,9 +85,7 @@
for (const path of ["/settings/cloud/audit", "/organizations/new"])
    await res.text();
    expect(res.status).toBe(303);
    const parts = (res.headers.get("server-timing") ?? "").split(", ");
    const dur = (n: string) => Number(parts.find((p) => p.startsWith(`${n};`))!.split("dur=")[1]);
    const names = ["route", "session", "render", "compress"];
    for (const name of names)
      expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
    expect(Math.abs(dur("app") - names.reduce((sum, n) => sum + dur(n), 0))).toBeLessThan(1.5);
  });
@@ -10,7 +10,6 @@
import { createApp as insertApp } from "../src/core/apps.ts";
import { createIssue } from "../src/core/issues.ts";
import { createUser, type User } from "../src/core/users.ts";
import { countStatements } from "../bench/run.ts";
import { quiet } from "./helpers/budget.ts";
import { quietLogs } from "./helpers/globals.ts";
import { proveAll } from "./helpers/verified.ts";
import { promote } from "./helpers/members.ts";
@@ -109,8 +108,6 @@
test("a warm rank of 500 public repositories is a fraction of a cold one and wit
    await explore(app, undefined, query);
    warm.push(performance.now() - t);
  }
  // The relative bound below holds under load; the absolute 15 ms only in a quiet cell.
  if (quiet()) expect(Math.min(...warm)).toBeLessThan(15);
  expect(Math.min(...warm) * 5).toBeLessThan(Math.min(...cold));
});
@@ -26,8 +26,6 @@
import {
  redFile,
  redIssueFor,
  readRun,
  TIMING_EXCLUDED,
  timingFiles,
  retries,
  ROOT_CAUSES,
  shuffled,
@@ -359,7 +357,6 @@
describe("no raw wall-clock budget in a test (#4429)", () => {
    expect(rawBudgets("x", "expect(dur(\"app\")).toBeLessThan(50);")).toHaveLength(1);
    expect(rawBudgets("x", "expect(total).toBeLessThan(50);")).toHaveLength(1);
    expect(rawBudgets("x", "test(`GET /a answers under 50 ms`, () => {});")).toHaveLength(1);
    expect(rawBudgets("x", "expectBudget(dur(\"app\"), { ms: 50, against });")).toEqual([]);
    expect(rawBudgets("x", "expect(elapsed).toBeLessThan(5_000);")).toEqual([]); // a hang guard
    expect(rawBudgets("x", "expect(res.status).toBeLessThan(400);")).toEqual([]);
    expect(rawBudgets("x", "expect(lines.length).toBeLessThan(20);")).toEqual([]);
@@ -427,44 +424,3 @@
describe("the scheduled run of the files outside the gate", () => {
    expect(redFile("Red: test/a.test.ts")).toBe("test/a.test.ts");
  });
});
describe("the nightly timing run (#7789)", () => {
  const run = (file: string, failing: string[] = []) =>
    readRun(file, 0, failing.length ? 1 : 0, failing.map((f) => `(fail) ${f}`).join("\n"));
  // Built, not written out, so this file is not itself a budget file.
  const helper = ["./helpers", "budget.ts"].join("/");
  const imports = `import { expectBudget } from "${helper}";\n`;
  const files: Record<string, string> = {
    "test/a-timing.test.ts": `${imports}expectBudget(dur, { ms: 50, against });\n`,
    "test/b-mentions.test.ts": 'expect(rawBudgets("x", "expectBudget(dur, { ms: 50, against });")).toEqual([]);\n',
    "test/c-imports-only.test.ts": `import { baselineMs } from "${helper}";\n`,
    "test/budget.test.ts": `${imports}expectBudget(150, { ms: 50, against: 5 });\n`,
  };
  test("holds only files that import the budget helper and call expectBudget, less the helper's own test", () => {
    expect(TIMING_EXCLUDED).toContain("test/budget.test.ts");
    expect(timingFiles(Object.keys(files), (f) => files[f]!)).toEqual(["test/a-timing.test.ts"]);
  });
  test("the real selection names budget files and nothing that merely mentions the name", () => {
    const picked = timingFiles(presentTests(), (f) => readFileSync(f, "utf8"));
    expect(picked.length).toBeGreaterThan(0);
    expect(picked).not.toContain("test/flake.test.ts");
    for (const f of TIMING_EXCLUDED) expect(picked).not.toContain(f);
  });
  test("a timing Red: issue says it is a budget and reproduces with timing on", () => {
    const red = run("test/a-timing.test.ts", ["page"]);
    const issue = redIssueFor(red, "http://run/2", "timing");
    expect(issue.title).toBe("Red: test/a-timing.test.ts");
    expect(issue.body).toContain("wall-clock budget");
    expect(issue.body).toContain("FORGE_TEST_TIMING=1 FORGE_TEST_QUIET=1 bun test");
    expect(issue.body).not.toContain("outside the gate");
  });
  test("flake-hunt names the timing mode in its usage", () => {
    const out = Bun.spawnSync(["bun", "scripts/flake-hunt.ts", "nope"]);
    expect(out.exitCode).toBe(2);
    expect(out.stderr.toString()).toContain("timing [--file-issues]");
  });
});
@@ -1,76 +0,0 @@
/**
 * The one way a test bounds a speed (#4429).
 *
 * A raw `expect(ms).toBeLessThan(50)` is a coin flip in a loaded check cell:
 * the same code is 3 ms on a quiet machine and 80 ms beside six busy cells.
 * `expectBudget` judges a measured time against a baseline taken in the same
 * run, so load that slows the page slows the baseline with it. The absolute
 * `ms` is asserted when the cell is quiet (`FORGE_TEST_QUIET=1`, which the
 * nightly `timing` workflow sets). Otherwise the limit is relative to the
 * baseline but never above `MAX_RELAXATION` times `ms`, so a 50 ms budget
 * cannot drift past 200 ms however slow the baseline.
 *
 * Wall-clock is judged only where it can be trusted (#7789): with
 * `FORGE_TEST_TIMING=1`, set only by the nightly `timing` workflow. In every
 * check the forge runs, a pull's and the merge queue's gate alike, a time over its limit is reported on stderr and the test
 * passes, so the machine being busy can never turn a pull or a landing red; what a page
 * costs is a matter for the performance watcher (#7559) reading real traffic,
 * and a test that must fail a regression counts work (statements, bytes,
 * phases) instead.
 *
 * With timing on, a time over its limit throws an error that starts with
 * `BUDGET_MARK`, so the nightly's `Red:` issue says it is a budget. A time
 * that is not a number is a missing phase, not a slow one: that fails as any
 * `expect` does, timing on or off.
 */
import { expect } from "bun:test";
/** What a budget red's error starts with. */
export const BUDGET_MARK = "over budget (expectBudget)";
/** How many times slower than the same-run baseline a page may be under load. */
export const BASELINE_FACTOR = 10;
/** The most a loaded cell may stretch a budget: the limit is never above this many times `ms`. */
export const MAX_RELAXATION = 4;
/** Whether the cell is quiet enough to hold a number to its absolute ms. */
export const quiet = (): boolean => process.env.FORGE_TEST_QUIET === "1";
/** Whether wall-clock budgets are enforced: only the scheduled run says so (`FORGE_TEST_TIMING=1`). */
export const timing = (): boolean => process.env.FORGE_TEST_TIMING === "1";
/** The ms a measured time may take: `ms` when quiet, else relative to `against`. */
export function budgetLimit(ms: number, against: number): number {
  if (quiet()) return ms;
  return Math.min(ms * MAX_RELAXATION, Math.max(ms, against * BASELINE_FACTOR));
}
/** Asserts `measured` (ms) fits the budget `ms`, judged against `against` (ms, same run) unless quiet. */
export function expectBudget(measured: number, opts: { ms: number; against: number }): void {
  if (!Number.isFinite(measured)) {
    expect(measured).toBeLessThan(budgetLimit(opts.ms, opts.against));
    return;
  }
  const limit = budgetLimit(opts.ms, opts.against);
  if (measured < limit) return;
  if (!timing()) {
    console.warn(`over budget, not enforced (FORGE_TEST_TIMING unset): ${measured.toFixed(1)} ms vs ${limit.toFixed(1)} ms (budget ${opts.ms} ms)`);
    return;
  }
  throw new Error(
    `${BUDGET_MARK}: ${measured.toFixed(1)} ms is not under ${limit.toFixed(1)} ms ` +
      `(budget ${opts.ms} ms, baseline ${opts.against.toFixed(1)} ms${quiet() ? ", quiet" : ""})`,
  );
}
/** The best wall-clock ms of a few loopback requests to `/healthz`: what a request costs right now. */
export async function baselineMs(url: string, runs = 5): Promise<number> {
  let best = Infinity;
  for (let i = 0; i < runs; i++) {
    const started = performance.now();
    await (await fetch(`${url}/healthz`)).text();
    best = Math.min(best, performance.now() - started);
  }
  return best;
}
@@ -0,0 +1,24 @@
/**
 * The way a test bounds what a page costs without a clock (#7824).
 *
 * Wall-clock is a coin flip in a loaded check cell and is never judged in a
 * pull (see `budget.ts`). Work is countable: the SQL statements a request
 * runs, the bytes it sends. A regression (an N+1, a page that stops using its
 * cache) raises the count in every run, so a count fails a pull reliably where
 * a stopwatch cannot.
 */
import type { Database } from "bun:sqlite";
import { expect } from "bun:test";
import { statements } from "./statements.ts";
/** The fewest SQL statements one request for `url` costs (server and test share `db`). */
export async function requestStatements(db: Database, url: string, init?: RequestInit): Promise<number> {
  return statements(db, async () => {
    await (await fetch(url, init)).text();
  });
}
/** Asserts counted work `measured` is no larger than `max`. */
export function expectWork(measured: number, opts: { max: number }): void {
      
        junipercrew
        12d ago
        outdated
      
      Debug print behind an env var, and the isFinite assert below is redundant with the <= check. Remove both.
    
  expect(measured).toBeLessThanOrEqual(opts.max);
}
@@ -1,8 +1,9 @@
/** #5806: the owner's home on a forge of 200 repositories and 100 open asks answers warm in within 300 ms, every slow phase named. */
/** #5806: the owner's home on a forge of 200 repositories and 100 open asks is read in a bounded number of statements warm, every slow phase named. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { assignIssue, openIssue } from "../src/actions/issues.ts";
import { waitingCache } from "../src/web/studio/home_cache.ts";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { snappyFixture, type SnappyFixture } from "../bench/snappy.ts";
let f: SnappyFixture;
@@ -21,30 +22,25 @@
afterAll(async () => {
  await f.close();
});
async function look(): Promise<{ ms: number; phases: { name: string; ms: number }[] }> {
  const t = performance.now();
async function look(): Promise<{ phases: { name: string; ms: number }[] }> {
  const res = await fetch(`${f.base}/`, { headers: { cookie: f.ownerCookie } });
  await res.arrayBuffer();
  const ms = performance.now() - t;
  const phases = (res.headers.get("server-timing") ?? "")
    .split(", ")
    .map((p) => ({ name: p.split(";")[0]!, ms: Number(/dur=([\d.]+)/.exec(p)?.[1] ?? 0) }));
  return { ms, phases };
  return { phases };
}
test("the second look at home as the owner is within its budget, and every phase over 20 ms is named", async () => {
test("the second look at home as the owner costs a bounded number of statements, and every phase over 20 ms is named", async () => {
  await look();
  const warm = await look();
  const slow = warm.phases.filter((p) => p.name !== "app" && p.ms > 20);
  console.log(`home warm ${warm.ms.toFixed(0)} ms:`, warm.phases.map((p) => `${p.name} ${p.ms}`).join(", "));
  expectBudget(warm.ms, { ms: 300, against: await baselineMs(f.base) });
  // Counted, not timed (#7824): 200 repositories and 100 asks are read in batches, not one statement each.
  expectWork(await statements(f.app.db, async () => void (await look())), { max: 70 });
  // The page's own total is the sum of named phases: nothing over 20 ms hides in "app" alone.
  // Every phase home.ts and the feed time: one missing here fails as a plain expect, which the gate's budget retry never re-runs.
  const known = new Set(["app", "spawned", "steps", "session", "repos", "asked", "kpis", "vitals", "waiting", "panels", "feed", "settled", "visit", "since", "render"]);
  for (const p of slow) expect(known.has(p.name)).toBe(true);
  const total = warm.phases.find((p) => p.name === "app")!.ms;
  const named = warm.phases.filter((p) => p.name !== "app").reduce((s, p) => s + p.ms, 0);
  expectBudget(total - named, { ms: 20, against: await baselineMs(f.base) });
});
const waiting = async (): Promise<number> => {
@@ -3,7 +3,8 @@
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { IDLE_PULLS_SQL, readIdlePulls } from "../src/flow/idle_pulls.ts";
@@ -61,14 +62,9 @@
test("the plan never scans event", () => {
  expect(plan.some((d) => d.includes("event_repo_kind_number_at"))).toBe(true);
});
test("answers inside its 20 ms budget on a seeded forge", () => {
  readIdlePulls(app.db, NOW, true);
  const b = performance.now();
  app.db.query("SELECT count(*) FROM pull").get();
  const against = performance.now() - b;
  const t = performance.now();
  const rows = readIdlePulls(app.db, NOW, true);
  expectBudget(performance.now() - t, { ms: 20, against });
test("answers in one statement on a seeded forge", async () => {
  let rows: ReturnType<typeof readIdlePulls> = [];
  expectWork(await statements(app.db, () => void (rows = readIdlePulls(app.db, NOW, true))), { max: 1 });
  expect(rows.length).toBe(20);
  expect(rows[0]!.hours).toBeGreaterThanOrEqual(rows[19]!.hours);
});
@@ -1,12 +1,13 @@
/**
 * #7627: the day's landings are read in one pass. On a forge of 20,000 merged
 * pulls `readKpis` answers within 100 ms, and its count is the one the old
 * pulls `readKpis` costs a bounded number of statements, and its count is the one the old
 * per-row `NOT EXISTS` gave: a commit landed twice, or landed before the day
 * and carried again today, counts once or not at all.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { readKpis } from "../src/flow/kpis.ts";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { snappyFixture, type SnappyFixture } from "../bench/snappy.ts";
const HOUR = 3_600_000;
@@ -73,12 +74,10 @@
afterAll(async () => {
  await f.close();
});
test("readKpis answers within 100 ms over 20,000 merged pulls", async () => {
test("readKpis costs a bounded number of statements over 20,000 merged pulls", async () => {
  readKpis(f.app.db, new Date(NOW));
  const t = performance.now();
  readKpis(f.app.db, new Date(NOW));
  const ms = performance.now() - t;
  expectBudget(ms, { ms: 100, against: await baselineMs(f.base) });
  // Counted, not timed (#7824): the figures are a few set-based reads, never one per merged pull.
  expectWork(await statements(f.app.db, () => void readKpis(f.app.db, new Date(NOW))), { max: 12 });
});
test("the count matches the per-row NOT EXISTS, duplicates counted once", () => {
@@ -3,7 +3,8 @@
 * week of merges answers fast when warm, with the same bytes as a cold
 * build, and a plan shows at once (#3321).
 */
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
@@ -79,7 +80,7 @@
afterAll(async () => {
  await rm(dataDir, { recursive: true, force: true });
});
test("warm, the seed's map answers within its budget with the bytes of a cold build", async () => {
test("warm, the seed's map answers from its snapshot with the bytes of a cold build", async () => {
  mapSnapshots(app.db).clear();
  const cold = await get();
  const data = JSON.parse(cold.body) as { issues: unknown[]; edges: unknown[]; pillars: { merged?: number }[] };
@@ -90,9 +91,10 @@
test("warm, the seed's map answers within its budget with the bytes of a cold bu
  const warm = await get();
  expect(mapSnapshots(app.db).hits).toBe(hits + 1);
  expect(warm.body).toBe(cold.body);
  let slowest = 0;
  for (let i = 0; i < 5; i++) slowest = Math.max(slowest, (await get()).ms);
  expectBudget(slowest, { ms: 150, against: await baselineMs(servers.url) });
  // Counted, not timed: a warm map is served from its snapshot, a hit each time and a few statements at most.
  const warmHits = mapSnapshots(app.db).hits;
  expectWork(await statements(app.db, async () => void (await get())), { max: 9 });
  expect(mapSnapshots(app.db).hits).toBeGreaterThan(warmHits);
  expect(warm.timing).toContain("asktolive");
});
@@ -7,10 +7,10 @@
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { openIssue } from "../src/actions/issues.ts";
import { planStamp, readPlan } from "../src/core/milestones.ts";
import { getIssuesOfRepo } from "../src/core/issues.ts";
import { getRepo } from "../src/core/repos.ts";
import { createUser, getUserByName } from "../src/core/users.ts";
import { expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { quietLogs } from "./helpers/globals.ts";
import { proveAll } from "./helpers/verified.ts";
@@ -92,23 +92,11 @@
const BEFORE = [
  ["P10 Pillar 10", 198, 180, 18, 36, 0, 63],
];
test("a plan of 2,000 closed and 300 open issues, 3,000 pulls and 1,500 runs reads in under 50 ms with the same limbs", async () => {
test("a plan of 2,000 closed and 300 open issues, 3,000 pulls and 1,500 runs reads in a bounded number of statements with the same limbs", async () => {
  const repo = getRepo(app.db, "alice", "forge")!;
  // the same run's yardstick: the plan's own first stretch, every issue of the repository read lean, which slows with the machine as the plan does
  let against = Infinity;
  for (let n = 0; n < 5; n++) {
    const t = performance.now();
    getIssuesOfRepo(app.db, repo.id, "issue", true);
    against = Math.min(against, performance.now() - t);
  }
  readPlan(app.db, repo, { spend: true });
  const times: number[] = [];
  for (let n = 0; n < 7; n++) {
    const start = performance.now();
    readPlan(app.db, repo, { spend: true });
    times.push(performance.now() - start);
  }
  expectBudget(Math.min(...times), { ms: 50, against });
  // Counted, not timed: the plan is a fixed number of reads however many issues, pulls and runs there are.
  expectWork(await statements(app.db, () => void readPlan(app.db, repo, { spend: true })), { max: 24 });
  expect(weights(true)).toEqual(BEFORE);
  expect(weights(false).map((w) => w.slice(0, 6))).toEqual(BEFORE.map((w) => w.slice(0, 6)));
});
@@ -126,9 +114,8 @@
test("a sealed issue is in no limb a viewer cannot see", () => {
test("the stamp is kept while nothing is written, and moves when an issue is", async () => {
  const repo = getRepo(app.db, "alice", "forge")!;
  const first = planStamp(app.db, repo);
  const start = performance.now();
  for (let n = 0; n < 20; n++) expect(planStamp(app.db, repo)).toBe(first);
  expectBudget((performance.now() - start) / 20, { ms: 2, against: 1 });
  expectWork((await statements(app.db, () => { for (let n = 0; n < 20; n++) planStamp(app.db, repo); })) / 20, { max: 1 });
  expect(openIssue(app, getUserByName(app.db, "alice")!, repo, { title: "Fresh" }).ok).toBe(true);
  expect(planStamp(app.db, repo)).not.toBe(first);
});
@@ -11,7 +11,8 @@
import { getRepo } from "../src/core/repos.ts";
import { createUser, getUserByName } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { clearMapCache, planBuildCount } from "../src/web/map_cache.ts";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { quietLogs } from "./helpers/globals.ts";
import { proveAll } from "./helpers/verified.ts";
@@ -85,28 +86,13 @@
const list = async () => {
  return res.status;
};
test("readPlan of 2,000 closed and 300 open issues under 10 pillars fits a 40 ms budget", async () => {
test("readPlan of 2,000 closed and 300 open issues under 10 pillars costs a bounded number of statements", async () => {
  const repo = getRepo(app.db, "alice", "forge")!;
  // The yardstick is the same machine reading the same rows raw, so load
  // stretches both and the verdict follows the code (#6690). The plan builds
  // 2,300 issue objects and walks their graph: ~25 ms quiet, so 30 sat on the
  // edge and flipped with the machine (#6811); 40 leaves room under the page's 50.
  const raw = Math.min(
    ...Array.from({ length: 7 }, () => {
      const start = performance.now();
      app.db.query("SELECT * FROM issue WHERE repo_id = ?1").all(repo.id);
      return performance.now() - start;
    }),
  );
  readPlan(app.db, repo, { spend: true });
  const times: number[] = [];
  for (let n = 0; n < 7; n++) {
    const start = performance.now();
    const plan = readPlan(app.db, repo, { spend: true });
    times.push(performance.now() - start);
    expect(plan.limbs).toHaveLength(PILLARS);
  }
  expectBudget(Math.min(...times), { ms: 40, against: raw });
  // Counted, not timed (#7824): the plan reads the repository's issues in a fixed number of
  // statements, never one per issue (2,300 issues here).
  let plan = readPlan(app.db, repo, { spend: true });
  expectWork(await statements(app.db, () => void (plan = readPlan(app.db, repo, { spend: true }))), { max: 22 });
  expect(plan.limbs).toHaveLength(PILLARS);
});
test("a write to another repository keeps the plan warm; a new issue here refreshes it", async () => {
@@ -139,13 +125,14 @@
test("a milestone page names session, repo, plan and render", async () => {
    expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
});
test("a cold plan names its stretches, stays under 50 ms, and keeps each limb's counts", async () => {
  const against = await baselineMs(servers.url);
test("a cold plan names its stretches, costs a bounded number of statements, and keeps each limb's counts", async () => {
  const stretches: string[] = [];
  const repo = getRepo(app.db, "alice", "forge")!;
  const start = performance.now();
  const plan = readPlan(app.db, repo, { spend: true, stretch: (name) => stretches.push(name) });
  expectBudget(performance.now() - start, { ms: 50, against });
  let plan = readPlan(app.db, repo, { spend: true });
  expectWork(await statements(app.db, () => {
    stretches.length = 0;
    plan = readPlan(app.db, repo, { spend: true, stretch: (name) => stretches.push(name) });
  }), { max: 22 });
  expect(stretches).toEqual(["graph", "crew", "week", "spend", "limbs"]);
  expect(plan.limbs).toHaveLength(PILLARS);
  for (const limb of plan.limbs)
@@ -4,13 +4,10 @@
import { tmpdir } from "node:os";
import { join } from "node:path";
import { readTips } from "../src/git/tips.ts";
import { pullMergeStates, type Pull } from "../src/core/pulls.ts";
import { expectBudget } from "./helpers/budget.ts";
import { gitClientEnv } from "./helpers/git-client.ts";
const env = gitClientEnv();
const N = 130;
/** The uncached read spawns git per pull; a cached one should cost a small fraction of it. */
const BASELINE_DIVISOR = 100;
let root: string;
      
        junipercrew
        12d ago
        outdated
      
      Orphan comment: the constant it described was removed. Delete it.
    
let dir: string;
@@ -57,26 +54,18 @@
beforeEach(async () => {
afterEach(() => rm(root, { recursive: true, force: true }));
test("a second read of 130 pulls does no git work and is cheap beside the first; a moved head recomputes only its pull", async () => {
test("a second read of 130 pulls does no git work and spawns nothing; a moved head recomputes only its pull", async () => {
  const items = Array.from({ length: N }, (_, i) => ({ dir, pull: pullOf(i) }));
  const spawn = spyOn(Bun, "spawn");
  try {
    const t1 = performance.now();
    const first = await pullMergeStates(items);
    const uncached = performance.now() - t1;
    expect(first.every((m) => m?.mergeable === true)).toBe(true);
    expect(spawn.mock.calls.length).toBeGreaterThan(0);
    spawn.mockClear();
    // Best of a few cached reads: a busy machine slows one, not all.
    let best = Infinity;
    // Counted, not timed: cached reads spawn no git at all.
    let second = first;
    for (let k = 0; k < 5; k++) {
      const t0 = performance.now();
      second = await pullMergeStates(items);
      best = Math.min(best, performance.now() - t0);
    }
    expectBudget(best, { ms: 20, against: uncached / BASELINE_DIVISOR });
    for (let k = 0; k < 5; k++) second = await pullMergeStates(items);
    expect(spawn.mock.calls.length).toBe(0);
    expect(second).toEqual(first);
@@ -1,6 +1,6 @@
/**
 * A page of pulls is worded from batched reads (#6151): `pullWords` gives the
 * word `pullWord` gives for every state, in few queries and well under 5 ms.
 * word `pullWord` gives for every state, in few queries.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
@@ -17,7 +17,8 @@
import { repoDir } from "../src/git/repos.ts";
import { pullWord, pullWords, servingHistory, type WordCtx } from "../src/web/studio/words.ts";
import { quietLogs } from "./helpers/globals.ts";
import { adminCookie } from "./helpers/admin.ts";
import { expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
quietLogs();
@@ -121,23 +122,10 @@
describe("a page of pulls, worded in one batch", () => {
    expect(batched[1]).toEqual({ word: "waiting", note: "merged; waiting for production to redeploy" });
  });
  test("the words phase for 50 pulls is under 5 ms", async () => {
  test("the words phase for 50 pulls costs a bounded number of statements", async () => {
    const pulls = page();
    await pullWords(ctx(), pulls); // warm statement caches
    let best = Infinity;
    for (let n = 0; n < 5; n++) {
      const c = ctx();
      const at = performance.now();
      await pullWords(c, pulls);
      best = Math.min(best, performance.now() - at);
    }
    // The same-run baseline: the best of a few trivial reads on this machine, right now.
    let against = Infinity;
    for (let n = 0; n < 20; n++) {
      const at = performance.now();
      app.db.query("SELECT count(*) FROM issue").get();
      against = Math.min(against, performance.now() - at);
    }
    expectBudget(best, { ms: 5, against });
    const cost = await statements(app.db, async () => void (await pullWords(ctx(), pulls)));
    expectWork(cost, { max: 5 });
  });
});
@@ -1,6 +1,6 @@
/** A repository's new issue, make-it-yours, report and toolbar pages name their phases and answer within budget (#4321). */
/** A repository's new issue, make-it-yours, report and toolbar pages name their phases and cost a bounded number of statements (#4321). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -56,6 +56,14 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
/** The most statements a request may run: today's count with room for a read or two. */
const MAX_STATEMENTS: Record<string, number> = {
  "/alice/forge/issues/new": 38,
  "/alice/forge/report": 36,
  "/alice/forge/toolbar": 21,
  "/alice/forge/make-it-yours": 27,
};
/** Path, who is signed in, the phases it names (bar `app` and `total`), in order. */
const PAGES: [path: string, who: string, names: string[]][] = [
  ["/alice/forge/issues/new", "alice", ["session", "repo", "render"]],
@@ -65,8 +73,7 @@
const PAGES: [path: string, who: string, names: string[]][] = [
];
for (const [path, who, names] of PAGES)
  test(`GET ${path} names its phases, they fit inside app, and it answers within its budget`, async () => {
    const against = await baselineMs(servers.url);
  test(`GET ${path} names its phases, and it costs a bounded number of statements`, async () => {
    const cookie = await signIn(who);
    const read = async (): Promise<string[]> => {
      const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
@@ -78,20 +85,12 @@
for (const [path, who, names] of PAGES)
      expect(part).toMatch(/^[a-z-]+;dur=\d+\.\d$/);
      return Number(part!.slice(part!.indexOf("=") + 1));
    };
    for (let i = 0; i < 5; i++) await read(); // warm
    const shares: number[] = [];
    for (let i = 0; i < 11; i++) {
    await read(); // warm
    for (let i = 0; i < 3; i++) {
      const parts = await read();
      const named = parts.map((p) => p.split(";")[0]!).filter((n) => n !== "app" && n !== "db");
      expect(named).toEqual(names);
      const total = dur(parts.find((p) => p.startsWith("app;")));
      const sum = parts.filter((p) => !/^(app|db);/.test(p)).reduce((s, p) => s + dur(p), 0);
      expectBudget(total, { ms: 50, against });
      expect(sum).toBeLessThanOrEqual(total + 0.1 * names.length);
      shares.push(total === 0 ? 1 : sum / total);
      for (const part of parts) dur(part);
    }
    // `app` also holds routing and the security floor, outside these pages;
    // the median request is what the page can name of itself.
    shares.sort((a, b) => a - b);
    expect(shares[Math.floor(shares.length / 2)]!).toBeGreaterThanOrEqual(0.4);
    expectWork(await requestStatements(app.db, `${servers.url}${path}`, { headers: { cookie } }), { max: MAX_STATEMENTS[path]! });
  });
@@ -1,6 +1,6 @@
/** A repository's settings pages name session, each read and render in Server-Timing, and answer within budget (#4318). */
/** A repository's settings pages name session, each read and render in Server-Timing, and cost a bounded number of statements (#4318). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -48,6 +48,15 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
/** The most statements a request may run: today's count with room for a read or two. */
const MAX_STATEMENTS: Record<string, number> = {
  "/alice/forge/settings": 62,
  "/alice/forge/settings/danger": 38,
  "/alice/forge/settings/hooks": 38,
  "/alice/forge/settings/labels": 38,
  "/alice/forge/settings/prompts": 38,
};
const PAGES: [string, string[]][] = [
  ["/alice/forge/settings", ["route", "session", "repo", "lead", "refs", "check", "mirror", "render", "compress"]],
  ["/alice/forge/settings/danger", ["route", "session", "repo", "render", "compress"]],
@@ -57,13 +66,12 @@
const PAGES: [string, string[]][] = [
];
for (const [path, names] of PAGES)
  test(`GET ${path} names its phases and answers within its budget`, async () => {
  test(`GET ${path} names its phases and costs a bounded number of statements`, async () => {
    const get = async () => {
      const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
      await res.text();
      return res;
    };
    const against = await baselineMs(servers.url);
    await get(); // warm
    const res = await get();
    expect(res.status).toBe(200);
@@ -75,8 +83,5 @@
for (const [path, names] of PAGES)
      expect(Number.isFinite(dur(name))).toBe(true);
    }
    expect(got.filter((n) => n === "render").length).toBe(1);
    expectBudget(dur("app"), { ms: 50, against });
    // The named phases cover the request (route and compress are the server's own); the issue's bound is 1 ms.
    const sum = names.reduce((t, n) => t + dur(n), 0);
    expect(Math.abs(dur("app") - sum)).toBeLessThanOrEqual(1);
    expectWork(await requestStatements(app.db, `${servers.url}${path}`, { headers: { cookie } }), { max: MAX_STATEMENTS[path]! });
  });
@@ -1,6 +1,6 @@
/** `/settings/keys`, `/settings/tokens` and `/settings/password` name their phases signed in and answer within budget (#4316). */
/** `/settings/keys`, `/settings/tokens` and `/settings/password` name their phases signed in and cost a bounded number of statements (#4316). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -45,6 +45,13 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
/** The most statements a request may run: today's count with room for a read or two. */
const MAX_STATEMENTS: Record<string, number> = {
  "/settings/keys": 12,
  "/settings/tokens": 13,
  "/settings/password": 10,
};
const PAGES: [string, string[]][] = [
  ["/settings/keys", ["session", "keys", "render"]],
  ["/settings/tokens", ["session", "tokens", "sessions", "render"]],
@@ -52,29 +59,15 @@
const PAGES: [string, string[]][] = [
];
for (const [path, names] of PAGES)
  test(`GET ${path} names its phases and answers within its budget`, async () => {
    const against = await baselineMs(servers.url);
  test(`GET ${path} names its phases and costs a bounded number of statements`, async () => {
    await fetch(`${servers.url}${path}`, { headers: { cookie } }); // warm
    // A GC pause or a busy core between phases lands in one request, not all: judge the best of three.
    const seen: { app: number; sum: number }[] = [];
    for (let i = 0; i < 3; i++) {
    {
      const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
      await res.text();
      expect(res.status).toBe(200);
      const parts = (res.headers.get("server-timing") ?? "").split(", ");
      const dur = (name: string) => {
        const part = parts.find((p) => p.startsWith(`${name};`));
        expect(part).toMatch(/^[a-z]+;dur=\d+\.\d$/);
        return Number(part!.slice(`${name};dur=`.length));
      };
      expect(parts.map((p) => p.split(";")[0]).filter((n) => n !== "app")).toEqual(names);
      seen.push({ app: dur("app"), sum: names.reduce((t, n) => t + dur(n), 0) });
      for (const name of names) expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
    }
    expectBudget(Math.min(...seen.map((s) => s.app)), { ms: 50, against });
    // The rest of `app` is the server's own routing and header work, outside any page (~1-2 ms on every page);
    // under load it stretches with the baseline, as the budget does.
    const slack = Math.max(0.3, against * 0.1);
    const room = Math.max(5, against); // accounting, not speed
    expect(Math.min(...seen.map((s) => s.sum - s.app))).toBeLessThanOrEqual(slack);
    expect(Math.min(...seen.map((s) => s.app - s.sum))).toBeLessThanOrEqual(room);
    expectWork(await requestStatements(app.db, `${servers.url}${path}`, { headers: { cookie } }), { max: MAX_STATEMENTS[path]! });
  });
@@ -15,8 +15,6 @@
import {
  crashedOutside,
  CRASH_RERUNS,
  quoteCrash,
  budgetLast,
  callsBudget,
  failingFiles,
  onlyTestFailures,
  WORKER_MEMORY_BYTES,
@@ -1042,7 +1040,7 @@
describe("every shard runs to the end, and a crashed one once more", () => {
  }, 30_000);
});
describe("budget files last, and a fixture suite in six shards (#7617)", () => {
describe("a fixture suite in six shards (#7617)", () => {
  const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname;
  const fixture = (files: Record<string, string>): string => {
    const dir = mkdtempSync(join(tmpdir(), "shards-fixture-"));
@@ -1068,27 +1066,10 @@
describe("budget files last, and a fixture suite in six shards (#7617)", () => {
  };
  const passing = (n: number): string =>
    `import { test } from "bun:test";\ntest("passes ${n}", () => {});\n`;
  const budgeted = `import { test } from "bun:test";\ntest("budgeted", () => { expectBudget(() => {}, 1); });\nfunction expectBudget(f: () => void, _s: number) { f(); }\n`;
  /** The files named on the `── shard N/M…: files` lines, in the order shards ran. */
  const finished = (stdout: string): string[] =>
    [...stdout.matchAll(/^── shard \d+\/\d+(?: \(exit \d+, [\d.]+s\))?: (.*)$/gm)].map((m) => m[1]!);
  test("callsBudget sees a call, not a mention in a comment or a string", () => {
    expect(callsBudget("expectBudget(1, 2)")).toBe(true);
    expect(callsBudget("await expectBudget (x)")).toBe(true);
    expect(callsBudget("// expectBudget(1)\nlet a = 1")).toBe(false);
    expect(callsBudget("/* expectBudget(1) */")).toBe(false);
    expect(callsBudget('const note = "expectBudget is a helper"')).toBe(false);
    expect(callsBudget(undefined)).toBe(false);
  });
  test("budgetLast sets aside the files that call expectBudget, in order", () => {
    const text: Record<string, string> = { "a": "x", "b": "expectBudget(1)", "c": "y", "d": "expectBudget(2)", "e": "// expectBudget(3)" };
    const got = budgetLast(["a", "b", "c", "d", "e", "gone"], (file) => text[file]);
    expect(got.rest).toEqual(["a", "c", "e", "gone"]);
    expect(got.budget).toEqual(["b", "d"]);
  });
  test("twelve files in six shards: the plan has six slices and each file runs exactly once", async () => {
    const names = Array.from({ length: 12 }, (_, i) => `s${i}.test.ts`);
    const dir = fixture(Object.fromEntries(names.map((name, i) => [name, passing(i)])));
@@ -1113,38 +1094,6 @@
describe("budget files last, and a fixture suite in six shards (#7617)", () => {
    expect(got.out).toContain("(fail) the broken one");
  }, 60_000);
  test("budget files run last, alone, after the others", async () => {
    const dir = fixture({
      "a.test.ts": passing(1),
      "b.test.ts": passing(2),
      "c.test.ts": passing(3),
      "slow.test.ts": budgeted,
    });
    const got = await run(dir, "4", ["slow.test.ts", "a.test.ts", "b.test.ts", "c.test.ts"]);
    rmSync(dir, { recursive: true, force: true });
    expect(got.code).toBe(0);
    expect(got.out).toContain("1 budget files last");
    const order = finished(got.stdout);
    expect(order.at(-1)).toBe("slow.test.ts");
    expect(order.slice(0, -1).join(" ").split(" ").sort()).toEqual(["a.test.ts", "b.test.ts", "c.test.ts"]);
  }, 60_000);
  test("a red shard does not skip the budget files: the run is red and nothing crashes", async () => {
    const dir = fixture({
      "bad.test.ts": `import { test, expect } from "bun:test";\ntest("the broken one", () => { expect(1).toBe(2); });\n`,
      "a.test.ts": passing(1),
      "slow.test.ts": budgeted,
    });
    const got = await run(dir, "2", ["bad.test.ts", "a.test.ts", "slow.test.ts"]);
    rmSync(dir, { recursive: true, force: true });
    expect(got.code).not.toBe(0);
    expect(got.out).toContain("(fail) the broken one");
    expect(got.out).not.toContain("fail-fast");
    expect(finished(got.stdout).flat().join(" ")).toContain("slow.test.ts");
    expect(got.out).not.toContain("undefined");
    expect(got.out).not.toContain("TypeError");
  }, 60_000);
  test("each shard gets its own empty FORGE_APP_DATA and TMPDIR, removed after", async () => {
    const probe = (n: number): string =>
      `import { test, expect } from "bun:test";
@@ -1,4 +1,4 @@
/** 40 busy apps: the shelf asks git once for production's ancestry and never again until production moves; its phase stays under 50 ms (#6290, for #6179). */
/** 40 busy apps: the shelf asks git once for production's ancestry and never again until production moves; the whole shelf costs a bounded number of statements (#6290, for #6179). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -9,13 +9,13 @@
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { AppsRuntime } from "../src/apps/runtime.ts";
import { createIssue } from "../src/core/issues.ts";
import { claimRun, createRun, finishRun } from "../src/core/runs.ts";
import { takePhases } from "../src/core/route_timing.ts";
import { createUser, type User } from "../src/core/users.ts";
import type { Repo } from "../src/core/repos.ts";
import type { Ctx } from "../src/http/context.ts";
import { ancestries } from "../src/web/studio/ancestry.ts";
import { shelfRows } from "../src/web/studio/shelf.ts";
import { expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { quietLogs } from "./helpers/globals.ts";
import { proveAll } from "./helpers/verified.ts";
@@ -137,15 +137,9 @@
test("a viewer who cannot enter a private app causes no read about it", async ()
  }
});
test("41 busy apps are drawn under 50 ms of shelf (a smoke check)", async () => {
test("41 busy apps are drawn in a bounded number of statements", async () => {
  await shelfRows(ctx(), repos);
  let best = Infinity;
  for (let i = 0; i < 5; i++) {
    const c = ctx();
    await shelfRows(c, repos);
    best = Math.min(best, takePhases(c.request).filter((p) => p.name.startsWith("shelf-")).reduce((n, p) => n + p.ms, 0));
  }
  const t = performance.now();
  app.db.query("SELECT 1").get();
  expectBudget(best, { ms: 50, against: performance.now() - t });
  // Measured, not timed: 70 statements for 41 busy apps today, a read or two per app. A regression to
  // several statements per app (or an unbatched read) lands well past this.
  expectWork(await statements(app.db, async () => void (await shelfRows(ctx(), repos))), { max: 72 });
});
@@ -1,4 +1,4 @@
/** The shelf of 40 apps is drawn in a fixed number of reads, under 40 ms, and says what it said before (#6123). */
/** The shelf of 40 apps is drawn in a fixed number of reads and says what it said before (#6123). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -13,7 +13,8 @@
import type { Repo } from "../src/core/repos.ts";
import type { Ctx } from "../src/http/context.ts";
import { takePhases } from "../src/core/route_timing.ts";
import { shelfRows } from "../src/web/studio/shelf.ts";
import { expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
import { appsForRepos } from "../src/core/apps.ts";
import { primaryHosts } from "../src/core/domains.ts";
import { quietLogs } from "./helpers/globals.ts";
@@ -73,26 +74,14 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
test("40 apps are drawn within budget, in the phases Server-Timing names", async () => {
test("40 apps are drawn in few statements, in the phases Server-Timing names", async () => {
  await shelfRows(ctx(), repos); // warm
  let best = Infinity;
  let phases: string[] = [];
  for (let i = 0; i < 5; i++) {
    const c = ctx();
    const started = performance.now();
    await shelfRows(c, repos);
    best = Math.min(best, performance.now() - started);
    phases = takePhases(c.request).map((p) => p.name);
  }
  const c = ctx();
  await shelfRows(c, repos);
  const phases = takePhases(c.request).map((p) => p.name);
  expect(phases).toEqual(["shelf-records", "shelf-runs", "shelf-production", "shelf-pulls", "shelf-rows"]);
  // The yardstick is one batched read of the same shelf, taken in this run.
  let against = Infinity;
  for (let i = 0; i < 5; i++) {
    const t = performance.now();
    appsForRepos(app.db, repos.map((r) => r.id));
    against = Math.min(against, performance.now() - t);
  }
  expectBudget(best, { ms: 40, against });
  // Counted, not timed: the shelf reads in batches, so 40 apps cost no more statements than a handful.
  expectWork(await statements(app.db, async () => void (await shelfRows(ctx(), repos))), { max: 10 });
});
test("its rows are byte-identical to the ones drawn before the batch", async () => {
@@ -1,6 +1,6 @@
/** `/signin` and `/signup` name session, each read and render in Server-Timing, and answer within budget (#4106). */
/** `/signin` and `/signup` name session, each read and render in Server-Timing, and cost a bounded number of statements (#4106). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -25,18 +25,22 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
/** The most statements a request may run: today's count with room for a read or two. */
const MAX_STATEMENTS: Record<string, number> = {
  "/signin": 12,
  "/signup": 14,
};
const PAGES: [string, string[]][] = [
  ["/signin", ["session", "settings", "claim", "render"]],
  ["/signup", ["session", "signup", "users", "claim", "render"]],
];
for (const [path, names] of PAGES)
  test(`GET ${path} names its phases and answers within its budget`, async () => {
    const against = await baselineMs(servers.url);
  test(`GET ${path} names its phases and costs a bounded number of statements`, async () => {
    await (await fetch(`${servers.url}${path}`)).text(); // warm
    // A loaded machine stalls the event loop mid-request and inflates any one
    // sample; the fastest of a few is what the handler itself costs.
    let best = Infinity;
    for (let i = 0; i < 7; i++) {
      const res = await fetch(`${servers.url}${path}`);
      await res.text();
@@ -44,8 +48,6 @@
for (const [path, names] of PAGES)
      const parts = (res.headers.get("server-timing") ?? "").split(", ");
      for (const name of ["app", ...names])
        expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
      const app = parts.find((p) => p.startsWith("app;"))!;
      best = Math.min(best, Number(app.slice("app;dur=".length)));
    }
    expectBudget(best, { ms: 10, against });
    expectWork(await requestStatements(app.db, `${servers.url}${path}`), { max: MAX_STATEMENTS[path]! });
  });
@@ -8,7 +8,8 @@
import { createRepo } from "../src/actions/repos.ts";
import { forgetListings } from "../src/core/listings.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork } from "./helpers/work.ts";
import { statements } from "./helpers/statements.ts";
let dataDir: string;
let app: App;
@@ -34,24 +35,23 @@
afterAll(async () => {
  await rm(dataDir, { recursive: true, force: true });
});
/** The whole request's time: `app` already holds the named phases, so they are not added again. */
const total = (h: string) => Number(/(?:^|, )app;dur=([\d.]+)/.exec(h)?.[1] ?? Infinity);
test("lists the 100 public repositories and none of the 20 private ones, built in under 15 ms", async () => {
  let best = Infinity;
test("lists the 100 public repositories and none of the 20 private ones, built in a bounded number of statements", async () => {
  let res!: Response;
  let body = "";
  for (let i = 0; i < 5; i++) {
    forgetListings(app.db);
    res = await fetch(`${servers.url}/sitemap.xml`);
    body = await res.text();
    best = Math.min(best, total(res.headers.get("server-timing") ?? ""));
  }
  expect(body.match(/\/alice\/pub\d+</g)?.length).toBe(100);
  expect(body).not.toContain("priv");
  const timing = res.headers.get("server-timing") ?? "";
  for (const phase of ["repos", "render", "etag"]) expect(timing).toContain(`${phase};dur=`);
  expectBudget(best, { ms: 15, against: await baselineMs(servers.url) });
  // Counted, not timed: a cold build reads once for all 120 repositories, not once each.
  expectWork(await statements(app.db, async () => {
    forgetListings(app.db);
    await (await fetch(`${servers.url}/sitemap.xml`)).text();
  }), { max: 8 });
});
test("a repeat with If-None-Match gets a 304, a stale tag gets the document", async () => {
@@ -13,7 +13,6 @@
import { notePeer } from "../src/http/peer.ts";
import type { App } from "../src/app.ts";
import { portInsteadOfSocket } from "../src/git/hook-address.ts";
import { missing } from "./helpers/tools.ts";
import { expectBudget } from "./helpers/budget.ts";
import { openBeatDoor, parentLocal } from "../src/spawn/beat-door.ts";
import { lastHeard, sendBeat } from "../src/spawn/heartbeat.ts";
@@ -338,13 +337,9 @@
describe("a child whose cell will not come up", () => {
      mkdirSync(join(dataDir, "spawn"), { recursive: true });
      const file = join(dataDir, "spawn", "kid.json");
      writeFileSync(file, JSON.stringify({ name: "kid", bodyId: "cell-f", done: true, placement: "cell", size: "small" }));
      // The baseline: a child with no ledger, answered from the same read with no start.
      const unknown = performance.now();
      expect((await answerFromCell(parent, new Request("https://nobody--f.test/", { headers: { host: "nobody--f.test" } }))).status).toBe(404);
      const against = performance.now() - unknown;
      const started = performance.now();
      // A first request that waited on the cell's start would come back hosted, not 503.
      const first = await answerFromCell(parent, request());
      expectBudget(performance.now() - started, { ms: 50, against });
      expect(first.status).toBe(503);
      expect(first.headers.get("retry-after")).toBe("5");
      // The start it set off fails here (no cells), and the ledger turns hosted.
@@ -2,7 +2,6 @@
import { expect, test } from "bun:test";
import { ASCII_LIMIT, readStl, sayStl, stlSize, TOO_LARGE } from "../src/text/stl.ts";
import { asciiBox, binaryBox } from "./helpers/stl.ts";
import { expectBudget } from "./helpers/budget.ts";
const ascii = (s: string) => new TextEncoder().encode(s);
@@ -61,20 +60,7 @@
test("an ASCII body over the limit is too large to describe, and one at the limi
  };
  const at = fill(ASCII_LIMIT);
  expect(at.bytes.length).toBe(ASCII_LIMIT);
  const best = (bytes: Uint8Array) => {
    let ms = Infinity;
    for (let i = 0; i < 3; i++) {
      const start = performance.now();
      readStl(bytes);
      ms = Math.min(ms, performance.now() - start);
    }
    return ms;
  };
  // The baseline is a binary model of the same byte size read in this run, so load slows both alike.
  const bin = new Uint8Array(84 + 50 * Math.floor((ASCII_LIMIT - 84) / 50));
  new DataView(bin.buffer).setUint32(80, (bin.length - 84) / 50, true);
  expect(readStl(at.bytes)).toEqual({ ok: true, value: { format: "ascii", triangles: at.n, min: [1, 2, 3], max: [7, 8, 9] } });
  expectBudget(best(at.bytes), { ms: 50, against: best(bin) });
  expect(readStl(fill(ASCII_LIMIT + 1).bytes)).toEqual({ ok: false, error: TOO_LARGE });
  expect(readStl(fill(5_000_000).bytes)).toEqual({ ok: false, error: TOO_LARGE });
  // Junk tokens are no slower to refuse: the limit is checked before any scan.
@@ -1,6 +1,6 @@
/** `/verify`, `/recover` and `/claim` name their phases in Server-Timing and answer within budget, signed in (#4317). */
/** `/verify`, `/recover` and `/claim` name their phases in Server-Timing and cost a bounded number of statements, signed in (#4317). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { expectWork, requestStatements } from "./helpers/work.ts";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -43,6 +43,14 @@
afterAll(() => {
  rmSync(root, { recursive: true, force: true });
});
/** The most statements a request may run: today's count with room for a read or two. */
const MAX_STATEMENTS: Record<string, number> = {
  "/verify?token=nope": 12,
  "/recover": 10,
  "/recover?token=nope": 12,
  "/claim": 10,
};
const PAGES: [string, string[]][] = [
  ["/verify?token=nope", ["session", "verify", "render"]],
  ["/recover", ["session", "render"]],
@@ -51,8 +59,7 @@
const PAGES: [string, string[]][] = [
];
for (const [path, names] of PAGES)
  test(`GET ${path} names its phases and answers within its budget`, async () => {
    const against = await baselineMs(servers.url);
  test(`GET ${path} names its phases and costs a bounded number of statements`, async () => {
    const get = () => fetch(`${servers.url}${path}`, { headers: { cookie }, redirect: "manual" });
    await (await get()).text(); // warm
    const res = await get();
@@ -60,7 +67,5 @@
for (const [path, names] of PAGES)
    const parts = (res.headers.get("server-timing") ?? "").split(", ");
    for (const name of ["app", ...names])
      expect(parts.find((p) => p.startsWith(`${name};`))).toMatch(/^[a-z]+;dur=\d+\.\d$/);
    const dur = (n: string) => Number(parts.find((p) => p.startsWith(`${n};`))!.split("=")[1]);
    expectBudget(dur("app"), { ms: 50, against });
    expect(names.reduce((s, n) => s + dur(n), 0)).toBeLessThanOrEqual(dur("app") + 1);
    expectWork(await requestStatements(app.db, `${servers.url}${path}`, { headers: { cookie }, redirect: "manual" }), { max: MAX_STATEMENTS[path]! });
  });
		@@ -1,39 +0,0 @@
1		name: timing
2		# Nightly: the wall-clock budgets (#7789). No check the forge runs, a pull's or
3		# the merge queue's gate, enforces them: it only warns of a time
4		# over its budget (test/helpers/budget.ts); here, one test file at a time on an
5		# otherwise idle runner, each budget is held to its absolute ms. A newly red
6		# file opens one `Red: test/….test.ts` issue and a green one closes it, as
7		# outside-gate does (scripts/flake-hunt.ts timing). Needs secret FORGE_TOKEN and
8		# variable FORGE_URL.
9		on:
10		schedule:
11		- cron: "23 4 * * *"
12		workflow_dispatch: {}
13
14		jobs:
15		timing:
16		runs-on: isolated
17		timeout-minutes: 30
18		env:
19		REGISTRY: forgejo-http.forgejo.svc.cluster.local:3000
20		FORGE_TEST_TIMING: "1"
21		FORGE_TEST_QUIET: "1"
22		steps:
23		- name: clone main
24		env:
25		TOKEN: ${{ github.token }}
26		run: \|
27		set -eu
28		git clone -q "http://x-access-token:$TOKEN@$REGISTRY/$GITHUB_REPOSITORY.git" src
29		cd src
30		bun install --frozen-lockfile
31
32		- name: budget tests, serially
33		working-directory: src
34		env:
35		FORGE_API: ${{ vars.FORGE_URL }}/api/v1
36		FORGE_REPO: ${{ github.repository }}
37		FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
38		FORGE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_number }}
39		run: bun scripts/flake-hunt.ts timing --file-issues
		@@ -106,14 +106,11 @@ bun run src/main.ts # boots on :3000, data in ./data
106	106	`EXCLUDED`, so never hand-edit that move. Never add `retry` — fix the
107	107	cause: a shared port, a real clock, order dependence, a leaked server, a
108	108	timeout near the real duration, a wall-clock bound under 200 ms on a
109		request or a spawn (`test/flake.test.ts` refuses one). A speed is bounded only
110		with `expectBudget` (`test/helpers/budget.ts`): relative to a same-run
111		baseline. No check the forge runs fails on it, a pull's or the merge queue's
112		gate: a time over budget is only warned. It is enforced only with
113		`FORGE_TEST_TIMING=1`, by the nightly `timing` workflow (one file at a time,
114		`FORGE_TEST_QUIET=1`, `Red:` issues). To fail a regression in a pull, count work
115		(statements, bytes, phases) or use a fake clock (#7789);
116		`test/flake.test.ts` refuses a raw `under N ms` title or bound.
	109	request or a spawn (`test/flake.test.ts` refuses one). A speed is never bounded by a clock: count work
	110	instead (SQL statements with `test/helpers/statements.ts`, spawns, bytes,
	111	phases), assert it with `expectWork` (`test/helpers/work.ts`), or use a fake
	112	clock (#7789, #7824); `test/flake.test.ts` refuses a raw `under N ms` title
	113	or bound.
117	114	- Every non-trivial change ships with a test. Prefer protocol-level tests
118	115	that boot a real app (`createApp` + `serve`) over unit mocks. Adversarial
119	116	tests (can this be bypassed?) are worth more than happy-path ones.
		@@ -210,12 +210,8 @@ longer hollow turns the check red, so the list only shrinks.
210	210	`bun scripts/hollow.ts` prints the same lines on their own.
211	211
212	212
213		## Budget tests run last
	213	## Shards
214	214
215		A test file that calls `expectBudget(` (a call in code, not a mention in a
216		comment or string) bounds a speed and only passes with the machine quiet. The
217		sharded gate (`scripts/test-shards.ts`) sets those files aside and runs them
218		one at a time after every other shard is green; a red shard skips them. The
219		rule is `callsBudget` in `scripts/shards.ts`. Each shard also gets its own
	215	Each shard gets its own
220	216	`TMPDIR` and `FORGE_APP_DATA` (nested under the cell's, when it set one),
221	217	removed when the shard ends.
		@@ -15,10 +15,6 @@
15	15	* `--file-issues` a newly red file opens one `Red: test/….test.ts`
16	16	* issue naming its first failing test, a file red on another test
17	17	* updates it, and a file green again closes it.
18		* bun scripts/flake-hunt.ts timing [--file-issues]
19		* The files that call `expectBudget`, one at a time, as `outside` does;
20		* the scheduled run sets FORGE_TEST_TIMING=1 FORGE_TEST_QUIET=1 so each
21		* budget is held to its absolute ms (#7789).
22	18	* bun scripts/flake-hunt.ts quarantine
23	19	* Puts every file an open `Flaky: test/….test.ts` issue the hunt filed
24	20	* (author is the token's user, not untrusted) names into
		@@ -43,8 +39,6 @@ import {
43	39	redActions,
44	40	redFile,
45	41	redIssueFor,
46		type RedRun,
47		timingFiles,
48	42	type OpenRed,
49	43	quarantineFlaky,
50	44	readRun,
		@@ -241,10 +235,10 @@ async function openRedIssues(): Promise<Map<string, OpenRed>> {
241	235	return found;
242	236	}
243	237
244		/** The files outside the gate (or, for `timing`, the budget files), one at a time; red ones reported to the forge. */
245		async function scheduled(kind: RedRun = "outside"): Promise<number> {
	238	/** The files outside the gate, one at a time; red ones reported to the forge. */
	239	async function scheduled(): Promise<number> {
246	240	const present = presentTests();
247		const files = kind === "timing" ? timingFiles(present, (file) => readFileSync(file, "utf8")) : outsideFiles(present);
	241	const files = outsideFiles(present);
248	242	const fileIssues = rest.includes("--file-issues");
249	243	const open = fileIssues ? await openRedIssues() : new Map<string, OpenRed>();
250	244	const runs: FileRun[] = [];
		@@ -257,12 +251,12 @@ async function scheduled(kind: RedRun = "outside"): Promise<number> {
257	251	for (const action of redActions(runs, open)) {
258	252	if (!fileIssues) continue;
259	253	if (action.kind === "open") {
260		const res = await forgeJson("POST", "/issues", redIssueFor(action.run, where, kind));
	254	const res = await forgeJson("POST", "/issues", redIssueFor(action.run, where));
261	255	const made = (await res.json().catch(() => ({}))) as { number?: unknown };
262	256	if (!res.ok) fail(`filing the issue for ${action.run.file}: ${res.status}`);
263	257	console.log(` filed #${made.number} for ${action.run.file}`);
264	258	} else if (action.kind === "update") {
265		const { body } = redIssueFor(action.run, where, kind);
	259	const { body } = redIssueFor(action.run, where);
266	260	const res = await forgeJson("PATCH", `/issues/${action.number}`, { body });
267	261	if (!res.ok) {
268	262	const note = await forgeJson("POST", `/issues/${action.number}/comments`, { body });
		@@ -279,7 +273,7 @@ async function scheduled(kind: RedRun = "outside"): Promise<number> {
279	273	}
280	274	}
281	275	const red = runs.filter((run) => !run.green).length;
282		console.log(`${files.length} files ${kind === "timing" ? "holding wall-clock budgets" : "outside the gate"}, ${red} red`);
	276	console.log(`${files.length} files outside the gate, ${red} red`);
283	277	return 0;
284	278	}
285	279
		@@ -376,11 +370,10 @@ async function admit(): Promise<number> {
376	370	}
377	371
378	372	if (mode === "outside") process.exit(await scheduled());
379		else if (mode === "timing") process.exit(await scheduled("timing"));
380	373	else if (mode === "hunt") process.exit(await hunt());
381	374	else if (mode === "quarantine") process.exit(await quarantineAll());
382	375	else if (mode === "admit") process.exit(await admit());
383	376	else
384	377	fail(
385		"usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] \| outside [--file-issues] \| timing [--file-issues] \| quarantine \| admit test/x.test.ts [--runs N]",
	378	"usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] \| outside [--file-issues] \| quarantine \| admit test/x.test.ts [--runs N]",
386	379	);
		@@ -295,7 +295,7 @@ const BUDGET_MEASURE =
295	295	/**
296	296	* Every line of `source` that holds a speed to a raw millisecond number: a
297	297	* title saying "under N ms", or a Server-Timing/elapsed time bounded by a
298		* literal under the floor (a hang guard above it stays a plain expect). `test/helpers/budget.ts` is the one way (#4429).
	298	* literal under the floor (a hang guard above it stays a plain expect). `expectWork()` in `test/helpers/work.ts` counts work instead (#7824).
299	299	*/
300	300	export function rawBudgets(name: string, source: string): string[] {
301	301	const found: string[] = [];
		@@ -307,8 +307,8 @@ export function rawBudgets(name: string, source: string): string[] {
307	307	Number(bound!.replaceAll("_", "")) < WALL_CLOCK_FLOOR_MS
308	308	)
309	309	raw = true;
310		if (raw && !/expectBudget/.test(line))
311		found.push(`${name}:${i + 1}: ${line.trim()} — bound speed with expectBudget() from test/helpers/budget.ts (#4429)`);
	310	if (raw)
	311	found.push(`${name}:${i + 1}: ${line.trim()} — count work (statements, spawns, bytes) with expectWork() from test/helpers/work.ts, not a clock (#7824)`);
312	312	});
313	313	return found;
314	314	}
		@@ -357,52 +357,20 @@ export function firstFailing(run: FileRun): string {
357	357	: "none named; the file failed outside a test";
358	358	}
359	359
360		/** Files that test the budget helper itself, not a speed: never in the timing run. */
361		export const TIMING_EXCLUDED: readonly string[] = ["test/budget.test.ts"];
362
363		/** How a file says it bounds a wall-clock speed: it imports the budget helper and calls it. */
364		const BUDGET_IMPORT = /from\s+["']\.\/helpers\/budget\.ts["']/;
365		const BUDGET_CALL = /\bexpectBudget\(/;
366
367		/**
368		* The files the nightly timing run holds to their wall-clock budgets (#7789):
369		* each present test that imports `test/helpers/budget.ts` and calls
370		* `expectBudget`, less {@link TIMING_EXCLUDED}. A file that only mentions the
371		* name, in a string or a comment, is not one.
372		*/
373		export function timingFiles(
374		present: readonly string[],
375		read: (file: string) => string,
376		): string[] {
377		return present.filter((file) => {
378		if (TIMING_EXCLUDED.includes(file)) return false;
379		const text = read(file);
380		return BUDGET_IMPORT.test(text) && BUDGET_CALL.test(text);
381		});
382		}
383
384		/** Which scheduled run found a file red: the files outside the gate, or the wall-clock budgets. */
385		export type RedRun = "outside" \| "timing";
386
387	360	/** The issue for a file the scheduled run found red. */
388	361	export function redIssueFor(
389	362	run: FileRun,
390	363	where: string,
391		kind: RedRun = "outside",
392	364	): { title: string; body: string } {
393	365	const fence = "```";
394	366	return {
395	367	title: `${RED_TITLE}${run.file}`,
396	368	body: [
397		kind === "timing"
398		? `\`${run.file}\` was over a wall-clock budget, or red, in the nightly timing run (one file at a time, absolute budgets). This issue closes itself when the file passes again.`
399		: `\`${run.file}\` is outside the gate and was red in the scheduled run. This issue closes itself when the file passes again.`,
	369	`\`${run.file}\` is outside the gate and was red in the scheduled run. This issue closes itself when the file passes again.`,
400	370	"",
401	371	`- first failing test: ${firstFailing(run)}`,
402	372	`- run: ${where \|\| "(a local run)"}`,
403		kind === "timing"
404		? `- reproduce: \`FORGE_TEST_TIMING=1 FORGE_TEST_QUIET=1 bun test --timeout 60000 ./${run.file}\``
405		: `- reproduce: \`bun test --timeout 60000 ./${run.file}\``,
	373	`- reproduce: \`bun test --timeout 60000 ./${run.file}\``,
406	374	"",
407	375	fence,
408	376	run.text \|\| "(no failure text captured)",
		@@ -188,32 +188,6 @@ export function splitCapped(
188	188	);
189	189	}
190	190
191		/** Whether a test file's code (comments dropped) calls `expectBudget(`. */
192		export function callsBudget(text: string \| undefined): boolean {
193		if (text === undefined) return false;
194		const code = text.replace(/\/\[\s\S]?\\/\|\/\/[^\n]/g, "");
195		return /\bexpectBudget\s*\(/.test(code);
196		}
197
198		/**
199		* Splits `files` into those that bound a speed (a call to `expectBudget(`, which only
200		* passes with the cell quiet enough) and the rest. The runner runs the
201		* budget files last, one at a time, after every other shard is done, so the
202		* load of the gate's own shards cannot redden them (#7617). `read` returns
203		* a file's text; one it cannot read counts as no budget. Each list keeps
204		* the input's order.
205		*/
206		export function budgetLast(
207		files: readonly string[],
208		read: (file: string) => string \| undefined,
209		): { rest: string[]; budget: string[] } {
210		const rest: string[] = [];
211		const budget: string[] = [];
212		for (const file of files)
213		(callsBudget(read(file)) ? budget : rest).push(file);
214		return { rest, budget };
215		}
216
217	191	/** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
218	192	export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
219	193
		@@ -415,7 +389,7 @@ export function crashedOutside(result: Pick<ShardResult, "code" \| "output">): bo
415	389	/**
416	390	* The crashed run's output, kept in the log for whoever reads it, each line
417	391	* behind ` │ ` so nothing that reads the log (the tally, the failing-test
418		* list, the budget re-run) counts its reds beside the re-run's.
	392	* list) counts its reds beside the re-run's.
419	393	*/
420	394	export function quoteCrash(where: string, code: number, output: string): string {
421	395	const lines = output.replace(/\n$/, "").split("\n");
		@@ -74,7 +74,6 @@ import {
74	74	tally,
75	75	weigher,
76	76	besideOthers,
77		budgetLast,
78	77	failingFiles,
79	78	onlyTestFailures,
80	79	crashedOutside,
		@@ -414,23 +413,13 @@ const serialFiles = !record && count.value === 1;
414	413	// refreshes the time of every file it ran (#1797).
415	414	const timed = !record && !serialFiles && gate && tier === "full" && "none" in shardSpec;
416	415	const perFile = record \|\| measure \|\| serialFiles \|\| timed;
417		// Files that bound a speed run last and alone, after every other shard,
418		// so the gate's own load cannot redden them (#7617).
419		const ordered = budgetLast(files, (file) => {
420		try {
421		return readFileSync(file, "utf8");
422		} catch {
423		return undefined;
424		}
425		});
426	416	const slicesOf = (list: string[]): string[][] =>
427	417	list.length === 0
428	418	? []
429	419	: perFile
430	420	? heaviestFirst(list, weight).map((file) => [file])
431	421	: splitCapped(list, count.value, weight);
432		const mainSlices = slicesOf(ordered.rest);
433		const slices = [...mainSlices, ...ordered.budget.map((file) => [file])];
	422	const slices = slicesOf(files);
434	423	const work = files.reduce((sum, file) => sum + weight(file), 0);
435	424
436	425	const banner = record
		@@ -447,8 +436,6 @@ const banner = record
447	436	`expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`;
448	437	console.log(banner);
449	438	process.stderr.write(banner + "\n");
450		if (ordered.budget.length > 0)
451		console.log(`test-shards: ${ordered.budget.length} budget files last, one at a time`);
452	439
453	440	const children = new Set<Bun.Subprocess>();
454	441	for (const signal of ["SIGINT", "SIGTERM"] as const)
		@@ -683,12 +670,12 @@ const started = performance.now();
683	670	const results = [
684	671	...(await pool(
685	672	// Never more side by side than the count: capped slices outnumber it.
686		serialFiles ? 1 : Math.max(1, Math.min(count.value, mainSlices.length)),
	673	serialFiles ? 1 : Math.max(1, Math.min(count.value, slices.length)),
687	674	0,
688		mainSlices.length,
	675	slices.length,
689	676	)),
690	677	// Budget files, one at a time, once the rest are done and green.
691		...(await pool(1, mainSlices.length, slices.length)),
	678	...(await pool(1, slices.length, slices.length)),
692	679	];
693	680	const wall = (performance.now() - started) / 1000;
694	681	const ending = tally(results, wall);
		@@ -9,7 +9,8 @@ import { addTeamMember, createOrg, createTeam } from "../src/core/orgs.ts";
9	9	import { canAdmin, getRepoById } from "../src/core/repos.ts";
10	10	import { createUser, getUserById, type User } from "../src/core/users.ts";
11	11	import { id } from "../src/kernel/id.ts";
12		import { expectBudget } from "./helpers/budget.ts";
	12	import { statements } from "./helpers/statements.ts";
	13	import { expectWork } from "./helpers/work.ts";
13	14	import { quietLogs } from "./helpers/globals.ts";
14	15
15	16	quietLogs();
		@@ -124,21 +125,11 @@ test("it is exactly one statement", () => {
124	125	expect(n).toBe(1);
125	126	});
126	127
127		test("on 2,000 repositories it costs about one scan of the repository table", () => {
	128	test("on 2,000 repositories it is still one statement", async () => {
128	129	app.db.transaction(() => {
129	130	for (let i = 0; i < 2000; i++) repo(i % 3 === 0 ? pat.id : root.id, `bulk${i}`);
130	131	})();
131		const best = (run: () => unknown): number => {
132		let least = Infinity;
133		for (let n = 0; n < 5; n++) {
134		const t = performance.now();
135		run();
136		least = Math.min(least, performance.now() - t);
137		}
138		return least;
139		};
140		// The baseline is a plain scan of the same table, taken in the same run.
141		const against = best(() => app.db.query("SELECT id FROM repo").all());
142		expectBudget(best(() => adminRepoIds(app.db, pat)), { ms: 5, against });
	132	// Counted, not timed: one statement however many repositories there are.
	133	expectWork(await statements(app.db, () => void adminRepoIds(app.db, pat)), { max: 1 });
143	134	expect(adminRepoIds(app.db, pat).size).toBe(667 + 3);
144	135	});
		@@ -10,7 +10,7 @@ import { createSession } from "../src/core/sessions.ts";
10	10	import type { Repo } from "../src/core/repos.ts";
11	11	import { createUser, type User } from "../src/core/users.ts";
12	12	import { serve, type Servers } from "../src/http/server.ts";
13		import { expectBudget } from "./helpers/budget.ts";
	13	import { expectWork, requestStatements } from "./helpers/work.ts";
14	14	import { quietLogs } from "./helpers/globals.ts";
15	15
16	16	quietLogs();
		@@ -109,20 +109,15 @@ test("130 open pulls read their mergeability from the cache: the merge phase is
109	109	});
110	110	if (!pull.ok) throw new Error(JSON.stringify(pull.error));
111	111	}
112		const mergeMs = async (): Promise<number> => {
113		const res = await fetch(`${servers.url}/api/v1/repos/alice/r/pulls?limit=200`, { headers: { cookie } });
114		const items = (JSON.parse(await res.text()) as { items: unknown[] }).items;
115		expect(items.length).toBeGreaterThanOrEqual(130);
116		const m = /merge;dur=([\d.]+)/.exec(res.headers.get("server-timing") ?? "");
117		return Number(m?.[1] ?? NaN);
118		};
119		const uncached = await mergeMs();
	112	const url = `${servers.url}/api/v1/repos/alice/r/pulls?limit=200`;
	113	const res = await fetch(url, { headers: { cookie } });
	114	expect((JSON.parse(await res.text()) as { items: unknown[] }).items.length).toBeGreaterThanOrEqual(130);
120	115	const spawn = spyOn(Bun, "spawn");
121	116	try {
122		let best = Infinity;
123		for (let k = 0; k < 5; k++) best = Math.min(best, await mergeMs());
	117	// Counted, not timed: one read per page, not one per pull (130 pulls, no N+1).
	118	const reads = await requestStatements(app.db, url, { headers: { cookie } });
124	119	expect(spawn.mock.calls.filter((c) => (c[0] as string[]).includes("merge-tree")).length).toBe(0);
125		expectBudget(best, { ms: 20, against: uncached / 100 });
	120	expectWork(reads, { max: 16 });
126	121	} finally {
127	122	spawn.mockRestore();
128	123	}
		@@ -12,7 +12,6 @@ import { getRepo } from "../src/core/repos.ts";
12	12	import { createUser } from "../src/core/users.ts";
13	13	import { serve, type Servers } from "../src/http/server.ts";
14	14	import { proveAll } from "./helpers/verified.ts";
15		import { baselineMs, expectBudget } from "./helpers/budget.ts";
16	15	import { quietLogs } from "./helpers/globals.ts";
17	16	import { quiet } from "./helpers/statements.ts";
18	17
		@@ -238,22 +237,7 @@ test("following stays cheap on a diverged fork (regression guard; this pull does
238	237	expect(Math.abs((await costOf(path)) - small[path]!)).toBeLessThanOrEqual(SLACK);
239	238	});
240	239
241		async function phases(path: string): Promise<Record<string, number>> {
242		const best: Record<string, number> = {};
243		for (let i = 0; i < 6; i++) {
244		const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
245		expect(res.status).toBe(200);
246		const body = await res.text();
247		if (path.endsWith("-fork")) expect(body.replace(/\s+/g, " ")).toContain("needs a decision");
248		for (const part of (res.headers.get("server-timing") ?? "").split(", ")) {
249		const m = /^([a-z]+);dur=([\d.]+)/.exec(part);
250		if (m) best[m[1]!] = Math.min(best[m[1]!] ?? Infinity, Number(m[2]));
251		}
252		}
253		return best;
254		}
255
256		test("now stays within budget on the live shape: 300 open issues, 2,000 closed, 3,500 edges, 20 pillars", async () => {
	240	test("now costs the same statements on the live shape: 300 open issues, 2,000 closed, 3,500 edges, 20 pillars", async () => {
257	241	const repo = getRepo(app.db, "alice", "vault")!;
258	242	const author = app.db.query<{ id: string }, []>("SELECT id FROM user WHERE username = 'alice'").get()!.id;
259	243	app.db.transaction(() => {
		@@ -264,10 +248,9 @@ test("now stays within budget on the live shape: 300 open issues, 2,000 closed,
264	248	).run(`${repo.id}i${i}`, repo.id, 5000 + i, `Tell ${i}`, author);
265	249	seedLive(app.db, repo.id, author, 0);
266	250	})();
267		const against = await baselineMs(servers.url);
268		const best = await phases("/alice/vault");
269		expectBudget(best.now!, { ms: 5, against });
270		expectBudget(best.shipped!, { ms: 5, against });
	251	// Counted, not timed: 2,000 closed issues and 3,500 edges cost four statements more than the small page
	252	// (measured), not one per row; a per-row read would add thousands.
	253	expect(Math.abs((await costOf("/alice/vault")) - small["/alice/vault"]!)).toBeLessThanOrEqual(4);
271	254	});
272	255
273	256	test("the lean deferred read says what the whole graph says, and something is deferred", () => {