c272f51
@@ -10,6 +10,7 @@
PORT=8080 FORGE_APP_DATA=./data bun run src/server.ts   # run it here
bun run scripts/build.ts                                # rebuild every static/<name>.<hash>.js
bun run scripts/train.ts --iters 1000                   # retrain the bots' policy
bun run scripts/train.ts --bench                        # how fast the arena steps
bun run scripts/bench.ts                                # how fast everything is, on this machine
bun test                                                # the check every pull runs
FORGE_APP_DATA=./data bun run src/tty.ts                # the terminal interface
```
@@ -112,6 +113,20 @@
the rest.
  `src/pages/<name>.ts` that calls the shell, and a route in `src/server.ts`**
  that renders it with `scripts["<name>"]` as the `script` — no inline script,
  the CSP is `default-src 'self'`.
- `scripts/bench.ts` is the answer to "did that get slower": one table, one row
  a case, each with its unit — microseconds a world step, an arena step, a
  policy decision and an A* path, milliseconds to build the level mesh and to
  paint the atlas, and the bytes of every committed bundle — plus the world
  step as a share of the 15.6 ms tick budget. Each case builds its inputs
  before the clock starts and reports a median and a p95 over the repetitions
  it names; the whole run takes about five seconds, and it writes nothing and
  rebuilds nothing. `--json` prints the same rows to paste into a pull and
  `--case <name>` runs one. **The numbers are the machine's, not a promise**:
  they are worth something as a before and an after from the same box and
  nothing at all against another box. Where a case cannot be measured here it
  says so instead of guessing — the per-frame path is one, because the
  accumulator and the input read live inside `boot()` in `src/client/main.ts`,
  which wants a canvas and a window.
- No dependencies. Nothing is installed to run this.
It also has a terminal: the exact command is in `FORGE_APP_SSH` when the app
@@ -0,0 +1,122 @@
import { expect, test } from "bun:test";
import { CASES, derive, measure, quantile, run, table, type Case, type Row, type SizeCase, type TimeCase } from "./bench.ts";
/**
 * These tests check the harness, not the speed of anything: every case here is
 * one repetition of a body that does nothing, so the whole file runs in
 * milliseconds and cannot fail because the machine was busy.
 */
const tiny = (name: string, run: () => void = () => {}): TimeCase => ({
  kind: "time", name, what: `${name}, for the test`, unit: "µs/op", reps: 1, warmup: 0,
  prepare: () => ({ run }),
});
const absent: TimeCase = {
  kind: "time", name: "frame-path", what: "a case whose module is not here", unit: "µs/frame", reps: 1, warmup: 0,
  prepare: () => ({ missing: "no headless module" }),
};
const size: SizeCase = {
  kind: "size", name: "bundle:app", what: "a committed bundle", unit: "bytes",
  measure: () => ({ bytes: 67479, note: "static/app.deadbeef00.js" }),
};
const TINY: Case[] = [tiny("noop"), tiny("add", () => void (1 + 1)), absent, size];
test("one tiny case, one repetition, and the median is a real number", () => {
  const row = measure(tiny("noop"), 1);
  expect(row.reps).toBe(1);
  expect(row.median).toBeGreaterThanOrEqual(0);
  expect(row.p95).toBeGreaterThanOrEqual(row.median as number);
  expect(Number.isFinite(row.median as number)).toBe(true);
});
test("the table has a row per case, with the unit stated", () => {
  const { code, out } = run([], TINY);
  expect(code).toBe(0);
  const lines = out.split("\n");
  const header = lines[0] as string;
  expect(header.split(/\s+/)).toEqual(["case", "unit", "median", "p95", "reps"]);
  for (const c of TINY) expect(lines.some((l) => l.startsWith(`${c.name} `) && l.includes(c.unit))).toBe(true);
  // The rows are the first block: header plus one line each, then a blank.
  expect(lines.indexOf("")).toBe(TINY.length + 1);
});
test("--json parses, and carries the same rows the table showed", () => {
  const { code, out } = run(["--json"], TINY);
  expect(code).toBe(0);
  const parsed = JSON.parse(out) as { seconds: number; rows: Row[]; derived: { name: string }[] };
  expect(parsed.rows.map((r) => r.name)).toEqual(TINY.map((c) => c.name));
  expect(parsed.seconds).toBeGreaterThanOrEqual(0);
  expect(parsed.derived).toHaveLength(2);
  const noop = parsed.rows[0] as Row;
  expect(noop.unit).toBe("µs/op");
  expect(noop.median).toBeGreaterThanOrEqual(0);
});
test("a case whose module is not here says so instead of guessing a number", () => {
  const rows = [measure(absent), measure({ ...size, measure: () => ({ missing: "no committed bundle" }) })];
  for (const row of rows) {
    expect(row.median).toBeNull();
    expect(row.p95).toBeNull();
    expect(row.reps).toBeNull();
  }
  expect(rows[0]?.note).toBe("no headless module");
  expect(table(rows)).toContain("—");
});
test("a byte size is one reading, not a population", () => {
  const row = measure(size);
  expect(row).toMatchObject({ unit: "bytes", reps: 1, median: 67479, p95: null });
  expect(table([row])).toContain("67479");
});
test("--case runs one case, and an unknown name exits non-zero with the list", () => {
  const one = run(["--case", "add"], TINY);
  expect(one.code).toBe(0);
  expect(one.out).toContain("add");
  expect(one.out).not.toContain("noop");
  for (const name of ["bogus", "../../etc/passwd", "world-step; rm -rf /"]) {
    const bad = run(["--case", name], TINY);
    expect(bad.code).not.toBe(0);
    expect(bad.out).toBe("");
    for (const c of TINY) expect(bad.err).toContain(c.name);
  }
  expect(run(["--case"], TINY).code).not.toBe(0);
  expect(run(["--rm", "-rf"], TINY).code).not.toBe(0);
  expect(run(["--rm", "-rf"], TINY).err).toContain("unknown argument");
});
test("the derived numbers are the share of the budget, by hand", () => {
  // A tenth of a 15.625 ms tick, and the same step in a 16.7 ms frame.
  const rows: Row[] = [
    { name: "world-step", what: "", unit: "µs/step", reps: 10, median: 1562.5, p95: 1600 },
    { name: "frame-path", what: "", unit: "µs/frame", reps: null, median: null, p95: null, note: "no headless module" },
  ];
  const [tick, frame] = derive(rows);
  expect(tick?.percent).toBeCloseTo(10, 6);
  expect(frame?.percent).toBeNull();
  // 16.7 / 15.625 = 1.07 steps a frame, so 1.5625 ms a step is 1.67 ms of it.
  expect(frame?.note).toContain("1.07 world steps");
  expect(frame?.note).toContain("1.67 ms");
  expect(frame?.note).toContain("not measured");
});
test("nearest-rank, so one repetition reports itself", () => {
  expect(quantile(Float64Array.from([7]), 0.5)).toBe(7);
  expect(quantile(Float64Array.from([1, 2, 3, 4]), 0.5)).toBe(2);
  expect(quantile(Float64Array.from([1, 2, 3, 4]), 0.95)).toBe(4);
  expect(quantile(new Float64Array(0), 0.5)).toBe(0);
});
test("the shipped cases are the ones a reviewer asks for, each with a unit and a note", () => {
  expect(CASES.map((c) => c.name)).toEqual([
    "world-step", "arena-step", "policy-decision", "level-mesh", "atlas-paint", "path", "frame-path", "bundle:app",
  ]);
  for (const c of CASES) {
    expect(c.unit).toMatch(/^(µs|ms|bytes)/);
    expect(c.what.length).toBeGreaterThan(20);
    if (c.kind === "time") expect(c.reps).toBeGreaterThan(0);
  }
});
@@ -0,0 +1,393 @@
/**
 * How fast is this? One command, one table.
 *
 *   bun run scripts/bench.ts                    # every case
 *   bun run scripts/bench.ts --json             # the same rows as JSON
 *   bun run scripts/bench.ts --case world-step  # one case
 *
 * Every case builds its inputs before the clock starts and times one
 * repetition at a time with `Bun.nanoseconds()`, so what comes out is a median
 * and a p95 over a population rather than one `Date.now()` difference. The
 * numbers are this machine's and nothing else: compare a before and an after
 * from the same box, and never an absolute against another machine's.
 *
 * Nothing here writes a file, rebuilds a bundle or takes a path from the
 * arguments, and the timing loop allocates nothing it can avoid.
 */
import { existsSync, readdirSync, statSync } from "node:fs";
import { join } from "node:path";
import { ENTRIES, isBundleOf, STATIC_DIR } from "./build.ts";
import { decode, HEADS, type Intent } from "../src/client/action.ts";
import * as map from "../src/client/map.ts";
import { SHORT } from "../src/client/match.ts";
import { materialTextures } from "../src/client/materials.ts";
import { levelMesh } from "../src/client/models.ts";
import { OBS_SIZE } from "../src/client/observe.ts";
import { forward, greedy, newActivations, unpack } from "../src/client/policy.ts";
import { PACKED } from "../src/client/weights.ts";
import { NO_INPUT, createWorld, update, type Input } from "../src/client/world.ts";
import { createArena, observeLearner, reset, step, SKIP, TICK } from "../src/rl/arena.ts";
// --- the shape of a case --------------------------------------------------
/** What a case hands the clock: something to run, or why it cannot be run. */
export type Prepared = { run: () => void } | { missing: string };
export type TimeCase = {
  kind: "time";
  name: string;
  /** One line, printed under the table: what exactly is inside the clock. */
  what: string;
  unit: string;
  /** Timed repetitions, and the untimed ones that come first. */
  reps: number;
  warmup: number;
  /** Builds the inputs. Everything it allocates is allocated before timing. */
  prepare: () => Prepared;
};
export type SizeCase = {
  kind: "size";
  name: string;
  what: string;
  unit: "bytes";
  measure: () => { bytes: number; note?: string } | { missing: string };
};
export type Case = TimeCase | SizeCase;
/** One line of the table, and one object in `--json`. */
export type Row = {
  name: string;
  what: string;
  unit: string;
  reps: number | null;
  median: number | null;
  p95: number | null;
  note?: string;
};
// --- the clock ------------------------------------------------------------
/** How many nanoseconds are in one of `unit`. Anything not in ms is µs. */
const scaleOf = (unit: string): number => (unit.startsWith("ms") ? 1e6 : 1e3);
/** Nearest-rank: the smallest sample at or above `q` of the population. */
export function quantile(sorted: Float64Array, q: number): number {
  if (sorted.length === 0) return 0;
  const at = Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1));
  return sorted[at] as number;
}
/**
 * Run one case and report it. `reps` overrides the case's own count and cuts
 * the warm-up down with it, which is how the test runs a case exactly once
 * without waiting for a warm-up meant for a measurement.
 */
export function measure(c: Case, reps?: number): Row {
  if (c.kind === "size") {
    const got = c.measure();
    if ("missing" in got)
      return { name: c.name, what: c.what, unit: c.unit, reps: null, median: null, p95: null, note: got.missing };
    return {
      name: c.name, what: c.what, unit: c.unit,
      reps: 1, median: got.bytes, p95: null, ...(got.note === undefined ? {} : { note: got.note }),
    };
  }
  const prepared = c.prepare();
  if ("missing" in prepared)
    return { name: c.name, what: c.what, unit: c.unit, reps: null, median: null, p95: null, note: prepared.missing };
  const { run } = prepared;
  const n = Math.max(1, Math.floor(reps ?? c.reps));
  const warmup = reps === undefined ? c.warmup : Math.min(c.warmup, 1);
  for (let i = 0; i < warmup; i++) run();
  const samples = new Float64Array(n);
  for (let i = 0; i < n; i++) {
    const started = Bun.nanoseconds();
    run();
    samples[i] = Bun.nanoseconds() - started;
  }
  samples.sort();
  const scale = scaleOf(c.unit);
  return {
    name: c.name, what: c.what, unit: c.unit, reps: n,
    median: quantile(samples, 0.5) / scale,
    p95: quantile(samples, 0.95) / scale,
  };
}
// --- the cases ------------------------------------------------------------
/** 64 Hz: one tick is 15.625 ms of wall clock, and that is the whole budget. */
const TICK_BUDGET_US = TICK * 1e6;
/** One frame at 60 Hz. The browser will not wait longer than this for us. */
const FRAME_BUDGET_US = 16_700;
const worldStep: TimeCase = {
  kind: "time",
  name: "world-step",
  what: `src/client/world.ts update() at 1/64 with ten players, nine of them bots thinking for themselves, in a live round of de_forge; the person's input is NO_INPUT. The warm-up steps past the ${SHORT.freezeTime}-second freeze time.`,
  unit: "µs/step",
  reps: 1500,
  warmup: Math.ceil((SHORT.freezeTime + 1) * 64),
  prepare: () => {
    const world = createWorld(SHORT, "bench", "ct", 5, 20260917);
    const input: Input = { ...NO_INPUT };
    return { run: () => update(world, TICK, input) };
  },
};
const arenaStep: TimeCase = {
  kind: "time",
  name: "arena-step",
  what: `src/rl/arena.ts step() — the same function scripts/train.ts --bench times — holding one decision for SKIP=${SKIP} ticks against the scripted opponent, resetting the duel when it ends.`,
  unit: "µs/step",
  reps: 3000,
  warmup: 300,
  prepare: () => {
    const arena = createArena(20260917);
    // A decision that moves and shoots, decoded once: a still intent would
    // measure a player standing in a corridor doing nothing.
    const actions = Int32Array.from([5, 3, 2, 1, 1, 0]);
    const intent: Intent = decode(actions);
    return {
      run: () => {
        if (step(arena, intent).done) reset(arena);
      },
    };
  },
};
const policyDecision: TimeCase = {
  kind: "time",
  name: "policy-decision",
  what: "src/client/policy.ts forward() over the shipped int8 weights, dequantised once before the clock, plus greedy() over the six heads. The observation is prepared before the clock, so this is the network and nothing else.",
  unit: "µs/decision",
  reps: 20000,
  warmup: 2000,
  prepare: () => {
    const policy = unpack(PACKED);
    const obs = new Float32Array(OBS_SIZE);
    observeLearner(createArena(20260917), obs);
    const act = newActivations(policy.shape);
    const actions = new Int32Array(HEADS.length);
    return {
      run: () => {
        forward(policy, obs, act);
        greedy(policy.shape, act.logits, actions);
      },
    };
  },
};
const levelMeshCase: TimeCase = {
  kind: "time",
  name: "level-mesh",
  what: `src/client/models.ts levelMesh() over de_forge's ${map.boxes.length} boxes, through the engine's box builder: once at start-up, before the first frame.`,
  unit: "ms/build",
  reps: 200,
  warmup: 20,
  prepare: () => {
    const boxes = map.boxes;
    return { run: () => void levelMesh(boxes) };
  },
};
const atlasCase: TimeCase = {
  kind: "time",
  name: "atlas-paint",
  what: "src/client/materials.ts materialTextures(): every material painted over hashed value noise by the engine's atlas painter. Once at start-up, and it is the slowest thing there.",
  unit: "ms/paint",
  reps: 30,
  warmup: 3,
  prepare: () => ({ run: () => void materialTextures() }),
};
const pathCase: TimeCase = {
  kind: "time",
  name: "path",
  what: `src/engine/grid.ts findPath() across de_forge (${map.GRID_W}×${map.GRID_H} cells), T spawn to the A site — about as far as a bot ever asks for.`,
  unit: "µs/path",
  reps: 800,
  warmup: 80,
  prepare: () => {
    const from = map.spawns.t[0];
    const to = map.sites.A;
    if (from === undefined) return { missing: "de_forge has no T spawn cell" };
    return { run: () => void map.findPath(from, to) };
  },
};
/**
 * The per-frame engine path. It is not measurable from here: the accumulator
 * and the input read live inside `boot()` in src/client/main.ts, over
 * `createControls` from src/client/input.ts, which wants a canvas and a
 * window. Saying so is the honest row; a number invented for it would be the
 * dishonest one. A headless module the page calls would make it measurable.
 */
const framePath: TimeCase = {
  kind: "time",
  name: "frame-path",
  what: "the fixed-step accumulator plus reading input, once a frame.",
  unit: "µs/frame",
  reps: 1,
  warmup: 0,
  prepare: () => ({
    missing:
      "no headless module: the accumulator and the input read live inside boot() in src/client/main.ts, and src/client/input.ts needs a canvas and a window.",
  }),
};
/** One row per committed bundle, read from static/ — nothing is rebuilt. */
export function bundleCases(): SizeCase[] {
  return ENTRIES.map(({ name }) => ({
    kind: "size" as const,
    name: `bundle:${name}`,
    what: `static/${name}.<hash>.js as committed, the bytes the browser downloads before it can start (not gzipped — what the wire costs depends on the proxy).`,
    unit: "bytes" as const,
    measure: () => {
      const files = existsSync(STATIC_DIR) ? readdirSync(STATIC_DIR).filter((f) => isBundleOf(name, f)) : [];
      const file = files[0];
      if (file === undefined) return { missing: `no committed bundle for ${name}: run bun run scripts/build.ts` };
      if (files.length > 1) return { missing: `${files.length} bundles for ${name} in static/: one of them is stale` };
      return { bytes: statSync(join(STATIC_DIR, file)).size, note: `static/${file}` };
    },
  }));
}
export const CASES: Case[] = [
  worldStep, arenaStep, policyDecision, levelMeshCase, atlasCase, pathCase, framePath, ...bundleCases(),
];
// --- the two numbers that matter ------------------------------------------
export type Derived = { name: string; what: string; percent: number | null; note?: string };
/** The share of its budget each hot path costs, from the rows just measured. */
export function derive(rows: Row[]): Derived[] {
  const at = (name: string): Row | undefined => rows.find((r) => r.name === name);
  const world = at("world-step");
  const frame = at("frame-path");
  const out: Derived[] = [];
  out.push({
    name: "world-step / tick budget",
    what: `one world step as a share of the ${(TICK_BUDGET_US / 1000).toFixed(1)} ms a 64 Hz tick may take`,
    percent: world?.median === undefined || world.median === null ? null : (world.median / TICK_BUDGET_US) * 100,
    ...(world === undefined ? { note: "world-step was not run" } : {}),
  });
  // The frame path is not measured (see framePath). What can be said without
  // a DOM is what the accumulator runs — the ticks themselves — in
  // milliseconds of the frame, and it is labelled as derived, not measured.
  // As a percentage it would be the row above by construction, since the tick
  // rate is fixed: 16.7 ms holds FRAME/TICK steps whatever a step costs.
  const ticksPerFrame = FRAME_BUDGET_US / TICK_BUDGET_US;
  const simulationMs =
    world?.median === undefined || world.median === null ? null : (world.median * ticksPerFrame) / 1000;
  out.push({
    name: "frame path / 16.7 ms frame",
    what: "the fixed-step accumulator plus reading input, as a share of one 60 Hz frame",
    percent: frame?.median ?? null,
    note:
      frame === undefined
        ? "frame-path was not run"
        : `${frame.note ?? "not measured"}` +
          (simulationMs === null
            ? ""
            : ` What the accumulator runs can be derived: ${ticksPerFrame.toFixed(2)} world steps fall in a 16.7 ms frame, so the simulation alone is ${simulationMs.toFixed(2)} ms of it — derived from world-step, not measured, and it counts none of the input read, the accumulator arithmetic or the draw.`),
  });
  return out;
}
// --- printing -------------------------------------------------------------
const pad = (text: string, width: number): string => text.padEnd(width);
const padLeft = (text: string, width: number): string => text.padStart(width);
/** Enough digits to compare two runs, and not one more. */
function show(row: Row, value: number | null): string {
  if (value === null) return "—";
  if (row.unit === "bytes") return String(Math.round(value));
  return value >= 100 ? value.toFixed(1) : value >= 1 ? value.toFixed(2) : value.toFixed(3);
}
export function table(rows: Row[]): string {
  const head = ["case", "unit", "median", "p95", "reps"];
  const cells = rows.map((r) => [r.name, r.unit, show(r, r.median), show(r, r.p95), r.reps === null ? "—" : String(r.reps)]);
  const width = head.map((h, i) => Math.max(h.length, ...cells.map((c) => (c[i] as string).length)));
  const line = (c: string[]): string =>
    `${pad(c[0] as string, width[0] as number)}  ${pad(c[1] as string, width[1] as number)}  ` +
    `${padLeft(c[2] as string, width[2] as number)}  ${padLeft(c[3] as string, width[3] as number)}  ` +
    `${padLeft(c[4] as string, width[4] as number)}`;
  return [line(head), ...cells.map(line)].join("\n");
}
function report(rows: Row[], derived: Derived[], seconds: number): string {
  const out = [table(rows), ""];
  for (const d of derived) {
    const value = d.percent === null ? "—" : `${d.percent.toFixed(1)}%`;
    out.push(`${d.name}: ${value}  (${d.what})${d.note === undefined ? "" : `\n  ${d.note}`}`);
  }
  out.push("", "What is in each case:");
  for (const r of rows) out.push(`  ${r.name} — ${r.what}${r.note === undefined ? "" : `\n      ${r.note}`}`);
  out.push(
    "",
    `${rows.length} case${rows.length === 1 ? "" : "s"} in ${seconds.toFixed(1)}s. Median and p95 are nearest-rank over the repetitions shown;`,
    "inputs are built before the clock. These are this machine's numbers, not a promise: compare a",
    "before and an after from the same box rather than trusting an absolute.",
  );
  return out.join("\n");
}
// --- the command ----------------------------------------------------------
export type Run = { code: number; out: string; err: string };
const USAGE = "usage: bun run scripts/bench.ts [--json] [--case <name>]";
/**
 * Parse the arguments, run what they asked for and hand back what to print.
 * Nothing here takes a path or a command from `argv`: `--case` is matched
 * against the names of the cases and everything else is rejected.
 */
export function run(argv: string[], cases: Case[] = CASES): Run {
  let json = false;
  let only: string | null = null;
  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];
    if (arg === "--json") json = true;
    else if (arg === "--case") {
      const name = argv[++i];
      if (name === undefined) return { code: 2, out: "", err: `--case wants a name\n${USAGE}` };
      only = name;
    } else return { code: 2, out: "", err: `unknown argument ${JSON.stringify(arg)}\n${USAGE}` };
  }
  let chosen = cases;
  if (only !== null) {
    chosen = cases.filter((c) => c.name === only);
    if (chosen.length === 0)
      return {
        code: 1,
        out: "",
        err: `no case named ${JSON.stringify(only)}. The cases are:\n${cases.map((c) => `  ${c.name}`).join("\n")}`,
      };
  }
  const started = Bun.nanoseconds();
  const rows = chosen.map((c) => measure(c));
  const seconds = (Bun.nanoseconds() - started) / 1e9;
  const derived = derive(rows);
  if (json) return { code: 0, out: JSON.stringify({ seconds: Number(seconds.toFixed(3)), rows, derived }, null, 2), err: "" };
  return { code: 0, out: report(rows, derived, seconds), err: "" };
}
if (import.meta.main) {
  const { code, out, err } = run(process.argv.slice(2));
  if (out !== "") console.log(out);
  if (err !== "") console.error(err);
  process.exit(code);
}
		@@ -10,6 +10,7 @@ PORT=8080 FORGE_APP_DATA=./data bun run src/server.ts # run it here
10	10	bun run scripts/build.ts # rebuild every static/<name>.<hash>.js
11	11	bun run scripts/train.ts --iters 1000 # retrain the bots' policy
12	12	bun run scripts/train.ts --bench # how fast the arena steps
	13	bun run scripts/bench.ts # how fast everything is, on this machine
13	14	bun test # the check every pull runs
14	15	FORGE_APP_DATA=./data bun run src/tty.ts # the terminal interface
15	16	```
		@@ -112,6 +113,20 @@ the rest.
112	113	`src/pages/<name>.ts` that calls the shell, and a route in `src/server.ts`**
113	114	that renders it with `scripts["<name>"]` as the `script` — no inline script,
114	115	the CSP is `default-src 'self'`.
	116	- `scripts/bench.ts` is the answer to "did that get slower": one table, one row
	117	a case, each with its unit — microseconds a world step, an arena step, a
	118	policy decision and an A* path, milliseconds to build the level mesh and to
	119	paint the atlas, and the bytes of every committed bundle — plus the world
	120	step as a share of the 15.6 ms tick budget. Each case builds its inputs
	121	before the clock starts and reports a median and a p95 over the repetitions
	122	it names; the whole run takes about five seconds, and it writes nothing and
	123	rebuilds nothing. `--json` prints the same rows to paste into a pull and
	124	`--case <name>` runs one. The numbers are the machine's, not a promise:
	125	they are worth something as a before and an after from the same box and
	126	nothing at all against another box. Where a case cannot be measured here it
	127	says so instead of guessing — the per-frame path is one, because the
	128	accumulator and the input read live inside `boot()` in `src/client/main.ts`,
	129	which wants a canvas and a window.
115	130	- No dependencies. Nothing is installed to run this.
116	131
117	132	It also has a terminal: the exact command is in `FORGE_APP_SSH` when the app
		@@ -0,0 +1,122 @@
	1	import { expect, test } from "bun:test";
	2	import { CASES, derive, measure, quantile, run, table, type Case, type Row, type SizeCase, type TimeCase } from "./bench.ts";
	3
	4	/**
	5	* These tests check the harness, not the speed of anything: every case here is
	6	* one repetition of a body that does nothing, so the whole file runs in
	7	* milliseconds and cannot fail because the machine was busy.
	8	*/
	9	const tiny = (name: string, run: () => void = () => {}): TimeCase => ({
	10	kind: "time", name, what: `${name}, for the test`, unit: "µs/op", reps: 1, warmup: 0,
	11	prepare: () => ({ run }),
	12	});
	13
	14	const absent: TimeCase = {
	15	kind: "time", name: "frame-path", what: "a case whose module is not here", unit: "µs/frame", reps: 1, warmup: 0,
	16	prepare: () => ({ missing: "no headless module" }),
	17	};
	18
	19	const size: SizeCase = {
	20	kind: "size", name: "bundle:app", what: "a committed bundle", unit: "bytes",
	21	measure: () => ({ bytes: 67479, note: "static/app.deadbeef00.js" }),
	22	};
	23
	24	const TINY: Case[] = [tiny("noop"), tiny("add", () => void (1 + 1)), absent, size];
	25
	26	test("one tiny case, one repetition, and the median is a real number", () => {
	27	const row = measure(tiny("noop"), 1);
	28	expect(row.reps).toBe(1);
	29	expect(row.median).toBeGreaterThanOrEqual(0);
	30	expect(row.p95).toBeGreaterThanOrEqual(row.median as number);
	31	expect(Number.isFinite(row.median as number)).toBe(true);
	32	});
	33
	34	test("the table has a row per case, with the unit stated", () => {
	35	const { code, out } = run([], TINY);
	36	expect(code).toBe(0);
	37	const lines = out.split("\n");
	38	const header = lines[0] as string;
	39	expect(header.split(/\s+/)).toEqual(["case", "unit", "median", "p95", "reps"]);
	40	for (const c of TINY) expect(lines.some((l) => l.startsWith(`${c.name} `) && l.includes(c.unit))).toBe(true);
	41	// The rows are the first block: header plus one line each, then a blank.
	42	expect(lines.indexOf("")).toBe(TINY.length + 1);
	43	});
	44
	45	test("--json parses, and carries the same rows the table showed", () => {
	46	const { code, out } = run(["--json"], TINY);
	47	expect(code).toBe(0);
	48	const parsed = JSON.parse(out) as { seconds: number; rows: Row[]; derived: { name: string }[] };
	49	expect(parsed.rows.map((r) => r.name)).toEqual(TINY.map((c) => c.name));
	50	expect(parsed.seconds).toBeGreaterThanOrEqual(0);
	51	expect(parsed.derived).toHaveLength(2);
	52	const noop = parsed.rows[0] as Row;
	53	expect(noop.unit).toBe("µs/op");
	54	expect(noop.median).toBeGreaterThanOrEqual(0);
	55	});
	56
	57	test("a case whose module is not here says so instead of guessing a number", () => {
	58	const rows = [measure(absent), measure({ ...size, measure: () => ({ missing: "no committed bundle" }) })];
	59	for (const row of rows) {
	60	expect(row.median).toBeNull();
	61	expect(row.p95).toBeNull();
	62	expect(row.reps).toBeNull();
	63	}
	64	expect(rows[0]?.note).toBe("no headless module");
	65	expect(table(rows)).toContain("—");
	66	});
	67
	68	test("a byte size is one reading, not a population", () => {
	69	const row = measure(size);
	70	expect(row).toMatchObject({ unit: "bytes", reps: 1, median: 67479, p95: null });
	71	expect(table([row])).toContain("67479");
	72	});
	73
	74	test("--case runs one case, and an unknown name exits non-zero with the list", () => {
	75	const one = run(["--case", "add"], TINY);
	76	expect(one.code).toBe(0);
	77	expect(one.out).toContain("add");
	78	expect(one.out).not.toContain("noop");
	79
	80	for (const name of ["bogus", "../../etc/passwd", "world-step; rm -rf /"]) {
	81	const bad = run(["--case", name], TINY);
	82	expect(bad.code).not.toBe(0);
	83	expect(bad.out).toBe("");
	84	for (const c of TINY) expect(bad.err).toContain(c.name);
	85	}
	86	expect(run(["--case"], TINY).code).not.toBe(0);
	87	expect(run(["--rm", "-rf"], TINY).code).not.toBe(0);
	88	expect(run(["--rm", "-rf"], TINY).err).toContain("unknown argument");
	89	});
	90
	91	test("the derived numbers are the share of the budget, by hand", () => {
	92	// A tenth of a 15.625 ms tick, and the same step in a 16.7 ms frame.
	93	const rows: Row[] = [
	94	{ name: "world-step", what: "", unit: "µs/step", reps: 10, median: 1562.5, p95: 1600 },
	95	{ name: "frame-path", what: "", unit: "µs/frame", reps: null, median: null, p95: null, note: "no headless module" },
	96	];
	97	const [tick, frame] = derive(rows);
	98	expect(tick?.percent).toBeCloseTo(10, 6);
	99	expect(frame?.percent).toBeNull();
	100	// 16.7 / 15.625 = 1.07 steps a frame, so 1.5625 ms a step is 1.67 ms of it.
	101	expect(frame?.note).toContain("1.07 world steps");
	102	expect(frame?.note).toContain("1.67 ms");
	103	expect(frame?.note).toContain("not measured");
	104	});
	105
	106	test("nearest-rank, so one repetition reports itself", () => {
	107	expect(quantile(Float64Array.from([7]), 0.5)).toBe(7);
	108	expect(quantile(Float64Array.from([1, 2, 3, 4]), 0.5)).toBe(2);
	109	expect(quantile(Float64Array.from([1, 2, 3, 4]), 0.95)).toBe(4);
	110	expect(quantile(new Float64Array(0), 0.5)).toBe(0);
	111	});
	112
	113	test("the shipped cases are the ones a reviewer asks for, each with a unit and a note", () => {
	114	expect(CASES.map((c) => c.name)).toEqual([
	115	"world-step", "arena-step", "policy-decision", "level-mesh", "atlas-paint", "path", "frame-path", "bundle:app",
	116	]);
	117	for (const c of CASES) {
	118	expect(c.unit).toMatch(/^(µs\|ms\|bytes)/);
	119	expect(c.what.length).toBeGreaterThan(20);
	120	if (c.kind === "time") expect(c.reps).toBeGreaterThan(0);
	121	}
	122	});
		@@ -0,0 +1,393 @@
	1	/**
	2	* How fast is this? One command, one table.
	3	*
	4	* bun run scripts/bench.ts # every case
	5	* bun run scripts/bench.ts --json # the same rows as JSON
	6	* bun run scripts/bench.ts --case world-step # one case
	7	*
	8	* Every case builds its inputs before the clock starts and times one
	9	* repetition at a time with `Bun.nanoseconds()`, so what comes out is a median
	10	* and a p95 over a population rather than one `Date.now()` difference. The
	11	* numbers are this machine's and nothing else: compare a before and an after
	12	* from the same box, and never an absolute against another machine's.
	13	*
	14	* Nothing here writes a file, rebuilds a bundle or takes a path from the
	15	* arguments, and the timing loop allocates nothing it can avoid.
	16	*/
	17
	18	import { existsSync, readdirSync, statSync } from "node:fs";
	19	import { join } from "node:path";
	20	import { ENTRIES, isBundleOf, STATIC_DIR } from "./build.ts";
	21	import { decode, HEADS, type Intent } from "../src/client/action.ts";
	22	import * as map from "../src/client/map.ts";
	23	import { SHORT } from "../src/client/match.ts";
	24	import { materialTextures } from "../src/client/materials.ts";
	25	import { levelMesh } from "../src/client/models.ts";
	26	import { OBS_SIZE } from "../src/client/observe.ts";
	27	import { forward, greedy, newActivations, unpack } from "../src/client/policy.ts";
	28	import { PACKED } from "../src/client/weights.ts";
	29	import { NO_INPUT, createWorld, update, type Input } from "../src/client/world.ts";
	30	import { createArena, observeLearner, reset, step, SKIP, TICK } from "../src/rl/arena.ts";
	31
	32	// --- the shape of a case --------------------------------------------------
	33
	34	/** What a case hands the clock: something to run, or why it cannot be run. */
	35	export type Prepared = { run: () => void } \| { missing: string };
	36
	37	export type TimeCase = {
	38	kind: "time";
	39	name: string;
	40	/** One line, printed under the table: what exactly is inside the clock. */
	41	what: string;
	42	unit: string;
	43	/** Timed repetitions, and the untimed ones that come first. */
	44	reps: number;
	45	warmup: number;
	46	/** Builds the inputs. Everything it allocates is allocated before timing. */
	47	prepare: () => Prepared;
	48	};
	49
	50	export type SizeCase = {
	51	kind: "size";
	52	name: string;
	53	what: string;
	54	unit: "bytes";
	55	measure: () => { bytes: number; note?: string } \| { missing: string };
	56	};
	57
	58	export type Case = TimeCase \| SizeCase;
	59
	60	/** One line of the table, and one object in `--json`. */
	61	export type Row = {
	62	name: string;
	63	what: string;
	64	unit: string;
	65	reps: number \| null;
	66	median: number \| null;
	67	p95: number \| null;
	68	note?: string;
	69	};
	70
	71	// --- the clock ------------------------------------------------------------
	72
	73	/** How many nanoseconds are in one of `unit`. Anything not in ms is µs. */
	74	const scaleOf = (unit: string): number => (unit.startsWith("ms") ? 1e6 : 1e3);
	75
	76	/** Nearest-rank: the smallest sample at or above `q` of the population. */
	77	export function quantile(sorted: Float64Array, q: number): number {
	78	if (sorted.length === 0) return 0;
	79	const at = Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1));
	80	return sorted[at] as number;
	81	}
	82
	83	/**
	84	* Run one case and report it. `reps` overrides the case's own count and cuts
	85	* the warm-up down with it, which is how the test runs a case exactly once
	86	* without waiting for a warm-up meant for a measurement.
	87	*/
	88	export function measure(c: Case, reps?: number): Row {
	89	if (c.kind === "size") {
	90	const got = c.measure();
	91	if ("missing" in got)
	92	return { name: c.name, what: c.what, unit: c.unit, reps: null, median: null, p95: null, note: got.missing };
	93	return {
	94	name: c.name, what: c.what, unit: c.unit,
	95	reps: 1, median: got.bytes, p95: null, ...(got.note === undefined ? {} : { note: got.note }),
	96	};
	97	}
	98	const prepared = c.prepare();
	99	if ("missing" in prepared)
	100	return { name: c.name, what: c.what, unit: c.unit, reps: null, median: null, p95: null, note: prepared.missing };
	101
	102	const { run } = prepared;
	103	const n = Math.max(1, Math.floor(reps ?? c.reps));
	104	const warmup = reps === undefined ? c.warmup : Math.min(c.warmup, 1);
	105	for (let i = 0; i < warmup; i++) run();
	106	const samples = new Float64Array(n);
	107	for (let i = 0; i < n; i++) {
	108	const started = Bun.nanoseconds();
	109	run();
	110	samples[i] = Bun.nanoseconds() - started;
	111	}
	112	samples.sort();
	113	const scale = scaleOf(c.unit);
	114	return {
	115	name: c.name, what: c.what, unit: c.unit, reps: n,
	116	median: quantile(samples, 0.5) / scale,
	117	p95: quantile(samples, 0.95) / scale,
	118	};
	119	}
	120
	121	// --- the cases ------------------------------------------------------------
	122
	123	/** 64 Hz: one tick is 15.625 ms of wall clock, and that is the whole budget. */
	124	const TICK_BUDGET_US = TICK * 1e6;
	125	/** One frame at 60 Hz. The browser will not wait longer than this for us. */
	126	const FRAME_BUDGET_US = 16_700;
	127
	128	const worldStep: TimeCase = {
	129	kind: "time",
	130	name: "world-step",
	131	what: `src/client/world.ts update() at 1/64 with ten players, nine of them bots thinking for themselves, in a live round of de_forge; the person's input is NO_INPUT. The warm-up steps past the ${SHORT.freezeTime}-second freeze time.`,
	132	unit: "µs/step",
	133	reps: 1500,
	134	warmup: Math.ceil((SHORT.freezeTime + 1) * 64),
	135	prepare: () => {
	136	const world = createWorld(SHORT, "bench", "ct", 5, 20260917);
	137	const input: Input = { ...NO_INPUT };
	138	return { run: () => update(world, TICK, input) };
	139	},
	140	};
	141
	142	const arenaStep: TimeCase = {
	143	kind: "time",
	144	name: "arena-step",
	145	what: `src/rl/arena.ts step() — the same function scripts/train.ts --bench times — holding one decision for SKIP=${SKIP} ticks against the scripted opponent, resetting the duel when it ends.`,
	146	unit: "µs/step",
	147	reps: 3000,
	148	warmup: 300,
	149	prepare: () => {
	150	const arena = createArena(20260917);
	151	// A decision that moves and shoots, decoded once: a still intent would
	152	// measure a player standing in a corridor doing nothing.
	153	const actions = Int32Array.from([5, 3, 2, 1, 1, 0]);
	154	const intent: Intent = decode(actions);
	155	return {
	156	run: () => {
	157	if (step(arena, intent).done) reset(arena);
	158	},
	159	};
	160	},
	161	};
	162
	163	const policyDecision: TimeCase = {
	164	kind: "time",
	165	name: "policy-decision",
	166	what: "src/client/policy.ts forward() over the shipped int8 weights, dequantised once before the clock, plus greedy() over the six heads. The observation is prepared before the clock, so this is the network and nothing else.",
	167	unit: "µs/decision",
	168	reps: 20000,
	169	warmup: 2000,
	170	prepare: () => {
	171	const policy = unpack(PACKED);
	172	const obs = new Float32Array(OBS_SIZE);
	173	observeLearner(createArena(20260917), obs);
	174	const act = newActivations(policy.shape);
	175	const actions = new Int32Array(HEADS.length);
	176	return {
	177	run: () => {
	178	forward(policy, obs, act);
	179	greedy(policy.shape, act.logits, actions);
	180	},
	181	};
	182	},
	183	};
	184
	185	const levelMeshCase: TimeCase = {
	186	kind: "time",
	187	name: "level-mesh",
	188	what: `src/client/models.ts levelMesh() over de_forge's ${map.boxes.length} boxes, through the engine's box builder: once at start-up, before the first frame.`,
	189	unit: "ms/build",
	190	reps: 200,
	191	warmup: 20,
	192	prepare: () => {
	193	const boxes = map.boxes;
	194	return { run: () => void levelMesh(boxes) };
	195	},
	196	};
	197
	198	const atlasCase: TimeCase = {
	199	kind: "time",
	200	name: "atlas-paint",
	201	what: "src/client/materials.ts materialTextures(): every material painted over hashed value noise by the engine's atlas painter. Once at start-up, and it is the slowest thing there.",
	202	unit: "ms/paint",
	203	reps: 30,
	204	warmup: 3,
	205	prepare: () => ({ run: () => void materialTextures() }),
	206	};
	207
	208	const pathCase: TimeCase = {
	209	kind: "time",
	210	name: "path",
	211	what: `src/engine/grid.ts findPath() across de_forge (${map.GRID_W}×${map.GRID_H} cells), T spawn to the A site — about as far as a bot ever asks for.`,
	212	unit: "µs/path",
	213	reps: 800,
	214	warmup: 80,
	215	prepare: () => {
	216	const from = map.spawns.t[0];
	217	const to = map.sites.A;
	218	if (from === undefined) return { missing: "de_forge has no T spawn cell" };
	219	return { run: () => void map.findPath(from, to) };
	220	},
	221	};
	222
	223	/**
	224	* The per-frame engine path. It is not measurable from here: the accumulator
	225	* and the input read live inside `boot()` in src/client/main.ts, over
	226	* `createControls` from src/client/input.ts, which wants a canvas and a
	227	* window. Saying so is the honest row; a number invented for it would be the
	228	* dishonest one. A headless module the page calls would make it measurable.
	229	*/
	230	const framePath: TimeCase = {
	231	kind: "time",
	232	name: "frame-path",
	233	what: "the fixed-step accumulator plus reading input, once a frame.",
	234	unit: "µs/frame",
	235	reps: 1,
	236	warmup: 0,
	237	prepare: () => ({
	238	missing:
	239	"no headless module: the accumulator and the input read live inside boot() in src/client/main.ts, and src/client/input.ts needs a canvas and a window.",
	240	}),
	241	};
	242
	243	/** One row per committed bundle, read from static/ — nothing is rebuilt. */
	244	export function bundleCases(): SizeCase[] {
	245	return ENTRIES.map(({ name }) => ({
	246	kind: "size" as const,
	247	name: `bundle:${name}`,
	248	what: `static/${name}.<hash>.js as committed, the bytes the browser downloads before it can start (not gzipped — what the wire costs depends on the proxy).`,
	249	unit: "bytes" as const,
	250	measure: () => {
	251	const files = existsSync(STATIC_DIR) ? readdirSync(STATIC_DIR).filter((f) => isBundleOf(name, f)) : [];
	252	const file = files[0];
	253	if (file === undefined) return { missing: `no committed bundle for ${name}: run bun run scripts/build.ts` };
	254	if (files.length > 1) return { missing: `${files.length} bundles for ${name} in static/: one of them is stale` };
	255	return { bytes: statSync(join(STATIC_DIR, file)).size, note: `static/${file}` };
	256	},
	257	}));
	258	}
	259
	260	export const CASES: Case[] = [
	261	worldStep, arenaStep, policyDecision, levelMeshCase, atlasCase, pathCase, framePath, ...bundleCases(),
	262	];
	263
	264	// --- the two numbers that matter ------------------------------------------
	265
	266	export type Derived = { name: string; what: string; percent: number \| null; note?: string };
	267
	268	/** The share of its budget each hot path costs, from the rows just measured. */
	269	export function derive(rows: Row[]): Derived[] {
	270	const at = (name: string): Row \| undefined => rows.find((r) => r.name === name);
	271	const world = at("world-step");
	272	const frame = at("frame-path");
	273	const out: Derived[] = [];
	274
	275	out.push({
	276	name: "world-step / tick budget",
	277	what: `one world step as a share of the ${(TICK_BUDGET_US / 1000).toFixed(1)} ms a 64 Hz tick may take`,
	278	percent: world?.median === undefined \|\| world.median === null ? null : (world.median / TICK_BUDGET_US) * 100,
	279	...(world === undefined ? { note: "world-step was not run" } : {}),
	280	});
	281
	282	// The frame path is not measured (see framePath). What can be said without
	283	// a DOM is what the accumulator runs — the ticks themselves — in
	284	// milliseconds of the frame, and it is labelled as derived, not measured.
	285	// As a percentage it would be the row above by construction, since the tick
	286	// rate is fixed: 16.7 ms holds FRAME/TICK steps whatever a step costs.
	287	const ticksPerFrame = FRAME_BUDGET_US / TICK_BUDGET_US;
	288	const simulationMs =
	289	world?.median === undefined \|\| world.median === null ? null : (world.median * ticksPerFrame) / 1000;
	290	out.push({
	291	name: "frame path / 16.7 ms frame",
	292	what: "the fixed-step accumulator plus reading input, as a share of one 60 Hz frame",
	293	percent: frame?.median ?? null,
	294	note:
	295	frame === undefined
	296	? "frame-path was not run"
	297	: `${frame.note ?? "not measured"}` +
	298	(simulationMs === null
	299	? ""
	300	: ` What the accumulator runs can be derived: ${ticksPerFrame.toFixed(2)} world steps fall in a 16.7 ms frame, so the simulation alone is ${simulationMs.toFixed(2)} ms of it — derived from world-step, not measured, and it counts none of the input read, the accumulator arithmetic or the draw.`),
	301	});
	302	return out;
	303	}
	304
	305	// --- printing -------------------------------------------------------------
	306
	307	const pad = (text: string, width: number): string => text.padEnd(width);
	308	const padLeft = (text: string, width: number): string => text.padStart(width);
	309
	310	/** Enough digits to compare two runs, and not one more. */
	311	function show(row: Row, value: number \| null): string {
	312	if (value === null) return "—";
	313	if (row.unit === "bytes") return String(Math.round(value));
	314	return value >= 100 ? value.toFixed(1) : value >= 1 ? value.toFixed(2) : value.toFixed(3);
	315	}
	316
	317	export function table(rows: Row[]): string {
	318	const head = ["case", "unit", "median", "p95", "reps"];
	319	const cells = rows.map((r) => [r.name, r.unit, show(r, r.median), show(r, r.p95), r.reps === null ? "—" : String(r.reps)]);
	320	const width = head.map((h, i) => Math.max(h.length, ...cells.map((c) => (c[i] as string).length)));
	321	const line = (c: string[]): string =>
	322	`${pad(c[0] as string, width[0] as number)} ${pad(c[1] as string, width[1] as number)} ` +
	323	`${padLeft(c[2] as string, width[2] as number)} ${padLeft(c[3] as string, width[3] as number)} ` +
	324	`${padLeft(c[4] as string, width[4] as number)}`;
	325	return [line(head), ...cells.map(line)].join("\n");
	326	}
	327
	328	function report(rows: Row[], derived: Derived[], seconds: number): string {
	329	const out = [table(rows), ""];
	330	for (const d of derived) {
	331	const value = d.percent === null ? "—" : `${d.percent.toFixed(1)}%`;
	332	out.push(`${d.name}: ${value} (${d.what})${d.note === undefined ? "" : `\n ${d.note}`}`);
	333	}
	334	out.push("", "What is in each case:");
	335	for (const r of rows) out.push(` ${r.name} — ${r.what}${r.note === undefined ? "" : `\n ${r.note}`}`);
	336	out.push(
	337	"",
	338	`${rows.length} case${rows.length === 1 ? "" : "s"} in ${seconds.toFixed(1)}s. Median and p95 are nearest-rank over the repetitions shown;`,
	339	"inputs are built before the clock. These are this machine's numbers, not a promise: compare a",
	340	"before and an after from the same box rather than trusting an absolute.",
	341	);
	342	return out.join("\n");
	343	}
	344
	345	// --- the command ----------------------------------------------------------
	346
	347	export type Run = { code: number; out: string; err: string };
	348
	349	const USAGE = "usage: bun run scripts/bench.ts [--json] [--case <name>]";
	350
	351	/**
	352	* Parse the arguments, run what they asked for and hand back what to print.
	353	* Nothing here takes a path or a command from `argv`: `--case` is matched
	354	* against the names of the cases and everything else is rejected.
	355	*/
	356	export function run(argv: string[], cases: Case[] = CASES): Run {
	357	let json = false;
	358	let only: string \| null = null;
	359	for (let i = 0; i < argv.length; i++) {
	360	const arg = argv[i];
	361	if (arg === "--json") json = true;
	362	else if (arg === "--case") {
	363	const name = argv[++i];
	364	if (name === undefined) return { code: 2, out: "", err: `--case wants a name\n${USAGE}` };
	365	only = name;
	366	} else return { code: 2, out: "", err: `unknown argument ${JSON.stringify(arg)}\n${USAGE}` };
	367	}
	368
	369	let chosen = cases;
	370	if (only !== null) {
	371	chosen = cases.filter((c) => c.name === only);
	372	if (chosen.length === 0)
	373	return {
	374	code: 1,
	375	out: "",
	376	err: `no case named ${JSON.stringify(only)}. The cases are:\n${cases.map((c) => ` ${c.name}`).join("\n")}`,
	377	};
	378	}
	379
	380	const started = Bun.nanoseconds();
	381	const rows = chosen.map((c) => measure(c));
	382	const seconds = (Bun.nanoseconds() - started) / 1e9;
	383	const derived = derive(rows);
	384	if (json) return { code: 0, out: JSON.stringify({ seconds: Number(seconds.toFixed(3)), rows, derived }, null, 2), err: "" };
	385	return { code: 0, out: report(rows, derived, seconds), err: "" };
	386	}
	387
	388	if (import.meta.main) {
	389	const { code, out, err } = run(process.argv.slice(2));
	390	if (out !== "") console.log(out);
	391	if (err !== "") console.error(err);
	392	process.exit(code);
	393	}