Split The code is public; the app answers only to the crew.
`bun run scripts/cost.ts`: what a tick of the simulation costs, and a guard so it stays that way #118
Check · Review · Land · Live done — Landed and live in production.
vesper/cs:vesper/90-bun-run-scripts-cost-ts-what-a-tick-of-t → main 1 commit fork
3 files
changed +280
−0
README.md +33
| @@ -30,6 +30,7 @@ bun run scripts/train.ts --iters 1000 # retrain the bots' policy | ||
| 30 | 30 | bun run scripts/train.ts --iters 1000 --workers 4 # or on four cores |
| 31 | 31 | bun run scripts/train.ts --bench # how fast the arena steps |
| 32 | 32 | bun run scripts/measure.ts # what a hundred rounds say — see below |
| 33 | bun run scripts/cost.ts # what a tick of the simulation costs — see below | |
| 33 | 34 | |
| 34 | 35 | |
| 35 | 36 | The client is bundled at start-up, in memory, from the sources on disk — the |
| @@ -517,3 +518,35 @@ terrorists' favour; treat the split between `detonated` and `elimination` | ||
| 517 | 518 | accordingly. This baseline drifts whenever the bots or the blast do — it |
| 518 | 519 | was 13% defused before #66 and #64 landed — so re-run it rather than trust |
| 519 | 520 | it after a change to either. |
| 521 | ||
| 522 | ## Measuring the tick | |
| 523 | ||
| 524 | `scripts/cost.ts` asks the other half of the question: not what a round looks | |
| 525 | like but what it costs. It steps a seeded ten-player world through the same | |
| 526 | `world.update` a browser steps, the person idle, and records every tick's wall | |
| 527 | clock as a number: | |
| 528 | ||
| 529 | sh | |
| 530 | bun run scripts/cost.ts # 7,680 ticks of a SHORT world | |
| 531 | bun run scripts/cost.ts --ticks 30000 --seed 7 # longer sample, a different seed | |
| 532 | bun run scripts/cost.ts --rules long # COMPETITIVE rules instead | |
| 533 | bun run scripts/cost.ts --difficulty hard # harder bots instead of normal | |
| 534 | ||
| 535 | ||
| 536 | Measured on the box that runs the check, the defaults (7,680 ticks — two | |
| 537 | minutes of a match — after a 256-tick warm-up): **mean 0.14 ms a tick, p50 | |
| 538 | 0.11, p95 0.29, p99 0.43**, about **7,200 ticks a second**, or **1% of the | |
| 539 | 15.6 ms a 64 Hz tick is allowed**. So the simulation is a rounding error in a | |
| 540 | frame and the renderer owns the rest of it — which is the point of writing the | |
| 541 | number down. The max swings between 3 ms and 30 ms run to run; that is the | |
| 542 | garbage collector and whatever else the box is doing, not the simulation, so | |
| 543 | read the mean and the percentiles and ignore it. | |
| 544 | ||
| 545 | The arithmetic is `summarize` in the script itself — a pure function over a | |
| 546 | list of milliseconds, tested on a known list with no clock in sight. Exactly | |
| 547 | one assertion in the suite depends on how fast the machine is, and it is in | |
| 548 | `scripts/cost.test.ts`: 600 sampled ticks must average under **2 ms**, roughly | |
| 549 | fourteen times today's cost. | |
| 550 | It is a regression alarm, not a target — it fails when a change to `bots.ts`, | |
| 551 | `trace.ts` or `world.ts` makes a tick an order of magnitude dearer, and it does | |
| 552 | not fail because CI was busy. It costs about a fifth of a second to run. |
scripts/cost.test.ts added +88
| @@ -0,0 +1,88 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { COMPETITIVE, SHORT } from "../src/client/match.ts"; | |
| 3 | import { | |
| 4 | TICK_BUDGET_MS, parseArgs, reportLines, rulesFor, sampleTicks, summarize, | |
| 5 | } from "./cost.ts"; | |
| 6 | ||
| 7 | test("no flags fall back to a short, quick, seeded default", () => { | |
| 8 | expect(parseArgs([])).toEqual({ ticks: 7680, seed: 20260917, rulesName: "short", difficulty: "normal" }); | |
| 9 | }); | |
| 10 | ||
| 11 | test("ticks, seed, rules and difficulty are read off the command line", () => { | |
| 12 | expect(parseArgs(["--ticks", "30000", "--seed", "7", "--rules", "long", "--difficulty", "hard"])).toEqual({ | |
| 13 | ticks: 30000, seed: 7, rulesName: "long", difficulty: "hard", | |
| 14 | }); | |
| 15 | }); | |
| 16 | ||
| 17 | test("a malformed or missing value falls back to the default rather than NaN", () => { | |
| 18 | expect(parseArgs(["--ticks", "nope"]).ticks).toBe(7680); | |
| 19 | expect(parseArgs(["--ticks", "-3"]).ticks).toBe(7680); | |
| 20 | expect(parseArgs(["--ticks"]).ticks).toBe(7680); | |
| 21 | expect(parseArgs(["--rules", "medium"]).rulesName).toBe("short"); | |
| 22 | expect(parseArgs(["--seed"]).seed).toBe(20260917); | |
| 23 | expect(parseArgs(["--seed", "-3"]).seed).toBe(20260917); | |
| 24 | expect(parseArgs(["--seed", "1.5"]).seed).toBe(20260917); | |
| 25 | expect(parseArgs(["--seed", "0"]).seed).toBe(0); // zero is a fine seed, not a falsy one | |
| 26 | expect(parseArgs(["--difficulty", "extreme"]).difficulty).toBe("normal"); | |
| 27 | expect(parseArgs(["--difficulty"]).difficulty).toBe("normal"); | |
| 28 | }); | |
| 29 | ||
| 30 | test("rulesFor maps the two names the same way measure.ts does", () => { | |
| 31 | expect(rulesFor("short")).toBe(SHORT); | |
| 32 | expect(rulesFor("long")).toBe(COMPETITIVE); | |
| 33 | }); | |
| 34 | ||
| 35 | test("an empty sample summarizes to zeros, not NaN or a thrown error", () => { | |
| 36 | expect(summarize([])).toEqual({ | |
| 37 | ticks: 0, meanMs: 0, p50Ms: 0, p95Ms: 0, p99Ms: 0, maxMs: 0, ticksPerSecond: 0, budgetShare: 0, | |
| 38 | }); | |
| 39 | }); | |
| 40 | ||
| 41 | test("percentiles of a known list are nearest-rank, and order does not matter", () => { | |
| 42 | // 1..100 shuffled by a fixed stride, so the list arrives out of order. | |
| 43 | const values = Array.from({ length: 100 }, (_, i) => ((i * 37) % 100) + 1); | |
| 44 | const cost = summarize(values); | |
| 45 | expect(cost.ticks).toBe(100); | |
| 46 | expect(cost.meanMs).toBeCloseTo(50.5, 10); | |
| 47 | expect(cost.p50Ms).toBe(50); | |
| 48 | expect(cost.p95Ms).toBe(95); | |
| 49 | expect(cost.p99Ms).toBe(99); | |
| 50 | expect(cost.maxMs).toBe(100); | |
| 51 | }); | |
| 52 | ||
| 53 | test("a single tick is its own mean, median and max", () => { | |
| 54 | const cost = summarize([4]); | |
| 55 | expect(cost).toMatchObject({ ticks: 1, meanMs: 4, p50Ms: 4, p95Ms: 4, p99Ms: 4, maxMs: 4 }); | |
| 56 | }); | |
| 57 | ||
| 58 | test("ticks a second and the budget share come off the mean", () => { | |
| 59 | const cost = summarize([1, 3]); // mean 2 ms | |
| 60 | expect(cost.ticksPerSecond).toBe(500); | |
| 61 | expect(cost.budgetShare).toBeCloseTo(2 / TICK_BUDGET_MS, 10); | |
| 62 | expect(TICK_BUDGET_MS).toBeCloseTo(15.625, 10); | |
| 63 | }); | |
| 64 | ||
| 65 | test("a Float64Array summarizes the same as the plain list of the same numbers", () => { | |
| 66 | expect(summarize(new Float64Array([3, 1, 2]))).toEqual(summarize([3, 1, 2])); | |
| 67 | }); | |
| 68 | ||
| 69 | test("the report says the ticks, the mean, the spread and the share of the budget", () => { | |
| 70 | const lines = reportLines(summarize([1, 3])); | |
| 71 | expect(lines[0]).toBe("ticks sampled 2"); | |
| 72 | expect(lines[1]).toContain("2.000 ms a tick (500 ticks/s)"); | |
| 73 | expect(lines[2]).toContain("p50 1.000 ms · p95 3.000 ms"); | |
| 74 | expect(lines[3]).toContain("12.8% of the 15.6 ms"); | |
| 75 | }); | |
| 76 | ||
| 77 | /** | |
| 78 | * The one wall-clock assertion in the repository. 2 ms is roughly fourteen | |
| 79 | * times what a tick costs today (about 0.13 ms on the box that runs the | |
| 80 | * check), so this is a regression alarm rather than a target: it fails when | |
| 81 | * a change to `bots.ts`, `trace.ts` or `world.ts` makes a tick an order of | |
| 82 | * magnitude dearer, and survives a CI box under load. | |
| 83 | */ | |
| 84 | test("a tick of the shipped simulation costs well under its 64 Hz budget", () => { | |
| 85 | const cost = summarize(sampleTicks({ ticks: 600, seed: 20260917, rulesName: "short", difficulty: "normal" })); | |
| 86 | expect(cost.ticks).toBe(600); | |
| 87 | expect(cost.meanMs).toBeLessThan(2); | |
| 88 | }); |
scripts/cost.ts added +159
| @@ -0,0 +1,159 @@ | ||
| 1 | /** | |
| 2 | * Measures what one tick of the simulation costs: `world.update` stepped at | |
| 3 | * 64 Hz on a seeded ten-player world, the person idle (`NO_INPUT`) the whole | |
| 4 | * way, with every tick's wall clock recorded as a number. Prints one table — | |
| 5 | * mean, p50, p95, p99 and max milliseconds a tick, the ticks a second that | |
| 6 | * implies, and the share of the 15.6 ms a 64 Hz tick is allowed. | |
| 7 | * | |
| 8 | * bun run scripts/cost.ts # 7,680 ticks of a SHORT world | |
| 9 | * bun run scripts/cost.ts --ticks 30000 --seed 7 # longer sample, a different seed | |
| 10 | * bun run scripts/cost.ts --rules long # COMPETITIVE rules instead | |
| 11 | * bun run scripts/cost.ts --difficulty hard # harder bots instead of normal | |
| 12 | * | |
| 13 | * Same shape as `scripts/measure.ts`: nothing here is imported by the game or | |
| 14 | * the server, the flags parse the same way, and the arithmetic is a pure | |
| 15 | * function (`summarize`) tested on a known list without touching a clock. One | |
| 16 | * world exists at a time and no tick is kept as anything but a number, so a | |
| 17 | * hundred thousand ticks cost one Float64Array rather than a hundred thousand | |
| 18 | * objects. | |
| 19 | */ | |
| 20 | ||
| 21 | import { COMPETITIVE, SHORT, type Rules } from "../src/client/match.ts"; | |
| 22 | import { | |
| 23 | DEFAULT_DIFFICULTY, DIFFICULTIES, NO_INPUT, createWorld, difficultyLabel, update, type Difficulty, | |
| 24 | } from "../src/client/world.ts"; | |
| 25 | ||
| 26 | export type RulesName = "short" | "long"; | |
| 27 | export type Options = { ticks: number; seed: number; rulesName: RulesName; difficulty: Difficulty }; | |
| 28 | ||
| 29 | /** 7,680 ticks is two minutes of a 64 Hz match: long enough to cross a plant. */ | |
| 30 | const DEFAULTS: Options = { ticks: 7680, seed: 20260917, rulesName: "short", difficulty: DEFAULT_DIFFICULTY }; | |
| 31 | ||
| 32 | /** `--ticks 30000 --seed 7 --rules long --difficulty hard`, missing or malformed flags keep the default. */ | |
| 33 | export function parseArgs(argv: string[]): Options { | |
| 34 | const value = (name: string): string | null => { | |
| 35 | const at = argv.indexOf(`--${name}`); | |
| 36 | return at < 0 ? null : (argv[at + 1] ?? null); | |
| 37 | }; | |
| 38 | const ticks = value("ticks") === null ? NaN : Number(value("ticks")); | |
| 39 | const seed = value("seed") === null ? NaN : Number(value("seed")); | |
| 40 | const rulesArg = value("rules"); | |
| 41 | const difficultyArg = value("difficulty"); | |
| 42 | return { | |
| 43 | ticks: Number.isFinite(ticks) && ticks > 0 ? Math.round(ticks) : DEFAULTS.ticks, | |
| 44 | seed: Number.isInteger(seed) && seed >= 0 ? seed : DEFAULTS.seed, | |
| 45 | rulesName: rulesArg === "long" ? "long" : rulesArg === "short" ? "short" : DEFAULTS.rulesName, | |
| 46 | difficulty: isDifficulty(difficultyArg) ? difficultyArg : DEFAULTS.difficulty, | |
| 47 | }; | |
| 48 | } | |
| 49 | ||
| 50 | export const rulesFor = (name: RulesName): Rules => (name === "long" ? COMPETITIVE : SHORT); | |
| 51 | ||
| 52 | const isDifficulty = (value: string | null): value is Difficulty => | |
| 53 | DIFFICULTIES.includes(value as Difficulty); | |
| 54 | ||
| 55 | const TICK = 1 / 64; | |
| 56 | ||
| 57 | /** The milliseconds a 64 Hz tick is allowed end to end — simulation and frame both. */ | |
| 58 | export const TICK_BUDGET_MS = 1000 * TICK; | |
| 59 | ||
| 60 | /** Ticks stepped and thrown away before the sample, so the JIT is warm when it counts. */ | |
| 61 | const WARMUP_TICKS = 256; | |
| 62 | ||
| 63 | export type Cost = { | |
| 64 | ticks: number; | |
| 65 | meanMs: number; | |
| 66 | p50Ms: number; | |
| 67 | p95Ms: number; | |
| 68 | p99Ms: number; | |
| 69 | maxMs: number; | |
| 70 | ticksPerSecond: number; // what the mean implies if nothing else ran | |
| 71 | budgetShare: number; // fraction of TICK_BUDGET_MS the mean uses | |
| 72 | }; | |
| 73 | ||
| 74 | /** | |
| 75 | * Nearest-rank percentile: the value at the smallest rank covering the given | |
| 76 | * share of the sorted sample, so p50 of four values is the second and the | |
| 77 | * top share is the max. No interpolation — every number printed is a tick | |
| 78 | * that actually happened. | |
| 79 | */ | |
| 80 | const percentile = (sorted: number[], share: number): number => | |
| 81 | sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(share * sorted.length) - 1))] as number; | |
| 82 | ||
| 83 | /** Pure arithmetic on a list of per-tick milliseconds. No clock is read here. */ | |
| 84 | export function summarize(millis: ArrayLike<number>): Cost { | |
| 85 | const values = Array.from(millis); | |
| 86 | if (values.length === 0) { | |
| 87 | return { ticks: 0, meanMs: 0, p50Ms: 0, p95Ms: 0, p99Ms: 0, maxMs: 0, ticksPerSecond: 0, budgetShare: 0 }; | |
| 88 | } | |
| 89 | const sorted = [...values].sort((a, b) => a - b); | |
| 90 | let total = 0; | |
| 91 | for (const ms of values) total += ms; | |
| 92 | const meanMs = total / values.length; | |
| 93 | return { | |
| 94 | ticks: values.length, | |
| 95 | meanMs, | |
| 96 | p50Ms: percentile(sorted, 0.5), | |
| 97 | p95Ms: percentile(sorted, 0.95), | |
| 98 | p99Ms: percentile(sorted, 0.99), | |
| 99 | maxMs: sorted[sorted.length - 1] as number, | |
| 100 | ticksPerSecond: meanMs === 0 ? Infinity : 1000 / meanMs, | |
| 101 | budgetShare: meanMs / TICK_BUDGET_MS, | |
| 102 | }; | |
| 103 | } | |
| 104 | ||
| 105 | /** The cost as one line a stat, the table this script prints. */ | |
| 106 | export function reportLines(cost: Cost): string[] { | |
| 107 | const ms = (value: number): string => `${value.toFixed(3)} ms`; | |
| 108 | return [ | |
| 109 | `ticks sampled ${cost.ticks}`, | |
| 110 | `mean ${ms(cost.meanMs)} a tick (${Math.round(cost.ticksPerSecond).toLocaleString("en-US")} ticks/s)`, | |
| 111 | `spread p50 ${ms(cost.p50Ms)} · p95 ${ms(cost.p95Ms)} · p99 ${ms(cost.p99Ms)} · max ${ms(cost.maxMs)}`, | |
| 112 | `budget ${(cost.budgetShare * 100).toFixed(1)}% of the ${TICK_BUDGET_MS.toFixed(1)} ms a 64 Hz tick is allowed`, | |
| 113 | ]; | |
| 114 | } | |
| 115 | ||
| 116 | /** | |
| 117 | * Step a seeded world `ticks` times and return each tick's milliseconds. The | |
| 118 | * world is replaced, untimed, whenever a match finishes, so a long sample | |
| 119 | * measures play rather than a `match_over` screen. | |
| 120 | */ | |
| 121 | export function sampleTicks(opts: Options): Float64Array { | |
| 122 | const rules = rulesFor(opts.rulesName); | |
| 123 | const newWorld = (restart: number) => | |
| 124 | // The same prime stride scripts/measure.ts uses, so a restart is not a | |
| 125 | // neighbour of the seed it followed. | |
| 126 | createWorld(rules, "cost", "ct", 5, opts.seed + restart * 7919, opts.difficulty); | |
| 127 | ||
| 128 | let world = newWorld(0); | |
| 129 | let restarts = 0; | |
| 130 | for (let i = 0; i < WARMUP_TICKS; i++) update(world, TICK, NO_INPUT); | |
| 131 | ||
| 132 | const millis = new Float64Array(opts.ticks); | |
| 133 | for (let i = 0; i < opts.ticks; i++) { | |
| 134 | const before = Bun.nanoseconds(); | |
| 135 | update(world, TICK, NO_INPUT); | |
| 136 | millis[i] = (Bun.nanoseconds() - before) / 1e6; | |
| 137 | if (world.match.phase === "match_over") world = newWorld(++restarts); | |
| 138 | } | |
| 139 | return millis; | |
| 140 | } | |
| 141 | ||
| 142 | function main(): void { | |
| 143 | const opts = parseArgs(process.argv.slice(2)); | |
| 144 | console.log( | |
| 145 | `${opts.ticks} ticks of a 10-player ${opts.rulesName.toUpperCase()} world, seed ${opts.seed}, ` + | |
| 146 | `${difficultyLabel(opts.difficulty)} bots, the person idle`, | |
| 147 | ); | |
| 148 | ||
| 149 | const started = Bun.nanoseconds(); | |
| 150 | const cost = summarize(sampleTicks(opts)); | |
| 151 | const totalSeconds = (Bun.nanoseconds() - started) / 1e9; | |
| 152 | ||
| 153 | console.log(""); | |
| 154 | for (const line of reportLines(cost)) console.log(line); | |
| 155 | console.log(""); | |
| 156 | console.log(`${totalSeconds.toFixed(1)}s wall clock, warm-up and sampling both`); | |
| 157 | } | |
| 158 | ||
| 159 | if (import.meta.main) main(); |
new above
offline