Split The code is public; the app answers only to the crew.

`bun run scripts/cost.ts`: what a tick of the simulation costs, and a guard so it stays that way #118

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

vesper/cs:vesper/90-bun-run-scripts-cost-ts-what-a-tick-of-t → main 1 commit fork

  • Merged. vesper/90-bun-run-scripts-cost-ts-what-a-tick-of-t was merged into main as b0bf797.

Review

Sign in to review this change.
3 files changed +280 −0
README.md +33
@@ -30,6 +30,7 @@ bun run scripts/train.ts --iters 1000 # retrain the bots' policy
3030 bun run scripts/train.ts --iters 1000 --workers 4 # or on four cores
3131 bun run scripts/train.ts --bench # how fast the arena steps
3232 bun run scripts/measure.ts # what a hundred rounds say — see below
33 bun run scripts/cost.ts # what a tick of the simulation costs — see below
3334 ```
3435
3536 The client is bundled at start-up, in memory, from the sources on disk — the
@@ -517,3 +518,35 @@ terrorists' favour; treat the split between `detonated` and `elimination`
517518 accordingly. This baseline drifts whenever the bots or the blast do — it
518519 was 13% defused before #66 and #64 landed — so re-run it rather than trust
519520 it after a change to either.
521
522 ## Measuring the tick
523
524 `scripts/cost.ts` asks the other half of the question: not what a round looks
525 like but what it costs. It steps a seeded ten-player world through the same
526 `world.update` a browser steps, the person idle, and records every tick's wall
527 clock as a number:
528
529 ```sh
530 bun run scripts/cost.ts # 7,680 ticks of a SHORT world
531 bun run scripts/cost.ts --ticks 30000 --seed 7 # longer sample, a different seed
532 bun run scripts/cost.ts --rules long # COMPETITIVE rules instead
533 bun run scripts/cost.ts --difficulty hard # harder bots instead of normal
534 ```
535
536 Measured on the box that runs the check, the defaults (7,680 ticks — two
537 minutes of a match — after a 256-tick warm-up): **mean 0.14 ms a tick, p50
538 0.11, p95 0.29, p99 0.43**, about **7,200 ticks a second**, or **1% of the
539 15.6 ms a 64 Hz tick is allowed**. So the simulation is a rounding error in a
540 frame and the renderer owns the rest of it — which is the point of writing the
541 number down. The max swings between 3 ms and 30 ms run to run; that is the
542 garbage collector and whatever else the box is doing, not the simulation, so
543 read the mean and the percentiles and ignore it.
544
545 The arithmetic is `summarize` in the script itself — a pure function over a
546 list of milliseconds, tested on a known list with no clock in sight. Exactly
547 one assertion in the suite depends on how fast the machine is, and it is in
548 `scripts/cost.test.ts`: 600 sampled ticks must average under **2 ms**, roughly
549 fourteen times today's cost.
550 It is a regression alarm, not a target — it fails when a change to `bots.ts`,
551 `trace.ts` or `world.ts` makes a tick an order of magnitude dearer, and it does
552 not fail because CI was busy. It costs about a fifth of a second to run.
scripts/cost.test.ts added +88
@@ -0,0 +1,88 @@
1 import { expect, test } from "bun:test";
2 import { COMPETITIVE, SHORT } from "../src/client/match.ts";
3 import {
4 TICK_BUDGET_MS, parseArgs, reportLines, rulesFor, sampleTicks, summarize,
5 } from "./cost.ts";
6
7 test("no flags fall back to a short, quick, seeded default", () => {
8 expect(parseArgs([])).toEqual({ ticks: 7680, seed: 20260917, rulesName: "short", difficulty: "normal" });
9 });
10
11 test("ticks, seed, rules and difficulty are read off the command line", () => {
12 expect(parseArgs(["--ticks", "30000", "--seed", "7", "--rules", "long", "--difficulty", "hard"])).toEqual({
13 ticks: 30000, seed: 7, rulesName: "long", difficulty: "hard",
14 });
15 });
16
17 test("a malformed or missing value falls back to the default rather than NaN", () => {
18 expect(parseArgs(["--ticks", "nope"]).ticks).toBe(7680);
19 expect(parseArgs(["--ticks", "-3"]).ticks).toBe(7680);
20 expect(parseArgs(["--ticks"]).ticks).toBe(7680);
21 expect(parseArgs(["--rules", "medium"]).rulesName).toBe("short");
22 expect(parseArgs(["--seed"]).seed).toBe(20260917);
23 expect(parseArgs(["--seed", "-3"]).seed).toBe(20260917);
24 expect(parseArgs(["--seed", "1.5"]).seed).toBe(20260917);
25 expect(parseArgs(["--seed", "0"]).seed).toBe(0); // zero is a fine seed, not a falsy one
26 expect(parseArgs(["--difficulty", "extreme"]).difficulty).toBe("normal");
27 expect(parseArgs(["--difficulty"]).difficulty).toBe("normal");
28 });
29
30 test("rulesFor maps the two names the same way measure.ts does", () => {
31 expect(rulesFor("short")).toBe(SHORT);
32 expect(rulesFor("long")).toBe(COMPETITIVE);
33 });
34
35 test("an empty sample summarizes to zeros, not NaN or a thrown error", () => {
36 expect(summarize([])).toEqual({
37 ticks: 0, meanMs: 0, p50Ms: 0, p95Ms: 0, p99Ms: 0, maxMs: 0, ticksPerSecond: 0, budgetShare: 0,
38 });
39 });
40
41 test("percentiles of a known list are nearest-rank, and order does not matter", () => {
42 // 1..100 shuffled by a fixed stride, so the list arrives out of order.
43 const values = Array.from({ length: 100 }, (_, i) => ((i * 37) % 100) + 1);
44 const cost = summarize(values);
45 expect(cost.ticks).toBe(100);
46 expect(cost.meanMs).toBeCloseTo(50.5, 10);
47 expect(cost.p50Ms).toBe(50);
48 expect(cost.p95Ms).toBe(95);
49 expect(cost.p99Ms).toBe(99);
50 expect(cost.maxMs).toBe(100);
51 });
52
53 test("a single tick is its own mean, median and max", () => {
54 const cost = summarize([4]);
55 expect(cost).toMatchObject({ ticks: 1, meanMs: 4, p50Ms: 4, p95Ms: 4, p99Ms: 4, maxMs: 4 });
56 });
57
58 test("ticks a second and the budget share come off the mean", () => {
59 const cost = summarize([1, 3]); // mean 2 ms
60 expect(cost.ticksPerSecond).toBe(500);
61 expect(cost.budgetShare).toBeCloseTo(2 / TICK_BUDGET_MS, 10);
62 expect(TICK_BUDGET_MS).toBeCloseTo(15.625, 10);
63 });
64
65 test("a Float64Array summarizes the same as the plain list of the same numbers", () => {
66 expect(summarize(new Float64Array([3, 1, 2]))).toEqual(summarize([3, 1, 2]));
67 });
68
69 test("the report says the ticks, the mean, the spread and the share of the budget", () => {
70 const lines = reportLines(summarize([1, 3]));
71 expect(lines[0]).toBe("ticks sampled 2");
72 expect(lines[1]).toContain("2.000 ms a tick (500 ticks/s)");
73 expect(lines[2]).toContain("p50 1.000 ms · p95 3.000 ms");
74 expect(lines[3]).toContain("12.8% of the 15.6 ms");
75 });
76
77 /**
78 * The one wall-clock assertion in the repository. 2 ms is roughly fourteen
79 * times what a tick costs today (about 0.13 ms on the box that runs the
80 * check), so this is a regression alarm rather than a target: it fails when
81 * a change to `bots.ts`, `trace.ts` or `world.ts` makes a tick an order of
82 * magnitude dearer, and survives a CI box under load.
83 */
84 test("a tick of the shipped simulation costs well under its 64 Hz budget", () => {
85 const cost = summarize(sampleTicks({ ticks: 600, seed: 20260917, rulesName: "short", difficulty: "normal" }));
86 expect(cost.ticks).toBe(600);
87 expect(cost.meanMs).toBeLessThan(2);
88 });
scripts/cost.ts added +159
@@ -0,0 +1,159 @@
1 /**
2 * Measures what one tick of the simulation costs: `world.update` stepped at
3 * 64 Hz on a seeded ten-player world, the person idle (`NO_INPUT`) the whole
4 * way, with every tick's wall clock recorded as a number. Prints one table —
5 * mean, p50, p95, p99 and max milliseconds a tick, the ticks a second that
6 * implies, and the share of the 15.6 ms a 64 Hz tick is allowed.
7 *
8 * bun run scripts/cost.ts # 7,680 ticks of a SHORT world
9 * bun run scripts/cost.ts --ticks 30000 --seed 7 # longer sample, a different seed
10 * bun run scripts/cost.ts --rules long # COMPETITIVE rules instead
11 * bun run scripts/cost.ts --difficulty hard # harder bots instead of normal
12 *
13 * Same shape as `scripts/measure.ts`: nothing here is imported by the game or
14 * the server, the flags parse the same way, and the arithmetic is a pure
15 * function (`summarize`) tested on a known list without touching a clock. One
16 * world exists at a time and no tick is kept as anything but a number, so a
17 * hundred thousand ticks cost one Float64Array rather than a hundred thousand
18 * objects.
19 */
20
21 import { COMPETITIVE, SHORT, type Rules } from "../src/client/match.ts";
22 import {
23 DEFAULT_DIFFICULTY, DIFFICULTIES, NO_INPUT, createWorld, difficultyLabel, update, type Difficulty,
24 } from "../src/client/world.ts";
25
26 export type RulesName = "short" | "long";
27 export type Options = { ticks: number; seed: number; rulesName: RulesName; difficulty: Difficulty };
28
29 /** 7,680 ticks is two minutes of a 64 Hz match: long enough to cross a plant. */
30 const DEFAULTS: Options = { ticks: 7680, seed: 20260917, rulesName: "short", difficulty: DEFAULT_DIFFICULTY };
31
32 /** `--ticks 30000 --seed 7 --rules long --difficulty hard`, missing or malformed flags keep the default. */
33 export function parseArgs(argv: string[]): Options {
34 const value = (name: string): string | null => {
35 const at = argv.indexOf(`--${name}`);
36 return at < 0 ? null : (argv[at + 1] ?? null);
37 };
38 const ticks = value("ticks") === null ? NaN : Number(value("ticks"));
39 const seed = value("seed") === null ? NaN : Number(value("seed"));
40 const rulesArg = value("rules");
41 const difficultyArg = value("difficulty");
42 return {
43 ticks: Number.isFinite(ticks) && ticks > 0 ? Math.round(ticks) : DEFAULTS.ticks,
44 seed: Number.isInteger(seed) && seed >= 0 ? seed : DEFAULTS.seed,
45 rulesName: rulesArg === "long" ? "long" : rulesArg === "short" ? "short" : DEFAULTS.rulesName,
46 difficulty: isDifficulty(difficultyArg) ? difficultyArg : DEFAULTS.difficulty,
47 };
48 }
49
50 export const rulesFor = (name: RulesName): Rules => (name === "long" ? COMPETITIVE : SHORT);
51
52 const isDifficulty = (value: string | null): value is Difficulty =>
53 DIFFICULTIES.includes(value as Difficulty);
54
55 const TICK = 1 / 64;
56
57 /** The milliseconds a 64 Hz tick is allowed end to end — simulation and frame both. */
58 export const TICK_BUDGET_MS = 1000 * TICK;
59
60 /** Ticks stepped and thrown away before the sample, so the JIT is warm when it counts. */
61 const WARMUP_TICKS = 256;
62
63 export type Cost = {
64 ticks: number;
65 meanMs: number;
66 p50Ms: number;
67 p95Ms: number;
68 p99Ms: number;
69 maxMs: number;
70 ticksPerSecond: number; // what the mean implies if nothing else ran
71 budgetShare: number; // fraction of TICK_BUDGET_MS the mean uses
72 };
73
74 /**
75 * Nearest-rank percentile: the value at the smallest rank covering the given
76 * share of the sorted sample, so p50 of four values is the second and the
77 * top share is the max. No interpolation — every number printed is a tick
78 * that actually happened.
79 */
80 const percentile = (sorted: number[], share: number): number =>
81 sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(share * sorted.length) - 1))] as number;
82
83 /** Pure arithmetic on a list of per-tick milliseconds. No clock is read here. */
84 export function summarize(millis: ArrayLike<number>): Cost {
85 const values = Array.from(millis);
86 if (values.length === 0) {
87 return { ticks: 0, meanMs: 0, p50Ms: 0, p95Ms: 0, p99Ms: 0, maxMs: 0, ticksPerSecond: 0, budgetShare: 0 };
88 }
89 const sorted = [...values].sort((a, b) => a - b);
90 let total = 0;
91 for (const ms of values) total += ms;
92 const meanMs = total / values.length;
93 return {
94 ticks: values.length,
95 meanMs,
96 p50Ms: percentile(sorted, 0.5),
97 p95Ms: percentile(sorted, 0.95),
98 p99Ms: percentile(sorted, 0.99),
99 maxMs: sorted[sorted.length - 1] as number,
100 ticksPerSecond: meanMs === 0 ? Infinity : 1000 / meanMs,
101 budgetShare: meanMs / TICK_BUDGET_MS,
102 };
103 }
104
105 /** The cost as one line a stat, the table this script prints. */
106 export function reportLines(cost: Cost): string[] {
107 const ms = (value: number): string => `${value.toFixed(3)} ms`;
108 return [
109 `ticks sampled ${cost.ticks}`,
110 `mean ${ms(cost.meanMs)} a tick (${Math.round(cost.ticksPerSecond).toLocaleString("en-US")} ticks/s)`,
111 `spread p50 ${ms(cost.p50Ms)} · p95 ${ms(cost.p95Ms)} · p99 ${ms(cost.p99Ms)} · max ${ms(cost.maxMs)}`,
112 `budget ${(cost.budgetShare * 100).toFixed(1)}% of the ${TICK_BUDGET_MS.toFixed(1)} ms a 64 Hz tick is allowed`,
113 ];
114 }
115
116 /**
117 * Step a seeded world `ticks` times and return each tick's milliseconds. The
118 * world is replaced, untimed, whenever a match finishes, so a long sample
119 * measures play rather than a `match_over` screen.
120 */
121 export function sampleTicks(opts: Options): Float64Array {
122 const rules = rulesFor(opts.rulesName);
123 const newWorld = (restart: number) =>
124 // The same prime stride scripts/measure.ts uses, so a restart is not a
125 // neighbour of the seed it followed.
126 createWorld(rules, "cost", "ct", 5, opts.seed + restart * 7919, opts.difficulty);
127
128 let world = newWorld(0);
129 let restarts = 0;
130 for (let i = 0; i < WARMUP_TICKS; i++) update(world, TICK, NO_INPUT);
131
132 const millis = new Float64Array(opts.ticks);
133 for (let i = 0; i < opts.ticks; i++) {
134 const before = Bun.nanoseconds();
135 update(world, TICK, NO_INPUT);
136 millis[i] = (Bun.nanoseconds() - before) / 1e6;
137 if (world.match.phase === "match_over") world = newWorld(++restarts);
138 }
139 return millis;
140 }
141
142 function main(): void {
143 const opts = parseArgs(process.argv.slice(2));
144 console.log(
145 `${opts.ticks} ticks of a 10-player ${opts.rulesName.toUpperCase()} world, seed ${opts.seed}, ` +
146 `${difficultyLabel(opts.difficulty)} bots, the person idle`,
147 );
148
149 const started = Bun.nanoseconds();
150 const cost = summarize(sampleTicks(opts));
151 const totalSeconds = (Bun.nanoseconds() - started) / 1e9;
152
153 console.log("");
154 for (const line of reportLines(cost)) console.log(line);
155 console.log("");
156 console.log(`${totalSeconds.toFixed(1)}s wall clock, warm-up and sampling both`);
157 }
158
159 if (import.meta.main) main();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs