7607472 · 63f0e0a
@@ -45,7 +45,8 @@
FORGE_APP_DATA=./data bun run src/tty.ts                # the terminal interface
  published shape so the habits transfer, and the rest is original.
- **Learned, and measured.** The bots do not aim by rule. A 32-input, two-layer
  policy decides where to point, when to shoot and which way to move, trained with
  PPO in `src/rl` against millions of ticks of this same simulation. The numbers on
  the engine's PPO against millions of ticks of this same simulation, in the arena
  under `src/rl`. The numbers on
  the page and in the terminal's `policy` tab are what the run that produced the
  shipped weights measured, on the same fixed duels for the policy and for the
  scripted bots it replaced.
@@ -402,10 +403,11 @@
renderer, no atlas, no mesh and no sound ship in that bundle, and
  `bundle.test.ts` boots the committed bundle and reads it for what is *not* in
  it (`webgl2`, `AudioContext`, `Math.tanh`): a door is only worth having if
  importing it costs what you use.
- `src/rl/` — training, which no game module imports: `arena` is the environment
  (a real duel on de_forge with the round and the economy taken out) and `ppo` is
  the algorithm, with its backward pass checked against finite differences.
  `scripts/train.ts` runs them and writes `src/client/weights.ts`. It is the
- `src/rl/` — training, which no game module imports: `arena` is the environment,
  a real duel on de_forge with the round and the economy taken out. The algorithm
  is not in here — PPO is `src/engine/learn.ts`, behind the door, because nothing
  in it is about shooting; what is here is this game's arena.
  `scripts/train.ts` runs the two together and writes `src/client/weights.ts`. It is the
  *shooter's* training and `src/independence.test.ts` counts it as the shooter:
  it reads `src/client/` module by module because that game is the environment,
  and for the same reason it may not read `src/range/`.
@@ -571,10 +573,14 @@
now a policy learned with PPO.
  movement, the real bullets and the real damage, and no round, economy or bomb.
  It steps about 13,000 ticks a second on one core, allocation-free; the policy
  decides every fourth tick and the decision is held in between.
- **How it learned** — `src/rl/ppo.ts`, about three hundred lines: GAE, a
  clipped surrogate, an entropy bonus, Adam, and a backward pass written out by
  hand and checked against finite differences in the tests. There is a
  curriculum on how far apart the duel starts and on how hard the opponent is,
- **How it learned** — `src/engine/learn.ts`, about three hundred lines behind
  the engine's door: GAE, a clipped surrogate, an entropy bonus, Adam, and a
  backward pass written out by hand and checked against finite differences in
  the tests. It is the engine's rather than this game's because nothing in it
  names a map, a tick or a weapon — the same door trains a two-armed bandit
  written inside `src/engine/learn.test.ts`, which is how that claim is checked.
  The curriculum on top of it is the shooter's and lives in `scripts/train.ts`:
  there is a curriculum on how far apart the duel starts and on how hard the opponent is,
  because handed a good opponent from the first step the policy learns to hide
  and never comes back from it.
- **What it costs you** — about eight kilobytes of int8 in the bundle, one
@@ -9,6 +9,10 @@
 * Nothing here is imported by the server or the page: this file runs on a
 * machine with a few minutes to spare, and its only output is a table of
 * numbers and about eight kilobytes of int8.
 *
 * The PPO is the engine's, reached through the door like everything else it
 * offers; what is the shooter's is the arena it collects from, the observation
 * it fills, the heads it decodes and the curriculum below.
 */
import { writeFileSync } from "node:fs";
@@ -16,17 +20,15 @@
import { join } from "node:path";
import { HEADS, decode } from "../src/client/action.ts";
import { botThink, makeBrain } from "../src/client/bots.ts";
import {
  copyPolicy, createPolicy, forward, greedy, newActivations, pack, rng, sample, unpack,
  type Policy, type Shape,
  computeAdvantages, copyPolicy, createAdam, createPolicy, createRollout, forward, greedy,
  newActivations, pack, rng, sample, unpack, update, DEFAULT_HYPER,
  type Hyper, type Policy, type Shape,
} from "../src/engine/index.ts";
import { OBS_SIZE } from "../src/client/observe.ts";
import {
  createArena, EPISODE_TICKS, observeFoe, observeLearner, policyFoe, reset, scriptedFoe, SKIP,
  step, stepInput, stillFoe, type Arena,
} from "../src/rl/arena.ts";
import {
  computeAdvantages, createAdam, createRollout, update, DEFAULT_HYPER, type Hyper,
} from "../src/rl/ppo.ts";
export const SHAPE: Shape = { obs: OBS_SIZE, hidden: 64, heads: HEADS };
@@ -420,7 +420,7 @@
It is the one group behind the door that a game may reasonably never open — th
shooter runs it every tick to fight with, and the range ships not a byte of it —
and it is here because nothing in it is about shooting. The example does not use
it either, which is why this section has no code block: read
`src/client/bots.ts` for a policy driving a game and `src/rl/ppo.ts` for one
`src/client/bots.ts` for a policy driving a game and `scripts/train.ts` for one
being trained.
**What a game brings first is a `Shape`:** how many numbers its observation has,
@@ -459,6 +459,54 @@
are and what each one is worth is `src/client/observe.ts`; which head is a turn
and which is a trigger is `src/client/action.ts`; both are the shooter's, and a
second game that wanted an agent would write its own two and share this.
### learn
PPO: how the weights the section above ships got their numbers. **A game needs
none of this.** A game ships a trained `Policy` and runs `forward`; training
happens once, on a machine with minutes to spare, in a script nothing serves.
Nothing in here ever reaches a browser — no committed bundle carries a byte of
it, which is a ceiling in `src/games.ts` and a test that reads the files — and
it is behind the door for one reason: not a line of it is about shooting. The
proof is in `src/engine/learn.test.ts`, which trains a two-armed bandit written
out inside the test, with no map, no tick and no weapon anywhere in it.
**What a caller brings is an environment and a `Rollout`.** `createRollout`
takes a `Shape`, how many environments are stepped side by side and how many
steps each one contributes, and hands back the flat arrays one batch of
experience is: the observations end to end, the action each head took, the
log-probability `sample` returned, the value `forward` predicted, the reward
that came back and whether the step ended an episode. The caller fills those by
stepping its own world — the engine never sees the world, only the numbers — and
writes into `Rollout.lastValues` what the policy thinks each unfinished episode
is still worth. Then `computeAdvantages` walks the slab backwards per
environment, discounting by `Hyper.gamma` and `Hyper.lambda` and never looking
through the seam where an episode ended, and fills in the advantages and the
returns.
**Then `update` is the step itself:** a few epochs of shuffled minibatches over
that rollout, each one a clipped surrogate with an entropy bonus and a value
loss, ending in an Adam step. It takes the policy, the rollout, an `Adam` from
`createAdam` and the same seeded generator the rollout was sampled with, and
returns an `UpdateStats` — the policy loss, the value loss, the entropy, the
fraction of samples that clipped and the gradient norm — which is the line a
training script prints and the first place a run that has gone wrong shows it.
**The tuning is data the caller passes.** `Hyper` is the nine numbers — the two
discounts, the clip, the entropy bonus, the value coefficient, the learning
rate, the epochs, the minibatch and the gradient-norm clip — and `DEFAULT_HYPER`
is a starting point rather than a recommendation: `createAdam` is handed the one
the caller means, and the shooter spreads `DEFAULT_HYPER` and overrides the
minibatch for its batch size. No constant in this module is the shooter's.
The pieces under those are exported too, because the hand-written backward pass
is the part most worth testing from outside: `Grads` is a `Policy` full of
gradients, `zeroGrads` makes one, `clearGrads` empties it between minibatches,
`backward` accumulates one sample into it given the derivative of the loss with
respect to the logits and the value, and `applyGrads` clips the global norm and
takes the Adam step, returning the norm it saw. The gradient check in
`src/engine/learn.test.ts` holds all of that against finite differences, which
is the only thing that catches an autodiff-free derivative being subtly wrong.
## The seam, named as pairs
Two games ship here, and every place they differ is a place the engine chose to
@@ -1,8 +1,8 @@
/**
 * The boundary. `src/engine/` is the part a second game could use, so a module
 * in here may import the engine and nothing else: not the game in
 * `src/client/`, not the training in `src/rl/`, and no package — the engine has
 * nothing to install. This test reads the sources rather than trusting the
 * `src/client/`, not the arena in `src/rl/` — the trainer is in here, what it
 * trains on is not — and no package, because the engine has nothing to install. This test reads the sources rather than trusting the
 * convention, because an import is one line and nobody reviews a line that
 * compiles.
 *
@@ -146,7 +146,7 @@
test("the engine has the sources this test is about", () => {
  expect(files).toContain("trace.ts");
});
test("no engine module imports the game, the training, or a package", () => {
test("no engine module imports the game, the arena, or a package", () => {
  const offences: string[] = [];
  for (const file of sources()) {
    const source = readFileSync(join(ENGINE, file), "utf8");
@@ -274,36 +274,32 @@
test("the example imports the door and nothing else, which is what it is for", (
 *   `newActivations`, `forward`, `headOffset`, `softmax`, `sample`, `greedy`,
 *   `pack`, `unpack`.
 *
 *   learn — `Hyper`, `DEFAULT_HYPER`, `Grads`, `zeroGrads`, `clearGrads`,
 *   `backward`, `Adam`, `createAdam`, `applyGrads`, `Rollout`, `createRollout`,
 *   `computeAdvantages`, `update`, `UpdateStats`.
 *
 * Exported because `guide.test.ts` holds `GUIDE.md` against the same list: one
 * copy of the surface, read twice, rather than two that drift.
 */
export const SURFACE = [
  "Activations",
  "Box", "Builder", "Burst", "Button", "ButtonOptions", "Camera", "Controls", "Ctor",
  "FakeCanvas", "FakeDocument", "FakeElement", "FakePage", "FakePaint", "FakeTarget", "FakeWindow",
  "Frames",
  "Grid", "Hit", "Host", "Instance", "KeyEventish", "Listens", "LockOwner", "Loop", "LoopOptions",
  "Mat4", "Mesh", "MouseEventish", "Options", "Overlay",
  "Packed", "PageOptions", "Paint", "Painter", "Policy", "Renderer",
  "RendererAssets",
  "Scene", "SceneCapacity", "Shader", "Shape", "SoundDevice", "Stick", "Surface", "Tone",
  "TouchEventish", "Touchish", "Vec3", "add", "addBox", "angleDelta", "angleVectors",
  "attachButtons", "attenuate", "builder", "cellAt", "clamp", "colOf",
  "copyPolicy", "createControls",
  "createLoop", "createPolicy", "createRenderer", "createScene", "createSoundDevice", "cross",
  "deg", "dist",
  "dist2d", "dot", "fakePage", "fbm",
  "findPath", "finish",
  "forward", "greedy", "hash", "headOffset", "inside", "length", "length2d", "lerp",
  "logitCount", "mat4", "mergeSolid",
  "modelMatrix", "mul", "multiply",
  "newActivations", "nextInstance", "noise", "normalize", "overlay",
  "pack", "paintAtlas", "painter", "paramCount",
  "perspective", "pushLine",
  "quadMesh", "rad", "rayBox", "resetScene", "rng", "rowOf",
  "sample", "scale", "segmentLength", "softmax", "sub", "tensorsOf",
  "traceWorld", "unpack",
  "v3", "viewMatrix", "visible", "worldOf", "zeroPolicy",
  "Activations", "Adam", "Box", "Builder", "Burst", "Button", "ButtonOptions", "Camera",
  "Controls", "Ctor", "DEFAULT_HYPER", "FakeCanvas", "FakeDocument", "FakeElement", "FakePage",
  "FakePaint", "FakeTarget", "FakeWindow", "Frames", "Grads", "Grid", "Hit", "Host", "Hyper",
  "Instance", "KeyEventish", "Listens", "LockOwner", "Loop", "LoopOptions", "Mat4", "Mesh",
  "MouseEventish", "Options", "Overlay", "Packed", "PageOptions", "Paint", "Painter", "Policy",
  "Renderer", "RendererAssets", "Rollout", "Scene", "SceneCapacity", "Shader", "Shape",
  "SoundDevice", "Stick", "Surface", "Tone", "TouchEventish", "Touchish", "UpdateStats", "Vec3",
  "add", "addBox", "angleDelta", "angleVectors", "applyGrads", "attachButtons", "attenuate",
  "backward", "builder", "cellAt", "clamp", "clearGrads", "colOf", "computeAdvantages",
  "copyPolicy", "createAdam", "createControls", "createLoop", "createPolicy", "createRenderer",
  "createRollout", "createScene", "createSoundDevice", "cross", "deg", "dist", "dist2d", "dot",
  "fakePage", "fbm", "findPath", "finish", "forward", "greedy", "hash", "headOffset", "inside",
  "length", "length2d", "lerp", "logitCount", "mat4", "mergeSolid", "modelMatrix", "mul",
  "multiply", "newActivations", "nextInstance", "noise", "normalize", "overlay", "pack",
  "paintAtlas", "painter", "paramCount", "perspective", "pushLine", "quadMesh", "rad", "rayBox",
  "resetScene", "rng", "rowOf", "sample", "scale", "segmentLength", "softmax", "sub",
  "tensorsOf", "traceWorld", "unpack", "update", "v3", "viewMatrix", "visible", "worldOf",
  "zeroGrads", "zeroPolicy",
];
test("the door offers exactly the names written down here", () => {
@@ -83,3 +83,14 @@
export {
  paramCount, sample, softmax, tensorsOf, unpack, zeroPolicy,
} from "./net.ts";
export type { Activations, Packed, Policy, Shape } from "./net.ts";
/**
 * PPO over a policy: a rollout a caller filled in, a clipped update out. A game
 * needs none of this — it ships weights, not a trainer — and no browser loads
 * it: the committed bundles carry not a byte of this module.
 */
export {
  applyGrads, backward, clearGrads, computeAdvantages, createAdam, createRollout, update,
  zeroGrads, DEFAULT_HYPER,
} from "./learn.ts";
export type { Adam, Grads, Hyper, Rollout, UpdateStats } from "./learn.ts";
@@ -0,0 +1,200 @@
/**
 * The trainer, checked from the two ends that can catch it being wrong: the
 * backward pass against finite differences, and a whole run against something
 * that learns. Everything here reaches the trainer through `index.ts`, the way
 * a caller outside the engine has to — the door is the surface under test.
 */
import { expect, test } from "bun:test";
import { readFileSync, readdirSync } from "node:fs";
import { join, resolve } from "node:path";
import {
  applyGrads, backward, computeAdvantages, createAdam, createPolicy, createRollout, forward,
  newActivations, rng, sample, tensorsOf, update, zeroGrads, DEFAULT_HYPER,
  type Hyper, type Policy, type Shape,
} from "./index.ts";
const SHAPE: Shape = { obs: 5, hidden: 6, heads: [3, 2] };
/**
 * The backward pass is written by hand, so it is checked against the only
 * thing that cannot be wrong in the same way: the function itself, evaluated
 * twice a hair apart.
 */
test("the analytic gradient matches finite differences everywhere", () => {
  const random = rng(4242);
  const p = createPolicy(SHAPE, random);
  // A weighted sum of the outputs, so dLoss/dLogits and dLoss/dValue are known.
  const dLogits = new Float32Array([0.7, -0.3, 0.2, -0.9, 0.4]);
  const dValue = 0.6;
  const obs = new Float32Array(SHAPE.obs);
  for (let i = 0; i < obs.length; i++) obs[i] = random() * 2 - 1;
  const loss = (policy: Policy): number => {
    const a = forward(policy, obs, newActivations(SHAPE));
    let sum = dValue * a.value;
    for (let i = 0; i < dLogits.length; i++) sum += (dLogits[i] as number) * (a.logits[i] as number);
    return sum;
  };
  const g = zeroGrads(SHAPE);
  const a = forward(p, obs, newActivations(SHAPE));
  backward(p, obs, 0, a, dLogits, dValue, g, {
    dh1: new Float32Array(SHAPE.hidden),
    dh2: new Float32Array(SHAPE.hidden),
  });
  const params = tensorsOf(p);
  const grads = tensorsOf(g);
  const eps = 1e-3;
  let checked = 0;
  for (let k = 0; k < params.length; k++) {
    const t = params[k] as Float32Array;
    const gt = grads[k] as Float32Array;
    for (let i = 0; i < t.length; i += Math.max(1, Math.floor(t.length / 7))) {
      const original = t[i] as number;
      t[i] = original + eps;
      const up = loss(p);
      t[i] = original - eps;
      const down = loss(p);
      t[i] = original;
      const numeric = (up - down) / (2 * eps);
      expect(Math.abs(numeric - (gt[i] as number))).toBeLessThan(2e-3 + Math.abs(numeric) * 5e-3);
      checked += 1;
    }
  }
  expect(checked).toBeGreaterThan(30);
});
test("advantages telescope backwards and stop at the end of an episode", () => {
  const r = createRollout(SHAPE, 1, 3);
  r.rewards.set([1, 1, 1]);
  r.values.set([0, 0, 0]);
  r.dones.set([0, 1, 0]);
  r.lastValues[0] = 0;
  computeAdvantages(r, 1, 1);
  // Step 2 is the last and is not terminal, so it bootstraps from lastValues=0.
  expect(r.advantages[2]).toBeCloseTo(1, 6);
  // Step 1 ended the episode: its advantage is its own reward and nothing past it.
  expect(r.advantages[1]).toBeCloseTo(1, 6);
  // Step 0 sees step 1 through the discount, but not step 2 through the seam.
  expect(r.advantages[0]).toBeCloseTo(2, 6);
  expect(r.returns[0]).toBeCloseTo(2, 6);
});
test("a gradient step moves the parameters and the norm is reported", () => {
  const p = createPolicy(SHAPE, rng(7));
  const g = zeroGrads(SHAPE);
  g.b1.fill(1);
  const before = (p.b1[0] as number);
  const adam = createAdam(p, DEFAULT_HYPER);
  const norm = applyGrads(p, g, adam, 1);
  expect(norm).toBeCloseTo(Math.sqrt(SHAPE.hidden), 5);
  // Adam's first step is about one learning rate, whatever the gradient size.
  expect(before - (p.b1[0] as number)).toBeCloseTo(DEFAULT_HYPER.learningRate, 5);
});
// --- an environment that is not a game ------------------------------------
/**
 * The proof that this module is the engine's rather than the shooter's: an
 * environment written out in this file, with no map, no tick and no weapon in
 * it, learned through the door and nothing else.
 *
 * It is a two-armed bandit with the answer in plain sight: the observation is
 * which arm pays this pull, one-hot, and pulling that arm pays 1 and the other
 * pays 0. A policy that ignores its observation scores a half however long it
 * trains, so the floor below is not something a drifting bias reaches — the
 * observation has to arrive at the logits through both layers for the number to
 * move. Every pull is its own episode, so GAE has nothing to telescope and what
 * is being tested is the surrogate and the optimiser.
 *
 * Small on purpose: 128 pulls a rollout, a hidden layer of eight, 40 updates,
 * which is a few hundredths of a second. The seed is fixed and nothing here
 * reads a clock, so the numbers it produces are the same on every machine.
 */
const BANDIT: Shape = { obs: 2, hidden: 8, heads: [2] };
const BANDIT_ENVS = 8;
const BANDIT_STEPS = 16;
const BANDIT_UPDATES = 40;
/** Train on the bandit from `seed`, and report the mean reward of each rollout. */
function trainBandit(seed: number): number[] {
  const random = rng(seed);
  // A bandit is not a duel: the batch is small, so the minibatch is the batch,
  // and the step is the one a toy can afford rather than the default.
  const hyper: Hyper = { ...DEFAULT_HYPER, learningRate: 0.02, minibatch: 64 };
  const policy = createPolicy(BANDIT, random);
  const adam = createAdam(policy, hyper);
  const rollout = createRollout(BANDIT, BANDIT_ENVS, BANDIT_STEPS);
  const act = newActivations(BANDIT);
  const actions = new Int32Array(1);
  const means: number[] = [];
  for (let iter = 0; iter < BANDIT_UPDATES; iter++) {
    let paid = 0;
    for (let t = 0; t < BANDIT_STEPS; t++)
      for (let e = 0; e < BANDIT_ENVS; e++) {
        const i = t * BANDIT_ENVS + e;
        const paying = random() < 0.5 ? 0 : 1;
        rollout.obs[i * BANDIT.obs] = paying === 0 ? 1 : 0;
        rollout.obs[i * BANDIT.obs + 1] = paying === 1 ? 1 : 0;
        forward(policy, rollout.obs, act, i * BANDIT.obs);
        rollout.values[i] = act.value;
        rollout.logProbs[i] = sample(BANDIT, act.logits, random, actions);
        const pulled = actions[0] as number;
        rollout.actions[i] = pulled;
        rollout.rewards[i] = pulled === paying ? 1 : 0;
        rollout.dones[i] = 1;
        paid += pulled === paying ? 1 : 0;
      }
    computeAdvantages(rollout, hyper.gamma, hyper.lambda);
    update(policy, rollout, adam, random);
    means.push(paid / (BANDIT_ENVS * BANDIT_STEPS));
  }
  return means;
}
test("the trainer teaches an environment that has never heard of this game", () => {
  const means = trainBandit(20260920);
  expect(means.length).toBe(BANDIT_UPDATES);
  // It starts where a coin starts: a fresh policy is nearly uniform.
  expect(means[0] as number).toBeGreaterThan(0.35);
  expect(means[0] as number).toBeLessThan(0.65);
  // And ends past a floor no observation-blind policy reaches, on the rollout
  // the updates were collected from — not a separate greedy evaluation, so the
  // number includes every pull the policy still spends exploring.
  const last = means[BANDIT_UPDATES - 1] as number;
  expect(last).toBeGreaterThan(0.9);
  // The rise is the run's, not one lucky rollout's: the last five average high.
  const tail = means.slice(-5).reduce((a, b) => a + b, 0) / 5;
  expect(tail).toBeGreaterThan(0.9);
});
test("the same seed trains the same way, and another seed learns too", () => {
  expect(trainBandit(20260920)).toEqual(trainBandit(20260920));
  const other = trainBandit(7);
  expect(other.slice(-5).reduce((a, b) => a + b, 0) / 5).toBeGreaterThan(0.9);
});
/**
 * The door re-exports a module no browser will ever load, and this is what says
 * so: the committed bundles, read the way `page.test.ts` reads them. A game
 * ships weights and runs them; if a byte of the trainer lands in a bundle the
 * design is wrong, not this test.
 */
test("no committed bundle carries the trainer", () => {
  const dir = join(resolve(import.meta.dir, "..", ".."), "static");
  const bundles = readdirSync(dir).filter((file) => file.endsWith(".js"));
  expect(bundles.length).toBeGreaterThanOrEqual(3);
  for (const file of bundles) {
    const code = readFileSync(join(dir, file), "utf8");
    // The reader is alive: a bundle is a bundle and not an empty file.
    expect(code.length).toBeGreaterThan(1000);
    // A minifier renames a variable and keeps a property, so these five are
    // this module's fingerprint: the hyperparameters it reads, the field a
    // caller bootstraps and the statistics it returns.
    for (const key of ["maxGradNorm", "valueCoef", "lastValues", "clipFraction", "policyLoss"])
      expect(code).not.toContain(key);
  }
});
@@ -6,13 +6,23 @@
 *
 * The backward pass is written out by hand because the network is three
 * matrices; an autodiff library would be more code to maintain than this file.
 * `gradientCheck` in the tests holds it honest against finite differences.
 * The gradient check in the tests holds it honest against finite differences.
 *
 * Nothing in here knows what an environment is. It is handed a `Rollout` — a
 * slab of numbers a caller filled by stepping something — and it hands back a
 * changed `Policy` and a line of statistics; what the observations mean, what
 * the actions do and where the rewards came from never arrive. `learn.test.ts`
 * trains a two-armed bandit written in the test itself to prove that, and the
 * game this repository ships trains through the same door with the same calls.
 *
 * No browser ever loads this: a game needs a trained `Policy`, not a trainer,
 * and the committed bundles carry none of it.
 */
import {
  forward, headOffset, logitCount, newActivations, softmax, zeroPolicy,
  tensorsOf, type Activations, type Policy, type Shape,
} from "../engine/index.ts";
} from "./net.ts";
export type Hyper = {
  gamma: number;
@@ -26,12 +36,17 @@
export type Hyper = {
  maxGradNorm: number;
};
/**
 * A starting point, not a recommendation: the numbers a caller gets when it
 * has no opinion yet. Tuning is the caller's, and `update` reads the `Hyper`
 * it was handed rather than these, so a game's settings never leak in here.
 */
export const DEFAULT_HYPER: Hyper = {
  gamma: 0.99,
  lambda: 0.95,
  clip: 0.2,
  // Summed over six heads, so it is worth six times what the same number means
  // for a single head: at 0.01 the bots never stop twitching long enough to aim.
  // The entropy bonus is summed over every head, so it is worth as many times
  // this number as a policy has heads: a six-headed policy at 0.01 twitches.
  entropy: 0.003,
  valueCoef: 0.5,
  learningRate: 1e-3,
@@ -114,6 +114,19 @@
export type Game = {
 * differing only in which short names the minifier handed out, because it
 * paints no texture and ships none of this. Both fit today's ceilings, so
 * neither moved.
 * #107 moved PPO behind the door too — `src/engine/learn.ts`, which no browser
 * will ever load — and not one of the three bundles gained a byte: app 72,048
 * raw, range 28,978 raw, example 8,754 raw, each the same number as before it
 * moved. The gzipped sizes moved by a byte or two (app 33,457 → 33,482, example
 * 4,228 → 4,227) because the minifier handed out different short names over a
 * module graph that changed shape, which is the same thing that happened for
 * #106. That the trainer is absent is a test rather than an inference:
 * `src/engine/learn.test.ts` reads every committed bundle for `maxGradNorm`,
 * `valueCoef`, `lastValues`, `clipFraction` and `policyLoss` and finds none of
 * them. Measured 2026-09-20 after #107, on the bundles this branch commits.
 * The range's numbers here are #121's moving plates and #125's painter at once:
 * both landed a range bundle and neither swept the other's, so static/ held two
 * and every reader of it failed until a plain `bun run build` resolved them.
 */
export const GAMES: readonly Game[] = [
  {
@@ -1,87 +0,0 @@
import { expect, test } from "bun:test";
import {
  createPolicy, forward, newActivations, rng, tensorsOf, type Policy, type Shape,
} from "../engine/index.ts";
import {
  applyGrads, backward, computeAdvantages, createAdam, createRollout, zeroGrads, DEFAULT_HYPER,
} from "./ppo.ts";
const SHAPE: Shape = { obs: 5, hidden: 6, heads: [3, 2] };
/**
 * The backward pass is written by hand, so it is checked against the only
 * thing that cannot be wrong in the same way: the function itself, evaluated
 * twice a hair apart.
 */
test("the analytic gradient matches finite differences everywhere", () => {
  const random = rng(4242);
  const p = createPolicy(SHAPE, random);
  // A weighted sum of the outputs, so dLoss/dLogits and dLoss/dValue are known.
  const dLogits = new Float32Array([0.7, -0.3, 0.2, -0.9, 0.4]);
  const dValue = 0.6;
  const obs = new Float32Array(SHAPE.obs);
  for (let i = 0; i < obs.length; i++) obs[i] = random() * 2 - 1;
  const loss = (policy: Policy): number => {
    const a = forward(policy, obs, newActivations(SHAPE));
    let sum = dValue * a.value;
    for (let i = 0; i < dLogits.length; i++) sum += (dLogits[i] as number) * (a.logits[i] as number);
    return sum;
  };
  const g = zeroGrads(SHAPE);
  const a = forward(p, obs, newActivations(SHAPE));
  backward(p, obs, 0, a, dLogits, dValue, g, {
    dh1: new Float32Array(SHAPE.hidden),
    dh2: new Float32Array(SHAPE.hidden),
  });
  const params = tensorsOf(p);
  const grads = tensorsOf(g);
  const eps = 1e-3;
  let checked = 0;
  for (let k = 0; k < params.length; k++) {
    const t = params[k] as Float32Array;
    const gt = grads[k] as Float32Array;
    for (let i = 0; i < t.length; i += Math.max(1, Math.floor(t.length / 7))) {
      const original = t[i] as number;
      t[i] = original + eps;
      const up = loss(p);
      t[i] = original - eps;
      const down = loss(p);
      t[i] = original;
      const numeric = (up - down) / (2 * eps);
      expect(Math.abs(numeric - (gt[i] as number))).toBeLessThan(2e-3 + Math.abs(numeric) * 5e-3);
      checked += 1;
    }
  }
  expect(checked).toBeGreaterThan(30);
});
test("advantages telescope backwards and stop at the end of an episode", () => {
  const r = createRollout(SHAPE, 1, 3);
  r.rewards.set([1, 1, 1]);
  r.values.set([0, 0, 0]);
  r.dones.set([0, 1, 0]);
  r.lastValues[0] = 0;
  computeAdvantages(r, 1, 1);
  // Step 2 is the last and is not terminal, so it bootstraps from lastValues=0.
  expect(r.advantages[2]).toBeCloseTo(1, 6);
  // Step 1 ended the episode: its advantage is its own reward and nothing past it.
  expect(r.advantages[1]).toBeCloseTo(1, 6);
  // Step 0 sees step 1 through the discount, but not step 2 through the seam.
  expect(r.advantages[0]).toBeCloseTo(2, 6);
  expect(r.returns[0]).toBeCloseTo(2, 6);
});
test("a gradient step moves the parameters and the norm is reported", () => {
  const p = createPolicy(SHAPE, rng(7));
  const g = zeroGrads(SHAPE);
  g.b1.fill(1);
  const before = (p.b1[0] as number);
  const adam = createAdam(p, DEFAULT_HYPER);
  const norm = applyGrads(p, g, adam, 1);
  expect(norm).toBeCloseTo(Math.sqrt(SHAPE.hidden), 5);
  // Adam's first step is about one learning rate, whatever the gradient size.
  expect(before - (p.b1[0] as number)).toBeCloseTo(DEFAULT_HYPER.learningRate, 5);
});
		@@ -45,7 +45,8 @@ FORGE_APP_DATA=./data bun run src/tty.ts # the terminal interface
45	45	published shape so the habits transfer, and the rest is original.
46	46	- Learned, and measured. The bots do not aim by rule. A 32-input, two-layer
47	47	policy decides where to point, when to shoot and which way to move, trained with
48		PPO in `src/rl` against millions of ticks of this same simulation. The numbers on
	48	the engine's PPO against millions of ticks of this same simulation, in the arena
	49	under `src/rl`. The numbers on
49	50	the page and in the terminal's `policy` tab are what the run that produced the
50	51	shipped weights measured, on the same fixed duels for the policy and for the
51	52	scripted bots it replaced.
		@@ -402,10 +403,11 @@ renderer, no atlas, no mesh and no sound ship in that bundle, and
402	403	`bundle.test.ts` boots the committed bundle and reads it for what is not in
403	404	it (`webgl2`, `AudioContext`, `Math.tanh`): a door is only worth having if
404	405	importing it costs what you use.
405		- `src/rl/` — training, which no game module imports: `arena` is the environment
406		(a real duel on de_forge with the round and the economy taken out) and `ppo` is
407		the algorithm, with its backward pass checked against finite differences.
408		`scripts/train.ts` runs them and writes `src/client/weights.ts`. It is the
	406	- `src/rl/` — training, which no game module imports: `arena` is the environment,
	407	a real duel on de_forge with the round and the economy taken out. The algorithm
	408	is not in here — PPO is `src/engine/learn.ts`, behind the door, because nothing
	409	in it is about shooting; what is here is this game's arena.
	410	`scripts/train.ts` runs the two together and writes `src/client/weights.ts`. It is the
409	411	shooter's training and `src/independence.test.ts` counts it as the shooter:
410	412	it reads `src/client/` module by module because that game is the environment,
411	413	and for the same reason it may not read `src/range/`.
		@@ -571,10 +573,14 @@ now a policy learned with PPO.
571	573	movement, the real bullets and the real damage, and no round, economy or bomb.
572	574	It steps about 13,000 ticks a second on one core, allocation-free; the policy
573	575	decides every fourth tick and the decision is held in between.
574		- How it learned — `src/rl/ppo.ts`, about three hundred lines: GAE, a
575		clipped surrogate, an entropy bonus, Adam, and a backward pass written out by
576		hand and checked against finite differences in the tests. There is a
577		curriculum on how far apart the duel starts and on how hard the opponent is,
	576	- How it learned — `src/engine/learn.ts`, about three hundred lines behind
	577	the engine's door: GAE, a clipped surrogate, an entropy bonus, Adam, and a
	578	backward pass written out by hand and checked against finite differences in
	579	the tests. It is the engine's rather than this game's because nothing in it
	580	names a map, a tick or a weapon — the same door trains a two-armed bandit
	581	written inside `src/engine/learn.test.ts`, which is how that claim is checked.
	582	The curriculum on top of it is the shooter's and lives in `scripts/train.ts`:
	583	there is a curriculum on how far apart the duel starts and on how hard the opponent is,
578	584	because handed a good opponent from the first step the policy learns to hide
579	585	and never comes back from it.
580	586	- What it costs you — about eight kilobytes of int8 in the bundle, one
		@@ -9,6 +9,10 @@
9	9	* Nothing here is imported by the server or the page: this file runs on a
10	10	* machine with a few minutes to spare, and its only output is a table of
11	11	* numbers and about eight kilobytes of int8.
	12	*
	13	* The PPO is the engine's, reached through the door like everything else it
	14	* offers; what is the shooter's is the arena it collects from, the observation
	15	* it fills, the heads it decodes and the curriculum below.
12	16	*/
13	17
14	18	import { writeFileSync } from "node:fs";
		@@ -16,17 +20,15 @@ import { join } from "node:path";
16	20	import { HEADS, decode } from "../src/client/action.ts";
17	21	import { botThink, makeBrain } from "../src/client/bots.ts";
18	22	import {
19		copyPolicy, createPolicy, forward, greedy, newActivations, pack, rng, sample, unpack,
20		type Policy, type Shape,
	23	computeAdvantages, copyPolicy, createAdam, createPolicy, createRollout, forward, greedy,
	24	newActivations, pack, rng, sample, unpack, update, DEFAULT_HYPER,
	25	type Hyper, type Policy, type Shape,
21	26	} from "../src/engine/index.ts";
22	27	import { OBS_SIZE } from "../src/client/observe.ts";
23	28	import {
24	29	createArena, EPISODE_TICKS, observeFoe, observeLearner, policyFoe, reset, scriptedFoe, SKIP,
25	30	step, stepInput, stillFoe, type Arena,
26	31	} from "../src/rl/arena.ts";
27		import {
28		computeAdvantages, createAdam, createRollout, update, DEFAULT_HYPER, type Hyper,
29		} from "../src/rl/ppo.ts";
30	32
31	33	export const SHAPE: Shape = { obs: OBS_SIZE, hidden: 64, heads: HEADS };
32	34
		@@ -420,7 +420,7 @@ It is the one group behind the door that a game may reasonably never open — th
420	420	shooter runs it every tick to fight with, and the range ships not a byte of it —
421	421	and it is here because nothing in it is about shooting. The example does not use
422	422	it either, which is why this section has no code block: read
423		`src/client/bots.ts` for a policy driving a game and `src/rl/ppo.ts` for one
	423	`src/client/bots.ts` for a policy driving a game and `scripts/train.ts` for one
424	424	being trained.
425	425
426	426	What a game brings first is a `Shape`: how many numbers its observation has,
		@@ -459,6 +459,54 @@ are and what each one is worth is `src/client/observe.ts`; which head is a turn
459	459	and which is a trigger is `src/client/action.ts`; both are the shooter's, and a
460	460	second game that wanted an agent would write its own two and share this.
461	461
	462	### learn
	463
	464	PPO: how the weights the section above ships got their numbers. **A game needs
	465	none of this.** A game ships a trained `Policy` and runs `forward`; training
	466	happens once, on a machine with minutes to spare, in a script nothing serves.
	467	Nothing in here ever reaches a browser — no committed bundle carries a byte of
	468	it, which is a ceiling in `src/games.ts` and a test that reads the files — and
	469	it is behind the door for one reason: not a line of it is about shooting. The
	470	proof is in `src/engine/learn.test.ts`, which trains a two-armed bandit written
	471	out inside the test, with no map, no tick and no weapon anywhere in it.
	472
	473	What a caller brings is an environment and a `Rollout`. `createRollout`
	474	takes a `Shape`, how many environments are stepped side by side and how many
	475	steps each one contributes, and hands back the flat arrays one batch of
	476	experience is: the observations end to end, the action each head took, the
	477	log-probability `sample` returned, the value `forward` predicted, the reward
	478	that came back and whether the step ended an episode. The caller fills those by
	479	stepping its own world — the engine never sees the world, only the numbers — and
	480	writes into `Rollout.lastValues` what the policy thinks each unfinished episode
	481	is still worth. Then `computeAdvantages` walks the slab backwards per
	482	environment, discounting by `Hyper.gamma` and `Hyper.lambda` and never looking
	483	through the seam where an episode ended, and fills in the advantages and the
	484	returns.
	485
	486	Then `update` is the step itself: a few epochs of shuffled minibatches over
	487	that rollout, each one a clipped surrogate with an entropy bonus and a value
	488	loss, ending in an Adam step. It takes the policy, the rollout, an `Adam` from
	489	`createAdam` and the same seeded generator the rollout was sampled with, and
	490	returns an `UpdateStats` — the policy loss, the value loss, the entropy, the
	491	fraction of samples that clipped and the gradient norm — which is the line a
	492	training script prints and the first place a run that has gone wrong shows it.
	493
	494	The tuning is data the caller passes. `Hyper` is the nine numbers — the two
	495	discounts, the clip, the entropy bonus, the value coefficient, the learning
	496	rate, the epochs, the minibatch and the gradient-norm clip — and `DEFAULT_HYPER`
	497	is a starting point rather than a recommendation: `createAdam` is handed the one
	498	the caller means, and the shooter spreads `DEFAULT_HYPER` and overrides the
	499	minibatch for its batch size. No constant in this module is the shooter's.
	500
	501	The pieces under those are exported too, because the hand-written backward pass
	502	is the part most worth testing from outside: `Grads` is a `Policy` full of
	503	gradients, `zeroGrads` makes one, `clearGrads` empties it between minibatches,
	504	`backward` accumulates one sample into it given the derivative of the loss with
	505	respect to the logits and the value, and `applyGrads` clips the global norm and
	506	takes the Adam step, returning the norm it saw. The gradient check in
	507	`src/engine/learn.test.ts` holds all of that against finite differences, which
	508	is the only thing that catches an autodiff-free derivative being subtly wrong.
	509
462	510	## The seam, named as pairs
463	511
464	512	Two games ship here, and every place they differ is a place the engine chose to
		@@ -1,8 +1,8 @@
1	1	/**
2	2	* The boundary. `src/engine/` is the part a second game could use, so a module
3	3	* in here may import the engine and nothing else: not the game in
4		* `src/client/`, not the training in `src/rl/`, and no package — the engine has
5		* nothing to install. This test reads the sources rather than trusting the
	4	* `src/client/`, not the arena in `src/rl/` — the trainer is in here, what it
	5	* trains on is not — and no package, because the engine has nothing to install. This test reads the sources rather than trusting the
6	6	* convention, because an import is one line and nobody reviews a line that
7	7	* compiles.
8	8	*
		@@ -146,7 +146,7 @@ test("the engine has the sources this test is about", () => {
146	146	expect(files).toContain("trace.ts");
147	147	});
148	148
149		test("no engine module imports the game, the training, or a package", () => {
	149	test("no engine module imports the game, the arena, or a package", () => {
150	150	const offences: string[] = [];
151	151	for (const file of sources()) {
152	152	const source = readFileSync(join(ENGINE, file), "utf8");
		@@ -274,36 +274,32 @@ test("the example imports the door and nothing else, which is what it is for", (
274	274	* `newActivations`, `forward`, `headOffset`, `softmax`, `sample`, `greedy`,
275	275	* `pack`, `unpack`.
276	276	*
	277	* learn — `Hyper`, `DEFAULT_HYPER`, `Grads`, `zeroGrads`, `clearGrads`,
	278	* `backward`, `Adam`, `createAdam`, `applyGrads`, `Rollout`, `createRollout`,
	279	* `computeAdvantages`, `update`, `UpdateStats`.
	280	*
277	281	* Exported because `guide.test.ts` holds `GUIDE.md` against the same list: one
278	282	* copy of the surface, read twice, rather than two that drift.
279	283	*/
280	284	export const SURFACE = [
281		"Activations",
282		"Box", "Builder", "Burst", "Button", "ButtonOptions", "Camera", "Controls", "Ctor",
283		"FakeCanvas", "FakeDocument", "FakeElement", "FakePage", "FakePaint", "FakeTarget", "FakeWindow",
284		"Frames",
285		"Grid", "Hit", "Host", "Instance", "KeyEventish", "Listens", "LockOwner", "Loop", "LoopOptions",
286		"Mat4", "Mesh", "MouseEventish", "Options", "Overlay",
287		"Packed", "PageOptions", "Paint", "Painter", "Policy", "Renderer",
288		"RendererAssets",
289		"Scene", "SceneCapacity", "Shader", "Shape", "SoundDevice", "Stick", "Surface", "Tone",
290		"TouchEventish", "Touchish", "Vec3", "add", "addBox", "angleDelta", "angleVectors",
291		"attachButtons", "attenuate", "builder", "cellAt", "clamp", "colOf",
292		"copyPolicy", "createControls",
293		"createLoop", "createPolicy", "createRenderer", "createScene", "createSoundDevice", "cross",
294		"deg", "dist",
295		"dist2d", "dot", "fakePage", "fbm",
296		"findPath", "finish",
297		"forward", "greedy", "hash", "headOffset", "inside", "length", "length2d", "lerp",
298		"logitCount", "mat4", "mergeSolid",
299		"modelMatrix", "mul", "multiply",
300		"newActivations", "nextInstance", "noise", "normalize", "overlay",
301		"pack", "paintAtlas", "painter", "paramCount",
302		"perspective", "pushLine",
303		"quadMesh", "rad", "rayBox", "resetScene", "rng", "rowOf",
304		"sample", "scale", "segmentLength", "softmax", "sub", "tensorsOf",
305		"traceWorld", "unpack",
306		"v3", "viewMatrix", "visible", "worldOf", "zeroPolicy",
	285	"Activations", "Adam", "Box", "Builder", "Burst", "Button", "ButtonOptions", "Camera",
	286	"Controls", "Ctor", "DEFAULT_HYPER", "FakeCanvas", "FakeDocument", "FakeElement", "FakePage",
	287	"FakePaint", "FakeTarget", "FakeWindow", "Frames", "Grads", "Grid", "Hit", "Host", "Hyper",
	288	"Instance", "KeyEventish", "Listens", "LockOwner", "Loop", "LoopOptions", "Mat4", "Mesh",
	289	"MouseEventish", "Options", "Overlay", "Packed", "PageOptions", "Paint", "Painter", "Policy",
	290	"Renderer", "RendererAssets", "Rollout", "Scene", "SceneCapacity", "Shader", "Shape",
	291	"SoundDevice", "Stick", "Surface", "Tone", "TouchEventish", "Touchish", "UpdateStats", "Vec3",
	292	"add", "addBox", "angleDelta", "angleVectors", "applyGrads", "attachButtons", "attenuate",
	293	"backward", "builder", "cellAt", "clamp", "clearGrads", "colOf", "computeAdvantages",
	294	"copyPolicy", "createAdam", "createControls", "createLoop", "createPolicy", "createRenderer",
	295	"createRollout", "createScene", "createSoundDevice", "cross", "deg", "dist", "dist2d", "dot",
	296	"fakePage", "fbm", "findPath", "finish", "forward", "greedy", "hash", "headOffset", "inside",
	297	"length", "length2d", "lerp", "logitCount", "mat4", "mergeSolid", "modelMatrix", "mul",
	298	"multiply", "newActivations", "nextInstance", "noise", "normalize", "overlay", "pack",
	299	"paintAtlas", "painter", "paramCount", "perspective", "pushLine", "quadMesh", "rad", "rayBox",
	300	"resetScene", "rng", "rowOf", "sample", "scale", "segmentLength", "softmax", "sub",
	301	"tensorsOf", "traceWorld", "unpack", "update", "v3", "viewMatrix", "visible", "worldOf",
	302	"zeroGrads", "zeroPolicy",
307	303	];
308	304
309	305	test("the door offers exactly the names written down here", () => {
		@@ -83,3 +83,14 @@ export {
83	83	paramCount, sample, softmax, tensorsOf, unpack, zeroPolicy,
84	84	} from "./net.ts";
85	85	export type { Activations, Packed, Policy, Shape } from "./net.ts";
	86
	87	/**
	88	* PPO over a policy: a rollout a caller filled in, a clipped update out. A game
	89	* needs none of this — it ships weights, not a trainer — and no browser loads
	90	* it: the committed bundles carry not a byte of this module.
	91	*/
	92	export {
	93	applyGrads, backward, clearGrads, computeAdvantages, createAdam, createRollout, update,
	94	zeroGrads, DEFAULT_HYPER,
	95	} from "./learn.ts";
	96	export type { Adam, Grads, Hyper, Rollout, UpdateStats } from "./learn.ts";
		@@ -0,0 +1,200 @@
	1	/**
	2	* The trainer, checked from the two ends that can catch it being wrong: the
	3	* backward pass against finite differences, and a whole run against something
	4	* that learns. Everything here reaches the trainer through `index.ts`, the way
	5	* a caller outside the engine has to — the door is the surface under test.
	6	*/
	7
	8	import { expect, test } from "bun:test";
	9	import { readFileSync, readdirSync } from "node:fs";
	10	import { join, resolve } from "node:path";
	11	import {
	12	applyGrads, backward, computeAdvantages, createAdam, createPolicy, createRollout, forward,
	13	newActivations, rng, sample, tensorsOf, update, zeroGrads, DEFAULT_HYPER,
	14	type Hyper, type Policy, type Shape,
	15	} from "./index.ts";
	16
	17	const SHAPE: Shape = { obs: 5, hidden: 6, heads: [3, 2] };
	18
	19	/**
	20	* The backward pass is written by hand, so it is checked against the only
	21	* thing that cannot be wrong in the same way: the function itself, evaluated
	22	* twice a hair apart.
	23	*/
	24	test("the analytic gradient matches finite differences everywhere", () => {
	25	const random = rng(4242);
	26	const p = createPolicy(SHAPE, random);
	27	// A weighted sum of the outputs, so dLoss/dLogits and dLoss/dValue are known.
	28	const dLogits = new Float32Array([0.7, -0.3, 0.2, -0.9, 0.4]);
	29	const dValue = 0.6;
	30	const obs = new Float32Array(SHAPE.obs);
	31	for (let i = 0; i < obs.length; i++) obs[i] = random() * 2 - 1;
	32
	33	const loss = (policy: Policy): number => {
	34	const a = forward(policy, obs, newActivations(SHAPE));
	35	let sum = dValue * a.value;
	36	for (let i = 0; i < dLogits.length; i++) sum += (dLogits[i] as number) * (a.logits[i] as number);
	37	return sum;
	38	};
	39
	40	const g = zeroGrads(SHAPE);
	41	const a = forward(p, obs, newActivations(SHAPE));
	42	backward(p, obs, 0, a, dLogits, dValue, g, {
	43	dh1: new Float32Array(SHAPE.hidden),
	44	dh2: new Float32Array(SHAPE.hidden),
	45	});
	46
	47	const params = tensorsOf(p);
	48	const grads = tensorsOf(g);
	49	const eps = 1e-3;
	50	let checked = 0;
	51	for (let k = 0; k < params.length; k++) {
	52	const t = params[k] as Float32Array;
	53	const gt = grads[k] as Float32Array;
	54	for (let i = 0; i < t.length; i += Math.max(1, Math.floor(t.length / 7))) {
	55	const original = t[i] as number;
	56	t[i] = original + eps;
	57	const up = loss(p);
	58	t[i] = original - eps;
	59	const down = loss(p);
	60	t[i] = original;
	61	const numeric = (up - down) / (2 * eps);
	62	expect(Math.abs(numeric - (gt[i] as number))).toBeLessThan(2e-3 + Math.abs(numeric) * 5e-3);
	63	checked += 1;
	64	}
	65	}
	66	expect(checked).toBeGreaterThan(30);
	67	});
	68
	69	test("advantages telescope backwards and stop at the end of an episode", () => {
	70	const r = createRollout(SHAPE, 1, 3);
	71	r.rewards.set([1, 1, 1]);
	72	r.values.set([0, 0, 0]);
	73	r.dones.set([0, 1, 0]);
	74	r.lastValues[0] = 0;
	75	computeAdvantages(r, 1, 1);
	76	// Step 2 is the last and is not terminal, so it bootstraps from lastValues=0.
	77	expect(r.advantages[2]).toBeCloseTo(1, 6);
	78	// Step 1 ended the episode: its advantage is its own reward and nothing past it.
	79	expect(r.advantages[1]).toBeCloseTo(1, 6);
	80	// Step 0 sees step 1 through the discount, but not step 2 through the seam.
	81	expect(r.advantages[0]).toBeCloseTo(2, 6);
	82	expect(r.returns[0]).toBeCloseTo(2, 6);
	83	});
	84
	85	test("a gradient step moves the parameters and the norm is reported", () => {
	86	const p = createPolicy(SHAPE, rng(7));
	87	const g = zeroGrads(SHAPE);
	88	g.b1.fill(1);
	89	const before = (p.b1[0] as number);
	90	const adam = createAdam(p, DEFAULT_HYPER);
	91	const norm = applyGrads(p, g, adam, 1);
	92	expect(norm).toBeCloseTo(Math.sqrt(SHAPE.hidden), 5);
	93	// Adam's first step is about one learning rate, whatever the gradient size.
	94	expect(before - (p.b1[0] as number)).toBeCloseTo(DEFAULT_HYPER.learningRate, 5);
	95	});
	96
	97	// --- an environment that is not a game ------------------------------------
	98
	99	/**
	100	* The proof that this module is the engine's rather than the shooter's: an
	101	* environment written out in this file, with no map, no tick and no weapon in
	102	* it, learned through the door and nothing else.
	103	*
	104	* It is a two-armed bandit with the answer in plain sight: the observation is
	105	* which arm pays this pull, one-hot, and pulling that arm pays 1 and the other
	106	* pays 0. A policy that ignores its observation scores a half however long it
	107	* trains, so the floor below is not something a drifting bias reaches — the
	108	* observation has to arrive at the logits through both layers for the number to
	109	* move. Every pull is its own episode, so GAE has nothing to telescope and what
	110	* is being tested is the surrogate and the optimiser.
	111	*
	112	* Small on purpose: 128 pulls a rollout, a hidden layer of eight, 40 updates,
	113	* which is a few hundredths of a second. The seed is fixed and nothing here
	114	* reads a clock, so the numbers it produces are the same on every machine.
	115	*/
	116	const BANDIT: Shape = { obs: 2, hidden: 8, heads: [2] };
	117	const BANDIT_ENVS = 8;
	118	const BANDIT_STEPS = 16;
	119	const BANDIT_UPDATES = 40;
	120
	121	/** Train on the bandit from `seed`, and report the mean reward of each rollout. */
	122	function trainBandit(seed: number): number[] {
	123	const random = rng(seed);
	124	// A bandit is not a duel: the batch is small, so the minibatch is the batch,
	125	// and the step is the one a toy can afford rather than the default.
	126	const hyper: Hyper = { ...DEFAULT_HYPER, learningRate: 0.02, minibatch: 64 };
	127	const policy = createPolicy(BANDIT, random);
	128	const adam = createAdam(policy, hyper);
	129	const rollout = createRollout(BANDIT, BANDIT_ENVS, BANDIT_STEPS);
	130	const act = newActivations(BANDIT);
	131	const actions = new Int32Array(1);
	132	const means: number[] = [];
	133
	134	for (let iter = 0; iter < BANDIT_UPDATES; iter++) {
	135	let paid = 0;
	136	for (let t = 0; t < BANDIT_STEPS; t++)
	137	for (let e = 0; e < BANDIT_ENVS; e++) {
	138	const i = t * BANDIT_ENVS + e;
	139	const paying = random() < 0.5 ? 0 : 1;
	140	rollout.obs[i * BANDIT.obs] = paying === 0 ? 1 : 0;
	141	rollout.obs[i * BANDIT.obs + 1] = paying === 1 ? 1 : 0;
	142	forward(policy, rollout.obs, act, i * BANDIT.obs);
	143	rollout.values[i] = act.value;
	144	rollout.logProbs[i] = sample(BANDIT, act.logits, random, actions);
	145	const pulled = actions[0] as number;
	146	rollout.actions[i] = pulled;
	147	rollout.rewards[i] = pulled === paying ? 1 : 0;
	148	rollout.dones[i] = 1;
	149	paid += pulled === paying ? 1 : 0;
	150	}
	151	computeAdvantages(rollout, hyper.gamma, hyper.lambda);
	152	update(policy, rollout, adam, random);
	153	means.push(paid / (BANDIT_ENVS * BANDIT_STEPS));
	154	}
	155	return means;
	156	}
	157
	158	test("the trainer teaches an environment that has never heard of this game", () => {
	159	const means = trainBandit(20260920);
	160	expect(means.length).toBe(BANDIT_UPDATES);
	161	// It starts where a coin starts: a fresh policy is nearly uniform.
	162	expect(means[0] as number).toBeGreaterThan(0.35);
	163	expect(means[0] as number).toBeLessThan(0.65);
	164	// And ends past a floor no observation-blind policy reaches, on the rollout
	165	// the updates were collected from — not a separate greedy evaluation, so the
	166	// number includes every pull the policy still spends exploring.
	167	const last = means[BANDIT_UPDATES - 1] as number;
	168	expect(last).toBeGreaterThan(0.9);
	169	// The rise is the run's, not one lucky rollout's: the last five average high.
	170	const tail = means.slice(-5).reduce((a, b) => a + b, 0) / 5;
	171	expect(tail).toBeGreaterThan(0.9);
	172	});
	173
	174	test("the same seed trains the same way, and another seed learns too", () => {
	175	expect(trainBandit(20260920)).toEqual(trainBandit(20260920));
	176	const other = trainBandit(7);
	177	expect(other.slice(-5).reduce((a, b) => a + b, 0) / 5).toBeGreaterThan(0.9);
	178	});
	179
	180	/**
	181	* The door re-exports a module no browser will ever load, and this is what says
	182	* so: the committed bundles, read the way `page.test.ts` reads them. A game
	183	* ships weights and runs them; if a byte of the trainer lands in a bundle the
	184	* design is wrong, not this test.
	185	*/
	186	test("no committed bundle carries the trainer", () => {
	187	const dir = join(resolve(import.meta.dir, "..", ".."), "static");
	188	const bundles = readdirSync(dir).filter((file) => file.endsWith(".js"));
	189	expect(bundles.length).toBeGreaterThanOrEqual(3);
	190	for (const file of bundles) {
	191	const code = readFileSync(join(dir, file), "utf8");
	192	// The reader is alive: a bundle is a bundle and not an empty file.
	193	expect(code.length).toBeGreaterThan(1000);
	194	// A minifier renames a variable and keeps a property, so these five are
	195	// this module's fingerprint: the hyperparameters it reads, the field a
	196	// caller bootstraps and the statistics it returns.
	197	for (const key of ["maxGradNorm", "valueCoef", "lastValues", "clipFraction", "policyLoss"])
	198	expect(code).not.toContain(key);
	199	}
	200	});
		@@ -6,13 +6,23 @@
6	6	*
7	7	* The backward pass is written out by hand because the network is three
8	8	* matrices; an autodiff library would be more code to maintain than this file.
9		* `gradientCheck` in the tests holds it honest against finite differences.
	9	* The gradient check in the tests holds it honest against finite differences.
	10	*
	11	* Nothing in here knows what an environment is. It is handed a `Rollout` — a
	12	* slab of numbers a caller filled by stepping something — and it hands back a
	13	* changed `Policy` and a line of statistics; what the observations mean, what
	14	* the actions do and where the rewards came from never arrive. `learn.test.ts`
	15	* trains a two-armed bandit written in the test itself to prove that, and the
	16	* game this repository ships trains through the same door with the same calls.
	17	*
	18	* No browser ever loads this: a game needs a trained `Policy`, not a trainer,
	19	* and the committed bundles carry none of it.
10	20	*/
11	21
12	22	import {
13	23	forward, headOffset, logitCount, newActivations, softmax, zeroPolicy,
14	24	tensorsOf, type Activations, type Policy, type Shape,
15		} from "../engine/index.ts";
	25	} from "./net.ts";
16	26
17	27	export type Hyper = {
18	28	gamma: number;
		@@ -26,12 +36,17 @@ export type Hyper = {
26	36	maxGradNorm: number;
27	37	};
28	38
	39	/**
	40	* A starting point, not a recommendation: the numbers a caller gets when it
	41	* has no opinion yet. Tuning is the caller's, and `update` reads the `Hyper`
	42	* it was handed rather than these, so a game's settings never leak in here.
	43	*/
29	44	export const DEFAULT_HYPER: Hyper = {
30	45	gamma: 0.99,
31	46	lambda: 0.95,
32	47	clip: 0.2,
33		// Summed over six heads, so it is worth six times what the same number means
34		// for a single head: at 0.01 the bots never stop twitching long enough to aim.
	48	// The entropy bonus is summed over every head, so it is worth as many times
	49	// this number as a policy has heads: a six-headed policy at 0.01 twitches.
35	50	entropy: 0.003,
36	51	valueCoef: 0.5,
37	52	learningRate: 1e-3,
		@@ -114,6 +114,19 @@ export type Game = {
114	114	* differing only in which short names the minifier handed out, because it
115	115	* paints no texture and ships none of this. Both fit today's ceilings, so
116	116	* neither moved.
	117	* #107 moved PPO behind the door too — `src/engine/learn.ts`, which no browser
	118	* will ever load — and not one of the three bundles gained a byte: app 72,048
	119	* raw, range 28,978 raw, example 8,754 raw, each the same number as before it
	120	* moved. The gzipped sizes moved by a byte or two (app 33,457 → 33,482, example
	121	* 4,228 → 4,227) because the minifier handed out different short names over a
	122	* module graph that changed shape, which is the same thing that happened for
	123	* #106. That the trainer is absent is a test rather than an inference:
	124	* `src/engine/learn.test.ts` reads every committed bundle for `maxGradNorm`,
	125	* `valueCoef`, `lastValues`, `clipFraction` and `policyLoss` and finds none of
	126	* them. Measured 2026-09-20 after #107, on the bundles this branch commits.
	127	* The range's numbers here are #121's moving plates and #125's painter at once:
	128	* both landed a range bundle and neither swept the other's, so static/ held two
	129	* and every reader of it failed until a plain `bun run build` resolved them.
117	130	*/
118	131	export const GAMES: readonly Game[] = [
119	132	{
		@@ -1,87 +0,0 @@
1		import { expect, test } from "bun:test";
2		import {
3		createPolicy, forward, newActivations, rng, tensorsOf, type Policy, type Shape,
4		} from "../engine/index.ts";
5		import {
6		applyGrads, backward, computeAdvantages, createAdam, createRollout, zeroGrads, DEFAULT_HYPER,
7		} from "./ppo.ts";
8
9		const SHAPE: Shape = { obs: 5, hidden: 6, heads: [3, 2] };
10
11		/**
12		* The backward pass is written by hand, so it is checked against the only
13		* thing that cannot be wrong in the same way: the function itself, evaluated
14		* twice a hair apart.
15		*/
16		test("the analytic gradient matches finite differences everywhere", () => {
17		const random = rng(4242);
18		const p = createPolicy(SHAPE, random);
19		// A weighted sum of the outputs, so dLoss/dLogits and dLoss/dValue are known.
20		const dLogits = new Float32Array([0.7, -0.3, 0.2, -0.9, 0.4]);
21		const dValue = 0.6;
22		const obs = new Float32Array(SHAPE.obs);
23		for (let i = 0; i < obs.length; i++) obs[i] = random() * 2 - 1;
24
25		const loss = (policy: Policy): number => {
26		const a = forward(policy, obs, newActivations(SHAPE));
27		let sum = dValue * a.value;
28		for (let i = 0; i < dLogits.length; i++) sum += (dLogits[i] as number) * (a.logits[i] as number);
29		return sum;
30		};
31
32		const g = zeroGrads(SHAPE);
33		const a = forward(p, obs, newActivations(SHAPE));
34		backward(p, obs, 0, a, dLogits, dValue, g, {
35		dh1: new Float32Array(SHAPE.hidden),
36		dh2: new Float32Array(SHAPE.hidden),
37		});
38
39		const params = tensorsOf(p);
40		const grads = tensorsOf(g);
41		const eps = 1e-3;
42		let checked = 0;
43		for (let k = 0; k < params.length; k++) {
44		const t = params[k] as Float32Array;
45		const gt = grads[k] as Float32Array;
46		for (let i = 0; i < t.length; i += Math.max(1, Math.floor(t.length / 7))) {
47		const original = t[i] as number;
48		t[i] = original + eps;
49		const up = loss(p);
50		t[i] = original - eps;
51		const down = loss(p);
52		t[i] = original;
53		const numeric = (up - down) / (2 * eps);
54		expect(Math.abs(numeric - (gt[i] as number))).toBeLessThan(2e-3 + Math.abs(numeric) * 5e-3);
55		checked += 1;
56		}
57		}
58		expect(checked).toBeGreaterThan(30);
59		});
60
61		test("advantages telescope backwards and stop at the end of an episode", () => {
62		const r = createRollout(SHAPE, 1, 3);
63		r.rewards.set([1, 1, 1]);
64		r.values.set([0, 0, 0]);
65		r.dones.set([0, 1, 0]);
66		r.lastValues[0] = 0;
67		computeAdvantages(r, 1, 1);
68		// Step 2 is the last and is not terminal, so it bootstraps from lastValues=0.
69		expect(r.advantages[2]).toBeCloseTo(1, 6);
70		// Step 1 ended the episode: its advantage is its own reward and nothing past it.
71		expect(r.advantages[1]).toBeCloseTo(1, 6);
72		// Step 0 sees step 1 through the discount, but not step 2 through the seam.
73		expect(r.advantages[0]).toBeCloseTo(2, 6);
74		expect(r.returns[0]).toBeCloseTo(2, 6);
75		});
76
77		test("a gradient step moves the parameters and the norm is reported", () => {
78		const p = createPolicy(SHAPE, rng(7));
79		const g = zeroGrads(SHAPE);
80		g.b1.fill(1);
81		const before = (p.b1[0] as number);
82		const adam = createAdam(p, DEFAULT_HYPER);
83		const norm = applyGrads(p, g, adam, 1);
84		expect(norm).toBeCloseTo(Math.sqrt(SHAPE.hidden), 5);
85		// Adam's first step is about one learning rate, whatever the gradient size.
86		expect(before - (p.b1[0] as number)).toBeCloseTo(DEFAULT_HYPER.learningRate, 5);
87		});