46588a1 · 01c7560
@@ -75,7 +75,9 @@
FORGE_APP_DATA=./data bun run src/tty.ts                # the terminal interface
  under `src/rl`. The numbers on
  the page and in the terminal's `policy` tab are what the run that produced the
  shipped weights measured, on the same fixed duels for the policy and for the
  scripted bots it replaced.
  scripted bots it replaced. Those numbers are a snapshot of one afternoon, and
  `src/rl/guard.ts` is what keeps them honest between afternoons: every push
  replays the shipped weights on named seeds and holds them to a floor.
- **Simulated.** The opponents are bots, and the match runs entirely in your
  browser: there is no multiplayer and no server-side authority. Scores are
  client-reported, so the server checks them against what a match can produce
@@ -732,7 +734,10 @@
never checked itself.
  a real duel with the round and the economy taken out, on whichever level it is
  handed, and `maps` is the two streams of levels it is handed — the ones a run
  learns on and the ones it is graded on, disjoint by construction and checked
  to be. The algorithm
  to be. `evaluate` is the grading harness — fixed duels in, one tally out — and
  it is one function rather than three because the trainer, `learned.test.ts`
  and `guard.ts` must all mean the same thing by "won 68%"; `guard` is the
  standing measurement of the weights the repository ships. The algorithm
  is not in here — PPO is `src/engine/learn.ts`, behind the door, because nothing
  in it is about shooting; what is here is this game's arena.
  `scripts/train.ts` runs the two together and writes `src/client/weights.ts`. It is the
@@ -1034,6 +1039,24 @@
exports, writes that file back character for character. What the *shipped
weights actually do* is a separate measurement and is `src/rl/learned.test.ts`:
seeded duels, run on every push.
**And the card is re-measured against, if not re-measured.** The weights are a
file and never move; the world they play in — the map generator, the
simulation, a weapon's numbers — moves every week, so a card can end up
boasting a number that was true of a game that no longer exists.
`src/rl/guard.ts` is the guard against that: it plays the weights
`src/client/weights.json` ships — through `useShipped`, so it is the document
a browser downloads and not a copy — greedily, through the trainer's own
`evaluate`, on a named sample — de_forge and two maps from the held-out stream, forty duels
each from seeds derived from the word `guard` — and fails if the share of them
won falls under 45%. The shipped weights win 68% of that sample and the
scripted bots win 46% of the same duels; the floor is five standard errors
below the measurement and about where the scripted bots sit, so it catches a
collapse and not an afternoon's variance. It adds about 0.6s to `bun test`, it
retrains nothing, and it writes nothing: when it fails it prints what it
measured, what `CARD` claims, and whether the scripted baseline fell with it —
which is the difference between a stale card and a broken game — and choosing
between retraining and reverting is a person's.
The trainer's flags are checked before it starts: `scripts/train.ts` parses
argv into options up front and a flag nobody knows, a count that is not a whole
number, or a value that never arrived ends the run with a sentence and exit 2
@@ -31,20 +31,19 @@
import { writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { HEADS, decode } from "../src/client/action.ts";
import { botThink, makeBrain } from "../src/client/bots.ts";
import {
  computeAdvantages, copyPolicy, createAdam, createPolicy, createRollout, forward, greedy,
  computeAdvantages, copyPolicy, createAdam, createPolicy, createRollout, forward,
  newActivations, pack, rng, sample, unpack, update, DEFAULT_HYPER,
  type Hyper, type Packed, type Policy,
  type Hyper, type Packed,
} from "../src/engine/index.ts";
import { OBS_SIZE } from "../src/client/observe.ts";
import { SHAPE } from "../src/client/packed.ts";
import {
  createArena, EPISODE_TICKS, observeFoe, observeLearner, policyFoe, relevel, reset, scriptedFoe, SKIP,
  step, stepInput, stillFoe, type Arena,
  createArena, observeFoe, observeLearner, policyFoe, relevel, reset, scriptedFoe, SKIP,
  step, stillFoe, type Arena,
} from "../src/rl/arena.ts";
import { drawPercent, evaluate, greedyDriver, wonPercent } from "../src/rl/evaluate.ts";
import { holdoutLevels, trainingLevels } from "../src/rl/maps.ts";
import { DE_FORGE } from "../src/client/map.ts";
import type { Level } from "../src/client/level.ts";
import type { Card } from "../src/client/weights.ts";
@@ -159,55 +158,6 @@
export function parseOptions(argv: readonly string[]): Options {
  return options;
}
// --- evaluation -----------------------------------------------------------
export type Outcome = { won: number; lost: number; draw: number; damage: number };
/**
 * Play `duels` fixed duels against the scripted bots, dealt round-robin over
 * `levels`. Both drivers get the same seeds and the same maps, so the same
 * spawns, the same distances and the same opponent skills: the difference in
 * the result is the difference in the driver. A duel that ends in neither a
 * win nor a loss is a draw, and a draw is the number to watch across maps — it
 * is six seconds in which the bot never finished the fight.
 */
export function evaluate(
  drive: ((arena: Arena) => void) | "scripted",
  duels: number,
  seed: number,
  levels: Level[] = [DE_FORGE],
): Outcome {
  const out: Outcome = { won: 0, lost: 0, draw: 0, damage: 0 };
  for (let i = 0; i < duels; i++) {
    const arena = createArena(seed + i * 7919, scriptedFoe, levels[i % levels.length] as Level);
    if (drive === "scripted") arena.learner.brain = makeBrain(0.65, seed + i, false);
    // Both drivers get the same six seconds; the scripted brain turns its own
    // view every tick, the policy decides once every SKIP ticks.
    while (arena.ticks < EPISODE_TICKS && arena.learner.alive && arena.foe.alive) {
      if (drive === "scripted") stepInput(arena, botThink(arena.world, arena.learner, 1 / 64));
      else drive(arena);
    }
    out.damage += arena.learner.damage;
    if (!arena.foe.alive) out.won += 1;
    else if (!arena.learner.alive) out.lost += 1;
    else out.draw += 1;
  }
  return out;
}
/** A driver that plays the policy greedily — no sampling, so it is repeatable. */
export function greedyDriver(policy: Policy): (arena: Arena) => void {
  const obs = new Float32Array(OBS_SIZE);
  const act = newActivations(policy.shape);
  const actions = new Int32Array(HEADS.length);
  return (arena) => {
    observeLearner(arena, obs);
    forward(policy, obs, act);
    greedy(policy.shape, act.logits, actions);
    step(arena, decode(actions));
  };
}
// --- the loop -------------------------------------------------------------
function benchmark({ maps, seed }: Options): void {
@@ -376,16 +326,14 @@
async function train(options: Options): Promise<void> {
  // map curriculum is for, and a draw there is the bot that got confused.
  const unseen = evaluate(greedyDriver(quantised), 240, SEED + 777, holdout);
  const unseenBaseline = evaluate("scripted", 240, SEED + 777, holdout);
  const rate = (o: typeof trained): number => Math.round((o.won / (o.won + o.lost + o.draw)) * 100);
  const draws = (o: typeof trained): number => Math.round((o.draw / (o.won + o.lost + o.draw)) * 100);
  console.log(`\nde_forge, the map it has always been measured on:`);
  console.log(`  trained  ${trained.won}/${trained.lost}/${trained.draw} (${rate(trained)}% won)`);
  console.log(`  shipped  ${shipped.won}/${shipped.lost}/${shipped.draw} (${rate(shipped)}% won) after int8 packing`);
  console.log(`  scripted ${baselineAgain.won}/${baselineAgain.lost}/${baselineAgain.draw} (${rate(baselineAgain)}% won)`);
  console.log(`  trained  ${trained.won}/${trained.lost}/${trained.draw} (${wonPercent(trained)}% won)`);
  console.log(`  shipped  ${shipped.won}/${shipped.lost}/${shipped.draw} (${wonPercent(shipped)}% won) after int8 packing`);
  console.log(`  scripted ${baselineAgain.won}/${baselineAgain.lost}/${baselineAgain.draw} (${wonPercent(baselineAgain)}% won)`);
  console.log(`${holdout.length} maps it has never seen:`);
  console.log(`  shipped  ${unseen.won}/${unseen.lost}/${unseen.draw} (${rate(unseen)}% won, ${draws(unseen)}% drawn)`);
  console.log(`  shipped  ${unseen.won}/${unseen.lost}/${unseen.draw} (${wonPercent(unseen)}% won, ${drawPercent(unseen)}% drawn)`);
  console.log(`  scripted ${unseenBaseline.won}/${unseenBaseline.lost}/${unseenBaseline.draw} ` +
    `(${rate(unseenBaseline)}% won, ${draws(unseenBaseline)}% drawn)`);
    `(${wonPercent(unseenBaseline)}% won, ${drawPercent(unseenBaseline)}% drawn)`);
  if (options.dry) return;
  // The two writes this script does, each to the path the options name: the
@@ -400,13 +348,13 @@
async function train(options: Options): Promise<void> {
    stepsPerSecond: Math.round((envSteps * SKIP) / seconds),
    minutes: Math.round(seconds / 6) / 10,
    duels: 240,
    winRate: rate(shipped),
    baselineWinRate: rate(baselineAgain),
    winRate: wonPercent(shipped),
    baselineWinRate: wonPercent(baselineAgain),
    maps: levels.length,
    unseenMaps: holdout.length,
    unseenWinRate: rate(unseen),
    unseenBaselineWinRate: rate(unseenBaseline),
    unseenDrawRate: draws(unseen),
    unseenWinRate: wonPercent(unseen),
    unseenBaselineWinRate: wonPercent(unseenBaseline),
    unseenDrawRate: drawPercent(unseen),
  }));
  console.log(`\nwrote ${options.policy} (${packed.bytes.length} base64 chars) and ${out}`);
  console.log("now run `bun run scripts/build.ts` and commit both files with what it writes into static/");
@@ -0,0 +1,42 @@
import { expect, test } from "bun:test";
import { useShipped } from "../client/shipped.ts";
import { DE_FORGE } from "../client/map.ts";
import { drawPercent, duelsOf, drawRate, evaluate, greedyDriver, wonPercent, wonRate } from "./evaluate.ts";
import { holdoutLevels } from "./maps.ts";
const outcome = (won: number, lost: number, draw: number) => ({ won, lost, draw, damage: 0 });
test("an outcome's rates are its counts, and an empty one has won nothing", () => {
  const o = outcome(6, 3, 1);
  expect(duelsOf(o)).toBe(10);
  expect(wonRate(o)).toBe(0.6);
  expect(drawRate(o)).toBe(0.1);
  expect(wonPercent(o)).toBe(60);
  expect(drawPercent(o)).toBe(10);
  const none = outcome(0, 0, 0);
  expect(wonRate(none)).toBe(0);
  expect(drawRate(none)).toBe(0);
});
test("every duel is counted once, whichever way it ended", () => {
  const out = evaluate("scripted", 6, 31337);
  expect(duelsOf(out)).toBe(6);
  expect(out.damage).toBeGreaterThanOrEqual(0);
});
test("the same seed is the same duels: a measurement can be reproduced", () => {
  const seed = 8191;
  expect(evaluate("scripted", 6, seed)).toEqual(evaluate("scripted", 6, seed));
  const shipped = greedyDriver(useShipped());
  // A fresh driver each time, because a driver carries its own scratch buffers
  // and nothing about the previous duel may survive into the next measurement.
  expect(evaluate(shipped, 6, seed)).toEqual(evaluate(greedyDriver(useShipped()), 6, seed));
});
test("a different map is a different measurement", () => {
  const seed = 60613;
  const here = evaluate("scripted", 8, seed, [DE_FORGE]);
  const there = evaluate("scripted", 8, seed, holdoutLevels(1));
  expect(duelsOf(here)).toBe(duelsOf(there));
  expect(here).not.toEqual(there);
});
@@ -0,0 +1,81 @@
/**
 * Grading a driver: fixed duels, one tally, no training anywhere in it.
 *
 * This was `scripts/train.ts`'s own function, and it had grown a second copy in
 * `learned.test.ts` and was about to grow a third in `guard.ts`. Three copies
 * of a measurement are three measurements, and the whole point of grading the
 * shipped weights on a push is that the number means the same thing the
 * trainer printed. So it lives here, beside the arena it steps: the trainer
 * imports it to grade the policy it just made, and the tests import it to
 * grade the policy the repository ships.
 *
 * Nothing in here writes a file, reads argv or touches the weights.
 */
import { HEADS, decode } from "../client/action.ts";
import { botThink, makeBrain } from "../client/bots.ts";
import { forward, greedy, newActivations, type Policy } from "../engine/index.ts";
import { OBS_SIZE } from "../client/observe.ts";
import { DE_FORGE } from "../client/map.ts";
import type { Level } from "../client/level.ts";
import { createArena, EPISODE_TICKS, observeLearner, scriptedFoe, step, stepInput, type Arena } from "./arena.ts";
export type Outcome = { won: number; lost: number; draw: number; damage: number };
/** How many duels an outcome is, which is every duel it counted. */
export const duelsOf = (o: Outcome): number => o.won + o.lost + o.draw;
/** The share of those duels it won, 0..1. An empty tally has won nothing. */
export const wonRate = (o: Outcome): number => (duelsOf(o) === 0 ? 0 : o.won / duelsOf(o));
/** The share it neither won nor lost: six seconds that never finished a fight. */
export const drawRate = (o: Outcome): number => (duelsOf(o) === 0 ? 0 : o.draw / duelsOf(o));
/** The same two, as the whole percents a card and a report print. */
export const wonPercent = (o: Outcome): number => Math.round(wonRate(o) * 100);
export const drawPercent = (o: Outcome): number => Math.round(drawRate(o) * 100);
/**
 * Play `duels` fixed duels against the scripted bots, dealt round-robin over
 * `levels`. Both drivers get the same seeds and the same maps, so the same
 * spawns, the same distances and the same opponent skills: the difference in
 * the result is the difference in the driver. A duel that ends in neither a
 * win nor a loss is a draw, and a draw is the number to watch across maps — it
 * is six seconds in which the bot never finished the fight.
 */
export function evaluate(
  drive: ((arena: Arena) => void) | "scripted",
  duels: number,
  seed: number,
  levels: Level[] = [DE_FORGE],
): Outcome {
  const out: Outcome = { won: 0, lost: 0, draw: 0, damage: 0 };
  for (let i = 0; i < duels; i++) {
    const arena = createArena(seed + i * 7919, scriptedFoe, levels[i % levels.length] as Level);
    if (drive === "scripted") arena.learner.brain = makeBrain(0.65, seed + i, false);
    // Both drivers get the same six seconds; the scripted brain turns its own
    // view every tick, the policy decides once every SKIP ticks.
    while (arena.ticks < EPISODE_TICKS && arena.learner.alive && arena.foe.alive) {
      if (drive === "scripted") stepInput(arena, botThink(arena.world, arena.learner, 1 / 64));
      else drive(arena);
    }
    out.damage += arena.learner.damage;
    if (!arena.foe.alive) out.won += 1;
    else if (!arena.learner.alive) out.lost += 1;
    else out.draw += 1;
  }
  return out;
}
/** A driver that plays the policy greedily — no sampling, so it is repeatable. */
export function greedyDriver(policy: Policy): (arena: Arena) => void {
  const obs = new Float32Array(OBS_SIZE);
  const act = newActivations(policy.shape);
  const actions = new Int32Array(HEADS.length);
  return (arena) => {
    observeLearner(arena, obs);
    forward(policy, obs, act);
    greedy(policy.shape, act.logits, actions);
    step(arena, decode(actions));
  };
}
@@ -0,0 +1,78 @@
import { expect, test } from "bun:test";
import { CARD } from "../client/weights.ts";
import { DE_FORGE } from "../client/map.ts";
import { wonRate, type Outcome } from "./evaluate.ts";
import { DUELS_PER_MAP, FLOOR, guardRounds, playGuard, totals, verdict, type Reading } from "./guard.ts";
import { holdoutLevels } from "./maps.ts";
/** An outcome with a given number of wins, the rest lost. */
const outcome = (won: number, of: number, draw = 0): Outcome =>
  ({ won, lost: of - won - draw, draw, damage: 0 });
const fake = (learnedWon: number, scriptedWon: number): Reading[] =>
  guardRounds(20).map((round) => ({
    round,
    learned: outcome(learnedWon, 20),
    scripted: outcome(scriptedWon, 20),
  }));
test("the sample is named: de_forge and two maps from the held-out stream", () => {
  const rounds = guardRounds();
  expect(rounds.map((r) => r.name)).toEqual([DE_FORGE.name, ...holdoutLevels(2).map((l) => l.name)]);
  // A generated map is in it on purpose: most maps a bot is ever played on are
  // carved out of a match id and did not exist when it trained.
  expect(rounds[1]?.name).toStartWith("gen_");
  // Named means reproducible: the same seeds and the same counts every run.
  expect(guardRounds().map((r) => r.seed)).toEqual(rounds.map((r) => r.seed));
  expect(new Set(rounds.map((r) => r.seed)).size).toBe(3);
  expect(rounds.every((r) => r.duels === DUELS_PER_MAP)).toBe(true);
});
/**
 * The guard itself, and the slowest test in the suite: 240 duels of the real
 * arena, about a second. It plays the weights `src/client/weights.json` ships —
 * not a freshly trained policy, and it trains nothing — against the scripted
 * bots they replaced, on the same seeds, and asks that the win rate has not
 * fallen through `FLOOR`. See that constant for why the floor is where it is;
 * the short of it is that this catches a collapse, not a wobble, because a
 * flaky guard on the bots is worse than no guard at all.
 *
 * It is a sample. What it proves is that the shipped policy still beats the
 * scripted bots convincingly on 120 duels across three maps, one hand-written
 * and two generated. It does not re-measure the card, and a pass here is not a
 * licence to leave the card's numbers alone forever.
 */
test("the shipped weights still beat the bots they replaced, on maps and seeds by name", () => {
  const readings = playGuard();
  expect(verdict(readings, CARD)).toBe("");
});
test("the guard's own sample is the size it says it is", () => {
  const { learned, scripted } = totals(fake(12, 8));
  expect(learned.won + learned.lost + learned.draw).toBe(60);
  expect(scripted.won).toBe(24);
  expect(wonRate(learned)).toBeCloseTo(0.6, 5);
});
test("a policy above the floor that still beats the baseline says nothing", () => {
  expect(verdict(fake(10, 8), CARD, FLOOR)).toBe("");
});
test("a collapse says what it measured, what the card claims, and on which map", () => {
  const report = verdict(fake(4, 9), CARD, FLOOR);
  expect(report).toContain("they won 20% of 60 duels, under the 45% floor");
  expect(report).toContain("they won 12 of 60 duels where the scripted bots they replaced won 27");
  expect(report).toContain(`${CARD.winRate}% against a ${CARD.baselineWinRate}% baseline`);
  expect(report).toContain(`${CARD.unseenWinRate}% against ${CARD.unseenBaselineWinRate}%`);
  expect(report).toContain(CARD.trainedAt);
  for (const round of guardRounds(20)) expect(report).toContain(`${round.name} @ ${round.seed}`);
  // It answers the question, and it proposes nothing: both fixes are a person's.
  expect(report).toContain("scripted baseline held");
  expect(report).toContain("Nothing here retrains");
});
test("a baseline that fell with it points at the game rather than at the policy", () => {
  const report = verdict(fake(4, 2), CARD, FLOOR);
  expect(report).toContain("scripted baseline fell with it");
  expect(report).toContain("before you look at the policy");
});
@@ -0,0 +1,167 @@
/**
 * The standing guard on the shipped weights: are the bots in the repository
 * still any good at the game in the repository?
 *
 * `CARD` in `src/client/weights.ts` is one afternoon's measurement — 96 layouts
 * learned, 24 graded, 71% against 42% on de_forge and 73% against 49% on maps
 * nobody wrote — and the page prints it on the first paint. Nothing re-measures
 * it. The weights are a file and they never move; the world they play in is not
 * a file and moves every week, because the map generator is tuned, the
 * simulation grows grenades and deploy times, and a weapon's numbers get
 * another pass. Any of those can leave the card boasting a number that was true
 * of a game that no longer exists.
 *
 * So this plays the weights the repository actually ships — `weights.json`,
 * through `useShipped`, which is the same document the browser downloads —
 * greedily, through the same `evaluate` the trainer grades with, on a named
 * sample, and holds the result to a floor. It does not retrain, does not write
 * the weights and does not touch the card: when it fails, the report below
 * says what it measured,
 * what the card claims, and whether the scripted baseline fell with it — which
 * is the difference between a stale card and a broken game — and a person
 * chooses between retraining and reverting whatever moved the world.
 */
import { seedFrom } from "../client/generate.ts";
import { useShipped } from "../client/shipped.ts";
import { DE_FORGE } from "../client/map.ts";
import type { Card } from "../client/weights.ts";
import type { Level } from "../client/level.ts";
import { drawPercent, duelsOf, evaluate, greedyDriver, wonPercent, wonRate, type Outcome } from "./evaluate.ts";
import { holdoutLevels } from "./maps.ts";
/**
 * Where the sample starts: the word `guard`, hashed the way a match id is, so
 * the seeds are named rather than picked. Each round walks off it by a large
 * stride, and `evaluate` spaces the duels within a round by 7919, so no two
 * duels in the sample are the same duel.
 */
export const GUARD_SEED = seedFrom("guard");
const STRIDE = 1_000_003;
/**
 * Duels per map per driver. Three maps, two drivers: 240 duels, about a second
 * on one core, which is what this adds to `bun test`. It is a sample and it is
 * a small one — the trainer grades on 240 duels a side — so the floor below is
 * set wide enough that the sample's own noise cannot reach it.
 */
export const DUELS_PER_MAP = 40;
/**
 * The floor, as a share of the sample won. On these seeds the shipped weights
 * win 68% of 120 duels (81/27/12) and the scripted baseline wins 46% of the
 * same ones; run with five other base seeds the policy lands between 63% and
 * 74%, so the sample's own spread is about five points either way — one
 * standard error at this size is sqrt(.68*.32/120) = 4.3 points.
 *
 * 45% is five of those below the measurement, and it is also about what the
 * scripted bots manage unaided, so falling through it means the shipped policy
 * is no better than the bots it replaced. That is a collapse. A bad afternoon
 * is 60%, and this must not fire on a bad afternoon: a flaky guard on the bots
 * gets deleted, and then nothing watches them at all.
 */
export const FLOOR = 0.45;
/** One map, one seed, and how many duels are played from it. */
export type Round = { name: string; level: Level; seed: number; duels: number };
/** What both drivers did on one round. Plain counts; the rates are derived. */
export type Reading = { round: Round; learned: Outcome; scripted: Outcome };
/**
 * The sample: de_forge, because it is the map the card's headline number is
 * measured on and the map the game is mostly played on, and two maps from the
 * held-out stream, because most maps a bot is ever played on are carved out of
 * a match id and did not exist when it trained. They are named — `gen_1nc0g0c`
 * and the rest are functions of `HOLDOUT_SEED` — so a failure names the map it
 * failed on and that map can be loaded in the game with `?map=`.
 */
export function guardRounds(duels = DUELS_PER_MAP): Round[] {
  const levels: Level[] = [DE_FORGE, ...holdoutLevels(2)];
  return levels.map((level, i) => ({
    name: level.name,
    level,
    seed: (GUARD_SEED + i * STRIDE) >>> 0,
    duels,
  }));
}
/**
 * Play the sample twice: once driven by the weights in `src/client/weights.json`,
 * once by the scripted brain that policy replaced, on the same seeds and so on
 * the same spawns, ranges and opponents. The baseline is not decoration — it is
 * how a failure tells "the world got harder for everyone" from "the weights
 * stopped working".
 */
export function playGuard(rounds = guardRounds()): Reading[] {
  const shipped = greedyDriver(useShipped());
  return rounds.map((round) => ({
    round,
    learned: evaluate(shipped, round.duels, round.seed, [round.level]),
    scripted: evaluate("scripted", round.duels, round.seed, [round.level]),
  }));
}
const add = (a: Outcome, b: Outcome): Outcome => ({
  won: a.won + b.won, lost: a.lost + b.lost, draw: a.draw + b.draw, damage: a.damage + b.damage,
});
const EMPTY: Outcome = { won: 0, lost: 0, draw: 0, damage: 0 };
/** The whole sample, both drivers, as two tallies. */
export function totals(readings: Reading[]): { learned: Outcome; scripted: Outcome } {
  return {
    learned: readings.reduce((sum, r) => add(sum, r.learned), EMPTY),
    scripted: readings.reduce((sum, r) => add(sum, r.scripted), EMPTY),
  };
}
const tally = (o: Outcome): string =>
  `${wonPercent(o)}% won (${o.won}/${o.lost}/${o.draw} won/lost/drawn, ${drawPercent(o)}% drawn)`;
/**
 * What the guard found, or the empty string when it found nothing wrong.
 *
 * The report is written for the one question a red suite asks first — is the
 * card stale or is the game broken? — so it prints the measurement, the card's
 * claim beside it, and the baseline's own answer, which is the evidence. It
 * proposes nothing: retraining and reverting are both a person's call.
 */
export function verdict(readings: Reading[], card: Card, floor = FLOOR): string {
  const { learned, scripted } = totals(readings);
  const count = duelsOf(learned);
  const reasons: string[] = [];
  if (wonRate(learned) < floor) {
    reasons.push(`they won ${wonPercent(learned)}% of ${count} duels, under the ${Math.round(floor * 100)}% floor`);
  }
  if (learned.won <= scripted.won) {
    reasons.push(`they won ${learned.won} of ${count} duels where the scripted bots they replaced won ${scripted.won} of the same ones`);
  }
  if (reasons.length === 0) return "";
  // Which way to look first. If the baseline fell too, the duels themselves got
  // harder and the weights are not the thing that changed; if it held, the
  // world moved under the weights alone and the card is describing a game that
  // is gone.
  const baselineFell = wonPercent(scripted) < card.baselineWinRate - 10;
  const where = baselineFell
    ? "The scripted baseline fell with it, so these duels got harder for everyone: look at what moved in the game — the generator, the simulation, a weapon — before you look at the policy."
    : "The scripted baseline held, so the duels are as winnable as they were: the shipped weights are what stopped working, and the card is describing a game that has moved on.";
  return [
    `the shipped weights are no longer worth what the card says: ${reasons.join("; and ")}.`,
    "",
    `measured now, greedily, on ${readings.length} maps from named seeds:`,
    `  shipped  ${tally(learned)}`,
    `  scripted ${tally(scripted)}`,
    ...readings.map((r) =>
      `  ${r.round.name} @ ${r.round.seed} (${r.round.duels} duels): shipped ${tally(r.learned)}, scripted ${tally(r.scripted)}`),
    "",
    `CARD says (trained ${card.trainedAt}, not re-measured since): ${card.winRate}% against a ${card.baselineWinRate}% baseline ` +
    `over ${card.duels} duels on de_forge, and ${card.unseenWinRate}% against ${card.unseenBaselineWinRate}% ` +
    `on ${card.unseenMaps} maps it never trained on.`,
    "",
    where,
    "Nothing here retrains and nothing here edits CARD: retrain with `bun run scripts/train.ts --iters 300` and `bun run scripts/build.ts`, or revert whatever moved the world.",
  ].join("\n");
}
@@ -1,11 +1,8 @@
import { expect, test } from "bun:test";
import { decode, HEADS } from "../client/action.ts";
import { botThink, makeBrain, policy } from "../client/bots.ts";
import { forward, greedy, newActivations } from "../engine/index.ts";
import { OBS_SIZE } from "../client/observe.ts";
import { useShipped } from "../client/shipped.ts";
import { DE_FORGE } from "../client/map.ts";
import type { Level } from "../client/level.ts";
import { createArena, EPISODE_TICKS, observeLearner, scriptedFoe, step, stepInput } from "./arena.ts";
import { evaluate, greedyDriver } from "./evaluate.ts";
import { holdoutLevels } from "./maps.ts";
/**
@@ -17,34 +14,13 @@
import { holdoutLevels } from "./maps.ts";
const DUELS = 40;
const SEED = 4242;
type Tally = { won: number; lost: number; draw: number };
/** `count` duels, dealt round-robin over `levels`, driven one way or the other. */
function duels(drive: "policy" | "scripted", count = DUELS, levels: Level[] = [DE_FORGE]): Tally {
  const net = policy();
  const obs = new Float32Array(OBS_SIZE);
  const act = newActivations(net.shape);
  const actions = new Int32Array(HEADS.length);
  const tally: Tally = { won: 0, lost: 0, draw: 0 };
  for (let i = 0; i < count; i++) {
    const arena = createArena(SEED + i * 7919, scriptedFoe, levels[i % levels.length] as Level);
    if (drive === "scripted") arena.learner.brain = makeBrain(0.65, SEED + i, false);
    while (arena.ticks < EPISODE_TICKS && arena.learner.alive && arena.foe.alive) {
      if (drive === "scripted") {
        stepInput(arena, botThink(arena.world, arena.learner, 1 / 64));
      } else {
        observeLearner(arena, obs);
        forward(net, obs, act);
        greedy(net.shape, act.logits, actions);
        step(arena, decode(actions));
      }
    }
    if (!arena.foe.alive) tally.won += 1;
    else if (!arena.learner.alive) tally.lost += 1;
    else tally.draw += 1;
  }
  return tally;
}
/**
 * `count` duels, dealt round-robin over `levels`, driven one way or the other:
 * the trainer's own harness, so the number here and the number the trainer
 * printed onto the card are the same measurement of the same thing.
 */
const duels = (drive: "policy" | "scripted", count = DUELS, levels: Level[] = [DE_FORGE]) =>
  evaluate(drive === "scripted" ? "scripted" : greedyDriver(useShipped()), count, SEED, levels);
test("the trained policy wins more of the same duels than the bots it replaced", () => {
  const learned = duels("policy");
		@@ -75,7 +75,9 @@ FORGE_APP_DATA=./data bun run src/tty.ts # the terminal interface
75	75	under `src/rl`. The numbers on
76	76	the page and in the terminal's `policy` tab are what the run that produced the
77	77	shipped weights measured, on the same fixed duels for the policy and for the
78		scripted bots it replaced.
	78	scripted bots it replaced. Those numbers are a snapshot of one afternoon, and
	79	`src/rl/guard.ts` is what keeps them honest between afternoons: every push
	80	replays the shipped weights on named seeds and holds them to a floor.
79	81	- Simulated. The opponents are bots, and the match runs entirely in your
80	82	browser: there is no multiplayer and no server-side authority. Scores are
81	83	client-reported, so the server checks them against what a match can produce
		@@ -732,7 +734,10 @@ never checked itself.
732	734	a real duel with the round and the economy taken out, on whichever level it is
733	735	handed, and `maps` is the two streams of levels it is handed — the ones a run
734	736	learns on and the ones it is graded on, disjoint by construction and checked
735		to be. The algorithm
	737	to be. `evaluate` is the grading harness — fixed duels in, one tally out — and
	738	it is one function rather than three because the trainer, `learned.test.ts`
	739	and `guard.ts` must all mean the same thing by "won 68%"; `guard` is the
	740	standing measurement of the weights the repository ships. The algorithm
736	741	is not in here — PPO is `src/engine/learn.ts`, behind the door, because nothing
737	742	in it is about shooting; what is here is this game's arena.
738	743	`scripts/train.ts` runs the two together and writes `src/client/weights.ts`. It is the
		@@ -1034,6 +1039,24 @@ exports, writes that file back character for character. What the *shipped
1034	1039	weights actually do* is a separate measurement and is `src/rl/learned.test.ts`:
1035	1040	seeded duels, run on every push.
1036	1041
	1042	And the card is re-measured against, if not re-measured. The weights are a
	1043	file and never move; the world they play in — the map generator, the
	1044	simulation, a weapon's numbers — moves every week, so a card can end up
	1045	boasting a number that was true of a game that no longer exists.
	1046	`src/rl/guard.ts` is the guard against that: it plays the weights
	1047	`src/client/weights.json` ships — through `useShipped`, so it is the document
	1048	a browser downloads and not a copy — greedily, through the trainer's own
	1049	`evaluate`, on a named sample — de_forge and two maps from the held-out stream, forty duels
	1050	each from seeds derived from the word `guard` — and fails if the share of them
	1051	won falls under 45%. The shipped weights win 68% of that sample and the
	1052	scripted bots win 46% of the same duels; the floor is five standard errors
	1053	below the measurement and about where the scripted bots sit, so it catches a
	1054	collapse and not an afternoon's variance. It adds about 0.6s to `bun test`, it
	1055	retrains nothing, and it writes nothing: when it fails it prints what it
	1056	measured, what `CARD` claims, and whether the scripted baseline fell with it —
	1057	which is the difference between a stale card and a broken game — and choosing
	1058	between retraining and reverting is a person's.
	1059
1037	1060	The trainer's flags are checked before it starts: `scripts/train.ts` parses
1038	1061	argv into options up front and a flag nobody knows, a count that is not a whole
1039	1062	number, or a value that never arrived ends the run with a sentence and exit 2
		@@ -31,20 +31,19 @@
31	31	import { writeFileSync } from "node:fs";
32	32	import { join, resolve } from "node:path";
33	33	import { HEADS, decode } from "../src/client/action.ts";
34		import { botThink, makeBrain } from "../src/client/bots.ts";
35	34	import {
36		computeAdvantages, copyPolicy, createAdam, createPolicy, createRollout, forward, greedy,
	35	computeAdvantages, copyPolicy, createAdam, createPolicy, createRollout, forward,
37	36	newActivations, pack, rng, sample, unpack, update, DEFAULT_HYPER,
38		type Hyper, type Packed, type Policy,
	37	type Hyper, type Packed,
39	38	} from "../src/engine/index.ts";
40	39	import { OBS_SIZE } from "../src/client/observe.ts";
41	40	import { SHAPE } from "../src/client/packed.ts";
42	41	import {
43		createArena, EPISODE_TICKS, observeFoe, observeLearner, policyFoe, relevel, reset, scriptedFoe, SKIP,
44		step, stepInput, stillFoe, type Arena,
	42	createArena, observeFoe, observeLearner, policyFoe, relevel, reset, scriptedFoe, SKIP,
	43	step, stillFoe, type Arena,
45	44	} from "../src/rl/arena.ts";
	45	import { drawPercent, evaluate, greedyDriver, wonPercent } from "../src/rl/evaluate.ts";
46	46	import { holdoutLevels, trainingLevels } from "../src/rl/maps.ts";
47		import { DE_FORGE } from "../src/client/map.ts";
48	47	import type { Level } from "../src/client/level.ts";
49	48	import type { Card } from "../src/client/weights.ts";
50	49
		@@ -159,55 +158,6 @@ export function parseOptions(argv: readonly string[]): Options {
159	158	return options;
160	159	}
161	160
162		// --- evaluation -----------------------------------------------------------
163
164		export type Outcome = { won: number; lost: number; draw: number; damage: number };
165
166		/**
167		* Play `duels` fixed duels against the scripted bots, dealt round-robin over
168		* `levels`. Both drivers get the same seeds and the same maps, so the same
169		* spawns, the same distances and the same opponent skills: the difference in
170		* the result is the difference in the driver. A duel that ends in neither a
171		* win nor a loss is a draw, and a draw is the number to watch across maps — it
172		* is six seconds in which the bot never finished the fight.
173		*/
174		export function evaluate(
175		drive: ((arena: Arena) => void) \| "scripted",
176		duels: number,
177		seed: number,
178		levels: Level[] = [DE_FORGE],
179		): Outcome {
180		const out: Outcome = { won: 0, lost: 0, draw: 0, damage: 0 };
181		for (let i = 0; i < duels; i++) {
182		const arena = createArena(seed + i * 7919, scriptedFoe, levels[i % levels.length] as Level);
183		if (drive === "scripted") arena.learner.brain = makeBrain(0.65, seed + i, false);
184		// Both drivers get the same six seconds; the scripted brain turns its own
185		// view every tick, the policy decides once every SKIP ticks.
186		while (arena.ticks < EPISODE_TICKS && arena.learner.alive && arena.foe.alive) {
187		if (drive === "scripted") stepInput(arena, botThink(arena.world, arena.learner, 1 / 64));
188		else drive(arena);
189		}
190		out.damage += arena.learner.damage;
191		if (!arena.foe.alive) out.won += 1;
192		else if (!arena.learner.alive) out.lost += 1;
193		else out.draw += 1;
194		}
195		return out;
196		}
197
198		/** A driver that plays the policy greedily — no sampling, so it is repeatable. */
199		export function greedyDriver(policy: Policy): (arena: Arena) => void {
200		const obs = new Float32Array(OBS_SIZE);
201		const act = newActivations(policy.shape);
202		const actions = new Int32Array(HEADS.length);
203		return (arena) => {
204		observeLearner(arena, obs);
205		forward(policy, obs, act);
206		greedy(policy.shape, act.logits, actions);
207		step(arena, decode(actions));
208		};
209		}
210
211	161	// --- the loop -------------------------------------------------------------
212	162
213	163	function benchmark({ maps, seed }: Options): void {
		@@ -376,16 +326,14 @@ async function train(options: Options): Promise<void> {
376	326	// map curriculum is for, and a draw there is the bot that got confused.
377	327	const unseen = evaluate(greedyDriver(quantised), 240, SEED + 777, holdout);
378	328	const unseenBaseline = evaluate("scripted", 240, SEED + 777, holdout);
379		const rate = (o: typeof trained): number => Math.round((o.won / (o.won + o.lost + o.draw)) * 100);
380		const draws = (o: typeof trained): number => Math.round((o.draw / (o.won + o.lost + o.draw)) * 100);
381	329	console.log(`\nde_forge, the map it has always been measured on:`);
382		console.log(` trained ${trained.won}/${trained.lost}/${trained.draw} (${rate(trained)}% won)`);
383		console.log(` shipped ${shipped.won}/${shipped.lost}/${shipped.draw} (${rate(shipped)}% won) after int8 packing`);
384		console.log(` scripted ${baselineAgain.won}/${baselineAgain.lost}/${baselineAgain.draw} (${rate(baselineAgain)}% won)`);
	330	console.log(` trained ${trained.won}/${trained.lost}/${trained.draw} (${wonPercent(trained)}% won)`);
	331	console.log(` shipped ${shipped.won}/${shipped.lost}/${shipped.draw} (${wonPercent(shipped)}% won) after int8 packing`);
	332	console.log(` scripted ${baselineAgain.won}/${baselineAgain.lost}/${baselineAgain.draw} (${wonPercent(baselineAgain)}% won)`);
385	333	console.log(`${holdout.length} maps it has never seen:`);
386		console.log(` shipped ${unseen.won}/${unseen.lost}/${unseen.draw} (${rate(unseen)}% won, ${draws(unseen)}% drawn)`);
	334	console.log(` shipped ${unseen.won}/${unseen.lost}/${unseen.draw} (${wonPercent(unseen)}% won, ${drawPercent(unseen)}% drawn)`);
387	335	console.log(` scripted ${unseenBaseline.won}/${unseenBaseline.lost}/${unseenBaseline.draw} ` +
388		`(${rate(unseenBaseline)}% won, ${draws(unseenBaseline)}% drawn)`);
	336	`(${wonPercent(unseenBaseline)}% won, ${drawPercent(unseenBaseline)}% drawn)`);
389	337
390	338	if (options.dry) return;
391	339	// The two writes this script does, each to the path the options name: the
		@@ -400,13 +348,13 @@ async function train(options: Options): Promise<void> {
400	348	stepsPerSecond: Math.round((envSteps * SKIP) / seconds),
401	349	minutes: Math.round(seconds / 6) / 10,
402	350	duels: 240,
403		winRate: rate(shipped),
404		baselineWinRate: rate(baselineAgain),
	351	winRate: wonPercent(shipped),
	352	baselineWinRate: wonPercent(baselineAgain),
405	353	maps: levels.length,
406	354	unseenMaps: holdout.length,
407		unseenWinRate: rate(unseen),
408		unseenBaselineWinRate: rate(unseenBaseline),
409		unseenDrawRate: draws(unseen),
	355	unseenWinRate: wonPercent(unseen),
	356	unseenBaselineWinRate: wonPercent(unseenBaseline),
	357	unseenDrawRate: drawPercent(unseen),
410	358	}));
411	359	console.log(`\nwrote ${options.policy} (${packed.bytes.length} base64 chars) and ${out}`);
412	360	console.log("now run `bun run scripts/build.ts` and commit both files with what it writes into static/");
		@@ -0,0 +1,42 @@
	1	import { expect, test } from "bun:test";
	2	import { useShipped } from "../client/shipped.ts";
	3	import { DE_FORGE } from "../client/map.ts";
	4	import { drawPercent, duelsOf, drawRate, evaluate, greedyDriver, wonPercent, wonRate } from "./evaluate.ts";
	5	import { holdoutLevels } from "./maps.ts";
	6
	7	const outcome = (won: number, lost: number, draw: number) => ({ won, lost, draw, damage: 0 });
	8
	9	test("an outcome's rates are its counts, and an empty one has won nothing", () => {
	10	const o = outcome(6, 3, 1);
	11	expect(duelsOf(o)).toBe(10);
	12	expect(wonRate(o)).toBe(0.6);
	13	expect(drawRate(o)).toBe(0.1);
	14	expect(wonPercent(o)).toBe(60);
	15	expect(drawPercent(o)).toBe(10);
	16	const none = outcome(0, 0, 0);
	17	expect(wonRate(none)).toBe(0);
	18	expect(drawRate(none)).toBe(0);
	19	});
	20
	21	test("every duel is counted once, whichever way it ended", () => {
	22	const out = evaluate("scripted", 6, 31337);
	23	expect(duelsOf(out)).toBe(6);
	24	expect(out.damage).toBeGreaterThanOrEqual(0);
	25	});
	26
	27	test("the same seed is the same duels: a measurement can be reproduced", () => {
	28	const seed = 8191;
	29	expect(evaluate("scripted", 6, seed)).toEqual(evaluate("scripted", 6, seed));
	30	const shipped = greedyDriver(useShipped());
	31	// A fresh driver each time, because a driver carries its own scratch buffers
	32	// and nothing about the previous duel may survive into the next measurement.
	33	expect(evaluate(shipped, 6, seed)).toEqual(evaluate(greedyDriver(useShipped()), 6, seed));
	34	});
	35
	36	test("a different map is a different measurement", () => {
	37	const seed = 60613;
	38	const here = evaluate("scripted", 8, seed, [DE_FORGE]);
	39	const there = evaluate("scripted", 8, seed, holdoutLevels(1));
	40	expect(duelsOf(here)).toBe(duelsOf(there));
	41	expect(here).not.toEqual(there);
	42	});
		@@ -0,0 +1,81 @@
	1	/**
	2	* Grading a driver: fixed duels, one tally, no training anywhere in it.
	3	*
	4	* This was `scripts/train.ts`'s own function, and it had grown a second copy in
	5	* `learned.test.ts` and was about to grow a third in `guard.ts`. Three copies
	6	* of a measurement are three measurements, and the whole point of grading the
	7	* shipped weights on a push is that the number means the same thing the
	8	* trainer printed. So it lives here, beside the arena it steps: the trainer
	9	* imports it to grade the policy it just made, and the tests import it to
	10	* grade the policy the repository ships.
	11	*
	12	* Nothing in here writes a file, reads argv or touches the weights.
	13	*/
	14
	15	import { HEADS, decode } from "../client/action.ts";
	16	import { botThink, makeBrain } from "../client/bots.ts";
	17	import { forward, greedy, newActivations, type Policy } from "../engine/index.ts";
	18	import { OBS_SIZE } from "../client/observe.ts";
	19	import { DE_FORGE } from "../client/map.ts";
	20	import type { Level } from "../client/level.ts";
	21	import { createArena, EPISODE_TICKS, observeLearner, scriptedFoe, step, stepInput, type Arena } from "./arena.ts";
	22
	23	export type Outcome = { won: number; lost: number; draw: number; damage: number };
	24
	25	/** How many duels an outcome is, which is every duel it counted. */
	26	export const duelsOf = (o: Outcome): number => o.won + o.lost + o.draw;
	27
	28	/** The share of those duels it won, 0..1. An empty tally has won nothing. */
	29	export const wonRate = (o: Outcome): number => (duelsOf(o) === 0 ? 0 : o.won / duelsOf(o));
	30
	31	/** The share it neither won nor lost: six seconds that never finished a fight. */
	32	export const drawRate = (o: Outcome): number => (duelsOf(o) === 0 ? 0 : o.draw / duelsOf(o));
	33
	34	/** The same two, as the whole percents a card and a report print. */
	35	export const wonPercent = (o: Outcome): number => Math.round(wonRate(o) * 100);
	36	export const drawPercent = (o: Outcome): number => Math.round(drawRate(o) * 100);
	37
	38	/**
	39	* Play `duels` fixed duels against the scripted bots, dealt round-robin over
	40	* `levels`. Both drivers get the same seeds and the same maps, so the same
	41	* spawns, the same distances and the same opponent skills: the difference in
	42	* the result is the difference in the driver. A duel that ends in neither a
	43	* win nor a loss is a draw, and a draw is the number to watch across maps — it
	44	* is six seconds in which the bot never finished the fight.
	45	*/
	46	export function evaluate(
	47	drive: ((arena: Arena) => void) \| "scripted",
	48	duels: number,
	49	seed: number,
	50	levels: Level[] = [DE_FORGE],
	51	): Outcome {
	52	const out: Outcome = { won: 0, lost: 0, draw: 0, damage: 0 };
	53	for (let i = 0; i < duels; i++) {
	54	const arena = createArena(seed + i * 7919, scriptedFoe, levels[i % levels.length] as Level);
	55	if (drive === "scripted") arena.learner.brain = makeBrain(0.65, seed + i, false);
	56	// Both drivers get the same six seconds; the scripted brain turns its own
	57	// view every tick, the policy decides once every SKIP ticks.
	58	while (arena.ticks < EPISODE_TICKS && arena.learner.alive && arena.foe.alive) {
	59	if (drive === "scripted") stepInput(arena, botThink(arena.world, arena.learner, 1 / 64));
	60	else drive(arena);
	61	}
	62	out.damage += arena.learner.damage;
	63	if (!arena.foe.alive) out.won += 1;
	64	else if (!arena.learner.alive) out.lost += 1;
	65	else out.draw += 1;
	66	}
	67	return out;
	68	}
	69
	70	/** A driver that plays the policy greedily — no sampling, so it is repeatable. */
	71	export function greedyDriver(policy: Policy): (arena: Arena) => void {
	72	const obs = new Float32Array(OBS_SIZE);
	73	const act = newActivations(policy.shape);
	74	const actions = new Int32Array(HEADS.length);
	75	return (arena) => {
	76	observeLearner(arena, obs);
	77	forward(policy, obs, act);
	78	greedy(policy.shape, act.logits, actions);
	79	step(arena, decode(actions));
	80	};
	81	}
		@@ -0,0 +1,78 @@
	1	import { expect, test } from "bun:test";
	2	import { CARD } from "../client/weights.ts";
	3	import { DE_FORGE } from "../client/map.ts";
	4	import { wonRate, type Outcome } from "./evaluate.ts";
	5	import { DUELS_PER_MAP, FLOOR, guardRounds, playGuard, totals, verdict, type Reading } from "./guard.ts";
	6	import { holdoutLevels } from "./maps.ts";
	7
	8	/** An outcome with a given number of wins, the rest lost. */
	9	const outcome = (won: number, of: number, draw = 0): Outcome =>
	10	({ won, lost: of - won - draw, draw, damage: 0 });
	11
	12	const fake = (learnedWon: number, scriptedWon: number): Reading[] =>
	13	guardRounds(20).map((round) => ({
	14	round,
	15	learned: outcome(learnedWon, 20),
	16	scripted: outcome(scriptedWon, 20),
	17	}));
	18
	19	test("the sample is named: de_forge and two maps from the held-out stream", () => {
	20	const rounds = guardRounds();
	21	expect(rounds.map((r) => r.name)).toEqual([DE_FORGE.name, ...holdoutLevels(2).map((l) => l.name)]);
	22	// A generated map is in it on purpose: most maps a bot is ever played on are
	23	// carved out of a match id and did not exist when it trained.
	24	expect(rounds[1]?.name).toStartWith("gen_");
	25	// Named means reproducible: the same seeds and the same counts every run.
	26	expect(guardRounds().map((r) => r.seed)).toEqual(rounds.map((r) => r.seed));
	27	expect(new Set(rounds.map((r) => r.seed)).size).toBe(3);
	28	expect(rounds.every((r) => r.duels === DUELS_PER_MAP)).toBe(true);
	29	});
	30
	31	/**
	32	* The guard itself, and the slowest test in the suite: 240 duels of the real
	33	* arena, about a second. It plays the weights `src/client/weights.json` ships —
	34	* not a freshly trained policy, and it trains nothing — against the scripted
	35	* bots they replaced, on the same seeds, and asks that the win rate has not
	36	* fallen through `FLOOR`. See that constant for why the floor is where it is;
	37	* the short of it is that this catches a collapse, not a wobble, because a
	38	* flaky guard on the bots is worse than no guard at all.
	39	*
	40	* It is a sample. What it proves is that the shipped policy still beats the
	41	* scripted bots convincingly on 120 duels across three maps, one hand-written
	42	* and two generated. It does not re-measure the card, and a pass here is not a
	43	* licence to leave the card's numbers alone forever.
	44	*/
	45	test("the shipped weights still beat the bots they replaced, on maps and seeds by name", () => {
	46	const readings = playGuard();
	47	expect(verdict(readings, CARD)).toBe("");
	48	});
	49
	50	test("the guard's own sample is the size it says it is", () => {
	51	const { learned, scripted } = totals(fake(12, 8));
	52	expect(learned.won + learned.lost + learned.draw).toBe(60);
	53	expect(scripted.won).toBe(24);
	54	expect(wonRate(learned)).toBeCloseTo(0.6, 5);
	55	});
	56
	57	test("a policy above the floor that still beats the baseline says nothing", () => {
	58	expect(verdict(fake(10, 8), CARD, FLOOR)).toBe("");
	59	});
	60
	61	test("a collapse says what it measured, what the card claims, and on which map", () => {
	62	const report = verdict(fake(4, 9), CARD, FLOOR);
	63	expect(report).toContain("they won 20% of 60 duels, under the 45% floor");
	64	expect(report).toContain("they won 12 of 60 duels where the scripted bots they replaced won 27");
	65	expect(report).toContain(`${CARD.winRate}% against a ${CARD.baselineWinRate}% baseline`);
	66	expect(report).toContain(`${CARD.unseenWinRate}% against ${CARD.unseenBaselineWinRate}%`);
	67	expect(report).toContain(CARD.trainedAt);
	68	for (const round of guardRounds(20)) expect(report).toContain(`${round.name} @ ${round.seed}`);
	69	// It answers the question, and it proposes nothing: both fixes are a person's.
	70	expect(report).toContain("scripted baseline held");
	71	expect(report).toContain("Nothing here retrains");
	72	});
	73
	74	test("a baseline that fell with it points at the game rather than at the policy", () => {
	75	const report = verdict(fake(4, 2), CARD, FLOOR);
	76	expect(report).toContain("scripted baseline fell with it");
	77	expect(report).toContain("before you look at the policy");
	78	});
		@@ -0,0 +1,167 @@
	1	/**
	2	* The standing guard on the shipped weights: are the bots in the repository
	3	* still any good at the game in the repository?
	4	*
	5	* `CARD` in `src/client/weights.ts` is one afternoon's measurement — 96 layouts
	6	* learned, 24 graded, 71% against 42% on de_forge and 73% against 49% on maps
	7	* nobody wrote — and the page prints it on the first paint. Nothing re-measures
	8	* it. The weights are a file and they never move; the world they play in is not
	9	* a file and moves every week, because the map generator is tuned, the
	10	* simulation grows grenades and deploy times, and a weapon's numbers get
	11	* another pass. Any of those can leave the card boasting a number that was true
	12	* of a game that no longer exists.
	13	*
	14	* So this plays the weights the repository actually ships — `weights.json`,
	15	* through `useShipped`, which is the same document the browser downloads —
	16	* greedily, through the same `evaluate` the trainer grades with, on a named
	17	* sample, and holds the result to a floor. It does not retrain, does not write
	18	* the weights and does not touch the card: when it fails, the report below
	19	* says what it measured,
	20	* what the card claims, and whether the scripted baseline fell with it — which
	21	* is the difference between a stale card and a broken game — and a person
	22	* chooses between retraining and reverting whatever moved the world.
	23	*/
	24
	25	import { seedFrom } from "../client/generate.ts";
	26	import { useShipped } from "../client/shipped.ts";
	27	import { DE_FORGE } from "../client/map.ts";
	28	import type { Card } from "../client/weights.ts";
	29	import type { Level } from "../client/level.ts";
	30	import { drawPercent, duelsOf, evaluate, greedyDriver, wonPercent, wonRate, type Outcome } from "./evaluate.ts";
	31	import { holdoutLevels } from "./maps.ts";
	32
	33	/**
	34	* Where the sample starts: the word `guard`, hashed the way a match id is, so
	35	* the seeds are named rather than picked. Each round walks off it by a large
	36	* stride, and `evaluate` spaces the duels within a round by 7919, so no two
	37	* duels in the sample are the same duel.
	38	*/
	39	export const GUARD_SEED = seedFrom("guard");
	40	const STRIDE = 1_000_003;
	41
	42	/**
	43	* Duels per map per driver. Three maps, two drivers: 240 duels, about a second
	44	* on one core, which is what this adds to `bun test`. It is a sample and it is
	45	* a small one — the trainer grades on 240 duels a side — so the floor below is
	46	* set wide enough that the sample's own noise cannot reach it.
	47	*/
	48	export const DUELS_PER_MAP = 40;
	49
	50	/**
	51	* The floor, as a share of the sample won. On these seeds the shipped weights
	52	* win 68% of 120 duels (81/27/12) and the scripted baseline wins 46% of the
	53	* same ones; run with five other base seeds the policy lands between 63% and
	54	* 74%, so the sample's own spread is about five points either way — one
	55	* standard error at this size is sqrt(.68*.32/120) = 4.3 points.
	56	*
	57	* 45% is five of those below the measurement, and it is also about what the
	58	* scripted bots manage unaided, so falling through it means the shipped policy
	59	* is no better than the bots it replaced. That is a collapse. A bad afternoon
	60	* is 60%, and this must not fire on a bad afternoon: a flaky guard on the bots
	61	* gets deleted, and then nothing watches them at all.
	62	*/
	63	export const FLOOR = 0.45;
	64
	65	/** One map, one seed, and how many duels are played from it. */
	66	export type Round = { name: string; level: Level; seed: number; duels: number };
	67
	68	/** What both drivers did on one round. Plain counts; the rates are derived. */
	69	export type Reading = { round: Round; learned: Outcome; scripted: Outcome };
	70
	71	/**
	72	* The sample: de_forge, because it is the map the card's headline number is
	73	* measured on and the map the game is mostly played on, and two maps from the
	74	* held-out stream, because most maps a bot is ever played on are carved out of
	75	* a match id and did not exist when it trained. They are named — `gen_1nc0g0c`
	76	* and the rest are functions of `HOLDOUT_SEED` — so a failure names the map it
	77	* failed on and that map can be loaded in the game with `?map=`.
	78	*/
	79	export function guardRounds(duels = DUELS_PER_MAP): Round[] {
	80	const levels: Level[] = [DE_FORGE, ...holdoutLevels(2)];
	81	return levels.map((level, i) => ({
	82	name: level.name,
	83	level,
	84	seed: (GUARD_SEED + i * STRIDE) >>> 0,
	85	duels,
	86	}));
	87	}
	88
	89	/**
	90	* Play the sample twice: once driven by the weights in `src/client/weights.json`,
	91	* once by the scripted brain that policy replaced, on the same seeds and so on
	92	* the same spawns, ranges and opponents. The baseline is not decoration — it is
	93	* how a failure tells "the world got harder for everyone" from "the weights
	94	* stopped working".
	95	*/
	96	export function playGuard(rounds = guardRounds()): Reading[] {
	97	const shipped = greedyDriver(useShipped());
	98	return rounds.map((round) => ({
	99	round,
	100	learned: evaluate(shipped, round.duels, round.seed, [round.level]),
	101	scripted: evaluate("scripted", round.duels, round.seed, [round.level]),
	102	}));
	103	}
	104
	105	const add = (a: Outcome, b: Outcome): Outcome => ({
	106	won: a.won + b.won, lost: a.lost + b.lost, draw: a.draw + b.draw, damage: a.damage + b.damage,
	107	});
	108
	109	const EMPTY: Outcome = { won: 0, lost: 0, draw: 0, damage: 0 };
	110
	111	/** The whole sample, both drivers, as two tallies. */
	112	export function totals(readings: Reading[]): { learned: Outcome; scripted: Outcome } {
	113	return {
	114	learned: readings.reduce((sum, r) => add(sum, r.learned), EMPTY),
	115	scripted: readings.reduce((sum, r) => add(sum, r.scripted), EMPTY),
	116	};
	117	}
	118
	119	const tally = (o: Outcome): string =>
	120	`${wonPercent(o)}% won (${o.won}/${o.lost}/${o.draw} won/lost/drawn, ${drawPercent(o)}% drawn)`;
	121
	122	/**
	123	* What the guard found, or the empty string when it found nothing wrong.
	124	*
	125	* The report is written for the one question a red suite asks first — is the
	126	* card stale or is the game broken? — so it prints the measurement, the card's
	127	* claim beside it, and the baseline's own answer, which is the evidence. It
	128	* proposes nothing: retraining and reverting are both a person's call.
	129	*/
	130	export function verdict(readings: Reading[], card: Card, floor = FLOOR): string {
	131	const { learned, scripted } = totals(readings);
	132	const count = duelsOf(learned);
	133	const reasons: string[] = [];
	134	if (wonRate(learned) < floor) {
	135	reasons.push(`they won ${wonPercent(learned)}% of ${count} duels, under the ${Math.round(floor * 100)}% floor`);
	136	}
	137	if (learned.won <= scripted.won) {
	138	reasons.push(`they won ${learned.won} of ${count} duels where the scripted bots they replaced won ${scripted.won} of the same ones`);
	139	}
	140	if (reasons.length === 0) return "";
	141
	142	// Which way to look first. If the baseline fell too, the duels themselves got
	143	// harder and the weights are not the thing that changed; if it held, the
	144	// world moved under the weights alone and the card is describing a game that
	145	// is gone.
	146	const baselineFell = wonPercent(scripted) < card.baselineWinRate - 10;
	147	const where = baselineFell
	148	? "The scripted baseline fell with it, so these duels got harder for everyone: look at what moved in the game — the generator, the simulation, a weapon — before you look at the policy."
	149	: "The scripted baseline held, so the duels are as winnable as they were: the shipped weights are what stopped working, and the card is describing a game that has moved on.";
	150
	151	return [
	152	`the shipped weights are no longer worth what the card says: ${reasons.join("; and ")}.`,
	153	"",
	154	`measured now, greedily, on ${readings.length} maps from named seeds:`,
	155	` shipped ${tally(learned)}`,
	156	` scripted ${tally(scripted)}`,
	157	...readings.map((r) =>
	158	` ${r.round.name} @ ${r.round.seed} (${r.round.duels} duels): shipped ${tally(r.learned)}, scripted ${tally(r.scripted)}`),
	159	"",
	160	`CARD says (trained ${card.trainedAt}, not re-measured since): ${card.winRate}% against a ${card.baselineWinRate}% baseline ` +
	161	`over ${card.duels} duels on de_forge, and ${card.unseenWinRate}% against ${card.unseenBaselineWinRate}% ` +
	162	`on ${card.unseenMaps} maps it never trained on.`,
	163	"",
	164	where,
	165	"Nothing here retrains and nothing here edits CARD: retrain with `bun run scripts/train.ts --iters 300` and `bun run scripts/build.ts`, or revert whatever moved the world.",
	166	].join("\n");
	167	}
		@@ -1,11 +1,8 @@
1	1	import { expect, test } from "bun:test";
2		import { decode, HEADS } from "../client/action.ts";
3		import { botThink, makeBrain, policy } from "../client/bots.ts";
4		import { forward, greedy, newActivations } from "../engine/index.ts";
5		import { OBS_SIZE } from "../client/observe.ts";
	2	import { useShipped } from "../client/shipped.ts";
6	3	import { DE_FORGE } from "../client/map.ts";
7	4	import type { Level } from "../client/level.ts";
8		import { createArena, EPISODE_TICKS, observeLearner, scriptedFoe, step, stepInput } from "./arena.ts";
	5	import { evaluate, greedyDriver } from "./evaluate.ts";
9	6	import { holdoutLevels } from "./maps.ts";
10	7
11	8	/**
		@@ -17,34 +14,13 @@ import { holdoutLevels } from "./maps.ts";
17	14	const DUELS = 40;
18	15	const SEED = 4242;
19	16
20		type Tally = { won: number; lost: number; draw: number };
21
22		/** `count` duels, dealt round-robin over `levels`, driven one way or the other. */
23		function duels(drive: "policy" \| "scripted", count = DUELS, levels: Level[] = [DE_FORGE]): Tally {
24		const net = policy();
25		const obs = new Float32Array(OBS_SIZE);
26		const act = newActivations(net.shape);
27		const actions = new Int32Array(HEADS.length);
28		const tally: Tally = { won: 0, lost: 0, draw: 0 };
29		for (let i = 0; i < count; i++) {
30		const arena = createArena(SEED + i * 7919, scriptedFoe, levels[i % levels.length] as Level);
31		if (drive === "scripted") arena.learner.brain = makeBrain(0.65, SEED + i, false);
32		while (arena.ticks < EPISODE_TICKS && arena.learner.alive && arena.foe.alive) {
33		if (drive === "scripted") {
34		stepInput(arena, botThink(arena.world, arena.learner, 1 / 64));
35		} else {
36		observeLearner(arena, obs);
37		forward(net, obs, act);
38		greedy(net.shape, act.logits, actions);
39		step(arena, decode(actions));
40		}
41		}
42		if (!arena.foe.alive) tally.won += 1;
43		else if (!arena.learner.alive) tally.lost += 1;
44		else tally.draw += 1;
45		}
46		return tally;
47		}
	17	/**
	18	* `count` duels, dealt round-robin over `levels`, driven one way or the other:
	19	* the trainer's own harness, so the number here and the number the trainer
	20	* printed onto the card are the same measurement of the same thing.
	21	*/
	22	const duels = (drive: "policy" \| "scripted", count = DUELS, levels: Level[] = [DE_FORGE]) =>
	23	evaluate(drive === "scripted" ? "scripted" : greedyDriver(useShipped()), count, SEED, levels);
48	24
49	25	test("the trained policy wins more of the same duels than the bots it replaced", () => {
50	26	const learned = duels("policy");