Split The code is public; the app answers only to the crew.
rl #7
Check · Review · Land · Live done — Landed and live in production.
vesper/cs:vesper/3-rl → main 3 commits fork
20 files
changed +1178
−291
README.md +28 −6
| @@ -27,6 +27,7 @@ bun test # the check every pull runs | ||
| 27 | 27 | bun run scripts/build.ts # rebuild static/app.<hash>.js |
| 28 | 28 | bun run src/tty.ts # the terminal interface, same database |
| 29 | 29 | bun run scripts/train.ts --iters 1000 # retrain the bots' policy |
| 30 | bun run scripts/train.ts --iters 1000 --workers 4 # or on four cores | |
| 30 | 31 | bun run scripts/train.ts --bench # how fast the arena steps |
| 31 | 32 | |
| 32 | 33 | |
| @@ -74,9 +75,11 @@ the rest. | ||
| 74 | 75 | `renderer`, `hud`, `input`, `audio`, `main`. Everything except the last five is |
| 75 | 76 | headless and tested. |
| 76 | 77 | - `src/rl/` — training, which the game never imports: `arena` is the environment |
| 77 | (a real duel on de_forge with the round and the economy taken out) and `ppo` is | |
| 78 | the algorithm, with its backward pass checked against finite differences. | |
| 79 | `scripts/train.ts` runs them and writes `src/client/weights.ts`. | |
| 78 | (a real duel on de_forge with the round and the economy taken out), `ppo` is | |
| 79 | the algorithm, with its backward pass checked against finite differences, and | |
| 80 | `shard`, `worker`, `pool` and `cores` are the same update spread over the | |
| 81 | cores this process is allowed. `scripts/train.ts` runs them and writes | |
| 82 | `src/client/weights.ts`. | |
| 80 | 83 | - `src/server.ts` — the HTTP server and the security floor. `src/db.ts` is the |
| 81 | 84 | schema and its queries, `src/result.ts` validates what a browser may claim, |
| 82 | 85 | `src/pages/game.ts` renders the page. |
| @@ -87,7 +90,8 @@ the rest. | ||
| 87 | 90 | It also has a terminal: the exact command is in `FORGE_APP_SSH` when the app |
| 88 | 91 | runs (the app is the login name, the forge is the host). It opens `src/tty.ts` |
| 89 | 92 | on a real pty over the same database: the same standings, the same matches, and |
| 90 | `x` removes one. | |
| 93 | `x` removes one. <kbd>Tab</kbd> walks the three tabs — the third is the policy | |
| 94 | card the page shows, which the tab key could not reach until now. | |
| 91 | 95 | |
| 92 | 96 | The forge sets `PORT`, `FORGE_APP_DATA`, `FORGE_APP_ENV` (`production` or |
| 93 | 97 | `preview`), `FORGE_APP_URL`, `FORGE_APP_SSH` and `FORGE_APP_COMMIT` when it |
| @@ -115,12 +119,27 @@ now a policy learned with PPO. | ||
| 115 | 119 | movement, the real bullets and the real damage, and no round, economy or bomb. |
| 116 | 120 | It steps about 13,000 ticks a second on one core, allocation-free; the policy |
| 117 | 121 | decides every fourth tick and the decision is held in between. |
| 118 | - **How it learned** — `src/rl/ppo.ts`, about three hundred lines: GAE, a | |
| 122 | - **How it learned** — `src/rl/ppo.ts`, about four hundred lines: GAE, a | |
| 119 | 123 | clipped surrogate, an entropy bonus, Adam, and a backward pass written out by |
| 120 | 124 | hand and checked against finite differences in the tests. There is a |
| 121 | 125 | curriculum on how far apart the duel starts and on how hard the opponent is, |
| 122 | 126 | because handed a good opponent from the first step the policy learns to hide |
| 123 | 127 | and never comes back from it. |
| 128 | - **Who it learned against** — a mix that moves with the run: targets that do | |
| 129 | not shoot back at the start, then the scripted bots, then a league of four | |
| 130 | frozen copies of itself, drawn at random. Practising only against your newest | |
| 131 | self is how a policy learns to beat one opponent and forgets the rest. | |
| 132 | - **How fast** — the run is cut into shards, one worker each, over the cores the | |
| 133 | process is actually allowed (`src/rl/cores.ts` reads the cgroup quota, because | |
| 134 | this container reports 80 cores and is allowed two). Each shard steps its own | |
| 135 | environments into its own rows of one shared rollout and differentiates its | |
| 136 | own slice of each minibatch; the plan, the normalisation and the Adam step | |
| 137 | stay on one thread. Nothing is copied — every buffer that crosses a thread is | |
| 138 | a view onto a `SharedArrayBuffer` — so the experience is identical bit for bit | |
| 139 | however many shards there are, which `src/rl/pool.test.ts` checks against both | |
| 140 | one shard and real workers. Measured on two allowed cores, whole updates and | |
| 141 | not just stepping: 9k ticks a second on one thread, 17k across two, and 5k | |
| 142 | across eight — oversubscribing a quota does not fail to help, it hurts. | |
| 124 | 143 | - **What it costs you** — about eight kilobytes of int8 in the bundle, one |
| 125 | 144 | dequantisation when the page loads, and roughly 7,700 multiply-adds per bot |
| 126 | 145 | per decision: nine bots deciding at 16 Hz is well under a millisecond a second. |
| @@ -137,4 +156,7 @@ The site at <https://puffer.ai/> could not be reached from this sandbox (the | ||
| 137 | 156 | network here is the forge and the package registry), so the influence is from |
| 138 | 157 | what PufferLib is known for rather than from its documentation: flat observation |
| 139 | 158 | and action spaces, environment throughput treated as the thing that decides |
| 140 | whether a run is possible, and plain PPO rather than a clever algorithm. | |
| 159 | whether a run is possible, and plain PPO rather than a clever algorithm. Hence | |
| 160 | the shape of `src/rl`: the same one-process PPO loop, an arena that steps | |
| 161 | without allocating, and the parallelism in how the work is cut up rather than in | |
| 162 | a framework. |
scripts/train.ts +74 −100
| @@ -5,6 +5,7 @@ | ||
| 5 | 5 | * bun run scripts/train.ts --bench # how fast the arena steps |
| 6 | 6 | * bun run scripts/train.ts --iters 300 # train and write the weights |
| 7 | 7 | * bun run scripts/train.ts --iters 300 --dry # train without writing |
| 8 | * bun run scripts/train.ts --workers 0 # on this thread, for debugging | |
| 8 | 9 | * |
| 9 | 10 | * Nothing here is imported by the server or the page: this file runs on a |
| 10 | 11 | * machine with a few minutes to spare, and its only output is a table of |
| @@ -18,16 +19,17 @@ import { botThink, makeBrain } from "../src/client/bots.ts"; | ||
| 18 | 19 | import { rng } from "../src/client/math.ts"; |
| 19 | 20 | import { OBS_SIZE } from "../src/client/observe.ts"; |
| 20 | 21 | import { |
| 21 | copyPolicy, createPolicy, forward, greedy, newActivations, pack, sample, unpack, | |
| 22 | type Policy, type Shape, | |
| 22 | createPolicy, forward, greedy, newActivations, pack, policyBytes, policyOver, sample, | |
| 23 | tensorsOf, unpack, type Policy, type Shape, | |
| 23 | 24 | } from "../src/client/policy.ts"; |
| 24 | 25 | import { |
| 25 | createArena, EPISODE_TICKS, observeFoe, observeLearner, policyFoe, reset, scriptedFoe, SKIP, | |
| 26 | step, stepInput, stillFoe, type Arena, | |
| 26 | createArena, EPISODE_TICKS, observeLearner, reset, scriptedFoe, SKIP, | |
| 27 | step, stepInput, type Arena, | |
| 27 | 28 | } from "../src/rl/arena.ts"; |
| 28 | import { | |
| 29 | computeAdvantages, createAdam, createRollout, update, DEFAULT_HYPER, type Hyper, | |
| 30 | } from "../src/rl/ppo.ts"; | |
| 29 | import { cores } from "../src/rl/cores.ts"; | |
| 30 | import { addTotals, createPool, noTotals, updateStep } from "../src/rl/pool.ts"; | |
| 31 | import { LEAGUE, type Curriculum } from "../src/rl/shard.ts"; | |
| 32 | import { createAdam, rolloutOver, DEFAULT_HYPER, type Hyper } from "../src/rl/ppo.ts"; | |
| 31 | 33 | |
| 32 | 34 | export const SHAPE: Shape = { obs: OBS_SIZE, hidden: 64, heads: HEADS }; |
| 33 | 35 | |
| @@ -43,6 +45,10 @@ const ENVS = arg("envs", 64); | ||
| 43 | 45 | const STEPS = arg("steps", 32); |
| 44 | 46 | const ITERS = arg("iters", 240); |
| 45 | 47 | const SEED = arg("seed", 20260914); |
| 48 | // One worker a shard, defaulting to the cores this process is allowed. Zero | |
| 49 | // runs the whole thing on this thread, which is the reference the parallel path | |
| 50 | // is tested against and the way to debug it. | |
| 51 | const WORKERS = arg("workers", cores()); | |
| 46 | 52 | |
| 47 | 53 | // --- evaluation ----------------------------------------------------------- |
| 48 | 54 | |
| @@ -116,120 +122,85 @@ function benchmark(): void { | ||
| 116 | 122 | async function train(): Promise<void> { |
| 117 | 123 | const hyper: Hyper = { ...DEFAULT_HYPER, minibatch: Math.max(256, (ENVS * STEPS) / 4) }; |
| 118 | 124 | const random = rng(SEED); |
| 119 | const policy = createPolicy(SHAPE, random); | |
| 120 | const adam = createAdam(policy, hyper); | |
| 121 | const rollout = createRollout(SHAPE, ENVS, STEPS); | |
| 122 | const act = newActivations(SHAPE); | |
| 123 | const foeObs = new Float32Array(OBS_SIZE); | |
| 124 | const foeAct = newActivations(SHAPE); | |
| 125 | const foeActions = new Int32Array(HEADS.length); | |
| 126 | const actions = new Int32Array(HEADS.length); | |
| 127 | const tail = new Float32Array(OBS_SIZE); | |
| 128 | ||
| 129 | let snapshot = copyPolicy(policy); | |
| 130 | const selfPlay = policyFoe((arena) => { | |
| 131 | observeFoe(arena, foeObs); | |
| 132 | forward(snapshot, foeObs, foeAct); | |
| 133 | sample(SHAPE, foeAct.logits, random, foeActions); | |
| 134 | return decode(foeActions); | |
| 125 | const pool = createPool({ | |
| 126 | shape: SHAPE, envs: ENVS, steps: STEPS, seed: SEED, hyper, | |
| 127 | shards: Math.max(1, WORKERS), threaded: WORKERS > 0, | |
| 135 | 128 | }); |
| 136 | // Who a bot practises against, and how hard, both move with the run: it | |
| 137 | // starts mostly against targets that do not shoot back, and ends mostly | |
| 138 | // against the scripted bots at full skill and against itself. Handed a hard | |
| 139 | // opponent from the first step, the policy learns to hide instead of to | |
| 140 | // shoot, and never comes back from it. | |
| 141 | const only = process.argv[process.argv.indexOf("--foe") + 1]; | |
| 142 | const opponentFor = (e: number, progress: number): Arena["opponent"] => { | |
| 143 | if (only === "still") return stillFoe; | |
| 144 | if (only === "scripted") return scriptedFoe; | |
| 145 | const still = Math.max(0.1, 0.6 - progress * 0.5); | |
| 146 | const scripted = 0.3 + progress * 0.3; | |
| 147 | const u = (e % 10) / 10; | |
| 148 | return u < still ? stillFoe : u < still + scripted ? scriptedFoe : selfPlay; | |
| 149 | }; | |
| 150 | ||
| 151 | const arenas = Array.from({ length: ENVS }, (_, e) => createArena(SEED + e * 104729, opponentFor(e, 0))); | |
| 129 | // The weights live in the shared buffer every shard reads, so there is no | |
| 130 | // broadcast: the Adam step below is visible to the workers the moment it lands. | |
| 131 | const policy = policyOver(SHAPE, pool.weights); | |
| 132 | const initial = createPolicy(SHAPE, random); | |
| 133 | tensorsOf(policy).forEach((t, i) => t.set(tensorsOf(initial)[i] as Float32Array)); | |
| 134 | const adam = createAdam(policy, hyper); | |
| 135 | const rollout = rolloutOver(SHAPE, ENVS, STEPS, pool.rollout); | |
| 136 | const league = Array.from({ length: LEAGUE }, (_, k) => policyOver(SHAPE, pool.league, k * policyBytes(SHAPE))); | |
| 137 | const freezeEvery = Math.max(1, Math.floor(ITERS / (LEAGUE * 6))); | |
| 138 | let filled = 0; | |
| 139 | let slot = 0; | |
| 152 | 140 | |
| 153 | 141 | const baseline = evaluate("scripted", 120, SEED + 555); |
| 142 | console.log( | |
| 143 | `${pool.shards} shard${pool.shards === 1 ? "" : "s"}, ${ENVS} environments, ` + | |
| 144 | `${ENVS * STEPS} decisions an update, minibatch ${hyper.minibatch}`, | |
| 145 | ); | |
| 154 | 146 | console.log( |
| 155 | 147 | `baseline (scripted brain in the same duels): ${baseline.won} won / ${baseline.lost} lost / ${baseline.draw} drawn`, |
| 156 | 148 | ); |
| 157 | 149 | |
| 158 | const episodeReturn = new Float64Array(ENVS); | |
| 159 | let returns: number[] = []; | |
| 160 | let wins = 0, losses = 0, finished = 0; | |
| 161 | let aimSum = 0, aimN = 0, fireSum = 0, fireN = 0; | |
| 150 | const n = ENVS * STEPS; | |
| 162 | 151 | const started = Bun.nanoseconds(); |
| 163 | 152 | let envSteps = 0; |
| 153 | let sinceLog = noTotals(); | |
| 164 | 154 | |
| 165 | 155 | for (let iter = 1; iter <= ITERS; iter++) { |
| 166 | // A curriculum on distance: near the start every duel is close enough that | |
| 167 | // the enemy is a wide target, and it widens to the full range by halfway. | |
| 168 | // Learning to shoot across the map before learning to shoot at all is the | |
| 169 | // sort of thing that eats a training run. | |
| 156 | // Two curricula, both stretched over the first half of the run: how far | |
| 157 | // apart a duel starts, and who it is against. Learning to shoot across the | |
| 158 | // map before learning to shoot at all is the sort of thing that eats a run, | |
| 159 | // and handed a good opponent from the first step the policy learns to hide. | |
| 170 | 160 | const progress = Math.min(1, iter / (ITERS * 0.5)); |
| 171 | for (const arena of arenas) { | |
| 172 | arena.maxRange = 600 + progress * 900; | |
| 173 | arena.maxSkill = 0.45 + progress * 0.55; | |
| 174 | } | |
| 175 | for (let t = 0; t < STEPS; t++) { | |
| 176 | for (let e = 0; e < ENVS; e++) { | |
| 177 | const arena = arenas[e] as Arena; | |
| 178 | const i = t * ENVS + e; | |
| 179 | observeLearner(arena, rollout.obs, i * OBS_SIZE); | |
| 180 | forward(policy, rollout.obs, act, i * OBS_SIZE); | |
| 181 | rollout.values[i] = act.value; | |
| 182 | rollout.logProbs[i] = sample(SHAPE, act.logits, random, actions); | |
| 183 | for (let h = 0; h < HEADS.length; h++) rollout.actions[i * HEADS.length + h] = actions[h] as number; | |
| 184 | if ((rollout.obs[i * OBS_SIZE + 2] as number) > 0.5) { | |
| 185 | aimSum += rollout.obs[i * OBS_SIZE + 4] as number; // cos of the aim error | |
| 186 | aimN += 1; | |
| 187 | fireSum += actions[4] === 1 ? 1 : 0; | |
| 188 | fireN += 1; | |
| 189 | } | |
| 190 | const result = step(arena, decode(actions)); | |
| 191 | rollout.rewards[i] = result.reward; | |
| 192 | rollout.dones[i] = result.done ? 1 : 0; | |
| 193 | episodeReturn[e] = (episodeReturn[e] as number) + result.reward; | |
| 194 | envSteps += 1; | |
| 195 | if (result.done) { | |
| 196 | returns.push(episodeReturn[e] as number); | |
| 197 | episodeReturn[e] = 0; | |
| 198 | finished += 1; | |
| 199 | if (result.won) wins += 1; | |
| 200 | if (result.lost) losses += 1; | |
| 201 | reset(arena); | |
| 202 | arena.opponent = opponentFor(e, progress); | |
| 203 | } | |
| 204 | } | |
| 205 | } | |
| 206 | // Bootstrap: what the policy thinks the unfinished episodes are still worth. | |
| 207 | for (let e = 0; e < ENVS; e++) { | |
| 208 | observeLearner(arenas[e] as Arena, tail); | |
| 209 | forward(policy, tail, act); | |
| 210 | rollout.lastValues[e] = act.value; | |
| 211 | } | |
| 212 | computeAdvantages(rollout, hyper.gamma, hyper.lambda); | |
| 213 | const stats = update(policy, rollout, adam, random); | |
| 214 | if (iter % 10 === 0) snapshot = copyPolicy(policy); | |
| 161 | const curriculum: Curriculum = { | |
| 162 | maxRange: 600 + progress * 900, | |
| 163 | maxSkill: 0.45 + progress * 0.55, | |
| 164 | still: Math.max(0.1, 0.6 - progress * 0.5), | |
| 165 | scripted: 0.3 + progress * 0.3, | |
| 166 | filled, | |
| 167 | }; | |
| 215 | 168 | |
| 169 | const { totals: t, gradNorm } = await updateStep(pool, policy, adam, rollout, curriculum, random); | |
| 170 | envSteps += n; | |
| 171 | ||
| 172 | // The league: a copy of the policy is frozen into the next slot every so | |
| 173 | // often, and self-play duels draw from the ones that are filled. Practising | |
| 174 | // only against your newest self is how a policy learns to beat one opponent | |
| 175 | // and forgets the rest; a handful of older selves keeps it honest. | |
| 176 | if (iter % freezeEvery === 0) { | |
| 177 | const into = league[slot % LEAGUE] as Policy; | |
| 178 | tensorsOf(into).forEach((tensor, i) => tensor.set(tensorsOf(policy)[i] as Float32Array)); | |
| 179 | slot += 1; | |
| 180 | filled = Math.min(LEAGUE, filled + 1); | |
| 181 | } | |
| 182 | ||
| 183 | sinceLog = addTotals(sinceLog, t); | |
| 216 | 184 | if (iter % 10 === 0 || iter === 1) { |
| 217 | const mean = returns.length === 0 ? 0 : returns.reduce((a, b) => a + b, 0) / returns.length; | |
| 218 | 185 | const seconds = (Bun.nanoseconds() - started) / 1e9; |
| 186 | const s = sinceLog; | |
| 187 | const rate = (x: number, of: number): string => (of === 0 ? "0" : Math.round((x / of) * 100).toString()); | |
| 219 | 188 | console.log( |
| 220 | `${String(iter).padStart(4)} ${((envSteps * SKIP) / 1000).toFixed(0).padStart(6)}k ticks ` + | |
| 221 | `${((envSteps * SKIP) / seconds / 1000).toFixed(0).padStart(4)}k/s return ${mean.toFixed(2).padStart(6)} ` + | |
| 222 | `won ${finished === 0 ? 0 : Math.round((wins / finished) * 100)}% lost ${finished === 0 ? 0 : Math.round((losses / finished) * 100)}% ` + | |
| 223 | `aim ${(aimN === 0 ? 0 : aimSum / aimN).toFixed(2)} fire ${(fireN === 0 ? 0 : fireSum / fireN).toFixed(2)} ` + | |
| 224 | `ent ${stats.entropy.toFixed(2)} vloss ${stats.valueLoss.toFixed(3)} clip ${stats.clipFraction.toFixed(2)}`, | |
| 189 | `${String(iter).padStart(4)} ${((envSteps * SKIP) / 1000).toFixed(0).padStart(7)}k ticks ` + | |
| 190 | `${((envSteps * SKIP) / seconds / 1000).toFixed(0).padStart(4)}k/s ` + | |
| 191 | `return ${(s.episodes === 0 ? 0 : s.returns / s.episodes).toFixed(2).padStart(6)} ` + | |
| 192 | `won ${rate(s.wins, s.episodes)}% lost ${rate(s.losses, s.episodes)}% ` + | |
| 193 | `aim ${(s.aimN === 0 ? 0 : s.aim / s.aimN).toFixed(2)} fire ${(s.fireN === 0 ? 0 : s.fire / s.fireN).toFixed(2)} ` + | |
| 194 | `ent ${(s.entropy / Math.max(1, s.samples)).toFixed(2)} ` + | |
| 195 | `vloss ${(s.valueLoss / Math.max(1, s.samples)).toFixed(3)} ` + | |
| 196 | `clip ${(s.clipped / Math.max(1, s.samples)).toFixed(2)} gn ${gradNorm.toFixed(2)}`, | |
| 225 | 197 | ); |
| 226 | returns = []; | |
| 227 | wins = 0; losses = 0; finished = 0; | |
| 228 | aimSum = 0; aimN = 0; fireSum = 0; fireN = 0; | |
| 198 | sinceLog = noTotals(); | |
| 229 | 199 | } |
| 230 | 200 | } |
| 231 | 201 | |
| 232 | 202 | const seconds = (Bun.nanoseconds() - started) / 1e9; |
| 203 | pool.close(); | |
| 233 | 204 | const packed = pack(policy); |
| 234 | 205 | const quantised = unpack(packed); |
| 235 | 206 | const trained = evaluate(greedyDriver(policy), 240, SEED + 555); |
| @@ -249,6 +220,7 @@ async function train(): Promise<void> { | ||
| 249 | 220 | updates: ITERS, |
| 250 | 221 | stepsPerSecond: Math.round((envSteps * SKIP) / seconds), |
| 251 | 222 | minutes: Math.round(seconds / 6) / 10, |
| 223 | workers: pool.shards, | |
| 252 | 224 | duels: 240, |
| 253 | 225 | winRate: rate(shipped), |
| 254 | 226 | baselineWinRate: rate(baselineAgain), |
| @@ -263,6 +235,7 @@ type Card = { | ||
| 263 | 235 | updates: number; |
| 264 | 236 | stepsPerSecond: number; |
| 265 | 237 | minutes: number; |
| 238 | workers: number; | |
| 266 | 239 | duels: number; |
| 267 | 240 | winRate: number; |
| 268 | 241 | baselineWinRate: number; |
| @@ -286,6 +259,7 @@ export type Card = { | ||
| 286 | 259 | updates: number; |
| 287 | 260 | stepsPerSecond: number; |
| 288 | 261 | minutes: number; |
| 262 | workers: number; | |
| 289 | 263 | duels: number; |
| 290 | 264 | winRate: number; |
| 291 | 265 | baselineWinRate: number; |
src/client/policy.ts +26 −10
| @@ -46,22 +46,38 @@ export function paramCount(shape: Shape): number { | ||
| 46 | 46 | export const tensorsOf = (p: Policy): Float32Array[] => [p.w1, p.b1, p.w2, p.b2, p.wp, p.bp, p.wv, p.bv]; |
| 47 | 47 | const tensors = tensorsOf; |
| 48 | 48 | |
| 49 | /** An empty policy of the right shape: all the arrays, all zero. */ | |
| 50 | export function zeroPolicy(shape: Shape): Policy { | |
| 49 | /** How many bytes the parameters take, laid out end to end as float32. */ | |
| 50 | export const policyBytes = (shape: Shape): number => paramCount(shape) * 4; | |
| 51 | ||
| 52 | /** | |
| 53 | * A policy whose tensors are views into `buffer`, in the order `tensorsOf` | |
| 54 | * walks. Over a SharedArrayBuffer this is how the training workers read the | |
| 55 | * same weights the main thread is updating, without a copy or a broadcast. | |
| 56 | */ | |
| 57 | export function policyOver(shape: Shape, buffer: ArrayBufferLike, at = 0): Policy { | |
| 51 | 58 | const l = logitCount(shape); |
| 59 | let off = at; | |
| 60 | const take = (n: number): Float32Array => { | |
| 61 | const a = new Float32Array(buffer, off, n); | |
| 62 | off += n * 4; | |
| 63 | return a; | |
| 64 | }; | |
| 52 | 65 | return { |
| 53 | 66 | shape, |
| 54 | w1: new Float32Array(shape.hidden * shape.obs), | |
| 55 | b1: new Float32Array(shape.hidden), | |
| 56 | w2: new Float32Array(shape.hidden * shape.hidden), | |
| 57 | b2: new Float32Array(shape.hidden), | |
| 58 | wp: new Float32Array(l * shape.hidden), | |
| 59 | bp: new Float32Array(l), | |
| 60 | wv: new Float32Array(shape.hidden), | |
| 61 | bv: new Float32Array(1), | |
| 67 | w1: take(shape.hidden * shape.obs), | |
| 68 | b1: take(shape.hidden), | |
| 69 | w2: take(shape.hidden * shape.hidden), | |
| 70 | b2: take(shape.hidden), | |
| 71 | wp: take(l * shape.hidden), | |
| 72 | bp: take(l), | |
| 73 | wv: take(shape.hidden), | |
| 74 | bv: take(1), | |
| 62 | 75 | }; |
| 63 | 76 | } |
| 64 | 77 | |
| 78 | /** An empty policy of the right shape: all the arrays, all zero. */ | |
| 79 | export const zeroPolicy = (shape: Shape): Policy => policyOver(shape, new ArrayBuffer(policyBytes(shape))); | |
| 80 | ||
| 65 | 81 | /** |
| 66 | 82 | * Orthogonal-ish initialisation on a budget: normal noise scaled by fan-in, |
| 67 | 83 | * with the output heads scaled far smaller so the first policy is nearly |
src/client/weights.ts +11 −9
Diff not shown — a line 10,799 characters wide (limit 2,000). +11 −9
View filesrc/client/world.test.ts +7 −1
| @@ -81,7 +81,13 @@ test("the terrorists plant, and a planted bomb keeps the round alive past the cl | ||
| 81 | 81 | world.me.hasBomb = true; |
| 82 | 82 | world.me.pos = [sites.A[0], sites.A[1], 2]; |
| 83 | 83 | expect(siteAt(world.me.pos)).toBe("A"); |
| 84 | run(world, SHORT.plantTime + 0.3, { ...NO_INPUT, use: true }); | |
| 84 | // Standing still on the site for three seconds is how you get shot, and the | |
| 85 | // CT bots do: they kill the planter a second in. The fight is not what this | |
| 86 | // test is about, so the planter is kept on their feet for the plant. | |
| 87 | for (let t = 0; t < SHORT.plantTime + 0.3; t += TICK) { | |
| 88 | update(world, TICK, { ...NO_INPUT, use: true }); | |
| 89 | world.me.health = 100; | |
| 90 | } | |
| 85 | 91 | expect(world.bomb.state).toBe("planted"); |
| 86 | 92 | expect(world.match.phase).toBe("planted"); |
| 87 | 93 | expect(world.me.money).toBeGreaterThan(0); |
src/pages/game.ts +2 −1
| @@ -156,7 +156,8 @@ export function gamePage(db: Database, options: PageOptions): string { | ||
| 156 | 156 | <p>The bots path to a site and plant or defuse on their own, but they do not aim by rule: |
| 157 | 157 | the fight is run by a small neural network trained with PPO against |
| 158 | 158 | ${CARD.envSteps.toLocaleString("en")} ticks of this same simulation, in |
| 159 | ${escape(String(CARD.minutes))} minutes on one core. In ${CARD.duels} fixed duels it wins | |
| 159 | ${escape(String(CARD.minutes))} minutes on ${CARD.workers === 1 ? "one core" : `${CARD.workers} cores`}. | |
| 160 | In ${CARD.duels} fixed duels it wins | |
| 160 | 161 | ${CARD.winRate}% against the scripted bots this game used to have, which win |
| 161 | 162 | ${CARD.baselineWinRate}% of the same duels. Its weights are about eight kilobytes |
| 162 | 163 | of int8 inside the bundle — there is no inference server and nothing is sent anywhere. Each |
src/rl/arena.test.ts +22 −1
| @@ -3,7 +3,8 @@ import { STILL, toInput } from "../client/action.ts"; | ||
| 3 | 3 | import { OBS_SIZE } from "../client/observe.ts"; |
| 4 | 4 | import { aimAt, bodyCentre, canSee } from "../client/world.ts"; |
| 5 | 5 | import { |
| 6 | createArena, EPISODE_TICKS, observeFoe, observeLearner, reset, SKIP, step, stepInput, stillFoe, | |
| 6 | createArena, EPISODE_TICKS, observeFoe, observeLearner, policyFoe, reset, SKIP, step, stepInput, | |
| 7 | stillFoe, | |
| 7 | 8 | } from "./arena.ts"; |
| 8 | 9 | |
| 9 | 10 | test("a duel starts with two live players in range of each other", () => { |
| @@ -32,6 +33,26 @@ test("a decision is held for SKIP ticks, so the clock moves four at a time", () | ||
| 32 | 33 | expect(arena.ticks).toBe(SKIP * 2); |
| 33 | 34 | }); |
| 34 | 35 | |
| 36 | /** | |
| 37 | * A policy opponent decides on the same beat the learner does — once every | |
| 38 | * SKIP ticks, holding the answer in between — because that is what a bot in | |
| 39 | * the game does, and because thinking every tick would cost four times as | |
| 40 | * much for a foe that is no better. | |
| 41 | */ | |
| 42 | test("a policy opponent thinks once a decision and holds it in between", () => { | |
| 43 | let thoughts = 0; | |
| 44 | const arena = createArena(4242, policyFoe(() => { | |
| 45 | thoughts += 1; | |
| 46 | return { ...STILL, yawRate: 120 }; | |
| 47 | })); | |
| 48 | const before = arena.foe.yaw; | |
| 49 | for (let d = 0; d < 5; d++) step(arena, STILL); | |
| 50 | expect(arena.ticks).toBe(5 * SKIP); | |
| 51 | expect(thoughts).toBe(5); | |
| 52 | // Held, not dropped: the foe turned on every tick in between. | |
| 53 | expect(arena.foe.yaw - before).toBeCloseTo((120 * 5 * SKIP) / 64, 3); | |
| 54 | }); | |
| 55 | ||
| 35 | 56 | test("turning is a rate: the view moves by the rate times the time it is held", () => { |
| 36 | 57 | const arena = createArena(4242, stillFoe); |
| 37 | 58 | const before = arena.learner.yaw; |
src/rl/arena.ts +10 −3
| @@ -12,7 +12,7 @@ | ||
| 12 | 12 | * rules tick. `bun run scripts/train.ts --bench` prints what it costs. |
| 13 | 13 | */ |
| 14 | 14 | |
| 15 | import { decode, toInput, type Intent } from "../client/action.ts"; | |
| 15 | import { decode, STILL, toInput, type Intent } from "../client/action.ts"; | |
| 16 | 16 | import { makeBrain, botThink } from "../client/bots.ts"; |
| 17 | 17 | import * as map from "../client/map.ts"; |
| 18 | 18 | import { angleDelta, clamp, dist, rng, type Vec3 } from "../client/math.ts"; |
| @@ -67,10 +67,17 @@ export const scriptedFoe = (arena: Arena, dt: number): Input => botThink(arena.w | ||
| 67 | 67 | /** A target that does nothing, for a sanity check that shooting is worth it. */ |
| 68 | 68 | export const stillFoe = (): Input => NO_INPUT; |
| 69 | 69 | |
| 70 | /** A frozen policy driving the other side: self-play without a second process. */ | |
| 70 | /** | |
| 71 | * A frozen policy driving the other side: self-play without a second process. | |
| 72 | * It thinks on the same beat the learner does — every SKIP ticks, holding the | |
| 73 | * answer in between — because that is what a bot in the game does, and because | |
| 74 | * thinking four times as often would cost four times as much for a worse foe. | |
| 75 | */ | |
| 71 | 76 | export function policyFoe(think: (arena: Arena) => Intent): (arena: Arena, dt: number) => Input { |
| 77 | let held: Intent = STILL; | |
| 72 | 78 | return (arena) => { |
| 73 | const intent = think(arena); | |
| 79 | if (arena.ticks % SKIP === 0) held = think(arena); | |
| 80 | const intent = held; | |
| 74 | 81 | const ammo = arena.foe.ammo[arena.foe.weapon]; |
| 75 | 82 | const foe = arena.foe; |
| 76 | 83 | foe.yaw += intent.yawRate * TICK; |
src/rl/cores.test.ts added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { cores, parseQuota } from "./cores.ts"; | |
| 3 | ||
| 4 | test("a cgroup quota reads as the cores it allows", () => { | |
| 5 | expect(parseQuota("200000 100000")).toBe(2); | |
| 6 | expect(parseQuota("100000 100000\n")).toBe(1); | |
| 7 | expect(parseQuota("1650000 100000")).toBe(16); | |
| 8 | // A fraction of a core is still one thread to run on. | |
| 9 | expect(parseQuota("50000 100000")).toBe(1); | |
| 10 | }); | |
| 11 | ||
| 12 | test("no quota, or a broken one, reads as no answer", () => { | |
| 13 | expect(parseQuota("max 100000")).toBe(null); | |
| 14 | expect(parseQuota("")).toBe(null); | |
| 15 | expect(parseQuota("200000")).toBe(null); | |
| 16 | expect(parseQuota("200000 0")).toBe(null); | |
| 17 | expect(parseQuota("-1 100000")).toBe(null); | |
| 18 | }); | |
| 19 | ||
| 20 | test("the core count is at least one and never past the cap", () => { | |
| 21 | expect(cores()).toBeGreaterThanOrEqual(1); | |
| 22 | expect(cores(4)).toBeLessThanOrEqual(4); | |
| 23 | expect(cores(1)).toBe(1); | |
| 24 | }); |
src/rl/cores.ts added +44
| @@ -0,0 +1,44 @@ | ||
| 1 | /** | |
| 2 | * How many cores this process may actually use. | |
| 3 | * | |
| 4 | * `navigator.hardwareConcurrency` counts the cores the machine has, which in a | |
| 5 | * container is a lie worth catching: this box reports 80 and is allowed two. | |
| 6 | * Oversubscribing a cgroup quota does not just fail to help, it hurts — the | |
| 7 | * scheduler hands out the quota and then stalls every thread until the next | |
| 8 | * period, and a training run that should take ten minutes takes thirty. | |
| 9 | * | |
| 10 | * Measured here at 64 environments: 9k ticks/s with one shard, 14k with two, | |
| 11 | * 16k with three, 12k with four, 5k with eight. So the default is the quota. | |
| 12 | */ | |
| 13 | ||
| 14 | import { readFileSync } from "node:fs"; | |
| 15 | ||
| 16 | /** cgroup v2: "<quota> <period>" in microseconds, or "max <period>" for none. */ | |
| 17 | export function parseQuota(text: string): number | null { | |
| 18 | const [quota, period] = text.trim().split(/\s+/); | |
| 19 | if (quota === undefined || period === undefined) return null; | |
| 20 | const q = Number(quota); | |
| 21 | const p = Number(period); | |
| 22 | if (!Number.isFinite(q) || !Number.isFinite(p) || q <= 0 || p <= 0) return null; | |
| 23 | return Math.max(1, Math.floor(q / p)); | |
| 24 | } | |
| 25 | ||
| 26 | const read = (path: string): string | null => { | |
| 27 | try { | |
| 28 | return readFileSync(path, "utf8"); | |
| 29 | } catch { | |
| 30 | return null; | |
| 31 | } | |
| 32 | }; | |
| 33 | ||
| 34 | /** | |
| 35 | * The quota if there is one, the machine's core count if there is not, and two | |
| 36 | * if neither can be read. Never more than `cap`: past the quota every extra | |
| 37 | * shard is another barrier to wait on and another thread to be throttled. | |
| 38 | */ | |
| 39 | export function cores(cap = 16): number { | |
| 40 | const v2 = read("/sys/fs/cgroup/cpu.max"); | |
| 41 | const quota = v2 === null ? null : parseQuota(v2); | |
| 42 | const hardware = globalThis.navigator?.hardwareConcurrency ?? 2; | |
| 43 | return Math.max(1, Math.min(cap, quota ?? hardware)); | |
| 44 | } |
src/rl/pool.test.ts added +168
| @@ -0,0 +1,168 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { HEADS } from "../client/action.ts"; | |
| 3 | import { rng } from "../client/math.ts"; | |
| 4 | import { OBS_SIZE } from "../client/observe.ts"; | |
| 5 | import { createPolicy, policyBytes, policyOver, tensorsOf, type Shape } from "../client/policy.ts"; | |
| 6 | import { scriptedFoe, stillFoe } from "./arena.ts"; | |
| 7 | import { addTotals, createPool, noTotals, totals, updateStep, type Pool } from "./pool.ts"; | |
| 8 | import { computeAdvantages, createAdam, normalise, rolloutOver, DEFAULT_HYPER, type Hyper } from "./ppo.ts"; | |
| 9 | import { createShard, opponentFor, split, LEAGUE, TALLY, TALLY_SIZE } from "./shard.ts"; | |
| 10 | ||
| 11 | const SHAPE: Shape = { obs: OBS_SIZE, hidden: 12, heads: HEADS }; | |
| 12 | const ENVS = 8; | |
| 13 | const STEPS = 6; | |
| 14 | const SEED = 31337; | |
| 15 | const HYPER: Hyper = { ...DEFAULT_HYPER, epochs: 1, minibatch: 16 }; | |
| 16 | ||
| 17 | const CURRICULUM = { maxRange: 900, maxSkill: 0.7, still: 0.3, scripted: 0.4, filled: 0 }; | |
| 18 | ||
| 19 | /** A pool of `shards`, with the same starting weights however it is cut up. */ | |
| 20 | function pool(shards: number, threaded = false): Pool { | |
| 21 | const p = createPool({ shape: SHAPE, envs: ENVS, steps: STEPS, seed: SEED, hyper: HYPER, shards, threaded }); | |
| 22 | const weights = policyOver(SHAPE, p.weights); | |
| 23 | const initial = createPolicy(SHAPE, rng(SEED)); | |
| 24 | tensorsOf(weights).forEach((t, i) => t.set(tensorsOf(initial)[i] as Float32Array)); | |
| 25 | return p; | |
| 26 | } | |
| 27 | ||
| 28 | const slab = (p: Pool) => rolloutOver(SHAPE, ENVS, STEPS, p.rollout); | |
| 29 | ||
| 30 | /** Every number a rollout holds, so a comparison cannot miss a field. */ | |
| 31 | const fields = (p: Pool): number[][] => { | |
| 32 | const r = slab(p); | |
| 33 | return [ | |
| 34 | Array.from(r.obs), Array.from(r.actions), Array.from(r.logProbs), Array.from(r.values), | |
| 35 | Array.from(r.rewards), Array.from(r.dones), Array.from(r.lastValues), | |
| 36 | ]; | |
| 37 | }; | |
| 38 | ||
| 39 | test("shards tile the environments exactly once, whatever the count", () => { | |
| 40 | for (const total of [1, 7, 8, 64, 100]) { | |
| 41 | for (const n of [1, 2, 3, 8, 16]) { | |
| 42 | expect(split(total, 0, n)).toBe(0); | |
| 43 | expect(split(total, n, n)).toBe(total); | |
| 44 | for (let k = 0; k < n; k++) expect(split(total, k + 1, n)).toBeGreaterThanOrEqual(split(total, k, n)); | |
| 45 | } | |
| 46 | } | |
| 47 | }); | |
| 48 | ||
| 49 | /** | |
| 50 | * The claim the whole parallel path rests on: cutting the work up does not | |
| 51 | * change the work. Every random draw is keyed to the environment it belongs | |
| 52 | * to, so four shards see exactly the experience one shard sees — bit for bit, | |
| 53 | * not nearly. | |
| 54 | */ | |
| 55 | test("four shards collect exactly the experience one shard collects", async () => { | |
| 56 | const one = pool(1); | |
| 57 | const four = pool(4); | |
| 58 | await one.run({ kind: "collect", curriculum: CURRICULUM }); | |
| 59 | await four.run({ kind: "collect", curriculum: CURRICULUM }); | |
| 60 | ||
| 61 | const a = fields(one); | |
| 62 | const b = fields(four); | |
| 63 | for (let k = 0; k < a.length; k++) expect(b[k]).toEqual(a[k] as number[]); | |
| 64 | // Not a slab of zeros: there was something to agree about. | |
| 65 | expect((a[3] as number[]).some((v) => v !== 0)).toBe(true); | |
| 66 | expect(totals(one)).toEqual(totals(four)); | |
| 67 | one.close(); | |
| 68 | four.close(); | |
| 69 | }); | |
| 70 | ||
| 71 | test("a minibatch's gradient is the same sum whether one shard takes it or four", async () => { | |
| 72 | const one = pool(1); | |
| 73 | const four = pool(4); | |
| 74 | await one.run({ kind: "collect", curriculum: CURRICULUM }); | |
| 75 | await four.run({ kind: "collect", curriculum: CURRICULUM }); | |
| 76 | for (const p of [one, four]) computeAdvantages(slab(p), HYPER.gamma, HYPER.lambda); | |
| 77 | ||
| 78 | const start = 0, end = 32; | |
| 79 | const { mean, std } = normalise(slab(one), one.order, start, end); | |
| 80 | await one.run({ kind: "gradients", start, end, mean, std }); | |
| 81 | await four.run({ kind: "gradients", start, end, mean, std }); | |
| 82 | ||
| 83 | const whole = tensorsOf(one.reduce()); | |
| 84 | const summed = tensorsOf(four.reduce()); | |
| 85 | let magnitude = 0; | |
| 86 | for (let k = 0; k < whole.length; k++) { | |
| 87 | const w = whole[k] as Float32Array; | |
| 88 | const s = summed[k] as Float32Array; | |
| 89 | for (let i = 0; i < w.length; i++) { | |
| 90 | magnitude += Math.abs(w[i] as number); | |
| 91 | // Summed in a different order, so float for float and not bit for bit. | |
| 92 | expect(s[i] as number).toBeCloseTo(w[i] as number, 5); | |
| 93 | } | |
| 94 | } | |
| 95 | expect(magnitude).toBeGreaterThan(0); | |
| 96 | // The slices tile the minibatch: the same samples were counted, exactly once. | |
| 97 | expect(totals(four).samples).toBe(end - start); | |
| 98 | expect(totals(four).samples).toBe(totals(one).samples); | |
| 99 | one.close(); | |
| 100 | four.close(); | |
| 101 | }); | |
| 102 | ||
| 103 | /** | |
| 104 | * The same check again, across real threads. This is the one that fails when | |
| 105 | * the worker wiring breaks — a buffer copied instead of shared, a message | |
| 106 | * dropped — and it is cheap enough to keep in the suite. | |
| 107 | */ | |
| 108 | test("two workers collect what the same pool collects on this thread", async () => { | |
| 109 | const here = pool(2, false); | |
| 110 | const there = pool(2, true); | |
| 111 | await here.run({ kind: "collect", curriculum: CURRICULUM }); | |
| 112 | await there.run({ kind: "collect", curriculum: CURRICULUM }); | |
| 113 | ||
| 114 | const a = fields(here); | |
| 115 | const b = fields(there); | |
| 116 | for (let k = 0; k < a.length; k++) expect(b[k]).toEqual(a[k] as number[]); | |
| 117 | expect(totals(there)).toEqual(totals(here)); | |
| 118 | here.close(); | |
| 119 | there.close(); | |
| 120 | }, 30_000); | |
| 121 | ||
| 122 | test("an update moves the weights and hands back what the shards counted", async () => { | |
| 123 | const p = pool(2); | |
| 124 | const policy = policyOver(SHAPE, p.weights); | |
| 125 | const before = (tensorsOf(policy)[0] as Float32Array).slice(); | |
| 126 | const adam = createAdam(policy, HYPER); | |
| 127 | const { totals: t, gradNorm } = await updateStep( | |
| 128 | p, policy, adam, slab(p), CURRICULUM, rng(7), | |
| 129 | ); | |
| 130 | ||
| 131 | expect(Number.isFinite(gradNorm)).toBe(true); | |
| 132 | expect(gradNorm).toBeGreaterThan(0); | |
| 133 | expect(t.samples).toBe(ENVS * STEPS); // one epoch, every sample once | |
| 134 | expect(t.aimN).toBeGreaterThan(0); | |
| 135 | expect(Number.isFinite(t.entropy)).toBe(true); | |
| 136 | const after = tensorsOf(policy)[0] as Float32Array; | |
| 137 | expect(Array.from(after)).not.toEqual(Array.from(before)); | |
| 138 | p.close(); | |
| 139 | }); | |
| 140 | ||
| 141 | test("the league is only drawn from once a slot has been frozen", () => { | |
| 142 | const p = pool(1); | |
| 143 | const shard = createShard({ | |
| 144 | shape: SHAPE, envs: ENVS, steps: STEPS, from: 0, to: ENVS, seed: SEED, hyper: HYPER, | |
| 145 | weights: p.weights, league: p.league, rollout: p.rollout, | |
| 146 | grads: new SharedArrayBuffer(policyBytes(SHAPE)), order: new SharedArrayBuffer(ENVS * STEPS * 4), | |
| 147 | tally: new SharedArrayBuffer(TALLY_SIZE * 8), index: 0, shards: 1, | |
| 148 | }); | |
| 149 | const empty = { ...CURRICULUM, still: 0, scripted: 0, filled: 0 }; | |
| 150 | // Nothing frozen yet: the scripted bots stand in, never an all-zero policy. | |
| 151 | for (let e = 0; e < ENVS; e++) expect(opponentFor(shard, e, empty)).toBe(scriptedFoe); | |
| 152 | const drawn = Array.from({ length: ENVS }, (_, e) => opponentFor(shard, e, { ...empty, filled: LEAGUE })); | |
| 153 | for (const foe of drawn) expect(foe === scriptedFoe || foe === stillFoe).toBe(false); | |
| 154 | // And the mix is still honoured: a fully still curriculum shoots at nothing. | |
| 155 | for (let e = 0; e < ENVS; e++) expect(opponentFor(shard, e, { ...empty, still: 1 })).toBe(stillFoe); | |
| 156 | p.close(); | |
| 157 | }); | |
| 158 | ||
| 159 | test("the tally adds up across shards and across updates", () => { | |
| 160 | const p = pool(3); | |
| 161 | for (let k = 0; k < 3; k++) p.tally[k * TALLY_SIZE + TALLY.episodes] = k + 1; | |
| 162 | expect(totals(p).episodes).toBe(6); | |
| 163 | expect(addTotals(totals(p), totals(p)).episodes).toBe(12); | |
| 164 | expect(noTotals().episodes).toBe(0); | |
| 165 | p.clearTally(); | |
| 166 | expect(totals(p).episodes).toBe(0); | |
| 167 | p.close(); | |
| 168 | }); |
src/rl/pool.ts added +205
| @@ -0,0 +1,205 @@ | ||
| 1 | /** | |
| 2 | * The pool: the same training step, spread over as many cores as you have. | |
| 3 | * | |
| 4 | * PufferLib's one big claim is that a reinforcement learning run is decided by | |
| 5 | * how many environment steps a second you can afford, and that the way to | |
| 6 | * afford more is to stop copying and to use every core. Both halves of an | |
| 7 | * update are parallel here: | |
| 8 | * | |
| 9 | * collect each shard steps its own environments into its own rows | |
| 10 | * gradient each shard differentiates its own slice of the same minibatch, | |
| 11 | * the main thread sums the gradients and takes the Adam step | |
| 12 | * | |
| 13 | * The minibatch plan, the advantage normalisation and the optimiser stay on the | |
| 14 | * main thread, so `workers` buys throughput and changes nothing else: the | |
| 15 | * experience collected is identical bit for bit however many shards there are, | |
| 16 | * and the gradient is the same sum added up in a different order. | |
| 17 | * Everything crossing a thread is a view onto a SharedArrayBuffer; the messages | |
| 18 | * are four words long and say only "go" and "done". | |
| 19 | * | |
| 20 | * More shards than cores is slower, not faster — see `src/rl/cores.ts`. | |
| 21 | */ | |
| 22 | ||
| 23 | import { policyBytes, policyOver, type Policy, type Shape } from "../client/policy.ts"; | |
| 24 | import { | |
| 25 | addGrads, applyGrads, computeAdvantages, normalise, rolloutBytes, shuffle, | |
| 26 | type Adam, type Hyper, type Rollout, | |
| 27 | } from "./ppo.ts"; | |
| 28 | import { | |
| 29 | collect, createShard, gradients, split, LEAGUE, TALLY, TALLY_SIZE, | |
| 30 | type Curriculum, type Shard, type ShardSpec, | |
| 31 | } from "./shard.ts"; | |
| 32 | ||
| 33 | export type PoolSpec = { | |
| 34 | shape: Shape; | |
| 35 | envs: number; | |
| 36 | steps: number; | |
| 37 | seed: number; | |
| 38 | hyper: Hyper; | |
| 39 | /** How many shards the work is cut into. */ | |
| 40 | shards: number; | |
| 41 | /** False runs them all on this thread: the reference, and what tests use. */ | |
| 42 | threaded: boolean; | |
| 43 | }; | |
| 44 | ||
| 45 | export type Job = | |
| 46 | | { kind: "collect"; curriculum: Curriculum } | |
| 47 | | { kind: "gradients"; start: number; end: number; mean: number; std: number }; | |
| 48 | ||
| 49 | export type Pool = { | |
| 50 | shards: number; | |
| 51 | /** The live weights every shard reads, and the main thread writes. */ | |
| 52 | weights: SharedArrayBuffer; | |
| 53 | /** LEAGUE frozen opponents, end to end. */ | |
| 54 | league: SharedArrayBuffer; | |
| 55 | rollout: SharedArrayBuffer; | |
| 56 | order: Int32Array; | |
| 57 | tally: Float64Array; | |
| 58 | grads: Policy[]; | |
| 59 | run: (job: Job) => Promise<void>; | |
| 60 | /** Sum every shard's gradients into the first one, and hand it back. */ | |
| 61 | reduce: () => Policy; | |
| 62 | clearTally: () => void; | |
| 63 | close: () => void; | |
| 64 | }; | |
| 65 | ||
| 66 | const shared = (bytes: number): SharedArrayBuffer => new SharedArrayBuffer(bytes); | |
| 67 | ||
| 68 | export function createPool(spec: PoolSpec): Pool { | |
| 69 | const shards = Math.max(1, spec.shards); | |
| 70 | const bytes = policyBytes(spec.shape); | |
| 71 | const weights = shared(bytes); | |
| 72 | const league = shared(bytes * LEAGUE); | |
| 73 | const rollout = shared(rolloutBytes(spec.shape, spec.envs, spec.steps)); | |
| 74 | const orderBuffer = shared(spec.envs * spec.steps * 4); | |
| 75 | const tallyBuffer = shared(shards * TALLY_SIZE * 8); | |
| 76 | const gradBuffers = Array.from({ length: shards }, () => shared(bytes)); | |
| 77 | ||
| 78 | const specs: ShardSpec[] = Array.from({ length: shards }, (_, k) => ({ | |
| 79 | shape: spec.shape, | |
| 80 | envs: spec.envs, | |
| 81 | steps: spec.steps, | |
| 82 | from: split(spec.envs, k, shards), | |
| 83 | to: split(spec.envs, k + 1, shards), | |
| 84 | seed: spec.seed, | |
| 85 | hyper: spec.hyper, | |
| 86 | weights, | |
| 87 | league, | |
| 88 | rollout, | |
| 89 | grads: gradBuffers[k] as SharedArrayBuffer, | |
| 90 | order: orderBuffer, | |
| 91 | tally: tallyBuffer, | |
| 92 | index: k, | |
| 93 | shards, | |
| 94 | })); | |
| 95 | ||
| 96 | const local: Shard[] = []; | |
| 97 | const workers: Worker[] = []; | |
| 98 | if (!spec.threaded) { | |
| 99 | for (const s of specs) local.push(createShard(s)); | |
| 100 | } else { | |
| 101 | for (const s of specs) { | |
| 102 | const worker = new Worker(new URL("./worker.ts", import.meta.url).href); | |
| 103 | worker.postMessage({ kind: "init", spec: s }); | |
| 104 | workers.push(worker); | |
| 105 | } | |
| 106 | } | |
| 107 | ||
| 108 | const runOne = (shard: Shard, job: Job): void => { | |
| 109 | if (job.kind === "collect") collect(shard, job.curriculum); | |
| 110 | else gradients(shard, job.start, job.end, job.mean, job.std); | |
| 111 | }; | |
| 112 | ||
| 113 | const run = async (job: Job): Promise<void> => { | |
| 114 | if (workers.length === 0) { | |
| 115 | for (const shard of local) runOne(shard, job); | |
| 116 | return; | |
| 117 | } | |
| 118 | await Promise.all(workers.map((worker) => new Promise<void>((resolve, reject) => { | |
| 119 | worker.onmessage = (event: MessageEvent): void => { | |
| 120 | const data = event.data as { kind: string; error?: string }; | |
| 121 | if (data.kind === "error") reject(new Error(data.error ?? "worker failed")); | |
| 122 | else resolve(); | |
| 123 | }; | |
| 124 | worker.postMessage(job); | |
| 125 | }))); | |
| 126 | }; | |
| 127 | ||
| 128 | const grads = gradBuffers.map((b) => policyOver(spec.shape, b)); | |
| 129 | const tally = new Float64Array(tallyBuffer); | |
| 130 | const order = new Int32Array(orderBuffer); | |
| 131 | for (let i = 0; i < order.length; i++) order[i] = i; | |
| 132 | ||
| 133 | return { | |
| 134 | shards, | |
| 135 | weights, | |
| 136 | league, | |
| 137 | rollout, | |
| 138 | order, | |
| 139 | tally, | |
| 140 | grads, | |
| 141 | run, | |
| 142 | reduce: () => { | |
| 143 | const first = grads[0] as Policy; | |
| 144 | for (let k = 1; k < grads.length; k++) addGrads(first, grads[k] as Policy); | |
| 145 | return first; | |
| 146 | }, | |
| 147 | clearTally: () => tally.fill(0), | |
| 148 | close: () => { | |
| 149 | for (const worker of workers) worker.terminate(); | |
| 150 | }, | |
| 151 | }; | |
| 152 | } | |
| 153 | ||
| 154 | /** | |
| 155 | * One PPO update, driven across the shards: collect a slab, work out the | |
| 156 | * advantages, then a few epochs of minibatches, each one differentiated in | |
| 157 | * parallel and applied once. The plan — the shuffle, the minibatch boundaries, | |
| 158 | * the normalisation and Adam — stays here on one thread, which is what makes | |
| 159 | * the result the same for one shard and for sixteen. | |
| 160 | */ | |
| 161 | export async function updateStep( | |
| 162 | pool: Pool, | |
| 163 | policy: Policy, | |
| 164 | adam: Adam, | |
| 165 | rollout: Rollout, | |
| 166 | curriculum: Curriculum, | |
| 167 | random: () => number, | |
| 168 | ): Promise<{ totals: Totals; gradNorm: number }> { | |
| 169 | const hyper = adam.hyper; | |
| 170 | const n = rollout.envs * rollout.steps; | |
| 171 | pool.clearTally(); | |
| 172 | await pool.run({ kind: "collect", curriculum }); | |
| 173 | computeAdvantages(rollout, hyper.gamma, hyper.lambda); | |
| 174 | ||
| 175 | let gradNorm = 0; | |
| 176 | for (let epoch = 0; epoch < hyper.epochs; epoch++) { | |
| 177 | shuffle(pool.order, random); | |
| 178 | for (let start = 0; start < n; start += hyper.minibatch) { | |
| 179 | const end = Math.min(n, start + hyper.minibatch); | |
| 180 | const size = end - start; | |
| 181 | if (size < 2) continue; | |
| 182 | const { mean, std } = normalise(rollout, pool.order, start, end); | |
| 183 | await pool.run({ kind: "gradients", start, end, mean, std }); | |
| 184 | gradNorm = applyGrads(policy, pool.reduce(), adam, 1 / size); | |
| 185 | } | |
| 186 | } | |
| 187 | return { totals: totals(pool), gradNorm }; | |
| 188 | } | |
| 189 | ||
| 190 | /** Every counter the shards kept this update, added up. */ | |
| 191 | export type Totals = Record<keyof typeof TALLY, number>; | |
| 192 | ||
| 193 | export const noTotals = (): Totals => | |
| 194 | Object.fromEntries(Object.keys(TALLY).map((k) => [k, 0])) as Totals; | |
| 195 | ||
| 196 | export function totals(pool: Pool): Totals { | |
| 197 | const out = noTotals(); | |
| 198 | for (let k = 0; k < pool.shards; k++) | |
| 199 | for (const [name, at] of Object.entries(TALLY)) | |
| 200 | out[name as keyof Totals] += pool.tally[k * TALLY_SIZE + at] as number; | |
| 201 | return out; | |
| 202 | } | |
| 203 | ||
| 204 | export const addTotals = (a: Totals, b: Totals): Totals => | |
| 205 | Object.fromEntries(Object.entries(a).map(([k, v]) => [k, v + b[k as keyof Totals]])) as Totals; |
src/rl/ppo.ts +193 −92
| @@ -49,6 +49,17 @@ export function clearGrads(g: Grads): void { | ||
| 49 | 49 | for (const t of tensorsOf(g)) t.fill(0); |
| 50 | 50 | } |
| 51 | 51 | |
| 52 | /** Add one set of gradients into another: the whole of data-parallel's reduce. */ | |
| 53 | export function addGrads(into: Grads, from: Grads): void { | |
| 54 | const a = tensorsOf(into); | |
| 55 | const b = tensorsOf(from); | |
| 56 | for (let k = 0; k < a.length; k++) { | |
| 57 | const t = a[k] as Float32Array; | |
| 58 | const u = b[k] as Float32Array; | |
| 59 | for (let i = 0; i < t.length; i++) t[i] = (t[i] as number) + (u[i] as number); | |
| 60 | } | |
| 61 | } | |
| 62 | ||
| 52 | 63 | /** |
| 53 | 64 | * Accumulate the gradient of one sample into `g`, given the activations that |
| 54 | 65 | * produced it and the derivative of the loss with respect to the logits and |
| @@ -171,22 +182,50 @@ export type Rollout = { | ||
| 171 | 182 | lastValues: Float32Array; |
| 172 | 183 | }; |
| 173 | 184 | |
| 174 | export function createRollout(shape: Shape, envs: number, steps: number): Rollout { | |
| 185 | const pad4 = (n: number): number => (n + 3) & ~3; | |
| 186 | ||
| 187 | /** How many bytes one slab takes, laid out end to end. */ | |
| 188 | export const rolloutBytes = (shape: Shape, envs: number, steps: number): number => { | |
| 175 | 189 | const n = envs * steps; |
| 190 | return n * shape.obs * 4 + n * shape.heads.length * 4 + n * 4 * 5 + pad4(n) + envs * 4; | |
| 191 | }; | |
| 192 | ||
| 193 | /** | |
| 194 | * A slab whose arrays are views into `buffer`. Over a SharedArrayBuffer every | |
| 195 | * worker writes its own environments' rows of the one slab, so collecting in | |
| 196 | * parallel costs no copy and no merge: the indices are already disjoint. | |
| 197 | */ | |
| 198 | export function rolloutOver(shape: Shape, envs: number, steps: number, buffer: ArrayBufferLike, at = 0): Rollout { | |
| 199 | const n = envs * steps; | |
| 200 | let off = at; | |
| 201 | const f32 = (len: number): Float32Array => { | |
| 202 | const a = new Float32Array(buffer, off, len); | |
| 203 | off += len * 4; | |
| 204 | return a; | |
| 205 | }; | |
| 206 | const i32 = (len: number): Int32Array => { | |
| 207 | const a = new Int32Array(buffer, off, len); | |
| 208 | off += len * 4; | |
| 209 | return a; | |
| 210 | }; | |
| 211 | const obs = f32(n * shape.obs); | |
| 212 | const actions = i32(n * shape.heads.length); | |
| 213 | const logProbs = f32(n); | |
| 214 | const values = f32(n); | |
| 215 | const rewards = f32(n); | |
| 216 | const dones = new Uint8Array(buffer, off, n); | |
| 217 | off += pad4(n); | |
| 176 | 218 | return { |
| 177 | shape, envs, steps, | |
| 178 | obs: new Float32Array(n * shape.obs), | |
| 179 | actions: new Int32Array(n * shape.heads.length), | |
| 180 | logProbs: new Float32Array(n), | |
| 181 | values: new Float32Array(n), | |
| 182 | rewards: new Float32Array(n), | |
| 183 | dones: new Uint8Array(n), | |
| 184 | advantages: new Float32Array(n), | |
| 185 | returns: new Float32Array(n), | |
| 186 | lastValues: new Float32Array(envs), | |
| 219 | shape, envs, steps, obs, actions, logProbs, values, rewards, dones, | |
| 220 | advantages: f32(n), | |
| 221 | returns: f32(n), | |
| 222 | lastValues: f32(envs), | |
| 187 | 223 | }; |
| 188 | 224 | } |
| 189 | 225 | |
| 226 | export const createRollout = (shape: Shape, envs: number, steps: number): Rollout => | |
| 227 | rolloutOver(shape, envs, steps, new ArrayBuffer(rolloutBytes(shape, envs, steps))); | |
| 228 | ||
| 190 | 229 | /** |
| 191 | 230 | * Generalised advantage estimation, walked backwards through time per |
| 192 | 231 | * environment. A step that ended an episode does not look through the seam. |
| @@ -214,101 +253,163 @@ export type UpdateStats = { | ||
| 214 | 253 | gradNorm: number; |
| 215 | 254 | }; |
| 216 | 255 | |
| 217 | /** The PPO update itself: a few epochs of minibatches over one rollout. */ | |
| 256 | /** Scratch one gradient pass needs, made once so a minibatch allocates nothing. */ | |
| 257 | export type Workspace = { | |
| 258 | a: Activations; | |
| 259 | dLogits: Float32Array; | |
| 260 | probs: Float32Array; | |
| 261 | scratch: { dh1: Float32Array; dh2: Float32Array }; | |
| 262 | }; | |
| 263 | ||
| 264 | export const createWorkspace = (shape: Shape): Workspace => ({ | |
| 265 | a: newActivations(shape), | |
| 266 | dLogits: new Float32Array(logitCount(shape)), | |
| 267 | probs: new Float32Array(Math.max(...shape.heads)), | |
| 268 | scratch: { dh1: new Float32Array(shape.hidden), dh2: new Float32Array(shape.hidden) }, | |
| 269 | }); | |
| 270 | ||
| 271 | /** What one slice of a minibatch contributed, before it is divided through. */ | |
| 272 | export type Sums = { policyLoss: number; valueLoss: number; entropy: number; clipped: number; samples: number }; | |
| 273 | ||
| 274 | export const noSums = (): Sums => ({ policyLoss: 0, valueLoss: 0, entropy: 0, clipped: 0, samples: 0 }); | |
| 275 | ||
| 276 | /** | |
| 277 | * The mean and standard deviation of the advantages in one minibatch. | |
| 278 | * Normalising inside the minibatch makes the scale of a reward stop mattering, | |
| 279 | * only its rank against its neighbours — so it is computed over the whole | |
| 280 | * minibatch once, never per shard, or every worker would normalise differently. | |
| 281 | */ | |
| 282 | export function normalise(r: Rollout, order: Int32Array, start: number, end: number): { mean: number; std: number } { | |
| 283 | const size = end - start; | |
| 284 | let mean = 0; | |
| 285 | for (let k = start; k < end; k++) mean += r.advantages[order[k] as number] as number; | |
| 286 | mean /= size; | |
| 287 | let variance = 0; | |
| 288 | for (let k = start; k < end; k++) variance += ((r.advantages[order[k] as number] as number) - mean) ** 2; | |
| 289 | return { mean, std: Math.sqrt(variance / size) + 1e-8 }; | |
| 290 | } | |
| 291 | ||
| 292 | /** | |
| 293 | * Accumulate the clipped-surrogate gradient for `order[start..end)` into `g`, | |
| 294 | * and add up what those samples cost. This is the whole of PPO's inner loop and | |
| 295 | * the only place it lives: one thread runs it over a minibatch, or sixteen run | |
| 296 | * it over a sixteenth each and the gradients are summed, which is the same sum. | |
| 297 | */ | |
| 298 | export function accumulate( | |
| 299 | p: Policy, | |
| 300 | r: Rollout, | |
| 301 | g: Grads, | |
| 302 | order: Int32Array, | |
| 303 | start: number, | |
| 304 | end: number, | |
| 305 | mean: number, | |
| 306 | std: number, | |
| 307 | hyper: Hyper, | |
| 308 | w: Workspace, | |
| 309 | sums: Sums, | |
| 310 | ): Sums { | |
| 311 | const heads = p.shape.heads; | |
| 312 | const { a, dLogits, probs, scratch } = w; | |
| 313 | for (let k = start; k < end; k++) { | |
| 314 | const i = order[k] as number; | |
| 315 | forward(p, r.obs, a, i * p.shape.obs); | |
| 316 | dLogits.fill(0); | |
| 317 | ||
| 318 | let logProb = 0; | |
| 319 | let entropy = 0; | |
| 320 | for (let h = 0; h < heads.length; h++) { | |
| 321 | const size_ = heads[h] as number; | |
| 322 | const at = headOffset(p.shape, h); | |
| 323 | softmax(a.logits, at, size_, probs); | |
| 324 | const pick = r.actions[i * heads.length + h] as number; | |
| 325 | logProb += Math.log(Math.max(1e-9, probs[pick] as number)); | |
| 326 | for (let q = 0; q < size_; q++) entropy -= (probs[q] as number) * Math.log(Math.max(1e-9, probs[q] as number)); | |
| 327 | } | |
| 328 | ||
| 329 | const advantage = ((r.advantages[i] as number) - mean) / std; | |
| 330 | const ratio = Math.exp(logProb - (r.logProbs[i] as number)); | |
| 331 | const low = 1 - hyper.clip, high = 1 + hyper.clip; | |
| 332 | const unclipped = ratio * advantage; | |
| 333 | const clippedTerm = Math.min(Math.max(ratio, low), high) * advantage; | |
| 334 | const useClipped = clippedTerm < unclipped; | |
| 335 | if (ratio < low || ratio > high) sums.clipped += 1; | |
| 336 | sums.policyLoss += -Math.min(unclipped, clippedTerm); | |
| 337 | sums.entropy += entropy; | |
| 338 | sums.samples += 1; | |
| 339 | ||
| 340 | // d(-min(rA, clip(r)A)) / d logProb, which is -A·r unless clipping bit. | |
| 341 | const dLogProb = useClipped && (ratio < low || ratio > high) ? 0 : -advantage * ratio; | |
| 342 | for (let h = 0; h < heads.length; h++) { | |
| 343 | const size_ = heads[h] as number; | |
| 344 | const at = headOffset(p.shape, h); | |
| 345 | softmax(a.logits, at, size_, probs); | |
| 346 | let headEntropy = 0; | |
| 347 | for (let q = 0; q < size_; q++) headEntropy -= (probs[q] as number) * Math.log(Math.max(1e-9, probs[q] as number)); | |
| 348 | const pick = r.actions[i * heads.length + h] as number; | |
| 349 | for (let q = 0; q < size_; q++) { | |
| 350 | const pq = probs[q] as number; | |
| 351 | const dLog = (q === pick ? 1 : 0) - pq; | |
| 352 | const dEnt = hyper.entropy * pq * (Math.log(Math.max(1e-9, pq)) + headEntropy); | |
| 353 | dLogits[at + q] = (dLogits[at + q] as number) + dLogProb * dLog + dEnt; | |
| 354 | } | |
| 355 | } | |
| 356 | ||
| 357 | const target = r.returns[i] as number; | |
| 358 | const diff = a.value - target; | |
| 359 | sums.valueLoss += 0.5 * diff * diff; | |
| 360 | backward(p, r.obs, i * p.shape.obs, a, dLogits, hyper.valueCoef * diff, g, scratch); | |
| 361 | } | |
| 362 | return sums; | |
| 363 | } | |
| 364 | ||
| 365 | /** Fisher-Yates over the sample indices, in place: one epoch's minibatch plan. */ | |
| 366 | export function shuffle(order: Int32Array, random: () => number): Int32Array { | |
| 367 | for (let i = order.length - 1; i > 0; i--) { | |
| 368 | const j = Math.floor(random() * (i + 1)); | |
| 369 | const tmp = order[i] as number; | |
| 370 | order[i] = order[j] as number; | |
| 371 | order[j] = tmp; | |
| 372 | } | |
| 373 | return order; | |
| 374 | } | |
| 375 | ||
| 376 | export const statsOf = (sums: Sums, gradNorm: number): UpdateStats => { | |
| 377 | const count = Math.max(1, sums.samples); | |
| 378 | return { | |
| 379 | policyLoss: sums.policyLoss / count, | |
| 380 | valueLoss: sums.valueLoss / count, | |
| 381 | entropy: sums.entropy / count, | |
| 382 | clipFraction: sums.clipped / count, | |
| 383 | gradNorm, | |
| 384 | }; | |
| 385 | }; | |
| 386 | ||
| 387 | /** | |
| 388 | * The PPO update itself: a few epochs of minibatches over one rollout, on one | |
| 389 | * thread. `src/rl/pool.ts` runs the same arithmetic across workers; this stays | |
| 390 | * as the reference the parallel path is tested against. | |
| 391 | */ | |
| 218 | 392 | export function update(p: Policy, r: Rollout, adam: Adam, random: () => number): UpdateStats { |
| 219 | 393 | const hyper = adam.hyper; |
| 220 | 394 | const n = r.envs * r.steps; |
| 221 | const heads = p.shape.heads; | |
| 222 | const l = logitCount(p.shape); | |
| 223 | 395 | const g = zeroGrads(p.shape); |
| 224 | const a = newActivations(p.shape); | |
| 225 | const dLogits = new Float32Array(l); | |
| 226 | const probs = new Float32Array(Math.max(...heads)); | |
| 227 | const scratch = { dh1: new Float32Array(p.shape.hidden), dh2: new Float32Array(p.shape.hidden) }; | |
| 396 | const w = createWorkspace(p.shape); | |
| 228 | 397 | const order = new Int32Array(n); |
| 229 | 398 | for (let i = 0; i < n; i++) order[i] = i; |
| 230 | ||
| 231 | let policyLoss = 0, valueLoss = 0, entropySum = 0, clipped = 0, gradNorm = 0, samples = 0; | |
| 399 | const sums = noSums(); | |
| 400 | let gradNorm = 0; | |
| 232 | 401 | |
| 233 | 402 | for (let epoch = 0; epoch < hyper.epochs; epoch++) { |
| 234 | for (let i = n - 1; i > 0; i--) { | |
| 235 | const j = Math.floor(random() * (i + 1)); | |
| 236 | const tmp = order[i] as number; | |
| 237 | order[i] = order[j] as number; | |
| 238 | order[j] = tmp; | |
| 239 | } | |
| 403 | shuffle(order, random); | |
| 240 | 404 | for (let start = 0; start < n; start += hyper.minibatch) { |
| 241 | 405 | const end = Math.min(n, start + hyper.minibatch); |
| 242 | 406 | const size = end - start; |
| 243 | 407 | if (size < 2) continue; |
| 244 | // Advantages are normalised inside the minibatch: the scale of a reward | |
| 245 | // then stops mattering, only its rank against its neighbours. | |
| 246 | let mean = 0; | |
| 247 | for (let k = start; k < end; k++) mean += r.advantages[order[k] as number] as number; | |
| 248 | mean /= size; | |
| 249 | let variance = 0; | |
| 250 | for (let k = start; k < end; k++) variance += ((r.advantages[order[k] as number] as number) - mean) ** 2; | |
| 251 | const std = Math.sqrt(variance / size) + 1e-8; | |
| 252 | ||
| 408 | const { mean, std } = normalise(r, order, start, end); | |
| 253 | 409 | clearGrads(g); |
| 254 | for (let k = start; k < end; k++) { | |
| 255 | const i = order[k] as number; | |
| 256 | forward(p, r.obs, a, i * p.shape.obs); | |
| 257 | dLogits.fill(0); | |
| 258 | ||
| 259 | let logProb = 0; | |
| 260 | let entropy = 0; | |
| 261 | for (let h = 0; h < heads.length; h++) { | |
| 262 | const size_ = heads[h] as number; | |
| 263 | const at = headOffset(p.shape, h); | |
| 264 | softmax(a.logits, at, size_, probs); | |
| 265 | const pick = r.actions[i * heads.length + h] as number; | |
| 266 | logProb += Math.log(Math.max(1e-9, probs[pick] as number)); | |
| 267 | for (let q = 0; q < size_; q++) entropy -= (probs[q] as number) * Math.log(Math.max(1e-9, probs[q] as number)); | |
| 268 | } | |
| 269 | ||
| 270 | const advantage = ((r.advantages[i] as number) - mean) / std; | |
| 271 | const ratio = Math.exp(logProb - (r.logProbs[i] as number)); | |
| 272 | const low = 1 - hyper.clip, high = 1 + hyper.clip; | |
| 273 | const unclipped = ratio * advantage; | |
| 274 | const clippedTerm = Math.min(Math.max(ratio, low), high) * advantage; | |
| 275 | const useClipped = clippedTerm < unclipped; | |
| 276 | if (ratio < low || ratio > high) clipped += 1; | |
| 277 | policyLoss += -Math.min(unclipped, clippedTerm); | |
| 278 | entropySum += entropy; | |
| 279 | samples += 1; | |
| 280 | ||
| 281 | // d(-min(rA, clip(r)A)) / d logProb, which is -A·r unless clipping bit. | |
| 282 | const dLogProb = useClipped && (ratio < low || ratio > high) ? 0 : -advantage * ratio; | |
| 283 | for (let h = 0; h < heads.length; h++) { | |
| 284 | const size_ = heads[h] as number; | |
| 285 | const at = headOffset(p.shape, h); | |
| 286 | softmax(a.logits, at, size_, probs); | |
| 287 | let headEntropy = 0; | |
| 288 | for (let q = 0; q < size_; q++) headEntropy -= (probs[q] as number) * Math.log(Math.max(1e-9, probs[q] as number)); | |
| 289 | const pick = r.actions[i * heads.length + h] as number; | |
| 290 | for (let q = 0; q < size_; q++) { | |
| 291 | const pq = probs[q] as number; | |
| 292 | const dLog = (q === pick ? 1 : 0) - pq; | |
| 293 | const dEnt = hyper.entropy * pq * (Math.log(Math.max(1e-9, pq)) + headEntropy); | |
| 294 | dLogits[at + q] = (dLogits[at + q] as number) + dLogProb * dLog + dEnt; | |
| 295 | } | |
| 296 | } | |
| 297 | ||
| 298 | const target = r.returns[i] as number; | |
| 299 | const diff = a.value - target; | |
| 300 | valueLoss += 0.5 * diff * diff; | |
| 301 | backward(p, r.obs, i * p.shape.obs, a, dLogits, hyper.valueCoef * diff, g, scratch); | |
| 302 | } | |
| 410 | accumulate(p, r, g, order, start, end, mean, std, hyper, w, sums); | |
| 303 | 411 | gradNorm = applyGrads(p, g, adam, 1 / size); |
| 304 | 412 | } |
| 305 | 413 | } |
| 306 | const count = Math.max(1, samples); | |
| 307 | return { | |
| 308 | policyLoss: policyLoss / count, | |
| 309 | valueLoss: valueLoss / count, | |
| 310 | entropy: entropySum / count, | |
| 311 | clipFraction: clipped / count, | |
| 312 | gradNorm, | |
| 313 | }; | |
| 414 | return statsOf(sums, gradNorm); | |
| 314 | 415 | } |
src/rl/shard.ts added +230
| @@ -0,0 +1,230 @@ | ||
| 1 | /** | |
| 2 | * A shard: one worker's share of a training step. | |
| 3 | * | |
| 4 | * It owns a contiguous range of environments and writes their rows of the one | |
| 5 | * shared rollout, then computes the gradient of one slice of one minibatch into | |
| 6 | * its own shared buffer. Nothing here knows whether it is running on the main | |
| 7 | * thread or in a worker — `src/rl/pool.ts` decides that — and nothing is copied | |
| 8 | * between the two, because the rollout, the weights and the gradients are all | |
| 9 | * views onto SharedArrayBuffers. | |
| 10 | * | |
| 11 | * Every random draw is keyed to the environment it belongs to, not to the | |
| 12 | * order the environments are visited in, so sixteen shards produce exactly the | |
| 13 | * same experience as one shard does. `pool.test.ts` holds that to it. | |
| 14 | */ | |
| 15 | ||
| 16 | import { decode, HEADS } from "../client/action.ts"; | |
| 17 | import { rng } from "../client/math.ts"; | |
| 18 | import { OBS_SIZE } from "../client/observe.ts"; | |
| 19 | import { | |
| 20 | forward, newActivations, policyOver, policyBytes, sample, | |
| 21 | type Activations, type Policy, type Shape, | |
| 22 | } from "../client/policy.ts"; | |
| 23 | import { | |
| 24 | createArena, observeFoe, observeLearner, policyFoe, reset, scriptedFoe, step, stillFoe, type Arena, | |
| 25 | } from "./arena.ts"; | |
| 26 | import { | |
| 27 | accumulate, clearGrads, createWorkspace, noSums, rolloutOver, | |
| 28 | type Hyper, type Rollout, type Workspace, | |
| 29 | } from "./ppo.ts"; | |
| 30 | ||
| 31 | /** How many frozen opponents the league holds. */ | |
| 32 | export const LEAGUE = 4; | |
| 33 | ||
| 34 | /** Where the run is: what the arenas are set to, and who they practise against. */ | |
| 35 | export type Curriculum = { | |
| 36 | maxRange: number; | |
| 37 | maxSkill: number; | |
| 38 | /** Fraction of environments facing a target that does not shoot back... */ | |
| 39 | still: number; | |
| 40 | /** ...and facing the scripted bots. The rest face the league. */ | |
| 41 | scripted: number; | |
| 42 | /** How many league slots have been filled, so an empty one is never drawn. */ | |
| 43 | filled: number; | |
| 44 | }; | |
| 45 | ||
| 46 | /** The buffers a shard is handed. All shared; all written by exactly one owner. */ | |
| 47 | export type ShardSpec = { | |
| 48 | shape: Shape; | |
| 49 | envs: number; | |
| 50 | steps: number; | |
| 51 | from: number; | |
| 52 | to: number; | |
| 53 | seed: number; | |
| 54 | hyper: Hyper; | |
| 55 | weights: SharedArrayBuffer; | |
| 56 | league: SharedArrayBuffer; | |
| 57 | rollout: SharedArrayBuffer; | |
| 58 | grads: SharedArrayBuffer; | |
| 59 | order: SharedArrayBuffer; | |
| 60 | tally: SharedArrayBuffer; | |
| 61 | /** Which shard this is, of how many: its slice of every minibatch. */ | |
| 62 | index: number; | |
| 63 | shards: number; | |
| 64 | }; | |
| 65 | ||
| 66 | /** Where shard `k` of `n` starts in a run of `total`: as even as it divides. */ | |
| 67 | export const split = (total: number, k: number, n: number): number => Math.round((total * k) / n); | |
| 68 | ||
| 69 | /** The counters a shard adds to, one row each, summed by the main thread. */ | |
| 70 | export const TALLY = { | |
| 71 | returns: 0, episodes: 1, wins: 2, losses: 3, | |
| 72 | aim: 4, aimN: 5, fire: 6, fireN: 7, | |
| 73 | policyLoss: 8, valueLoss: 9, entropy: 10, clipped: 11, samples: 12, | |
| 74 | } as const; | |
| 75 | /** Thirteen counters, padded to two cache lines so two shards never share one. */ | |
| 76 | export const TALLY_SIZE = 16; | |
| 77 | ||
| 78 | export type Shard = { | |
| 79 | spec: ShardSpec; | |
| 80 | policy: Policy; | |
| 81 | league: Policy[]; | |
| 82 | rollout: Rollout; | |
| 83 | grads: Policy; | |
| 84 | order: Int32Array; | |
| 85 | tally: Float64Array; | |
| 86 | arenas: Arena[]; | |
| 87 | randoms: (() => number)[]; | |
| 88 | act: Activations; | |
| 89 | actions: Int32Array; | |
| 90 | workspace: Workspace; | |
| 91 | tail: Float32Array; | |
| 92 | episodeReturn: Float64Array; | |
| 93 | foeObs: Float32Array; | |
| 94 | foeAct: Activations; | |
| 95 | foeActions: Int32Array; | |
| 96 | }; | |
| 97 | ||
| 98 | /** The seeds an environment gets, wherever it is run. */ | |
| 99 | const arenaSeed = (seed: number, e: number): number => (seed + e * 104729) | 0; | |
| 100 | const streamSeed = (seed: number, e: number): number => (seed ^ (e * 2654435761)) | 0; | |
| 101 | ||
| 102 | export function createShard(spec: ShardSpec): Shard { | |
| 103 | const { shape, envs, steps, from, to, seed } = spec; | |
| 104 | const shard: Shard = { | |
| 105 | spec, | |
| 106 | policy: policyOver(shape, spec.weights), | |
| 107 | league: Array.from({ length: LEAGUE }, (_, k) => policyOver(shape, spec.league, k * policyBytes(shape))), | |
| 108 | rollout: rolloutOver(shape, envs, steps, spec.rollout), | |
| 109 | grads: policyOver(shape, spec.grads), | |
| 110 | order: new Int32Array(spec.order), | |
| 111 | tally: new Float64Array(spec.tally, spec.index * TALLY_SIZE * 8, TALLY_SIZE), | |
| 112 | arenas: [], | |
| 113 | randoms: [], | |
| 114 | act: newActivations(shape), | |
| 115 | actions: new Int32Array(HEADS.length), | |
| 116 | workspace: createWorkspace(shape), | |
| 117 | tail: new Float32Array(OBS_SIZE), | |
| 118 | episodeReturn: new Float64Array(to - from), | |
| 119 | foeObs: new Float32Array(OBS_SIZE), | |
| 120 | foeAct: newActivations(shape), | |
| 121 | foeActions: new Int32Array(HEADS.length), | |
| 122 | }; | |
| 123 | for (let e = from; e < to; e++) { | |
| 124 | shard.randoms.push(rng(streamSeed(seed, e))); | |
| 125 | shard.arenas.push(createArena(arenaSeed(seed, e), stillFoe)); | |
| 126 | } | |
| 127 | return shard; | |
| 128 | } | |
| 129 | ||
| 130 | /** | |
| 131 | * Who environment `e` practises against this episode. The mix moves with the | |
| 132 | * run: mostly targets that do not shoot back at the start, mostly the scripted | |
| 133 | * bots and the league by the end. Handed a good opponent from the first step | |
| 134 | * the policy learns to hide, and never comes back from it. | |
| 135 | */ | |
| 136 | export function opponentFor(shard: Shard, e: number, c: Curriculum): Arena["opponent"] { | |
| 137 | const u = (e % 10) / 10; | |
| 138 | if (u < c.still) return stillFoe; | |
| 139 | if (u < c.still + c.scripted || c.filled === 0) return scriptedFoe; | |
| 140 | const slot = shard.randoms[e - shard.spec.from] as () => number; | |
| 141 | const pick = Math.min(c.filled - 1, Math.floor(slot() * c.filled)); | |
| 142 | return policyFoe((arena) => { | |
| 143 | const foe = shard.league[pick] as Policy; | |
| 144 | observeFoe(arena, shard.foeObs); | |
| 145 | forward(foe, shard.foeObs, shard.foeAct); | |
| 146 | sample(shard.spec.shape, shard.foeAct.logits, slot, shard.foeActions); | |
| 147 | return decode(shard.foeActions); | |
| 148 | }); | |
| 149 | } | |
| 150 | ||
| 151 | /** Point every environment at the curriculum it is meant to be on. */ | |
| 152 | export function retune(shard: Shard, c: Curriculum): void { | |
| 153 | for (const arena of shard.arenas) { | |
| 154 | arena.maxRange = c.maxRange; | |
| 155 | arena.maxSkill = c.maxSkill; | |
| 156 | } | |
| 157 | } | |
| 158 | ||
| 159 | /** | |
| 160 | * Fill this shard's rows of the rollout: `steps` decisions in each of its | |
| 161 | * environments, with the live policy, keeping the episode counters as it goes. | |
| 162 | */ | |
| 163 | export function collect(shard: Shard, c: Curriculum): void { | |
| 164 | const { spec, rollout, policy, act, actions, tally } = shard; | |
| 165 | const { envs, from, to } = spec; | |
| 166 | retune(shard, c); | |
| 167 | for (let t = 0; t < spec.steps; t++) { | |
| 168 | for (let e = from; e < to; e++) { | |
| 169 | const local = e - from; | |
| 170 | const arena = shard.arenas[local] as Arena; | |
| 171 | const random = shard.randoms[local] as () => number; | |
| 172 | const i = t * envs + e; | |
| 173 | observeLearner(arena, rollout.obs, i * OBS_SIZE); | |
| 174 | forward(policy, rollout.obs, act, i * OBS_SIZE); | |
| 175 | rollout.values[i] = act.value; | |
| 176 | rollout.logProbs[i] = sample(spec.shape, act.logits, random, actions); | |
| 177 | for (let h = 0; h < HEADS.length; h++) rollout.actions[i * HEADS.length + h] = actions[h] as number; | |
| 178 | if ((rollout.obs[i * OBS_SIZE + 2] as number) > 0.5) { | |
| 179 | tally[TALLY.aim] = (tally[TALLY.aim] as number) + (rollout.obs[i * OBS_SIZE + 4] as number); | |
| 180 | tally[TALLY.aimN] = (tally[TALLY.aimN] as number) + 1; | |
| 181 | tally[TALLY.fire] = (tally[TALLY.fire] as number) + (actions[4] === 1 ? 1 : 0); | |
| 182 | tally[TALLY.fireN] = (tally[TALLY.fireN] as number) + 1; | |
| 183 | } | |
| 184 | const result = step(arena, decode(actions)); | |
| 185 | rollout.rewards[i] = result.reward; | |
| 186 | rollout.dones[i] = result.done ? 1 : 0; | |
| 187 | shard.episodeReturn[local] = (shard.episodeReturn[local] as number) + result.reward; | |
| 188 | if (result.done) { | |
| 189 | tally[TALLY.returns] = (tally[TALLY.returns] as number) + (shard.episodeReturn[local] as number); | |
| 190 | tally[TALLY.episodes] = (tally[TALLY.episodes] as number) + 1; | |
| 191 | if (result.won) tally[TALLY.wins] = (tally[TALLY.wins] as number) + 1; | |
| 192 | if (result.lost) tally[TALLY.losses] = (tally[TALLY.losses] as number) + 1; | |
| 193 | shard.episodeReturn[local] = 0; | |
| 194 | reset(arena); | |
| 195 | arena.opponent = opponentFor(shard, e, c); | |
| 196 | } | |
| 197 | } | |
| 198 | } | |
| 199 | // What the policy thinks the unfinished episodes are still worth. | |
| 200 | for (let e = from; e < to; e++) { | |
| 201 | observeLearner(shard.arenas[e - from] as Arena, shard.tail); | |
| 202 | forward(policy, shard.tail, act); | |
| 203 | rollout.lastValues[e] = act.value; | |
| 204 | } | |
| 205 | } | |
| 206 | ||
| 207 | /** | |
| 208 | * The gradient of this shard's share of `order[start..end)` — the minibatch the | |
| 209 | * main thread laid out — into its own buffer, with the losses tallied. Every | |
| 210 | * shard is given the whole minibatch and takes the same fraction of it that it | |
| 211 | * takes of the environments, so the slices tile it exactly once. | |
| 212 | */ | |
| 213 | export function gradients(shard: Shard, start: number, end: number, mean: number, std: number): void { | |
| 214 | const { index, shards } = shard.spec; | |
| 215 | const size = end - start; | |
| 216 | const mine = start + split(size, index, shards); | |
| 217 | const until = start + split(size, index + 1, shards); | |
| 218 | const sums = noSums(); | |
| 219 | clearGrads(shard.grads); | |
| 220 | accumulate( | |
| 221 | shard.policy, shard.rollout, shard.grads, shard.order, mine, until, mean, std, | |
| 222 | shard.spec.hyper, shard.workspace, sums, | |
| 223 | ); | |
| 224 | const t = shard.tally; | |
| 225 | t[TALLY.policyLoss] = (t[TALLY.policyLoss] as number) + sums.policyLoss; | |
| 226 | t[TALLY.valueLoss] = (t[TALLY.valueLoss] as number) + sums.valueLoss; | |
| 227 | t[TALLY.entropy] = (t[TALLY.entropy] as number) + sums.entropy; | |
| 228 | t[TALLY.clipped] = (t[TALLY.clipped] as number) + sums.clipped; | |
| 229 | t[TALLY.samples] = (t[TALLY.samples] as number) + sums.samples; | |
| 230 | } |
src/rl/worker.ts added +29
| @@ -0,0 +1,29 @@ | ||
| 1 | /** | |
| 2 | * One training worker. It builds its shard once and then does as it is told: | |
| 3 | * collect, or differentiate a slice. It owns no data of its own — every buffer | |
| 4 | * it touches was handed to it by `pool.ts` and is shared with the main thread — | |
| 5 | * so the only thing on the wire is the word "done". | |
| 6 | */ | |
| 7 | ||
| 8 | import { collect, createShard, gradients, type Shard, type ShardSpec } from "./shard.ts"; | |
| 9 | import type { Job } from "./pool.ts"; | |
| 10 | ||
| 11 | declare const self: Worker; | |
| 12 | ||
| 13 | let shard: Shard | null = null; | |
| 14 | ||
| 15 | self.onmessage = (event: MessageEvent): void => { | |
| 16 | const message = event.data as Job | { kind: "init"; spec: ShardSpec }; | |
| 17 | try { | |
| 18 | if (message.kind === "init") { | |
| 19 | shard = createShard(message.spec); | |
| 20 | return; | |
| 21 | } | |
| 22 | if (shard === null) throw new Error("worker was asked to work before it was given a shard"); | |
| 23 | if (message.kind === "collect") collect(shard, message.curriculum); | |
| 24 | else gradients(shard, message.start, message.end, message.mean, message.std); | |
| 25 | postMessage({ kind: "done" }); | |
| 26 | } catch (error) { | |
| 27 | postMessage({ kind: "error", error: error instanceof Error ? error.message : String(error) }); | |
| 28 | } | |
| 29 | }; |
src/tabs.test.ts added +23
| @@ -0,0 +1,23 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { readFileSync } from "node:fs"; | |
| 3 | import { join } from "node:path"; | |
| 4 | import { nextTab, TABS, type Tab } from "./tabs.ts"; | |
| 5 | ||
| 6 | test("pressing tab reaches every tab and comes back round", () => { | |
| 7 | let tab: Tab = "standings"; | |
| 8 | const seen: Tab[] = [tab]; | |
| 9 | for (let i = 0; i < TABS.length - 1; i++) { | |
| 10 | tab = nextTab(tab); | |
| 11 | seen.push(tab); | |
| 12 | } | |
| 13 | expect(new Set(seen).size).toBe(TABS.length); | |
| 14 | for (const t of TABS) expect(seen).toContain(t); | |
| 15 | expect(nextTab(tab)).toBe("standings"); | |
| 16 | }); | |
| 17 | ||
| 18 | test("the terminal draws every tab it lets you reach", () => { | |
| 19 | // The bar and the body are written out tab by tab, so a tab the keyboard can | |
| 20 | // reach and the body cannot render would be a blank screen. | |
| 21 | const tty = readFileSync(join(import.meta.dir, "tty.ts"), "utf8"); | |
| 22 | for (const t of TABS) expect(tty).toContain(`"${t}"`); | |
| 23 | }); |
src/tabs.ts added +16
| @@ -0,0 +1,16 @@ | ||
| 1 | /** | |
| 2 | * The terminal's tabs, and the order the tab key walks them. | |
| 3 | * | |
| 4 | * Its own module because `src/tty.ts` opens the database the moment it is | |
| 5 | * imported, which a test cannot do, and because the tab key missing a tab is | |
| 6 | * exactly the sort of thing that hides a whole page from the keyboard: the | |
| 7 | * policy card was drawn and named in the bar for a week without being | |
| 8 | * reachable. | |
| 9 | */ | |
| 10 | ||
| 11 | export type Tab = "standings" | "matches" | "policy"; | |
| 12 | ||
| 13 | export const TABS: readonly Tab[] = ["standings", "matches", "policy"]; | |
| 14 | ||
| 15 | /** The next tab round, wrapping: what one press of tab does. */ | |
| 16 | export const nextTab = (tab: Tab): Tab => TABS[(TABS.indexOf(tab) + 1) % TABS.length] as Tab; |
src/tty.ts +8 −10
| @@ -1,13 +1,15 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * cs, in a terminal. The forge runs this on a pty when someone `ssh`es to the |
| 3 | 3 | * app, with FORGE_APP_DATA pointing at the same database the web page uses: |
| 4 | * the same leaderboard, the same matches. j/k move, tab switches between the | |
| 5 | * standings and the match list, x removes the selected match, r refreshes, | |
| 6 | * q leaves. Plain ANSI, no dependencies. | |
| 4 | * the same leaderboard, the same matches. j/k move, tab walks the standings, | |
| 5 | * the match list and the policy card (`src/tabs.ts` keeps that order), x | |
| 6 | * removes the selected match, r refreshes, q leaves. Plain ANSI, no | |
| 7 | * dependencies. | |
| 7 | 8 | */ |
| 8 | 9 | |
| 9 | 10 | import { headshotShare, matchCount, openDatabase, ratio, recentMatches, standings, deleteMatch } from "./db.ts"; |
| 10 | 11 | import { CARD } from "./client/weights.ts"; |
| 12 | import { nextTab, TABS, type Tab } from "./tabs.ts"; | |
| 11 | 13 | |
| 12 | 14 | const db = openDatabase(); |
| 13 | 15 | const where = process.env.FORGE_APP_ENV === "preview" |
| @@ -19,7 +21,6 @@ const esc = "\x1b["; | ||
| 19 | 21 | const out = (s: string): void => { process.stdout.write(s); }; |
| 20 | 22 | const pad = (s: string, n: number): string => (s.length > n ? s.slice(0, n - 1) + "…" : s.padEnd(n)); |
| 21 | 23 | |
| 22 | type Tab = "standings" | "matches" | "policy"; | |
| 23 | 24 | let tab: Tab = "standings"; |
| 24 | 25 | let cursor = 0; |
| 25 | 26 | let note = ""; |
| @@ -33,11 +34,7 @@ function draw(): void { | ||
| 33 | 34 | const total = matchCount(db); |
| 34 | 35 | lines.push(`${esc}1mcs${esc}0m ${esc}2m${where} · ${total} match${total === 1 ? "" : "es"}${esc}0m`); |
| 35 | 36 | lines.push(""); |
| 36 | lines.push( | |
| 37 | `${tab === "standings" ? `${esc}7m standings ${esc}0m` : `${esc}2m standings ${esc}0m`}` + | |
| 38 | `${tab === "matches" ? `${esc}7m matches ${esc}0m` : `${esc}2m matches ${esc}0m`}` + | |
| 39 | `${tab === "policy" ? `${esc}7m policy ${esc}0m` : `${esc}2m policy ${esc}0m`}`, | |
| 40 | ); | |
| 37 | lines.push(TABS.map((t) => `${esc}${t === tab ? "7" : "2"}m ${t} ${esc}0m`).join("")); | |
| 41 | 38 | lines.push(""); |
| 42 | 39 | if (tab === "policy") { |
| 43 | 40 | const line = (k: string, v: string): string => ` ${esc}2m${pad(k, 22)}${esc}0m${v}`; |
| @@ -46,6 +43,7 @@ function draw(): void { | ||
| 46 | 43 | lines.push(line("trained", CARD.trainedAt)); |
| 47 | 44 | lines.push(line("simulation ticks", `${CARD.envSteps.toLocaleString("en")} in ${CARD.minutes} min`)); |
| 48 | 45 | lines.push(line("throughput", `${CARD.stepsPerSecond.toLocaleString("en")} ticks a second`)); |
| 46 | lines.push(line("parallel shards", `${CARD.workers} worker${CARD.workers === 1 ? "" : "s"}`)); | |
| 49 | 47 | lines.push(line("PPO updates", String(CARD.updates))); |
| 50 | 48 | lines.push(line("duels evaluated", String(CARD.duels))); |
| 51 | 49 | lines.push(line("won by the policy", `${CARD.winRate}%`)); |
| @@ -92,7 +90,7 @@ process.stdin.on("data", (key: string) => { | ||
| 92 | 90 | if (key === "j" || key === "\x1b[B") cursor += 1; |
| 93 | 91 | else if (key === "k" || key === "\x1b[A") cursor = Math.max(0, cursor - 1); |
| 94 | 92 | else if (key === "\t") { |
| 95 | tab = tab === "standings" ? "matches" : "standings"; | |
| 93 | tab = nextTab(tab); | |
| 96 | 94 | cursor = 0; |
| 97 | 95 | } else if (key === "x" && tab === "matches") { |
| 98 | 96 | const match = recentMatches(db, 20)[cursor]; |
static/app.85c83ea9c3.js added +58
Diff not shown — a line 55,766 characters wide (limit 2,000). +58
View filestatic/app.e8ca965610.js deleted −58
Diff not shown — a line 55,651 characters wide (limit 2,000). −58
View file
new above
offline