Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
/metrics.prom counts /follow frames received and dropped, and the relay logs summaryLine() once a minute #272
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/247-metrics-prom-counts-follow-frames-receiv → main 1 commit fork 1 thread · 1 unresolved
3 files
changed +22
−2
Software/so101/bench/fake-serve.ts +1 −1
| @@ -65,6 +65,6 @@ try { | ||
| 65 | 65 | } catch (err) { |
| 66 | 66 | console.error(`${(err as Error).message}; cameras stay MJPEG`); |
| 67 | 67 | } |
| 68 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video }); | |
| 68 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video, summaryEveryMs: 60_000 }); | |
| 69 | 69 | const { mode, host: armHost } = relay.link(); |
| 70 | 70 | console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`); |
Software/so101/src/frame-limit.test.ts +13
| @@ -160,6 +160,19 @@ describe("the relay's /follow", () => { | ||
| 160 | 160 | return { ws, got, ended, stop: () => (clearInterval(pump), wss.clients.forEach((c) => c.terminate())) }; |
| 161 | 161 | }; |
| 162 | 162 | |
| 163 | test("a flood of /follow frames shows received and dropped on /metrics.prom", async () => { | |
| 164 | const arm = await fakeArm(); | |
| 165 | try { | |
| 166 | for (let i = 0; i < BURST + 40; i++) arm.ws.send(JSON.stringify({ 1: 2048 })); | |
| 167 | await sleep(100); | |
| 168 | const text = await (await fetch(`http://127.0.0.1:${relay!.port}/metrics.prom`)).text(); | |
| 169 | expect(Number(/^follow_frames_received_per_second\{\} (\S+)$/m.exec(text)?.[1])).toBeGreaterThan(0); | |
| 170 | expect(Number(/^follow_frames_dropped_per_second\{\} (\S+)$/m.exec(text)?.[1])).toBeGreaterThan(0); | |
| 171 | } finally { | |
| 172 | arm.stop(); | |
| 173 | } | |
| 174 | }); | |
| 175 | ||
| 163 | 176 | test("an oversized frame is not passed on: the stream closes with 1009 and the arm holds", async () => { |
| 164 | 177 | const arm = await fakeArm(); |
| 165 | 178 | try { |
Software/so101/src/relay.ts +8 −1
| @@ -128,6 +128,8 @@ export interface RelayOptions { | ||
| 128 | 128 | leaders?: LeaderCalibrations; |
| 129 | 129 | /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */ |
| 130 | 130 | statusEveryMs?: number; |
| 131 | /** How often the relay writes its metrics summary line to stdout; off (tests) when unset. */ | |
| 132 | summaryEveryMs?: number; | |
| 131 | 133 | /** Where HTTP_PROXY and NO_PROXY are read from; process.env by default. */ |
| 132 | 134 | env?: Record<string, string | undefined>; |
| 133 | 135 | } |
| @@ -710,6 +712,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 710 | 712 | let overLimit = false; |
| 711 | 713 | browser.on("message", (raw) => { |
| 712 | 714 | if (overLimit) return; |
| 715 | metrics.count("follow_frames_received"); | |
| 713 | 716 | if (frameBytes(raw) > MAX_FRAME_BYTES) { |
| 714 | 717 | overLimit = true; |
| 715 | 718 | machine.note("clamp", "not passed on", { why: "frame too large", bytes: frameBytes(raw) }); |
| @@ -718,10 +721,11 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 718 | 721 | const pace = rate.take(); |
| 719 | 722 | if (pace === "flood") { |
| 720 | 723 | overLimit = true; |
| 724 | metrics.count("follow_frames_dropped"); | |
| 721 | 725 | machine.note("clamp", "not passed on", { why: "too many frames" }); |
| 722 | 726 | return browser.close(FLOOD_CLOSE, "too many frames: the stream was closed"); |
| 723 | 727 | } |
| 724 | if (pace === "drop") return; | |
| 728 | if (pace === "drop") return void metrics.count("follow_frames_dropped"); | |
| 725 | 729 | if (driver !== null) { |
| 726 | 730 | // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on. |
| 727 | 731 | if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void machine.note("clamp", "not passed on", { why: "the turn ended" }); |
| @@ -862,6 +866,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 862 | 866 | timer?.unref?.(); |
| 863 | 867 | const pusher = lobby ? setInterval(() => void push(), statusEveryMs) : undefined; |
| 864 | 868 | pusher?.unref?.(); |
| 869 | const summary = options.summaryEveryMs ? setInterval(() => console.log(metrics.summaryLine()), options.summaryEveryMs) : undefined; | |
| 870 | summary?.unref?.(); | |
| 865 | 871 | await new Promise<void>((resolve) => http.listen(options.port ?? 8102, options.host ?? "127.0.0.1", resolve)); |
| 866 | 872 | const address = http.address(); |
| 867 | 873 | return { |
| @@ -871,6 +877,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 871 | 877 | new Promise<void>((resolve) => { |
| 872 | 878 | clearInterval(timer); |
| 873 | 879 | clearInterval(pusher); |
| 880 | clearInterval(summary); | |
| 874 | 881 | settingsChanged?.(); |
| 875 | 882 | turnEnded?.(); |
| 876 | 883 | arrived?.(); |
new above
offline
Oversized frames end the stream here without bumping follow_frames_dropped; count them as dropped so the metric matches what is discarded.