Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

/metrics.prom counts /follow frames received and dropped, and the relay logs summaryLine() once a minute #272

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/247-metrics-prom-counts-follow-frames-receiv → main 1 commit fork 1 thread · 1 unresolved

  • Merged. smith/247-metrics-prom-counts-follow-frames-receiv was merged into main as 185e3c3.

Review

Sign in to review this change.
3 files changed +22 −2
Software/so101/bench/fake-serve.ts +1 −1
@@ -65,6 +65,6 @@ try {
6565 } catch (err) {
6666 console.error(`${(err as Error).message}; cameras stay MJPEG`);
6767 }
68 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video });
68 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video, summaryEveryMs: 60_000 });
6969 const { mode, host: armHost } = relay.link();
7070 console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`);
Software/so101/src/frame-limit.test.ts +13
@@ -160,6 +160,19 @@ describe("the relay's /follow", () => {
160160 return { ws, got, ended, stop: () => (clearInterval(pump), wss.clients.forEach((c) => c.terminate())) };
161161 };
162162
163 test("a flood of /follow frames shows received and dropped on /metrics.prom", async () => {
164 const arm = await fakeArm();
165 try {
166 for (let i = 0; i < BURST + 40; i++) arm.ws.send(JSON.stringify({ 1: 2048 }));
167 await sleep(100);
168 const text = await (await fetch(`http://127.0.0.1:${relay!.port}/metrics.prom`)).text();
169 expect(Number(/^follow_frames_received_per_second\{\} (\S+)$/m.exec(text)?.[1])).toBeGreaterThan(0);
170 expect(Number(/^follow_frames_dropped_per_second\{\} (\S+)$/m.exec(text)?.[1])).toBeGreaterThan(0);
171 } finally {
172 arm.stop();
173 }
174 });
175
163176 test("an oversized frame is not passed on: the stream closes with 1009 and the arm holds", async () => {
164177 const arm = await fakeArm();
165178 try {
Software/so101/src/relay.ts +8 −1
@@ -128,6 +128,8 @@ export interface RelayOptions {
128128 leaders?: LeaderCalibrations;
129129 /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */
130130 statusEveryMs?: number;
131 /** How often the relay writes its metrics summary line to stdout; off (tests) when unset. */
132 summaryEveryMs?: number;
131133 /** Where HTTP_PROXY and NO_PROXY are read from; process.env by default. */
132134 env?: Record<string, string | undefined>;
133135 }
@@ -710,6 +712,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
710712 let overLimit = false;
711713 browser.on("message", (raw) => {
712714 if (overLimit) return;
715 metrics.count("follow_frames_received");
713716 if (frameBytes(raw) > MAX_FRAME_BYTES) {

Oversized frames end the stream here without bumping follow_frames_dropped; count them as dropped so the metric matches what is discarded.

714717 overLimit = true;
715718 machine.note("clamp", "not passed on", { why: "frame too large", bytes: frameBytes(raw) });
@@ -718,10 +721,11 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
718721 const pace = rate.take();
719722 if (pace === "flood") {
720723 overLimit = true;
724 metrics.count("follow_frames_dropped");
721725 machine.note("clamp", "not passed on", { why: "too many frames" });
722726 return browser.close(FLOOD_CLOSE, "too many frames: the stream was closed");
723727 }
724 if (pace === "drop") return;
728 if (pace === "drop") return void metrics.count("follow_frames_dropped");
725729 if (driver !== null) {
726730 // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on.
727731 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void machine.note("clamp", "not passed on", { why: "the turn ended" });
@@ -862,6 +866,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
862866 timer?.unref?.();
863867 const pusher = lobby ? setInterval(() => void push(), statusEveryMs) : undefined;
864868 pusher?.unref?.();
869 const summary = options.summaryEveryMs ? setInterval(() => console.log(metrics.summaryLine()), options.summaryEveryMs) : undefined;
870 summary?.unref?.();
865871 await new Promise<void>((resolve) => http.listen(options.port ?? 8102, options.host ?? "127.0.0.1", resolve));
866872 const address = http.address();
867873 return {
@@ -871,6 +877,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
871877 new Promise<void>((resolve) => {
872878 clearInterval(timer);
873879 clearInterval(pusher);
880 clearInterval(summary);
874881 settingsChanged?.();
875882 turnEnded?.();
876883 arrived?.();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs