Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

The relay feeds src/live-metrics.ts and serves it: Pi round trip, camera fps in, viewers, follow frames dropped, bytes out at /metrics.prom, one log line a minute #238

Parked opened this
Closed without merging

smith/arm:smith/110-the-relay-feeds-src-live-metrics-ts-and → main 2 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
5 files changed +113 −16
Software/so101/bench/fake-serve.ts +4 −2
@@ -19,6 +19,7 @@ import { Accounts } from "../src/accounts.ts";
1919 import { FeetechBus } from "../src/bus.ts";
2020 import { FakeBus } from "../src/fake-bus.ts";
2121 import { CameraHub } from "../src/cameras.ts";
22 import { LiveMetrics } from "../src/live-metrics.ts";
2223 import { LeaderCalibrations } from "../src/leader-calibrations.ts";
2324 import { Lobby } from "../src/lobby.ts";
2425 import { Moderator, modelFromEnv } from "../src/moderator.ts";
@@ -56,9 +57,10 @@ const lobby = new Lobby({
5657 accounts: new Accounts({ file: join(dir, "accounts.sqlite") }),
5758 moderator: new Moderator({ file: join(dir, "crew.json"), model: modelFromEnv(process.env) }),
5859 });
59 const cameras = new CameraHub({ file: join(dir, "cameras.json"), seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
60 const metrics = new LiveMetrics();
61 const cameras = new CameraHub({ feed: { metrics }, file: join(dir, "cameras.json"), seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
6062
6163 const leaders = new LeaderCalibrations({ file: join(dir, "leader-calibrations.json") });
62 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders });
64 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, metrics, summaryEveryMs: 60_000 });
6365 const { mode, host: armHost } = relay.link();
6466 console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`);
Software/so101/src/camera-feed.ts +13 −2
@@ -10,6 +10,7 @@
1010 */
1111
1212 import type { IncomingMessage, ServerResponse } from "node:http";
13 import type { LiveMetrics } from "./live-metrics.ts";
1314 import { boundaryOf, MjpegParser } from "./mjpeg.ts";
1415 import { OFFLINE_FRAME } from "./offline-frame.ts";
1516
@@ -26,6 +27,8 @@ export interface FeedOptions {
2627 lingerMs?: number;
2728 /** Bytes queued for one viewer past which it skips frames. */
2829 maxBacklog?: number;
30 /** Counts `camera_frames_in` (parts the source sent) and `camera_bytes_out` (bytes written to viewers). */
31 metrics?: LiveMetrics;
2932 }
3033
3134 /** Something that takes the feed's pictures whole rather than as a multipart body (the camera WebSocket). */
@@ -76,7 +79,7 @@ export class CameraFeed {
7679 private watchdog: ReturnType<typeof setInterval> | undefined;
7780 private lastByte = 0;
7881 private closed = false;
79 private readonly o: Required<FeedOptions>;
82 private readonly o: Required<Omit<FeedOptions, "metrics">> & Pick<FeedOptions, "metrics">;
8083 private readonly source: string | null;
8184 private readonly name: string;
8285
@@ -158,7 +161,14 @@ export class CameraFeed {
158161 }
159162
160163 private write(viewer: Viewer, bytes: Buffer) {
161 if (!viewer.res.destroyed && !viewer.res.writableEnded) viewer.res.write(bytes);
164 if (viewer.res.destroyed || viewer.res.writableEnded) return;
165 viewer.res.write(bytes);
166 this.countOut(bytes.length);
167 }
168
169 /** Bytes sent to a viewer that is not on the multipart stream (the camera WebSocket) count the same. */
170 countOut(bytes: number) {
171 this.o.metrics?.count("camera_bytes_out", bytes);
162172 }
163173
164174 private whole(viewer: Viewer, type: string, body: Buffer) {
@@ -263,6 +273,7 @@ export class CameraFeed {
263273 clearInterval(this.repeat);
264274 this.repeat = undefined;
265275 }
276 this.o.metrics?.count("camera_frames_in");
266277 for (const v of this.viewers) if (v.inPart) this.write(v, NEXT), (v.inPart = false);
267278 if (this.part) {
268279 const frame = Buffer.concat(this.part);
Software/so101/src/relay.test.ts +50
@@ -418,6 +418,56 @@ describe("relay", () => {
418418 }
419419 });
420420
421 test("/metrics.prom shows a slow arm in the round trip, a /follow flood in the dropped count, and the camera's frames and bytes", async () => {
422 // A fake arm that answers each ping 80 ms late, and a fake MJPEG source that sends one frame a few times a second.
423 const wss = new WebSocketServer({ noServer: true, autoPong: false });
424 camera = createServer((_req, res) => res.writeHead(404).end());
425 camera.on("upgrade", (req, socket, head) =>
426 wss.handleUpgrade(req, socket, head, (ws) => {
427 if (req.url === "/watch") ws.on("ping", (data) => setTimeout(() => ws.readyState === WebSocket.OPEN && ws.pong(data), 80));
428 }),
429 );
430 const pump = setInterval(() => wss.clients.forEach((c) => c.send('{"joints":[]}')), 10);
431 const mjpeg = createServer((_req, res) => {
432 res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=frame" });
433 const part = Buffer.concat([Buffer.from(`--frame\r\ncontent-type: image/jpeg\r\ncontent-length: ${JPEG.length}\r\n\r\n`), JPEG, Buffer.from("\r\n")]);
434 const send = setInterval(() => res.write(part), 20);
435 res.on("close", () => clearInterval(send));
436 });
437 try {
438 const armPort = await listening(camera);
439 relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(armPort, await listening(mjpeg)), link: { ...FAST, staleMs: 500, pingEveryMs: 100 } });
440 const base = `http://127.0.0.1:${relay.port}`;
441 const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/follow`);
442 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
443 const stamp = stamper();
444 for (let i = 0; i < 150; i++) ws.send(stamp({ 1: 2000 + (i % 50) }));
445 await cameraShows(`${base}/camera/1/stream`, JPEG);
446 await sleep(450);
447
448 const res = await fetch(`${base}/metrics.prom`);
449 expect(res.status).toBe(200);
450 expect(res.headers.get("content-type")).toStartWith("text/plain; version=0.0.4");
451 expect(res.headers.get("x-content-type-options")).toBe("nosniff");
452 expect(res.headers.get("content-security-policy")).toContain("frame-ancestors 'none'");
453 const text = await res.text();
454 const value = (name: string) => Number(new RegExp(`^${name} (\\S+)$`, "m").exec(text)?.[1]);
455 expect(value('arm_round_trip_ms\\{quantile="0.95"\\}')).toBeGreaterThanOrEqual(70);
456 expect(value("follow_frames_received_per_second\\{\\}") * 60).toBe(150);
457 expect(value("follow_frames_dropped_per_second\\{\\}") * 60).toBeGreaterThanOrEqual(50);
458 expect(value("camera_frames_in_per_second\\{\\}")).toBeGreaterThan(0);
459 expect(value("camera_bytes_out_per_second\\{\\}")).toBeGreaterThan(0);
460 expect(text).toContain('viewers{kind="follow"} 1');
461 // The JSON /metrics stays the arm server's own.
462 expect(await (await fetch(`${base}/metrics`)).json()).not.toHaveProperty("arm_round_trip_ms");
463 ws.close();
464 } finally {
465 clearInterval(pump);
466 for (const c of wss.clients) c.terminate();
467 await shut({ close: () => new Promise<void>((r) => (mjpeg.close(() => r()), mjpeg.closeAllConnections())) });
468 }
469 });
470
421471 test("a bad setting is reported and the arm is simulated", async () => {
422472 relay = await startRelay({ port: 0, simulation: simBase, problem: 'ARM_SERVER "ftp://x" must start with http://' });
423473 expect(await link(`http://127.0.0.1:${relay.port}`)).toEqual({ mode: "simulated", host: null, problem: 'ARM_SERVER "ftp://x" must start with http://' });
Software/so101/src/relay.ts +36 −10
@@ -57,6 +57,7 @@ import { Readable } from "node:stream";
5757 import { pipeline } from "node:stream/promises";
5858 import WebSocket, { WebSocketServer } from "ws";
5959 import { ArmLink, type LinkTimings } from "./arm-link.ts";
60 import { LiveMetrics } from "./live-metrics.ts";
6061 import { MachineFeed } from "./machine-feed.ts";
6162 import { perServo } from "./machine-log.ts";
6263 import { parseFrame } from "./safety.ts";
@@ -124,6 +125,10 @@ export interface RelayOptions {
124125 statusEveryMs?: number;
125126 /** Where HTTP_PROXY and NO_PROXY are read from; process.env by default. */
126127 env?: Record<string, string | undefined>;
128 /** The live numbers; made here by default. Hand in the one a `cameras` hub's feeds count into, so they share it. */
129 metrics?: LiveMetrics;
130 /** How often the one-line summary of the live numbers goes to stdout; 0 (the default) for never. */
131 summaryEveryMs?: number;
127132 }
128133
129134 export interface RunningRelay {
@@ -225,7 +230,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
225230 const readTimeoutMs = options.readTimeoutMs ?? 2500;
226231 const lobby = options.lobby ?? null;
227232 const heartbeatMs = options.heartbeatMs ?? 1000;
228 const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
233 const live = options.metrics ?? new LiveMetrics();
234 const cameras = options.cameras ?? new CameraHub({ feed: { ...options.camera, metrics: live }, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
229235 const leaders = options.leaders ?? new LeaderCalibrations();
230236
231237 // ------------------------------------------------------------------ the link
@@ -244,6 +250,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
244250 remote,
245251 open: (url, timeoutMs) => open(url, timeoutMs),
246252 timings: options.link,
253 metrics: live,
247254 onStatus: (status) => onArm() && machine.status(status),
248255 onMachine: (entries, run) => onArm() && machine.arm(entries, run),
249256 onChange: (state, previous) => {
@@ -572,6 +579,10 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
572579 by = who!.name;
573580 }
574581 if (req.method === "GET" && url.pathname === "/link") return json(res, 200, link());
582 if (reading && url.pathname === "/metrics.prom") {
583 res.writeHead(200, { "content-type": "text/plain; version=0.0.4; charset=utf-8", "cache-control": "no-store" });
584 return res.end(req.method === "HEAD" ? undefined : live.prometheus());
585 }
575586 if (req.method === "GET" && url.pathname === "/status") return await status(res, path);
576587
577588 const camera = CAMERA.exec(url.pathname);
@@ -693,29 +704,33 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
693704 const rate = new FrameRate();
694705 /** Set when this relay ends the stream for breaking the limits: the arm then holds, as for a stream closed on purpose. */
695706 let overLimit = false;
707 const dropped = () => live.count("follow_frames_dropped");
696708 browser.on("message", (raw) => {
697709 if (overLimit) return;
710 live.count("follow_frames_received");
698711 if (frameBytes(raw) > MAX_FRAME_BYTES) {
699712 overLimit = true;
713 dropped();
700714 machine.note("clamp", "not passed on", { why: "frame too large", bytes: frameBytes(raw) });
701715 return browser.close(TOO_BIG_CLOSE, `a frame is at most ${MAX_FRAME_BYTES} bytes`);
702716 }
703717 const pace = rate.take();
704718 if (pace === "flood") {
705719 overLimit = true;
720 dropped();
706721 machine.note("clamp", "not passed on", { why: "too many frames" });
707722 return browser.close(FLOOD_CLOSE, "too many frames: the stream was closed");
708723 }
709 if (pace === "drop") return;
724 if (pace === "drop") return dropped();
710725 if (driver !== null) {
711726 // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on.
712 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void machine.note("clamp", "not passed on", { why: "the turn ended" });
727 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void (dropped(), machine.note("clamp", "not passed on", { why: "the turn ended" }));
713728 lobby?.activity(driver);
714729 }
715730 const verdict = admission.admit(String(raw));
716 if (!verdict.ok) return void machine.note("clamp", "not passed on", { why: verdict.drop });
717 if (arm.readyState !== WebSocket.OPEN) return;
718 if (real && mode() !== "live") return void machine.note("clamp", "not passed on", { why: RECONNECTING });
731 if (!verdict.ok) return void (dropped(), machine.note("clamp", "not passed on", { why: verdict.drop }));
732 if (arm.readyState !== WebSocket.OPEN) return dropped();
733 if (real && mode() !== "live") return void (dropped(), machine.note("clamp", "not passed on", { why: RECONNECTING }));
719734 const data = JSON.stringify(Object.fromEntries(verdict.joints));
720735 const percent = lobby?.maxSpeedPercent ?? 100;
721736 const text = String(data);
@@ -728,7 +743,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
728743 if (!seeded) seed();
729744 const capped = cap.cap(text, percent);
730745 if (capped !== text) machine.note("clamp", "speed cap", { percent, asked: values(text), sent: capped === null ? {} : values(capped) });
731 if (capped === null) return;
746 if (capped === null) return dropped();
732747 machine.note("sent", "follow frame", { values: values(capped) });
733748 arm.send(capped);
734749 });
@@ -746,6 +761,12 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
746761 arm.on("error", () => browser.terminate());
747762 browser.on("error", () => arm.terminate());
748763 };
764 /** Viewers per kind of socket, reported as a gauge while it is open. */
765 const open_ = { live: 0, follow: 0, machine: 0, camera: 0 };
766 const viewing = (kind: keyof typeof open_, socket: Duplex) => {
767 live.gauge("viewers", ++open_[kind], { kind });
768 socket.once("close", () => live.gauge("viewers", --open_[kind], { kind }));
769 };
749770 const refuse = (socket: Duplex, status: string) => {
750771 socket.end(`HTTP/1.1 ${status}\r\nconnection: close\r\ncontent-length: 0\r\n\r\n`);
751772 };
@@ -759,7 +780,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
759780 watched.set(id, (w = { fan, viewers: 0, detach }));
760781 }
761782 const mine = w;
762 const viewer = { send: (data: Uint8Array) => browser.send(data, () => mine.fan.drain(viewer)), getBufferedAmount: () => browser.bufferedAmount };
783 const viewer = { send: (data: Uint8Array) => (cameras.feed(id)?.countOut(data.length), browser.send(data, () => mine.fan.drain(viewer))), getBufferedAmount: () => browser.bufferedAmount };
763784 mine.viewers++;
764785 mine.fan.add(viewer);
765786 browser.on("error", () => browser.terminate());
@@ -778,13 +799,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
778799 const id = Number(cameraWs[1]);
779800 if (!cameras.has(id)) return refuse(socket, "404 Not Found");
780801 socket.on("error", () => socket.destroy());
802 viewing("camera", socket);
781803 return wss.handleUpgrade(req, socket, head, (browser) => watchCamera(id, browser));
782804 }
783805 if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found");
784806 if (!sameOrigin(req)) return refuse(socket, "403 Forbidden");
785807 socket.on("error", () => socket.destroy());
786 if (url.pathname === "/live") return lobby!.upgrade(req, socket, head);
787 if (url.pathname === "/machine") return machine.upgrade(req, socket, head);
808 if (url.pathname === "/live") return viewing("live", socket), lobby!.upgrade(req, socket, head);
809 if (url.pathname === "/machine") return viewing("machine", socket), machine.upgrade(req, socket, head);
788810 let driver: string | null = null;
789811 if (lobby) {
790812 const who = lobby.who(req);
@@ -814,6 +836,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
814836 }
815837 }
816838 if (socket.destroyed) return arm.close();
839 viewing("follow", socket);
817840 wss.handleUpgrade(req, socket, head, (browser) => bridge(browser, arm, base, driver));
818841 })();
819842 });
@@ -839,6 +862,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
839862 timer?.unref?.();
840863 const pusher = lobby ? setInterval(() => void push(), statusEveryMs) : undefined;
841864 pusher?.unref?.();
865 const summary = options.summaryEveryMs ? setInterval(() => console.log(`live: ${live.summaryLine()}`), options.summaryEveryMs) : undefined;
866 summary?.unref?.();
842867 await new Promise<void>((resolve) => http.listen(options.port ?? 8102, options.host ?? "127.0.0.1", resolve));
843868 const address = http.address();
844869 return {
@@ -848,6 +873,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
848873 new Promise<void>((resolve) => {
849874 clearInterval(timer);
850875 clearInterval(pusher);
876 clearInterval(summary);
851877 settingsChanged?.();
852878 turnEnded?.();
853879 arrived?.();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs