Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

/metrics.prom counts camera frames in, bytes out and viewers per socket kind #273

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/246-metrics-prom-counts-camera-frames-in-byt → main 1 commit fork

  • Merged. smith/246-metrics-prom-counts-camera-frames-in-byt was merged into main as d217ec3.

Review

Sign in to review this change.
3 files changed +52 −4
Software/so101/src/camera-feed.ts +12 −2
@@ -10,6 +10,7 @@
1010 */
1111
1212 import type { IncomingMessage, ServerResponse } from "node:http";
13 import type { LiveMetrics } from "./live-metrics.ts";
1314 import { boundaryOf, MjpegParser } from "./mjpeg.ts";
1415 import { OFFLINE_FRAME } from "./offline-frame.ts";
1516
@@ -26,6 +27,8 @@ export interface FeedOptions {
2627 lingerMs?: number;
2728 /** Bytes queued for one viewer past which it skips frames. */
2829 maxBacklog?: number;
30 /** Where to count frames in from the source and bytes out to viewers (`camera_frames_in`, `camera_bytes_out`). */
31 metrics?: LiveMetrics;
2932 }
3033
3134 /** Something that takes the feed's pictures whole rather than as a multipart body (the camera WebSocket). */
@@ -76,7 +79,8 @@ export class CameraFeed {
7679 private watchdog: ReturnType<typeof setInterval> | undefined;
7780 private lastByte = 0;
7881 private closed = false;
79 private readonly o: Required<FeedOptions>;
82 private readonly o: Required<Omit<FeedOptions, "metrics">>;
83 private readonly metrics: LiveMetrics | undefined;
8084 private readonly source: string | null;
8185 private readonly name: string;
8286
@@ -84,6 +88,7 @@ export class CameraFeed {
8488 constructor(source: string | null, name = "camera", options: FeedOptions = {}) {
8589 this.source = source;
8690 this.name = name;
91 this.metrics = options.metrics;
8792 this.o = { connectTimeoutMs: 1500, staleMs: 2000, retryMs: 2000, offlineEveryMs: 2000, lingerMs: 2000, maxBacklog: 1 << 20, ...options };
8893 this._state = source ? "idle" : "offline";
8994 }
@@ -158,7 +163,10 @@ export class CameraFeed {
158163 }
159164
160165 private write(viewer: Viewer, bytes: Buffer) {
161 if (!viewer.res.destroyed && !viewer.res.writableEnded) viewer.res.write(bytes);
166 if (!viewer.res.destroyed && !viewer.res.writableEnded) {
167 viewer.res.write(bytes);
168 this.metrics?.count("camera_bytes_out", bytes.length);
169 }
162170 }
163171
164172 private whole(viewer: Viewer, type: string, body: Buffer) {
@@ -257,6 +265,7 @@ export class CameraFeed {
257265 for (const v of this.viewers) if (v.inPart) this.write(v, bytes);
258266 },
259267 end: () => {
268 this.metrics?.count("camera_frames_in");
260269 if (this._state !== "live") {
261270 if (this._state === "offline") console.log(`${this.name} ${this.source}: live`);
262271 this._state = "live";
@@ -268,6 +277,7 @@ export class CameraFeed {
268277 const frame = Buffer.concat(this.part);
269278 this.part = null;
270279 for (const s of this.sinks) s.frame(frame);
280 this.metrics?.count("camera_bytes_out", frame.length * this.sinks.size);
271281 }
272282 },
273283 });
Software/so101/src/relay.test.ts +28
@@ -204,6 +204,34 @@ describe("relay", () => {
204204 expect(await (await fetch(`${base}/camera/1/state`)).json()).toEqual({ ok: true });
205205 });
206206
207 test("/metrics.prom counts camera frames in and bytes out, and the open sockets per kind rise and fall", async () => {
208 remote = await arm([1, 2, 6]);
209 camera = createServer((req, res) => {
210 res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=frame" });
211 const timer = setInterval(() => res.write(Buffer.concat([Buffer.from(`--frame\r\ncontent-type: image/jpeg\r\ncontent-length: ${JPEG.length}\r\n\r\n`), JPEG, Buffer.from("\r\n")])), 20);
212 res.on("close", () => clearInterval(timer));
213 });
214 relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(remote.port, await listening(camera)) });
215 const base = `http://127.0.0.1:${relay.port}`;
216 const value = async (series: string) => {
217 const text = await (await fetch(`${base}/metrics.prom`)).text();
218 const line = text.split("\n").find((l) => l.startsWith(`${series} `));
219 return line ? Number(line.slice(series.length + 1)) : NaN;
220 };
221 const gauge = 'sockets_open{kind="camera"}';
222 expect(await value(gauge)).toBe(0);
223
224 const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/camera/1/ws`);
225 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
226 await sleep(300);
227 expect(await value(gauge)).toBe(1);
228 expect(await value("camera_frames_in_per_second{}")).toBeGreaterThan(0);
229 expect(await value("camera_bytes_out_per_second{}")).toBeGreaterThan(0);
230 ws.close();
231 await sleep(100);
232 expect(await value(gauge)).toBe(0);
233 });
234
207235 test("WS /camera/<id>/ws: two viewers get whole JPEGs from one upstream connection; an unknown id is refused", async () => {
208236 let connections = 0;
209237 camera = createServer((req, res) => {
Software/so101/src/relay.ts +12 −2
@@ -232,12 +232,12 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
232232 const readTimeoutMs = options.readTimeoutMs ?? 2500;
233233 const lobby = options.lobby ?? null;
234234 const heartbeatMs = options.heartbeatMs ?? 1000;
235 const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
235 const metrics = new LiveMetrics();
236 const cameras = options.cameras ?? new CameraHub({ feed: { ...options.camera, metrics }, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
236237 const leaders = options.leaders ?? new LeaderCalibrations();
237238 const videos = new Map<number, VideoFeed>(Object.entries(options.video ?? {}).map(([id, sources]) => [Number(id), new VideoFeed(sources, {}, options.env ?? process.env)]));
238239
239240 // ------------------------------------------------------------------ the link
240 const metrics = new LiveMetrics();
241241 let armLink: ArmLink | null = null;
242242 /** The simulation's own watch, held only while someone has the Machine panel open and the simulation is the arm. */
243243 let simLink: ArmLink | null = null;
@@ -761,6 +761,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
761761 arm.on("error", () => browser.terminate());
762762 browser.on("error", () => arm.terminate());
763763 };
764 /** Open sockets per kind, kept as the `sockets_open` gauge: counted from the upgrade to the socket's close. */
765 const socketsOpen = { live: 0, follow: 0, machine: 0, camera: 0 };
766 type SocketKind = keyof typeof socketsOpen;
767 for (const kind of Object.keys(socketsOpen) as SocketKind[]) metrics.gauge("sockets_open", 0, { kind });
768 const track = (kind: SocketKind, socket: Duplex) => {
769 metrics.gauge("sockets_open", ++socketsOpen[kind], { kind });
770 socket.once("close", () => metrics.gauge("sockets_open", --socketsOpen[kind], { kind }));
771 };
764772 const refuse = (socket: Duplex, status: string) => {
765773 socket.end(`HTTP/1.1 ${status}\r\nconnection: close\r\ncontent-length: 0\r\n\r\n`);
766774 };
@@ -794,6 +802,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
794802 const id = Number(cameraWs[1]);
795803 if (!cameras.has(id)) return refuse(socket, "404 Not Found");
796804 socket.on("error", () => socket.destroy());
805 track("camera", socket);
797806 return wss.handleUpgrade(req, socket, head, (browser) => watchCamera(id, browser));
798807 }
799808 if (url.pathname !== "/follow" && url.pathname !== "/machine" && !video && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found");
@@ -806,6 +815,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
806815 const driver = !!who && !!lobby?.turns.isDriver(who.id);
807816 return wss.handleUpgrade(req, socket, head, (browser) => feed.serve(browser, driver));
808817 }
818 if (!video) track(url.pathname === "/live" ? "live" : url.pathname === "/machine" ? "machine" : "follow", socket);
809819 if (url.pathname === "/live") return lobby!.upgrade(req, socket, head);
810820 if (url.pathname === "/machine") return machine.upgrade(req, socket, head);
811821 let driver: string | null = null;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs