Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
/metrics.prom counts camera frames in, bytes out and viewers per socket kind #273
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/246-metrics-prom-counts-camera-frames-in-byt → main 1 commit fork
3 files
changed +52
−4
Software/so101/src/camera-feed.ts +12 −2
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | */ |
| 11 | 11 | |
| 12 | 12 | import type { IncomingMessage, ServerResponse } from "node:http"; |
| 13 | import type { LiveMetrics } from "./live-metrics.ts"; | |
| 13 | 14 | import { boundaryOf, MjpegParser } from "./mjpeg.ts"; |
| 14 | 15 | import { OFFLINE_FRAME } from "./offline-frame.ts"; |
| 15 | 16 | |
| @@ -26,6 +27,8 @@ export interface FeedOptions { | ||
| 26 | 27 | lingerMs?: number; |
| 27 | 28 | /** Bytes queued for one viewer past which it skips frames. */ |
| 28 | 29 | maxBacklog?: number; |
| 30 | /** Where to count frames in from the source and bytes out to viewers (`camera_frames_in`, `camera_bytes_out`). */ | |
| 31 | metrics?: LiveMetrics; | |
| 29 | 32 | } |
| 30 | 33 | |
| 31 | 34 | /** Something that takes the feed's pictures whole rather than as a multipart body (the camera WebSocket). */ |
| @@ -76,7 +79,8 @@ export class CameraFeed { | ||
| 76 | 79 | private watchdog: ReturnType<typeof setInterval> | undefined; |
| 77 | 80 | private lastByte = 0; |
| 78 | 81 | private closed = false; |
| 79 | private readonly o: Required<FeedOptions>; | |
| 82 | private readonly o: Required<Omit<FeedOptions, "metrics">>; | |
| 83 | private readonly metrics: LiveMetrics | undefined; | |
| 80 | 84 | private readonly source: string | null; |
| 81 | 85 | private readonly name: string; |
| 82 | 86 | |
| @@ -84,6 +88,7 @@ export class CameraFeed { | ||
| 84 | 88 | constructor(source: string | null, name = "camera", options: FeedOptions = {}) { |
| 85 | 89 | this.source = source; |
| 86 | 90 | this.name = name; |
| 91 | this.metrics = options.metrics; | |
| 87 | 92 | this.o = { connectTimeoutMs: 1500, staleMs: 2000, retryMs: 2000, offlineEveryMs: 2000, lingerMs: 2000, maxBacklog: 1 << 20, ...options }; |
| 88 | 93 | this._state = source ? "idle" : "offline"; |
| 89 | 94 | } |
| @@ -158,7 +163,10 @@ export class CameraFeed { | ||
| 158 | 163 | } |
| 159 | 164 | |
| 160 | 165 | private write(viewer: Viewer, bytes: Buffer) { |
| 161 | if (!viewer.res.destroyed && !viewer.res.writableEnded) viewer.res.write(bytes); | |
| 166 | if (!viewer.res.destroyed && !viewer.res.writableEnded) { | |
| 167 | viewer.res.write(bytes); | |
| 168 | this.metrics?.count("camera_bytes_out", bytes.length); | |
| 169 | } | |
| 162 | 170 | } |
| 163 | 171 | |
| 164 | 172 | private whole(viewer: Viewer, type: string, body: Buffer) { |
| @@ -257,6 +265,7 @@ export class CameraFeed { | ||
| 257 | 265 | for (const v of this.viewers) if (v.inPart) this.write(v, bytes); |
| 258 | 266 | }, |
| 259 | 267 | end: () => { |
| 268 | this.metrics?.count("camera_frames_in"); | |
| 260 | 269 | if (this._state !== "live") { |
| 261 | 270 | if (this._state === "offline") console.log(`${this.name} ${this.source}: live`); |
| 262 | 271 | this._state = "live"; |
| @@ -268,6 +277,7 @@ export class CameraFeed { | ||
| 268 | 277 | const frame = Buffer.concat(this.part); |
| 269 | 278 | this.part = null; |
| 270 | 279 | for (const s of this.sinks) s.frame(frame); |
| 280 | this.metrics?.count("camera_bytes_out", frame.length * this.sinks.size); | |
| 271 | 281 | } |
| 272 | 282 | }, |
| 273 | 283 | }); |
Software/so101/src/relay.test.ts +28
| @@ -204,6 +204,34 @@ describe("relay", () => { | ||
| 204 | 204 | expect(await (await fetch(`${base}/camera/1/state`)).json()).toEqual({ ok: true }); |
| 205 | 205 | }); |
| 206 | 206 | |
| 207 | test("/metrics.prom counts camera frames in and bytes out, and the open sockets per kind rise and fall", async () => { | |
| 208 | remote = await arm([1, 2, 6]); | |
| 209 | camera = createServer((req, res) => { | |
| 210 | res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=frame" }); | |
| 211 | const timer = setInterval(() => res.write(Buffer.concat([Buffer.from(`--frame\r\ncontent-type: image/jpeg\r\ncontent-length: ${JPEG.length}\r\n\r\n`), JPEG, Buffer.from("\r\n")])), 20); | |
| 212 | res.on("close", () => clearInterval(timer)); | |
| 213 | }); | |
| 214 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(remote.port, await listening(camera)) }); | |
| 215 | const base = `http://127.0.0.1:${relay.port}`; | |
| 216 | const value = async (series: string) => { | |
| 217 | const text = await (await fetch(`${base}/metrics.prom`)).text(); | |
| 218 | const line = text.split("\n").find((l) => l.startsWith(`${series} `)); | |
| 219 | return line ? Number(line.slice(series.length + 1)) : NaN; | |
| 220 | }; | |
| 221 | const gauge = 'sockets_open{kind="camera"}'; | |
| 222 | expect(await value(gauge)).toBe(0); | |
| 223 | ||
| 224 | const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/camera/1/ws`); | |
| 225 | await new Promise((r, j) => (ws.once("open", r), ws.once("error", j))); | |
| 226 | await sleep(300); | |
| 227 | expect(await value(gauge)).toBe(1); | |
| 228 | expect(await value("camera_frames_in_per_second{}")).toBeGreaterThan(0); | |
| 229 | expect(await value("camera_bytes_out_per_second{}")).toBeGreaterThan(0); | |
| 230 | ws.close(); | |
| 231 | await sleep(100); | |
| 232 | expect(await value(gauge)).toBe(0); | |
| 233 | }); | |
| 234 | ||
| 207 | 235 | test("WS /camera/<id>/ws: two viewers get whole JPEGs from one upstream connection; an unknown id is refused", async () => { |
| 208 | 236 | let connections = 0; |
| 209 | 237 | camera = createServer((req, res) => { |
Software/so101/src/relay.ts +12 −2
| @@ -232,12 +232,12 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 232 | 232 | const readTimeoutMs = options.readTimeoutMs ?? 2500; |
| 233 | 233 | const lobby = options.lobby ?? null; |
| 234 | 234 | const heartbeatMs = options.heartbeatMs ?? 1000; |
| 235 | const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); | |
| 235 | const metrics = new LiveMetrics(); | |
| 236 | const cameras = options.cameras ?? new CameraHub({ feed: { ...options.camera, metrics }, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); | |
| 236 | 237 | const leaders = options.leaders ?? new LeaderCalibrations(); |
| 237 | 238 | const videos = new Map<number, VideoFeed>(Object.entries(options.video ?? {}).map(([id, sources]) => [Number(id), new VideoFeed(sources, {}, options.env ?? process.env)])); |
| 238 | 239 | |
| 239 | 240 | // ------------------------------------------------------------------ the link |
| 240 | const metrics = new LiveMetrics(); | |
| 241 | 241 | let armLink: ArmLink | null = null; |
| 242 | 242 | /** The simulation's own watch, held only while someone has the Machine panel open and the simulation is the arm. */ |
| 243 | 243 | let simLink: ArmLink | null = null; |
| @@ -761,6 +761,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 761 | 761 | arm.on("error", () => browser.terminate()); |
| 762 | 762 | browser.on("error", () => arm.terminate()); |
| 763 | 763 | }; |
| 764 | /** Open sockets per kind, kept as the `sockets_open` gauge: counted from the upgrade to the socket's close. */ | |
| 765 | const socketsOpen = { live: 0, follow: 0, machine: 0, camera: 0 }; | |
| 766 | type SocketKind = keyof typeof socketsOpen; | |
| 767 | for (const kind of Object.keys(socketsOpen) as SocketKind[]) metrics.gauge("sockets_open", 0, { kind }); | |
| 768 | const track = (kind: SocketKind, socket: Duplex) => { | |
| 769 | metrics.gauge("sockets_open", ++socketsOpen[kind], { kind }); | |
| 770 | socket.once("close", () => metrics.gauge("sockets_open", --socketsOpen[kind], { kind })); | |
| 771 | }; | |
| 764 | 772 | const refuse = (socket: Duplex, status: string) => { |
| 765 | 773 | socket.end(`HTTP/1.1 ${status}\r\nconnection: close\r\ncontent-length: 0\r\n\r\n`); |
| 766 | 774 | }; |
| @@ -794,6 +802,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 794 | 802 | const id = Number(cameraWs[1]); |
| 795 | 803 | if (!cameras.has(id)) return refuse(socket, "404 Not Found"); |
| 796 | 804 | socket.on("error", () => socket.destroy()); |
| 805 | track("camera", socket); | |
| 797 | 806 | return wss.handleUpgrade(req, socket, head, (browser) => watchCamera(id, browser)); |
| 798 | 807 | } |
| 799 | 808 | if (url.pathname !== "/follow" && url.pathname !== "/machine" && !video && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); |
| @@ -806,6 +815,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 806 | 815 | const driver = !!who && !!lobby?.turns.isDriver(who.id); |
| 807 | 816 | return wss.handleUpgrade(req, socket, head, (browser) => feed.serve(browser, driver)); |
| 808 | 817 | } |
| 818 | if (!video) track(url.pathname === "/live" ? "live" : url.pathname === "/machine" ? "machine" : "follow", socket); | |
| 809 | 819 | if (url.pathname === "/live") return lobby!.upgrade(req, socket, head); |
| 810 | 820 | if (url.pathname === "/machine") return machine.upgrade(req, socket, head); |
| 811 | 821 | let driver: string | null = null; |
new above
offline