Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

The app counts its own numbers: Pi round trip, camera fps in, viewers, frames dropped and bytes out, at /metrics.prom and a one-line log each minute #100

Closed opened this
Closed without merging

smith/arm:smith/88-the-app-counts-its-own-numbers-pi-round → main 1 commit fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
6 files changed +287 −11
Software/so101/src/camera-feed.ts +6 −2
@@ -26,6 +26,9 @@ export interface FeedOptions {
2626 lingerMs?: number;
2727 /** Bytes queued for one viewer past which it skips frames. */
2828 maxBacklog?: number;
29 /** Called for each frame that comes in from the source, and with the bytes written to a viewer. */
30 onFrame?: () => void;
31 onBytes?: (bytes: number) => void;
2932 }
3033
3134 export type FeedState = "idle" | "connecting" | "live" | "offline";
@@ -65,7 +68,7 @@ export class CameraFeed {
6568 private watchdog: ReturnType<typeof setInterval> | undefined;
6669 private lastByte = 0;
6770 private closed = false;
68 private readonly o: Required<FeedOptions>;
71 private readonly o: Required<Omit<FeedOptions, "onFrame" | "onBytes">> & Pick<FeedOptions, "onFrame" | "onBytes">;
6972 private readonly source: string | null;
7073 private readonly name: string;
7174
@@ -128,7 +131,7 @@ export class CameraFeed {
128131 }
129132
130133 private write(viewer: Viewer, bytes: Buffer) {
131 if (!viewer.res.destroyed && !viewer.res.writableEnded) viewer.res.write(bytes);
134 if (!viewer.res.destroyed && !viewer.res.writableEnded) (viewer.res.write(bytes), this.o.onBytes?.(bytes.length));
132135 }
133136
134137 private whole(viewer: Viewer, type: string, body: Buffer) {
@@ -223,6 +226,7 @@ export class CameraFeed {
223226 for (const v of this.viewers) if (v.inPart) this.write(v, bytes);
224227 },
225228 end: () => {
229 this.o.onFrame?.();
226230 if (this._state !== "live") {
227231 if (this._state === "offline") console.log(`${this.name} ${this.source}: live`);
228232 this._state = "live";
Software/so101/src/live-metrics.test.ts added +94
@@ -0,0 +1,94 @@
1 import { describe, expect, test } from "bun:test";
2 import { LiveMetrics, quantile } from "./live-metrics.ts";
3
4 /** Every sample line of Prometheus text: name, labels, value. */
5 function parse(text: string) {
6 const line = /^([a-z_][a-z0-9_]*)(\{[a-z_]+="[^"]*"(?:,[a-z_]+="[^"]*")*\})? (-?\d+(?:\.\d+)?)$/;
7 return text
8 .trimEnd()
9 .split("\n")
10 .filter((l) => !l.startsWith("#"))
11 .map((l) => {
12 const m = line.exec(l);
13 if (!m) throw new Error(`not a sample: ${l}`);
14 return { name: m[1]!, labels: m[2] ?? "", value: Number(m[3]) };
15 });
16 }
17
18 describe("live metrics", () => {
19 test("quantile of nothing is null; of a list, the value at that rank", () => {
20 expect(quantile([], 0.5)).toBeNull();
21 expect(quantile([1, 2, 3, 4], 0.5)).toBe(2);
22 expect(quantile([1, 2, 3, 4], 0.95)).toBe(4);
23 });
24
25 test("injected link delays show up in p95, and p50 stays at the usual", () => {
26 const m = new LiveMetrics();
27 for (let i = 0; i < 95; i++) m.rtt(10);
28 for (let i = 0; i < 5; i++) m.rtt(400);
29 const s = m.snapshot();
30 expect(s.rttP50Ms).toBe(10);
31 expect(s.rttP95Ms).toBe(10);
32 m.rtt(400);
33 m.rtt(400);
34 expect(m.snapshot().rttP95Ms).toBe(400);
35 expect(m.snapshot().rttP50Ms).toBe(10);
36 });
37
38 test("a bad round trip is not a sample", () => {
39 const m = new LiveMetrics();
40 m.rtt(NaN);
41 m.rtt(-1);
42 expect(m.snapshot().rttSamples).toBe(0);
43 expect(m.snapshot().rttP95Ms).toBeNull();
44 });
45
46 test("camera frames per second are over the last ten seconds, and old ones age out", () => {
47 let now = 1_000_000;
48 const m = new LiveMetrics(() => now);
49 for (let s = 0; s < 10; s++) {
50 for (let i = 0; i < 20; i++) m.frame();
51 if (s < 9) now += 1000;
52 }
53 expect(m.snapshot().cameraFps).toBe(20);
54 now += 20_000;
55 expect(m.snapshot().cameraFps).toBe(0);
56 expect(m.snapshot().cameraFramesTotal).toBe(200);
57 });
58
59 test("viewers, commands and bytes are counted", () => {
60 const m = new LiveMetrics();
61 m.viewer("follow", 1);
62 m.viewer("camera", 1);
63 m.viewer("camera", 1);
64 m.viewer("camera", -1);
65 m.viewer("live", -1);
66 m.command();
67 m.command();
68 m.drop();
69 m.out(1500);
70 const s = m.snapshot();
71 expect(s.viewers).toEqual({ follow: 1, live: 0, machine: 0, camera: 1 });
72 expect([s.followReceived, s.followDropped, s.bytesOut]).toEqual([2, 1, 1500]);
73 });
74
75 test("the Prometheus text parses: a name, optional labels, a value on every line", () => {
76 const m = new LiveMetrics();
77 m.rtt(12.5);
78 m.viewer("machine", 1);
79 m.out(7);
80 const samples = parse(m.prometheus());
81 const get = (name: string, labels = "") => samples.find((x) => x.name === name && x.labels === labels)?.value;
82 expect(get("arm_link_rtt_ms", '{quantile="0.95"}')).toBe(12.5);
83 expect(get("viewers", '{kind="machine"}')).toBe(1);
84 expect(get("viewer_bytes_sent_total")).toBe(7);
85 expect(new LiveMetrics().prometheus()).not.toContain("arm_link_rtt_ms{");
86 });
87
88 test("the summary is one line", () => {
89 const m = new LiveMetrics();
90 m.rtt(5);
91 expect(m.summary()).not.toContain("\n");
92 expect(m.summary()).toContain("p95=5ms");
93 });
94 });
Software/so101/src/live-metrics.ts added +140
@@ -0,0 +1,140 @@
1 /**
2 * The relay's own numbers, counted as it runs: the arm link's round trip, camera frames in, viewers per kind of socket,
3 * /follow commands received and dropped, and bytes sent to viewers. Plain data in, plain data out: the hot paths only
4 * bump a number (no allocation per frame); percentiles are worked out when someone asks.
5 *
6 * `prometheus()` is the text format for GET /metrics.prom; `summary()` the one line the relay logs each minute.
7 */
8
9 const RTT_SAMPLES = 512;
10 const RATE_SECONDS = 10;
11
12 export type ViewerKind = "follow" | "live" | "machine" | "camera";
13 export const VIEWER_KINDS: readonly ViewerKind[] = ["follow", "live", "machine", "camera"];
14
15 export interface LiveSnapshot {
16 rttP50Ms: number | null;
17 rttP95Ms: number | null;
18 rttSamples: number;
19 cameraFramesTotal: number;
20 /** Camera frames per second over the last ten seconds. */
21 cameraFps: number;
22 viewers: Record<ViewerKind, number>;
23 followReceived: number;
24 followDropped: number;
25 bytesOut: number;
26 }
27
28 /** The value at fraction `q` (0-1) of an ascending list; null when it is empty. */
29 export function quantile(sorted: ArrayLike<number>, q: number): number | null {
30 if (!sorted.length) return null;
31 return sorted[Math.min(sorted.length - 1, Math.ceil(q * sorted.length) - 1)]!;
32 }
33
34 export class LiveMetrics {
35 private readonly rtts = new Float64Array(RTT_SAMPLES);
36 private rttCount = 0;
37 private rttNext = 0;
38 private frames = 0;
39 private readonly perSecond = new Uint32Array(RATE_SECONDS);
40 private second = 0;
41 private readonly viewerCount: Record<ViewerKind, number> = { follow: 0, live: 0, machine: 0, camera: 0 };
42 private received = 0;
43 private dropped = 0;
44 private bytes = 0;
45 private readonly now: () => number;
46
47 constructor(now: () => number = Date.now) {
48 this.now = now;
49 this.second = Math.floor(now() / 1000);
50 }
51
52 /** A round trip to the arm server, ms. */
53 rtt(ms: number) {
54 if (!Number.isFinite(ms) || ms < 0) return;
55 this.rtts[this.rttNext] = ms;
56 this.rttNext = (this.rttNext + 1) % RTT_SAMPLES;
57 if (this.rttCount < RTT_SAMPLES) this.rttCount++;
58 }
59
60 /** One camera frame came in from a source. */
61 frame() {
62 this.frames++;
63 this.advance();
64 this.perSecond[this.second % RATE_SECONDS]!++;
65 }
66
67 /** A viewer of `kind` connected (+1) or left (-1). */
68 viewer(kind: ViewerKind, delta: 1 | -1) {
69 this.viewerCount[kind] = Math.max(0, this.viewerCount[kind] + delta);
70 }
71
72 /** A /follow command arrived from a browser. */
73 command() {
74 this.received++;
75 }
76
77 /** A /follow command was not passed on to the arm. */
78 drop() {
79 this.dropped++;
80 }
81
82 /** Bytes written to a viewer. */
83 out(bytes: number) {
84 this.bytes += bytes;
85 }
86
87 private advance() {
88 const s = Math.floor(this.now() / 1000);
89 const gap = Math.min(s - this.second, RATE_SECONDS);
90 for (let i = 1; i <= gap; i++) this.perSecond[(this.second + i) % RATE_SECONDS] = 0;
91 if (s > this.second) this.second = s;
92 }
93
94 snapshot(): LiveSnapshot {
95 this.advance();
96 const sorted = this.rtts.slice(0, this.rttCount).sort();
97 let inWindow = 0;
98 for (const n of this.perSecond) inWindow += n;
99 return {
100 rttP50Ms: quantile(sorted, 0.5),
101 rttP95Ms: quantile(sorted, 0.95),
102 rttSamples: this.rttCount,
103 cameraFramesTotal: this.frames,
104 cameraFps: Math.round((inWindow / RATE_SECONDS) * 10) / 10,
105 viewers: { ...this.viewerCount },
106 followReceived: this.received,
107 followDropped: this.dropped,
108 bytesOut: this.bytes,
109 };
110 }
111
112 /** Prometheus text exposition format, one sample per line. */
113 prometheus(): string {
114 const s = this.snapshot();
115 const out: string[] = [];
116 const metric = (name: string, type: "gauge" | "counter", help: string, samples: Array<[string, number | null]>) => {
117 out.push(`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`);
118 for (const [labels, value] of samples) if (value !== null) out.push(`${name}${labels} ${value}`);
119 };
120 metric("arm_link_rtt_ms", "gauge", "Round trip to the arm server over the recent samples, ms.", [
121 ['{quantile="0.5"}', s.rttP50Ms],
122 ['{quantile="0.95"}', s.rttP95Ms],
123 ]);
124 metric("arm_link_rtt_samples", "gauge", "Round trips in the window the quantiles come from.", [["", s.rttSamples]]);
125 metric("camera_frames_in_total", "counter", "Camera frames received from sources.", [["", s.cameraFramesTotal]]);
126 metric("camera_frames_in_per_second", "gauge", "Camera frames per second over the last ten seconds.", [["", s.cameraFps]]);
127 metric("viewers", "gauge", "Connected viewers by kind of socket.", VIEWER_KINDS.map((k) => [`{kind="${k}"}`, s.viewers[k]]));
128 metric("follow_commands_received_total", "counter", "/follow commands received from browsers.", [["", s.followReceived]]);
129 metric("follow_commands_dropped_total", "counter", "/follow commands not passed on to the arm.", [["", s.followDropped]]);
130 metric("viewer_bytes_sent_total", "counter", "Bytes sent to viewers (camera streams and /follow replies).", [["", s.bytesOut]]);
131 return out.join("\n") + "\n";
132 }
133
134 /** The one-line summary, for the log. */
135 summary(): string {
136 const s = this.snapshot();
137 const v = VIEWER_KINDS.map((k) => `${k}=${s.viewers[k]}`).join(" ");
138 return `metrics: rtt p50=${s.rttP50Ms ?? "-"}ms p95=${s.rttP95Ms ?? "-"}ms camera=${s.cameraFps}fps viewers ${v} follow received=${s.followReceived} dropped=${s.followDropped} out=${s.bytesOut}B`;
139 }
140 }
Software/so101/src/relay.test.ts +11
@@ -136,6 +136,17 @@ describe("relay", () => {
136136 cameras.close();
137137 });
138138
139 test("/metrics.prom answers the relay's own numbers as Prometheus text", async () => {
140 relay = await startRelay({ port: 0, simulation: simBase });
141 const r = await fetch(`http://127.0.0.1:${relay.port}/metrics.prom`);
142 expect(r.status).toBe(200);
143 expect(r.headers.get("content-type")).toContain("text/plain");
144 const text = await r.text();
145 expect(text).toContain("# TYPE viewers gauge");
146 expect(text).toMatch(/^viewers\{kind="follow"\} 0$/m);
147 expect(text).toMatch(/^follow_commands_received_total 0$/m);
148 });
149
139150 test("unset: everything is the simulation, and the page says so", async () => {
140151 relay = await startRelay({ port: 0, simulation: simBase });
141152 const base = `http://127.0.0.1:${relay.port}`;
Software/so101/src/relay.ts +30 −8
@@ -55,6 +55,7 @@ import type { Duplex } from "node:stream";
5555 import { Readable } from "node:stream";
5656 import { pipeline } from "node:stream/promises";
5757 import WebSocket, { WebSocketServer } from "ws";
58 import { LiveMetrics, type ViewerKind } from "./live-metrics.ts";
5859 import { ArmLink, type LinkTimings } from "./arm-link.ts";
5960 import { MachineFeed } from "./machine-feed.ts";
6061 import { perServo } from "./machine-log.ts";
@@ -112,6 +113,8 @@ export interface RelayOptions {
112113 readTimeoutMs?: number;
113114 /** Who may command the arm; without one, anyone who reaches the relay may (a run on one's own machine). */
114115 lobby?: Lobby | null;
116 /** How often the one-line summary of the relay's own numbers is logged. */
117 summaryEveryMs?: number;
115118 /** How often each /follow browser is pinged; one that misses a ping is a lost connection. */
116119 heartbeatMs?: number;
117120 /** Timings of the shared camera feeds. */
@@ -223,7 +226,13 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
223226 const readTimeoutMs = options.readTimeoutMs ?? 2500;
224227 const lobby = options.lobby ?? null;
225228 const heartbeatMs = options.heartbeatMs ?? 1000;
226 const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
229 const live = new LiveMetrics();
230 /** Counts a viewer in until `closing` says it has gone. */
231 const watch = (kind: ViewerKind, closing: { once(event: "close", fn: () => void): unknown }) => {
232 live.viewer(kind, 1);
233 closing.once("close", () => live.viewer(kind, -1));
234 };
235 const cameras = options.cameras ?? new CameraHub({ feed: { ...options.camera, onFrame: () => live.frame(), onBytes: (n) => live.out(n) }, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
227236 const leaders = options.leaders ?? new LeaderCalibrations();
228237
229238 // ------------------------------------------------------------------ the link
@@ -242,6 +251,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
242251 remote,
243252 open: (url, timeoutMs) => open(url, timeoutMs),
244253 timings: options.link,
254 onRtt: (ms) => live.rtt(ms),
245255 onStatus: (status) => onArm() && machine.status(status),
246256 onMachine: (entries, run) => onArm() && machine.arm(entries, run),
247257 onChange: (state, previous) => {
@@ -626,6 +636,10 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
626636 if (refused) return json(res, refused.status, { error: refused.error });
627637 by = who!.name;
628638 }
639 if (req.method === "GET" && url.pathname === "/metrics.prom") {
640 res.writeHead(200, { "content-type": "text/plain; version=0.0.4; charset=utf-8", "cache-control": "no-store" });
641 return res.end(live.prometheus());
642 }
629643 if (req.method === "GET" && url.pathname === "/link") return json(res, 200, link());
630644 if (req.method === "GET" && url.pathname === "/status") return await status(res, path);
631645
@@ -635,6 +649,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
635649 const id = Number(camera[1]);
636650 const source = cameras.source(id);
637651 if (source === null) return json(res, 404, { error: "no such camera" });
652 if (camera[2] === "stream") watch("camera", res);
638653 if (camera[2] === "stream") return cameras.feed(id)!.serve(req, res, parseFps(url.searchParams.get("fps")));
639654 if (camera[2] === "state") return json(res, 200, { ok: cameras.list().find((c) => c.id === id)?.online ?? null });
640655 const target = source.replace(/\/[^/]*$/, "/snapshot") + url.search;
@@ -722,7 +737,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
722737 streams.delete(browser);
723738 admission.close();
724739 });
725 arm.on("message", (data, binary) => browser.readyState === WebSocket.OPEN && browser.send(data, { binary }));
740 arm.on("message", (data, binary) => browser.readyState === WebSocket.OPEN && (live.out(frameBytes(data)), browser.send(data, { binary })));
726741 const cap = new FrameCap();
727742 let seeded = false;
728743 let asking = false;
@@ -750,27 +765,30 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
750765 let overLimit = false;
751766 browser.on("message", (raw) => {
752767 if (overLimit) return;
768 live.command();
753769 if (frameBytes(raw) > MAX_FRAME_BYTES) {
754770 overLimit = true;
771 live.drop();
755772 machine.note("clamp", "not passed on", { why: "frame too large", bytes: frameBytes(raw) });
756773 return browser.close(TOO_BIG_CLOSE, `a frame is at most ${MAX_FRAME_BYTES} bytes`);
757774 }
758775 const pace = rate.take();
759776 if (pace === "flood") {
760777 overLimit = true;
778 live.drop();
761779 machine.note("clamp", "not passed on", { why: "too many frames" });
762780 return browser.close(FLOOD_CLOSE, "too many frames: the stream was closed");
763781 }
764 if (pace === "drop") return;
782 if (pace === "drop") return void live.drop();
765783 if (driver !== null) {
766784 // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on.
767 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void machine.note("clamp", "not passed on", { why: "the turn ended" });
785 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void (live.drop(), machine.note("clamp", "not passed on", { why: "the turn ended" }));
768786 lobby?.activity(driver);
769787 }
770788 const verdict = admission.admit(String(raw));
771 if (!verdict.ok) return void machine.note("clamp", "not passed on", { why: verdict.drop });
772 if (arm.readyState !== WebSocket.OPEN) return;
773 if (real && mode() !== "live") return void machine.note("clamp", "not passed on", { why: RECONNECTING });
789 if (!verdict.ok) return void (live.drop(), machine.note("clamp", "not passed on", { why: verdict.drop }));
790 if (arm.readyState !== WebSocket.OPEN) return void live.drop();
791 if (real && mode() !== "live") return void (live.drop(), machine.note("clamp", "not passed on", { why: RECONNECTING }));
774792 const data = JSON.stringify(Object.fromEntries(verdict.joints));
775793 const percent = lobby?.maxSpeedPercent ?? 100;
776794 const text = String(data);
@@ -783,7 +801,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
783801 if (!seeded) seed();
784802 const capped = cap.cap(text, percent);
785803 if (capped !== text) machine.note("clamp", "speed cap", { percent, asked: values(text), sent: capped === null ? {} : values(capped) });
786 if (capped === null) return;
804 if (capped === null) return void live.drop();
787805 machine.note("sent", "follow frame", { values: values(capped) });
788806 arm.send(capped);
789807 });
@@ -809,6 +827,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
809827 if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found");
810828 if (!sameOrigin(req)) return refuse(socket, "403 Forbidden");
811829 socket.on("error", () => socket.destroy());
830 watch(url.pathname === "/live" ? "live" : url.pathname === "/machine" ? "machine" : "follow", socket);
812831 if (url.pathname === "/live") return lobby!.upgrade(req, socket, head);
813832 if (url.pathname === "/machine") return machine.upgrade(req, socket, head);
814833 let driver: string | null = null;
@@ -861,6 +880,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
861880 if (lobby) follow();
862881 if (!armLink) connect();
863882 await (armLink as ArmLink | null)?.ready; // bounded by the first attempt's timeout: the first page already knows which arm it has
883 const summaryTimer = setInterval(() => console.log(live.summary()), options.summaryEveryMs ?? 60_000);
884 summaryTimer.unref?.();
864885 const timer = lobby ? setInterval(follow, followEveryMs) : undefined;
865886 timer?.unref?.();
866887 const pusher = lobby ? setInterval(() => void push(), statusEveryMs) : undefined;
@@ -874,6 +895,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
874895 new Promise<void>((resolve) => {
875896 clearInterval(timer);
876897 clearInterval(pusher);
898 clearInterval(summaryTimer);
877899 settingsChanged?.();
878900 turnEnded?.();
879901 arrived?.();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs