Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
The relay holds one LiveMetrics, reports the Pi round trip from arm-link, and serves it at GET /metrics.prom #261
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/245-the-relay-holds-one-livemetrics-reports → main 2 commits fork 2 threads · 2 unresolved
4 files
changed +60
−2
Software/README.md +15
| @@ -732,6 +732,21 @@ The vision kit in `Software/vision` works against this server and stream: | ||
| 732 | 732 | `SO101_ARM=http://<pi>:8101 SO101_CAMERA=http://<pi>:8080/stream`, then the |
| 733 | 733 | `just vision-*` recipes. See `Software/vision/QUEST.md`. |
| 734 | 734 | |
| 735 | ## Metrics | |
| 736 | ||
| 737 | `GET /metrics.prom` on the relay answers Prometheus text (`text/plain; version=0.0.4`, `no-store`, with the usual | |
| 738 | security headers). It carries the round trip to the arm server, which `src/arm-link.ts` measures on every ping (or | |
| 739 | `GET /status` when polling) and reports as `pi_rtt_ms`: the series `pi_rtt_ms{quantile="0.5"}`, | |
| 740 | `pi_rtt_ms{quantile="0.95"}`, `pi_rtt_ms_sum` and `pi_rtt_ms_count`, over the last 256 samples. It is empty until the | |
| 741 | first round trip is measured. | |
| 742 | ||
| 743 | ||
| 744 | curl https://arm--forge-trevato-apps.open-platform.sh/metrics.prom | |
| 745 | ||
| 746 | ||
| 747 | The endpoint is unauthenticated, like `/link` and `/status`: it exposes only timing aggregates, no names, tokens or | |
| 748 | commands, and cannot move the arm. `GET /metrics` is unchanged and is still the arm server's own, proxied. | |
| 749 | ||
| 735 | 750 | ## Launch check |
| 736 | 751 | |
| 737 | 752 | `so101/bench/launch-check.ts` checks a deployed page as an anonymous visitor and |
Software/so101/src/arm-link.ts +10 −2
| @@ -18,6 +18,7 @@ | ||
| 18 | 18 | |
| 19 | 19 | import type WebSocket from "ws"; |
| 20 | 20 | import { whyNot } from "./egress.ts"; |
| 21 | import type { LiveMetrics } from "./live-metrics.ts"; | |
| 21 | 22 | import { parseEntries, type MachineEntry } from "./machine-log.ts"; |
| 22 | 23 | |
| 23 | 24 | export type LinkState = "live" | "reconnecting" | "offline"; |
| @@ -55,6 +56,8 @@ export interface ArmLinkOptions { | ||
| 55 | 56 | /** Opens a WebSocket (through the egress proxy when there is one); rejects on failure or after the timeout. */ |
| 56 | 57 | open: (url: string, timeoutMs: number) => Promise<WebSocket>; |
| 57 | 58 | timings?: Partial<LinkTimings>; |
| 59 | /** Each round trip is reported to it as `pi_rtt_ms`. */ | |
| 60 | metrics?: LiveMetrics; | |
| 58 | 61 | /** Called on every change of state, after it is made. */ |
| 59 | 62 | onChange?: (state: LinkState, previous: LinkState) => void; |
| 60 | 63 | /** Called with every status that comes in. */ |
| @@ -166,7 +169,7 @@ export class ArmLink { | ||
| 166 | 169 | ws.ping(); |
| 167 | 170 | }, this.t.pingEveryMs); |
| 168 | 171 | ws.on("pong", () => { |
| 169 | if (pingedAt) this.rttMs = Math.round((performance.now() - pingedAt) * 10) / 10; | |
| 172 | if (pingedAt) this.round(performance.now() - pingedAt); | |
| 170 | 173 | }); |
| 171 | 174 | ws.on("message", (data) => { |
| 172 | 175 | const text = String(data); |
| @@ -189,11 +192,16 @@ export class ArmLink { | ||
| 189 | 192 | }); |
| 190 | 193 | } |
| 191 | 194 | |
| 195 | private round(ms: number) { | |
| 196 | this.rttMs = Math.round(ms * 10) / 10; | |
| 197 | this.o.metrics?.observe("pi_rtt_ms", ms); | |
| 198 | } | |
| 199 | ||
| 192 | 200 | private async poll() { |
| 193 | 201 | try { |
| 194 | 202 | const asked = performance.now(); |
| 195 | 203 | const r = await fetch(`${this.o.remote.http}/status`, { signal: AbortSignal.timeout(this.t.connectTimeoutMs), cache: "no-store" }); |
| 196 | this.rttMs = Math.round((performance.now() - asked) * 10) / 10; | |
| 204 | this.round(performance.now() - asked); | |
| 197 | 205 | if (!r.ok) throw new Error(await whyNot(r, "/status")); |
| 198 | 206 | const body = parseStatus(await r.text()); |
| 199 | 207 | if (!body) throw new Error("/status did not answer an arm's status"); |
Software/so101/src/relay.test.ts +27
| @@ -418,6 +418,33 @@ describe("relay", () => { | ||
| 418 | 418 | } |
| 419 | 419 | }); |
| 420 | 420 | |
| 421 | test("a fake arm that waits 50 ms shows in the round-trip p95 on /metrics.prom, and /metrics stays the arm's JSON", async () => { | |
| 422 | const wss = new WebSocketServer({ noServer: true, autoPong: false }); | |
| 423 | camera = createServer((_req, res) => res.writeHead(404).end()); | |
| 424 | camera.on("upgrade", (req, socket, head) => | |
| 425 | wss.handleUpgrade(req, socket, head, (ws) => { | |
| 426 | ws.on("ping", (data) => void setTimeout(() => ws.readyState === WebSocket.OPEN && ws.pong(data), 50)); | |
| 427 | }), | |
| 428 | ); | |
| 429 | const pump = setInterval(() => wss.clients.forEach((c) => c.send('{"joints":[]}')), 10); | |
| 430 | try { | |
| 431 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(await listening(camera)), link: { ...FAST, staleMs: 500, pingEveryMs: 100 } }); | |
| 432 | await sleep(700); | |
| 433 | const res = await fetch(`http://127.0.0.1:${relay.port}/metrics.prom`); | |
| 434 | expect(res.status).toBe(200); | |
| 435 | expect(res.headers.get("content-type")).toStartWith("text/plain; version=0.0.4"); | |
| 436 | expect(res.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 437 | const text = await res.text(); | |
| 438 | const p95 = Number(/^pi_rtt_ms\{quantile="0.95"\} (\S+)$/m.exec(text)?.[1]); | |
| 439 | expect(p95).toBeGreaterThanOrEqual(45); | |
| 440 | expect(p95).toBeLessThan(500); | |
| 441 | expect((await fetch(`http://127.0.0.1:${relay.port}/metrics`)).status).toBe(404); // still the arm server's, which this fake does not have | |
| 442 | } finally { | |
| 443 | clearInterval(pump); | |
| 444 | for (const c of wss.clients) c.terminate(); | |
| 445 | } | |
| 446 | }); | |
| 447 | ||
| 421 | 448 | test("a bad setting is reported and the arm is simulated", async () => { |
| 422 | 449 | relay = await startRelay({ port: 0, simulation: simBase, problem: 'ARM_SERVER "ftp://x" must start with http://' }); |
| 423 | 450 | expect(await link(`http://127.0.0.1:${relay.port}`)).toEqual({ mode: "simulated", host: null, problem: 'ARM_SERVER "ftp://x" must start with http://' }); |
Software/so101/src/relay.ts +8
| @@ -56,6 +56,7 @@ import type { Duplex } from "node:stream"; | ||
| 56 | 56 | import { Readable } from "node:stream"; |
| 57 | 57 | import { pipeline } from "node:stream/promises"; |
| 58 | 58 | import WebSocket, { WebSocketServer } from "ws"; |
| 59 | import { LiveMetrics } from "./live-metrics.ts"; | |
| 59 | 60 | import { ArmLink, type LinkTimings } from "./arm-link.ts"; |
| 60 | 61 | import { MachineFeed } from "./machine-feed.ts"; |
| 61 | 62 | import { perServo } from "./machine-log.ts"; |
| @@ -229,6 +230,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 229 | 230 | const leaders = options.leaders ?? new LeaderCalibrations(); |
| 230 | 231 | |
| 231 | 232 | // ------------------------------------------------------------------ the link |
| 233 | const metrics = new LiveMetrics(); | |
| 232 | 234 | let armLink: ArmLink | null = null; |
| 233 | 235 | /** The simulation's own watch, held only while someone has the Machine panel open and the simulation is the arm. */ |
| 234 | 236 | let simLink: ArmLink | null = null; |
| @@ -244,6 +246,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 244 | 246 | remote, |
| 245 | 247 | open: (url, timeoutMs) => open(url, timeoutMs), |
| 246 | 248 | timings: options.link, |
| 249 | metrics, | |
| 247 | 250 | onStatus: (status) => onArm() && machine.status(status), |
| 248 | 251 | onMachine: (entries, run) => onArm() && machine.arm(entries, run), |
| 249 | 252 | onChange: (state, previous) => { |
| @@ -571,6 +574,11 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 571 | 574 | if (refused) return json(res, refused.status, { error: refused.error }); |
| 572 | 575 | by = who!.name; |
| 573 | 576 | } |
| 577 | // Public on purpose, like /link: timing aggregates only (README, "Metrics"). | |
outdated
This sits after the lobby check, but for a GET the check is skipped when reading, so the route is public. That is fine for aggregate timings, but document it in the README and add a line saying so. |
||
| 578 | if (req.method === "GET" && url.pathname === "/metrics.prom") { | |
| 579 | res.writeHead(200, { "content-type": "text/plain; version=0.0.4; charset=utf-8", "cache-control": "no-store" }); | |
| 580 | return res.end(metrics.prometheus()); | |
| 581 | } | |
| 574 | 582 | if (req.method === "GET" && url.pathname === "/link") return json(res, 200, link()); |
| 575 | 583 | if (req.method === "GET" && url.pathname === "/status") return await status(res, path); |
| 576 | 584 |
new above
offline
round() runs before the r.ok check, so a fast 5xx counts as a healthy round trip and pulls p95 down. Move it after the ok check.