Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

/follow: size and rate checks #44

Parked opened this
Closed without merging

smith/arm:smith/36-follow-size-and-rate-checks → main 7 commits fork 3 threads · 3 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
7 files changed +176 −5
.gitignore +1
@@ -5,6 +5,7 @@ node_modules/
55 poses.json
66 outputs/
77 __pycache__/
8 metrics.jsonl
89
910 # sandbox home-directory placeholders: never part of the app
1011 /.bash_profile
Software/README.md +10
@@ -143,6 +143,16 @@ held to the speed and acceleration limits, and holds the arm after a second
143143 of silence (see **Safety limits** below). There is no authentication: keep it
144144 on a LAN or behind Tailscale.
145145
146 `/follow` bounds what a client may send. A frame is at most 1 KiB (a six-joint
147 frame is under 100 bytes); a larger message closes the socket with code 1009.
148 Frames are accepted at up to 200 a second, with a burst of 200 after a quiet
149 spell (a leader sends 50-100); surplus frames are dropped, not queued, and
150 counted as `refused` under `stream` in `/status`. The allowance is shared by all
151 the sockets of one peer. The relay keeps it per driver (per address without a
152 lobby); the arm server only sees its peer's address, so behind the relay that
153 is one allowance for the relay's streams together, and the relay's per-driver
154 limit is what keeps one noisy client from starving the others.
155
146156 `GET /` is the browser app: up to two camera panes, a live 3D model of the
147157 arm (its own URDF and meshes, drag a joint and the arm follows), a slider and
148158 hold-to-jog buttons per joint that stream targets like a leader does, a
Software/so101/src/frame-limit.test.ts added +40
@@ -0,0 +1,40 @@
1 import { describe, expect, test } from "bun:test";
2 import WebSocket from "ws";
3 import { FRAME_BURST, FrameRate, MAX_FRAME_BYTES } from "./frame-limit.ts";
4 import { FeetechBus } from "./bus.ts";
5 import { FakeBus } from "./fake-bus.ts";
6 import { startServer } from "./server.ts";
7
8 describe("frame limits", () => {
9 test("a burst passes, then frames are dropped until the bucket refills", () => {
10 const rate = new FrameRate(100, 0);
11 for (let i = 0; i < FRAME_BURST; i++) expect(rate.take(0)).toBe(true);
12 expect(rate.take(0)).toBe(false);
13 expect(rate.take(10)).toBe(true); // 10 ms at 100/s is one frame
14 expect(rate.take(10)).toBe(false);
15 expect(rate.take(5000)).toBe(true);
16 });
17
18 test("an oversized message closes the /follow socket with 1009", async () => {
19 const server = await startServer(new FeetechBus(new FakeBus([1])), { host: "127.0.0.1", port: 0, ids: [1] });
20 const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`);
21 const code = await new Promise<number>((resolve) => {
22 ws.on("open", () => ws.send("x".repeat(MAX_FRAME_BYTES + 1)));
23 ws.on("close", (c) => resolve(c));
24 });
25 expect(code).toBe(1009);
26 await Promise.race([server.close(), new Promise((r) => setTimeout(r, 500))]);
27 });
28
29 test("frames past the allowance are dropped and counted as refused", async () => {
30 const server = await startServer(new FeetechBus(new FakeBus([1])), { host: "127.0.0.1", port: 0, ids: [1] });
31 const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`);
32 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
33 for (let i = 0; i < FRAME_BURST * 2; i++) ws.send(JSON.stringify({ "1": 2048 }));
34 await new Promise((r) => setTimeout(r, 150));
35 const status = (await (await fetch(`http://127.0.0.1:${server.port}/status`)).json()) as { stream: { refused: number } };
36 expect(status.stream.refused).toBeGreaterThan(FRAME_BURST / 2);
37 ws.close();
38 await Promise.race([server.close(), new Promise((r) => setTimeout(r, 500))]);
39 });
40 });
Software/so101/src/frame-limit.ts added +53
@@ -0,0 +1,53 @@
1 /**
2 * The bounds on a /follow stream: a frame is a handful of joint targets, so a large message is refused by the socket
3 * itself (close 1009), and frames faster than any leader sends them are dropped before they reach the arm.
4 */
5
6 /** Six joints as JSON is under 100 bytes; this leaves room for every key and a long number. */
7 export const MAX_FRAME_BYTES = 1024;
8 /** A leader streams at 50-100 frames a second; a client may go well past that before it is held back. */
9 export const MAX_FRAMES_PER_SECOND = 200;
10 /** Frames let through at once after a quiet spell. */
11 export const FRAME_BURST = 200;
12
13 /** The size of a received message in bytes, whichever shape the socket delivers it in. */
14 export const frameBytes = (data: unknown): number =>
15 Array.isArray(data) ? data.reduce((n: number, b: Buffer) => n + b.length, 0) : data instanceof ArrayBuffer ? data.byteLength : (data as Buffer).length;
16
17 /** A token bucket: `take()` is true when a frame may pass. */
18 export class FrameRate {
19 private tokens = FRAME_BURST;
20 private at: number;
21 private readonly perSecond: number;
22 constructor(perSecond = MAX_FRAMES_PER_SECOND, now = Date.now()) {
outdated

The bucket is per connection, so many sockets multiply the allowance. Key it on the driver identity or socket address, or document that only one driver can stream at a time.

23 this.perSecond = perSecond;
24 this.at = now;
25 }
26 take(now = Date.now()): boolean {
27 this.tokens = Math.min(FRAME_BURST, this.tokens + Math.max(0, now - this.at) * (this.perSecond / 1000));
28 this.at = now;
29 if (this.tokens < 1) return false;
30 this.tokens -= 1;
31 return true;
32 }
33 }
34
35 /** One bucket per key (a driver, or a client address), shared by all the sockets that key opens, so more sockets are no more frames. */
36 export class FrameRates {
37 private readonly buckets = new Map<string, { rate: FrameRate; sockets: number }>();
38 /** Joins `key`'s bucket; the returned `leave` drops the bucket once its last socket is gone. */
39 join(key: string): { take: () => boolean; leave: () => void } {
40 const entry = this.buckets.get(key) ?? { rate: new FrameRate(), sockets: 0 };
41 entry.sockets++;
42 this.buckets.set(key, entry);
43 let left = false;
44 return {
45 take: () => entry.rate.take(),
46 leave: () => {
47 if (left) return;
48 left = true;
49 if (--entry.sockets === 0) this.buckets.delete(key);
50 },
51 };
52 }
53 }
Software/so101/src/relay.test.ts +39
@@ -6,6 +6,7 @@ import { tmpdir } from "node:os";
66 import { join } from "node:path";
77 import { gunzipSync } from "node:zlib";
88 import WebSocket, { WebSocketServer } from "ws";
9 import { FRAME_BURST, MAX_FRAME_BYTES } from "./frame-limit.ts";
910 import { FeetechBus } from "./bus.ts";
1011 import { FakeBus } from "./fake-bus.ts";
1112 import { OFFLINE_FRAME } from "./offline-frame.ts";
@@ -357,6 +358,44 @@ describe("relay", () => {
357358 }
358359 });
359360
361 test("/follow: an oversized message closes with 1009 and a burst past the allowance never reaches the arm", async () => {
362 const received: string[] = [];
363 const upstream = createServer();
364 const arms = new WebSocketServer({ server: upstream, path: "/follow" });
365 arms.on("connection", (ws) => ws.on("message", (d) => received.push(String(d))));
366 const port = await listening(upstream);
367 try {
368 relay = await startRelay({ port: 0, simulation: `http://127.0.0.1:${port}` });
369 const open = async () => {
370 const ws = new WebSocket(`ws://127.0.0.1:${relay!.port}/follow`);
371 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
372 return ws;
373 };
374 const big = await open();
375 const closed = new Promise<number>((r) => big.once("close", r));
376 big.send("x".repeat(MAX_FRAME_BYTES + 1));
377 expect(await closed).toBe(1009);
378 await sleep(50);
379 expect(received).toEqual([]);
380
381 // two sockets from one address share one allowance
382 const [a, b] = [await open(), await open()];
383 for (let i = 0; i < FRAME_BURST; i++) (i % 2 ? a : b).send(JSON.stringify({ "1": i }));
384 for (let i = 0; i < FRAME_BURST; i++) (i % 2 ? a : b).send(JSON.stringify({ "1": 9999 }));
385 await sleep(150);
386 expect(received.length).toBeGreaterThanOrEqual(FRAME_BURST);
387 expect(received.length).toBeLessThan(FRAME_BURST + 10); // the refill in the meantime, not the 40 extra
388 expect(relay.droppedFrames).toBeGreaterThan(FRAME_BURST / 2);
389 a.close();
390 b.close();
391 } finally {
392 for (const c of arms.clients) c.terminate();
393 arms.close();
394 upstream.closeAllConnections();
395 upstream.close();
396 }
397 });
398
360399 test("a bad setting is reported and the arm is simulated", async () => {
361400 relay = await startRelay({ port: 0, simulation: simBase, problem: 'ARM_SERVER "ftp://x" must start with http://' });
362401 expect(await link(`http://127.0.0.1:${relay.port}`)).toEqual({ mode: "simulated", host: null, problem: 'ARM_SERVER "ftp://x" must start with http://' });
Software/so101/src/relay.ts +20 −3
@@ -53,6 +53,7 @@ import type { FeedOptions } from "./camera-feed.ts";
5353 import { CameraError, CameraHub } from "./cameras.ts";
5454 import { LOCAL as LOCAL_PERSON } from "./identity.ts";
5555 import { LeaderCalibrations } from "./leader-calibrations.ts";
56 import { FrameRates, frameBytes, MAX_FRAME_BYTES } from "./frame-limit.ts";
5657 import { proxyFor, whyNot } from "./egress.ts";
5758 import { CalibrationError } from "./leader-map.js";
5859 import type { Lobby } from "./lobby.ts";
@@ -109,6 +110,7 @@ export interface RelayOptions {
109110 export interface RunningRelay {
110111 readonly port: number;
111112 link(): Link;
113 readonly droppedFrames: number;
112114 close(): Promise<void>;
113115 }
114116
@@ -556,7 +558,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
556558 });
557559
558560 // ------------------------------------------------------------------ WebSocket /follow
559 const wss = new WebSocketServer({ noServer: true });
561 const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_BYTES });
560562 const env = options.env ?? process.env;
561563 const open = (url: string, timeoutMs: number) =>
562564 new Promise<WebSocket>((resolve, reject) => {
@@ -580,11 +582,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
580582 const sendable = (code: number) => (code >= 1000 && code <= 1003) || (code >= 1007 && code <= 1014) || (code >= 3000 && code <= 4999);
581583 /** Each open /follow stream and whose it is, so a turn that ends can close it. */
582584 const streams = new Map<WebSocket, string>();
585 /** One frame allowance per driver (per address when there is no lobby), however many sockets they open. */
586 const rates = new FrameRates();
587 let droppedFrames = 0;
583588 /**
584589 * `base` is the arm server's HTTP side, asked where the joints are so the speed cap starts from there. On the real arm,
585590 * frames are not passed on while the link reconnects: the stream goes quiet, and the arm server's dead-man holds the arm.
586591 */
587 const bridge = (browser: WebSocket, arm: WebSocket, base: string, driver: string | null) => {
592 const bridge = (browser: WebSocket, arm: WebSocket, base: string, driver: string | null, address: string) => {
588593 const real = base !== simulation;
589594 if (driver !== null) {
590595 streams.set(browser, driver);
@@ -611,7 +616,15 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
611616 .finally(() => (asking = false));
612617 };
613618 seed();
619 const rate = rates.join(driver ?? address);
620 browser.on("close", rate.leave);
614621 browser.on("message", (data, binary) => {
622 // Checked after `ws` has buffered the message: Bun ignores `maxPayload`, so this bounds what is used, not what is held in memory.
623 if (frameBytes(data) > MAX_FRAME_BYTES) return browser.close(1009, "frame too large");
624 if (!rate.take()) {
625 droppedFrames++;
626 return;
627 }
615628 if (driver !== null) {
616629 // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on.
617630 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return;
@@ -679,7 +692,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
679692 }
680693 }
681694 if (socket.destroyed) return arm.close();
682 wss.handleUpgrade(req, socket, head, (browser) => bridge(browser, arm, base, driver));
695 wss.handleUpgrade(req, socket, head, (browser) => bridge(browser, arm, base, driver, req.socket.remoteAddress ?? "?"));
683696 })();
684697 });
685698
@@ -706,6 +719,10 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
706719 return {
707720 port: typeof address === "object" && address ? address.port : options.port ?? 8102,
708721 link,
722 /** Frames dropped for coming faster than the allowance. */
723 get droppedFrames() {
724 return droppedFrames;
725 },
709726 close: () =>
710727 new Promise<void>((resolve) => {
711728 clearInterval(timer);
Software/so101/src/server.ts +13 −2
@@ -51,6 +51,7 @@ import { fileURLToPath } from "node:url";
5151 import { WebSocketServer, type WebSocket } from "ws";
5252 import { browserModule } from "./browser-modules.ts";
5353 import type { FeetechBus } from "./bus.ts";
54 import { FrameRates, frameBytes, MAX_FRAME_BYTES } from "./frame-limit.ts";
5455 import { ArmModel, gripperTicksToRadians, ticksToRadians, type Violation } from "./kinematics.ts";
5556 import { clamp, type Frame } from "./mapping.ts";
5657 import { Metrics } from "./metrics.ts";
@@ -897,8 +898,9 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
897898 // Frames are merged down to the command rate; each joint's target is pulled inside its bounds, judged by the zone,
898899 // and handed to the governor, which moves the goals toward it within the speed and acceleration limits.
899900 // Silence makes the arm hold (the dead-man); a connection that is lost rather than closed makes it park.
900 const wss = new WebSocketServer({ noServer: true, maxPayload: 4096 });
901 wss.on("connection", (ws: WebSocket) => {
901 const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_BYTES });
902 const rates = new FrameRates();
903 wss.on("connection", (ws: WebSocket, req: IncomingMessage) => {
902904 if (estop) {
903905 ws.close(ESTOP_CLOSE, stopped(estop));
904906 return;
@@ -1108,7 +1110,16 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
11081110 lastTaken = Date.now();
11091111 follow(frame);
11101112 };
1113 // No identity reaches this server; behind the relay every stream shares the relay's address (and so one allowance).
1114 const rate = rates.join(req.socket.remoteAddress ?? "?");
1115 ws.on("close", rate.leave);
11111116 ws.on("message", (data) => {
1117 // Checked after `ws` has buffered the message: Bun ignores `maxPayload`, so this bounds what is used, not what is held in memory.
1118 if (frameBytes(data) > MAX_FRAME_BYTES) return ws.close(1009, "frame too large");
1119 if (!rate.take()) {
1120 stats.refused++;
1121 return;
1122 }
11121123 const frame = parseFrame(String(data));
11131124 if (!frame) return;
11141125 clearTimeout(timer);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs