Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
/follow: size and rate checks #44
smith/arm:smith/36-follow-size-and-rate-checks → main 7 commits fork 3 threads · 3 unresolved
7 files
changed +176
−5
.gitignore +1
| @@ -5,6 +5,7 @@ node_modules/ | ||
| 5 | 5 | poses.json |
| 6 | 6 | outputs/ |
| 7 | 7 | __pycache__/ |
| 8 | metrics.jsonl | |
| 8 | 9 | |
| 9 | 10 | # sandbox home-directory placeholders: never part of the app |
| 10 | 11 | /.bash_profile |
Software/README.md +10
| @@ -143,6 +143,16 @@ held to the speed and acceleration limits, and holds the arm after a second | ||
| 143 | 143 | of silence (see **Safety limits** below). There is no authentication: keep it |
| 144 | 144 | on a LAN or behind Tailscale. |
| 145 | 145 | |
| 146 | `/follow` bounds what a client may send. A frame is at most 1 KiB (a six-joint | |
| 147 | frame is under 100 bytes); a larger message closes the socket with code 1009. | |
| 148 | Frames are accepted at up to 200 a second, with a burst of 200 after a quiet | |
| 149 | spell (a leader sends 50-100); surplus frames are dropped, not queued, and | |
| 150 | counted as `refused` under `stream` in `/status`. The allowance is shared by all | |
| 151 | the sockets of one peer. The relay keeps it per driver (per address without a | |
| 152 | lobby); the arm server only sees its peer's address, so behind the relay that | |
| 153 | is one allowance for the relay's streams together, and the relay's per-driver | |
| 154 | limit is what keeps one noisy client from starving the others. | |
| 155 | ||
| 146 | 156 | `GET /` is the browser app: up to two camera panes, a live 3D model of the |
| 147 | 157 | arm (its own URDF and meshes, drag a joint and the arm follows), a slider and |
| 148 | 158 | hold-to-jog buttons per joint that stream targets like a leader does, a |
Software/so101/src/frame-limit.test.ts added +40
| @@ -0,0 +1,40 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import WebSocket from "ws"; | |
| 3 | import { FRAME_BURST, FrameRate, MAX_FRAME_BYTES } from "./frame-limit.ts"; | |
| 4 | import { FeetechBus } from "./bus.ts"; | |
| 5 | import { FakeBus } from "./fake-bus.ts"; | |
| 6 | import { startServer } from "./server.ts"; | |
| 7 | ||
| 8 | describe("frame limits", () => { | |
| 9 | test("a burst passes, then frames are dropped until the bucket refills", () => { | |
| 10 | const rate = new FrameRate(100, 0); | |
| 11 | for (let i = 0; i < FRAME_BURST; i++) expect(rate.take(0)).toBe(true); | |
| 12 | expect(rate.take(0)).toBe(false); | |
| 13 | expect(rate.take(10)).toBe(true); // 10 ms at 100/s is one frame | |
| 14 | expect(rate.take(10)).toBe(false); | |
| 15 | expect(rate.take(5000)).toBe(true); | |
| 16 | }); | |
| 17 | ||
| 18 | test("an oversized message closes the /follow socket with 1009", async () => { | |
| 19 | const server = await startServer(new FeetechBus(new FakeBus([1])), { host: "127.0.0.1", port: 0, ids: [1] }); | |
| 20 | const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`); | |
| 21 | const code = await new Promise<number>((resolve) => { | |
| 22 | ws.on("open", () => ws.send("x".repeat(MAX_FRAME_BYTES + 1))); | |
| 23 | ws.on("close", (c) => resolve(c)); | |
| 24 | }); | |
| 25 | expect(code).toBe(1009); | |
| 26 | await Promise.race([server.close(), new Promise((r) => setTimeout(r, 500))]); | |
| 27 | }); | |
| 28 | ||
| 29 | test("frames past the allowance are dropped and counted as refused", async () => { | |
| 30 | const server = await startServer(new FeetechBus(new FakeBus([1])), { host: "127.0.0.1", port: 0, ids: [1] }); | |
| 31 | const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`); | |
| 32 | await new Promise((r, j) => (ws.once("open", r), ws.once("error", j))); | |
| 33 | for (let i = 0; i < FRAME_BURST * 2; i++) ws.send(JSON.stringify({ "1": 2048 })); | |
| 34 | await new Promise((r) => setTimeout(r, 150)); | |
| 35 | const status = (await (await fetch(`http://127.0.0.1:${server.port}/status`)).json()) as { stream: { refused: number } }; | |
| 36 | expect(status.stream.refused).toBeGreaterThan(FRAME_BURST / 2); | |
| 37 | ws.close(); | |
| 38 | await Promise.race([server.close(), new Promise((r) => setTimeout(r, 500))]); | |
| 39 | }); | |
| 40 | }); |
Software/so101/src/frame-limit.ts added +53
| @@ -0,0 +1,53 @@ | ||
| 1 | /** | |
| 2 | * The bounds on a /follow stream: a frame is a handful of joint targets, so a large message is refused by the socket | |
| 3 | * itself (close 1009), and frames faster than any leader sends them are dropped before they reach the arm. | |
| 4 | */ | |
| 5 | ||
| 6 | /** Six joints as JSON is under 100 bytes; this leaves room for every key and a long number. */ | |
| 7 | export const MAX_FRAME_BYTES = 1024; | |
| 8 | /** A leader streams at 50-100 frames a second; a client may go well past that before it is held back. */ | |
| 9 | export const MAX_FRAMES_PER_SECOND = 200; | |
| 10 | /** Frames let through at once after a quiet spell. */ | |
| 11 | export const FRAME_BURST = 200; | |
| 12 | ||
| 13 | /** The size of a received message in bytes, whichever shape the socket delivers it in. */ | |
| 14 | export const frameBytes = (data: unknown): number => | |
| 15 | Array.isArray(data) ? data.reduce((n: number, b: Buffer) => n + b.length, 0) : data instanceof ArrayBuffer ? data.byteLength : (data as Buffer).length; | |
| 16 | ||
| 17 | /** A token bucket: `take()` is true when a frame may pass. */ | |
| 18 | export class FrameRate { | |
| 19 | private tokens = FRAME_BURST; | |
| 20 | private at: number; | |
| 21 | private readonly perSecond: number; | |
| 22 | constructor(perSecond = MAX_FRAMES_PER_SECOND, now = Date.now()) { | |
| 23 | this.perSecond = perSecond; | |
| 24 | this.at = now; | |
| 25 | } | |
| 26 | take(now = Date.now()): boolean { | |
| 27 | this.tokens = Math.min(FRAME_BURST, this.tokens + Math.max(0, now - this.at) * (this.perSecond / 1000)); | |
| 28 | this.at = now; | |
| 29 | if (this.tokens < 1) return false; | |
| 30 | this.tokens -= 1; | |
| 31 | return true; | |
| 32 | } | |
| 33 | } | |
| 34 | ||
| 35 | /** One bucket per key (a driver, or a client address), shared by all the sockets that key opens, so more sockets are no more frames. */ | |
| 36 | export class FrameRates { | |
| 37 | private readonly buckets = new Map<string, { rate: FrameRate; sockets: number }>(); | |
| 38 | /** Joins `key`'s bucket; the returned `leave` drops the bucket once its last socket is gone. */ | |
| 39 | join(key: string): { take: () => boolean; leave: () => void } { | |
| 40 | const entry = this.buckets.get(key) ?? { rate: new FrameRate(), sockets: 0 }; | |
| 41 | entry.sockets++; | |
| 42 | this.buckets.set(key, entry); | |
| 43 | let left = false; | |
| 44 | return { | |
| 45 | take: () => entry.rate.take(), | |
| 46 | leave: () => { | |
| 47 | if (left) return; | |
| 48 | left = true; | |
| 49 | if (--entry.sockets === 0) this.buckets.delete(key); | |
| 50 | }, | |
| 51 | }; | |
| 52 | } | |
| 53 | } |
Software/so101/src/relay.test.ts +39
| @@ -6,6 +6,7 @@ import { tmpdir } from "node:os"; | ||
| 6 | 6 | import { join } from "node:path"; |
| 7 | 7 | import { gunzipSync } from "node:zlib"; |
| 8 | 8 | import WebSocket, { WebSocketServer } from "ws"; |
| 9 | import { FRAME_BURST, MAX_FRAME_BYTES } from "./frame-limit.ts"; | |
| 9 | 10 | import { FeetechBus } from "./bus.ts"; |
| 10 | 11 | import { FakeBus } from "./fake-bus.ts"; |
| 11 | 12 | import { OFFLINE_FRAME } from "./offline-frame.ts"; |
| @@ -357,6 +358,44 @@ describe("relay", () => { | ||
| 357 | 358 | } |
| 358 | 359 | }); |
| 359 | 360 | |
| 361 | test("/follow: an oversized message closes with 1009 and a burst past the allowance never reaches the arm", async () => { | |
| 362 | const received: string[] = []; | |
| 363 | const upstream = createServer(); | |
| 364 | const arms = new WebSocketServer({ server: upstream, path: "/follow" }); | |
| 365 | arms.on("connection", (ws) => ws.on("message", (d) => received.push(String(d)))); | |
| 366 | const port = await listening(upstream); | |
| 367 | try { | |
| 368 | relay = await startRelay({ port: 0, simulation: `http://127.0.0.1:${port}` }); | |
| 369 | const open = async () => { | |
| 370 | const ws = new WebSocket(`ws://127.0.0.1:${relay!.port}/follow`); | |
| 371 | await new Promise((r, j) => (ws.once("open", r), ws.once("error", j))); | |
| 372 | return ws; | |
| 373 | }; | |
| 374 | const big = await open(); | |
| 375 | const closed = new Promise<number>((r) => big.once("close", r)); | |
| 376 | big.send("x".repeat(MAX_FRAME_BYTES + 1)); | |
| 377 | expect(await closed).toBe(1009); | |
| 378 | await sleep(50); | |
| 379 | expect(received).toEqual([]); | |
| 380 | ||
| 381 | // two sockets from one address share one allowance | |
| 382 | const [a, b] = [await open(), await open()]; | |
| 383 | for (let i = 0; i < FRAME_BURST; i++) (i % 2 ? a : b).send(JSON.stringify({ "1": i })); | |
| 384 | for (let i = 0; i < FRAME_BURST; i++) (i % 2 ? a : b).send(JSON.stringify({ "1": 9999 })); | |
| 385 | await sleep(150); | |
| 386 | expect(received.length).toBeGreaterThanOrEqual(FRAME_BURST); | |
| 387 | expect(received.length).toBeLessThan(FRAME_BURST + 10); // the refill in the meantime, not the 40 extra | |
| 388 | expect(relay.droppedFrames).toBeGreaterThan(FRAME_BURST / 2); | |
| 389 | a.close(); | |
| 390 | b.close(); | |
| 391 | } finally { | |
| 392 | for (const c of arms.clients) c.terminate(); | |
| 393 | arms.close(); | |
| 394 | upstream.closeAllConnections(); | |
| 395 | upstream.close(); | |
| 396 | } | |
| 397 | }); | |
| 398 | ||
| 360 | 399 | test("a bad setting is reported and the arm is simulated", async () => { |
| 361 | 400 | relay = await startRelay({ port: 0, simulation: simBase, problem: 'ARM_SERVER "ftp://x" must start with http://' }); |
| 362 | 401 | expect(await link(`http://127.0.0.1:${relay.port}`)).toEqual({ mode: "simulated", host: null, problem: 'ARM_SERVER "ftp://x" must start with http://' }); |
Software/so101/src/relay.ts +20 −3
| @@ -53,6 +53,7 @@ import type { FeedOptions } from "./camera-feed.ts"; | ||
| 53 | 53 | import { CameraError, CameraHub } from "./cameras.ts"; |
| 54 | 54 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 55 | 55 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| 56 | import { FrameRates, frameBytes, MAX_FRAME_BYTES } from "./frame-limit.ts"; | |
| 56 | 57 | import { proxyFor, whyNot } from "./egress.ts"; |
| 57 | 58 | import { CalibrationError } from "./leader-map.js"; |
| 58 | 59 | import type { Lobby } from "./lobby.ts"; |
| @@ -109,6 +110,7 @@ export interface RelayOptions { | ||
| 109 | 110 | export interface RunningRelay { |
| 110 | 111 | readonly port: number; |
| 111 | 112 | link(): Link; |
| 113 | readonly droppedFrames: number; | |
| 112 | 114 | close(): Promise<void>; |
| 113 | 115 | } |
| 114 | 116 | |
| @@ -556,7 +558,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 556 | 558 | }); |
| 557 | 559 | |
| 558 | 560 | // ------------------------------------------------------------------ WebSocket /follow |
| 559 | const wss = new WebSocketServer({ noServer: true }); | |
| 561 | const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_BYTES }); | |
| 560 | 562 | const env = options.env ?? process.env; |
| 561 | 563 | const open = (url: string, timeoutMs: number) => |
| 562 | 564 | new Promise<WebSocket>((resolve, reject) => { |
| @@ -580,11 +582,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 580 | 582 | const sendable = (code: number) => (code >= 1000 && code <= 1003) || (code >= 1007 && code <= 1014) || (code >= 3000 && code <= 4999); |
| 581 | 583 | /** Each open /follow stream and whose it is, so a turn that ends can close it. */ |
| 582 | 584 | const streams = new Map<WebSocket, string>(); |
| 585 | /** One frame allowance per driver (per address when there is no lobby), however many sockets they open. */ | |
| 586 | const rates = new FrameRates(); | |
| 587 | let droppedFrames = 0; | |
| 583 | 588 | /** |
| 584 | 589 | * `base` is the arm server's HTTP side, asked where the joints are so the speed cap starts from there. On the real arm, |
| 585 | 590 | * frames are not passed on while the link reconnects: the stream goes quiet, and the arm server's dead-man holds the arm. |
| 586 | 591 | */ |
| 587 | const bridge = (browser: WebSocket, arm: WebSocket, base: string, driver: string | null) => { | |
| 592 | const bridge = (browser: WebSocket, arm: WebSocket, base: string, driver: string | null, address: string) => { | |
| 588 | 593 | const real = base !== simulation; |
| 589 | 594 | if (driver !== null) { |
| 590 | 595 | streams.set(browser, driver); |
| @@ -611,7 +616,15 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 611 | 616 | .finally(() => (asking = false)); |
| 612 | 617 | }; |
| 613 | 618 | seed(); |
| 619 | const rate = rates.join(driver ?? address); | |
| 620 | browser.on("close", rate.leave); | |
| 614 | 621 | browser.on("message", (data, binary) => { |
| 622 | // Checked after `ws` has buffered the message: Bun ignores `maxPayload`, so this bounds what is used, not what is held in memory. | |
| 623 | if (frameBytes(data) > MAX_FRAME_BYTES) return browser.close(1009, "frame too large"); | |
| 624 | if (!rate.take()) { | |
| 625 | droppedFrames++; | |
| 626 | return; | |
| 627 | } | |
| 615 | 628 | if (driver !== null) { |
| 616 | 629 | // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on. |
| 617 | 630 | if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return; |
| @@ -679,7 +692,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 679 | 692 | } |
| 680 | 693 | } |
| 681 | 694 | if (socket.destroyed) return arm.close(); |
| 682 | wss.handleUpgrade(req, socket, head, (browser) => bridge(browser, arm, base, driver)); | |
| 695 | wss.handleUpgrade(req, socket, head, (browser) => bridge(browser, arm, base, driver, req.socket.remoteAddress ?? "?")); | |
| 683 | 696 | })(); |
| 684 | 697 | }); |
| 685 | 698 | |
| @@ -706,6 +719,10 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 706 | 719 | return { |
| 707 | 720 | port: typeof address === "object" && address ? address.port : options.port ?? 8102, |
| 708 | 721 | link, |
| 722 | /** Frames dropped for coming faster than the allowance. */ | |
| 723 | get droppedFrames() { | |
| 724 | return droppedFrames; | |
| 725 | }, | |
| 709 | 726 | close: () => |
| 710 | 727 | new Promise<void>((resolve) => { |
| 711 | 728 | clearInterval(timer); |
Software/so101/src/server.ts +13 −2
| @@ -51,6 +51,7 @@ import { fileURLToPath } from "node:url"; | ||
| 51 | 51 | import { WebSocketServer, type WebSocket } from "ws"; |
| 52 | 52 | import { browserModule } from "./browser-modules.ts"; |
| 53 | 53 | import type { FeetechBus } from "./bus.ts"; |
| 54 | import { FrameRates, frameBytes, MAX_FRAME_BYTES } from "./frame-limit.ts"; | |
| 54 | 55 | import { ArmModel, gripperTicksToRadians, ticksToRadians, type Violation } from "./kinematics.ts"; |
| 55 | 56 | import { clamp, type Frame } from "./mapping.ts"; |
| 56 | 57 | import { Metrics } from "./metrics.ts"; |
| @@ -897,8 +898,9 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}): | ||
| 897 | 898 | // Frames are merged down to the command rate; each joint's target is pulled inside its bounds, judged by the zone, |
| 898 | 899 | // and handed to the governor, which moves the goals toward it within the speed and acceleration limits. |
| 899 | 900 | // Silence makes the arm hold (the dead-man); a connection that is lost rather than closed makes it park. |
| 900 | const wss = new WebSocketServer({ noServer: true, maxPayload: 4096 }); | |
| 901 | wss.on("connection", (ws: WebSocket) => { | |
| 901 | const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_BYTES }); | |
| 902 | const rates = new FrameRates(); | |
| 903 | wss.on("connection", (ws: WebSocket, req: IncomingMessage) => { | |
| 902 | 904 | if (estop) { |
| 903 | 905 | ws.close(ESTOP_CLOSE, stopped(estop)); |
| 904 | 906 | return; |
| @@ -1108,7 +1110,16 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}): | ||
| 1108 | 1110 | lastTaken = Date.now(); |
| 1109 | 1111 | follow(frame); |
| 1110 | 1112 | }; |
| 1113 | // No identity reaches this server; behind the relay every stream shares the relay's address (and so one allowance). | |
| 1114 | const rate = rates.join(req.socket.remoteAddress ?? "?"); | |
| 1115 | ws.on("close", rate.leave); | |
| 1111 | 1116 | ws.on("message", (data) => { |
| 1117 | // Checked after `ws` has buffered the message: Bun ignores `maxPayload`, so this bounds what is used, not what is held in memory. | |
| 1118 | if (frameBytes(data) > MAX_FRAME_BYTES) return ws.close(1009, "frame too large"); | |
| 1119 | if (!rate.take()) { | |
| 1120 | stats.refused++; | |
| 1121 | return; | |
| 1122 | } | |
| 1112 | 1123 | const frame = parseFrame(String(data)); |
| 1113 | 1124 | if (!frame) return; |
| 1114 | 1125 | clearTimeout(timer); |
new above
offline
The bucket is per connection, so many sockets multiply the allowance. Key it on the driver identity or socket address, or document that only one driver can stream at a time.