Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

Fast on a phone over 4G: first view under 2 s, the camera adapts to the connection, nothing loads until it is needed #69

Merged opened this in answer to

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

vesper/arm:vesper/53-fast-on-a-phone-over-4g-first-view-under → main 4 commits fork

  • Merged. vesper/53-fast-on-a-phone-over-4g-first-view-under was merged into main as 56e9837.

Review

Sign in to review this change.
13 files changed +612 −141
Software/README.md +24 −1
@@ -147,7 +147,7 @@ on a LAN or behind Tailscale.
147147 arm (its own URDF and meshes, drag a joint and the arm follows), a slider and
148148 hold-to-jog buttons per joint that stream targets like a leader does, a
149149 button per saved pose plus save and delete, hold and release, record and
150 play, a stop button (space bar), fullscreen (F), a metrics panel, a settings
150 play, a stop button while a move or a playback runs (space bar), fullscreen (F), a metrics panel, a settings
151151 panel for the safe zone, and a 3D-calibration panel that explains the vision
152152 quest and shows its status. `bench/fake-serve.ts` runs the same server on a
153153 simulated bus for working on the page. The page expects MJPEG streams on
@@ -195,6 +195,29 @@ again every 2 s and the live picture comes back in the same stream, without
195195 a reload. The camera is independent of the arm server: a camera that is down
196196 does not put the arm on the simulation, and the other way round.
197197
198 **On a phone over 4G.** The first load is the page and three small modules,
199 under 50 KB gzipped (a test in `src/server.test.ts` holds it there). three.js,
200 the leader-arm mapping and the 3D-calibration panel (`src/ui/calibration.js`)
201 load only when they're used. The relay writes the link and the camera list
202 into the page (`<script type="application/json" id="boot">`), so the camera
203 starts as soon as the script runs. The arm's limits, settings, poses and
204 model are all requested at once after that. One WebSocket, `/live`, carries
205 the turn, the queue, the chat and the arm's status. The status is pushed only
206 when it changes, so a page whose socket is open doesn't poll `/status`. A new
207 page gets the last 30 chat lines; **Earlier messages** fetches 50 more at a
208 time. The camera follows the connection (`src/ui/camera-plan.js`). On a slow
209 link (round trips over 400 ms, or the browser reporting 3G or a low
210 downlink), the relay holds that viewer's stream to 5 frames a second
211 (`/camera/<id>/stream?fps=1-30`), or 2 when the browser asks to save data.
212 On a crawl (2G, or round trips over a second), the page fetches one snapshot
213 at a time. The **Auto / Saver / Full** button in the top bar overrides this,
214 and the choice is kept in the browser. A hidden page (another tab, a locked
215 phone) fetches no video at all.
216
217 There is one **Stop** on screen: the red emergency stop. A second button
218 appears only while a move, a playback, a recording or a park is running, and
219 says which one it ends (**End playback**, …). It holds the arm where it is.
220
198221 **Cameras are a list the owner edits** on `/admin/camera-list` (name + source, such as
199222 `<arm host>:8080`), kept in `cameras.json` under `FORGE_APP_DATA`; the first
200223 start lists the arm host's camera 1. Each is streamed once from its source
Software/so101/src/camera-feed.test.ts +27 −2
@@ -1,7 +1,7 @@
11 import { afterEach, describe, expect, test } from "bun:test";
22 import { createServer, type Server, type ServerResponse } from "node:http";
33 import { createServer as createTcpServer, type Server as TcpServer } from "node:net";
4 import { CameraFeed, type FeedOptions } from "./camera-feed.ts";
4 import { CameraFeed, parseFps, type FeedOptions } from "./camera-feed.ts";
55 import { OFFLINE_FRAME } from "./offline-frame.ts";
66
77 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
@@ -82,7 +82,7 @@ afterEach(() => {
8282 /** Serve a feed of `source` on a port of its own, as the relay does. */
8383 async function serveFeed(source: string | null, options: FeedOptions = {}) {
8484 feed = new CameraFeed(source, "camera 1", options);
85 front = createServer((req, res) => feed!.serve(req, res));
85 front = createServer((req, res) => feed!.serve(req, res, parseFps(new URL(req.url ?? "/", "http://x").searchParams.get("fps"))));
8686 return `http://127.0.0.1:${await listening(front)}/`;
8787 }
8888
@@ -105,6 +105,31 @@ describe("a camera feed", () => {
105105 await b.cancel();
106106 });
107107
108 test("a viewer on a slow link asks for fewer frames: it gets whole frames at that rate, the others every frame", async () => {
109 camera = fakeCamera(); // a frame every 30 ms, about 33 a second
110 const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`);
111 const full = await watch(url);
112 const slow = await watch(`${url}?fps=5&1729`); // the page adds a cache-buster after it
113 await full.until(hasFrame);
114 await sleep(1000);
115 const count = (w: { seen: Buffer }) => (w.seen.toString("latin1").match(/frame-/g) ?? []).length;
116 await full.until((s) => (s.toString("latin1").match(/frame-/g) ?? []).length >= 20);
117 await slow.until((s) => (s.toString("latin1").match(/frame-/g) ?? []).length >= 4);
118 const n = count(slow);
119 expect(n).toBeLessThanOrEqual(8);
120 expect(count(full)).toBeGreaterThan(3 * n);
121 expect(slow.seen.toString("latin1")).toMatch(/^--armframe\r\n(Content-Type: image\/jpeg\r\n\r\n\xff\xd8frame-\d+\xff\xd9\r\n--armframe\r\n)+/);
122 expect(camera.connections).toBe(1);
123 await full.cancel();
124 await slow.cancel();
125 });
126
127 test("fps is a whole number 1-30; anything else is every frame", () => {
128 expect(parseFps("5")).toBe(5);
129 expect(parseFps("30")).toBe(30);
130 for (const bad of [null, "", "0", "31", "2.5", "-1", "1e1", "5x", "999"]) expect(parseFps(bad)).toBeNull();
131 });
132
108133 test("a viewer who joins later shares the same connection and starts on a whole frame", async () => {
109134 camera = fakeCamera();
110135 const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`);
Software/so101/src/camera-feed.ts +28 −6
@@ -1,7 +1,8 @@
11 /**
22 * One camera, shared: a single upstream MJPEG connection fanned out to every viewer, part by part, as the bytes
33 * arrive. No frame is gathered whole; a viewer who joins mid-frame starts at the next one, and a viewer who cannot
4 * keep up skips frames rather than holding the others back.
4 * keep up skips frames rather than holding the others back. A viewer on a slow link may ask for fewer frames a
5 * second (`fps`): the frames in between are skipped for it alone, whole, so what it gets is always a full picture.
56 *
67 * When there is no source, or it does not answer, or it stops sending, every viewer gets the "camera offline"
78 * picture at once and again every few seconds, while the feed keeps trying the source; when it answers again the
@@ -37,6 +38,20 @@ interface Viewer {
3738 res: ServerResponse;
3839 /** Mid-way through a live part; otherwise it stands right after a boundary. */
3940 inPart: boolean;
41 /** The least time between two frames this viewer is sent, ms; 0 for every frame. */
42 gapMs: number;
43 /** When it was last sent the start of a frame. */
44 sentAt: number;
45 }
46
47 /** The most frames a second a viewer may ask to be held to; above it, it gets every frame. */
48 export const MAX_FPS = 30;
49
50 /** A viewer's `fps` from a query value: a whole number 1-30, or null (every frame) for anything else. */
51 export function parseFps(text: string | null): number | null {
52 if (text === null || !/^\d{1,2}$/.test(text)) return null;
53 const n = Number(text);
54 return n >= 1 && n <= MAX_FPS ? n : null;
4055 }
4156
4257 export class CameraFeed {
@@ -71,8 +86,11 @@ export class CameraFeed {
7186 return this.viewers.size;
7287 }
7388
74 /** Answer one viewer's GET (or HEAD) with the shared stream; it runs until the viewer leaves or the feed closes. */
75 serve(req: IncomingMessage, res: ServerResponse): void {
89 /**
90 * Answer one viewer's GET (or HEAD) with the shared stream; it runs until the viewer leaves or the feed closes.
91 * `fps` holds this viewer to at most that many frames a second.
92 */
93 serve(req: IncomingMessage, res: ServerResponse, fps: number | null = null): void {
7694 res.writeHead(200, {
7795 "content-type": `multipart/x-mixed-replace;boundary=${BOUNDARY}`,
7896 "cache-control": "no-store, no-transform",
@@ -80,7 +98,7 @@ export class CameraFeed {
8098 });
8199 if (req.method === "HEAD" || this.closed) return void res.end();
82100 res.write(`--${BOUNDARY}\r\n`);
83 const viewer: Viewer = { res, inPart: false };
101 const viewer: Viewer = { res, inPart: false, gapMs: fps ? 1000 / fps : 0, sentAt: -Infinity };
84102 this.viewers.add(viewer);
85103 // Bun's node:http may only tell the socket that the viewer hung up, not the response: listen to both.
86104 res.on("close", () => this.leave(viewer));
@@ -192,9 +210,13 @@ export class CameraFeed {
192210 }
193211 const parser = new MjpegParser(boundary, {
194212 start: (type) => {
213 const now = performance.now();
195214 for (const v of this.viewers) {
196 v.inPart = v.res.writableLength <= this.o.maxBacklog;
197 if (v.inPart) this.write(v, head(type));
215 // A frame a little early still counts as on time: a 10 fps source held to 5 sends every other frame, not every third.
216 v.inPart = v.res.writableLength <= this.o.maxBacklog && now - v.sentAt >= v.gapMs * 0.8;
217 if (!v.inPart) continue;
218 v.sentAt = now;
219 this.write(v, head(type));
198220 }
199221 },
200222 data: (bytes) => {
Software/so101/src/lobby.test.ts +52 −1
@@ -10,7 +10,7 @@ import { Chat } from "./chat.ts";
1010 import { FakeBus } from "./fake-bus.ts";
1111 import { sign } from "./identity.ts";
1212 import { Accounts } from "./accounts.ts";
13 import { Lobby } from "./lobby.ts";
13 import { CHAT_FIRST, HISTORY_PAGE, Lobby } from "./lobby.ts";
1414 import { startRelay, type RunningRelay } from "./relay.ts";
1515 import { startServer, type RunningServer } from "./server.ts";
1616 import { Turns } from "./turns.ts";
@@ -277,6 +277,57 @@ describe("lobby", () => {
277277 expect((await late.first((m) => m.type === "chat")).messages).toEqual([]);
278278 });
279279
280 test("chat history is lazy: a new page gets the last lines, anyone asks for older pages, oldest first", async () => {
281 const chat = new Chat();
282 for (let i = 0; i < 100; i++) chat.system(`line ${i}`);
283 const port = await start(undefined, chat);
284 const anon = await join(port);
285 const first = await anon.first((m) => m.type === "chat");
286 expect(first.messages.length).toBe(CHAT_FIRST);
287 expect(first.messages.at(-1).text).toBe("line 99");
288 expect(first.more).toBe(true);
289 anon.send({ type: "history", before: first.messages[0].id });
290 const older = await anon.next((m) => m.type === "history");
291 expect(older.messages.length).toBe(HISTORY_PAGE);
292 expect(older.messages.at(-1).id).toBe(first.messages[0].id - 1);
293 expect(older.more).toBe(true);
294 anon.send({ type: "history", before: older.messages[0].id });
295 const oldest = await anon.next((m) => m.type === "history");
296 expect(oldest.messages.map((m: Msg) => m.text)).toEqual(Array.from({ length: 100 - CHAT_FIRST - HISTORY_PAGE }, (_, i) => `line ${i}`));
297 expect(oldest.more).toBe(false);
298 anon.send({ type: "history", before: "x" });
299 expect((await anon.next((m) => m.type === "error")).error).toContain("history");
300 });
301
302 test("publish reaches every page on /live, signed in or not", async () => {
303 const port = await start();
304 const anon = await join(port);
305 const alice = await join(port, ALICE);
306 await Promise.all([anon.first((m) => m.type === "chat"), alice.first((m) => m.type === "chat")]);
307 expect(lobby!.watching).toBe(2);
308 const heard = [anon, alice].map((c) => c.next((m) => m.type === "status"));
309 lobby!.publish({ type: "status", joints: [] });
310 for (const m of await Promise.all(heard)) expect(m.joints).toEqual([]);
311 });
312
313 test("one socket: the relay pushes the arm's status over /live, once, and again only when it changes", async () => {
314 const port = await start();
315 const anon = await join(port);
316 const first = await anon.first((m) => m.type === "status");
317 expect(first.status.joints.length).toBe(3);
318 expect(first.status.source).toStartWith("simulated");
319 await sleep(700); // three ticks of an arm that sits still
320 expect(anon.heard.filter((m) => m.type === "status").length).toBe(1);
321 const owner = await join(port, OWNER);
322 await owner.first((m) => m.type === "status"); // a page that opens later hears it at once, not at the next change
323 owner.send({ type: "join" });
324 await owner.next(drivingIs("trevato"));
325 const moved = anon.next((m) => m.type === "status" && m.status.joints[0].position !== first.status.joints[0].position);
326 const r = await fetch(`http://127.0.0.1:${port}/move`, { method: "POST", headers: { ...OWNER, "content-type": "application/json" }, body: JSON.stringify({ targets: { 1: first.status.joints[0].position + 100 } }) });
327 expect(r.status).toBe(200);
328 expect((await moved).status.joints[0].position).toBe(first.status.joints[0].position + 100);
329 });
330
280331 test("the driver and any admin may hit the emergency stop; only an admin clears it or changes the settings", async () => {
281332 const port = await start();
282333 const alice = await join(port, ALICE);
Software/so101/src/lobby.ts +47 −4
@@ -21,9 +21,12 @@
2121 * {"type": "ban", "id": "<user id>", "on": true|false} owner: bars their account, name and address; they still watch
2222 * {"type": "words", "words": ["…"]} owner: the chat's word filter; a line with a listed word is refused
2323 * {"type": "pong"} the answer to {"type": "ping"}
24 * {"type": "history", "before": 12} anyone: up to HISTORY_PAGE lines older than that one
2425 * Server → client: hello (who you are), state (the turn, the line, pause, the announcement and the speed cap),
25 * chat (the log), said, unsaid, muted, admin (owners only: the settings and the admin log), crew (owners only:
26 * settings, hidden lines, mute suggestions), banned and words (owners only), ping, error.
26 * chat (the last CHAT_FIRST lines of the log, and whether there are more), history (an older page, the same way),
27 * said, unsaid, muted, admin (owners only: the settings and the admin log), crew (owners only:
28 * settings, hidden lines, mute suggestions), banned and words (owners only), ping, error; and whatever the relay
29 * `publish`es to everyone: the arm's status, so a page needs this one socket for the pose, the chat and the queue.
2730 */
2831
2932 import type { IncomingMessage } from "node:http";
@@ -69,6 +72,10 @@ interface Seat {
6972 }
7073
7174 const MAX_FRAME = 4096;
75 /** Lines of chat a new page is sent: enough to fill a phone's chat, little enough for a slow link. */
76 export const CHAT_FIRST = 30;
77 /** Lines per "history" answer, when someone scrolls up for more. */
78 export const HISTORY_PAGE = 50;
7279 const USER_ID = /^[A-Za-z0-9_-]{1,64}$/;
7380
7481 export class Lobby {
@@ -84,6 +91,7 @@ export class Lobby {
8491 private readonly seats = new Set<Seat>();
8592 private readonly changeListeners = new Set<() => void>();
8693 private readonly endListeners = new Set<(who: Person) => void>();
94 private readonly arriveListeners = new Set<(tell: (text: string) => void) => void>();
8795 private readonly pinger: ReturnType<typeof setInterval>;
8896 private readonly silentMs: number;
8997 private lastActivityState = 0;
@@ -214,6 +222,12 @@ export class Lobby {
214222 return () => this.endListeners.delete(listener);
215223 }
216224
225 /** Called as each page opens the room, with a way to tell that page alone something (JSON text): what was published before it came. */
226 onArrive(listener: (tell: (text: string) => void) => void): () => void {
227 this.arriveListeners.add(listener);
228 return () => this.arriveListeners.delete(listener);
229 }
230
217231 upgrade(req: IncomingMessage, socket: Duplex, head: Buffer): void {
218232 const who = this.who(req);
219233 const from = visitor(req);
@@ -235,7 +249,7 @@ export class Lobby {
235249 if (who) this.turns.connect(who.id);
236250 this.send(seat, this.hello(seat));
237251 this.send(seat, this.state());
238 this.send(seat, { type: "chat", messages: this.chat.messages });
252 this.send(seat, this.chatLog());
239253 if (who?.owner) {
240254 this.send(seat, this.mutedList());
241255 this.send(seat, this.bannedList());
@@ -243,6 +257,7 @@ export class Lobby {
243257 this.send(seat, this.adminState());
244258 this.send(seat, this.crewState());
245259 }
260 for (const listener of this.arriveListeners) listener((text) => seat.ws.readyState === WebSocket.OPEN && seat.ws.send(text));
246261 ws.on("message", (data, binary) => {
247262 seat.heard = Date.now();
248263 if (binary) return;
@@ -272,6 +287,10 @@ export class Lobby {
272287 const m = message as Record<string, unknown>;
273288 const who = seat.who;
274289 if (m.type === "pong") return;
290 if (m.type === "history") {
291 if (!Number.isSafeInteger(m.before)) return this.send(seat, { type: "error", error: "history before which line?" });
292 return this.send(seat, { type: "history", ...this.page(m.before as number, HISTORY_PAGE) });
293 }
275294 if (!who) return this.send(seat, { type: "error", error: "join to take part" });
276295 if (m.type === "leave") return void this.turns.leave(who.id);
277296 if (this.barred(who, seat.from)) return this.send(seat, { type: "error", error: "you are banned: you can still watch" });
@@ -367,7 +386,7 @@ export class Lobby {
367386 const back = Number.isSafeInteger(m.id) ? this.moderator?.restore(m.id as number) : null;
368387 if (!back) return this.send(seat, { type: "error", error: "that line is not in the log" });
369388 this.chat.restore(back);
370 this.broadcast({ type: "chat", messages: this.chat.messages });
389 this.broadcast(this.chatLog());
371390 return;
372391 }
373392 case "dismiss":
@@ -439,6 +458,30 @@ export class Lobby {
439458 }
440459 }
441460
461 /** Up to `count` lines before the one with id `before` (all of them with no `before`), oldest first, and whether older ones remain. */
462 private page(before: number | null, count: number) {
463 const all = this.chat.messages;
464 const end = before === null ? all.length : all.findIndex((m) => m.id >= before);
465 const stop = end < 0 ? all.length : end;
466 const start = Math.max(0, stop - count);
467 return { messages: all.slice(start, stop), more: start > 0 };
468 }
469
470 private chatLog() {
471 return { type: "chat", ...this.page(null, CHAT_FIRST) };
472 }
473
474 /** Send every open page the same thing, serialized once however many watch; a string is taken as JSON already. */
475 publish(body: unknown): void {
476 const text = typeof body === "string" ? body : JSON.stringify(body);
477 for (const s of this.seats) if (s.ws.readyState === WebSocket.OPEN) s.ws.send(text);
478 }
479
480 /** How many pages have the room open. */
481 get watching(): number {
482 return this.seats.size;
483 }
484
442485 private send(seat: Seat, body: unknown): void {
443486 if (seat.ws.readyState === WebSocket.OPEN) seat.ws.send(JSON.stringify(body));
444487 }
Software/so101/src/relay.test.ts +18
@@ -7,6 +7,7 @@ import { join } from "node:path";
77 import { gunzipSync } from "node:zlib";
88 import WebSocket, { WebSocketServer } from "ws";
99 import { FeetechBus } from "./bus.ts";
10 import { CameraHub } from "./cameras.ts";
1011 import { FakeBus } from "./fake-bus.ts";
1112 import { OFFLINE_FRAME } from "./offline-frame.ts";
1213 import { parseRemote, sameOrigin, startRelay, type Remote, type RunningRelay } from "./relay.ts";
@@ -117,6 +118,23 @@ describe("relay", () => {
117118 relay = remote = camera = hang = undefined;
118119 });
119120
121 test("the page carries its first view's facts: which arm, and the cameras, escaped as inert JSON", async () => {
122 const cameras = new CameraHub({ seed: [{ name: "Desk </script><img src=x onerror=alert(1)>", source: "http://127.0.0.1:9/stream" }] });
123 relay = await startRelay({ port: 0, simulation: simBase, cameras });
124 const base = `http://127.0.0.1:${relay.port}`;
125 for (const encoding of ["identity", "gzip"]) {
126 const page = await fetch(`${base}/`, { headers: { "accept-encoding": encoding }, decompress: false } as RequestInit);
127 const html = encoding === "gzip" ? (expect(page.headers.get("content-encoding")).toBe("gzip"), gunzipSync(Buffer.from(await page.arrayBuffer())).toString("utf8")) : await page.text();
128 expect(page.headers.get("cache-control")).toBe("no-store");
129 expect(page.headers.get("content-security-policy")).toContain("frame-ancestors 'none'");
130 const found = /<script type="application\/json" id="boot">(.*?)<\/script>\n<\/head>/.exec(html);
131 expect(found).not.toBeNull();
132 expect(found![1]).not.toContain("<");
133 expect(JSON.parse(found![1]!)).toEqual({ link: { mode: "simulated", host: null, problem: null }, cameras: [{ id: 1, name: "Desk </script><img src=x onerror=alert(1)>", online: null }] }); // not watched yet: unknown
134 }
135 cameras.close();
136 });
137
120138 test("unset: everything is the simulation, and the page says so", async () => {
121139 relay = await startRelay({ port: 0, simulation: simBase });
122140 const base = `http://127.0.0.1:${relay.port}`;
Software/so101/src/relay.ts +66 −9
@@ -21,7 +21,7 @@
2121 *
2222 * The owner's speed cap (src/speed-cap.ts) is applied here too, to moves, poses, grips, playbacks and every stream frame.
2323 *
24 * WS /live the lobby: the turn, the queue, the chat
24 * WS /live the lobby: the turn, the queue, the chat, and the arm's /status pushed when it changes
2525 * POST /join {"name": "…"}: become someone here, no forge account needed (src/accounts.ts); the answer
2626 * sets the HttpOnly session cookie and names you: {"you": {"id", "name", "owner"}}
2727 * GET /admin, POST /admin the owner's page: pause, end a turn, take control, the settings, the log
@@ -31,7 +31,7 @@
3131 * there is one), or "simulated: <why>" ("simulated: offline, <why>" after 60 s down)
3232 * GET /cameras the listed cameras: [{"id", "name", "online"}]
3333 * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts);
34 * the "camera offline" picture when its source is down
34 * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate
3535 * GET /admin/camera-list the owner's page for the camera list
3636 * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds
3737 * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time,
@@ -49,7 +49,7 @@ import { acceptsGzip, GzipCache, worthCompressing } from "./compress.ts";
4949 import { AccountError, sessionCookie, visitor } from "./accounts.ts";
5050 import { AdminError, changeFromForm } from "./admin.ts";
5151 import { ADMIN_CSP, DONE, renderAdmin, renderRefusal, type Done } from "./admin-page.ts";
52 import type { FeedOptions } from "./camera-feed.ts";
52 import { parseFps, type FeedOptions } from "./camera-feed.ts";
5353 import { CameraError, CameraHub } from "./cameras.ts";
5454 import { LOCAL as LOCAL_PERSON } from "./identity.ts";
5555 import { LeaderCalibrations } from "./leader-calibrations.ts";
@@ -102,6 +102,8 @@ export interface RelayOptions {
102102 camera?: FeedOptions;
103103 /** Every driver's leader calibration; without one they are kept in memory. */
104104 leaders?: LeaderCalibrations;
105 /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */
106 statusEveryMs?: number;
105107 /** Where HTTP_PROXY and NO_PROXY are read from; process.env by default. */
106108 env?: Record<string, string | undefined>;
107109 }
@@ -120,6 +122,8 @@ const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url),
120122 const MAX_FORM = 16_384;
121123 /** Served from this process whichever arm is live: they describe the app, not the arm. */
122124 const LOCAL = /^\/(?:index\.html|leader-map\.js)?$|^\/(?:model|vendor|printables|ui|app)(?:\/|$)/;
125 /** The page itself, which the relay writes the first view's facts into. */
126 const PAGE = /^\/(?:index\.html)?$/;
123127 /** The page's static files: what a first visit downloads, and what gzip is for. */
124128 const STATIC = /^\/(?:index\.html)?$|^\/(?:model|vendor|ui)\//;
125129 const LEADER = "/leader/calibration";
@@ -313,16 +317,44 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
313317 * The arm's /status, saying which arm it is: from the link while it is live, the last one it had while reconnecting,
314318 * and from the simulation otherwise.
315319 */
316 const status = async (res: ServerResponse, path: string) => {
320 const statusOf = async (path = "/status"): Promise<{ code: number; body: Record<string, unknown> }> => {
317321 if (onArm()) {
318322 const last = armLink!.last;
319323 const reconnecting = mode() === "reconnecting";
320 if (!last) return json(res, 503, { error: `${RECONNECTING}…`, source: source(), reconnecting });
321 return json(res, 200, { ...last, source: source(), reconnecting, ...(reconnecting ? { seenAt: new Date(armLink!.lastAt).toISOString() } : {}) });
324 if (!last) return { code: 503, body: { error: `${RECONNECTING}…`, source: source(), reconnecting } };
325 return { code: 200, body: { ...last, source: source(), reconnecting, ...(reconnecting ? { seenAt: new Date(armLink!.lastAt).toISOString() } : {}) } };
322326 }
323 const answer = await fetch(simulation + path, { cache: "no-store" });
327 const answer = await fetch(simulation + path, { cache: "no-store", signal: AbortSignal.timeout(readTimeoutMs) });
324328 const body = (await answer.json()) as Record<string, unknown>;
325 return json(res, answer.status, { ...body, source: source() });
329 return { code: answer.status, body: { ...body, source: source() } };
330 };
331 const status = async (res: ServerResponse, path: string) => {
332 const { code, body } = await statusOf(path);
333 return json(res, code, body);
334 };
335
336 /**
337 * The same status pushed over /live, so a page needs one socket for the pose, the chat and the queue rather than
338 * polling: read once per tick however many watch, sent only when it changed. {"type": "status", "status": {...}|null, "error"?}
339 */
340 const statusEveryMs = options.statusEveryMs ?? 200;
341 let pushed = "";
342 let pushing = false;
343 const push = async () => {
344 if (!lobby || pushing) return;
345 if (lobby.watching === 0) return void (pushed = ""); // the next page to open the room hears it at once
346 pushing = true;
347 let text: string;
348 try {
349 const { code, body } = await statusOf();
350 text = JSON.stringify(code === 200 ? { type: "status", status: body } : { type: "status", status: null, error: body.error ?? `status ${code}` });
351 } catch (err) {
352 text = JSON.stringify({ type: "status", status: null, error: reason(err) });
353 } finally {
354 pushing = false;
355 }
356 if (text !== pushed) lobby.publish(text);
357 pushed = text;
326358 };
327359
328360 /** A static answer, gzipped: whole, so the length is known; kept when the browser may cache it too. */
@@ -342,6 +374,25 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
342374 res.end(out);
343375 };
344376
377 /**
378 * The page, with what its first view needs written into it: which arm this is and the cameras. It starts the camera
379 * as soon as its script runs, without asking /link and /cameras first: two round trips fewer on a phone.
380 * The JSON is inert data (type application/json), with every "<" escaped so nothing in a name can close the element.
381 */
382 const page = async (req: IncomingMessage, res: ServerResponse, { response }: Awaited<ReturnType<typeof ask>>) => {
383 const html = await response.text();
384 const boot = JSON.stringify({ link: link(), cameras: cameras.list().map(({ id, name, online }) => ({ id, name, online })) }).replace(/</g, "\\u003c");
385 const body = Buffer.from(response.status === 200 ? html.replace("</head>", `<script type="application/json" id="boot">${boot}</script>\n</head>`) : html);
386 const headers: Record<string, string> = { "content-type": response.headers.get("content-type") ?? "text/html; charset=utf-8", "cache-control": "no-store", vary: "accept-encoding" };
387 if (!acceptsGzip(req.headers)) {
388 res.writeHead(response.status, headers);
389 return res.end(body);
390 }
391 const out = gzipped.gzip(null, body);
392 res.writeHead(response.status, { ...headers, "content-encoding": "gzip", "content-length": String(out.length) });
393 res.end(out);
394 };
395
345396 /** The owner's changes to the camera list. */
346397 const administer = async (req: IncomingMessage, res: ServerResponse, id: number | null) => {
347398 const who = lobby ? lobby.who(req) : null;
@@ -525,7 +576,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
525576 const id = Number(camera[1]);
526577 const source = cameras.source(id);
527578 if (source === null) return json(res, 404, { error: "no such camera" });
528 if (camera[2] === "stream") return cameras.feed(id)!.serve(req, res);
579 if (camera[2] === "stream") return cameras.feed(id)!.serve(req, res, parseFps(url.searchParams.get("fps")));
529580 if (camera[2] === "state") return json(res, 200, { ok: cameras.list().find((c) => c.id === id)?.online ?? null });
530581 const target = source.replace(/\/[^/]*$/, "/snapshot") + url.search;
531582 try {
@@ -535,6 +586,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
535586 }
536587 }
537588
589 if (req.method === "GET" && PAGE.test(url.pathname)) return await page(req, res, await ask(simulation + path, req, undefined, null, false));
538590 if (req.method === "GET" && STATIC.test(url.pathname) && acceptsGzip(req.headers)) return await packed(res, await ask(simulation + path, req, undefined, null, false), path);
539591 // Who stopped or cleared the arm is the relay's to say, never the page's; every other command is held to the admin's speed cap.
540592 const raw = reading || route === "POST /estop" || route === "POST /estop/clear" ? undefined : await readBody(req);
@@ -686,6 +738,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
686738 // ------------------------------------------------------------------ a turn that ends
687739 // Its streams close (the arm server holds the arm when a stream ends) and a move, playback or recording stops.
688740 const settingsChanged = lobby?.onSettingsChange(follow);
741 const arrived = lobby?.onArrive((tell) => (pushed ? tell(pushed) : void push()));
689742 const turnEnded = lobby?.onTurnEnd((who) => {
690743 for (const [browser, id] of streams) if (id === who.id) browser.close(1000, "your turn ended");
691744 const base = remote && onArm() ? remote.http : simulation;
@@ -701,6 +754,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
701754 await (armLink as ArmLink | null)?.ready; // bounded by the first attempt's timeout: the first page already knows which arm it has
702755 const timer = lobby ? setInterval(follow, followEveryMs) : undefined;
703756 timer?.unref?.();
757 const pusher = lobby ? setInterval(() => void push(), statusEveryMs) : undefined;
758 pusher?.unref?.();
704759 await new Promise<void>((resolve) => http.listen(options.port ?? 8102, options.host ?? "127.0.0.1", resolve));
705760 const address = http.address();
706761 return {
@@ -709,8 +764,10 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
709764 close: () =>
710765 new Promise<void>((resolve) => {
711766 clearInterval(timer);
767 clearInterval(pusher);
712768 settingsChanged?.();
713769 turnEnded?.();
770 arrived?.();
714771 armLink?.close();
715772 if (!options.cameras) cameras.close();
716773 for (const c of inFlight) c.abort();
Software/so101/src/server.test.ts +25 −1
@@ -3,6 +3,7 @@ import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
33 import { connect } from "node:net";
44 import { tmpdir } from "node:os";
55 import { join } from "node:path";
6 import { gzipSync } from "node:zlib";
67 import WebSocket from "ws";
78 import { FeetechBus } from "./bus.ts";
89 import { FakeBus } from "./fake-bus.ts";
@@ -134,6 +135,24 @@ describe("viewer page", () => {
134135 expect(html).toContain('$("events").replaceChildren(');
135136 });
136137
138 test("a first visit downloads under 50 KB gzipped: the page and the modules it imports up front; the rest comes when needed", async () => {
139 const page = await (await fetch(`${base}/`)).text();
140 const script = page.slice(page.indexOf('<script type="module">'));
141 const upfront = new Set([...page.matchAll(/<link rel="modulepreload" href="([^"]+)">/g), ...script.matchAll(/^\s*import [^;]*? from "([^"]+)";/gm)].map((m) => m[1]!));
142 expect(upfront.size).toBeGreaterThan(0);
143 let bytes = gzipSync(page, { level: 6 }).length;
144 for (const path of upfront) {
145 const r = await fetch(`${base}${path}`);
146 expect(r.status).toBe(200);
147 bytes += gzipSync(Buffer.from(await r.arrayBuffer()), { level: 6 }).length;
148 }
149 expect(bytes).toBeLessThan(50_000);
150 // three.js, the leader's mapping and the calibration panel are imported where they are used, not up front.
151 for (const lazy of ["three", "/leader-map.js", "calibration.js", "reach.js"]) expect([...upfront].some((p) => p === lazy || p.endsWith(`/${lazy}`))).toBe(false);
152 expect(script).toContain('import("/leader-map.js")');
153 expect(script).toMatch(/import\("\/ui\/[0-9a-f]{12}\/calibration\.js"\)/);
154 });
155
137156 test("names its modules by their content, serves them for good, and serves nothing else from src/ui", async () => {
138157 const page = await (await fetch(`${base}/`)).text();
139158 expect(page).not.toContain("__VERSION__");
@@ -172,7 +191,12 @@ describe("viewer page", () => {
172191 expect(page).toMatch(/\.stick \{[^}]*touch-action: none/);
173192 expect(page).toMatch(/#grips button \{[^}]*min-height: (7[2-9]|[89]\d)px/);
174193 expect(page).toMatch(/#chips button small \{[^}]*white-space: nowrap/); // a long label never wraps and shifts the row
175 expect(page).toMatch(/@media \(max-width: 760px\), \(max-height: 500px\) \{ #stop \{ display: none/); // the E-stop is the one stop on a phone
194 // One "Stop": the E-stop. The other button starts hidden, comes up only while a move, a playback, a recording or a park
195 // runs (which the phone had no other way to end), and is named for what it ends.
196 expect(page).toContain('<button id="stop" class="danger" hidden');
197 expect(page).toContain('const ENDS_LABEL = { move: "End move", play: "End playback", record: "End recording", park: "End park" };');
198 expect(page).toContain('$("stop").hidden = !label || !(mayDrive() || isOwner());');
199 expect(page.match(/>Stop</g)?.length).toBe(1);
176200 });
177201 });
178202
Software/so101/src/ui/calibration.js added +109
@@ -0,0 +1,109 @@
1 // @ts-check
2 /**
3 * The 3D calibration panel: what to print, what is solved, and the steps on the computer that runs the vision kit.
4 * Only admins open it, so it is not part of the page's first load: the page imports this module on the first click.
5 * The words are fixed; everything that comes from the server goes in as text nodes.
6 */
7
8 const ARTICLE = `
9 <article>
10 <h2>3D calibration: teach the arm where the camera is</h2>
11 <p>The camera sees a marker. The arm needs that as a position in its own space. Two calibrations give it that.</p>
12 <ol>
13 <li><b>Camera.</b> The lens geometry: focal length and distortion. Found from 15 views of the printed <em>ChArUco board</em>: a checkerboard with a small marker in each white square. Download it below and print it at 100 %. Output: <code>camera.json</code>.</li>
14 <li><b>Hand-eye.</b> The arm holds a marker in 6 to 8 poses. The camera says where the marker is. The arm's model says where the gripper is. The solver finds the camera's position relative to the arm's base. Output: <code>handeye.json</code>.</li>
15 </ol>
16 <p>After both, a cube the camera sees becomes a target the arm can reach. With two cameras, each one gets its own pair of calibrations (<code>camera2</code>, <code>handeye2</code>); two views then give a full orientation.</p>
17 <h3>Print these</h3>
18 <p>Three sheets, at 100 % scale on matte paper. Check with a ruler: a cube marker's black square is 40 mm, a board square 30 mm. Tape the six cube markers on a 40 to 50 mm block, and the gripper marker flat on the outside of the fixed jaw.</p>
19 <div class="row" id="printables">loading…</div>
20 <h3>Status</h3>
21 <div class="status" id="calStatus"></div>
22 <h3>Steps, on the Mac</h3>
23 <pre id="calSteps"></pre>
24 <p>For the poses: press <b>Release</b>, move the arm by hand, press <b>Save pose</b>, and uncheck <em>with gripper</em>. The hand-eye poses need different tilts and rolls of the gripper, with the marker facing the camera.</p>
25 <h3>References</h3>
26 <ul class="refs">
27 <li>Camera calibration: Z. Zhang, <em>A Flexible New Technique for Camera Calibration</em>, IEEE TPAMI 2000, <a href="https://doi.org/10.1109/34.888718" target="_blank" rel="noopener">doi:10.1109/34.888718</a>. OpenCV tutorial: <a href="https://docs.opencv.org/4.x/da/d13/tutorial_aruco_calibration.html" target="_blank" rel="noopener">Calibration with ArUco and ChArUco</a>.</li>
28 <li>ArUco markers and ChArUco boards: S. Garrido-Jurado et al., <em>Automatic generation and detection of highly reliable fiducial markers under occlusion</em>, Pattern Recognition 2014, <a href="https://doi.org/10.1016/j.patcog.2014.01.005" target="_blank" rel="noopener">doi:10.1016/j.patcog.2014.01.005</a>.</li>
29 <li>Pose of one square marker: T. Collins, A. Bartoli, <em>Infinitesimal Plane-Based Pose Estimation</em>, IJCV 2014, <a href="https://doi.org/10.1007/s11263-014-0725-5" target="_blank" rel="noopener">doi:10.1007/s11263-014-0725-5</a> (OpenCV's <code>SOLVEPNP_IPPE_SQUARE</code>).</li>
30 <li>Hand-eye calibration: Tsai &amp; Lenz 1989 <a href="https://doi.org/10.1109/70.34770" target="_blank" rel="noopener">doi:10.1109/70.34770</a>; Park &amp; Martin 1994 <a href="https://doi.org/10.1109/70.326576" target="_blank" rel="noopener">doi:10.1109/70.326576</a>; Daniilidis 1999 <a href="https://doi.org/10.1177/02783649922066213" target="_blank" rel="noopener">doi:10.1177/02783649922066213</a>.</li>
31 <li>Robot-world and hand-eye in one solve, what this kit uses: M. Shah 2013, <a href="https://doi.org/10.1115/1.4024473" target="_blank" rel="noopener">doi:10.1115/1.4024473</a>; A. Li, L. Wang, D. Wu 2010. OpenCV: <a href="https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html" target="_blank" rel="noopener"><code>calibrateRobotWorldHandEye</code></a>.</li>
32 <li>Two cameras, a full orientation: Hartley &amp; Zisserman, <em>Multiple View Geometry in Computer Vision</em>, 2nd ed. 2004 (stereo calibration, triangulation).</li>
33 </ul>
34 <h3>About</h3>
35 <p>This page drives an <a href="https://github.com/TheRobotStudio/SO-ARM100" target="_blank" rel="noopener">SO-101</a>, the open robot arm by The Robot Studio, with the calibration conventions of <a href="https://huggingface.co/docs/lerobot/so101" target="_blank" rel="noopener">Hugging Face LeRobot</a>. The 3D view uses the arm's own URDF through <a href="https://github.com/gkjohnson/urdf-loaders" target="_blank" rel="noopener">urdf-loader</a> and three.js. The code, the build notes and the quest are in the repository's <code>Software/</code> folder.</p>
36 </article>`;
37
38 /** The page's document: this module only runs there (the type check has no DOM, as for src/ui/drive.js). @type {any} */
39 const doc = /** @type {any} */ (globalThis).document;
40
41 /** @param {string} tag @param {Record<string, unknown>} [props] @param {...any} children @returns {any} */
42 const el = (tag, props = {}, ...children) => { const e = Object.assign(doc.createElement(tag), props); e.append(...children); return e; };
43
44 /** The panel, built once and kept. @returns {any} */
45 function panel() {
46 const found = doc.getElementById("panel");
47 if (found) return found;
48 const close = el("button", { className: "close", textContent: "Close" });
49 const box = el("div", { id: "panel", className: "panel", hidden: true }, close);
50 box.insertAdjacentHTML("beforeend", ARTICLE); // fixed markup from this file, no data in it
51 close.onclick = () => (box.hidden = true);
52 (doc.getElementById("shell") ?? doc.body).append(box);
53 return box;
54 }
55
56 /**
57 * Open the panel and fill it from the arm.
58 * @param {{ api: (path: string) => Promise<any>, host: string, cameras: string[] }} page
59 */
60 export async function openCalibration({ api, host, cameras }) {
61 const box = panel();
62 box.hidden = false;
63 const $ = (/** @type {string} */ id) => box.querySelector(`#${id}`);
64 $("calSteps").textContent = steps(host, cameras[0] ?? null).join("\n");
65 const [sheets, cal, saved] = await Promise.all([api("/printables"), api("/calibration"), api("/poses")]);
66 $("printables").replaceChildren(...(Array.isArray(sheets) && sheets.length
67 ? sheets.map((/** @type {any} */ s) => el("a", { href: String(s.url), download: String(s.file) }, el("button", { className: "go", textContent: `⬇ ${String(s.file).replace(/_/g, " ").replace(/\.png$/, "")}` })))
68 : [el("span", { textContent: "no printables on this server: copy Software/vision/printables next to the package" })]));
69 $("calStatus").replaceChildren(...statusRows(cal, saved, cameras.length).flatMap(([what, done, text]) => [el("b", { textContent: what }), el("span", {}, el("span", { className: done ? "ok" : "no", textContent: text }))]));
70 }
71
72 /** The commands, with this arm's address and its first camera filled in. @param {string} host @param {string | null} camera */
73 export function steps(host, camera) {
74 return [
75 `export SO101_ARM=http://${host}`,
76 `export SO101_CAMERA=${camera ?? "<the camera's stream URL>"}`,
77 `just vision-calibrate # hold the printed board in front of the camera, tilt it`,
78 `# save poses from this page: home, drop, he1 … he8 (without gripper)`,
79 `just vision-handeye he1,he2,he3,he4,he5,he6,he7,he8`,
80 `just vision-rehearse # sees the cube, plans the joints, moves nothing`,
81 `just vision-pick`,
82 ``,
83 `# a second camera: SO101_CAMERA=<its stream URL> and --publish-as camera2 / handeye2`,
84 ];
85 }
86
87 /**
88 * What is done and what is missing, one row each: [what, done, how it stands].
89 * @param {Record<string, any>} cal @param {Record<string, { ids: number[] }>} saved @param {number} cameraCount
90 * @returns {Array<[string, boolean, string]>}
91 */
92 export function statusRows(cal, saved, cameraCount) {
93 const he = Object.keys(saved).filter((n) => /^he\d+$/.test(n) && !saved[n]?.ids.includes(6)).length;
94 /** @returns {[boolean, string]} */
95 const armOnly = (/** @type {string} */ name) => !saved[name] ? [false, "missing"] : saved[name].ids.includes(6) ? [false, "saved with the gripper: save it again without"] : [true, "saved"];
96 /** @returns {[boolean, string]} */
97 const camLine = (/** @type {any} */ c) => c ? [true, `calibrated ${String(c.saved ?? "").slice(0, 10)} · ${c.width}×${c.height} · ${Number(c.rms).toFixed(2)} px error`] : [false, "not yet"];
98 const mean = (/** @type {unknown} */ r) => Array.isArray(r) && r.length ? (r.reduce((a, b) => a + Number(b), 0) / r.length).toFixed(1) : "?";
99 /** @returns {[boolean, string]} */
100 const heLine = (/** @type {any} */ h) => h ? [true, `solved ${String(h.saved ?? "").slice(0, 10)} · ${h.poses?.length ?? "?"} poses · mean ${mean(h.residuals_mm)} mm`] : [false, "not yet"];
101 /** @type {Array<[string, [boolean, string]]>} */
102 const rows = [
103 ["camera 1", camLine(cal.camera)], ["hand-eye 1", heLine(cal.handeye)],
104 ...(cameraCount < 2 ? [] : /** @type {Array<[string, [boolean, string]]>} */ ([["camera 2", camLine(cal.camera2)], ["hand-eye 2", heLine(cal.handeye2)]])),
105 ["pose home", armOnly("home")], ["pose drop", armOnly("drop")],
106 ["poses he1…", [he >= 6, he >= 6 ? `${he} saved` : `${he} of 6 to 8 saved (without the gripper)`]],
107 ];
108 return rows.map(([what, [done, text]]) => [what, done, text]);
109 }
Software/so101/src/ui/calibration.test.ts added +23
@@ -0,0 +1,23 @@
1 import { describe, expect, test } from "bun:test";
2 import { statusRows, steps } from "./calibration.js";
3
4 describe("the calibration panel", () => {
5 test("the steps carry this arm's address and its first camera, or say what to put there", () => {
6 expect(steps("arm.example:8102", "https://arm.example/camera/1/stream").slice(0, 2)).toEqual(["export SO101_ARM=http://arm.example:8102", "export SO101_CAMERA=https://arm.example/camera/1/stream"]);
7 expect(steps("h", null)[1]).toBe("export SO101_CAMERA=<the camera's stream URL>");
8 });
9
10 test("says what is done and what is missing; a hand-eye pose saved with the gripper does not count", () => {
11 const he = Object.fromEntries([1, 2, 3, 4, 5].map((n) => [`he${n}`, { ids: [1, 2, 3, 4, 5] }]));
12 const rows = statusRows({ camera: { saved: "2026-10-01T10:00:00Z", width: 1280, height: 720, rms: 0.4123 }, handeye: null }, { ...he, he6: { ids: [1, 2, 3, 4, 5, 6] }, home: { ids: [1, 2, 3, 4, 5, 6] } }, 1);
13 expect(rows).toEqual([
14 ["camera 1", true, "calibrated 2026-10-01 · 1280×720 · 0.41 px error"],
15 ["hand-eye 1", false, "not yet"],
16 ["pose home", false, "saved with the gripper: save it again without"],
17 ["pose drop", false, "missing"],
18 ["poses he1…", false, "5 of 6 to 8 saved (without the gripper)"],
19 ]);
20 expect(statusRows({ handeye: { saved: "2026-10-02", poses: [1, 2, 3], residuals_mm: [1, 2] } }, {}, 2).map((r) => r[0])).toEqual(["camera 1", "hand-eye 1", "camera 2", "hand-eye 2", "pose home", "pose drop", "poses he1…"]);
21 expect(statusRows({ handeye: { saved: "2026-10-02", poses: [1, 2, 3], residuals_mm: [1, 2] } }, {}, 1)[1]).toEqual(["hand-eye 1", true, "solved 2026-10-02 · 3 poses · mean 1.5 mm"]);
22 });
23 });
Software/so101/src/ui/camera-plan.js +37 −13
@@ -1,30 +1,54 @@
11 /**
2 * How the page fetches a camera on the connection it has: the full MJPEG stream on a good link, or one
3 * snapshot at a time, further apart the slower the link. Plain ESM, served at /ui/ and imported by the tests.
2 * How the page fetches a camera on the connection it has: the full MJPEG stream on a good link; on a slow one the
3 * stream held to a few frames a second where the relay can do that (`/camera/<id>/stream?fps=`), and one snapshot
4 * at a time where it cannot or the link is very slow. Plain ESM, served at /ui/ and imported by the tests.
45 *
5 * The signals are the browser's own estimate where it gives one (navigator.connection: Chrome on Android)
6 * and the round trips of the page's /status polls, which every browser has.
6 * The signals are the viewer's own choice (the quality button: auto, saver, full), the browser's estimate where it
7 * gives one (navigator.connection: Chrome on Android) and the round trips the page measures, which every browser has.
78 */
89
910 /** Round trips above this mean the stream would queue up behind itself. */
1011 export const SLOW_RTT_MS = 400;
1112 /** And below this the stream comes back. The gap keeps the plan from flapping. */
1213 export const FAST_RTT_MS = 200;
14 /** Round trips above this, or a 2G link, get snapshots even where the stream could be held to a lower rate. */
15 export const CRAWL_RTT_MS = 1000;
16 /** Frames a second of a held stream: on a slow link, and when the viewer or the browser asks to save data. */
17 export const SLOW_FPS = 5;
18 export const SAVER_FPS = 2;
19
20 /** The quality choices, in the order the button steps through them. */
21 export const QUALITIES = /** @type {const} */ (["auto", "saver", "full"]);
1322
1423 /**
24 * @typedef {{ mode: "stream", fps?: number } | { mode: "snapshot", everyMs: number }} Plan
1525 * @param {{ effectiveType?: string, saveData?: boolean, downlink?: number } | null | undefined} net
16 * @param {number[]} rtts recent /status round trips, milliseconds
17 * @param {"stream" | "snapshot"} [current]
18 * @returns {{ mode: "stream" } | { mode: "snapshot", everyMs: number }}
26 * @param {number[]} rtts recent round trips, milliseconds
27 * @param {"stream" | "snapshot" | Plan} [current] the plan in force (or its mode): once held back, the link must be clearly fast to come back
28 * @param {{ quality?: string, throttle?: boolean }} [options] the viewer's choice; whether the stream takes ?fps= (through the relay)
29 * @returns {Plan}
1930 */
20 export function cameraPlan(net, rtts, current = "stream") {
31 export function cameraPlan(net, rtts, current = "stream", options = {}) {
32 const quality = QUALITIES.includes(/** @type {any} */ (options.quality)) ? options.quality : "auto";
33 const throttle = options.throttle === true;
34 if (quality === "full") return { mode: "stream" };
2135 const rtt = median(rtts);
22 const told = net?.saveData === true || net?.effectiveType === "slow-2g" || net?.effectiveType === "2g" || net?.effectiveType === "3g" || (typeof net?.downlink === "number" && net.downlink > 0 && net.downlink < 1.5);
23 const slow = told || (rtt !== null && rtt > (current === "snapshot" ? FAST_RTT_MS : SLOW_RTT_MS));
36 if (quality === "saver") return throttle ? { mode: "stream", fps: SAVER_FPS } : snapshots(Math.max(rtt ?? 0, 125));
37 const crawling = net?.effectiveType === "slow-2g" || net?.effectiveType === "2g" || (rtt !== null && rtt > CRAWL_RTT_MS);
38 const told = net?.saveData === true || net?.effectiveType === "3g" || (typeof net?.downlink === "number" && net.downlink > 0 && net.downlink < 1.5);
39 const held = typeof current === "string" ? current !== "stream" : current.mode !== "stream" || current.fps !== undefined;
40 const slow = crawling || told || (rtt !== null && rtt > (held ? FAST_RTT_MS : SLOW_RTT_MS));
2441 if (!slow) return { mode: "stream" };
25 // A frame every four round trips, between 4 and 0.5 frames a second.
26 const everyMs = Math.round(Math.max(250, Math.min(2000, 4 * (rtt ?? SLOW_RTT_MS))));
27 return { mode: "snapshot", everyMs };
42 if (throttle && !crawling) return { mode: "stream", fps: net?.saveData === true ? SAVER_FPS : SLOW_FPS };
43 return snapshots(rtt ?? SLOW_RTT_MS);
44 }
45
46 /** A frame every four round trips, between 4 and 0.5 frames a second. @param {number} rtt @returns {Plan} */
47 const snapshots = (rtt) => ({ mode: "snapshot", everyMs: Math.round(Math.max(250, Math.min(2000, 4 * rtt))) });
48
49 /** The same plan, so nothing need restart. @param {Plan} a @param {Plan} b */
50 export function samePlan(a, b) {
51 return a.mode === b.mode && (a.mode === "stream" ? a.fps === /** @type {any} */ (b).fps : a.everyMs === /** @type {any} */ (b).everyMs);
2852 }
2953
3054 /** @param {number[]} values */
Software/so101/src/ui/camera-plan.test.ts +36 −1
@@ -1,5 +1,5 @@
11 import { describe, expect, test } from "bun:test";
2 import { cameraPlan } from "./camera-plan.js";
2 import { cameraPlan, samePlan, SAVER_FPS, SLOW_FPS } from "./camera-plan.js";
33
44 describe("the camera plan", () => {
55 test("a good link, or too little to judge, gets the full stream", () => {
@@ -25,3 +25,38 @@ describe("the camera plan", () => {
2525 expect(cameraPlan(null, [120, 150, 130], "snapshot").mode).toBe("stream");
2626 });
2727 });
28
29 describe("the camera plan through the relay, which can hold a stream to fewer frames", () => {
30 const relay = { throttle: true };
31 test("a slow link keeps the stream, at a few frames a second; saving data, fewer still", () => {
32 expect(cameraPlan(null, [500, 520, 480], "stream", relay)).toEqual({ mode: "stream", fps: SLOW_FPS });
33 expect(cameraPlan({ effectiveType: "3g" }, [], "stream", relay)).toEqual({ mode: "stream", fps: SLOW_FPS });
34 expect(cameraPlan({ saveData: true }, [30, 30, 30], "stream", relay)).toEqual({ mode: "stream", fps: SAVER_FPS });
35 expect(cameraPlan({ effectiveType: "4g" }, [40, 50, 60], "stream", relay)).toEqual({ mode: "stream" });
36 });
37
38 test("a crawl (2G, or round trips over a second) still gets snapshots", () => {
39 expect(cameraPlan({ effectiveType: "2g" }, [], "stream", relay).mode).toBe("snapshot");
40 expect(cameraPlan(null, [1500, 1600, 1400], "stream", relay)).toEqual({ mode: "snapshot", everyMs: 2000 });
41 });
42
43 test("a held stream comes back to every frame only once the link is clearly fast", () => {
44 const held = { mode: "stream", fps: SLOW_FPS } as const;
45 expect(cameraPlan(null, [300, 300, 300], held, relay)).toEqual(held);
46 expect(cameraPlan(null, [120, 150, 130], held, relay)).toEqual({ mode: "stream" });
47 });
48
49 test("the viewer's choice wins: full is every frame on any link, saver is the fewest", () => {
50 expect(cameraPlan({ effectiveType: "2g", saveData: true }, [2000, 2000, 2000], "stream", { ...relay, quality: "full" })).toEqual({ mode: "stream" });
51 expect(cameraPlan({ effectiveType: "4g" }, [20, 20, 20], "stream", { ...relay, quality: "saver" })).toEqual({ mode: "stream", fps: SAVER_FPS });
52 expect(cameraPlan(null, [], "stream", { quality: "saver" }).mode).toBe("snapshot"); // straight to an arm server: no ?fps=
53 expect(cameraPlan(null, [500, 500, 500], "stream", { ...relay, quality: "bogus" })).toEqual({ mode: "stream", fps: SLOW_FPS }); // unknown is auto
54 });
55
56 test("samePlan tells whether anything need restart", () => {
57 expect(samePlan({ mode: "stream" }, { mode: "stream" })).toBe(true);
58 expect(samePlan({ mode: "stream" }, { mode: "stream", fps: 5 })).toBe(false);
59 expect(samePlan({ mode: "snapshot", everyMs: 500 }, { mode: "snapshot", everyMs: 500 })).toBe(true);
60 expect(samePlan({ mode: "snapshot", everyMs: 500 }, { mode: "snapshot", everyMs: 750 })).toBe(false);
61 });
62 });
Software/so101/src/viewer.html +120 −103
@@ -20,7 +20,6 @@
2020 <link rel="modulepreload" href="/ui/__VERSION__/camera-plan.js">
2121 <link rel="modulepreload" href="/ui/__VERSION__/drive.js">
2222 <link rel="modulepreload" href="/ui/__VERSION__/frames.js">
23 <link rel="modulepreload" href="/leader-map.js">
2423 <style>
2524 /* Phone portrait first: the view on top, a status strip with the E-stop under it, a sheet of tabs below that.
2625 Landscape on a phone: the view fills the screen and the strip and the sheet float over it.
@@ -178,6 +177,7 @@
178177 #oneName { font-size: 20px; font-weight: 700; } #onePos { font-size: 28px; font-weight: 650; font-variant-numeric: tabular-nums; }
179178 #oneHint { color: var(--dim); font-size: 12px; }
180179 #lag { color: #f2cc60; font-variant-numeric: tabular-nums; }
180 #chat .earlier { display: block; margin: 2px auto 8px; font-size: 13px; }
181181 /* on a phone on its side: two thumb sticks over the picture, the gripper between them */
182182 #sticks { display: none; position: absolute; inset: 0; z-index: 6; pointer-events: none; --stick: min(40vh, 164px); }
183183 #sticks > * { pointer-events: auto; }
@@ -270,8 +270,7 @@
270270 #pad { width: min(30vh, 150px); }
271271 }
272272 @media (max-width: 640px) and (orientation: portrait) { #touch { display: grid; } #reach, #joints { display: none !important; } }
273 /* One stop on a phone: the E-stop. Letting go stops the joints; Hold in Controls ends a move or a playback. */
274 @media (max-width: 760px), (max-height: 500px) { #stop { display: none !important; } }
273 /* One Stop: the E-stop. The other button appears only while a move, a playback, a recording or a park runs, and says which it ends. */
275274 @media (orientation: landscape) and (max-height: 500px) {
276275 #shell.driving.bare #sticks { display: block; }
277276 #shell.driving.bare #top { top: 6px; bottom: auto; left: auto; right: calc(6px + var(--safe-r)); max-width: 42%; justify-content: flex-end; }
@@ -291,6 +290,7 @@
291290 <div id="top">
292291 <span class="seg" role="group" aria-label="Views"><button id="v3" aria-pressed="false">3D</button></span>
293292 <span class="info"><span id="link" hidden></span><span id="live" class="label"></span><span class="dot" id="dot"></span><span id="state">…</span><span id="lag" hidden></span></span>
293 <button id="quality" type="button" hidden title="Video quality: Auto follows the connection, Saver sends 2 frames a second, Full every frame">Auto</button>
294294 <button id="fs" title="F">Fullscreen</button>
295295 </div>
296296 <div id="driverTag" aria-hidden="true"></div>
@@ -309,7 +309,7 @@
309309 <div class="who"><div id="driverLine" class="none">Connecting…</div><div id="clock" aria-live="off"></div><div id="capLine"></div></div>
310310 <div id="turnActions"></div>
311311 <button id="sheetToggle" hidden aria-controls="sheet" aria-expanded="true">Hide</button>
312 <button id="stop" class="danger" hidden title="End what the arm is doing and hold (space)">Stop</button>
312 <button id="stop" class="danger" hidden title="End what the arm is doing and hold (space)">End move</button>
313313 <button id="estop" class="estop" title="Emergency stop (Esc): halt, hold with torque, until an admin clears it" aria-label="Emergency stop">Stop</button>
314314 </section>
315315
@@ -448,38 +448,6 @@
448448 </article>
449449 </div>
450450
451 <div id="panel" class="panel" hidden>
452 <button class="close" data-close="panel">Close</button>
453 <article>
454 <h2>3D calibration: teach the arm where the camera is</h2>
455 <p>The camera sees a marker. The arm needs that as a position in its own space. Two calibrations give it that.</p>
456 <ol>
457 <li><b>Camera.</b> The lens geometry: focal length and distortion. Found from 15 views of the printed <em>ChArUco board</em>: a checkerboard with a small marker in each white square. Download it below and print it at 100 %. Output: <code>camera.json</code>.</li>
458 <li><b>Hand-eye.</b> The arm holds a marker in 6 to 8 poses. The camera says where the marker is. The arm's model says where the gripper is. The solver finds the camera's position relative to the arm's base. Output: <code>handeye.json</code>.</li>
459 </ol>
460 <p>After both, a cube the camera sees becomes a target the arm can reach. With two cameras, each one gets its own pair of calibrations (<code>camera2</code>, <code>handeye2</code>); two views then give a full orientation.</p>
461 <h3>Print these</h3>
462 <p>Three sheets, at 100 % scale on matte paper. Check with a ruler: a cube marker's black square is 40 mm, a board square 30 mm. Tape the six cube markers on a 40 to 50 mm block, and the gripper marker flat on the outside of the fixed jaw.</p>
463 <div class="row" id="printables">loading…</div>
464 <h3>Status</h3>
465 <div class="status" id="calStatus"></div>
466 <h3>Steps, on the Mac</h3>
467 <pre id="calSteps"></pre>
468 <p>For the poses: press <b>Release</b>, move the arm by hand, press <b>Save pose</b>, and uncheck <em>with gripper</em>. The hand-eye poses need different tilts and rolls of the gripper, with the marker facing the camera.</p>
469 <h3>References</h3>
470 <ul class="refs">
471 <li>Camera calibration: Z. Zhang, <em>A Flexible New Technique for Camera Calibration</em>, IEEE TPAMI 2000, <a href="https://doi.org/10.1109/34.888718" target="_blank" rel="noopener">doi:10.1109/34.888718</a>. OpenCV tutorial: <a href="https://docs.opencv.org/4.x/da/d13/tutorial_aruco_calibration.html" target="_blank" rel="noopener">Calibration with ArUco and ChArUco</a>.</li>
472 <li>ArUco markers and ChArUco boards: S. Garrido-Jurado et al., <em>Automatic generation and detection of highly reliable fiducial markers under occlusion</em>, Pattern Recognition 2014, <a href="https://doi.org/10.1016/j.patcog.2014.01.005" target="_blank" rel="noopener">doi:10.1016/j.patcog.2014.01.005</a>.</li>
473 <li>Pose of one square marker: T. Collins, A. Bartoli, <em>Infinitesimal Plane-Based Pose Estimation</em>, IJCV 2014, <a href="https://doi.org/10.1007/s11263-014-0725-5" target="_blank" rel="noopener">doi:10.1007/s11263-014-0725-5</a> (OpenCV's <code>SOLVEPNP_IPPE_SQUARE</code>).</li>
474 <li>Hand-eye calibration: Tsai &amp; Lenz 1989 <a href="https://doi.org/10.1109/70.34770" target="_blank" rel="noopener">doi:10.1109/70.34770</a>; Park &amp; Martin 1994 <a href="https://doi.org/10.1109/70.326576" target="_blank" rel="noopener">doi:10.1109/70.326576</a>; Daniilidis 1999 <a href="https://doi.org/10.1177/02783649922066213" target="_blank" rel="noopener">doi:10.1177/02783649922066213</a>.</li>
475 <li>Robot-world and hand-eye in one solve, what this kit uses: M. Shah 2013, <a href="https://doi.org/10.1115/1.4024473" target="_blank" rel="noopener">doi:10.1115/1.4024473</a>; A. Li, L. Wang, D. Wu 2010. OpenCV: <a href="https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html" target="_blank" rel="noopener"><code>calibrateRobotWorldHandEye</code></a>.</li>
476 <li>Two cameras, a full orientation: Hartley &amp; Zisserman, <em>Multiple View Geometry in Computer Vision</em>, 2nd ed. 2004 (stereo calibration, triangulation).</li>
477 </ul>
478 <h3>About</h3>
479 <p>This page drives an <a href="https://github.com/TheRobotStudio/SO-ARM100" target="_blank" rel="noopener">SO-101</a>, the open robot arm by The Robot Studio, with the calibration conventions of <a href="https://huggingface.co/docs/lerobot/so101" target="_blank" rel="noopener">Hugging Face LeRobot</a>. The 3D view uses the arm's own URDF through <a href="https://github.com/gkjohnson/urdf-loaders" target="_blank" rel="noopener">urdf-loader</a> and three.js. The code, the build notes and the quest are in the repository's <code>Software/</code> folder.</p>
480 </article>
481 </div>
482
483451 <dialog id="saveDlg">
484452 <form method="dialog">
485453 <h3>Save the current pose</h3>
@@ -527,10 +495,9 @@
527495 </dialog>
528496 </div>
529497 <script type="module">
530 import { cameraPlan } from "/ui/__VERSION__/camera-plan.js";
498 import { cameraPlan, QUALITIES, samePlan } from "/ui/__VERSION__/camera-plan.js";
531499 import { bindSurface, canDrive, Driver, shape, UP } from "/ui/__VERSION__/drive.js";
532500 import { Frames } from "/ui/__VERSION__/frames.js";
533 import { mapLeaderFrame, parseCalibration } from "/leader-map.js";
534501
535502 const $ = (id) => document.getElementById(id);
536503 const q = new URLSearchParams(location.search);
@@ -587,7 +554,9 @@
587554
588555 // ---------------------------------------------------------------- which arm
589556 // Behind the hosted app's relay (/link answers), the arm is a live `so101 serve` or the simulation, and the cameras come through the relay.
590 const relayed = await fetch("/link", { cache: "no-store" }).then((r) => r.ok, () => false);
557 // The relay writes the link and the camera list into the page (#boot), so the first view needs no round trip to start.
558 const boot = (() => { try { return JSON.parse($("boot")?.textContent ?? "null"); } catch { return null; } })();
559 const relayed = boot !== null || await fetch("/link", { cache: "no-store" }).then((r) => r.ok, () => false);
591560 // A real arm that drops is "reconnecting": the same arm, its last pose kept, driving paused. Only after a minute down
592561 // does the relay hand the page the simulation, and then it says OFFLINE in red.
593562 let linkMode = null, linkNow = null;
@@ -615,14 +584,28 @@
615584 // The owner lists the cameras (/admin/camera-list); this page polls the list, so a camera added or removed shows within seconds.
616585 // Which views are on is this viewer's own choice, kept in this browser: one at a time on a phone, several side by side on a wider screen.
617586 // On a good link a plain <img> stream, which every browser plays; a canvas comparison once a second notices a frozen
618 // picture and reconnects. On a slow one (src/ui/camera-plan.js) one snapshot at a time, further apart the slower it is.
587 // picture and reconnects. On a slow one (src/ui/camera-plan.js) the relay holds the stream to a few frames a second,
588 // and on a crawl one snapshot at a time. The quality button overrides it. A hidden page fetches no video at all.
619589 const cams = [];
620590 const direct = (n) => q.get(`stream${n}`) || q.get(n === 1 ? "stream" : "") || `${location.protocol}//${location.hostname}:${8079 + n}/stream`;
621591 const canvas = document.createElement("canvas"); canvas.width = 32; canvas.height = 18;
622592 const ctx = canvas.getContext("2d", { willReadFrequently: true });
623593 const fresh = (url) => url + (url.includes("?") ? "&" : "?") + Date.now();
624594 const rtts = [];
595 /** A round trip, from a small request the page makes anyway: the time to its first byte. */
596 const timed = async (url) => {
597 const started = performance.now();
598 const r = await fetch(url, { cache: "no-store" });
599 rtts.push(performance.now() - started); if (rtts.length > 10) rtts.shift();
600 return r;
601 };
625602 let plan = { mode: "stream" };
603 const QUALITY = { auto: "Auto", saver: "Saver", full: "Full" };
604 let quality = (() => { try { const v = localStorage.getItem("so101.quality"); return Object.hasOwn(QUALITY, v ?? "") ? v : "auto"; } catch { return "auto"; } })();
605 // The first frame already follows what the browser says about the link (and the viewer's choice), not just later ones.
606 plan = cameraPlan(navigator.connection, rtts, "stream", { quality, throttle: relayed });
607 /** The stream's address under the plan: through the relay, ?fps= holds it to fewer frames. */
608 const streamUrl = (c) => fresh(plan.mode === "stream" && plan.fps && relayed ? `${c.url}?fps=${plan.fps}` : c.url);
626609 const saved = (() => { try { const s = JSON.parse(localStorage.getItem("so101.panes") || "null"); return Array.isArray(s) ? s : null; } catch { return null; } })();
627610 // Pane keys: "cam:<id>" and "pane3". Until the startup below knows the model and the link, cameras are listed but not shown.
628611 let chosen = saved ? new Set(saved) : null, viewsReady = false, cameraFirst = true;
@@ -631,7 +614,7 @@
631614 const paneOn = (c) => c.pane.classList.contains("on");
632615 async function snap(c) {
633616 clearTimeout(c.timer);
634 if (!paneOn(c) || plan.mode !== "snapshot") return;
617 if (!paneOn(c) || plan.mode !== "snapshot" || document.hidden) return;
635618 const started = performance.now();
636619 try {
637620 const r = await fetch(fresh(c.snapshot), { cache: "no-store" });
@@ -645,9 +628,9 @@
645628 const showCam = (c, on) => {
646629 c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on));
647630 clearTimeout(c.timer);
648 if (!on) c.img.src = "";
631 if (!on || document.hidden) c.img.src = "";
649632 else if (plan.mode === "snapshot" && c.snapshot) snap(c);
650 else c.img.src = fresh(c.url);
633 else c.img.src = streamUrl(c);
651634 };
652635 function setPane(key, on) {
653636 if (key === "pane3") {
@@ -687,7 +670,7 @@
687670 c.tag = el("span", { className: "tag" });
688671 c.pane = el("div", { className: "pane" }, c.img, c.tag); c.pane.dataset.key = `cam:${info.id}`;
689672 c.btn = el("button", { type: "button" }); c.btn.setAttribute("aria-pressed", "false");
690 c.img.onerror = () => { if (paneOn(c) && plan.mode === "stream") setTimeout(() => { if (paneOn(c) && plan.mode === "stream") c.img.src = fresh(c.url); }, 3000); };
673 c.img.onerror = () => { if (paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); };
691674 c.btn.onclick = () => pick(c.pane.dataset.key);
692675 $("stage").insertBefore(c.pane, $("pane3"));
693676 $("top").querySelector(".seg").insertBefore(c.btn, $("v3"));
@@ -716,6 +699,7 @@
716699 }
717700 label(c, info);
718701 }
702 showQuality();
719703 fallback();
720704 }
721705 /** Once the model and the link are known: the views the viewer wants. */
@@ -728,19 +712,30 @@
728712 }
729713 async function pollCameras() {
730714 try {
731 const r = await fetch("/cameras", { cache: "no-store" });
715 const r = await timed("/cameras");
732716 if (r.ok) reconcile(await r.json());
733717 } catch { /* the next poll tries again */ }
734718 setTimeout(pollCameras, document.hidden ? 10000 : 3000);
735719 }
736720 function replan() {
737 const next = cameraPlan(navigator.connection, rtts, plan.mode);
738 if (next.mode === plan.mode && (next.mode === "stream" || next.everyMs === plan.everyMs)) return;
739 const was = plan.mode; plan = next;
740 if (was !== next.mode) for (const c of cams) if (paneOn(c) && c.snapshot) showCam(c, true);
721 const next = cameraPlan(navigator.connection, rtts, plan, { quality, throttle: relayed });
722 if (samePlan(next, plan)) return;
723 const was = plan; plan = next;
724 // A new frame rate is a new stream; a new snapshot interval is taken at the next snapshot.
725 if (was.mode !== next.mode || next.mode === "stream") for (const c of cams) if (paneOn(c) && (c.snapshot || next.mode === "stream")) showCam(c, true);
741726 }
742727 setInterval(replan, 5000);
743728 navigator.connection?.addEventListener?.("change", replan);
729 const qualityButton = $("quality");
730 const showQuality = () => { qualityButton.textContent = QUALITY[quality]; qualityButton.hidden = !cams.length; };
731 qualityButton.onclick = () => {
732 quality = QUALITIES[(QUALITIES.indexOf(quality) + 1) % QUALITIES.length];
733 try { localStorage.setItem("so101.quality", quality); } catch {}
734 showQuality(); replan();
735 msg(quality === "auto" ? "video: follows the connection" : quality === "saver" ? "video: data saver, a couple of frames a second" : "video: every frame");
736 };
737 // Hidden (another tab, the phone locked): no video at all; back on screen, it picks up where the plan says.
738 document.addEventListener("visibilitychange", () => { for (const c of cams) if (paneOn(c)) showCam(c, true); });
744739 setInterval(() => {
745740 for (const c of cams) {
746741 if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue;
@@ -752,12 +747,14 @@
752747 } catch { /* nothing decodable yet */ }
753748 c.unchanged = changed ? 0 : c.unchanged + 1;
754749 c.live = changed && c.online !== false;
755 if (c.unchanged >= 15) { c.img.src = fresh(c.url); c.unchanged = 0; }
750 if (c.unchanged >= Math.max(15, plan.fps ? 30 / plan.fps : 0) && !document.hidden) { c.img.src = streamUrl(c); c.unchanged = 0; }
756751 }
757 const slow = plan.mode === "snapshot" ? ` · ${(1000 / plan.everyMs).toFixed(plan.everyMs > 1000 ? 1 : 0)} fps, slow link` : "";
752 const why = quality === "saver" ? "data saver" : "slow link";
753 const slow = plan.mode === "snapshot" ? ` · ${(1000 / plan.everyMs).toFixed(plan.everyMs > 1000 ? 1 : 0)} fps, ${why}` : plan.fps ? ` · ${plan.fps} fps, ${why}` : "";
758754 $("live").textContent = cams.filter(paneOn).map((c) => (c.live ? `live${slow}` : c.online === false ? "offline" : "waiting")).join(" · ");
759755 }, 1000);
760 if (relayed) pollCameras();
756 if (boot) { reconcile(boot.cameras); setTimeout(pollCameras, 3000); }
757 else if (relayed) pollCameras();
761758 else {
762759 reconcile([{ id: 1, name: "Cam 1", url: direct(1), online: null }]);
763760 // Straight to an arm server, camera 2 appears when its stream answers.
@@ -846,6 +843,13 @@
846843 if (r.saved) loadRecordings();
847844 }
848845 $("stop").onclick = stop;
846 /** The second button, only while there is something besides the E-stop to end, named for what it ends. */
847 const ENDS_LABEL = { move: "End move", play: "End playback", record: "End recording", park: "End park" };
848 function showStop() {
849 const label = ENDS_LABEL[driving()];
850 $("stop").hidden = !label || !(mayDrive() || isOwner());
851 if (label) $("stop").textContent = label;
852 }
849853 // The emergency stop: the server halts whatever drives the arm and holds it with torque on until an admin clears it.
850854 async function eStop() {
851855 driver.releaseAll();
@@ -1054,8 +1058,11 @@
10541058 }
10551059 if (step !== "check") showOnModel(previewOf(step));
10561060 }
1061 /** src/leader-map.js, fetched the first time someone calibrates a leader: nobody else needs it. */
1062 let leaderMap = null;
10571063 async function openCalib() {
10581064 if (!leader) return;
1065 try { leaderMap ??= await import("/leader-map.js"); } catch { return msg("the calibration did not load; try again", true); }
10591066 if (!Object.keys(limits).length) return msg("the arm's limits have not loaded yet", true);
10601067 const mine = await api("/leader/calibration").catch(() => null);
10611068 if (mine?.status === 401) return msg("join to calibrate your leader: the calibration is kept with your name", true);
@@ -1079,7 +1086,7 @@
10791086 function calibReading(pose) {
10801087 const c = calib;
10811088 for (const [id, t] of pose) c.reading.set(id, t);
1082 if (STEPS[c.step] === "check") showOnModel(mapLeaderFrame(c.reading, draft(), limits));
1089 if (STEPS[c.step] === "check") showOnModel(leaderMap.mapLeaderFrame(c.reading, draft(), limits));
10831090 for (const [i, span] of [...$("lcRead").children].entries()) span.textContent = `${LABELS[LEADER_IDS[i]].split(" ")[0]} ${c.reading.get(LEADER_IDS[i]) ?? "–"}`;
10841091 }
10851092 async function nextCalib() {
@@ -1099,7 +1106,7 @@
10991106 const j = c.joints[step.id];
11001107 j[step.end] = c.reading.get(step.id);
11011108 if (step.end === "high") {
1102 try { parseCalibration({ joints: { [step.id]: step.id === 6 ? { low: j.low, high: j.high, centre: Math.round((j.low + j.high) / 2) } : j } }); }
1109 try { leaderMap.parseCalibration({ joints: { [step.id]: step.id === 6 ? { low: j.low, high: j.high, centre: Math.round((j.low + j.high) / 2) } : j } }); }
11031110 catch (err) { delete j.low; delete j.high; c.step--; c.error = `${err.message.replace(/^joint \d+: /, "")}. Try ${LABELS[step.id].toLowerCase()} again.`; return renderCalib(); }
11041111 }
11051112 }
@@ -1477,7 +1484,7 @@
14771484 // The page follows suit: the Controls tab appears on your turn. Everything here arrives on one socket.
14781485 let wordsSent = false;
14791486 $("wordsForm").addEventListener("submit", (e) => { e.preventDefault(); wordsSent = true; say({ type: "words", words: $("wordsForm").elements.words.value.split(",").map((w) => w.trim()).filter(Boolean) }); });
1480 const lobby = { ws: null, hello: null, state: null, offset: 0, retry: 500, mutedList: [], bannedList: [], admin: null };
1487 const lobby = { ws: null, hello: null, state: null, offset: 0, retry: 500, mutedList: [], bannedList: [], admin: null, statusSeen: false };
14811488 function mayDrive() { return !relayed || (lobby.hello !== null && (!lobby.hello.signin || (lobby.hello.you !== null && lobby.state?.driver?.id === lobby.hello.you.id))); }
14821489 function isOwner() { return !relayed || Boolean(lobby.hello?.you?.owner); }
14831490 const say = (m) => { if (lobby.ws?.readyState === 1) lobby.ws.send(JSON.stringify(m)); };
@@ -1491,7 +1498,7 @@
14911498 shell.classList.toggle("driving", can);
14921499 if (!can) driver.releaseAll();
14931500 wake(can);
1494 $("stop").hidden = !(can || owner);
1501 showStop();
14951502 $("estop").disabled = !(mayDrive() || owner);
14961503 $("estop").title = $("estop").disabled ? "Emergency stop: the driver and the admins can press it" : "Emergency stop (Esc): halt, hold with torque, until an admin clears it";
14971504 // The server takes settings, the safety limits among them, from admins only.
@@ -1649,11 +1656,26 @@
16491656 if (m.from && me?.owner) line.tabIndex = 0;
16501657 return line;
16511658 }
1659 // A new page gets the last lines of the chat; older ones come a page at a time, when asked for.
1660 const earlier = el("button", { type: "button", className: "earlier", textContent: "Earlier messages" });
1661 earlier.onclick = () => {
1662 const first = chatBox.querySelector(".line[data-id]");
1663 if (!first) return;
1664 earlier.disabled = true;
1665 say({ type: "history", before: Number(first.dataset.id) });
1666 };
1667 function showEarlier(more) { earlier.disabled = false; if (more) chatBox.prepend(earlier); else earlier.remove(); }
1668 function addOlder(lines, more) {
1669 const below = chatBox.scrollHeight - chatBox.scrollTop; // the reader stays on the line they were reading
1670 chatBox.prepend(...lines.map(chatLine));
1671 chatBox.scrollTop = chatBox.scrollHeight - below;
1672 showEarlier(more);
1673 }
16521674 function addChat(lines, replace = false) {
16531675 const atEnd = chatBox.scrollHeight - chatBox.scrollTop - chatBox.clientHeight < 40;
16541676 if (replace) chatBox.replaceChildren();
16551677 chatBox.append(...lines.map(chatLine));
1656 while (chatBox.children.length > 200) chatBox.firstChild.remove();
1678 if (!replace) while (chatBox.querySelectorAll(".line").length > 200) chatBox.querySelector(".line").remove();
16571679 if (atEnd || replace) chatBox.scrollTop = chatBox.scrollHeight;
16581680 if (!replace && tab !== "chat") $("unread").textContent = String(Math.min(99, Number($("unread").textContent || 0) + lines.length));
16591681 }
@@ -1695,9 +1717,11 @@
16951717 ws.onmessage = (e) => {
16961718 const m = JSON.parse(e.data);
16971719 if (m.type === "ping") return say({ type: "pong" });
1720 if (m.type === "status") { lobby.statusSeen = true; return m.status ? applyStatus(m.status) : noStatus(); }
16981721 if (m.type === "hello") lobby.hello = m;
16991722 else if (m.type === "state") { lobby.offset = m.now - Date.now(); lobby.state = m; }
1700 else if (m.type === "chat") return addChat(m.messages, true);
1723 else if (m.type === "chat") return addChat(m.messages, true), showEarlier(m.more);
1724 else if (m.type === "history") return addOlder(m.messages, m.more);
17011725 else if (m.type === "said") return addChat([m.message]);
17021726 else if (m.type === "unsaid") return chatBox.querySelector(`[data-id="${Number(m.id)}"]`)?.remove();
17031727 else if (m.type === "muted") lobby.mutedList = m.muted;
@@ -1710,7 +1734,7 @@
17101734 };
17111735 ws.onclose = () => {
17121736 if (lobby.ws !== ws) return;
1713 lobby.ws = null; lobby.hello = null;
1737 lobby.ws = null; lobby.hello = null; lobby.statusSeen = false; // polling takes over until it is back
17141738 applyLock();
17151739 $("driverLine").className = "none"; $("driverLine").textContent = "Reconnecting…";
17161740 setTimeout(lobbyConnect, lobby.retry);
@@ -1743,37 +1767,12 @@
17431767 };
17441768
17451769 // ---------------------------------------------------------------- 3D calibration
1770 // The panel and its words are fetched the first time someone opens it: src/ui/calibration.js.
17461771 $("calib").onclick = async () => {
1747 $("panel").hidden = false;
1748 $("calSteps").textContent = [
1749 `export SO101_ARM=http://${location.host}`,
1750 `export SO101_CAMERA=${cams[0] ? new URL(cams[0].url, location.href).href : "<the camera's stream URL>"}`,
1751 `just vision-calibrate # hold the printed board in front of the camera, tilt it`,
1752 `# save poses from this page: home, drop, he1 … he8 (without gripper)`,
1753 `just vision-handeye he1,he2,he3,he4,he5,he6,he7,he8`,
1754 `just vision-rehearse # sees the cube, plans the joints, moves nothing`,
1755 `just vision-pick`,
1756 ``,
1757 `# a second camera: SO101_CAMERA=<its stream URL> and --publish-as camera2 / handeye2`,
1758 ].join("\n");
1759 const sheets = await api("/printables");
1760 $("printables").innerHTML = Array.isArray(sheets) && sheets.length
1761 ? sheets.map((s) => `<a href="${s.url}" download="${s.file}"><button class="go">⬇ ${s.file.replace(/_/g, " ").replace(/\.png$/, "")}</button></a>`).join("")
1762 : "<span>no printables on this server: copy Software/vision/printables next to the package</span>";
1763 const cal = await api("/calibration");
1764 const saved = await api("/poses");
1765 const poseNames = Object.keys(saved);
1766 const he = poseNames.filter((n) => /^he\d+$/.test(n) && !saved[n].ids.includes(6)).length;
1767 const mark = (ok, yes, no) => `<span class="${ok ? "ok" : "no"}">${ok ? yes : no}</span>`;
1768 const armOnly = (name) => !saved[name] ? mark(false, "", "missing") : saved[name].ids.includes(6) ? mark(false, "", "saved with the gripper: save it again without") : mark(true, "saved");
1769 const camLine = (c) => c ? mark(true, `calibrated ${(c.saved ?? "").slice(0, 10)} · ${c.width}×${c.height} · ${Number(c.rms).toFixed(2)} px error`) : mark(false, "", "not yet");
1770 const heLine = (h) => h ? mark(true, `solved ${(h.saved ?? "").slice(0, 10)} · ${h.poses?.length ?? "?"} poses · mean ${h.residuals_mm ? (h.residuals_mm.reduce((a, b) => a + b, 0) / h.residuals_mm.length).toFixed(1) : "?"} mm`) : mark(false, "", "not yet");
1771 $("calStatus").innerHTML = [
1772 ["camera 1", camLine(cal.camera)], ["hand-eye 1", heLine(cal.handeye)],
1773 ...(cams.length < 2 ? [] : [["camera 2", camLine(cal.camera2)], ["hand-eye 2", heLine(cal.handeye2)]]),
1774 ["pose home", armOnly("home")], ["pose drop", armOnly("drop")],
1775 ["poses he1…", mark(he >= 6, `${he} saved`, `${he} of 6 to 8 saved (without the gripper)`)],
1776 ].map(([k, v]) => `<b>${k}</b><span>${v}</span>`).join("");
1772 try {
1773 const { openCalibration } = await import("/ui/__VERSION__/calibration.js");
1774 await openCalibration({ api, host: location.host, cameras: cams.map((c) => new URL(c.url, location.href).href) });
1775 } catch (err) { console.error(err); msg("the calibration panel did not load", true); }
17771776 };
17781777
17791778 // ---------------------------------------------------------------- live status, sliders and jog
@@ -1834,13 +1833,14 @@
18341833 else if (d.violation) { b.className = "warn"; b.textContent = `the arm is ${say(d.violation)} (zone: ${d.zone})`; }
18351834 else b.className = "";
18361835 }
1837 async function poll() {
1838 const started = performance.now();
1839 try {
1840 const d = await api("/status");
1841 if (!d.ok) throw new Error();
1836 /** The arm's status, from /live (pushed when it changes) or, without it, from polling /status. */
1837 // Until the limits, the settings and the model are in, a status is kept aside, not drawn; then it is applied whole,
1838 // so an E-stop it carries still locks the controls.
1839 let started = false, early = null;
1840 function applyStatus(d) {
1841 if (!started) return void (early = d);
1842 {
18421843 const wasStopped = Boolean(status?.estop);
1843 rtts.push(performance.now() - started); if (rtts.length > 10) rtts.shift(); // the link's round trip, for the camera plan
18441844 status = d;
18451845 driver.seed(d.joints.filter((j) => j.position != null).map((j) => [j.id, j.position]));
18461846 if (Boolean(d.estop) !== wasStopped) applyLock();
@@ -1852,28 +1852,45 @@
18521852 $("dot").className = d.reconnecting ? "dot warn" : "dot on";
18531853 $("state").textContent = d.reconnecting ? "reconnecting…" : d.driving === null ? "idle" : d.driving === "record" ? `recording ${d.recording}` : leaderStreaming() ? `leader · ${leader.frames} frames` : d.driving;
18541854 $("estop").classList.toggle("on", Boolean(d.estop)); $("estop").textContent = d.estop ? "Stopped" : "Stop";
1855 showStop();
18551856 $("record").textContent = d.driving === "record" ? "■ Stop recording" : "● Record";
18561857 for (const id of ["hold", "releaseBtn", "play"]) $(id).disabled = d.driving !== null;
18571858 if (tab === "controls") { render(d); renderTouch(d); }
18581859 banner(d); applyModel(d);
18591860 $("readout").innerHTML = `<span>lowest part <b>${Number.isFinite(d.lowestMm) ? d.lowestMm : "—"} mm</b> above the table</span><span>zone <b>${d.zone === "enforce" || d.zone === "warn" || d.zone === "off" ? d.zone : "—"}</b></span>`;
1860 } catch {
1861 const reconnecting = linkNow === "reconnecting";
1862 $("dot").className = reconnecting ? "dot warn" : "dot"; $("state").textContent = reconnecting ? "reconnecting…" : "no arm";
18631861 }
1862 }
1863 function noStatus() {
1864 const reconnecting = linkNow === "reconnecting";
1865 $("dot").className = reconnecting ? "dot warn" : "dot"; $("state").textContent = reconnecting ? "reconnecting…" : "no arm";
1866 }
1867 async function poll() {
1868 // The room's socket carries the status: nothing to poll while it is open and has said where the arm is.
1869 if (lobby.statusSeen && lobby.ws?.readyState === 1) return void setTimeout(poll, 1000);
1870 try {
1871 const r = await timed("/status");
1872 const d = await r.json();
1873 if (!r.ok) throw new Error();
1874 applyStatus(d);
1875 } catch { noStatus(); }
18641876 // Hidden, the page keeps the arm's state fresh at a walk: nobody is looking, and a phone's battery and data matter.
18651877 setTimeout(poll, document.hidden ? 2000 : 200);
18661878 }
18671879 (async () => {
18681880 applyLock();
18691881 if (relayed) lobbyConnect(); else showTab("controls");
1870 limits = await api("/limits");
1871 await loadSettings();
1872 await loadPoses();
1873 modelInfo = await api("/model.json");
18741882 // The views: the ones chosen last time, else one on a phone (the camera when there is a real one), else the camera and the model.
1875 cameraFirst = !relayed || (await api("/link").catch(() => ({}))).mode === "live" || q.has("stream");
1883 // With the page's own facts (#boot) the camera starts now, before anything else is asked; the rest is asked all at once.
1884 const linked = boot ? Promise.resolve(boot.link) : relayed ? api("/link").catch(() => ({})) : Promise.resolve(null);
1885 if (boot) {
1886 cameraFirst = boot.link.mode === "live" || q.has("stream");
1887 if (cameraFirst) { viewsReady = true; for (const c of cams) if (!paneOn(c) && wanted(c.pane.dataset.key)) showCam(c, true); }
1888 }
1889 const [l, mi, link] = await Promise.all([api("/limits"), api("/model.json"), linked, loadSettings(), loadPoses()]);
1890 limits = l; modelInfo = mi; started = true;
1891 cameraFirst = !relayed || link?.mode === "live" || q.has("stream");
18761892 startViews();
1893 if (early) applyStatus(early);
18771894 poll();
18781895 if (relayed) pollLink();
18791896 // The model is fetched once the first view is up, whether or not it is on screen: the joystick needs it.

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs