Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
Fast on a phone over 4G: first view under 2 s, the camera adapts to the connection, nothing loads until it is needed #69
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
vesper/arm:vesper/53-fast-on-a-phone-over-4g-first-view-under → main 4 commits fork
13 files
changed +612
−141
Software/README.md +24 −1
| @@ -147,7 +147,7 @@ on a LAN or behind Tailscale. | ||
| 147 | 147 | arm (its own URDF and meshes, drag a joint and the arm follows), a slider and |
| 148 | 148 | hold-to-jog buttons per joint that stream targets like a leader does, a |
| 149 | 149 | button per saved pose plus save and delete, hold and release, record and |
| 150 | play, a stop button (space bar), fullscreen (F), a metrics panel, a settings | |
| 150 | play, a stop button while a move or a playback runs (space bar), fullscreen (F), a metrics panel, a settings | |
| 151 | 151 | panel for the safe zone, and a 3D-calibration panel that explains the vision |
| 152 | 152 | quest and shows its status. `bench/fake-serve.ts` runs the same server on a |
| 153 | 153 | simulated bus for working on the page. The page expects MJPEG streams on |
| @@ -195,6 +195,29 @@ again every 2 s and the live picture comes back in the same stream, without | ||
| 195 | 195 | a reload. The camera is independent of the arm server: a camera that is down |
| 196 | 196 | does not put the arm on the simulation, and the other way round. |
| 197 | 197 | |
| 198 | **On a phone over 4G.** The first load is the page and three small modules, | |
| 199 | under 50 KB gzipped (a test in `src/server.test.ts` holds it there). three.js, | |
| 200 | the leader-arm mapping and the 3D-calibration panel (`src/ui/calibration.js`) | |
| 201 | load only when they're used. The relay writes the link and the camera list | |
| 202 | into the page (`<script type="application/json" id="boot">`), so the camera | |
| 203 | starts as soon as the script runs. The arm's limits, settings, poses and | |
| 204 | model are all requested at once after that. One WebSocket, `/live`, carries | |
| 205 | the turn, the queue, the chat and the arm's status. The status is pushed only | |
| 206 | when it changes, so a page whose socket is open doesn't poll `/status`. A new | |
| 207 | page gets the last 30 chat lines; **Earlier messages** fetches 50 more at a | |
| 208 | time. The camera follows the connection (`src/ui/camera-plan.js`). On a slow | |
| 209 | link (round trips over 400 ms, or the browser reporting 3G or a low | |
| 210 | downlink), the relay holds that viewer's stream to 5 frames a second | |
| 211 | (`/camera/<id>/stream?fps=1-30`), or 2 when the browser asks to save data. | |
| 212 | On a crawl (2G, or round trips over a second), the page fetches one snapshot | |
| 213 | at a time. The **Auto / Saver / Full** button in the top bar overrides this, | |
| 214 | and the choice is kept in the browser. A hidden page (another tab, a locked | |
| 215 | phone) fetches no video at all. | |
| 216 | ||
| 217 | There is one **Stop** on screen: the red emergency stop. A second button | |
| 218 | appears only while a move, a playback, a recording or a park is running, and | |
| 219 | says which one it ends (**End playback**, …). It holds the arm where it is. | |
| 220 | ||
| 198 | 221 | **Cameras are a list the owner edits** on `/admin/camera-list` (name + source, such as |
| 199 | 222 | `<arm host>:8080`), kept in `cameras.json` under `FORGE_APP_DATA`; the first |
| 200 | 223 | start lists the arm host's camera 1. Each is streamed once from its source |
Software/so101/src/camera-feed.test.ts +27 −2
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | import { afterEach, describe, expect, test } from "bun:test"; |
| 2 | 2 | import { createServer, type Server, type ServerResponse } from "node:http"; |
| 3 | 3 | import { createServer as createTcpServer, type Server as TcpServer } from "node:net"; |
| 4 | import { CameraFeed, type FeedOptions } from "./camera-feed.ts"; | |
| 4 | import { CameraFeed, parseFps, type FeedOptions } from "./camera-feed.ts"; | |
| 5 | 5 | import { OFFLINE_FRAME } from "./offline-frame.ts"; |
| 6 | 6 | |
| 7 | 7 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); |
| @@ -82,7 +82,7 @@ afterEach(() => { | ||
| 82 | 82 | /** Serve a feed of `source` on a port of its own, as the relay does. */ |
| 83 | 83 | async function serveFeed(source: string | null, options: FeedOptions = {}) { |
| 84 | 84 | feed = new CameraFeed(source, "camera 1", options); |
| 85 | front = createServer((req, res) => feed!.serve(req, res)); | |
| 85 | front = createServer((req, res) => feed!.serve(req, res, parseFps(new URL(req.url ?? "/", "http://x").searchParams.get("fps")))); | |
| 86 | 86 | return `http://127.0.0.1:${await listening(front)}/`; |
| 87 | 87 | } |
| 88 | 88 | |
| @@ -105,6 +105,31 @@ describe("a camera feed", () => { | ||
| 105 | 105 | await b.cancel(); |
| 106 | 106 | }); |
| 107 | 107 | |
| 108 | test("a viewer on a slow link asks for fewer frames: it gets whole frames at that rate, the others every frame", async () => { | |
| 109 | camera = fakeCamera(); // a frame every 30 ms, about 33 a second | |
| 110 | const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`); | |
| 111 | const full = await watch(url); | |
| 112 | const slow = await watch(`${url}?fps=5&1729`); // the page adds a cache-buster after it | |
| 113 | await full.until(hasFrame); | |
| 114 | await sleep(1000); | |
| 115 | const count = (w: { seen: Buffer }) => (w.seen.toString("latin1").match(/frame-/g) ?? []).length; | |
| 116 | await full.until((s) => (s.toString("latin1").match(/frame-/g) ?? []).length >= 20); | |
| 117 | await slow.until((s) => (s.toString("latin1").match(/frame-/g) ?? []).length >= 4); | |
| 118 | const n = count(slow); | |
| 119 | expect(n).toBeLessThanOrEqual(8); | |
| 120 | expect(count(full)).toBeGreaterThan(3 * n); | |
| 121 | expect(slow.seen.toString("latin1")).toMatch(/^--armframe\r\n(Content-Type: image\/jpeg\r\n\r\n\xff\xd8frame-\d+\xff\xd9\r\n--armframe\r\n)+/); | |
| 122 | expect(camera.connections).toBe(1); | |
| 123 | await full.cancel(); | |
| 124 | await slow.cancel(); | |
| 125 | }); | |
| 126 | ||
| 127 | test("fps is a whole number 1-30; anything else is every frame", () => { | |
| 128 | expect(parseFps("5")).toBe(5); | |
| 129 | expect(parseFps("30")).toBe(30); | |
| 130 | for (const bad of [null, "", "0", "31", "2.5", "-1", "1e1", "5x", "999"]) expect(parseFps(bad)).toBeNull(); | |
| 131 | }); | |
| 132 | ||
| 108 | 133 | test("a viewer who joins later shares the same connection and starts on a whole frame", async () => { |
| 109 | 134 | camera = fakeCamera(); |
| 110 | 135 | const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`); |
Software/so101/src/camera-feed.ts +28 −6
| @@ -1,7 +1,8 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * One camera, shared: a single upstream MJPEG connection fanned out to every viewer, part by part, as the bytes |
| 3 | 3 | * arrive. No frame is gathered whole; a viewer who joins mid-frame starts at the next one, and a viewer who cannot |
| 4 | * keep up skips frames rather than holding the others back. | |
| 4 | * keep up skips frames rather than holding the others back. A viewer on a slow link may ask for fewer frames a | |
| 5 | * second (`fps`): the frames in between are skipped for it alone, whole, so what it gets is always a full picture. | |
| 5 | 6 | * |
| 6 | 7 | * When there is no source, or it does not answer, or it stops sending, every viewer gets the "camera offline" |
| 7 | 8 | * picture at once and again every few seconds, while the feed keeps trying the source; when it answers again the |
| @@ -37,6 +38,20 @@ interface Viewer { | ||
| 37 | 38 | res: ServerResponse; |
| 38 | 39 | /** Mid-way through a live part; otherwise it stands right after a boundary. */ |
| 39 | 40 | inPart: boolean; |
| 41 | /** The least time between two frames this viewer is sent, ms; 0 for every frame. */ | |
| 42 | gapMs: number; | |
| 43 | /** When it was last sent the start of a frame. */ | |
| 44 | sentAt: number; | |
| 45 | } | |
| 46 | ||
| 47 | /** The most frames a second a viewer may ask to be held to; above it, it gets every frame. */ | |
| 48 | export const MAX_FPS = 30; | |
| 49 | ||
| 50 | /** A viewer's `fps` from a query value: a whole number 1-30, or null (every frame) for anything else. */ | |
| 51 | export function parseFps(text: string | null): number | null { | |
| 52 | if (text === null || !/^\d{1,2}$/.test(text)) return null; | |
| 53 | const n = Number(text); | |
| 54 | return n >= 1 && n <= MAX_FPS ? n : null; | |
| 40 | 55 | } |
| 41 | 56 | |
| 42 | 57 | export class CameraFeed { |
| @@ -71,8 +86,11 @@ export class CameraFeed { | ||
| 71 | 86 | return this.viewers.size; |
| 72 | 87 | } |
| 73 | 88 | |
| 74 | /** Answer one viewer's GET (or HEAD) with the shared stream; it runs until the viewer leaves or the feed closes. */ | |
| 75 | serve(req: IncomingMessage, res: ServerResponse): void { | |
| 89 | /** | |
| 90 | * Answer one viewer's GET (or HEAD) with the shared stream; it runs until the viewer leaves or the feed closes. | |
| 91 | * `fps` holds this viewer to at most that many frames a second. | |
| 92 | */ | |
| 93 | serve(req: IncomingMessage, res: ServerResponse, fps: number | null = null): void { | |
| 76 | 94 | res.writeHead(200, { |
| 77 | 95 | "content-type": `multipart/x-mixed-replace;boundary=${BOUNDARY}`, |
| 78 | 96 | "cache-control": "no-store, no-transform", |
| @@ -80,7 +98,7 @@ export class CameraFeed { | ||
| 80 | 98 | }); |
| 81 | 99 | if (req.method === "HEAD" || this.closed) return void res.end(); |
| 82 | 100 | res.write(`--${BOUNDARY}\r\n`); |
| 83 | const viewer: Viewer = { res, inPart: false }; | |
| 101 | const viewer: Viewer = { res, inPart: false, gapMs: fps ? 1000 / fps : 0, sentAt: -Infinity }; | |
| 84 | 102 | this.viewers.add(viewer); |
| 85 | 103 | // Bun's node:http may only tell the socket that the viewer hung up, not the response: listen to both. |
| 86 | 104 | res.on("close", () => this.leave(viewer)); |
| @@ -192,9 +210,13 @@ export class CameraFeed { | ||
| 192 | 210 | } |
| 193 | 211 | const parser = new MjpegParser(boundary, { |
| 194 | 212 | start: (type) => { |
| 213 | const now = performance.now(); | |
| 195 | 214 | for (const v of this.viewers) { |
| 196 | v.inPart = v.res.writableLength <= this.o.maxBacklog; | |
| 197 | if (v.inPart) this.write(v, head(type)); | |
| 215 | // A frame a little early still counts as on time: a 10 fps source held to 5 sends every other frame, not every third. | |
| 216 | v.inPart = v.res.writableLength <= this.o.maxBacklog && now - v.sentAt >= v.gapMs * 0.8; | |
| 217 | if (!v.inPart) continue; | |
| 218 | v.sentAt = now; | |
| 219 | this.write(v, head(type)); | |
| 198 | 220 | } |
| 199 | 221 | }, |
| 200 | 222 | data: (bytes) => { |
Software/so101/src/lobby.test.ts +52 −1
| @@ -10,7 +10,7 @@ import { Chat } from "./chat.ts"; | ||
| 10 | 10 | import { FakeBus } from "./fake-bus.ts"; |
| 11 | 11 | import { sign } from "./identity.ts"; |
| 12 | 12 | import { Accounts } from "./accounts.ts"; |
| 13 | import { Lobby } from "./lobby.ts"; | |
| 13 | import { CHAT_FIRST, HISTORY_PAGE, Lobby } from "./lobby.ts"; | |
| 14 | 14 | import { startRelay, type RunningRelay } from "./relay.ts"; |
| 15 | 15 | import { startServer, type RunningServer } from "./server.ts"; |
| 16 | 16 | import { Turns } from "./turns.ts"; |
| @@ -277,6 +277,57 @@ describe("lobby", () => { | ||
| 277 | 277 | expect((await late.first((m) => m.type === "chat")).messages).toEqual([]); |
| 278 | 278 | }); |
| 279 | 279 | |
| 280 | test("chat history is lazy: a new page gets the last lines, anyone asks for older pages, oldest first", async () => { | |
| 281 | const chat = new Chat(); | |
| 282 | for (let i = 0; i < 100; i++) chat.system(`line ${i}`); | |
| 283 | const port = await start(undefined, chat); | |
| 284 | const anon = await join(port); | |
| 285 | const first = await anon.first((m) => m.type === "chat"); | |
| 286 | expect(first.messages.length).toBe(CHAT_FIRST); | |
| 287 | expect(first.messages.at(-1).text).toBe("line 99"); | |
| 288 | expect(first.more).toBe(true); | |
| 289 | anon.send({ type: "history", before: first.messages[0].id }); | |
| 290 | const older = await anon.next((m) => m.type === "history"); | |
| 291 | expect(older.messages.length).toBe(HISTORY_PAGE); | |
| 292 | expect(older.messages.at(-1).id).toBe(first.messages[0].id - 1); | |
| 293 | expect(older.more).toBe(true); | |
| 294 | anon.send({ type: "history", before: older.messages[0].id }); | |
| 295 | const oldest = await anon.next((m) => m.type === "history"); | |
| 296 | expect(oldest.messages.map((m: Msg) => m.text)).toEqual(Array.from({ length: 100 - CHAT_FIRST - HISTORY_PAGE }, (_, i) => `line ${i}`)); | |
| 297 | expect(oldest.more).toBe(false); | |
| 298 | anon.send({ type: "history", before: "x" }); | |
| 299 | expect((await anon.next((m) => m.type === "error")).error).toContain("history"); | |
| 300 | }); | |
| 301 | ||
| 302 | test("publish reaches every page on /live, signed in or not", async () => { | |
| 303 | const port = await start(); | |
| 304 | const anon = await join(port); | |
| 305 | const alice = await join(port, ALICE); | |
| 306 | await Promise.all([anon.first((m) => m.type === "chat"), alice.first((m) => m.type === "chat")]); | |
| 307 | expect(lobby!.watching).toBe(2); | |
| 308 | const heard = [anon, alice].map((c) => c.next((m) => m.type === "status")); | |
| 309 | lobby!.publish({ type: "status", joints: [] }); | |
| 310 | for (const m of await Promise.all(heard)) expect(m.joints).toEqual([]); | |
| 311 | }); | |
| 312 | ||
| 313 | test("one socket: the relay pushes the arm's status over /live, once, and again only when it changes", async () => { | |
| 314 | const port = await start(); | |
| 315 | const anon = await join(port); | |
| 316 | const first = await anon.first((m) => m.type === "status"); | |
| 317 | expect(first.status.joints.length).toBe(3); | |
| 318 | expect(first.status.source).toStartWith("simulated"); | |
| 319 | await sleep(700); // three ticks of an arm that sits still | |
| 320 | expect(anon.heard.filter((m) => m.type === "status").length).toBe(1); | |
| 321 | const owner = await join(port, OWNER); | |
| 322 | await owner.first((m) => m.type === "status"); // a page that opens later hears it at once, not at the next change | |
| 323 | owner.send({ type: "join" }); | |
| 324 | await owner.next(drivingIs("trevato")); | |
| 325 | const moved = anon.next((m) => m.type === "status" && m.status.joints[0].position !== first.status.joints[0].position); | |
| 326 | const r = await fetch(`http://127.0.0.1:${port}/move`, { method: "POST", headers: { ...OWNER, "content-type": "application/json" }, body: JSON.stringify({ targets: { 1: first.status.joints[0].position + 100 } }) }); | |
| 327 | expect(r.status).toBe(200); | |
| 328 | expect((await moved).status.joints[0].position).toBe(first.status.joints[0].position + 100); | |
| 329 | }); | |
| 330 | ||
| 280 | 331 | test("the driver and any admin may hit the emergency stop; only an admin clears it or changes the settings", async () => { |
| 281 | 332 | const port = await start(); |
| 282 | 333 | const alice = await join(port, ALICE); |
Software/so101/src/lobby.ts +47 −4
| @@ -21,9 +21,12 @@ | ||
| 21 | 21 | * {"type": "ban", "id": "<user id>", "on": true|false} owner: bars their account, name and address; they still watch |
| 22 | 22 | * {"type": "words", "words": ["…"]} owner: the chat's word filter; a line with a listed word is refused |
| 23 | 23 | * {"type": "pong"} the answer to {"type": "ping"} |
| 24 | * {"type": "history", "before": 12} anyone: up to HISTORY_PAGE lines older than that one | |
| 24 | 25 | * Server → client: hello (who you are), state (the turn, the line, pause, the announcement and the speed cap), |
| 25 | * chat (the log), said, unsaid, muted, admin (owners only: the settings and the admin log), crew (owners only: | |
| 26 | * settings, hidden lines, mute suggestions), banned and words (owners only), ping, error. | |
| 26 | * chat (the last CHAT_FIRST lines of the log, and whether there are more), history (an older page, the same way), | |
| 27 | * said, unsaid, muted, admin (owners only: the settings and the admin log), crew (owners only: | |
| 28 | * settings, hidden lines, mute suggestions), banned and words (owners only), ping, error; and whatever the relay | |
| 29 | * `publish`es to everyone: the arm's status, so a page needs this one socket for the pose, the chat and the queue. | |
| 27 | 30 | */ |
| 28 | 31 | |
| 29 | 32 | import type { IncomingMessage } from "node:http"; |
| @@ -69,6 +72,10 @@ interface Seat { | ||
| 69 | 72 | } |
| 70 | 73 | |
| 71 | 74 | const MAX_FRAME = 4096; |
| 75 | /** Lines of chat a new page is sent: enough to fill a phone's chat, little enough for a slow link. */ | |
| 76 | export const CHAT_FIRST = 30; | |
| 77 | /** Lines per "history" answer, when someone scrolls up for more. */ | |
| 78 | export const HISTORY_PAGE = 50; | |
| 72 | 79 | const USER_ID = /^[A-Za-z0-9_-]{1,64}$/; |
| 73 | 80 | |
| 74 | 81 | export class Lobby { |
| @@ -84,6 +91,7 @@ export class Lobby { | ||
| 84 | 91 | private readonly seats = new Set<Seat>(); |
| 85 | 92 | private readonly changeListeners = new Set<() => void>(); |
| 86 | 93 | private readonly endListeners = new Set<(who: Person) => void>(); |
| 94 | private readonly arriveListeners = new Set<(tell: (text: string) => void) => void>(); | |
| 87 | 95 | private readonly pinger: ReturnType<typeof setInterval>; |
| 88 | 96 | private readonly silentMs: number; |
| 89 | 97 | private lastActivityState = 0; |
| @@ -214,6 +222,12 @@ export class Lobby { | ||
| 214 | 222 | return () => this.endListeners.delete(listener); |
| 215 | 223 | } |
| 216 | 224 | |
| 225 | /** Called as each page opens the room, with a way to tell that page alone something (JSON text): what was published before it came. */ | |
| 226 | onArrive(listener: (tell: (text: string) => void) => void): () => void { | |
| 227 | this.arriveListeners.add(listener); | |
| 228 | return () => this.arriveListeners.delete(listener); | |
| 229 | } | |
| 230 | ||
| 217 | 231 | upgrade(req: IncomingMessage, socket: Duplex, head: Buffer): void { |
| 218 | 232 | const who = this.who(req); |
| 219 | 233 | const from = visitor(req); |
| @@ -235,7 +249,7 @@ export class Lobby { | ||
| 235 | 249 | if (who) this.turns.connect(who.id); |
| 236 | 250 | this.send(seat, this.hello(seat)); |
| 237 | 251 | this.send(seat, this.state()); |
| 238 | this.send(seat, { type: "chat", messages: this.chat.messages }); | |
| 252 | this.send(seat, this.chatLog()); | |
| 239 | 253 | if (who?.owner) { |
| 240 | 254 | this.send(seat, this.mutedList()); |
| 241 | 255 | this.send(seat, this.bannedList()); |
| @@ -243,6 +257,7 @@ export class Lobby { | ||
| 243 | 257 | this.send(seat, this.adminState()); |
| 244 | 258 | this.send(seat, this.crewState()); |
| 245 | 259 | } |
| 260 | for (const listener of this.arriveListeners) listener((text) => seat.ws.readyState === WebSocket.OPEN && seat.ws.send(text)); | |
| 246 | 261 | ws.on("message", (data, binary) => { |
| 247 | 262 | seat.heard = Date.now(); |
| 248 | 263 | if (binary) return; |
| @@ -272,6 +287,10 @@ export class Lobby { | ||
| 272 | 287 | const m = message as Record<string, unknown>; |
| 273 | 288 | const who = seat.who; |
| 274 | 289 | if (m.type === "pong") return; |
| 290 | if (m.type === "history") { | |
| 291 | if (!Number.isSafeInteger(m.before)) return this.send(seat, { type: "error", error: "history before which line?" }); | |
| 292 | return this.send(seat, { type: "history", ...this.page(m.before as number, HISTORY_PAGE) }); | |
| 293 | } | |
| 275 | 294 | if (!who) return this.send(seat, { type: "error", error: "join to take part" }); |
| 276 | 295 | if (m.type === "leave") return void this.turns.leave(who.id); |
| 277 | 296 | if (this.barred(who, seat.from)) return this.send(seat, { type: "error", error: "you are banned: you can still watch" }); |
| @@ -367,7 +386,7 @@ export class Lobby { | ||
| 367 | 386 | const back = Number.isSafeInteger(m.id) ? this.moderator?.restore(m.id as number) : null; |
| 368 | 387 | if (!back) return this.send(seat, { type: "error", error: "that line is not in the log" }); |
| 369 | 388 | this.chat.restore(back); |
| 370 | this.broadcast({ type: "chat", messages: this.chat.messages }); | |
| 389 | this.broadcast(this.chatLog()); | |
| 371 | 390 | return; |
| 372 | 391 | } |
| 373 | 392 | case "dismiss": |
| @@ -439,6 +458,30 @@ export class Lobby { | ||
| 439 | 458 | } |
| 440 | 459 | } |
| 441 | 460 | |
| 461 | /** Up to `count` lines before the one with id `before` (all of them with no `before`), oldest first, and whether older ones remain. */ | |
| 462 | private page(before: number | null, count: number) { | |
| 463 | const all = this.chat.messages; | |
| 464 | const end = before === null ? all.length : all.findIndex((m) => m.id >= before); | |
| 465 | const stop = end < 0 ? all.length : end; | |
| 466 | const start = Math.max(0, stop - count); | |
| 467 | return { messages: all.slice(start, stop), more: start > 0 }; | |
| 468 | } | |
| 469 | ||
| 470 | private chatLog() { | |
| 471 | return { type: "chat", ...this.page(null, CHAT_FIRST) }; | |
| 472 | } | |
| 473 | ||
| 474 | /** Send every open page the same thing, serialized once however many watch; a string is taken as JSON already. */ | |
| 475 | publish(body: unknown): void { | |
| 476 | const text = typeof body === "string" ? body : JSON.stringify(body); | |
| 477 | for (const s of this.seats) if (s.ws.readyState === WebSocket.OPEN) s.ws.send(text); | |
| 478 | } | |
| 479 | ||
| 480 | /** How many pages have the room open. */ | |
| 481 | get watching(): number { | |
| 482 | return this.seats.size; | |
| 483 | } | |
| 484 | ||
| 442 | 485 | private send(seat: Seat, body: unknown): void { |
| 443 | 486 | if (seat.ws.readyState === WebSocket.OPEN) seat.ws.send(JSON.stringify(body)); |
| 444 | 487 | } |
Software/so101/src/relay.test.ts +18
| @@ -7,6 +7,7 @@ import { join } from "node:path"; | ||
| 7 | 7 | import { gunzipSync } from "node:zlib"; |
| 8 | 8 | import WebSocket, { WebSocketServer } from "ws"; |
| 9 | 9 | import { FeetechBus } from "./bus.ts"; |
| 10 | import { CameraHub } from "./cameras.ts"; | |
| 10 | 11 | import { FakeBus } from "./fake-bus.ts"; |
| 11 | 12 | import { OFFLINE_FRAME } from "./offline-frame.ts"; |
| 12 | 13 | import { parseRemote, sameOrigin, startRelay, type Remote, type RunningRelay } from "./relay.ts"; |
| @@ -117,6 +118,23 @@ describe("relay", () => { | ||
| 117 | 118 | relay = remote = camera = hang = undefined; |
| 118 | 119 | }); |
| 119 | 120 | |
| 121 | test("the page carries its first view's facts: which arm, and the cameras, escaped as inert JSON", async () => { | |
| 122 | const cameras = new CameraHub({ seed: [{ name: "Desk </script><img src=x onerror=alert(1)>", source: "http://127.0.0.1:9/stream" }] }); | |
| 123 | relay = await startRelay({ port: 0, simulation: simBase, cameras }); | |
| 124 | const base = `http://127.0.0.1:${relay.port}`; | |
| 125 | for (const encoding of ["identity", "gzip"]) { | |
| 126 | const page = await fetch(`${base}/`, { headers: { "accept-encoding": encoding }, decompress: false } as RequestInit); | |
| 127 | const html = encoding === "gzip" ? (expect(page.headers.get("content-encoding")).toBe("gzip"), gunzipSync(Buffer.from(await page.arrayBuffer())).toString("utf8")) : await page.text(); | |
| 128 | expect(page.headers.get("cache-control")).toBe("no-store"); | |
| 129 | expect(page.headers.get("content-security-policy")).toContain("frame-ancestors 'none'"); | |
| 130 | const found = /<script type="application\/json" id="boot">(.*?)<\/script>\n<\/head>/.exec(html); | |
| 131 | expect(found).not.toBeNull(); | |
| 132 | expect(found![1]).not.toContain("<"); | |
| 133 | expect(JSON.parse(found![1]!)).toEqual({ link: { mode: "simulated", host: null, problem: null }, cameras: [{ id: 1, name: "Desk </script><img src=x onerror=alert(1)>", online: null }] }); // not watched yet: unknown | |
| 134 | } | |
| 135 | cameras.close(); | |
| 136 | }); | |
| 137 | ||
| 120 | 138 | test("unset: everything is the simulation, and the page says so", async () => { |
| 121 | 139 | relay = await startRelay({ port: 0, simulation: simBase }); |
| 122 | 140 | const base = `http://127.0.0.1:${relay.port}`; |
Software/so101/src/relay.ts +66 −9
| @@ -21,7 +21,7 @@ | ||
| 21 | 21 | * |
| 22 | 22 | * The owner's speed cap (src/speed-cap.ts) is applied here too, to moves, poses, grips, playbacks and every stream frame. |
| 23 | 23 | * |
| 24 | * WS /live the lobby: the turn, the queue, the chat | |
| 24 | * WS /live the lobby: the turn, the queue, the chat, and the arm's /status pushed when it changes | |
| 25 | 25 | * POST /join {"name": "…"}: become someone here, no forge account needed (src/accounts.ts); the answer |
| 26 | 26 | * sets the HttpOnly session cookie and names you: {"you": {"id", "name", "owner"}} |
| 27 | 27 | * GET /admin, POST /admin the owner's page: pause, end a turn, take control, the settings, the log |
| @@ -31,7 +31,7 @@ | ||
| 31 | 31 | * there is one), or "simulated: <why>" ("simulated: offline, <why>" after 60 s down) |
| 32 | 32 | * GET /cameras the listed cameras: [{"id", "name", "online"}] |
| 33 | 33 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 34 | * the "camera offline" picture when its source is down | |
| 34 | * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate | |
| 35 | 35 | * GET /admin/camera-list the owner's page for the camera list |
| 36 | 36 | * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds |
| 37 | 37 | * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time, |
| @@ -49,7 +49,7 @@ import { acceptsGzip, GzipCache, worthCompressing } from "./compress.ts"; | ||
| 49 | 49 | import { AccountError, sessionCookie, visitor } from "./accounts.ts"; |
| 50 | 50 | import { AdminError, changeFromForm } from "./admin.ts"; |
| 51 | 51 | import { ADMIN_CSP, DONE, renderAdmin, renderRefusal, type Done } from "./admin-page.ts"; |
| 52 | import type { FeedOptions } from "./camera-feed.ts"; | |
| 52 | import { parseFps, type FeedOptions } from "./camera-feed.ts"; | |
| 53 | 53 | import { CameraError, CameraHub } from "./cameras.ts"; |
| 54 | 54 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 55 | 55 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| @@ -102,6 +102,8 @@ export interface RelayOptions { | ||
| 102 | 102 | camera?: FeedOptions; |
| 103 | 103 | /** Every driver's leader calibration; without one they are kept in memory. */ |
| 104 | 104 | leaders?: LeaderCalibrations; |
| 105 | /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */ | |
| 106 | statusEveryMs?: number; | |
| 105 | 107 | /** Where HTTP_PROXY and NO_PROXY are read from; process.env by default. */ |
| 106 | 108 | env?: Record<string, string | undefined>; |
| 107 | 109 | } |
| @@ -120,6 +122,8 @@ const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), | ||
| 120 | 122 | const MAX_FORM = 16_384; |
| 121 | 123 | /** Served from this process whichever arm is live: they describe the app, not the arm. */ |
| 122 | 124 | const LOCAL = /^\/(?:index\.html|leader-map\.js)?$|^\/(?:model|vendor|printables|ui|app)(?:\/|$)/; |
| 125 | /** The page itself, which the relay writes the first view's facts into. */ | |
| 126 | const PAGE = /^\/(?:index\.html)?$/; | |
| 123 | 127 | /** The page's static files: what a first visit downloads, and what gzip is for. */ |
| 124 | 128 | const STATIC = /^\/(?:index\.html)?$|^\/(?:model|vendor|ui)\//; |
| 125 | 129 | const LEADER = "/leader/calibration"; |
| @@ -313,16 +317,44 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 313 | 317 | * The arm's /status, saying which arm it is: from the link while it is live, the last one it had while reconnecting, |
| 314 | 318 | * and from the simulation otherwise. |
| 315 | 319 | */ |
| 316 | const status = async (res: ServerResponse, path: string) => { | |
| 320 | const statusOf = async (path = "/status"): Promise<{ code: number; body: Record<string, unknown> }> => { | |
| 317 | 321 | if (onArm()) { |
| 318 | 322 | const last = armLink!.last; |
| 319 | 323 | const reconnecting = mode() === "reconnecting"; |
| 320 | if (!last) return json(res, 503, { error: `${RECONNECTING}…`, source: source(), reconnecting }); | |
| 321 | return json(res, 200, { ...last, source: source(), reconnecting, ...(reconnecting ? { seenAt: new Date(armLink!.lastAt).toISOString() } : {}) }); | |
| 324 | if (!last) return { code: 503, body: { error: `${RECONNECTING}…`, source: source(), reconnecting } }; | |
| 325 | return { code: 200, body: { ...last, source: source(), reconnecting, ...(reconnecting ? { seenAt: new Date(armLink!.lastAt).toISOString() } : {}) } }; | |
| 322 | 326 | } |
| 323 | const answer = await fetch(simulation + path, { cache: "no-store" }); | |
| 327 | const answer = await fetch(simulation + path, { cache: "no-store", signal: AbortSignal.timeout(readTimeoutMs) }); | |
| 324 | 328 | const body = (await answer.json()) as Record<string, unknown>; |
| 325 | return json(res, answer.status, { ...body, source: source() }); | |
| 329 | return { code: answer.status, body: { ...body, source: source() } }; | |
| 330 | }; | |
| 331 | const status = async (res: ServerResponse, path: string) => { | |
| 332 | const { code, body } = await statusOf(path); | |
| 333 | return json(res, code, body); | |
| 334 | }; | |
| 335 | ||
| 336 | /** | |
| 337 | * The same status pushed over /live, so a page needs one socket for the pose, the chat and the queue rather than | |
| 338 | * polling: read once per tick however many watch, sent only when it changed. {"type": "status", "status": {...}|null, "error"?} | |
| 339 | */ | |
| 340 | const statusEveryMs = options.statusEveryMs ?? 200; | |
| 341 | let pushed = ""; | |
| 342 | let pushing = false; | |
| 343 | const push = async () => { | |
| 344 | if (!lobby || pushing) return; | |
| 345 | if (lobby.watching === 0) return void (pushed = ""); // the next page to open the room hears it at once | |
| 346 | pushing = true; | |
| 347 | let text: string; | |
| 348 | try { | |
| 349 | const { code, body } = await statusOf(); | |
| 350 | text = JSON.stringify(code === 200 ? { type: "status", status: body } : { type: "status", status: null, error: body.error ?? `status ${code}` }); | |
| 351 | } catch (err) { | |
| 352 | text = JSON.stringify({ type: "status", status: null, error: reason(err) }); | |
| 353 | } finally { | |
| 354 | pushing = false; | |
| 355 | } | |
| 356 | if (text !== pushed) lobby.publish(text); | |
| 357 | pushed = text; | |
| 326 | 358 | }; |
| 327 | 359 | |
| 328 | 360 | /** A static answer, gzipped: whole, so the length is known; kept when the browser may cache it too. */ |
| @@ -342,6 +374,25 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 342 | 374 | res.end(out); |
| 343 | 375 | }; |
| 344 | 376 | |
| 377 | /** | |
| 378 | * The page, with what its first view needs written into it: which arm this is and the cameras. It starts the camera | |
| 379 | * as soon as its script runs, without asking /link and /cameras first: two round trips fewer on a phone. | |
| 380 | * The JSON is inert data (type application/json), with every "<" escaped so nothing in a name can close the element. | |
| 381 | */ | |
| 382 | const page = async (req: IncomingMessage, res: ServerResponse, { response }: Awaited<ReturnType<typeof ask>>) => { | |
| 383 | const html = await response.text(); | |
| 384 | const boot = JSON.stringify({ link: link(), cameras: cameras.list().map(({ id, name, online }) => ({ id, name, online })) }).replace(/</g, "\\u003c"); | |
| 385 | const body = Buffer.from(response.status === 200 ? html.replace("</head>", `<script type="application/json" id="boot">${boot}</script>\n</head>`) : html); | |
| 386 | const headers: Record<string, string> = { "content-type": response.headers.get("content-type") ?? "text/html; charset=utf-8", "cache-control": "no-store", vary: "accept-encoding" }; | |
| 387 | if (!acceptsGzip(req.headers)) { | |
| 388 | res.writeHead(response.status, headers); | |
| 389 | return res.end(body); | |
| 390 | } | |
| 391 | const out = gzipped.gzip(null, body); | |
| 392 | res.writeHead(response.status, { ...headers, "content-encoding": "gzip", "content-length": String(out.length) }); | |
| 393 | res.end(out); | |
| 394 | }; | |
| 395 | ||
| 345 | 396 | /** The owner's changes to the camera list. */ |
| 346 | 397 | const administer = async (req: IncomingMessage, res: ServerResponse, id: number | null) => { |
| 347 | 398 | const who = lobby ? lobby.who(req) : null; |
| @@ -525,7 +576,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 525 | 576 | const id = Number(camera[1]); |
| 526 | 577 | const source = cameras.source(id); |
| 527 | 578 | if (source === null) return json(res, 404, { error: "no such camera" }); |
| 528 | if (camera[2] === "stream") return cameras.feed(id)!.serve(req, res); | |
| 579 | if (camera[2] === "stream") return cameras.feed(id)!.serve(req, res, parseFps(url.searchParams.get("fps"))); | |
| 529 | 580 | if (camera[2] === "state") return json(res, 200, { ok: cameras.list().find((c) => c.id === id)?.online ?? null }); |
| 530 | 581 | const target = source.replace(/\/[^/]*$/, "/snapshot") + url.search; |
| 531 | 582 | try { |
| @@ -535,6 +586,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 535 | 586 | } |
| 536 | 587 | } |
| 537 | 588 | |
| 589 | if (req.method === "GET" && PAGE.test(url.pathname)) return await page(req, res, await ask(simulation + path, req, undefined, null, false)); | |
| 538 | 590 | if (req.method === "GET" && STATIC.test(url.pathname) && acceptsGzip(req.headers)) return await packed(res, await ask(simulation + path, req, undefined, null, false), path); |
| 539 | 591 | // Who stopped or cleared the arm is the relay's to say, never the page's; every other command is held to the admin's speed cap. |
| 540 | 592 | const raw = reading || route === "POST /estop" || route === "POST /estop/clear" ? undefined : await readBody(req); |
| @@ -686,6 +738,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 686 | 738 | // ------------------------------------------------------------------ a turn that ends |
| 687 | 739 | // Its streams close (the arm server holds the arm when a stream ends) and a move, playback or recording stops. |
| 688 | 740 | const settingsChanged = lobby?.onSettingsChange(follow); |
| 741 | const arrived = lobby?.onArrive((tell) => (pushed ? tell(pushed) : void push())); | |
| 689 | 742 | const turnEnded = lobby?.onTurnEnd((who) => { |
| 690 | 743 | for (const [browser, id] of streams) if (id === who.id) browser.close(1000, "your turn ended"); |
| 691 | 744 | const base = remote && onArm() ? remote.http : simulation; |
| @@ -701,6 +754,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 701 | 754 | await (armLink as ArmLink | null)?.ready; // bounded by the first attempt's timeout: the first page already knows which arm it has |
| 702 | 755 | const timer = lobby ? setInterval(follow, followEveryMs) : undefined; |
| 703 | 756 | timer?.unref?.(); |
| 757 | const pusher = lobby ? setInterval(() => void push(), statusEveryMs) : undefined; | |
| 758 | pusher?.unref?.(); | |
| 704 | 759 | await new Promise<void>((resolve) => http.listen(options.port ?? 8102, options.host ?? "127.0.0.1", resolve)); |
| 705 | 760 | const address = http.address(); |
| 706 | 761 | return { |
| @@ -709,8 +764,10 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 709 | 764 | close: () => |
| 710 | 765 | new Promise<void>((resolve) => { |
| 711 | 766 | clearInterval(timer); |
| 767 | clearInterval(pusher); | |
| 712 | 768 | settingsChanged?.(); |
| 713 | 769 | turnEnded?.(); |
| 770 | arrived?.(); | |
| 714 | 771 | armLink?.close(); |
| 715 | 772 | if (!options.cameras) cameras.close(); |
| 716 | 773 | for (const c of inFlight) c.abort(); |
Software/so101/src/server.test.ts +25 −1
| @@ -3,6 +3,7 @@ import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; | ||
| 3 | 3 | import { connect } from "node:net"; |
| 4 | 4 | import { tmpdir } from "node:os"; |
| 5 | 5 | import { join } from "node:path"; |
| 6 | import { gzipSync } from "node:zlib"; | |
| 6 | 7 | import WebSocket from "ws"; |
| 7 | 8 | import { FeetechBus } from "./bus.ts"; |
| 8 | 9 | import { FakeBus } from "./fake-bus.ts"; |
| @@ -134,6 +135,24 @@ describe("viewer page", () => { | ||
| 134 | 135 | expect(html).toContain('$("events").replaceChildren('); |
| 135 | 136 | }); |
| 136 | 137 | |
| 138 | test("a first visit downloads under 50 KB gzipped: the page and the modules it imports up front; the rest comes when needed", async () => { | |
| 139 | const page = await (await fetch(`${base}/`)).text(); | |
| 140 | const script = page.slice(page.indexOf('<script type="module">')); | |
| 141 | const upfront = new Set([...page.matchAll(/<link rel="modulepreload" href="([^"]+)">/g), ...script.matchAll(/^\s*import [^;]*? from "([^"]+)";/gm)].map((m) => m[1]!)); | |
| 142 | expect(upfront.size).toBeGreaterThan(0); | |
| 143 | let bytes = gzipSync(page, { level: 6 }).length; | |
| 144 | for (const path of upfront) { | |
| 145 | const r = await fetch(`${base}${path}`); | |
| 146 | expect(r.status).toBe(200); | |
| 147 | bytes += gzipSync(Buffer.from(await r.arrayBuffer()), { level: 6 }).length; | |
| 148 | } | |
| 149 | expect(bytes).toBeLessThan(50_000); | |
| 150 | // three.js, the leader's mapping and the calibration panel are imported where they are used, not up front. | |
| 151 | for (const lazy of ["three", "/leader-map.js", "calibration.js", "reach.js"]) expect([...upfront].some((p) => p === lazy || p.endsWith(`/${lazy}`))).toBe(false); | |
| 152 | expect(script).toContain('import("/leader-map.js")'); | |
| 153 | expect(script).toMatch(/import\("\/ui\/[0-9a-f]{12}\/calibration\.js"\)/); | |
| 154 | }); | |
| 155 | ||
| 137 | 156 | test("names its modules by their content, serves them for good, and serves nothing else from src/ui", async () => { |
| 138 | 157 | const page = await (await fetch(`${base}/`)).text(); |
| 139 | 158 | expect(page).not.toContain("__VERSION__"); |
| @@ -172,7 +191,12 @@ describe("viewer page", () => { | ||
| 172 | 191 | expect(page).toMatch(/\.stick \{[^}]*touch-action: none/); |
| 173 | 192 | expect(page).toMatch(/#grips button \{[^}]*min-height: (7[2-9]|[89]\d)px/); |
| 174 | 193 | expect(page).toMatch(/#chips button small \{[^}]*white-space: nowrap/); // a long label never wraps and shifts the row |
| 175 | expect(page).toMatch(/@media \(max-width: 760px\), \(max-height: 500px\) \{ #stop \{ display: none/); // the E-stop is the one stop on a phone | |
| 194 | // One "Stop": the E-stop. The other button starts hidden, comes up only while a move, a playback, a recording or a park | |
| 195 | // runs (which the phone had no other way to end), and is named for what it ends. | |
| 196 | expect(page).toContain('<button id="stop" class="danger" hidden'); | |
| 197 | expect(page).toContain('const ENDS_LABEL = { move: "End move", play: "End playback", record: "End recording", park: "End park" };'); | |
| 198 | expect(page).toContain('$("stop").hidden = !label || !(mayDrive() || isOwner());'); | |
| 199 | expect(page.match(/>Stop</g)?.length).toBe(1); | |
| 176 | 200 | }); |
| 177 | 201 | }); |
| 178 | 202 |
Software/so101/src/ui/calibration.js added +109
| @@ -0,0 +1,109 @@ | ||
| 1 | // @ts-check | |
| 2 | /** | |
| 3 | * The 3D calibration panel: what to print, what is solved, and the steps on the computer that runs the vision kit. | |
| 4 | * Only admins open it, so it is not part of the page's first load: the page imports this module on the first click. | |
| 5 | * The words are fixed; everything that comes from the server goes in as text nodes. | |
| 6 | */ | |
| 7 | ||
| 8 | const ARTICLE = ` | |
| 9 | <article> | |
| 10 | <h2>3D calibration: teach the arm where the camera is</h2> | |
| 11 | <p>The camera sees a marker. The arm needs that as a position in its own space. Two calibrations give it that.</p> | |
| 12 | <ol> | |
| 13 | <li><b>Camera.</b> The lens geometry: focal length and distortion. Found from 15 views of the printed <em>ChArUco board</em>: a checkerboard with a small marker in each white square. Download it below and print it at 100 %. Output: <code>camera.json</code>.</li> | |
| 14 | <li><b>Hand-eye.</b> The arm holds a marker in 6 to 8 poses. The camera says where the marker is. The arm's model says where the gripper is. The solver finds the camera's position relative to the arm's base. Output: <code>handeye.json</code>.</li> | |
| 15 | </ol> | |
| 16 | <p>After both, a cube the camera sees becomes a target the arm can reach. With two cameras, each one gets its own pair of calibrations (<code>camera2</code>, <code>handeye2</code>); two views then give a full orientation.</p> | |
| 17 | <h3>Print these</h3> | |
| 18 | <p>Three sheets, at 100 % scale on matte paper. Check with a ruler: a cube marker's black square is 40 mm, a board square 30 mm. Tape the six cube markers on a 40 to 50 mm block, and the gripper marker flat on the outside of the fixed jaw.</p> | |
| 19 | <div class="row" id="printables">loading…</div> | |
| 20 | <h3>Status</h3> | |
| 21 | <div class="status" id="calStatus"></div> | |
| 22 | <h3>Steps, on the Mac</h3> | |
| 23 | <pre id="calSteps"></pre> | |
| 24 | <p>For the poses: press <b>Release</b>, move the arm by hand, press <b>Save pose</b>, and uncheck <em>with gripper</em>. The hand-eye poses need different tilts and rolls of the gripper, with the marker facing the camera.</p> | |
| 25 | <h3>References</h3> | |
| 26 | <ul class="refs"> | |
| 27 | <li>Camera calibration: Z. Zhang, <em>A Flexible New Technique for Camera Calibration</em>, IEEE TPAMI 2000, <a href="https://doi.org/10.1109/34.888718" target="_blank" rel="noopener">doi:10.1109/34.888718</a>. OpenCV tutorial: <a href="https://docs.opencv.org/4.x/da/d13/tutorial_aruco_calibration.html" target="_blank" rel="noopener">Calibration with ArUco and ChArUco</a>.</li> | |
| 28 | <li>ArUco markers and ChArUco boards: S. Garrido-Jurado et al., <em>Automatic generation and detection of highly reliable fiducial markers under occlusion</em>, Pattern Recognition 2014, <a href="https://doi.org/10.1016/j.patcog.2014.01.005" target="_blank" rel="noopener">doi:10.1016/j.patcog.2014.01.005</a>.</li> | |
| 29 | <li>Pose of one square marker: T. Collins, A. Bartoli, <em>Infinitesimal Plane-Based Pose Estimation</em>, IJCV 2014, <a href="https://doi.org/10.1007/s11263-014-0725-5" target="_blank" rel="noopener">doi:10.1007/s11263-014-0725-5</a> (OpenCV's <code>SOLVEPNP_IPPE_SQUARE</code>).</li> | |
| 30 | <li>Hand-eye calibration: Tsai & Lenz 1989 <a href="https://doi.org/10.1109/70.34770" target="_blank" rel="noopener">doi:10.1109/70.34770</a>; Park & Martin 1994 <a href="https://doi.org/10.1109/70.326576" target="_blank" rel="noopener">doi:10.1109/70.326576</a>; Daniilidis 1999 <a href="https://doi.org/10.1177/02783649922066213" target="_blank" rel="noopener">doi:10.1177/02783649922066213</a>.</li> | |
| 31 | <li>Robot-world and hand-eye in one solve, what this kit uses: M. Shah 2013, <a href="https://doi.org/10.1115/1.4024473" target="_blank" rel="noopener">doi:10.1115/1.4024473</a>; A. Li, L. Wang, D. Wu 2010. OpenCV: <a href="https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html" target="_blank" rel="noopener"><code>calibrateRobotWorldHandEye</code></a>.</li> | |
| 32 | <li>Two cameras, a full orientation: Hartley & Zisserman, <em>Multiple View Geometry in Computer Vision</em>, 2nd ed. 2004 (stereo calibration, triangulation).</li> | |
| 33 | </ul> | |
| 34 | <h3>About</h3> | |
| 35 | <p>This page drives an <a href="https://github.com/TheRobotStudio/SO-ARM100" target="_blank" rel="noopener">SO-101</a>, the open robot arm by The Robot Studio, with the calibration conventions of <a href="https://huggingface.co/docs/lerobot/so101" target="_blank" rel="noopener">Hugging Face LeRobot</a>. The 3D view uses the arm's own URDF through <a href="https://github.com/gkjohnson/urdf-loaders" target="_blank" rel="noopener">urdf-loader</a> and three.js. The code, the build notes and the quest are in the repository's <code>Software/</code> folder.</p> | |
| 36 | </article>`; | |
| 37 | ||
| 38 | /** The page's document: this module only runs there (the type check has no DOM, as for src/ui/drive.js). @type {any} */ | |
| 39 | const doc = /** @type {any} */ (globalThis).document; | |
| 40 | ||
| 41 | /** @param {string} tag @param {Record<string, unknown>} [props] @param {...any} children @returns {any} */ | |
| 42 | const el = (tag, props = {}, ...children) => { const e = Object.assign(doc.createElement(tag), props); e.append(...children); return e; }; | |
| 43 | ||
| 44 | /** The panel, built once and kept. @returns {any} */ | |
| 45 | function panel() { | |
| 46 | const found = doc.getElementById("panel"); | |
| 47 | if (found) return found; | |
| 48 | const close = el("button", { className: "close", textContent: "Close" }); | |
| 49 | const box = el("div", { id: "panel", className: "panel", hidden: true }, close); | |
| 50 | box.insertAdjacentHTML("beforeend", ARTICLE); // fixed markup from this file, no data in it | |
| 51 | close.onclick = () => (box.hidden = true); | |
| 52 | (doc.getElementById("shell") ?? doc.body).append(box); | |
| 53 | return box; | |
| 54 | } | |
| 55 | ||
| 56 | /** | |
| 57 | * Open the panel and fill it from the arm. | |
| 58 | * @param {{ api: (path: string) => Promise<any>, host: string, cameras: string[] }} page | |
| 59 | */ | |
| 60 | export async function openCalibration({ api, host, cameras }) { | |
| 61 | const box = panel(); | |
| 62 | box.hidden = false; | |
| 63 | const $ = (/** @type {string} */ id) => box.querySelector(`#${id}`); | |
| 64 | $("calSteps").textContent = steps(host, cameras[0] ?? null).join("\n"); | |
| 65 | const [sheets, cal, saved] = await Promise.all([api("/printables"), api("/calibration"), api("/poses")]); | |
| 66 | $("printables").replaceChildren(...(Array.isArray(sheets) && sheets.length | |
| 67 | ? sheets.map((/** @type {any} */ s) => el("a", { href: String(s.url), download: String(s.file) }, el("button", { className: "go", textContent: `⬇ ${String(s.file).replace(/_/g, " ").replace(/\.png$/, "")}` }))) | |
| 68 | : [el("span", { textContent: "no printables on this server: copy Software/vision/printables next to the package" })])); | |
| 69 | $("calStatus").replaceChildren(...statusRows(cal, saved, cameras.length).flatMap(([what, done, text]) => [el("b", { textContent: what }), el("span", {}, el("span", { className: done ? "ok" : "no", textContent: text }))])); | |
| 70 | } | |
| 71 | ||
| 72 | /** The commands, with this arm's address and its first camera filled in. @param {string} host @param {string | null} camera */ | |
| 73 | export function steps(host, camera) { | |
| 74 | return [ | |
| 75 | `export SO101_ARM=http://${host}`, | |
| 76 | `export SO101_CAMERA=${camera ?? "<the camera's stream URL>"}`, | |
| 77 | `just vision-calibrate # hold the printed board in front of the camera, tilt it`, | |
| 78 | `# save poses from this page: home, drop, he1 … he8 (without gripper)`, | |
| 79 | `just vision-handeye he1,he2,he3,he4,he5,he6,he7,he8`, | |
| 80 | `just vision-rehearse # sees the cube, plans the joints, moves nothing`, | |
| 81 | `just vision-pick`, | |
| 82 | ``, | |
| 83 | `# a second camera: SO101_CAMERA=<its stream URL> and --publish-as camera2 / handeye2`, | |
| 84 | ]; | |
| 85 | } | |
| 86 | ||
| 87 | /** | |
| 88 | * What is done and what is missing, one row each: [what, done, how it stands]. | |
| 89 | * @param {Record<string, any>} cal @param {Record<string, { ids: number[] }>} saved @param {number} cameraCount | |
| 90 | * @returns {Array<[string, boolean, string]>} | |
| 91 | */ | |
| 92 | export function statusRows(cal, saved, cameraCount) { | |
| 93 | const he = Object.keys(saved).filter((n) => /^he\d+$/.test(n) && !saved[n]?.ids.includes(6)).length; | |
| 94 | /** @returns {[boolean, string]} */ | |
| 95 | const armOnly = (/** @type {string} */ name) => !saved[name] ? [false, "missing"] : saved[name].ids.includes(6) ? [false, "saved with the gripper: save it again without"] : [true, "saved"]; | |
| 96 | /** @returns {[boolean, string]} */ | |
| 97 | const camLine = (/** @type {any} */ c) => c ? [true, `calibrated ${String(c.saved ?? "").slice(0, 10)} · ${c.width}×${c.height} · ${Number(c.rms).toFixed(2)} px error`] : [false, "not yet"]; | |
| 98 | const mean = (/** @type {unknown} */ r) => Array.isArray(r) && r.length ? (r.reduce((a, b) => a + Number(b), 0) / r.length).toFixed(1) : "?"; | |
| 99 | /** @returns {[boolean, string]} */ | |
| 100 | const heLine = (/** @type {any} */ h) => h ? [true, `solved ${String(h.saved ?? "").slice(0, 10)} · ${h.poses?.length ?? "?"} poses · mean ${mean(h.residuals_mm)} mm`] : [false, "not yet"]; | |
| 101 | /** @type {Array<[string, [boolean, string]]>} */ | |
| 102 | const rows = [ | |
| 103 | ["camera 1", camLine(cal.camera)], ["hand-eye 1", heLine(cal.handeye)], | |
| 104 | ...(cameraCount < 2 ? [] : /** @type {Array<[string, [boolean, string]]>} */ ([["camera 2", camLine(cal.camera2)], ["hand-eye 2", heLine(cal.handeye2)]])), | |
| 105 | ["pose home", armOnly("home")], ["pose drop", armOnly("drop")], | |
| 106 | ["poses he1…", [he >= 6, he >= 6 ? `${he} saved` : `${he} of 6 to 8 saved (without the gripper)`]], | |
| 107 | ]; | |
| 108 | return rows.map(([what, [done, text]]) => [what, done, text]); | |
| 109 | } |
Software/so101/src/ui/calibration.test.ts added +23
| @@ -0,0 +1,23 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { statusRows, steps } from "./calibration.js"; | |
| 3 | ||
| 4 | describe("the calibration panel", () => { | |
| 5 | test("the steps carry this arm's address and its first camera, or say what to put there", () => { | |
| 6 | expect(steps("arm.example:8102", "https://arm.example/camera/1/stream").slice(0, 2)).toEqual(["export SO101_ARM=http://arm.example:8102", "export SO101_CAMERA=https://arm.example/camera/1/stream"]); | |
| 7 | expect(steps("h", null)[1]).toBe("export SO101_CAMERA=<the camera's stream URL>"); | |
| 8 | }); | |
| 9 | ||
| 10 | test("says what is done and what is missing; a hand-eye pose saved with the gripper does not count", () => { | |
| 11 | const he = Object.fromEntries([1, 2, 3, 4, 5].map((n) => [`he${n}`, { ids: [1, 2, 3, 4, 5] }])); | |
| 12 | const rows = statusRows({ camera: { saved: "2026-10-01T10:00:00Z", width: 1280, height: 720, rms: 0.4123 }, handeye: null }, { ...he, he6: { ids: [1, 2, 3, 4, 5, 6] }, home: { ids: [1, 2, 3, 4, 5, 6] } }, 1); | |
| 13 | expect(rows).toEqual([ | |
| 14 | ["camera 1", true, "calibrated 2026-10-01 · 1280×720 · 0.41 px error"], | |
| 15 | ["hand-eye 1", false, "not yet"], | |
| 16 | ["pose home", false, "saved with the gripper: save it again without"], | |
| 17 | ["pose drop", false, "missing"], | |
| 18 | ["poses he1…", false, "5 of 6 to 8 saved (without the gripper)"], | |
| 19 | ]); | |
| 20 | expect(statusRows({ handeye: { saved: "2026-10-02", poses: [1, 2, 3], residuals_mm: [1, 2] } }, {}, 2).map((r) => r[0])).toEqual(["camera 1", "hand-eye 1", "camera 2", "hand-eye 2", "pose home", "pose drop", "poses he1…"]); | |
| 21 | expect(statusRows({ handeye: { saved: "2026-10-02", poses: [1, 2, 3], residuals_mm: [1, 2] } }, {}, 1)[1]).toEqual(["hand-eye 1", true, "solved 2026-10-02 · 3 poses · mean 1.5 mm"]); | |
| 22 | }); | |
| 23 | }); |
Software/so101/src/ui/camera-plan.js +37 −13
| @@ -1,30 +1,54 @@ | ||
| 1 | 1 | /** |
| 2 | * How the page fetches a camera on the connection it has: the full MJPEG stream on a good link, or one | |
| 3 | * snapshot at a time, further apart the slower the link. Plain ESM, served at /ui/ and imported by the tests. | |
| 2 | * How the page fetches a camera on the connection it has: the full MJPEG stream on a good link; on a slow one the | |
| 3 | * stream held to a few frames a second where the relay can do that (`/camera/<id>/stream?fps=`), and one snapshot | |
| 4 | * at a time where it cannot or the link is very slow. Plain ESM, served at /ui/ and imported by the tests. | |
| 4 | 5 | * |
| 5 | * The signals are the browser's own estimate where it gives one (navigator.connection: Chrome on Android) | |
| 6 | * and the round trips of the page's /status polls, which every browser has. | |
| 6 | * The signals are the viewer's own choice (the quality button: auto, saver, full), the browser's estimate where it | |
| 7 | * gives one (navigator.connection: Chrome on Android) and the round trips the page measures, which every browser has. | |
| 7 | 8 | */ |
| 8 | 9 | |
| 9 | 10 | /** Round trips above this mean the stream would queue up behind itself. */ |
| 10 | 11 | export const SLOW_RTT_MS = 400; |
| 11 | 12 | /** And below this the stream comes back. The gap keeps the plan from flapping. */ |
| 12 | 13 | export const FAST_RTT_MS = 200; |
| 14 | /** Round trips above this, or a 2G link, get snapshots even where the stream could be held to a lower rate. */ | |
| 15 | export const CRAWL_RTT_MS = 1000; | |
| 16 | /** Frames a second of a held stream: on a slow link, and when the viewer or the browser asks to save data. */ | |
| 17 | export const SLOW_FPS = 5; | |
| 18 | export const SAVER_FPS = 2; | |
| 19 | ||
| 20 | /** The quality choices, in the order the button steps through them. */ | |
| 21 | export const QUALITIES = /** @type {const} */ (["auto", "saver", "full"]); | |
| 13 | 22 | |
| 14 | 23 | /** |
| 24 | * @typedef {{ mode: "stream", fps?: number } | { mode: "snapshot", everyMs: number }} Plan | |
| 15 | 25 | * @param {{ effectiveType?: string, saveData?: boolean, downlink?: number } | null | undefined} net |
| 16 | * @param {number[]} rtts recent /status round trips, milliseconds | |
| 17 | * @param {"stream" | "snapshot"} [current] | |
| 18 | * @returns {{ mode: "stream" } | { mode: "snapshot", everyMs: number }} | |
| 26 | * @param {number[]} rtts recent round trips, milliseconds | |
| 27 | * @param {"stream" | "snapshot" | Plan} [current] the plan in force (or its mode): once held back, the link must be clearly fast to come back | |
| 28 | * @param {{ quality?: string, throttle?: boolean }} [options] the viewer's choice; whether the stream takes ?fps= (through the relay) | |
| 29 | * @returns {Plan} | |
| 19 | 30 | */ |
| 20 | export function cameraPlan(net, rtts, current = "stream") { | |
| 31 | export function cameraPlan(net, rtts, current = "stream", options = {}) { | |
| 32 | const quality = QUALITIES.includes(/** @type {any} */ (options.quality)) ? options.quality : "auto"; | |
| 33 | const throttle = options.throttle === true; | |
| 34 | if (quality === "full") return { mode: "stream" }; | |
| 21 | 35 | const rtt = median(rtts); |
| 22 | const told = net?.saveData === true || net?.effectiveType === "slow-2g" || net?.effectiveType === "2g" || net?.effectiveType === "3g" || (typeof net?.downlink === "number" && net.downlink > 0 && net.downlink < 1.5); | |
| 23 | const slow = told || (rtt !== null && rtt > (current === "snapshot" ? FAST_RTT_MS : SLOW_RTT_MS)); | |
| 36 | if (quality === "saver") return throttle ? { mode: "stream", fps: SAVER_FPS } : snapshots(Math.max(rtt ?? 0, 125)); | |
| 37 | const crawling = net?.effectiveType === "slow-2g" || net?.effectiveType === "2g" || (rtt !== null && rtt > CRAWL_RTT_MS); | |
| 38 | const told = net?.saveData === true || net?.effectiveType === "3g" || (typeof net?.downlink === "number" && net.downlink > 0 && net.downlink < 1.5); | |
| 39 | const held = typeof current === "string" ? current !== "stream" : current.mode !== "stream" || current.fps !== undefined; | |
| 40 | const slow = crawling || told || (rtt !== null && rtt > (held ? FAST_RTT_MS : SLOW_RTT_MS)); | |
| 24 | 41 | if (!slow) return { mode: "stream" }; |
| 25 | // A frame every four round trips, between 4 and 0.5 frames a second. | |
| 26 | const everyMs = Math.round(Math.max(250, Math.min(2000, 4 * (rtt ?? SLOW_RTT_MS)))); | |
| 27 | return { mode: "snapshot", everyMs }; | |
| 42 | if (throttle && !crawling) return { mode: "stream", fps: net?.saveData === true ? SAVER_FPS : SLOW_FPS }; | |
| 43 | return snapshots(rtt ?? SLOW_RTT_MS); | |
| 44 | } | |
| 45 | ||
| 46 | /** A frame every four round trips, between 4 and 0.5 frames a second. @param {number} rtt @returns {Plan} */ | |
| 47 | const snapshots = (rtt) => ({ mode: "snapshot", everyMs: Math.round(Math.max(250, Math.min(2000, 4 * rtt))) }); | |
| 48 | ||
| 49 | /** The same plan, so nothing need restart. @param {Plan} a @param {Plan} b */ | |
| 50 | export function samePlan(a, b) { | |
| 51 | return a.mode === b.mode && (a.mode === "stream" ? a.fps === /** @type {any} */ (b).fps : a.everyMs === /** @type {any} */ (b).everyMs); | |
| 28 | 52 | } |
| 29 | 53 | |
| 30 | 54 | /** @param {number[]} values */ |
Software/so101/src/ui/camera-plan.test.ts +36 −1
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | import { describe, expect, test } from "bun:test"; |
| 2 | import { cameraPlan } from "./camera-plan.js"; | |
| 2 | import { cameraPlan, samePlan, SAVER_FPS, SLOW_FPS } from "./camera-plan.js"; | |
| 3 | 3 | |
| 4 | 4 | describe("the camera plan", () => { |
| 5 | 5 | test("a good link, or too little to judge, gets the full stream", () => { |
| @@ -25,3 +25,38 @@ describe("the camera plan", () => { | ||
| 25 | 25 | expect(cameraPlan(null, [120, 150, 130], "snapshot").mode).toBe("stream"); |
| 26 | 26 | }); |
| 27 | 27 | }); |
| 28 | ||
| 29 | describe("the camera plan through the relay, which can hold a stream to fewer frames", () => { | |
| 30 | const relay = { throttle: true }; | |
| 31 | test("a slow link keeps the stream, at a few frames a second; saving data, fewer still", () => { | |
| 32 | expect(cameraPlan(null, [500, 520, 480], "stream", relay)).toEqual({ mode: "stream", fps: SLOW_FPS }); | |
| 33 | expect(cameraPlan({ effectiveType: "3g" }, [], "stream", relay)).toEqual({ mode: "stream", fps: SLOW_FPS }); | |
| 34 | expect(cameraPlan({ saveData: true }, [30, 30, 30], "stream", relay)).toEqual({ mode: "stream", fps: SAVER_FPS }); | |
| 35 | expect(cameraPlan({ effectiveType: "4g" }, [40, 50, 60], "stream", relay)).toEqual({ mode: "stream" }); | |
| 36 | }); | |
| 37 | ||
| 38 | test("a crawl (2G, or round trips over a second) still gets snapshots", () => { | |
| 39 | expect(cameraPlan({ effectiveType: "2g" }, [], "stream", relay).mode).toBe("snapshot"); | |
| 40 | expect(cameraPlan(null, [1500, 1600, 1400], "stream", relay)).toEqual({ mode: "snapshot", everyMs: 2000 }); | |
| 41 | }); | |
| 42 | ||
| 43 | test("a held stream comes back to every frame only once the link is clearly fast", () => { | |
| 44 | const held = { mode: "stream", fps: SLOW_FPS } as const; | |
| 45 | expect(cameraPlan(null, [300, 300, 300], held, relay)).toEqual(held); | |
| 46 | expect(cameraPlan(null, [120, 150, 130], held, relay)).toEqual({ mode: "stream" }); | |
| 47 | }); | |
| 48 | ||
| 49 | test("the viewer's choice wins: full is every frame on any link, saver is the fewest", () => { | |
| 50 | expect(cameraPlan({ effectiveType: "2g", saveData: true }, [2000, 2000, 2000], "stream", { ...relay, quality: "full" })).toEqual({ mode: "stream" }); | |
| 51 | expect(cameraPlan({ effectiveType: "4g" }, [20, 20, 20], "stream", { ...relay, quality: "saver" })).toEqual({ mode: "stream", fps: SAVER_FPS }); | |
| 52 | expect(cameraPlan(null, [], "stream", { quality: "saver" }).mode).toBe("snapshot"); // straight to an arm server: no ?fps= | |
| 53 | expect(cameraPlan(null, [500, 500, 500], "stream", { ...relay, quality: "bogus" })).toEqual({ mode: "stream", fps: SLOW_FPS }); // unknown is auto | |
| 54 | }); | |
| 55 | ||
| 56 | test("samePlan tells whether anything need restart", () => { | |
| 57 | expect(samePlan({ mode: "stream" }, { mode: "stream" })).toBe(true); | |
| 58 | expect(samePlan({ mode: "stream" }, { mode: "stream", fps: 5 })).toBe(false); | |
| 59 | expect(samePlan({ mode: "snapshot", everyMs: 500 }, { mode: "snapshot", everyMs: 500 })).toBe(true); | |
| 60 | expect(samePlan({ mode: "snapshot", everyMs: 500 }, { mode: "snapshot", everyMs: 750 })).toBe(false); | |
| 61 | }); | |
| 62 | }); |
Software/so101/src/viewer.html +120 −103
| @@ -20,7 +20,6 @@ | ||
| 20 | 20 | <link rel="modulepreload" href="/ui/__VERSION__/camera-plan.js"> |
| 21 | 21 | <link rel="modulepreload" href="/ui/__VERSION__/drive.js"> |
| 22 | 22 | <link rel="modulepreload" href="/ui/__VERSION__/frames.js"> |
| 23 | <link rel="modulepreload" href="/leader-map.js"> | |
| 24 | 23 | <style> |
| 25 | 24 | /* Phone portrait first: the view on top, a status strip with the E-stop under it, a sheet of tabs below that. |
| 26 | 25 | Landscape on a phone: the view fills the screen and the strip and the sheet float over it. |
| @@ -178,6 +177,7 @@ | ||
| 178 | 177 | #oneName { font-size: 20px; font-weight: 700; } #onePos { font-size: 28px; font-weight: 650; font-variant-numeric: tabular-nums; } |
| 179 | 178 | #oneHint { color: var(); font-size: 12px; } |
| 180 | 179 | #lag { color: #f2cc60; font-variant-numeric: tabular-nums; } |
| 180 | #chat .earlier { display: block; margin: 2px auto 8px; font-size: 13px; } | |
| 181 | 181 | /* on a phone on its side: two thumb sticks over the picture, the gripper between them */ |
| 182 | 182 | #sticks { display: none; position: absolute; inset: 0; z-index: 6; pointer-events: none; : min(40vh, 164px); } |
| 183 | 183 | #sticks > * { pointer-events: auto; } |
| @@ -270,8 +270,7 @@ | ||
| 270 | 270 | #pad { width: min(30vh, 150px); } |
| 271 | 271 | } |
| 272 | 272 | @media (max-width: 640px) and (orientation: portrait) { #touch { display: grid; } #reach, #joints { display: none ; } } |
| 273 | /* One stop on a phone: the E-stop. Letting go stops the joints; Hold in Controls ends a move or a playback. */ | |
| 274 | @media (max-width: 760px), (max-height: 500px) { #stop { display: none ; } } | |
| 273 | /* One Stop: the E-stop. The other button appears only while a move, a playback, a recording or a park runs, and says which it ends. */ | |
| 275 | 274 | @media (orientation: landscape) and (max-height: 500px) { |
| 276 | 275 | #shell.driving.bare #sticks { display: block; } |
| 277 | 276 | #shell.driving.bare #top { top: 6px; bottom: auto; left: auto; right: calc(6px + var()); max-width: 42%; justify-content: flex-end; } |
| @@ -291,6 +290,7 @@ | ||
| 291 | 290 | <div id="top"> |
| 292 | 291 | <span class="seg" role="group" aria-label="Views"><button id="v3" aria-pressed="false">3D</button></span> |
| 293 | 292 | <span class="info"><span id="link" hidden></span><span id="live" class="label"></span><span class="dot" id="dot"></span><span id="state">…</span><span id="lag" hidden></span></span> |
| 293 | <button id="quality" type="button" hidden title="Video quality: Auto follows the connection, Saver sends 2 frames a second, Full every frame">Auto</button> | |
| 294 | 294 | <button id="fs" title="F">Fullscreen</button> |
| 295 | 295 | </div> |
| 296 | 296 | <div id="driverTag" aria-hidden="true"></div> |
| @@ -309,7 +309,7 @@ | ||
| 309 | 309 | <div class="who"><div id="driverLine" class="none">Connecting…</div><div id="clock" aria-live="off"></div><div id="capLine"></div></div> |
| 310 | 310 | <div id="turnActions"></div> |
| 311 | 311 | <button id="sheetToggle" hidden aria-controls="sheet" aria-expanded="true">Hide</button> |
| 312 | <button id="stop" class="danger" hidden title="End what the arm is doing and hold (space)">Stop</button> | |
| 312 | <button id="stop" class="danger" hidden title="End what the arm is doing and hold (space)">End move</button> | |
| 313 | 313 | <button id="estop" class="estop" title="Emergency stop (Esc): halt, hold with torque, until an admin clears it" aria-label="Emergency stop">Stop</button> |
| 314 | 314 | </section> |
| 315 | 315 | |
| @@ -448,38 +448,6 @@ | ||
| 448 | 448 | </article> |
| 449 | 449 | </div> |
| 450 | 450 | |
| 451 | <div id="panel" class="panel" hidden> | |
| 452 | <button class="close" data-close="panel">Close</button> | |
| 453 | <article> | |
| 454 | <h2>3D calibration: teach the arm where the camera is</h2> | |
| 455 | <p>The camera sees a marker. The arm needs that as a position in its own space. Two calibrations give it that.</p> | |
| 456 | <ol> | |
| 457 | <li><b>Camera.</b> The lens geometry: focal length and distortion. Found from 15 views of the printed <em>ChArUco board</em>: a checkerboard with a small marker in each white square. Download it below and print it at 100 %. Output: <code>camera.json</code>.</li> | |
| 458 | <li><b>Hand-eye.</b> The arm holds a marker in 6 to 8 poses. The camera says where the marker is. The arm's model says where the gripper is. The solver finds the camera's position relative to the arm's base. Output: <code>handeye.json</code>.</li> | |
| 459 | </ol> | |
| 460 | <p>After both, a cube the camera sees becomes a target the arm can reach. With two cameras, each one gets its own pair of calibrations (<code>camera2</code>, <code>handeye2</code>); two views then give a full orientation.</p> | |
| 461 | <h3>Print these</h3> | |
| 462 | <p>Three sheets, at 100 % scale on matte paper. Check with a ruler: a cube marker's black square is 40 mm, a board square 30 mm. Tape the six cube markers on a 40 to 50 mm block, and the gripper marker flat on the outside of the fixed jaw.</p> | |
| 463 | <div class="row" id="printables">loading…</div> | |
| 464 | <h3>Status</h3> | |
| 465 | <div class="status" id="calStatus"></div> | |
| 466 | <h3>Steps, on the Mac</h3> | |
| 467 | <pre id="calSteps"></pre> | |
| 468 | <p>For the poses: press <b>Release</b>, move the arm by hand, press <b>Save pose</b>, and uncheck <em>with gripper</em>. The hand-eye poses need different tilts and rolls of the gripper, with the marker facing the camera.</p> | |
| 469 | <h3>References</h3> | |
| 470 | <ul class="refs"> | |
| 471 | <li>Camera calibration: Z. Zhang, <em>A Flexible New Technique for Camera Calibration</em>, IEEE TPAMI 2000, <a href="https://doi.org/10.1109/34.888718" target="_blank" rel="noopener">doi:10.1109/34.888718</a>. OpenCV tutorial: <a href="https://docs.opencv.org/4.x/da/d13/tutorial_aruco_calibration.html" target="_blank" rel="noopener">Calibration with ArUco and ChArUco</a>.</li> | |
| 472 | <li>ArUco markers and ChArUco boards: S. Garrido-Jurado et al., <em>Automatic generation and detection of highly reliable fiducial markers under occlusion</em>, Pattern Recognition 2014, <a href="https://doi.org/10.1016/j.patcog.2014.01.005" target="_blank" rel="noopener">doi:10.1016/j.patcog.2014.01.005</a>.</li> | |
| 473 | <li>Pose of one square marker: T. Collins, A. Bartoli, <em>Infinitesimal Plane-Based Pose Estimation</em>, IJCV 2014, <a href="https://doi.org/10.1007/s11263-014-0725-5" target="_blank" rel="noopener">doi:10.1007/s11263-014-0725-5</a> (OpenCV's <code>SOLVEPNP_IPPE_SQUARE</code>).</li> | |
| 474 | <li>Hand-eye calibration: Tsai & Lenz 1989 <a href="https://doi.org/10.1109/70.34770" target="_blank" rel="noopener">doi:10.1109/70.34770</a>; Park & Martin 1994 <a href="https://doi.org/10.1109/70.326576" target="_blank" rel="noopener">doi:10.1109/70.326576</a>; Daniilidis 1999 <a href="https://doi.org/10.1177/02783649922066213" target="_blank" rel="noopener">doi:10.1177/02783649922066213</a>.</li> | |
| 475 | <li>Robot-world and hand-eye in one solve, what this kit uses: M. Shah 2013, <a href="https://doi.org/10.1115/1.4024473" target="_blank" rel="noopener">doi:10.1115/1.4024473</a>; A. Li, L. Wang, D. Wu 2010. OpenCV: <a href="https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html" target="_blank" rel="noopener"><code>calibrateRobotWorldHandEye</code></a>.</li> | |
| 476 | <li>Two cameras, a full orientation: Hartley & Zisserman, <em>Multiple View Geometry in Computer Vision</em>, 2nd ed. 2004 (stereo calibration, triangulation).</li> | |
| 477 | </ul> | |
| 478 | <h3>About</h3> | |
| 479 | <p>This page drives an <a href="https://github.com/TheRobotStudio/SO-ARM100" target="_blank" rel="noopener">SO-101</a>, the open robot arm by The Robot Studio, with the calibration conventions of <a href="https://huggingface.co/docs/lerobot/so101" target="_blank" rel="noopener">Hugging Face LeRobot</a>. The 3D view uses the arm's own URDF through <a href="https://github.com/gkjohnson/urdf-loaders" target="_blank" rel="noopener">urdf-loader</a> and three.js. The code, the build notes and the quest are in the repository's <code>Software/</code> folder.</p> | |
| 480 | </article> | |
| 481 | </div> | |
| 482 | ||
| 483 | 451 | <dialog id="saveDlg"> |
| 484 | 452 | <form method="dialog"> |
| 485 | 453 | <h3>Save the current pose</h3> |
| @@ -527,10 +495,9 @@ | ||
| 527 | 495 | </dialog> |
| 528 | 496 | </div> |
| 529 | 497 | <script type="module"> |
| 530 | import { cameraPlan } from "/ui/__VERSION__/camera-plan.js"; | |
| 498 | import { cameraPlan, QUALITIES, samePlan } from "/ui/__VERSION__/camera-plan.js"; | |
| 531 | 499 | import { bindSurface, canDrive, Driver, shape, UP } from "/ui/__VERSION__/drive.js"; |
| 532 | 500 | import { Frames } from "/ui/__VERSION__/frames.js"; |
| 533 | import { mapLeaderFrame, parseCalibration } from "/leader-map.js"; | |
| 534 | 501 | |
| 535 | 502 | const $ = (id) => document.getElementById(id); |
| 536 | 503 | const q = new URLSearchParams(location.search); |
| @@ -587,7 +554,9 @@ | ||
| 587 | 554 | |
| 588 | 555 | // ---------------------------------------------------------------- which arm |
| 589 | 556 | // Behind the hosted app's relay (/link answers), the arm is a live `so101 serve` or the simulation, and the cameras come through the relay. |
| 590 | const relayed = await fetch("/link", { cache: "no-store" }).then((r) => r.ok, () => false); | |
| 557 | // The relay writes the link and the camera list into the page (#boot), so the first view needs no round trip to start. | |
| 558 | const boot = (() => { try { return JSON.parse($("boot")?.textContent ?? "null"); } catch { return null; } })(); | |
| 559 | const relayed = boot !== null || await fetch("/link", { cache: "no-store" }).then((r) => r.ok, () => false); | |
| 591 | 560 | // A real arm that drops is "reconnecting": the same arm, its last pose kept, driving paused. Only after a minute down |
| 592 | 561 | // does the relay hand the page the simulation, and then it says OFFLINE in red. |
| 593 | 562 | let linkMode = null, linkNow = null; |
| @@ -615,14 +584,28 @@ | ||
| 615 | 584 | // The owner lists the cameras (/admin/camera-list); this page polls the list, so a camera added or removed shows within seconds. |
| 616 | 585 | // Which views are on is this viewer's own choice, kept in this browser: one at a time on a phone, several side by side on a wider screen. |
| 617 | 586 | // On a good link a plain <img> stream, which every browser plays; a canvas comparison once a second notices a frozen |
| 618 | // picture and reconnects. On a slow one (src/ui/camera-plan.js) one snapshot at a time, further apart the slower it is. | |
| 587 | // picture and reconnects. On a slow one (src/ui/camera-plan.js) the relay holds the stream to a few frames a second, | |
| 588 | // and on a crawl one snapshot at a time. The quality button overrides it. A hidden page fetches no video at all. | |
| 619 | 589 | const cams = []; |
| 620 | 590 | const direct = (n) => q.get(`stream${n}`) || q.get(n === 1 ? "stream" : "") || `${location.protocol}//${location.hostname}:${8079 + n}/stream`; |
| 621 | 591 | const canvas = document.createElement("canvas"); canvas.width = 32; canvas.height = 18; |
| 622 | 592 | const ctx = canvas.getContext("2d", { willReadFrequently: true }); |
| 623 | 593 | const fresh = (url) => url + (url.includes("?") ? "&" : "?") + Date.now(); |
| 624 | 594 | const rtts = []; |
| 595 | /** A round trip, from a small request the page makes anyway: the time to its first byte. */ | |
| 596 | const timed = async (url) => { | |
| 597 | const started = performance.now(); | |
| 598 | const r = await fetch(url, { cache: "no-store" }); | |
| 599 | rtts.push(performance.now() - started); if (rtts.length > 10) rtts.shift(); | |
| 600 | return r; | |
| 601 | }; | |
| 625 | 602 | let plan = { mode: "stream" }; |
| 603 | const QUALITY = { auto: "Auto", saver: "Saver", full: "Full" }; | |
| 604 | let quality = (() => { try { const v = localStorage.getItem("so101.quality"); return Object.hasOwn(QUALITY, v ?? "") ? v : "auto"; } catch { return "auto"; } })(); | |
| 605 | // The first frame already follows what the browser says about the link (and the viewer's choice), not just later ones. | |
| 606 | plan = cameraPlan(navigator.connection, rtts, "stream", { quality, throttle: relayed }); | |
| 607 | /** The stream's address under the plan: through the relay, ?fps= holds it to fewer frames. */ | |
| 608 | const streamUrl = (c) => fresh(plan.mode === "stream" && plan.fps && relayed ? `${c.url}?fps=${plan.fps}` : c.url); | |
| 626 | 609 | const saved = (() => { try { const s = JSON.parse(localStorage.getItem("so101.panes") || "null"); return Array.isArray(s) ? s : null; } catch { return null; } })(); |
| 627 | 610 | // Pane keys: "cam:<id>" and "pane3". Until the startup below knows the model and the link, cameras are listed but not shown. |
| 628 | 611 | let chosen = saved ? new Set(saved) : null, viewsReady = false, cameraFirst = true; |
| @@ -631,7 +614,7 @@ | ||
| 631 | 614 | const paneOn = (c) => c.pane.classList.contains("on"); |
| 632 | 615 | async function snap(c) { |
| 633 | 616 | clearTimeout(c.timer); |
| 634 | if (!paneOn(c) || plan.mode !== "snapshot") return; | |
| 617 | if (!paneOn(c) || plan.mode !== "snapshot" || document.hidden) return; | |
| 635 | 618 | const started = performance.now(); |
| 636 | 619 | try { |
| 637 | 620 | const r = await fetch(fresh(c.snapshot), { cache: "no-store" }); |
| @@ -645,9 +628,9 @@ | ||
| 645 | 628 | const showCam = (c, on) => { |
| 646 | 629 | c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on)); |
| 647 | 630 | clearTimeout(c.timer); |
| 648 | if (!on) c.img.src = ""; | |
| 631 | if (!on || document.hidden) c.img.src = ""; | |
| 649 | 632 | else if (plan.mode === "snapshot" && c.snapshot) snap(c); |
| 650 | else c.img.src = fresh(c.url); | |
| 633 | else c.img.src = streamUrl(c); | |
| 651 | 634 | }; |
| 652 | 635 | function setPane(key, on) { |
| 653 | 636 | if (key === "pane3") { |
| @@ -687,7 +670,7 @@ | ||
| 687 | 670 | c.tag = el("span", { className: "tag" }); |
| 688 | 671 | c.pane = el("div", { className: "pane" }, c.img, c.tag); c.pane.dataset.key = `cam:${info.id}`; |
| 689 | 672 | c.btn = el("button", { type: "button" }); c.btn.setAttribute("aria-pressed", "false"); |
| 690 | c.img.onerror = () => { if (paneOn(c) && plan.mode === "stream") setTimeout(() => { if (paneOn(c) && plan.mode === "stream") c.img.src = fresh(c.url); }, 3000); }; | |
| 673 | c.img.onerror = () => { if (paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); }; | |
| 691 | 674 | c.btn.onclick = () => pick(c.pane.dataset.key); |
| 692 | 675 | $("stage").insertBefore(c.pane, $("pane3")); |
| 693 | 676 | $("top").querySelector(".seg").insertBefore(c.btn, $("v3")); |
| @@ -716,6 +699,7 @@ | ||
| 716 | 699 | } |
| 717 | 700 | label(c, info); |
| 718 | 701 | } |
| 702 | showQuality(); | |
| 719 | 703 | fallback(); |
| 720 | 704 | } |
| 721 | 705 | /** Once the model and the link are known: the views the viewer wants. */ |
| @@ -728,19 +712,30 @@ | ||
| 728 | 712 | } |
| 729 | 713 | async function pollCameras() { |
| 730 | 714 | try { |
| 731 | const r = await fetch("/cameras", { cache: "no-store" }); | |
| 715 | const r = await timed("/cameras"); | |
| 732 | 716 | if (r.ok) reconcile(await r.json()); |
| 733 | 717 | } catch { /* the next poll tries again */ } |
| 734 | 718 | setTimeout(pollCameras, document.hidden ? 10000 : 3000); |
| 735 | 719 | } |
| 736 | 720 | function replan() { |
| 737 | const next = cameraPlan(navigator.connection, rtts, plan.mode); | |
| 738 | if (next.mode === plan.mode && (next.mode === "stream" || next.everyMs === plan.everyMs)) return; | |
| 739 | const was = plan.mode; plan = next; | |
| 740 | if (was !== next.mode) for (const c of cams) if (paneOn(c) && c.snapshot) showCam(c, true); | |
| 721 | const next = cameraPlan(navigator.connection, rtts, plan, { quality, throttle: relayed }); | |
| 722 | if (samePlan(next, plan)) return; | |
| 723 | const was = plan; plan = next; | |
| 724 | // A new frame rate is a new stream; a new snapshot interval is taken at the next snapshot. | |
| 725 | if (was.mode !== next.mode || next.mode === "stream") for (const c of cams) if (paneOn(c) && (c.snapshot || next.mode === "stream")) showCam(c, true); | |
| 741 | 726 | } |
| 742 | 727 | setInterval(replan, 5000); |
| 743 | 728 | navigator.connection?.addEventListener?.("change", replan); |
| 729 | const qualityButton = $("quality"); | |
| 730 | const showQuality = () => { qualityButton.textContent = QUALITY[quality]; qualityButton.hidden = !cams.length; }; | |
| 731 | qualityButton.onclick = () => { | |
| 732 | quality = QUALITIES[(QUALITIES.indexOf(quality) + 1) % QUALITIES.length]; | |
| 733 | try { localStorage.setItem("so101.quality", quality); } catch {} | |
| 734 | showQuality(); replan(); | |
| 735 | msg(quality === "auto" ? "video: follows the connection" : quality === "saver" ? "video: data saver, a couple of frames a second" : "video: every frame"); | |
| 736 | }; | |
| 737 | // Hidden (another tab, the phone locked): no video at all; back on screen, it picks up where the plan says. | |
| 738 | document.addEventListener("visibilitychange", () => { for (const c of cams) if (paneOn(c)) showCam(c, true); }); | |
| 744 | 739 | setInterval(() => { |
| 745 | 740 | for (const c of cams) { |
| 746 | 741 | if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue; |
| @@ -752,12 +747,14 @@ | ||
| 752 | 747 | } catch { /* nothing decodable yet */ } |
| 753 | 748 | c.unchanged = changed ? 0 : c.unchanged + 1; |
| 754 | 749 | c.live = changed && c.online !== false; |
| 755 | if (c.unchanged >= 15) { c.img.src = fresh(c.url); c.unchanged = 0; } | |
| 750 | if (c.unchanged >= Math.max(15, plan.fps ? 30 / plan.fps : 0) && !document.hidden) { c.img.src = streamUrl(c); c.unchanged = 0; } | |
| 756 | 751 | } |
| 757 | const slow = plan.mode === "snapshot" ? ` · ${(1000 / plan.everyMs).toFixed(plan.everyMs > 1000 ? 1 : 0)} fps, slow link` : ""; | |
| 752 | const why = quality === "saver" ? "data saver" : "slow link"; | |
| 753 | const slow = plan.mode === "snapshot" ? ` · ${(1000 / plan.everyMs).toFixed(plan.everyMs > 1000 ? 1 : 0)} fps, ${why}` : plan.fps ? ` · ${plan.fps} fps, ${why}` : ""; | |
| 758 | 754 | $("live").textContent = cams.filter(paneOn).map((c) => (c.live ? `live${slow}` : c.online === false ? "offline" : "waiting")).join(" · "); |
| 759 | 755 | }, 1000); |
| 760 | if (relayed) pollCameras(); | |
| 756 | if (boot) { reconcile(boot.cameras); setTimeout(pollCameras, 3000); } | |
| 757 | else if (relayed) pollCameras(); | |
| 761 | 758 | else { |
| 762 | 759 | reconcile([{ id: 1, name: "Cam 1", url: direct(1), online: null }]); |
| 763 | 760 | // Straight to an arm server, camera 2 appears when its stream answers. |
| @@ -846,6 +843,13 @@ | ||
| 846 | 843 | if (r.saved) loadRecordings(); |
| 847 | 844 | } |
| 848 | 845 | $("stop").onclick = stop; |
| 846 | /** The second button, only while there is something besides the E-stop to end, named for what it ends. */ | |
| 847 | const ENDS_LABEL = { move: "End move", play: "End playback", record: "End recording", park: "End park" }; | |
| 848 | function showStop() { | |
| 849 | const label = ENDS_LABEL[driving()]; | |
| 850 | $("stop").hidden = !label || !(mayDrive() || isOwner()); | |
| 851 | if (label) $("stop").textContent = label; | |
| 852 | } | |
| 849 | 853 | // The emergency stop: the server halts whatever drives the arm and holds it with torque on until an admin clears it. |
| 850 | 854 | async function eStop() { |
| 851 | 855 | driver.releaseAll(); |
| @@ -1054,8 +1058,11 @@ | ||
| 1054 | 1058 | } |
| 1055 | 1059 | if (step !== "check") showOnModel(previewOf(step)); |
| 1056 | 1060 | } |
| 1061 | /** src/leader-map.js, fetched the first time someone calibrates a leader: nobody else needs it. */ | |
| 1062 | let leaderMap = null; | |
| 1057 | 1063 | async function openCalib() { |
| 1058 | 1064 | if (!leader) return; |
| 1065 | try { leaderMap ??= await import("/leader-map.js"); } catch { return msg("the calibration did not load; try again", true); } | |
| 1059 | 1066 | if (!Object.keys(limits).length) return msg("the arm's limits have not loaded yet", true); |
| 1060 | 1067 | const mine = await api("/leader/calibration").catch(() => null); |
| 1061 | 1068 | if (mine?.status === 401) return msg("join to calibrate your leader: the calibration is kept with your name", true); |
| @@ -1079,7 +1086,7 @@ | ||
| 1079 | 1086 | function calibReading(pose) { |
| 1080 | 1087 | const c = calib; |
| 1081 | 1088 | for (const [id, t] of pose) c.reading.set(id, t); |
| 1082 | if (STEPS[c.step] === "check") showOnModel(mapLeaderFrame(c.reading, draft(), limits)); | |
| 1089 | if (STEPS[c.step] === "check") showOnModel(leaderMap.mapLeaderFrame(c.reading, draft(), limits)); | |
| 1083 | 1090 | for (const [i, span] of [...$("lcRead").children].entries()) span.textContent = `${LABELS[LEADER_IDS[i]].split(" ")[0]} ${c.reading.get(LEADER_IDS[i]) ?? "–"}`; |
| 1084 | 1091 | } |
| 1085 | 1092 | async function nextCalib() { |
| @@ -1099,7 +1106,7 @@ | ||
| 1099 | 1106 | const j = c.joints[step.id]; |
| 1100 | 1107 | j[step.end] = c.reading.get(step.id); |
| 1101 | 1108 | if (step.end === "high") { |
| 1102 | try { parseCalibration({ joints: { [step.id]: step.id === 6 ? { low: j.low, high: j.high, centre: Math.round((j.low + j.high) / 2) } : j } }); } | |
| 1109 | try { leaderMap.parseCalibration({ joints: { [step.id]: step.id === 6 ? { low: j.low, high: j.high, centre: Math.round((j.low + j.high) / 2) } : j } }); } | |
| 1103 | 1110 | catch (err) { delete j.low; delete j.high; c.step--; c.error = `${err.message.replace(/^joint \d+: /, "")}. Try ${LABELS[step.id].toLowerCase()} again.`; return renderCalib(); } |
| 1104 | 1111 | } |
| 1105 | 1112 | } |
| @@ -1477,7 +1484,7 @@ | ||
| 1477 | 1484 | // The page follows suit: the Controls tab appears on your turn. Everything here arrives on one socket. |
| 1478 | 1485 | let wordsSent = false; |
| 1479 | 1486 | $("wordsForm").addEventListener("submit", (e) => { e.preventDefault(); wordsSent = true; say({ type: "words", words: $("wordsForm").elements.words.value.split(",").map((w) => w.trim()).filter(Boolean) }); }); |
| 1480 | const lobby = { ws: null, hello: null, state: null, offset: 0, retry: 500, mutedList: [], bannedList: [], admin: null }; | |
| 1487 | const lobby = { ws: null, hello: null, state: null, offset: 0, retry: 500, mutedList: [], bannedList: [], admin: null, statusSeen: false }; | |
| 1481 | 1488 | function mayDrive() { return !relayed || (lobby.hello !== null && (!lobby.hello.signin || (lobby.hello.you !== null && lobby.state?.driver?.id === lobby.hello.you.id))); } |
| 1482 | 1489 | function isOwner() { return !relayed || Boolean(lobby.hello?.you?.owner); } |
| 1483 | 1490 | const say = (m) => { if (lobby.ws?.readyState === 1) lobby.ws.send(JSON.stringify(m)); }; |
| @@ -1491,7 +1498,7 @@ | ||
| 1491 | 1498 | shell.classList.toggle("driving", can); |
| 1492 | 1499 | if (!can) driver.releaseAll(); |
| 1493 | 1500 | wake(can); |
| 1494 | $("stop").hidden = !(can || owner); | |
| 1501 | showStop(); | |
| 1495 | 1502 | $("estop").disabled = !(mayDrive() || owner); |
| 1496 | 1503 | $("estop").title = $("estop").disabled ? "Emergency stop: the driver and the admins can press it" : "Emergency stop (Esc): halt, hold with torque, until an admin clears it"; |
| 1497 | 1504 | // The server takes settings, the safety limits among them, from admins only. |
| @@ -1649,11 +1656,26 @@ | ||
| 1649 | 1656 | if (m.from && me?.owner) line.tabIndex = 0; |
| 1650 | 1657 | return line; |
| 1651 | 1658 | } |
| 1659 | // A new page gets the last lines of the chat; older ones come a page at a time, when asked for. | |
| 1660 | const earlier = el("button", { type: "button", className: "earlier", textContent: "Earlier messages" }); | |
| 1661 | earlier.onclick = () => { | |
| 1662 | const first = chatBox.querySelector(".line[data-id]"); | |
| 1663 | if (!first) return; | |
| 1664 | earlier.disabled = true; | |
| 1665 | say({ type: "history", before: Number(first.dataset.id) }); | |
| 1666 | }; | |
| 1667 | function showEarlier(more) { earlier.disabled = false; if (more) chatBox.prepend(earlier); else earlier.remove(); } | |
| 1668 | function addOlder(lines, more) { | |
| 1669 | const below = chatBox.scrollHeight - chatBox.scrollTop; // the reader stays on the line they were reading | |
| 1670 | chatBox.prepend(...lines.map(chatLine)); | |
| 1671 | chatBox.scrollTop = chatBox.scrollHeight - below; | |
| 1672 | showEarlier(more); | |
| 1673 | } | |
| 1652 | 1674 | function addChat(lines, replace = false) { |
| 1653 | 1675 | const atEnd = chatBox.scrollHeight - chatBox.scrollTop - chatBox.clientHeight < 40; |
| 1654 | 1676 | if (replace) chatBox.replaceChildren(); |
| 1655 | 1677 | chatBox.append(...lines.map(chatLine)); |
| 1656 | while (chatBox.children.length > 200) chatBox.firstChild.remove(); | |
| 1678 | if (!replace) while (chatBox.querySelectorAll(".line").length > 200) chatBox.querySelector(".line").remove(); | |
| 1657 | 1679 | if (atEnd || replace) chatBox.scrollTop = chatBox.scrollHeight; |
| 1658 | 1680 | if (!replace && tab !== "chat") $("unread").textContent = String(Math.min(99, Number($("unread").textContent || 0) + lines.length)); |
| 1659 | 1681 | } |
| @@ -1695,9 +1717,11 @@ | ||
| 1695 | 1717 | ws.onmessage = (e) => { |
| 1696 | 1718 | const m = JSON.parse(e.data); |
| 1697 | 1719 | if (m.type === "ping") return say({ type: "pong" }); |
| 1720 | if (m.type === "status") { lobby.statusSeen = true; return m.status ? applyStatus(m.status) : noStatus(); } | |
| 1698 | 1721 | if (m.type === "hello") lobby.hello = m; |
| 1699 | 1722 | else if (m.type === "state") { lobby.offset = m.now - Date.now(); lobby.state = m; } |
| 1700 | else if (m.type === "chat") return addChat(m.messages, true); | |
| 1723 | else if (m.type === "chat") return addChat(m.messages, true), showEarlier(m.more); | |
| 1724 | else if (m.type === "history") return addOlder(m.messages, m.more); | |
| 1701 | 1725 | else if (m.type === "said") return addChat([m.message]); |
| 1702 | 1726 | else if (m.type === "unsaid") return chatBox.querySelector(`[data-id="${Number(m.id)}"]`)?.remove(); |
| 1703 | 1727 | else if (m.type === "muted") lobby.mutedList = m.muted; |
| @@ -1710,7 +1734,7 @@ | ||
| 1710 | 1734 | }; |
| 1711 | 1735 | ws.onclose = () => { |
| 1712 | 1736 | if (lobby.ws !== ws) return; |
| 1713 | lobby.ws = null; lobby.hello = null; | |
| 1737 | lobby.ws = null; lobby.hello = null; lobby.statusSeen = false; // polling takes over until it is back | |
| 1714 | 1738 | applyLock(); |
| 1715 | 1739 | $("driverLine").className = "none"; $("driverLine").textContent = "Reconnecting…"; |
| 1716 | 1740 | setTimeout(lobbyConnect, lobby.retry); |
| @@ -1743,37 +1767,12 @@ | ||
| 1743 | 1767 | }; |
| 1744 | 1768 | |
| 1745 | 1769 | // ---------------------------------------------------------------- 3D calibration |
| 1770 | // The panel and its words are fetched the first time someone opens it: src/ui/calibration.js. | |
| 1746 | 1771 | $("calib").onclick = async () => { |
| 1747 | $("panel").hidden = false; | |
| 1748 | $("calSteps").textContent = [ | |
| 1749 | `export SO101_ARM=http://${location.host}`, | |
| 1750 | `export SO101_CAMERA=${cams[0] ? new URL(cams[0].url, location.href).href : "<the camera's stream URL>"}`, | |
| 1751 | `just vision-calibrate # hold the printed board in front of the camera, tilt it`, | |
| 1752 | `# save poses from this page: home, drop, he1 … he8 (without gripper)`, | |
| 1753 | `just vision-handeye he1,he2,he3,he4,he5,he6,he7,he8`, | |
| 1754 | `just vision-rehearse # sees the cube, plans the joints, moves nothing`, | |
| 1755 | `just vision-pick`, | |
| 1756 | ``, | |
| 1757 | `# a second camera: SO101_CAMERA=<its stream URL> and --publish-as camera2 / handeye2`, | |
| 1758 | ].join("\n"); | |
| 1759 | const sheets = await api("/printables"); | |
| 1760 | $("printables").innerHTML = Array.isArray(sheets) && sheets.length | |
| 1761 | ? sheets.map((s) => `<a href="${s.url}" download="${s.file}"><button class="go">⬇ ${s.file.replace(/_/g, " ").replace(/\.png$/, "")}</button></a>`).join("") | |
| 1762 | : "<span>no printables on this server: copy Software/vision/printables next to the package</span>"; | |
| 1763 | const cal = await api("/calibration"); | |
| 1764 | const saved = await api("/poses"); | |
| 1765 | const poseNames = Object.keys(saved); | |
| 1766 | const he = poseNames.filter((n) => /^he\d+$/.test(n) && !saved[n].ids.includes(6)).length; | |
| 1767 | const mark = (ok, yes, no) => `<span class="${ok ? "ok" : "no"}">${ok ? yes : no}</span>`; | |
| 1768 | const armOnly = (name) => !saved[name] ? mark(false, "", "missing") : saved[name].ids.includes(6) ? mark(false, "", "saved with the gripper: save it again without") : mark(true, "saved"); | |
| 1769 | const camLine = (c) => c ? mark(true, `calibrated ${(c.saved ?? "").slice(0, 10)} · ${c.width}×${c.height} · ${Number(c.rms).toFixed(2)} px error`) : mark(false, "", "not yet"); | |
| 1770 | const heLine = (h) => h ? mark(true, `solved ${(h.saved ?? "").slice(0, 10)} · ${h.poses?.length ?? "?"} poses · mean ${h.residuals_mm ? (h.residuals_mm.reduce((a, b) => a + b, 0) / h.residuals_mm.length).toFixed(1) : "?"} mm`) : mark(false, "", "not yet"); | |
| 1771 | $("calStatus").innerHTML = [ | |
| 1772 | ["camera 1", camLine(cal.camera)], ["hand-eye 1", heLine(cal.handeye)], | |
| 1773 | ...(cams.length < 2 ? [] : [["camera 2", camLine(cal.camera2)], ["hand-eye 2", heLine(cal.handeye2)]]), | |
| 1774 | ["pose home", armOnly("home")], ["pose drop", armOnly("drop")], | |
| 1775 | ["poses he1…", mark(he >= 6, `${he} saved`, `${he} of 6 to 8 saved (without the gripper)`)], | |
| 1776 | ].map(([k, v]) => `<b>${k}</b><span>${v}</span>`).join(""); | |
| 1772 | try { | |
| 1773 | const { openCalibration } = await import("/ui/__VERSION__/calibration.js"); | |
| 1774 | await openCalibration({ api, host: location.host, cameras: cams.map((c) => new URL(c.url, location.href).href) }); | |
| 1775 | } catch (err) { console.error(err); msg("the calibration panel did not load", true); } | |
| 1777 | 1776 | }; |
| 1778 | 1777 | |
| 1779 | 1778 | // ---------------------------------------------------------------- live status, sliders and jog |
| @@ -1834,13 +1833,14 @@ | ||
| 1834 | 1833 | else if (d.violation) { b.className = "warn"; b.textContent = `the arm is ${say(d.violation)} (zone: ${d.zone})`; } |
| 1835 | 1834 | else b.className = ""; |
| 1836 | 1835 | } |
| 1837 | async function poll() { | |
| 1838 | const started = performance.now(); | |
| 1839 | try { | |
| 1840 | const d = await api("/status"); | |
| 1841 | if (!d.ok) throw new Error(); | |
| 1836 | /** The arm's status, from /live (pushed when it changes) or, without it, from polling /status. */ | |
| 1837 | // Until the limits, the settings and the model are in, a status is kept aside, not drawn; then it is applied whole, | |
| 1838 | // so an E-stop it carries still locks the controls. | |
| 1839 | let started = false, early = null; | |
| 1840 | function applyStatus(d) { | |
| 1841 | if (!started) return void (early = d); | |
| 1842 | { | |
| 1842 | 1843 | const wasStopped = Boolean(status?.estop); |
| 1843 | rtts.push(performance.now() - started); if (rtts.length > 10) rtts.shift(); // the link's round trip, for the camera plan | |
| 1844 | 1844 | status = d; |
| 1845 | 1845 | driver.seed(d.joints.filter((j) => j.position != null).map((j) => [j.id, j.position])); |
| 1846 | 1846 | if (Boolean(d.estop) !== wasStopped) applyLock(); |
| @@ -1852,28 +1852,45 @@ | ||
| 1852 | 1852 | $("dot").className = d.reconnecting ? "dot warn" : "dot on"; |
| 1853 | 1853 | $("state").textContent = d.reconnecting ? "reconnecting…" : d.driving === null ? "idle" : d.driving === "record" ? `recording ${d.recording}` : leaderStreaming() ? `leader · ${leader.frames} frames` : d.driving; |
| 1854 | 1854 | $("estop").classList.toggle("on", Boolean(d.estop)); $("estop").textContent = d.estop ? "Stopped" : "Stop"; |
| 1855 | showStop(); | |
| 1855 | 1856 | $("record").textContent = d.driving === "record" ? "■ Stop recording" : "● Record"; |
| 1856 | 1857 | for (const id of ["hold", "releaseBtn", "play"]) $(id).disabled = d.driving !== null; |
| 1857 | 1858 | if (tab === "controls") { render(d); renderTouch(d); } |
| 1858 | 1859 | banner(d); applyModel(d); |
| 1859 | 1860 | $("readout").innerHTML = `<span>lowest part <b>${Number.isFinite(d.lowestMm) ? d.lowestMm : "—"} mm</b> above the table</span><span>zone <b>${d.zone === "enforce" || d.zone === "warn" || d.zone === "off" ? d.zone : "—"}</b></span>`; |
| 1860 | } catch { | |
| 1861 | const reconnecting = linkNow === "reconnecting"; | |
| 1862 | $("dot").className = reconnecting ? "dot warn" : "dot"; $("state").textContent = reconnecting ? "reconnecting…" : "no arm"; | |
| 1863 | 1861 | } |
| 1862 | } | |
| 1863 | function noStatus() { | |
| 1864 | const reconnecting = linkNow === "reconnecting"; | |
| 1865 | $("dot").className = reconnecting ? "dot warn" : "dot"; $("state").textContent = reconnecting ? "reconnecting…" : "no arm"; | |
| 1866 | } | |
| 1867 | async function poll() { | |
| 1868 | // The room's socket carries the status: nothing to poll while it is open and has said where the arm is. | |
| 1869 | if (lobby.statusSeen && lobby.ws?.readyState === 1) return void setTimeout(poll, 1000); | |
| 1870 | try { | |
| 1871 | const r = await timed("/status"); | |
| 1872 | const d = await r.json(); | |
| 1873 | if (!r.ok) throw new Error(); | |
| 1874 | applyStatus(d); | |
| 1875 | } catch { noStatus(); } | |
| 1864 | 1876 | // Hidden, the page keeps the arm's state fresh at a walk: nobody is looking, and a phone's battery and data matter. |
| 1865 | 1877 | setTimeout(poll, document.hidden ? 2000 : 200); |
| 1866 | 1878 | } |
| 1867 | 1879 | (async () => { |
| 1868 | 1880 | applyLock(); |
| 1869 | 1881 | if (relayed) lobbyConnect(); else showTab("controls"); |
| 1870 | limits = await api("/limits"); | |
| 1871 | await loadSettings(); | |
| 1872 | await loadPoses(); | |
| 1873 | modelInfo = await api("/model.json"); | |
| 1874 | 1882 | // The views: the ones chosen last time, else one on a phone (the camera when there is a real one), else the camera and the model. |
| 1875 | cameraFirst = !relayed || (await api("/link").catch(() => ({}))).mode === "live" || q.has("stream"); | |
| 1883 | // With the page's own facts (#boot) the camera starts now, before anything else is asked; the rest is asked all at once. | |
| 1884 | const linked = boot ? Promise.resolve(boot.link) : relayed ? api("/link").catch(() => ({})) : Promise.resolve(null); | |
| 1885 | if (boot) { | |
| 1886 | cameraFirst = boot.link.mode === "live" || q.has("stream"); | |
| 1887 | if (cameraFirst) { viewsReady = true; for (const c of cams) if (!paneOn(c) && wanted(c.pane.dataset.key)) showCam(c, true); } | |
| 1888 | } | |
| 1889 | const [l, mi, link] = await Promise.all([api("/limits"), api("/model.json"), linked, loadSettings(), loadPoses()]); | |
| 1890 | limits = l; modelInfo = mi; started = true; | |
| 1891 | cameraFirst = !relayed || link?.mode === "live" || q.has("stream"); | |
| 1876 | 1892 | startViews(); |
| 1893 | if (early) applyStatus(early); | |
| 1877 | 1894 | poll(); |
| 1878 | 1895 | if (relayed) pollLink(); |
| 1879 | 1896 | // The model is fetched once the first view is up, whether or not it is on screen: the joystick needs it. |
new above
offline