Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

The page draws the camera from /camera/<id>/ws when relayed and reconnects after 2 s without a frame #182

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/140-the-page-draws-the-camera-from-camera-id → main 2 commits fork 2 threads · 2 unresolved

  • Merged. smith/140-the-page-draws-the-camera-from-camera-id was merged into main as 8671eed.

Review

Sign in to review this change.
4 files changed +262 −3
Software/README.md +7
@@ -218,6 +218,13 @@ again every 2 s and the live picture comes back in the same stream, without
218218 a reload. The camera is independent of the arm server: a camera that is down
219219 does not put the arm on the simulation, and the other way round.
220220
221 **In the browser.** Behind the relay the page draws each camera from
222 `WS /camera/<id>/ws`: whole JPEG frames, drawn as they arrive. If no frame
223 comes for 2 s the page closes that socket and opens a new one, and it retries
224 a closed socket after 0.5 s (`src/ui/video.js`, loaded only when a camera is
225 first shown). A server run on the machine itself (`so101 serve`, no relay)
226 keeps the plain MJPEG `<img src=/camera/<id>/stream>`.
227
221228 **On a phone over 4G.** The first load is the page and three small modules,
222229 under 50 KB gzipped (a test in `src/server.test.ts` holds it there). three.js,
223230 the leader-arm mapping and the 3D-calibration panel (`src/ui/calibration.js`)
Software/so101/src/ui/video.js added +96
@@ -0,0 +1,96 @@
1 // @ts-check
2 /**
3 * The camera when the page is behind the relay: whole JPEG frames over WS /camera/<id>/ws, drawn as they come. With no
4 * frame for QUIET_MS (a stalled socket, a relay that restarted, a source that went away) the socket is opened again.
5 * Beside the relay, on the machine itself, the page keeps the plain <img src=/camera/<id>/stream>. Loaded by dynamic
6 * import, so the first visit does not carry it. Plain ESM, served at /ui/ and imported by the tests.
7 */
8
9 /** No frame for this long and the socket is replaced. */
10 export const QUIET_MS = 2000;
11 /** The wait before trying again after a socket that closed or failed, so a relay that is down is not hammered. */
12 export const RETRY_MS = 500;
13
14 /** The socket's address on the page's own origin. @param {{ protocol: string, host: string }} loc @param {number | string} id */
outdated

Dead in production: only the test calls cameraSource, and viewer.html uses relayed directly. Remove it or use it.

15 export function socketUrl(loc, id) {
16 return `${loc.protocol === "https:" ? "wss:" : "ws:"}//${loc.host}/camera/${encodeURIComponent(String(id))}/ws`;
17 }
18
19 /**
20 * Keep one camera's socket open and hand every frame (a Blob of JPEG or PNG) to `onFrame`.
21 * @param {{
22 * url: string, onFrame: (frame: Blob) => void, maxFps?: number,
23 * open?: (url: string) => WebSocket, now?: () => number,
24 * after?: (fn: () => void, ms: number) => any, cancel?: (timer: any) => void,
25 * }} options the last four are for the tests
26 * @returns {{ close(): void }}
27 */
28 export function openFeed(options) {
29 const { url, onFrame } = options;
30 const open = options.open ?? ((u) => new WebSocket(u));
31 const now = options.now ?? Date.now;
32 const after = options.after ?? ((fn, ms) => setTimeout(fn, ms));
33 const cancel = options.cancel ?? ((t) => clearTimeout(t));
34 const gap = options.maxFps && options.maxFps > 0 ? 1000 / options.maxFps : 0;
35 let ws = /** @type {WebSocket | null} */ (null), watchdog = /** @type {any} */ (null), closed = false, shown = -Infinity;
36
37 const stop = () => {
38 cancel(watchdog); watchdog = null;
39 const w = ws; ws = null;
40 if (w) { w.onmessage = w.onclose = w.onerror = null; try { w.close(); } catch { /* already gone */ } }
41 };
42 /** Replace the socket, after `ms`. */
43 const again = (ms) => {
44 stop();
45 if (!closed) watchdog = after(connect, ms);
46 };
47 function connect() {
48 watchdog = null;
49 if (closed) return;
50 let w;
51 try { w = open(url); } catch { return again(RETRY_MS); }
52 ws = w;
53 w.binaryType = "blob";
54 w.onmessage = (e) => {
55 if (ws !== w || !(e.data instanceof Blob)) return;
56 cancel(watchdog); watchdog = after(() => again(0), QUIET_MS);
57 const t = now();
58 if (t - shown < gap) return; // held to fewer frames on a slow link
59 shown = t;
60 onFrame(e.data);
61 };
62 w.onclose = w.onerror = () => { if (ws === w) again(RETRY_MS); };
63 watchdog = after(() => again(0), QUIET_MS); // the first frame is due too
64 }
65 connect();
66 return { close() { closed = true; stop(); } };
67 }
68
69 /**
70 * Draw one frame in a camera's <img> and let go of the one before it, so memory does not grow with the frames.
71 * @param {{ img: { src: string }, blob?: string, lastFrame?: number }} cam
72 * @param {Blob} frame
73 * @param {{ create(b: Blob): string, revoke(u: string): void, now(): number }} env
74 */
75 export function drawFrame(cam, frame, env) {
76 const url = env.create(frame);
77 cam.img.src = url;
78 if (cam.blob) env.revoke(cam.blob);
79 cam.blob = url;
80 cam.lastFrame = env.now();
81 }
82
83 /** Whether a socket-drawn camera is live: a frame lately, and the relay not saying it is offline. @param {{ lastFrame?: number, online?: boolean | null }} cam */
84 export const isLive = (cam, now) => now - (cam.lastFrame ?? -Infinity) < QUIET_MS + 500 && cam.online !== false;
85
86 /**
87 * Start a camera's socket, unless the page moved on while this module was loading: `epoch` is the camera's count when
88 * the load began, and the page counts up whenever it shows or hides the camera again.
89 * @param {{ id: number | string, epoch: number, feed: any, img: { src: string }, blob?: string, lastFrame?: number }} cam
90 * @param {number} epoch
91 * @param {{ visible(): boolean, loc: { protocol: string, host: string }, maxFps?: number, create(b: Blob): string, revoke(u: string): void, now(): number }} env
92 */
93 export function attach(cam, epoch, env) {
94 if (cam.epoch !== epoch || !env.visible()) return;
95 cam.feed = openFeed({ url: socketUrl(env.loc, cam.id), maxFps: env.maxFps, onFrame: (frame) => drawFrame(cam, frame, env) });
96 }
Software/so101/src/ui/video.test.ts added +146
@@ -0,0 +1,146 @@
1 import { describe, expect, test } from "bun:test";
2 import { attach, drawFrame, isLive, openFeed, QUIET_MS, RETRY_MS, socketUrl } from "./video.js";
3
4 /** A clock and a socket factory the test drives by hand. */
5 function rig(maxFps?: number) {
6 let t = 0;
7 const timers: { at: number; fn: () => void; id: number }[] = [];
8 let next = 0;
9 const sockets: any[] = [];
10 const frames: Blob[] = [];
11 const feed = openFeed({
12 url: "ws://x/camera/1/ws", maxFps, onFrame: (f) => frames.push(f),
13 now: () => t,
14 after: (fn, ms) => { const id = ++next; timers.push({ at: t + ms, fn, id }); return id; },
15 cancel: (id) => { const i = timers.findIndex((x) => x.id === id); if (i >= 0) timers.splice(i, 1); },
16 open: (url) => { const w: any = { url, closed: false, close() { w.closed = true; } }; sockets.push(w); return w; },
17 });
18 const advance = (ms: number) => {
19 const end = t + ms;
20 for (;;) {
21 timers.sort((a, b) => a.at - b.at);
22 const due = timers[0];
23 if (!due || due.at > end) break;
24 timers.shift(); t = due.at; due.fn();
25 }
26 t = end;
27 };
28 const frame = (w = sockets.at(-1)) => w.onmessage({ data: new Blob([new Uint8Array([0xff, 0xd8])]) });
29 return { feed, sockets, frames, advance, frame };
30 }
31
32 describe("the relayed camera's address", () => {
33 test("the socket address is on the page's own origin", () => {
34 expect(socketUrl({ protocol: "https:", host: "a.example" }, 2)).toBe("wss://a.example/camera/2/ws");
35 expect(socketUrl({ protocol: "http:", host: "localhost:8080" }, "1")).toBe("ws://localhost:8080/camera/1/ws");
36 expect(socketUrl({ protocol: "http:", host: "h" }, "../x")).toBe("ws://h/camera/..%2Fx/ws");
37 });
38 });
39
40 describe("the relayed feed", () => {
41 test("draws frames as they come and keeps its socket while they do", () => {
42 const r = rig();
43 r.frame(); r.advance(QUIET_MS - 1); r.frame(); r.advance(QUIET_MS - 1); r.frame();
44 expect(r.frames.length).toBe(3);
45 expect(r.sockets.length).toBe(1);
46 });
47 test("reconnects after 2 s without a frame, and again if the new socket stays silent", () => {
48 const r = rig();
49 r.frame();
50 r.advance(QUIET_MS - 1);
51 expect(r.sockets.length).toBe(1);
52 r.advance(1);
53 expect(r.sockets[0].closed).toBe(true);
54 r.advance(0);
55 expect(r.sockets.length).toBe(2);
56 r.advance(QUIET_MS);
57 expect(r.sockets.length).toBe(3);
58 });
59 test("a socket that closes is replaced after a short wait; a late frame from the old one is ignored", () => {
60 const r = rig();
61 const old = r.sockets[0];
62 old.onclose();
63 expect(r.sockets.length).toBe(1);
64 r.advance(RETRY_MS);
65 expect(r.sockets.length).toBe(2);
66 old.onmessage?.({ data: new Blob([]) });
67 expect(r.frames.length).toBe(0);
68 });
69 test("close stops everything", () => {
70 const r = rig();
71 r.feed.close();
72 r.advance(10 * QUIET_MS);
73 expect(r.sockets.length).toBe(1);
74 expect(r.sockets[0].closed).toBe(true);
75 });
76 test("a slow link gets fewer frames", () => {
77 const r = rig(2);
78 r.frame(); r.advance(100); r.frame(); r.advance(500); r.frame();
79 expect(r.frames.length).toBe(2);
80 });
81 test("text is not a frame", () => {
82 const r = rig();
83 r.sockets[0].onmessage({ data: "hello" });
84 expect(r.frames.length).toBe(0);
85 });
86 });
87
88 /** What the page hands in, with the object URLs counted. */
89 function env(visible = true) {
90 let n = 0, t = 1000;
91 const live = new Set<string>();
92 return { live, tick: (ms: number) => (t += ms), e: { visible: () => visible, loc: { protocol: "https:", host: "h" }, maxFps: undefined, create: () => { const u = `blob:${++n}`; live.add(u); return u; }, revoke: (u: string) => void live.delete(u), now: () => t } };
93 }
94
95 describe("drawing frames in the page", () => {
96 test("each frame replaces the picture and frees the one before it", () => {
97 const { e, live } = env();
98 const cam: any = { img: { src: "" } };
99 drawFrame(cam, new Blob(["a"]), e);
100 drawFrame(cam, new Blob(["b"]), e);
101 drawFrame(cam, new Blob(["c"]), e);
102 expect(cam.img.src).toBe("blob:3");
103 expect([...live]).toEqual(["blob:3"]);
104 expect(cam.lastFrame).toBe(1000);
105 });
106 test("a camera is live while frames come, and not when the relay says it is offline", () => {
107 expect(isLive({}, 5000)).toBe(false);
108 expect(isLive({ lastFrame: 4000 }, 5000)).toBe(true);
109 expect(isLive({ lastFrame: 4000, online: false }, 5000)).toBe(false);
110 expect(isLive({ lastFrame: 1000 }, 5000)).toBe(false);
111 });
112 test("attach opens the socket and draws what it sends", () => {
113 const { e } = env();
114 const cam: any = { id: 2, epoch: 4, feed: null, img: { src: "" } };
115 const sockets: any[] = [];
116 (globalThis as any).WebSocket = class { constructor(public url: string) { sockets.push(this); } close() {} };
117 attach(cam, 4, e);
118 expect(sockets[0].url).toBe("wss://h/camera/2/ws");
119 sockets[0].onmessage({ data: new Blob(["x"]) });
120 expect(cam.img.src).toBe("blob:1");
121 cam.feed.close();
122 });
123 test("attach does nothing when the page moved on while the module loaded", () => {
124 const { e } = env();
125 const cam: any = { id: 1, epoch: 5, feed: null, img: { src: "" } };
126 attach(cam, 4, e); // shown again since: a newer load owns it
127 expect(cam.feed).toBeNull();
128 attach(cam, 5, env(false).e); // hidden or switched off since
129 expect(cam.feed).toBeNull();
130 });
131 });
132
133 describe("the page's wiring", () => {
134 const page = require("node:fs").readFileSync(new URL("../viewer.html", import.meta.url), "utf8") as string;
135 test("a relayed camera in stream mode goes through video.js; a local one keeps the img stream", () => {
136 expect(page).toMatch(/else if \(relayed\) startFeed\(c\);\n\s*else c\.img\.src = streamUrl\(c\);/);
137 expect(page).toMatch(/import\("\/ui\/__VERSION__\/video\.js"\)/);
138 expect(page).toContain("video.attach(c, epoch,");
139 expect(page).toContain("video.isLive(c, Date.now())");
140 });
141 test("showing, hiding or removing a camera stops its socket, which also voids a load in flight", () => {
142 expect(page).toContain("c.epoch = (c.epoch ?? 0) + 1; c.feed?.close?.()");
143 expect(page).toMatch(/clearTimeout\(c\.timer\); stopFeed\(c\);\n\s*if \(!on/);
144 expect(page).toContain("clearTimeout(c.timer); stopFeed(c); c.img.src");
145 });
146 });
Software/so101/src/viewer.html +13 −3
@@ -643,11 +643,20 @@
643643 } catch { c.live = false; }
644644 if (plan.mode === "snapshot") c.timer = setTimeout(() => snap(c), Math.max(0, plan.everyMs - (performance.now() - started)));
645645 }
646 // Behind the relay the camera is frames over WS /camera/<id>/ws (src/ui/video.js, imported when first needed); on the machine itself, the plain <img> stream.
647 let video = null;
648 const stopFeed = (c) => { c.epoch = (c.epoch ?? 0) + 1; c.feed?.close?.(); c.feed = null; };
649 async function startFeed(c) {
650 const epoch = c.epoch;
651 try { video ??= await import("/ui/__VERSION__/video.js"); } catch { return; }
652 video.attach(c, epoch, { visible: () => paneOn(c) && !document.hidden, loc: location, maxFps: plan.fps, create: URL.createObjectURL, revoke: URL.revokeObjectURL, now: Date.now });
653 }
outdated

This onFrame path (object URL, revoke the previous, lastFrame) has no test. Move it into video.js and cover it.

646654 const showCam = (c, on) => {
647655 c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on));
648 clearTimeout(c.timer);
656 clearTimeout(c.timer); stopFeed(c);
649657 if (!on || document.hidden) c.img.src = "";
650658 else if (plan.mode === "snapshot" && c.snapshot) snap(c);
659 else if (relayed) startFeed(c);
651660 else c.img.src = streamUrl(c);
652661 };
653662 function setPane(key, on) {
@@ -688,7 +697,7 @@
688697 c.tag = el("span", { className: "tag" });
689698 c.pane = el("div", { className: "pane" }, c.img, c.tag); c.pane.dataset.key = `cam:${info.id}`;
690699 c.btn = el("button", { type: "button" }); c.btn.setAttribute("aria-pressed", "false");
691 c.img.onerror = () => { if (paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); };
700 c.img.onerror = () => { if (!relayed && paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); };
692701 c.btn.onclick = () => pick(c.pane.dataset.key);
693702 $("stage").insertBefore(c.pane, $("pane3"));
694703 $("top").querySelector(".seg").insertBefore(c.btn, $("v3"));
@@ -696,7 +705,7 @@
696705 return c;
697706 }
698707 function removeCam(c) {
699 clearTimeout(c.timer); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob);
708 clearTimeout(c.timer); stopFeed(c); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob);
700709 c.pane.remove(); c.btn.remove();
701710 cams.splice(cams.indexOf(c), 1);
702711 fallback();
@@ -756,6 +765,7 @@
756765 document.addEventListener("visibilitychange", () => { for (const c of cams) if (paneOn(c)) showCam(c, true); });
757766 setInterval(() => {
758767 for (const c of cams) {
768 if (c.feed && paneOn(c)) { c.live = video.isLive(c, Date.now()); continue; }
759769 if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue;
760770 let changed = false;
761771 try {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs