Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
The page draws the camera from /camera/<id>/ws when relayed and reconnects after 2 s without a frame #182
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/140-the-page-draws-the-camera-from-camera-id → main 2 commits fork 2 threads · 2 unresolved
4 files
changed +262
−3
Software/README.md +7
| @@ -218,6 +218,13 @@ again every 2 s and the live picture comes back in the same stream, without | ||
| 218 | 218 | a reload. The camera is independent of the arm server: a camera that is down |
| 219 | 219 | does not put the arm on the simulation, and the other way round. |
| 220 | 220 | |
| 221 | **In the browser.** Behind the relay the page draws each camera from | |
| 222 | `WS /camera/<id>/ws`: whole JPEG frames, drawn as they arrive. If no frame | |
| 223 | comes for 2 s the page closes that socket and opens a new one, and it retries | |
| 224 | a closed socket after 0.5 s (`src/ui/video.js`, loaded only when a camera is | |
| 225 | first shown). A server run on the machine itself (`so101 serve`, no relay) | |
| 226 | keeps the plain MJPEG `<img src=/camera/<id>/stream>`. | |
| 227 | ||
| 221 | 228 | **On a phone over 4G.** The first load is the page and three small modules, |
| 222 | 229 | under 50 KB gzipped (a test in `src/server.test.ts` holds it there). three.js, |
| 223 | 230 | the leader-arm mapping and the 3D-calibration panel (`src/ui/calibration.js`) |
Software/so101/src/ui/video.js added +96
| @@ -0,0 +1,96 @@ | ||
| 1 | // @ts-check | |
| 2 | /** | |
| 3 | * The camera when the page is behind the relay: whole JPEG frames over WS /camera/<id>/ws, drawn as they come. With no | |
| 4 | * frame for QUIET_MS (a stalled socket, a relay that restarted, a source that went away) the socket is opened again. | |
| 5 | * Beside the relay, on the machine itself, the page keeps the plain <img src=/camera/<id>/stream>. Loaded by dynamic | |
| 6 | * import, so the first visit does not carry it. Plain ESM, served at /ui/ and imported by the tests. | |
| 7 | */ | |
| 8 | ||
| 9 | /** No frame for this long and the socket is replaced. */ | |
| 10 | export const QUIET_MS = 2000; | |
| 11 | /** The wait before trying again after a socket that closed or failed, so a relay that is down is not hammered. */ | |
| 12 | export const RETRY_MS = 500; | |
| 13 | ||
| 14 | /** The socket's address on the page's own origin. @param {{ protocol: string, host: string }} loc @param {number | string} id */ | |
| 15 | export function socketUrl(loc, id) { | |
| 16 | return `${loc.protocol === "https:" ? "wss:" : "ws:"}//${loc.host}/camera/${encodeURIComponent(String(id))}/ws`; | |
| 17 | } | |
| 18 | ||
| 19 | /** | |
| 20 | * Keep one camera's socket open and hand every frame (a Blob of JPEG or PNG) to `onFrame`. | |
| 21 | * @param {{ | |
| 22 | * url: string, onFrame: (frame: Blob) => void, maxFps?: number, | |
| 23 | * open?: (url: string) => WebSocket, now?: () => number, | |
| 24 | * after?: (fn: () => void, ms: number) => any, cancel?: (timer: any) => void, | |
| 25 | * }} options the last four are for the tests | |
| 26 | * @returns {{ close(): void }} | |
| 27 | */ | |
| 28 | export function openFeed(options) { | |
| 29 | const { url, onFrame } = options; | |
| 30 | const open = options.open ?? ((u) => new WebSocket(u)); | |
| 31 | const now = options.now ?? Date.now; | |
| 32 | const after = options.after ?? ((fn, ms) => setTimeout(fn, ms)); | |
| 33 | const cancel = options.cancel ?? ((t) => clearTimeout(t)); | |
| 34 | const gap = options.maxFps && options.maxFps > 0 ? 1000 / options.maxFps : 0; | |
| 35 | let ws = /** @type {WebSocket | null} */ (null), watchdog = /** @type {any} */ (null), closed = false, shown = -Infinity; | |
| 36 | ||
| 37 | const stop = () => { | |
| 38 | cancel(watchdog); watchdog = null; | |
| 39 | const w = ws; ws = null; | |
| 40 | if (w) { w.onmessage = w.onclose = w.onerror = null; try { w.close(); } catch { /* already gone */ } } | |
| 41 | }; | |
| 42 | /** Replace the socket, after `ms`. */ | |
| 43 | const again = (ms) => { | |
| 44 | stop(); | |
| 45 | if (!closed) watchdog = after(connect, ms); | |
| 46 | }; | |
| 47 | function connect() { | |
| 48 | watchdog = null; | |
| 49 | if (closed) return; | |
| 50 | let w; | |
| 51 | try { w = open(url); } catch { return again(RETRY_MS); } | |
| 52 | ws = w; | |
| 53 | w.binaryType = "blob"; | |
| 54 | w.onmessage = (e) => { | |
| 55 | if (ws !== w || !(e.data instanceof Blob)) return; | |
| 56 | cancel(watchdog); watchdog = after(() => again(0), QUIET_MS); | |
| 57 | const t = now(); | |
| 58 | if (t - shown < gap) return; // held to fewer frames on a slow link | |
| 59 | shown = t; | |
| 60 | onFrame(e.data); | |
| 61 | }; | |
| 62 | w.onclose = w.onerror = () => { if (ws === w) again(RETRY_MS); }; | |
| 63 | watchdog = after(() => again(0), QUIET_MS); // the first frame is due too | |
| 64 | } | |
| 65 | connect(); | |
| 66 | return { close() { closed = true; stop(); } }; | |
| 67 | } | |
| 68 | ||
| 69 | /** | |
| 70 | * Draw one frame in a camera's <img> and let go of the one before it, so memory does not grow with the frames. | |
| 71 | * @param {{ img: { src: string }, blob?: string, lastFrame?: number }} cam | |
| 72 | * @param {Blob} frame | |
| 73 | * @param {{ create(b: Blob): string, revoke(u: string): void, now(): number }} env | |
| 74 | */ | |
| 75 | export function drawFrame(cam, frame, env) { | |
| 76 | const url = env.create(frame); | |
| 77 | cam.img.src = url; | |
| 78 | if (cam.blob) env.revoke(cam.blob); | |
| 79 | cam.blob = url; | |
| 80 | cam.lastFrame = env.now(); | |
| 81 | } | |
| 82 | ||
| 83 | /** Whether a socket-drawn camera is live: a frame lately, and the relay not saying it is offline. @param {{ lastFrame?: number, online?: boolean | null }} cam */ | |
| 84 | export const isLive = (cam, now) => now - (cam.lastFrame ?? -Infinity) < QUIET_MS + 500 && cam.online !== false; | |
| 85 | ||
| 86 | /** | |
| 87 | * Start a camera's socket, unless the page moved on while this module was loading: `epoch` is the camera's count when | |
| 88 | * the load began, and the page counts up whenever it shows or hides the camera again. | |
| 89 | * @param {{ id: number | string, epoch: number, feed: any, img: { src: string }, blob?: string, lastFrame?: number }} cam | |
| 90 | * @param {number} epoch | |
| 91 | * @param {{ visible(): boolean, loc: { protocol: string, host: string }, maxFps?: number, create(b: Blob): string, revoke(u: string): void, now(): number }} env | |
| 92 | */ | |
| 93 | export function attach(cam, epoch, env) { | |
| 94 | if (cam.epoch !== epoch || !env.visible()) return; | |
| 95 | cam.feed = openFeed({ url: socketUrl(env.loc, cam.id), maxFps: env.maxFps, onFrame: (frame) => drawFrame(cam, frame, env) }); | |
| 96 | } |
Software/so101/src/ui/video.test.ts added +146
| @@ -0,0 +1,146 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { attach, drawFrame, isLive, openFeed, QUIET_MS, RETRY_MS, socketUrl } from "./video.js"; | |
| 3 | ||
| 4 | /** A clock and a socket factory the test drives by hand. */ | |
| 5 | function rig(maxFps?: number) { | |
| 6 | let t = 0; | |
| 7 | const timers: { at: number; fn: () => void; id: number }[] = []; | |
| 8 | let next = 0; | |
| 9 | const sockets: any[] = []; | |
| 10 | const frames: Blob[] = []; | |
| 11 | const feed = openFeed({ | |
| 12 | url: "ws://x/camera/1/ws", maxFps, onFrame: (f) => frames.push(f), | |
| 13 | now: () => t, | |
| 14 | after: (fn, ms) => { const id = ++next; timers.push({ at: t + ms, fn, id }); return id; }, | |
| 15 | cancel: (id) => { const i = timers.findIndex((x) => x.id === id); if (i >= 0) timers.splice(i, 1); }, | |
| 16 | open: (url) => { const w: any = { url, closed: false, close() { w.closed = true; } }; sockets.push(w); return w; }, | |
| 17 | }); | |
| 18 | const advance = (ms: number) => { | |
| 19 | const end = t + ms; | |
| 20 | for (;;) { | |
| 21 | timers.sort((a, b) => a.at - b.at); | |
| 22 | const due = timers[0]; | |
| 23 | if (!due || due.at > end) break; | |
| 24 | timers.shift(); t = due.at; due.fn(); | |
| 25 | } | |
| 26 | t = end; | |
| 27 | }; | |
| 28 | const frame = (w = sockets.at(-1)) => w.onmessage({ data: new Blob([new Uint8Array([0xff, 0xd8])]) }); | |
| 29 | return { feed, sockets, frames, advance, frame }; | |
| 30 | } | |
| 31 | ||
| 32 | describe("the relayed camera's address", () => { | |
| 33 | test("the socket address is on the page's own origin", () => { | |
| 34 | expect(socketUrl({ protocol: "https:", host: "a.example" }, 2)).toBe("wss://a.example/camera/2/ws"); | |
| 35 | expect(socketUrl({ protocol: "http:", host: "localhost:8080" }, "1")).toBe("ws://localhost:8080/camera/1/ws"); | |
| 36 | expect(socketUrl({ protocol: "http:", host: "h" }, "../x")).toBe("ws://h/camera/..%2Fx/ws"); | |
| 37 | }); | |
| 38 | }); | |
| 39 | ||
| 40 | describe("the relayed feed", () => { | |
| 41 | test("draws frames as they come and keeps its socket while they do", () => { | |
| 42 | const r = rig(); | |
| 43 | r.frame(); r.advance(QUIET_MS - 1); r.frame(); r.advance(QUIET_MS - 1); r.frame(); | |
| 44 | expect(r.frames.length).toBe(3); | |
| 45 | expect(r.sockets.length).toBe(1); | |
| 46 | }); | |
| 47 | test("reconnects after 2 s without a frame, and again if the new socket stays silent", () => { | |
| 48 | const r = rig(); | |
| 49 | r.frame(); | |
| 50 | r.advance(QUIET_MS - 1); | |
| 51 | expect(r.sockets.length).toBe(1); | |
| 52 | r.advance(1); | |
| 53 | expect(r.sockets[0].closed).toBe(true); | |
| 54 | r.advance(0); | |
| 55 | expect(r.sockets.length).toBe(2); | |
| 56 | r.advance(QUIET_MS); | |
| 57 | expect(r.sockets.length).toBe(3); | |
| 58 | }); | |
| 59 | test("a socket that closes is replaced after a short wait; a late frame from the old one is ignored", () => { | |
| 60 | const r = rig(); | |
| 61 | const old = r.sockets[0]; | |
| 62 | old.onclose(); | |
| 63 | expect(r.sockets.length).toBe(1); | |
| 64 | r.advance(RETRY_MS); | |
| 65 | expect(r.sockets.length).toBe(2); | |
| 66 | old.onmessage?.({ data: new Blob([]) }); | |
| 67 | expect(r.frames.length).toBe(0); | |
| 68 | }); | |
| 69 | test("close stops everything", () => { | |
| 70 | const r = rig(); | |
| 71 | r.feed.close(); | |
| 72 | r.advance(10 * QUIET_MS); | |
| 73 | expect(r.sockets.length).toBe(1); | |
| 74 | expect(r.sockets[0].closed).toBe(true); | |
| 75 | }); | |
| 76 | test("a slow link gets fewer frames", () => { | |
| 77 | const r = rig(2); | |
| 78 | r.frame(); r.advance(100); r.frame(); r.advance(500); r.frame(); | |
| 79 | expect(r.frames.length).toBe(2); | |
| 80 | }); | |
| 81 | test("text is not a frame", () => { | |
| 82 | const r = rig(); | |
| 83 | r.sockets[0].onmessage({ data: "hello" }); | |
| 84 | expect(r.frames.length).toBe(0); | |
| 85 | }); | |
| 86 | }); | |
| 87 | ||
| 88 | /** What the page hands in, with the object URLs counted. */ | |
| 89 | function env(visible = true) { | |
| 90 | let n = 0, t = 1000; | |
| 91 | const live = new Set<string>(); | |
| 92 | return { live, tick: (ms: number) => (t += ms), e: { visible: () => visible, loc: { protocol: "https:", host: "h" }, maxFps: undefined, create: () => { const u = `blob:${++n}`; live.add(u); return u; }, revoke: (u: string) => void live.delete(u), now: () => t } }; | |
| 93 | } | |
| 94 | ||
| 95 | describe("drawing frames in the page", () => { | |
| 96 | test("each frame replaces the picture and frees the one before it", () => { | |
| 97 | const { e, live } = env(); | |
| 98 | const cam: any = { img: { src: "" } }; | |
| 99 | drawFrame(cam, new Blob(["a"]), e); | |
| 100 | drawFrame(cam, new Blob(["b"]), e); | |
| 101 | drawFrame(cam, new Blob(["c"]), e); | |
| 102 | expect(cam.img.src).toBe("blob:3"); | |
| 103 | expect([...live]).toEqual(["blob:3"]); | |
| 104 | expect(cam.lastFrame).toBe(1000); | |
| 105 | }); | |
| 106 | test("a camera is live while frames come, and not when the relay says it is offline", () => { | |
| 107 | expect(isLive({}, 5000)).toBe(false); | |
| 108 | expect(isLive({ lastFrame: 4000 }, 5000)).toBe(true); | |
| 109 | expect(isLive({ lastFrame: 4000, online: false }, 5000)).toBe(false); | |
| 110 | expect(isLive({ lastFrame: 1000 }, 5000)).toBe(false); | |
| 111 | }); | |
| 112 | test("attach opens the socket and draws what it sends", () => { | |
| 113 | const { e } = env(); | |
| 114 | const cam: any = { id: 2, epoch: 4, feed: null, img: { src: "" } }; | |
| 115 | const sockets: any[] = []; | |
| 116 | (globalThis as any).WebSocket = class { constructor(public url: string) { sockets.push(this); } close() {} }; | |
| 117 | attach(cam, 4, e); | |
| 118 | expect(sockets[0].url).toBe("wss://h/camera/2/ws"); | |
| 119 | sockets[0].onmessage({ data: new Blob(["x"]) }); | |
| 120 | expect(cam.img.src).toBe("blob:1"); | |
| 121 | cam.feed.close(); | |
| 122 | }); | |
| 123 | test("attach does nothing when the page moved on while the module loaded", () => { | |
| 124 | const { e } = env(); | |
| 125 | const cam: any = { id: 1, epoch: 5, feed: null, img: { src: "" } }; | |
| 126 | attach(cam, 4, e); // shown again since: a newer load owns it | |
| 127 | expect(cam.feed).toBeNull(); | |
| 128 | attach(cam, 5, env(false).e); // hidden or switched off since | |
| 129 | expect(cam.feed).toBeNull(); | |
| 130 | }); | |
| 131 | }); | |
| 132 | ||
| 133 | describe("the page's wiring", () => { | |
| 134 | const page = require("node:fs").readFileSync(new URL("../viewer.html", import.meta.url), "utf8") as string; | |
| 135 | test("a relayed camera in stream mode goes through video.js; a local one keeps the img stream", () => { | |
| 136 | expect(page).toMatch(/else if \(relayed\) startFeed\(c\);\n\s*else c\.img\.src = streamUrl\(c\);/); | |
| 137 | expect(page).toMatch(/import\("\/ui\/__VERSION__\/video\.js"\)/); | |
| 138 | expect(page).toContain("video.attach(c, epoch,"); | |
| 139 | expect(page).toContain("video.isLive(c, Date.now())"); | |
| 140 | }); | |
| 141 | test("showing, hiding or removing a camera stops its socket, which also voids a load in flight", () => { | |
| 142 | expect(page).toContain("c.epoch = (c.epoch ?? 0) + 1; c.feed?.close?.()"); | |
| 143 | expect(page).toMatch(/clearTimeout\(c\.timer\); stopFeed\(c\);\n\s*if \(!on/); | |
| 144 | expect(page).toContain("clearTimeout(c.timer); stopFeed(c); c.img.src"); | |
| 145 | }); | |
| 146 | }); |
Software/so101/src/viewer.html +13 −3
| @@ -643,11 +643,20 @@ | ||
| 643 | 643 | } catch { c.live = false; } |
| 644 | 644 | if (plan.mode === "snapshot") c.timer = setTimeout(() => snap(c), Math.max(0, plan.everyMs - (performance.now() - started))); |
| 645 | 645 | } |
| 646 | // Behind the relay the camera is frames over WS /camera/<id>/ws (src/ui/video.js, imported when first needed); on the machine itself, the plain <img> stream. | |
| 647 | let video = null; | |
| 648 | const stopFeed = (c) => { c.epoch = (c.epoch ?? 0) + 1; c.feed?.close?.(); c.feed = null; }; | |
| 649 | async function startFeed(c) { | |
| 650 | const epoch = c.epoch; | |
| 651 | try { video ??= await import("/ui/__VERSION__/video.js"); } catch { return; } | |
| 652 | video.attach(c, epoch, { visible: () => paneOn(c) && !document.hidden, loc: location, maxFps: plan.fps, create: URL.createObjectURL, revoke: URL.revokeObjectURL, now: Date.now }); | |
| 653 | } | |
outdated
This onFrame path (object URL, revoke the previous, lastFrame) has no test. Move it into video.js and cover it. |
||
| 646 | 654 | const showCam = (c, on) => { |
| 647 | 655 | c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on)); |
| 648 | clearTimeout(c.timer); | |
| 656 | clearTimeout(c.timer); stopFeed(c); | |
| 649 | 657 | if (!on || document.hidden) c.img.src = ""; |
| 650 | 658 | else if (plan.mode === "snapshot" && c.snapshot) snap(c); |
| 659 | else if (relayed) startFeed(c); | |
| 651 | 660 | else c.img.src = streamUrl(c); |
| 652 | 661 | }; |
| 653 | 662 | function setPane(key, on) { |
| @@ -688,7 +697,7 @@ | ||
| 688 | 697 | c.tag = el("span", { className: "tag" }); |
| 689 | 698 | c.pane = el("div", { className: "pane" }, c.img, c.tag); c.pane.dataset.key = `cam:${info.id}`; |
| 690 | 699 | c.btn = el("button", { type: "button" }); c.btn.setAttribute("aria-pressed", "false"); |
| 691 | c.img.onerror = () => { if (paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); }; | |
| 700 | c.img.onerror = () => { if (!relayed && paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); }; | |
| 692 | 701 | c.btn.onclick = () => pick(c.pane.dataset.key); |
| 693 | 702 | $("stage").insertBefore(c.pane, $("pane3")); |
| 694 | 703 | $("top").querySelector(".seg").insertBefore(c.btn, $("v3")); |
| @@ -696,7 +705,7 @@ | ||
| 696 | 705 | return c; |
| 697 | 706 | } |
| 698 | 707 | function removeCam(c) { |
| 699 | clearTimeout(c.timer); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob); | |
| 708 | clearTimeout(c.timer); stopFeed(c); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob); | |
| 700 | 709 | c.pane.remove(); c.btn.remove(); |
| 701 | 710 | cams.splice(cams.indexOf(c), 1); |
| 702 | 711 | fallback(); |
| @@ -756,6 +765,7 @@ | ||
| 756 | 765 | document.addEventListener("visibilitychange", () => { for (const c of cams) if (paneOn(c)) showCam(c, true); }); |
| 757 | 766 | setInterval(() => { |
| 758 | 767 | for (const c of cams) { |
| 768 | if (c.feed && paneOn(c)) { c.live = video.isLive(c, Date.now()); continue; } | |
| 759 | 769 | if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue; |
| 760 | 770 | let changed = false; |
| 761 | 771 | try { |
new above
offline
Dead in production: only the test calls cameraSource, and viewer.html uses
relayeddirectly. Remove it or use it.