Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
The camera reaches the browser over a WebSocket, latest frame wins, so it never freezes behind Cloudflare #103
smith/arm:smith/87-the-camera-reaches-the-browser-over-a-we → main 7 commits fork
8 files
changed +326
−7
Software/so101/src/camera-feed.ts +41 −4
| @@ -54,8 +54,14 @@ export function parseFps(text: string | null): number | null { | ||
| 54 | 54 | return n >= 1 && n <= MAX_FPS ? n : null; |
| 55 | 55 | } |
| 56 | 56 | |
| 57 | /** Gets each whole picture as it completes: its image type and bytes (the offline picture is a PNG). */ | |
| 58 | export type FrameListener = (type: string, body: Buffer) => void; | |
| 59 | ||
| 57 | 60 | export class CameraFeed { |
| 58 | 61 | private readonly viewers = new Set<Viewer>(); |
| 62 | private readonly listeners = new Set<FrameListener>(); | |
| 63 | /** The pieces of the part in flight, gathered only while a listener is there to take it whole. */ | |
| 64 | private part: { type: string; chunks: Buffer[] } | null = null; | |
| 59 | 65 | private _state: FeedState; |
| 60 | 66 | private attempt = 0; |
| 61 | 67 | private controller: AbortController | null = null; |
| @@ -83,7 +89,7 @@ export class CameraFeed { | ||
| 83 | 89 | |
| 84 | 90 | /** How many viewers are watching right now. */ |
| 85 | 91 | get watching(): number { |
| 86 | return this.viewers.size; | |
| 92 | return this.viewers.size + this.listeners.size; | |
| 87 | 93 | } |
| 88 | 94 | |
| 89 | 95 | /** |
| @@ -110,19 +116,42 @@ export class CameraFeed { | ||
| 110 | 116 | } else if (this._state === "idle") this.connect(); |
| 111 | 117 | } |
| 112 | 118 | |
| 119 | /** | |
| 120 | * Take whole pictures instead of a stream: `listener` is called once per complete frame, and at once with the offline | |
| 121 | * picture when the camera is down. A frame in flight when it subscribes is not sent to it half-way. Returns the unsubscribe. | |
| 122 | */ | |
| 123 | subscribe(listener: FrameListener): () => void { | |
| 124 | this.listeners.add(listener); | |
| 125 | clearTimeout(this.linger); | |
| 126 | if (this.closed) return () => void this.listeners.delete(listener); | |
| 127 | if (this._state === "offline") { | |
| 128 | listener("image/png", OFFLINE_FRAME); | |
| 129 | this.offline(); | |
| 130 | } else if (this._state === "idle") this.connect(); | |
| 131 | return () => { | |
| 132 | if (this.listeners.delete(listener)) this.idleCheck(); | |
| 133 | }; | |
| 134 | } | |
| 135 | ||
| 113 | 136 | close(): void { |
| 114 | 137 | this.closed = true; |
| 115 | 138 | this.stop(); |
| 116 | 139 | for (const v of this.viewers) v.res.end(); |
| 117 | 140 | this.viewers.clear(); |
| 141 | this.listeners.clear(); | |
| 118 | 142 | } |
| 119 | 143 | |
| 120 | 144 | // ------------------------------------------------------------------ viewers |
| 121 | 145 | private leave(viewer: Viewer) { |
| 122 | if (!this.viewers.delete(viewer) || this.viewers.size) return; | |
| 146 | if (this.viewers.delete(viewer)) this.idleCheck(); | |
| 147 | } | |
| 148 | ||
| 149 | /** With nobody watching, the source stays open a moment (a page reload keeps it), then goes. */ | |
| 150 | private idleCheck() { | |
| 151 | if (this.watching) return; | |
| 123 | 152 | clearTimeout(this.linger); |
| 124 | 153 | this.linger = setTimeout(() => { |
| 125 | if (!this.viewers.size) this.stop(); | |
| 154 | if (!this.watching) this.stop(); | |
| 126 | 155 | }, this.o.lingerMs); |
| 127 | 156 | this.linger.unref?.(); |
| 128 | 157 | } |
| @@ -139,6 +168,8 @@ export class CameraFeed { | ||
| 139 | 168 | |
| 140 | 169 | /** Show the offline picture; `cut` closes the parts a failed source left half-sent, otherwise those viewers wait. */ |
| 141 | 170 | private broadcastOffline(cut: boolean) { |
| 171 | this.part = null; | |
| 172 | for (const l of this.listeners) l("image/png", OFFLINE_FRAME); | |
| 142 | 173 | for (const v of this.viewers) { |
| 143 | 174 | if (v.inPart && !cut) continue; |
| 144 | 175 | if (v.inPart) this.write(v, NEXT); |
| @@ -156,6 +187,7 @@ export class CameraFeed { | ||
| 156 | 187 | for (const t of [this.retry, this.linger]) clearTimeout(t); |
| 157 | 188 | for (const t of [this.repeat, this.watchdog]) clearInterval(t); |
| 158 | 189 | this.retry = this.linger = this.repeat = this.watchdog = undefined; |
| 190 | this.part = null; | |
| 159 | 191 | this._state = this.source ? "idle" : "offline"; |
| 160 | 192 | } |
| 161 | 193 | |
| @@ -167,7 +199,7 @@ export class CameraFeed { | ||
| 167 | 199 | if (this.source && !this.retry && !this.closed) { |
| 168 | 200 | this.retry = setTimeout(() => { |
| 169 | 201 | this.retry = undefined; |
| 170 | if (this.viewers.size && this._state === "offline" && !this.controller) this.connect(); | |
| 202 | if (this.watching && this._state === "offline" && !this.controller) this.connect(); | |
| 171 | 203 | }, this.o.retryMs); |
| 172 | 204 | this.retry.unref?.(); |
| 173 | 205 | } |
| @@ -210,6 +242,7 @@ export class CameraFeed { | ||
| 210 | 242 | } |
| 211 | 243 | const parser = new MjpegParser(boundary, { |
| 212 | 244 | start: (type) => { |
| 245 | this.part = this.listeners.size ? { type, chunks: [] } : null; | |
| 213 | 246 | const now = performance.now(); |
| 214 | 247 | for (const v of this.viewers) { |
| 215 | 248 | // A frame a little early still counts as on time: a 10 fps source held to 5 sends every other frame, not every third. |
| @@ -220,9 +253,12 @@ export class CameraFeed { | ||
| 220 | 253 | } |
| 221 | 254 | }, |
| 222 | 255 | data: (bytes) => { |
| 256 | this.part?.chunks.push(Buffer.from(bytes)); | |
| 223 | 257 | for (const v of this.viewers) if (v.inPart) this.write(v, bytes); |
| 224 | 258 | }, |
| 225 | 259 | end: () => { |
| 260 | const whole = this.part; | |
| 261 | this.part = null; | |
| 226 | 262 | if (this._state !== "live") { |
| 227 | 263 | if (this._state === "offline") console.log(`${this.name} ${this.source}: live`); |
| 228 | 264 | this._state = "live"; |
| @@ -230,6 +266,7 @@ export class CameraFeed { | ||
| 230 | 266 | this.repeat = undefined; |
| 231 | 267 | } |
| 232 | 268 | for (const v of this.viewers) if (v.inPart) this.write(v, NEXT), (v.inPart = false); |
| 269 | if (whole) for (const l of this.listeners) l(whole.type, Buffer.concat(whole.chunks)); | |
| 233 | 270 | }, |
| 234 | 271 | }); |
| 235 | 272 | this.lastByte = Date.now(); |
Software/so101/src/camera-socket.test.ts added +118
| @@ -0,0 +1,118 @@ | ||
| 1 | import { afterEach, describe, expect, test } from "bun:test"; | |
| 2 | import { createServer, type Server, type ServerResponse } from "node:http"; | |
| 3 | import { EventEmitter } from "node:events"; | |
| 4 | import WebSocket, { WebSocketServer } from "ws"; | |
| 5 | import { CameraFeed } from "./camera-feed.ts"; | |
| 6 | import { serveCameraSocket } from "./camera-socket.ts"; | |
| 7 | import { OFFLINE_FRAME } from "./offline-frame.ts"; | |
| 8 | ||
| 9 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 10 | const listening = (server: Server) => new Promise<number>((r) => server.listen(0, "127.0.0.1", () => r((server.address() as { port: number }).port))); | |
| 11 | const PART = (body: Buffer) => Buffer.concat([Buffer.from(`--b\r\nContent-Type: image/jpeg\r\nContent-Length: ${body.length}\r\n\r\n`), body, Buffer.from("\r\n")]); | |
| 12 | /** A JPEG-shaped frame: SOI, its number repeated, EOI. Whole means every byte after SOI up to EOI is the same number. */ | |
| 13 | const frame = (n: number, size = 64) => Buffer.concat([Buffer.from([0xff, 0xd8]), Buffer.alloc(size, n % 250), Buffer.from([0xff, 0xd9])]); | |
| 14 | const whole = (m: Buffer) => m[0] === 0xff && m[1] === 0xd8 && m[m.length - 2] === 0xff && m[m.length - 1] === 0xd9 && m.subarray(2, -2).every((b) => b === m[2]); | |
| 15 | ||
| 16 | /** A fake MJPEG source: `every` ms a frame of `size` bytes to each client, optionally cut in two with a pause between. */ | |
| 17 | function fakeSource(every: number, size: number, split = false) { | |
| 18 | const clients = new Set<ServerResponse>(); | |
| 19 | let n = 0; | |
| 20 | const server = createServer((_req, res) => { | |
| 21 | res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=b" }); | |
| 22 | clients.add(res); | |
| 23 | res.socket?.once("close", () => clients.delete(res)); | |
| 24 | }); | |
| 25 | const timer = setInterval(() => { | |
| 26 | const part = PART(frame(++n, size)); | |
| 27 | for (const res of clients) { | |
| 28 | if (!split) res.write(part); | |
| 29 | else { | |
| 30 | res.write(part.subarray(0, part.length >> 1)); | |
| 31 | setTimeout(() => res.write(part.subarray(part.length >> 1)), every / 2); | |
| 32 | } | |
| 33 | } | |
| 34 | }, every); | |
| 35 | return { server, freeze: () => clearInterval(timer), close: () => (clearInterval(timer), server.closeAllConnections(), server.close()) }; | |
| 36 | } | |
| 37 | ||
| 38 | const toClose: (() => void)[] = []; | |
| 39 | afterEach(() => { | |
| 40 | for (const f of toClose.splice(0)) f(); | |
| 41 | }); | |
| 42 | ||
| 43 | /** The relay's camera socket over a feed from `source`; returns a way to connect viewers. */ | |
| 44 | async function setup(source: ReturnType<typeof fakeSource>, options = {}) { | |
| 45 | const port = await listening(source.server); | |
| 46 | const feed = new CameraFeed(`http://127.0.0.1:${port}/`, "fake", { staleMs: 300, retryMs: 100, offlineEveryMs: 100, ...options }); | |
| 47 | const wss = new WebSocketServer({ noServer: true }); | |
| 48 | const front = createServer(); | |
| 49 | front.on("upgrade", (req, socket, head) => wss.handleUpgrade(req, socket, head, (ws) => serveCameraSocket(feed, ws))); | |
| 50 | const frontPort = await listening(front); | |
| 51 | toClose.push(() => (feed.close(), front.closeAllConnections(), front.close(), source.close())); | |
| 52 | return { | |
| 53 | feed, | |
| 54 | async viewer() { | |
| 55 | const ws = new WebSocket(`ws://127.0.0.1:${frontPort}/`); | |
| 56 | const got: Buffer[] = []; | |
| 57 | ws.on("message", (data, binary) => binary && got.push(data as Buffer)); | |
| 58 | await new Promise((r) => ws.once("open", r)); | |
| 59 | toClose.push(() => ws.terminate()); | |
| 60 | return { ws, got }; | |
| 61 | }, | |
| 62 | }; | |
| 63 | } | |
| 64 | ||
| 65 | describe("WS camera", () => { | |
| 66 | test("a slow viewer drops frames without slowing a fast one", async () => { | |
| 67 | const { viewer, feed } = await setup(fakeSource(20, 4096)); | |
| 68 | // A viewer whose socket takes 200 ms to finish sending each message, as a phone on a bad link does. | |
| 69 | const sent: Buffer[] = []; | |
| 70 | const slow = Object.assign(new EventEmitter(), { | |
| 71 | OPEN: 1, | |
| 72 | readyState: 1, | |
| 73 | send: (data: Buffer, _o: unknown, done: (err?: Error) => void) => (sent.push(data), setTimeout(done, 200)), | |
| 74 | ping: () => {}, | |
| 75 | terminate: () => {}, | |
| 76 | }); | |
| 77 | serveCameraSocket(feed, slow as unknown as WebSocket); | |
| 78 | const fast = await viewer(); | |
| 79 | await sleep(1500); | |
| 80 | // ~75 frames were made: the fast viewer has most of them, the slow one about one in ten, each whole | |
| 81 | expect(fast.got.length).toBeGreaterThan(30); | |
| 82 | expect(sent.length).toBeGreaterThan(3); | |
| 83 | expect(sent.length).toBeLessThan(fast.got.length / 2); | |
| 84 | for (const m of [...fast.got, ...sent]) expect(whole(m)).toBe(true); | |
| 85 | // the slow one's frames are the latest at the time, not a backlog: its last is near the newest | |
| 86 | expect(sent.at(-1)![2]).toBeGreaterThan(sent[0]![2]!); | |
| 87 | slow.emit("close"); | |
| 88 | }); | |
| 89 | ||
| 90 | test("a viewer joining mid-frame gets only whole JPEGs", async () => { | |
| 91 | const { viewer } = await setup(fakeSource(40, 4096, true)); | |
| 92 | await sleep(70); // lands between two halves of a frame | |
| 93 | const v = await viewer(); | |
| 94 | await sleep(500); | |
| 95 | expect(v.got.length).toBeGreaterThan(3); | |
| 96 | for (const m of v.got) expect(whole(m)).toBe(true); | |
| 97 | }); | |
| 98 | ||
| 99 | test("a stalled source shows the offline picture, and live frames come back with it", async () => { | |
| 100 | const source = fakeSource(20, 64); | |
| 101 | const { viewer } = await setup(source); | |
| 102 | const v = await viewer(); | |
| 103 | await sleep(200); | |
| 104 | expect(v.got.some(whole)).toBe(true); | |
| 105 | source.freeze(); | |
| 106 | await sleep(800); | |
| 107 | expect(v.got.some((m) => m.equals(OFFLINE_FRAME))).toBe(true); | |
| 108 | }); | |
| 109 | ||
| 110 | test("a camera that is not there is offline at once", async () => { | |
| 111 | const feed = new CameraFeed(null, "none"); | |
| 112 | const frames: string[] = []; | |
| 113 | const off = feed.subscribe((type, body) => frames.push(`${type}:${body.equals(OFFLINE_FRAME)}`)); | |
| 114 | expect(frames).toEqual(["image/png:true"]); | |
| 115 | off(); | |
| 116 | feed.close(); | |
| 117 | }); | |
| 118 | }); |
Software/so101/src/camera-socket.ts added +51
| @@ -0,0 +1,51 @@ | ||
| 1 | /** | |
| 2 | * A camera over a WebSocket, for the browsers that reach the relay through a proxy that buffers or cuts a long HTTP | |
| 3 | * response (Cloudflare does, with MJPEG). Each message is one whole picture, JPEG (or the PNG "camera offline"), read | |
| 4 | * from the shared feed (src/camera-feed.ts) once for everyone. Latest frame wins: a viewer holds at most one frame | |
| 5 | * waiting, and while its socket is still sending the last one, newer frames replace the waiting one. Nothing queues. `fps` holds a viewer to fewer frames a second. | |
| 6 | */ | |
| 7 | ||
| 8 | import type WebSocket from "ws"; | |
| 9 | import type { CameraFeed } from "./camera-feed.ts"; | |
| 10 | ||
| 11 | /** Send each frame of `feed` to `ws` as one binary message, latest wins; ends with the socket. */ | |
| 12 | export function serveCameraSocket(feed: CameraFeed, ws: WebSocket, fps: number | null = null): void { | |
| 13 | const gapMs = fps ? 1000 / fps : 0; | |
| 14 | let sentAt = -Infinity; | |
| 15 | let sending = false; | |
| 16 | let waiting: Buffer | null = null; | |
| 17 | const send = (body: Buffer) => { | |
| 18 | sending = true; | |
| 19 | ws.send(body, { binary: true }, (err) => { | |
| 20 | sending = false; | |
| 21 | if (err) return void ws.terminate(); | |
| 22 | const next = waiting; | |
| 23 | waiting = null; | |
| 24 | if (next) send(next); | |
| 25 | }); | |
| 26 | }; | |
| 27 | const unsubscribe = feed.subscribe((type, body) => { | |
| 28 | if (ws.readyState !== ws.OPEN) return; | |
| 29 | const now = performance.now(); | |
| 30 | if (type !== "image/png" && now - sentAt < gapMs * 0.8) return; // held to `fps`: the frames between are skipped whole | |
| 31 | sentAt = now; | |
| 32 | if (sending) waiting = body; | |
| 33 | else send(body); | |
| 34 | }); | |
| 35 | const done = () => { | |
| 36 | unsubscribe(); | |
| 37 | waiting = null; | |
| 38 | }; | |
| 39 | ws.on("close", done); | |
| 40 | ws.on("error", () => ws.terminate()); | |
| 41 | // A client that stops answering pings is gone, however quiet its socket. | |
| 42 | let alive = true; | |
| 43 | ws.on("pong", () => (alive = true)); | |
| 44 | const beat = setInterval(() => { | |
| 45 | if (!alive) return ws.terminate(); | |
| 46 | alive = false; | |
| 47 | ws.ping(); | |
| 48 | }, 15000); | |
| 49 | beat.unref?.(); | |
| 50 | ws.on("close", () => clearInterval(beat)); | |
| 51 | } |
Software/so101/src/relay.ts +13
| @@ -43,6 +43,8 @@ | ||
| 43 | 43 | * GET /cameras the listed cameras: [{"id", "name", "online"}] |
| 44 | 44 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 45 | 45 | * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate |
| 46 | * WS /camera/<id>/ws the same camera as whole pictures, one binary message each, latest frame wins: for browsers behind a | |
| 47 | * proxy that buffers or cuts a long HTTP stream (src/camera-socket.ts) | |
| 46 | 48 | * GET /admin/camera-list the owner's page for the camera list |
| 47 | 49 | * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds |
| 48 | 50 | * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time, |
| @@ -65,6 +67,7 @@ import { AdminError, changeFromForm } from "./admin.ts"; | ||
| 65 | 67 | import { ADMIN_CSP, DONE, renderAdmin, renderRefusal, type Done } from "./admin-page.ts"; |
| 66 | 68 | import { parseFps, type FeedOptions } from "./camera-feed.ts"; |
| 67 | 69 | import { CameraError, CameraHub } from "./cameras.ts"; |
| 70 | import { serveCameraSocket } from "./camera-socket.ts"; | |
| 68 | 71 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 69 | 72 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| 70 | 73 | import { proxyFor, whyNot } from "./egress.ts"; |
| @@ -133,6 +136,7 @@ export interface RunningRelay { | ||
| 133 | 136 | const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" }; |
| 134 | 137 | const MAX_BODY = 1_000_000; |
| 135 | 138 | const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/; |
| 139 | const CAMERA_WS = /^\/camera\/(\d{1,9})\/ws$/; | |
| 136 | 140 | const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/; |
| 137 | 141 | const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8"); |
| 138 | 142 | const MAX_FORM = 16_384; |
| @@ -707,6 +711,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 707 | 711 | /** Close codes a server may send on; the rest (1005, 1006, 1015) only describe a connection that died. */ |
| 708 | 712 | const sendable = (code: number) => (code >= 1000 && code <= 1003) || (code >= 1007 && code <= 1014) || (code >= 3000 && code <= 4999); |
| 709 | 713 | /** Each open /follow stream and whose it is, so a turn that ends can close it. */ |
| 714 | const cameraWss = new WebSocketServer({ noServer: true, maxPayload: 1024 }); // the browser sends nothing | |
| 710 | 715 | const streams = new Map<WebSocket, string>(); |
| 711 | 716 | /** |
| 712 | 717 | * `base` is the arm server's HTTP side, asked where the joints are so the speed cap starts from there. On the real arm, |
| @@ -806,6 +811,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 806 | 811 | }; |
| 807 | 812 | http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => { |
| 808 | 813 | const url = new URL(req.url ?? "/", "http://x"); |
| 814 | const cameraWs = CAMERA_WS.exec(url.pathname); | |
| 815 | if (cameraWs) { | |
| 816 | const feed = cameras.feed(Number(cameraWs[1])); | |
| 817 | if (!feed) return refuse(socket, "404 Not Found"); | |
| 818 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); | |
| 819 | socket.on("error", () => socket.destroy()); | |
| 820 | return cameraWss.handleUpgrade(req, socket, head, (browser) => serveCameraSocket(feed, browser, parseFps(url.searchParams.get("fps")))); | |
| 821 | } | |
| 809 | 822 | if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); |
| 810 | 823 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); |
| 811 | 824 | socket.on("error", () => socket.destroy()); |
Software/so101/src/ui/video.js added +46
| @@ -0,0 +1,46 @@ | ||
| 1 | /** The relayed camera: whole pictures over a WebSocket (/camera/<id>/ws) drawn to an <img>; a proxy cannot buffer or cut it. */ | |
| 2 | ||
| 3 | /** Silence this long (ms) and the socket is reopened. */ | |
| 4 | export const STALL_MS = 2000; | |
| 5 | ||
| 6 | /** The image type from the first bytes: the offline picture is a PNG, the rest JPEG. */ | |
| 7 | export const typeOf = (/** @type {Uint8Array} */ b) => (b[0] === 0x89 ? "image/png" : "image/jpeg"); | |
| 8 | ||
| 9 | /** | |
| 10 | * @param {string} url | |
| 11 | * @param {{ src: string }} img | |
| 12 | * @param {{ WebSocket?: any, URL?: any, location?: any, onFrame?: () => void }} [deps] | |
| 13 | */ | |
| 14 | export function showCameraSocket(url, img, deps = {}) { | |
| 15 | const { WebSocket: Socket = WebSocket, URL: blobs = URL, location: where = /** @type {any} */ (globalThis).location, onFrame } = deps; | |
| 16 | /** @type {any} */ let ws = null; | |
| 17 | let shown = "", last = 0, off = false; | |
| 18 | const open = () => { | |
| 19 | if (off) return; | |
| 20 | const s = (ws = new Socket(`${where.protocol === "https:" ? "wss:" : "ws:"}//${where.host}${url}`)); | |
| 21 | s.binaryType = "arraybuffer"; | |
| 22 | last = Date.now(); | |
| 23 | s.onmessage = (/** @type {any} */ e) => { | |
| 24 | if (s !== ws) return; | |
| 25 | last = Date.now(); | |
| 26 | const next = blobs.createObjectURL(new Blob([e.data], { type: typeOf(new Uint8Array(e.data, 0, 1)) })); | |
| 27 | img.src = next; | |
| 28 | if (shown) blobs.revokeObjectURL(shown); | |
| 29 | shown = next; | |
| 30 | onFrame?.(); | |
| 31 | }; | |
| 32 | }; | |
| 33 | const drop = () => { const s = ws; ws = null; s?.close(); }; | |
| 34 | // A closed or silent socket is opened again by this check. | |
| 35 | const timer = setInterval(() => { if (!ws || Date.now() - last > STALL_MS) { drop(); open(); } }, STALL_MS / 4); | |
| 36 | open(); | |
| 37 | return { | |
| 38 | close() { | |
| 39 | off = true; | |
| 40 | clearInterval(timer); | |
| 41 | drop(); | |
| 42 | if (shown) blobs.revokeObjectURL(shown); | |
| 43 | img.src = shown = ""; | |
| 44 | }, | |
| 45 | }; | |
| 46 | } |
Software/so101/src/ui/video.test.ts added +49
| @@ -0,0 +1,49 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { showCameraSocket, STALL_MS, typeOf } from "./video.js"; | |
| 3 | ||
| 4 | class FakeSocket { | |
| 5 | static all: FakeSocket[] = []; | |
| 6 | binaryType = ""; | |
| 7 | closed = false; | |
| 8 | onmessage: ((e: { data: unknown }) => void) | null = null; | |
| 9 | url: string; | |
| 10 | constructor(url: string) { | |
| 11 | this.url = url; | |
| 12 | FakeSocket.all.push(this); | |
| 13 | } | |
| 14 | close() { | |
| 15 | this.closed = true; | |
| 16 | } | |
| 17 | } | |
| 18 | const jpeg = () => new Uint8Array([0xff, 0xd8, 1, 2, 0xff, 0xd9]).buffer; | |
| 19 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 20 | ||
| 21 | test("types come from the first bytes", () => { | |
| 22 | expect(typeOf(new Uint8Array([0x89, 0x50]))).toBe("image/png"); | |
| 23 | expect(typeOf(new Uint8Array([0xff, 0xd8]))).toBe("image/jpeg"); | |
| 24 | }); | |
| 25 | ||
| 26 | test("frames are drawn, old pictures released, and a silent socket is reopened after 2 s", async () => { | |
| 27 | FakeSocket.all = []; | |
| 28 | const made: string[] = [], revoked: string[] = []; | |
| 29 | const img = { src: "" }; | |
| 30 | const show = showCameraSocket("/camera/1/ws", img, { | |
| 31 | WebSocket: FakeSocket, | |
| 32 | URL: { createObjectURL: () => { made.push(`blob:${made.length}`); return made.at(-1)!; }, revokeObjectURL: (u: string) => revoked.push(u) }, | |
| 33 | location: { protocol: "https:", host: "arm.example" }, | |
| 34 | }); | |
| 35 | const first = FakeSocket.all[0]!; | |
| 36 | expect(first.url).toBe("wss://arm.example/camera/1/ws"); | |
| 37 | first.onmessage!({ data: jpeg() }); | |
| 38 | first.onmessage!({ data: jpeg() }); | |
| 39 | expect(img.src).toBe("blob:1"); | |
| 40 | expect(revoked).toEqual(["blob:0"]); | |
| 41 | await sleep(STALL_MS + 700); | |
| 42 | expect(first.closed).toBe(true); | |
| 43 | expect(FakeSocket.all.length).toBeGreaterThan(1); | |
| 44 | show.close(); | |
| 45 | expect(img.src).toBe(""); | |
| 46 | const count = FakeSocket.all.length; | |
| 47 | await sleep(STALL_MS); | |
| 48 | expect(FakeSocket.all.length).toBe(count); | |
| 49 | }, 10000); |
Software/so101/src/viewer.html +7 −3
| @@ -602,8 +602,9 @@ | ||
| 602 | 602 | } |
| 603 | 603 | |
| 604 | 604 | // ---------------------------------------------------------------- cameras |
| 605 | // On a good link a plain <img> stream, which every browser plays; a canvas comparison once a second notices a frozen picture and reconnects; a slow link gets fewer frames (src/ui/camera-plan.js). | |
| 605 | // Through the relay the camera is a WebSocket of whole pictures (src/ui/video.js), which proxies do not buffer or cut; straight to an arm server, a plain <img> stream, which every browser plays; a canvas comparison once a second notices a frozen picture and reconnects; a slow link gets fewer frames (src/ui/camera-plan.js). | |
| 606 | 606 | const cams = []; |
| 607 | const video = import("/ui/__VERSION__/video.js"); // starts now, beside the page; only the relayed camera needs it | |
| 607 | 608 | const direct = (n) => q.get(`stream${n}`) || q.get(n === 1 ? "stream" : "") || `${location.protocol}//${location.hostname}:${8079 + n}/stream`; |
| 608 | 609 | const canvas = document.createElement("canvas"); canvas.width = 32; canvas.height = 18; |
| 609 | 610 | const ctx = canvas.getContext("2d", { willReadFrequently: true }); |
| @@ -645,8 +646,10 @@ | ||
| 645 | 646 | const showCam = (c, on) => { |
| 646 | 647 | c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on)); |
| 647 | 648 | clearTimeout(c.timer); |
| 649 | c.sock?.close(); c.sock = null; // closing blanks the picture too | |
| 648 | 650 | if (!on || document.hidden) c.img.src = ""; |
| 649 | 651 | else if (plan.mode === "snapshot" && c.snapshot) snap(c); |
| 652 | else if (relayed) video.then(({ showCameraSocket }) => { if (c.sock === null && paneOn(c) && !document.hidden && plan.mode === "stream") c.sock = showCameraSocket(`/camera/${c.id}/ws${plan.fps ? `?fps=${plan.fps}` : ""}`, c.img, { onFrame: () => { c.frameAt = Date.now(); } }); }); | |
| 650 | 653 | else c.img.src = streamUrl(c); |
| 651 | 654 | }; |
| 652 | 655 | function setPane(key, on) { |
| @@ -681,7 +684,7 @@ | ||
| 681 | 684 | else if (cams[0]) showCam(cams[0], true); |
| 682 | 685 | } |
| 683 | 686 | function addCam(info) { |
| 684 | const c = { id: info.id, url: info.url, sig: "", unchanged: 0, live: false, online: info.online, timer: null, blob: "" }; | |
| 687 | const c = { id: info.id, url: info.url, sig: "", unchanged: 0, live: false, online: info.online, timer: null, blob: "", sock: null, frameAt: 0 }; | |
| 685 | 688 | c.snapshot = /\/stream(?=$|\?)/.test(c.url) ? c.url.replace(/\/stream(?=$|\?)/, "/snapshot") : null; |
| 686 | 689 | c.img = el("img", { alt: info.name, decoding: "async", crossOrigin: "anonymous" }); |
| 687 | 690 | c.tag = el("span", { className: "tag" }); |
| @@ -695,7 +698,7 @@ | ||
| 695 | 698 | return c; |
| 696 | 699 | } |
| 697 | 700 | function removeCam(c) { |
| 698 | clearTimeout(c.timer); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob); | |
| 701 | clearTimeout(c.timer); c.sock?.close(); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob); | |
| 699 | 702 | c.pane.remove(); c.btn.remove(); |
| 700 | 703 | cams.splice(cams.indexOf(c), 1); |
| 701 | 704 | fallback(); |
| @@ -756,6 +759,7 @@ | ||
| 756 | 759 | setInterval(() => { |
| 757 | 760 | for (const c of cams) { |
| 758 | 761 | if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue; |
| 762 | if (relayed) { c.live = Date.now() - c.frameAt < 3000 && c.online !== false; continue; } // the socket reconnects itself when frames stop | |
| 759 | 763 | let changed = false; |
| 760 | 764 | try { |
| 761 | 765 | ctx.drawImage(c.img, 0, 0, 32, 18); |
forge.json +1
| @@ -16,6 +16,7 @@ | ||
| 16 | 16 | "100.89.95.68:8101" |
| 17 | 17 | ], |
| 18 | 18 | "check": { |
| 19 | "run": "bun install --cwd Software/so101 --frozen-lockfile && cd Software/so101 && bun x tsc --noEmit && bun test", | |
| 19 | 20 | "limits": { |
| 20 | 21 | "cpus": 6, |
| 21 | 22 | "memory": 6144 |
new above
offline