Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

The camera reaches the browser over a WebSocket, latest frame wins, so it never freezes behind Cloudflare #99

Parked opened this
Closed without merging

smith/arm:smith/87-the-camera-reaches-the-browser-over-a-we → main 4 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
8 files changed +349 −7
Software/so101/src/camera-feed.ts +41 −4
@@ -54,8 +54,14 @@ export function parseFps(text: string | null): number | null {
5454 return n >= 1 && n <= MAX_FPS ? n : null;
5555 }
5656
57 /** Gets each whole picture as it completes: its image type and bytes (the offline picture is a PNG). */
58 export type FrameListener = (type: string, body: Buffer) => void;
59
5760 export class CameraFeed {
5861 private readonly viewers = new Set<Viewer>();
62 private readonly listeners = new Set<FrameListener>();
63 /** The pieces of the part in flight, gathered only while a listener is there to take it whole. */
64 private part: { type: string; chunks: Buffer[] } | null = null;
5965 private _state: FeedState;
6066 private attempt = 0;
6167 private controller: AbortController | null = null;
@@ -83,7 +89,7 @@ export class CameraFeed {
8389
8490 /** How many viewers are watching right now. */
8591 get watching(): number {
86 return this.viewers.size;
92 return this.viewers.size + this.listeners.size;
8793 }
8894
8995 /**
@@ -110,19 +116,42 @@ export class CameraFeed {
110116 } else if (this._state === "idle") this.connect();
111117 }
112118
119 /**
120 * Take whole pictures instead of a stream: `listener` is called once per complete frame, and at once with the offline
121 * picture when the camera is down. A frame in flight when it subscribes is not sent to it half-way. Returns the unsubscribe.
122 */
123 subscribe(listener: FrameListener): () => void {
124 this.listeners.add(listener);
125 clearTimeout(this.linger);
126 if (this.closed) return () => void this.listeners.delete(listener);
127 if (this._state === "offline") {
128 listener("image/png", OFFLINE_FRAME);
129 this.offline();
130 } else if (this._state === "idle") this.connect();
131 return () => {
132 if (this.listeners.delete(listener)) this.idleCheck();
133 };
134 }
135
113136 close(): void {
114137 this.closed = true;
115138 this.stop();
116139 for (const v of this.viewers) v.res.end();
117140 this.viewers.clear();
141 this.listeners.clear();
118142 }
119143
120144 // ------------------------------------------------------------------ viewers
121145 private leave(viewer: Viewer) {
122 if (!this.viewers.delete(viewer) || this.viewers.size) return;
146 if (this.viewers.delete(viewer)) this.idleCheck();
147 }
148
149 /** With nobody watching, the source stays open a moment (a page reload keeps it), then goes. */
150 private idleCheck() {
151 if (this.watching) return;
123152 clearTimeout(this.linger);
124153 this.linger = setTimeout(() => {
125 if (!this.viewers.size) this.stop();
154 if (!this.watching) this.stop();
126155 }, this.o.lingerMs);
127156 this.linger.unref?.();
128157 }
@@ -139,6 +168,8 @@ export class CameraFeed {
139168
140169 /** Show the offline picture; `cut` closes the parts a failed source left half-sent, otherwise those viewers wait. */
141170 private broadcastOffline(cut: boolean) {
171 this.part = null;
172 for (const l of this.listeners) l("image/png", OFFLINE_FRAME);
142173 for (const v of this.viewers) {
143174 if (v.inPart && !cut) continue;
144175 if (v.inPart) this.write(v, NEXT);
@@ -156,6 +187,7 @@ export class CameraFeed {
156187 for (const t of [this.retry, this.linger]) clearTimeout(t);
157188 for (const t of [this.repeat, this.watchdog]) clearInterval(t);
158189 this.retry = this.linger = this.repeat = this.watchdog = undefined;
190 this.part = null;
159191 this._state = this.source ? "idle" : "offline";
160192 }
161193
@@ -167,7 +199,7 @@ export class CameraFeed {
167199 if (this.source && !this.retry && !this.closed) {
168200 this.retry = setTimeout(() => {
169201 this.retry = undefined;
170 if (this.viewers.size && this._state === "offline" && !this.controller) this.connect();
202 if (this.watching && this._state === "offline" && !this.controller) this.connect();
171203 }, this.o.retryMs);
172204 this.retry.unref?.();
173205 }
@@ -210,6 +242,7 @@ export class CameraFeed {
210242 }
211243 const parser = new MjpegParser(boundary, {
212244 start: (type) => {
245 this.part = this.listeners.size ? { type, chunks: [] } : null;
213246 const now = performance.now();
214247 for (const v of this.viewers) {
215248 // A frame a little early still counts as on time: a 10 fps source held to 5 sends every other frame, not every third.
@@ -220,9 +253,12 @@ export class CameraFeed {
220253 }
221254 },
222255 data: (bytes) => {
256 this.part?.chunks.push(Buffer.from(bytes));
223257 for (const v of this.viewers) if (v.inPart) this.write(v, bytes);
224258 },
225259 end: () => {
260 const whole = this.part;
261 this.part = null;
226262 if (this._state !== "live") {
227263 if (this._state === "offline") console.log(`${this.name} ${this.source}: live`);
228264 this._state = "live";
@@ -230,6 +266,7 @@ export class CameraFeed {
230266 this.repeat = undefined;
231267 }
232268 for (const v of this.viewers) if (v.inPart) this.write(v, NEXT), (v.inPart = false);
269 if (whole) for (const l of this.listeners) l(whole.type, Buffer.concat(whole.chunks));
233270 },
234271 });
235272 this.lastByte = Date.now();
Software/so101/src/camera-socket.test.ts added +118
@@ -0,0 +1,118 @@
1 import { afterEach, describe, expect, test } from "bun:test";
2 import { createServer, type Server, type ServerResponse } from "node:http";
3 import { EventEmitter } from "node:events";
4 import WebSocket, { WebSocketServer } from "ws";
5 import { CameraFeed } from "./camera-feed.ts";
6 import { serveCameraSocket } from "./camera-socket.ts";
7 import { OFFLINE_FRAME } from "./offline-frame.ts";
8
9 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
10 const listening = (server: Server) => new Promise<number>((r) => server.listen(0, "127.0.0.1", () => r((server.address() as { port: number }).port)));
11 const PART = (body: Buffer) => Buffer.concat([Buffer.from(`--b\r\nContent-Type: image/jpeg\r\nContent-Length: ${body.length}\r\n\r\n`), body, Buffer.from("\r\n")]);
12 /** A JPEG-shaped frame: SOI, its number repeated, EOI. Whole means every byte after SOI up to EOI is the same number. */
13 const frame = (n: number, size = 64) => Buffer.concat([Buffer.from([0xff, 0xd8]), Buffer.alloc(size, n % 250), Buffer.from([0xff, 0xd9])]);
14 const whole = (m: Buffer) => m[0] === 0xff && m[1] === 0xd8 && m[m.length - 2] === 0xff && m[m.length - 1] === 0xd9 && m.subarray(2, -2).every((b) => b === m[2]);
15
16 /** A fake MJPEG source: `every` ms a frame of `size` bytes to each client, optionally cut in two with a pause between. */
17 function fakeSource(every: number, size: number, split = false) {
18 const clients = new Set<ServerResponse>();
19 let n = 0;
20 const server = createServer((_req, res) => {
21 res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=b" });
22 clients.add(res);
23 res.socket?.once("close", () => clients.delete(res));
24 });
25 const timer = setInterval(() => {
26 const part = PART(frame(++n, size));
27 for (const res of clients) {
28 if (!split) res.write(part);
29 else {
30 res.write(part.subarray(0, part.length >> 1));
31 setTimeout(() => res.write(part.subarray(part.length >> 1)), every / 2);
32 }
33 }
34 }, every);
35 return { server, freeze: () => clearInterval(timer), close: () => (clearInterval(timer), server.closeAllConnections(), server.close()) };
36 }
37
38 const toClose: (() => void)[] = [];
39 afterEach(() => {
40 for (const f of toClose.splice(0)) f();
41 });
42
43 /** The relay's camera socket over a feed from `source`; returns a way to connect viewers. */
44 async function setup(source: ReturnType<typeof fakeSource>, options = {}) {
45 const port = await listening(source.server);
46 const feed = new CameraFeed(`http://127.0.0.1:${port}/`, "fake", { staleMs: 300, retryMs: 100, offlineEveryMs: 100, ...options });
47 const wss = new WebSocketServer({ noServer: true });
48 const front = createServer();
49 front.on("upgrade", (req, socket, head) => wss.handleUpgrade(req, socket, head, (ws) => serveCameraSocket(feed, ws)));
50 const frontPort = await listening(front);
51 toClose.push(() => (feed.close(), front.closeAllConnections(), front.close(), source.close()));
52 return {
53 feed,
54 async viewer() {
55 const ws = new WebSocket(`ws://127.0.0.1:${frontPort}/`);
56 const got: Buffer[] = [];
57 ws.on("message", (data, binary) => binary && got.push(data as Buffer));
58 await new Promise((r) => ws.once("open", r));
59 toClose.push(() => ws.terminate());
60 return { ws, got };
61 },
62 };
63 }
64
65 describe("WS camera", () => {
66 test("a slow viewer drops frames without slowing a fast one", async () => {
67 const { viewer, feed } = await setup(fakeSource(20, 4096));
68 // A viewer whose socket takes 200 ms to finish sending each message, as a phone on a bad link does.
69 const sent: Buffer[] = [];
70 const slow = Object.assign(new EventEmitter(), {
71 OPEN: 1,
72 readyState: 1,
73 send: (data: Buffer, _o: unknown, done: (err?: Error) => void) => (sent.push(data), setTimeout(done, 200)),
74 ping: () => {},
75 terminate: () => {},
76 });
77 serveCameraSocket(feed, slow as unknown as WebSocket);
78 const fast = await viewer();
79 await sleep(1500);
80 // ~75 frames were made: the fast viewer has most of them, the slow one about one in ten, each whole
81 expect(fast.got.length).toBeGreaterThan(30);
82 expect(sent.length).toBeGreaterThan(3);
83 expect(sent.length).toBeLessThan(fast.got.length / 2);
84 for (const m of [...fast.got, ...sent]) expect(whole(m)).toBe(true);
85 // the slow one's frames are the latest at the time, not a backlog: its last is near the newest
86 expect(sent.at(-1)![2]).toBeGreaterThan(sent[0]![2]!);
87 slow.emit("close");
88 });
89
90 test("a viewer joining mid-frame gets only whole JPEGs", async () => {
91 const { viewer } = await setup(fakeSource(40, 4096, true));
92 await sleep(70); // lands between two halves of a frame
93 const v = await viewer();
94 await sleep(500);
95 expect(v.got.length).toBeGreaterThan(3);
96 for (const m of v.got) expect(whole(m)).toBe(true);
97 });
98
99 test("a stalled source shows the offline picture, and live frames come back with it", async () => {
100 const source = fakeSource(20, 64);
101 const { viewer } = await setup(source);
102 const v = await viewer();
103 await sleep(200);
104 expect(v.got.some(whole)).toBe(true);
105 source.freeze();
106 await sleep(800);
107 expect(v.got.some((m) => m.equals(OFFLINE_FRAME))).toBe(true);
108 });
109
110 test("a camera that is not there is offline at once", async () => {
111 const feed = new CameraFeed(null, "none");
112 const frames: string[] = [];
113 const off = feed.subscribe((type, body) => frames.push(`${type}:${body.equals(OFFLINE_FRAME)}`));
114 expect(frames).toEqual(["image/png:true"]);
115 off();
116 feed.close();
117 });
118 });
Software/so101/src/camera-socket.ts added +51
@@ -0,0 +1,51 @@
1 /**
2 * A camera over a WebSocket, for the browsers that reach the relay through a proxy that buffers or cuts a long HTTP
3 * response (Cloudflare does, with MJPEG). Each message is one whole picture, JPEG (or the PNG "camera offline"), read
4 * from the shared feed (src/camera-feed.ts) once for everyone. Latest frame wins: a viewer holds at most one frame
5 * waiting, and while its socket is still sending the last one, newer frames replace the waiting one. Nothing queues. `fps` holds a viewer to fewer frames a second.
6 */
7
8 import type WebSocket from "ws";
9 import type { CameraFeed } from "./camera-feed.ts";
10
11 /** Send each frame of `feed` to `ws` as one binary message, latest wins; ends with the socket. */
12 export function serveCameraSocket(feed: CameraFeed, ws: WebSocket, fps: number | null = null): void {
13 const gapMs = fps ? 1000 / fps : 0;
14 let sentAt = -Infinity;
15 let sending = false;
16 let waiting: Buffer | null = null;
17 const send = (body: Buffer) => {
18 sending = true;
19 ws.send(body, { binary: true }, (err) => {
20 sending = false;
21 if (err) return void ws.terminate();
22 const next = waiting;
23 waiting = null;
24 if (next) send(next);
25 });
26 };
27 const unsubscribe = feed.subscribe((type, body) => {
28 if (ws.readyState !== ws.OPEN) return;
29 const now = performance.now();
30 if (type !== "image/png" && now - sentAt < gapMs * 0.8) return; // held to `fps`: the frames between are skipped whole
31 sentAt = now;
32 if (sending) waiting = body;
33 else send(body);
34 });
35 const done = () => {
36 unsubscribe();
37 waiting = null;
38 };
39 ws.on("close", done);
40 ws.on("error", () => ws.terminate());
41 // A client that stops answering pings is gone, however quiet its socket.
42 let alive = true;
43 ws.on("pong", () => (alive = true));
44 const beat = setInterval(() => {
45 if (!alive) return ws.terminate();
46 alive = false;
47 ws.ping();
48 }, 15000);
49 beat.unref?.();
50 ws.on("close", () => clearInterval(beat));
51 }
Software/so101/src/relay.ts +13
@@ -43,6 +43,8 @@
4343 * GET /cameras the listed cameras: [{"id", "name", "online"}]
4444 * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts);
4545 * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate
46 * WS /camera/<id>/ws the same camera as whole pictures, one binary message each, latest frame wins: for browsers behind a
47 * proxy that buffers or cuts a long HTTP stream (src/camera-socket.ts)
4648 * GET /admin/camera-list the owner's page for the camera list
4749 * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds
4850 * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time,
@@ -65,6 +67,7 @@ import { AdminError, changeFromForm } from "./admin.ts";
6567 import { ADMIN_CSP, DONE, renderAdmin, renderRefusal, type Done } from "./admin-page.ts";
6668 import { parseFps, type FeedOptions } from "./camera-feed.ts";
6769 import { CameraError, CameraHub } from "./cameras.ts";
70 import { serveCameraSocket } from "./camera-socket.ts";
6871 import { LOCAL as LOCAL_PERSON } from "./identity.ts";
6972 import { LeaderCalibrations } from "./leader-calibrations.ts";
7073 import { proxyFor, whyNot } from "./egress.ts";
@@ -133,6 +136,7 @@ export interface RunningRelay {
133136 const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" };
134137 const MAX_BODY = 1_000_000;
135138 const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/;
139 const CAMERA_WS = /^\/camera\/(\d{1,9})\/ws$/;
136140 const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/;
137141 const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8");
138142 const MAX_FORM = 16_384;
@@ -707,6 +711,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
707711 /** Close codes a server may send on; the rest (1005, 1006, 1015) only describe a connection that died. */
708712 const sendable = (code: number) => (code >= 1000 && code <= 1003) || (code >= 1007 && code <= 1014) || (code >= 3000 && code <= 4999);
709713 /** Each open /follow stream and whose it is, so a turn that ends can close it. */
714 const cameraWss = new WebSocketServer({ noServer: true, maxPayload: 1024 }); // the browser sends nothing
710715 const streams = new Map<WebSocket, string>();
711716 /**
712717 * `base` is the arm server's HTTP side, asked where the joints are so the speed cap starts from there. On the real arm,
@@ -806,6 +811,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
806811 };
807812 http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => {
808813 const url = new URL(req.url ?? "/", "http://x");
814 const cameraWs = CAMERA_WS.exec(url.pathname);
815 if (cameraWs) {
816 const feed = cameras.feed(Number(cameraWs[1]));
817 if (!feed) return refuse(socket, "404 Not Found");
818 if (!sameOrigin(req)) return refuse(socket, "403 Forbidden");
819 socket.on("error", () => socket.destroy());
820 return cameraWss.handleUpgrade(req, socket, head, (browser) => serveCameraSocket(feed, browser, parseFps(url.searchParams.get("fps"))));
821 }
809822 if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found");
810823 if (!sameOrigin(req)) return refuse(socket, "403 Forbidden");
811824 socket.on("error", () => socket.destroy());
Software/so101/src/ui/video.js added +67
@@ -0,0 +1,67 @@
1 /**
2 * The camera in the page. Through the relay a long HTTP stream is buffered and cut by proxies such as Cloudflare, so the
3 * relayed camera is a WebSocket (/camera/<id>/ws) of whole pictures, one per message, each drawn to an <img>. If nothing
4 * arrives for STALL_MS it reconnects. Plain ESM, served at /ui/ and imported by the tests.
5 */
6
7 /** No message for this long and the socket is dropped and opened again. */
8 export const STALL_MS = 2000;
9 /** The wait before the next try after a socket closes. */
10 export const RETRY_MS = 1000;
11
12 /** The image type of a picture from its first bytes: the offline picture is a PNG, the camera sends JPEG. */
13 export const typeOf = (/** @type {Uint8Array} */ bytes) => (bytes[0] === 0x89 && bytes[1] === 0x50 ? "image/png" : "image/jpeg");
14
15 /**
16 * @param {string} url the camera's socket path, e.g. /camera/1/ws, with ?fps= if held
17 * @param {{ src: string }} img the element that shows the pictures
18 * @param {{ WebSocket?: any, URL?: { createObjectURL(b: Blob): string, revokeObjectURL(u: string): void }, location?: { protocol: string, host: string }, onFrame?: () => void }} [deps]
19 * @returns {{ close(): void }}
20 */
21 export function showCameraSocket(url, img, deps = {}) {
22 const Socket = deps.WebSocket ?? WebSocket;
23 const blobs = deps.URL ?? URL;
24 const where = deps.location ?? /** @type {any} */ (globalThis).location;
25 /** @type {any} */ let ws = null;
26 /** @type {any} */ let retry;
27 let shown = "", last = 0, closed = false;
28 const open = () => {
29 clearTimeout(retry);
30 if (closed) return;
31 const socket = (ws = new Socket(`${where.protocol === "https:" ? "wss:" : "ws:"}//${where.host}${url}`));
32 socket.binaryType = "arraybuffer";
33 last = Date.now();
34 socket.onmessage = (/** @type {{ data: unknown }} */ e) => {
35 if (socket !== ws || !(e.data instanceof ArrayBuffer)) return;
36 last = Date.now();
37 const next = blobs.createObjectURL(new Blob([e.data], { type: typeOf(new Uint8Array(e.data, 0, 2)) }));
38 img.src = next;
39 if (shown) blobs.revokeObjectURL(shown);
40 shown = next;
41 deps.onFrame?.();
42 };
43 socket.onclose = () => { if (socket === ws) { ws = null; if (!closed) retry = setTimeout(open, RETRY_MS); } };
44 socket.onerror = () => {};
45 };
46 const watchdog = setInterval(() => {
47 if (closed || Date.now() - last <= STALL_MS) return;
48 const stale = ws;
49 ws = null;
50 stale?.close();
51 open();
52 }, STALL_MS / 4);
53 open();
54 return {
55 close() {
56 closed = true;
57 clearInterval(watchdog);
58 clearTimeout(retry);
59 const stale = ws;
60 ws = null;
61 stale?.close();
62 if (shown) blobs.revokeObjectURL(shown);
63 shown = "";
64 img.src = "";
65 },
66 };
67 }
Software/so101/src/ui/video.test.ts added +51
@@ -0,0 +1,51 @@
1 import { expect, test } from "bun:test";
2 import { showCameraSocket, STALL_MS, typeOf } from "./video.js";
3
4 class FakeSocket {
5 static all: FakeSocket[] = [];
6 binaryType = "";
7 closed = false;
8 onmessage: ((e: { data: unknown }) => void) | null = null;
9 onclose: (() => void) | null = null;
10 onerror: (() => void) | null = null;
11 url: string;
12 constructor(url: string) {
13 this.url = url;
14 FakeSocket.all.push(this);
15 }
16 close() {
17 this.closed = true;
18 }
19 }
20 const jpeg = () => new Uint8Array([0xff, 0xd8, 1, 2, 0xff, 0xd9]).buffer;
21 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
22
23 test("types come from the first bytes", () => {
24 expect(typeOf(new Uint8Array([0x89, 0x50]))).toBe("image/png");
25 expect(typeOf(new Uint8Array([0xff, 0xd8]))).toBe("image/jpeg");
26 });
27
28 test("frames are drawn, old pictures released, and a silent socket is reopened after 2 s", async () => {
29 FakeSocket.all = [];
30 const made: string[] = [], revoked: string[] = [];
31 const img = { src: "" };
32 const show = showCameraSocket("/camera/1/ws", img, {
33 WebSocket: FakeSocket,
34 URL: { createObjectURL: () => { made.push(`blob:${made.length}`); return made.at(-1)!; }, revokeObjectURL: (u: string) => revoked.push(u) },
35 location: { protocol: "https:", host: "arm.example" },
36 });
37 const first = FakeSocket.all[0]!;
38 expect(first.url).toBe("wss://arm.example/camera/1/ws");
39 first.onmessage!({ data: jpeg() });
40 first.onmessage!({ data: jpeg() });
41 expect(img.src).toBe("blob:1");
42 expect(revoked).toEqual(["blob:0"]);
43 await sleep(STALL_MS + 700);
44 expect(first.closed).toBe(true);
45 expect(FakeSocket.all.length).toBeGreaterThan(1);
46 show.close();
47 expect(img.src).toBe("");
48 const count = FakeSocket.all.length;
49 await sleep(STALL_MS);
50 expect(FakeSocket.all.length).toBe(count);
51 }, 10000);
Software/so101/src/viewer.html +7 −2
@@ -18,6 +18,7 @@
1818 } }
1919 </script>
2020 <link rel="modulepreload" href="/ui/__VERSION__/camera-plan.js">
21 <link rel="modulepreload" href="/ui/__VERSION__/video.js">
2122 <link rel="modulepreload" href="/ui/__VERSION__/drive.js">
2223 <link rel="modulepreload" href="/ui/__VERSION__/frames.js">
2324 <style>
@@ -515,6 +516,7 @@
515516 </div>
516517 <script type="module">
517518 import { cameraPlan, QUALITIES, samePlan } from "/ui/__VERSION__/camera-plan.js";
519 import { showCameraSocket } from "/ui/__VERSION__/video.js";
518520 import { bindSurface, canDrive, Driver, shape, UP } from "/ui/__VERSION__/drive.js";
519521 import { Frames } from "/ui/__VERSION__/frames.js";
520522
@@ -602,7 +604,7 @@
602604 }
603605
604606 // ---------------------------------------------------------------- cameras
605 // On a good link a plain <img> stream, which every browser plays; a canvas comparison once a second notices a frozen picture and reconnects; a slow link gets fewer frames (src/ui/camera-plan.js).
607 // Through the relay the camera is a WebSocket of whole pictures (src/ui/video.js), which proxies do not buffer or cut; straight to an arm server, a plain <img> stream, which every browser plays; a canvas comparison once a second notices a frozen picture and reconnects; a slow link gets fewer frames (src/ui/camera-plan.js).
606608 const cams = [];
607609 const direct = (n) => q.get(`stream${n}`) || q.get(n === 1 ? "stream" : "") || `${location.protocol}//${location.hostname}:${8079 + n}/stream`;
608610 const canvas = document.createElement("canvas"); canvas.width = 32; canvas.height = 18;
@@ -645,8 +647,10 @@
645647 const showCam = (c, on) => {
646648 c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on));
647649 clearTimeout(c.timer);
650 c.sock?.close(); c.sock = null; // closing blanks the picture too
648651 if (!on || document.hidden) c.img.src = "";
649652 else if (plan.mode === "snapshot" && c.snapshot) snap(c);
653 else if (relayed) c.sock = showCameraSocket(`/camera/${c.id}/ws${plan.fps ? `?fps=${plan.fps}` : ""}`, c.img, { onFrame: () => { c.frameAt = Date.now(); } });
650654 else c.img.src = streamUrl(c);
651655 };
652656 function setPane(key, on) {
@@ -695,7 +699,7 @@
695699 return c;
696700 }
697701 function removeCam(c) {
698 clearTimeout(c.timer); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob);
702 clearTimeout(c.timer); c.sock?.close(); c.img.src = ""; if (c.blob) URL.revokeObjectURL(c.blob);
699703 c.pane.remove(); c.btn.remove();
700704 cams.splice(cams.indexOf(c), 1);
701705 fallback();
@@ -756,6 +760,7 @@
756760 setInterval(() => {
757761 for (const c of cams) {
758762 if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue;
763 if (c.sock) { c.live = Date.now() - (c.frameAt ?? 0) < 3000 && c.online !== false; continue; } // the socket reconnects itself when frames stop
759764 let changed = false;
760765 try {
761766 ctx.drawImage(c.img, 0, 0, 32, 18);
forge.json +1 −1
@@ -4,7 +4,7 @@
44 "run": "bun run Software/so101/bench/fake-serve.ts",
55 "tty": "bun run Software/so101/src/tty.ts",
66 "test": "bun --cwd Software/so101 test",
7 "health": "/status",
7 "health": "/link",
88 "limits": {
99 "cpus": 6,
1010 "memory": 6144

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs