Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

The relay serves WS /camera/<id>/ws from the shared camera feed, and the offline picture when the Pi camera is down #172

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/139-the-relay-serves-ws-camera-id-ws-from-th → main 2 commits fork

  • Merged. smith/139-the-relay-serves-ws-camera-id-ws-from-th was merged into main as 46c4128.

Review

Sign in to review this change.
4 files changed +121 −4
Software/so101/src/camera-feed.ts +43 −4
@@ -28,6 +28,14 @@ export interface FeedOptions {
2828 maxBacklog?: number;
2929 }
3030
31 /** Something that takes the feed's pictures whole rather than as a multipart body (the camera WebSocket). */
32 export interface FrameSink {
33 /** One whole part of the live source, as the source sent it. */
34 frame(bytes: Uint8Array): void;
35 /** The source is down: show the offline picture. Called again every few seconds while it stays down. */
36 offline(): void;
37 }
38
3139 export type FeedState = "idle" | "connecting" | "live" | "offline";
3240
3341 const BOUNDARY = "armframe";
@@ -56,6 +64,9 @@ export function parseFps(text: string | null): number | null {
5664
5765 export class CameraFeed {
5866 private readonly viewers = new Set<Viewer>();
67 private readonly sinks = new Set<FrameSink>();
68 /** The live part being gathered for the sinks, or null when none is wanted. */
69 private part: Uint8Array[] | null = null;
5970 private _state: FeedState;
6071 private attempt = 0;
6172 private controller: AbortController | null = null;
@@ -83,7 +94,21 @@ export class CameraFeed {
8394
8495 /** How many viewers are watching right now. */
8596 get watching(): number {
86 return this.viewers.size;
97 return this.viewers.size + this.sinks.size;
98 }
99
100 /** Take whole frames from the same upstream connection as every other viewer. Returns the function that stops. */
101 attach(sink: FrameSink): () => void {
102 if (this.closed) return () => {};
103 this.sinks.add(sink);
104 clearTimeout(this.linger);
105 if (this._state === "offline") {
106 sink.offline();
107 this.offline();
108 } else if (this._state === "idle") this.connect();
109 return () => {
110 if (this.sinks.delete(sink)) this.idleCheck();
111 };
87112 }
88113
89114 /**
@@ -115,14 +140,19 @@ export class CameraFeed {
115140 this.stop();
116141 for (const v of this.viewers) v.res.end();
117142 this.viewers.clear();
143 this.sinks.clear();
118144 }
119145
120146 // ------------------------------------------------------------------ viewers
121147 private leave(viewer: Viewer) {
122 if (!this.viewers.delete(viewer) || this.viewers.size) return;
148 if (this.viewers.delete(viewer)) this.idleCheck();
149 }
150
151 private idleCheck() {
152 if (this.watching) return;
123153 clearTimeout(this.linger);
124154 this.linger = setTimeout(() => {
125 if (!this.viewers.size) this.stop();
155 if (!this.watching) this.stop();
126156 }, this.o.lingerMs);
127157 this.linger.unref?.();
128158 }
@@ -139,6 +169,8 @@ export class CameraFeed {
139169
140170 /** Show the offline picture; `cut` closes the parts a failed source left half-sent, otherwise those viewers wait. */
141171 private broadcastOffline(cut: boolean) {
172 this.part = null;
173 for (const s of this.sinks) s.offline();
142174 for (const v of this.viewers) {
143175 if (v.inPart && !cut) continue;
144176 if (v.inPart) this.write(v, NEXT);
@@ -167,7 +199,7 @@ export class CameraFeed {
167199 if (this.source && !this.retry && !this.closed) {
168200 this.retry = setTimeout(() => {
169201 this.retry = undefined;
170 if (this.viewers.size && this._state === "offline" && !this.controller) this.connect();
202 if (this.watching && this._state === "offline" && !this.controller) this.connect();
171203 }, this.o.retryMs);
172204 this.retry.unref?.();
173205 }
@@ -210,6 +242,7 @@ export class CameraFeed {
210242 }
211243 const parser = new MjpegParser(boundary, {
212244 start: (type) => {
245 this.part = this.sinks.size ? [] : null;
213246 const now = performance.now();
214247 for (const v of this.viewers) {
215248 // A frame a little early still counts as on time: a 10 fps source held to 5 sends every other frame, not every third.
@@ -220,6 +253,7 @@ export class CameraFeed {
220253 }
221254 },
222255 data: (bytes) => {
256 this.part?.push(bytes.slice());
223257 for (const v of this.viewers) if (v.inPart) this.write(v, bytes);
224258 },
225259 end: () => {
@@ -230,6 +264,11 @@ export class CameraFeed {
230264 this.repeat = undefined;
231265 }
232266 for (const v of this.viewers) if (v.inPart) this.write(v, NEXT), (v.inPart = false);
267 if (this.part) {
268 const frame = Buffer.concat(this.part);
269 this.part = null;
270 for (const s of this.sinks) s.frame(frame);
271 }
233272 },
234273 });
235274 this.lastByte = Date.now();
Software/so101/src/camera-socket.ts +6
@@ -48,6 +48,12 @@ export class CameraSocket {
4848 return true;
4949 }
5050
51 /** Show a picture that is not a camera frame (the offline one) to everyone now, and to whoever joins until the next frame. */
52 replace(picture: Uint8Array): void {
53 this.latest = picture;
54 for (const ws of this.viewers.keys()) this.offer(ws, picture);
55 }
56
5157 /** The viewer's socket has room again: send the frame it was holding, if any. */
5258 drain(ws: ViewerSocket): void {
5359 const pending = this.viewers.get(ws);
Software/so101/src/relay.test.ts +39
@@ -204,6 +204,45 @@ describe("relay", () => {
204204 expect(await (await fetch(`${base}/camera/1/state`)).json()).toEqual({ ok: true });
205205 });
206206
207 test("WS /camera/<id>/ws: two viewers get whole JPEGs from one upstream connection; an unknown id is refused", async () => {
208 let connections = 0;
209 camera = createServer((req, res) => {
210 connections++;
211 res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=frame" });
212 const timer = setInterval(() => res.write(Buffer.concat([Buffer.from(`--frame\r\ncontent-type: image/jpeg\r\ncontent-length: ${JPEG.length}\r\n\r\n`), JPEG, Buffer.from("\r\n")])), 20);
213 res.on("close", () => clearInterval(timer));
214 });
215 const cameras = new CameraHub({ seed: [{ name: "Desk", source: `http://127.0.0.1:${await listening(camera)}/stream` }] });
216 relay = await startRelay({ port: 0, simulation: simBase, cameras });
217 const first = async (url: string) => {
218 const ws = new WebSocket(url);
219 const frame = await new Promise<Buffer>((r, j) => (ws.once("message", (d) => r(d as Buffer)), ws.once("error", j)));
220 return { ws, frame };
221 };
222 const [a, b] = await Promise.all([first(`ws://127.0.0.1:${relay.port}/camera/1/ws`), first(`ws://127.0.0.1:${relay.port}/camera/1/ws`)]);
223 expect(Buffer.compare(a.frame, JPEG)).toBe(0);
224 expect(Buffer.compare(b.frame, JPEG)).toBe(0);
225 expect(connections).toBe(1);
226 a.ws.close();
227 b.ws.close();
228
229 const unknown = new WebSocket(`ws://127.0.0.1:${relay.port}/camera/9/ws`);
230 const opened = await new Promise<boolean>((r) => (unknown.on("open", () => r(true)), unknown.on("error", () => r(false))));
231 expect(opened).toBe(false);
232 cameras.close();
233 });
234
235 test("WS /camera/<id>/ws: a stalled upstream gives the offline frame", async () => {
236 hang = createTcpServer(() => {}); // accepts and never answers
237 const cameras = new CameraHub({ seed: [{ name: "Desk", source: `http://127.0.0.1:${await listening(hang)}/stream` }], feed: { connectTimeoutMs: 50 } });
238 relay = await startRelay({ port: 0, simulation: simBase, cameras });
239 const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/camera/1/ws`);
240 const frame = await new Promise<Buffer>((r, j) => (ws.once("message", (d) => r(d as Buffer)), ws.once("error", j)));
241 expect(Buffer.compare(frame, OFFLINE_FRAME)).toBe(0);
242 ws.close();
243 cameras.close();
244 });
245
207246 test("set, with the camera down: the offline picture within 3 s while the arm stays live", async () => {
208247 remote = await arm([1, 2, 6]);
209248 hang = createTcpServer(() => {}); // a camera port that accepts and never answers
Software/so101/src/relay.ts +33
@@ -43,6 +43,7 @@
4343 * GET /cameras the listed cameras: [{"id", "name", "online"}]
4444 * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts);
4545 * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate
46 * WS /camera/<id>/ws the same feed as whole binary frames (JPEG; the offline PNG when its source is down), latest frame wins
4647 * GET /admin/camera-list the owner's page for the camera list
4748 * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds
4849 * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time,
@@ -61,6 +62,8 @@ import { perServo } from "./machine-log.ts";
6162 import { parseFrame } from "./safety.ts";
6263 import { acceptsGzip, GzipCache, worthCompressing } from "./compress.ts";
6364 import { AccountError, sessionCookie, visitor } from "./accounts.ts";
65 import { CameraSocket } from "./camera-socket.ts";
66 import { OFFLINE_FRAME } from "./offline-frame.ts";
6467 import { handleAdmin } from "./admin-routes.ts";
6568 import { parseFps, type FeedOptions } from "./camera-feed.ts";
6669 import { CameraError, CameraHub } from "./cameras.ts";
@@ -131,6 +134,7 @@ export interface RunningRelay {
131134
132135 const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" };
133136 const MAX_BODY = 1_000_000;
137 const CAMERA_WS = /^\/camera\/(\d{1,9})\/ws$/;
134138 const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/;
135139 const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/;
136140 const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8");
@@ -737,8 +741,37 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
737741 const refuse = (socket: Duplex, status: string) => {
738742 socket.end(`HTTP/1.1 ${status}\r\nconnection: close\r\ncontent-length: 0\r\n\r\n`);
739743 };
744 /** Per camera, while anyone watches over WebSocket: the fan-out and the one tap on the shared feed. */
745 const watched = new Map<number, { fan: CameraSocket; viewers: number; detach: () => void }>();
746 const watchCamera = (id: number, browser: WebSocket) => {
747 let w = watched.get(id);
748 if (!w) {
749 const fan = new CameraSocket();
750 const detach = cameras.feed(id)!.attach({ frame: (bytes) => void fan.push(bytes), offline: () => fan.replace(OFFLINE_FRAME) });
751 watched.set(id, (w = { fan, viewers: 0, detach }));
752 }
753 const mine = w;
754 const viewer = { send: (data: Uint8Array) => browser.send(data, () => mine.fan.drain(viewer)), getBufferedAmount: () => browser.bufferedAmount };
755 mine.viewers++;
756 mine.fan.add(viewer);
757 browser.on("error", () => browser.terminate());
758 browser.once("close", () => {
759 mine.fan.remove(viewer);
760 if (--mine.viewers > 0) return;
761 mine.detach();
762 if (watched.get(id) === mine) watched.delete(id);
763 });
764 };
740765 http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => {
741766 const url = new URL(req.url ?? "/", "http://x");
767 const cameraWs = CAMERA_WS.exec(url.pathname);
768 if (cameraWs) {
769 if (!sameOrigin(req)) return refuse(socket, "403 Forbidden");
770 const id = Number(cameraWs[1]);
771 if (!cameras.has(id)) return refuse(socket, "404 Not Found");
772 socket.on("error", () => socket.destroy());
773 return wss.handleUpgrade(req, socket, head, (browser) => watchCamera(id, browser));
774 }
742775 if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found");
743776 if (!sameOrigin(req)) return refuse(socket, "403 Forbidden");
744777 socket.on("error", () => socket.destroy());

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs