Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
The relay serves WS /camera/<id>/ws from the shared camera feed, and the offline picture when the Pi camera is down #172
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/139-the-relay-serves-ws-camera-id-ws-from-th → main 2 commits fork
4 files
changed +121
−4
Software/so101/src/camera-feed.ts +43 −4
| @@ -28,6 +28,14 @@ export interface FeedOptions { | ||
| 28 | 28 | maxBacklog?: number; |
| 29 | 29 | } |
| 30 | 30 | |
| 31 | /** Something that takes the feed's pictures whole rather than as a multipart body (the camera WebSocket). */ | |
| 32 | export interface FrameSink { | |
| 33 | /** One whole part of the live source, as the source sent it. */ | |
| 34 | frame(bytes: Uint8Array): void; | |
| 35 | /** The source is down: show the offline picture. Called again every few seconds while it stays down. */ | |
| 36 | offline(): void; | |
| 37 | } | |
| 38 | ||
| 31 | 39 | export type FeedState = "idle" | "connecting" | "live" | "offline"; |
| 32 | 40 | |
| 33 | 41 | const BOUNDARY = "armframe"; |
| @@ -56,6 +64,9 @@ export function parseFps(text: string | null): number | null { | ||
| 56 | 64 | |
| 57 | 65 | export class CameraFeed { |
| 58 | 66 | private readonly viewers = new Set<Viewer>(); |
| 67 | private readonly sinks = new Set<FrameSink>(); | |
| 68 | /** The live part being gathered for the sinks, or null when none is wanted. */ | |
| 69 | private part: Uint8Array[] | null = null; | |
| 59 | 70 | private _state: FeedState; |
| 60 | 71 | private attempt = 0; |
| 61 | 72 | private controller: AbortController | null = null; |
| @@ -83,7 +94,21 @@ export class CameraFeed { | ||
| 83 | 94 | |
| 84 | 95 | /** How many viewers are watching right now. */ |
| 85 | 96 | get watching(): number { |
| 86 | return this.viewers.size; | |
| 97 | return this.viewers.size + this.sinks.size; | |
| 98 | } | |
| 99 | ||
| 100 | /** Take whole frames from the same upstream connection as every other viewer. Returns the function that stops. */ | |
| 101 | attach(sink: FrameSink): () => void { | |
| 102 | if (this.closed) return () => {}; | |
| 103 | this.sinks.add(sink); | |
| 104 | clearTimeout(this.linger); | |
| 105 | if (this._state === "offline") { | |
| 106 | sink.offline(); | |
| 107 | this.offline(); | |
| 108 | } else if (this._state === "idle") this.connect(); | |
| 109 | return () => { | |
| 110 | if (this.sinks.delete(sink)) this.idleCheck(); | |
| 111 | }; | |
| 87 | 112 | } |
| 88 | 113 | |
| 89 | 114 | /** |
| @@ -115,14 +140,19 @@ export class CameraFeed { | ||
| 115 | 140 | this.stop(); |
| 116 | 141 | for (const v of this.viewers) v.res.end(); |
| 117 | 142 | this.viewers.clear(); |
| 143 | this.sinks.clear(); | |
| 118 | 144 | } |
| 119 | 145 | |
| 120 | 146 | // ------------------------------------------------------------------ viewers |
| 121 | 147 | private leave(viewer: Viewer) { |
| 122 | if (!this.viewers.delete(viewer) || this.viewers.size) return; | |
| 148 | if (this.viewers.delete(viewer)) this.idleCheck(); | |
| 149 | } | |
| 150 | ||
| 151 | private idleCheck() { | |
| 152 | if (this.watching) return; | |
| 123 | 153 | clearTimeout(this.linger); |
| 124 | 154 | this.linger = setTimeout(() => { |
| 125 | if (!this.viewers.size) this.stop(); | |
| 155 | if (!this.watching) this.stop(); | |
| 126 | 156 | }, this.o.lingerMs); |
| 127 | 157 | this.linger.unref?.(); |
| 128 | 158 | } |
| @@ -139,6 +169,8 @@ export class CameraFeed { | ||
| 139 | 169 | |
| 140 | 170 | /** Show the offline picture; `cut` closes the parts a failed source left half-sent, otherwise those viewers wait. */ |
| 141 | 171 | private broadcastOffline(cut: boolean) { |
| 172 | this.part = null; | |
| 173 | for (const s of this.sinks) s.offline(); | |
| 142 | 174 | for (const v of this.viewers) { |
| 143 | 175 | if (v.inPart && !cut) continue; |
| 144 | 176 | if (v.inPart) this.write(v, NEXT); |
| @@ -167,7 +199,7 @@ export class CameraFeed { | ||
| 167 | 199 | if (this.source && !this.retry && !this.closed) { |
| 168 | 200 | this.retry = setTimeout(() => { |
| 169 | 201 | this.retry = undefined; |
| 170 | if (this.viewers.size && this._state === "offline" && !this.controller) this.connect(); | |
| 202 | if (this.watching && this._state === "offline" && !this.controller) this.connect(); | |
| 171 | 203 | }, this.o.retryMs); |
| 172 | 204 | this.retry.unref?.(); |
| 173 | 205 | } |
| @@ -210,6 +242,7 @@ export class CameraFeed { | ||
| 210 | 242 | } |
| 211 | 243 | const parser = new MjpegParser(boundary, { |
| 212 | 244 | start: (type) => { |
| 245 | this.part = this.sinks.size ? [] : null; | |
| 213 | 246 | const now = performance.now(); |
| 214 | 247 | for (const v of this.viewers) { |
| 215 | 248 | // A frame a little early still counts as on time: a 10 fps source held to 5 sends every other frame, not every third. |
| @@ -220,6 +253,7 @@ export class CameraFeed { | ||
| 220 | 253 | } |
| 221 | 254 | }, |
| 222 | 255 | data: (bytes) => { |
| 256 | this.part?.push(bytes.slice()); | |
| 223 | 257 | for (const v of this.viewers) if (v.inPart) this.write(v, bytes); |
| 224 | 258 | }, |
| 225 | 259 | end: () => { |
| @@ -230,6 +264,11 @@ export class CameraFeed { | ||
| 230 | 264 | this.repeat = undefined; |
| 231 | 265 | } |
| 232 | 266 | for (const v of this.viewers) if (v.inPart) this.write(v, NEXT), (v.inPart = false); |
| 267 | if (this.part) { | |
| 268 | const frame = Buffer.concat(this.part); | |
| 269 | this.part = null; | |
| 270 | for (const s of this.sinks) s.frame(frame); | |
| 271 | } | |
| 233 | 272 | }, |
| 234 | 273 | }); |
| 235 | 274 | this.lastByte = Date.now(); |
Software/so101/src/camera-socket.ts +6
| @@ -48,6 +48,12 @@ export class CameraSocket { | ||
| 48 | 48 | return true; |
| 49 | 49 | } |
| 50 | 50 | |
| 51 | /** Show a picture that is not a camera frame (the offline one) to everyone now, and to whoever joins until the next frame. */ | |
| 52 | replace(picture: Uint8Array): void { | |
| 53 | this.latest = picture; | |
| 54 | for (const ws of this.viewers.keys()) this.offer(ws, picture); | |
| 55 | } | |
| 56 | ||
| 51 | 57 | /** The viewer's socket has room again: send the frame it was holding, if any. */ |
| 52 | 58 | drain(ws: ViewerSocket): void { |
| 53 | 59 | const pending = this.viewers.get(ws); |
Software/so101/src/relay.test.ts +39
| @@ -204,6 +204,45 @@ describe("relay", () => { | ||
| 204 | 204 | expect(await (await fetch(`${base}/camera/1/state`)).json()).toEqual({ ok: true }); |
| 205 | 205 | }); |
| 206 | 206 | |
| 207 | test("WS /camera/<id>/ws: two viewers get whole JPEGs from one upstream connection; an unknown id is refused", async () => { | |
| 208 | let connections = 0; | |
| 209 | camera = createServer((req, res) => { | |
| 210 | connections++; | |
| 211 | res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=frame" }); | |
| 212 | const timer = setInterval(() => res.write(Buffer.concat([Buffer.from(`--frame\r\ncontent-type: image/jpeg\r\ncontent-length: ${JPEG.length}\r\n\r\n`), JPEG, Buffer.from("\r\n")])), 20); | |
| 213 | res.on("close", () => clearInterval(timer)); | |
| 214 | }); | |
| 215 | const cameras = new CameraHub({ seed: [{ name: "Desk", source: `http://127.0.0.1:${await listening(camera)}/stream` }] }); | |
| 216 | relay = await startRelay({ port: 0, simulation: simBase, cameras }); | |
| 217 | const first = async (url: string) => { | |
| 218 | const ws = new WebSocket(url); | |
| 219 | const frame = await new Promise<Buffer>((r, j) => (ws.once("message", (d) => r(d as Buffer)), ws.once("error", j))); | |
| 220 | return { ws, frame }; | |
| 221 | }; | |
| 222 | const [a, b] = await Promise.all([first(`ws://127.0.0.1:${relay.port}/camera/1/ws`), first(`ws://127.0.0.1:${relay.port}/camera/1/ws`)]); | |
| 223 | expect(Buffer.compare(a.frame, JPEG)).toBe(0); | |
| 224 | expect(Buffer.compare(b.frame, JPEG)).toBe(0); | |
| 225 | expect(connections).toBe(1); | |
| 226 | a.ws.close(); | |
| 227 | b.ws.close(); | |
| 228 | ||
| 229 | const unknown = new WebSocket(`ws://127.0.0.1:${relay.port}/camera/9/ws`); | |
| 230 | const opened = await new Promise<boolean>((r) => (unknown.on("open", () => r(true)), unknown.on("error", () => r(false)))); | |
| 231 | expect(opened).toBe(false); | |
| 232 | cameras.close(); | |
| 233 | }); | |
| 234 | ||
| 235 | test("WS /camera/<id>/ws: a stalled upstream gives the offline frame", async () => { | |
| 236 | hang = createTcpServer(() => {}); // accepts and never answers | |
| 237 | const cameras = new CameraHub({ seed: [{ name: "Desk", source: `http://127.0.0.1:${await listening(hang)}/stream` }], feed: { connectTimeoutMs: 50 } }); | |
| 238 | relay = await startRelay({ port: 0, simulation: simBase, cameras }); | |
| 239 | const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/camera/1/ws`); | |
| 240 | const frame = await new Promise<Buffer>((r, j) => (ws.once("message", (d) => r(d as Buffer)), ws.once("error", j))); | |
| 241 | expect(Buffer.compare(frame, OFFLINE_FRAME)).toBe(0); | |
| 242 | ws.close(); | |
| 243 | cameras.close(); | |
| 244 | }); | |
| 245 | ||
| 207 | 246 | test("set, with the camera down: the offline picture within 3 s while the arm stays live", async () => { |
| 208 | 247 | remote = await arm([1, 2, 6]); |
| 209 | 248 | hang = createTcpServer(() => {}); // a camera port that accepts and never answers |
Software/so101/src/relay.ts +33
| @@ -43,6 +43,7 @@ | ||
| 43 | 43 | * GET /cameras the listed cameras: [{"id", "name", "online"}] |
| 44 | 44 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 45 | 45 | * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate |
| 46 | * WS /camera/<id>/ws the same feed as whole binary frames (JPEG; the offline PNG when its source is down), latest frame wins | |
| 46 | 47 | * GET /admin/camera-list the owner's page for the camera list |
| 47 | 48 | * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds |
| 48 | 49 | * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time, |
| @@ -61,6 +62,8 @@ import { perServo } from "./machine-log.ts"; | ||
| 61 | 62 | import { parseFrame } from "./safety.ts"; |
| 62 | 63 | import { acceptsGzip, GzipCache, worthCompressing } from "./compress.ts"; |
| 63 | 64 | import { AccountError, sessionCookie, visitor } from "./accounts.ts"; |
| 65 | import { CameraSocket } from "./camera-socket.ts"; | |
| 66 | import { OFFLINE_FRAME } from "./offline-frame.ts"; | |
| 64 | 67 | import { handleAdmin } from "./admin-routes.ts"; |
| 65 | 68 | import { parseFps, type FeedOptions } from "./camera-feed.ts"; |
| 66 | 69 | import { CameraError, CameraHub } from "./cameras.ts"; |
| @@ -131,6 +134,7 @@ export interface RunningRelay { | ||
| 131 | 134 | |
| 132 | 135 | const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" }; |
| 133 | 136 | const MAX_BODY = 1_000_000; |
| 137 | const CAMERA_WS = /^\/camera\/(\d{1,9})\/ws$/; | |
| 134 | 138 | const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/; |
| 135 | 139 | const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/; |
| 136 | 140 | const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8"); |
| @@ -737,8 +741,37 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 737 | 741 | const refuse = (socket: Duplex, status: string) => { |
| 738 | 742 | socket.end(`HTTP/1.1 ${status}\r\nconnection: close\r\ncontent-length: 0\r\n\r\n`); |
| 739 | 743 | }; |
| 744 | /** Per camera, while anyone watches over WebSocket: the fan-out and the one tap on the shared feed. */ | |
| 745 | const watched = new Map<number, { fan: CameraSocket; viewers: number; detach: () => void }>(); | |
| 746 | const watchCamera = (id: number, browser: WebSocket) => { | |
| 747 | let w = watched.get(id); | |
| 748 | if (!w) { | |
| 749 | const fan = new CameraSocket(); | |
| 750 | const detach = cameras.feed(id)!.attach({ frame: (bytes) => void fan.push(bytes), offline: () => fan.replace(OFFLINE_FRAME) }); | |
| 751 | watched.set(id, (w = { fan, viewers: 0, detach })); | |
| 752 | } | |
| 753 | const mine = w; | |
| 754 | const viewer = { send: (data: Uint8Array) => browser.send(data, () => mine.fan.drain(viewer)), getBufferedAmount: () => browser.bufferedAmount }; | |
| 755 | mine.viewers++; | |
| 756 | mine.fan.add(viewer); | |
| 757 | browser.on("error", () => browser.terminate()); | |
| 758 | browser.once("close", () => { | |
| 759 | mine.fan.remove(viewer); | |
| 760 | if (--mine.viewers > 0) return; | |
| 761 | mine.detach(); | |
| 762 | if (watched.get(id) === mine) watched.delete(id); | |
| 763 | }); | |
| 764 | }; | |
| 740 | 765 | http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => { |
| 741 | 766 | const url = new URL(req.url ?? "/", "http://x"); |
| 767 | const cameraWs = CAMERA_WS.exec(url.pathname); | |
| 768 | if (cameraWs) { | |
| 769 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); | |
| 770 | const id = Number(cameraWs[1]); | |
| 771 | if (!cameras.has(id)) return refuse(socket, "404 Not Found"); | |
| 772 | socket.on("error", () => socket.destroy()); | |
| 773 | return wss.handleUpgrade(req, socket, head, (browser) => watchCamera(id, browser)); | |
| 774 | } | |
| 742 | 775 | if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); |
| 743 | 776 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); |
| 744 | 777 | socket.on("error", () => socket.destroy()); |
new above
offline