Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
The live camera streams through the app: MJPEG from the arm server's host, simulated frame when unreachable #21
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
quill/arm:quill/7-the-live-camera-streams-through-the-app → main 2 commits fork 1 thread · 1 unresolved
9 files
changed +801
−15
Software/README.md +17 −2
| @@ -174,8 +174,23 @@ is. The forge has to be able to reach `<host>`: put the arm machine on the | ||
| 174 | 174 | same tailnet as the forge (Tailscale, `ws://<machine>.<tailnet>.ts.net:8101`) |
| 175 | 175 | or on an address the forge can route to. Don't put it on the open internet: |
| 176 | 176 | `so101 serve` has no login. The cameras default to ports 8080 and 8081 on the |
| 177 | same host. `ARM_CAMERA_1` and `ARM_CAMERA_2` take full stream URLs | |
| 178 | (`http://<host>:8080/stream`) when they live elsewhere. | |
| 177 | same host. `ARM_CAMERA` (or `ARM_CAMERA_1`) and `ARM_CAMERA_2` name them when | |
| 178 | they live elsewhere, as a stream URL (`http://<host>:8080/stream`) or a bare | |
| 179 | `<host>:<port>`, whose `/stream` it is. | |
| 180 | ||
| 181 | **The camera through the app.** `/camera/1/stream` (and `/camera/2/stream`) | |
| 182 | is one MJPEG connection from the app to the camera, shared by every viewer: | |
| 183 | ten people watching cost the camera one stream, and none of them needs to | |
| 184 | reach the arm machine. Frames are passed on as their bytes arrive, never | |
| 185 | gathered whole; a viewer who joins mid-frame starts on the next one, and a | |
| 186 | viewer on a slow connection skips frames instead of slowing the rest. The | |
| 187 | connection closes 2 s after the last viewer leaves. With `ARM_SERVER` unset, | |
| 188 | or when the camera refuses, doesn't answer within 1.5 s, or sends nothing for | |
| 189 | 2 s, every viewer sees a **CAMERA OFFLINE** picture straight away (a PNG the | |
| 190 | app draws when it starts), sent again every 2 s; the app tries the camera | |
| 191 | again every 2 s and the live picture comes back in the same stream, without | |
| 192 | a reload. The camera is independent of the arm server: a camera that is down | |
| 193 | does not put the arm on the simulation, and the other way round. | |
| 179 | 194 | |
| 180 | 195 | With `ARM_SERVER` unset, the app runs the simulated arm and says |
| 181 | 196 | **Simulated**. When it is set but the server doesn't answer `/status` within |
Software/so101/src/camera-feed.test.ts added +203
| @@ -0,0 +1,203 @@ | ||
| 1 | import { afterEach, describe, expect, test } from "bun:test"; | |
| 2 | import { createServer, type Server, type ServerResponse } from "node:http"; | |
| 3 | import { createServer as createTcpServer, type Server as TcpServer } from "node:net"; | |
| 4 | import { CameraFeed, type FeedOptions } from "./camera-feed.ts"; | |
| 5 | import { OFFLINE_FRAME } from "./offline-frame.ts"; | |
| 6 | ||
| 7 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 8 | const listening = (server: Server | TcpServer) => new Promise<number>((r) => server.listen(0, "127.0.0.1", () => r((server.address() as { port: number }).port))); | |
| 9 | const PNG = OFFLINE_FRAME.subarray(0, 8); | |
| 10 | const frame = (n: number) => Buffer.from(`\xff\xd8frame-${n}\xff\xd9`, "latin1"); | |
| 11 | ||
| 12 | /** A µStreamer stand-in: frames every 30 ms to each request, counting the connections it gets. */ | |
| 13 | function fakeCamera() { | |
| 14 | const clients = new Set<ServerResponse>(); | |
| 15 | let connections = 0; | |
| 16 | let n = 0; | |
| 17 | const server = createServer((_req, res) => { | |
| 18 | connections++; | |
| 19 | res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=boundarydonotcross" }); | |
| 20 | clients.add(res); | |
| 21 | res.socket?.once("close", () => clients.delete(res)); // Bun's node:http tells the socket, not the response | |
| 22 | }); | |
| 23 | const timer = setInterval(() => { | |
| 24 | const body = frame(++n); | |
| 25 | for (const res of clients) res.write(Buffer.concat([Buffer.from(`--boundarydonotcross\r\nContent-Type: image/jpeg\r\nContent-Length: ${body.length}\r\n\r\n`), body, Buffer.from("\r\n")])); | |
| 26 | }, 30); | |
| 27 | return { | |
| 28 | server, | |
| 29 | clients, | |
| 30 | get connections() { | |
| 31 | return connections; | |
| 32 | }, | |
| 33 | /** Stop sending but keep the connections open: a camera that froze. */ | |
| 34 | freeze: () => clearInterval(timer), | |
| 35 | close: () => (clearInterval(timer), server.closeAllConnections(), server.close()), | |
| 36 | }; | |
| 37 | } | |
| 38 | ||
| 39 | /** One viewer's stream from the feed; `until` waits for bytes that satisfy the test. */ | |
| 40 | async function watch(url: string) { | |
| 41 | const hangUp = new AbortController(); // Bun's fetch keeps the connection on cancel(); abort is a browser leaving | |
| 42 | const response = await fetch(url, { signal: hangUp.signal }); | |
| 43 | const reader = response.body!.getReader(); | |
| 44 | let seen = Buffer.alloc(0); | |
| 45 | return { | |
| 46 | response, | |
| 47 | get seen() { | |
| 48 | return seen; | |
| 49 | }, | |
| 50 | async until(test: (seen: Buffer) => boolean, timeoutMs = 3000) { | |
| 51 | const deadline = Date.now() + timeoutMs; | |
| 52 | while (!test(seen)) { | |
| 53 | const left = deadline - Date.now(); | |
| 54 | if (left <= 0) throw new Error(`timed out; saw ${seen.length} bytes`); | |
| 55 | const next = await Promise.race([reader.read(), sleep(left).then(() => null)]); | |
| 56 | if (!next) continue; | |
| 57 | if (next.done) throw new Error("the stream ended"); | |
| 58 | seen = Buffer.concat([seen, next.value]); | |
| 59 | } | |
| 60 | return seen; | |
| 61 | }, | |
| 62 | cancel: () => hangUp.abort(), | |
| 63 | }; | |
| 64 | } | |
| 65 | const hasFrame = (seen: Buffer) => seen.includes("frame-"); | |
| 66 | const hasOffline = (seen: Buffer) => seen.includes(PNG); | |
| 67 | const after = (seen: Buffer, mark: Buffer) => seen.subarray(seen.lastIndexOf(mark)); | |
| 68 | ||
| 69 | let camera: ReturnType<typeof fakeCamera> | undefined; | |
| 70 | let feed: CameraFeed | undefined; | |
| 71 | let front: Server | undefined; | |
| 72 | let hang: TcpServer | undefined; | |
| 73 | afterEach(() => { | |
| 74 | feed?.close(); | |
| 75 | front?.closeAllConnections(); | |
| 76 | front?.close(); | |
| 77 | camera?.close(); | |
| 78 | hang?.close(); | |
| 79 | camera = feed = front = hang = undefined; | |
| 80 | }); | |
| 81 | ||
| 82 | /** Serve a feed of `source` on a port of its own, as the relay does. */ | |
| 83 | async function serveFeed(source: string | null, options: FeedOptions = {}) { | |
| 84 | feed = new CameraFeed(source, "camera 1", options); | |
| 85 | front = createServer((req, res) => feed!.serve(req, res)); | |
| 86 | return `http://127.0.0.1:${await listening(front)}/`; | |
| 87 | } | |
| 88 | ||
| 89 | describe("a camera feed", () => { | |
| 90 | test("frames reach two viewers from one upstream connection", async () => { | |
| 91 | camera = fakeCamera(); | |
| 92 | const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`); | |
| 93 | const a = await watch(url); | |
| 94 | const b = await watch(url); | |
| 95 | expect(a.response.headers.get("content-type")).toBe("multipart/x-mixed-replace;boundary=armframe"); | |
| 96 | expect(a.response.headers.get("cache-control")).toContain("no-store"); | |
| 97 | await a.until((s) => (s.toString("latin1").match(/frame-/g) ?? []).length >= 3); | |
| 98 | await b.until((s) => (s.toString("latin1").match(/frame-/g) ?? []).length >= 3); | |
| 99 | expect(camera.connections).toBe(1); | |
| 100 | expect(feed!.watching).toBe(2); | |
| 101 | expect(feed!.state).toBe("live"); | |
| 102 | // Re-framed whole: every live part has its type, its bytes, and the next boundary. | |
| 103 | expect(a.seen.toString("latin1")).toMatch(/^--armframe\r\nContent-Type: image\/jpeg\r\n\r\n\xff\xd8frame-\d+\xff\xd9\r\n--armframe\r\n/); | |
| 104 | await a.cancel(); | |
| 105 | await b.cancel(); | |
| 106 | }); | |
| 107 | ||
| 108 | test("a viewer who joins later shares the same connection and starts on a whole frame", async () => { | |
| 109 | camera = fakeCamera(); | |
| 110 | const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`); | |
| 111 | const a = await watch(url); | |
| 112 | await a.until(hasFrame); | |
| 113 | await sleep(45); | |
| 114 | const b = await watch(url); | |
| 115 | await b.until(hasFrame); | |
| 116 | expect(b.seen.toString("latin1")).toMatch(/^--armframe\r\nContent-Type: image\/jpeg\r\n\r\n\xff\xd8frame-/); | |
| 117 | expect(camera.connections).toBe(1); | |
| 118 | await a.cancel(); | |
| 119 | await b.cancel(); | |
| 120 | }); | |
| 121 | ||
| 122 | test("source down: the offline frame within 3 s", async () => { | |
| 123 | const closed = createServer(); | |
| 124 | const port = await listening(closed); | |
| 125 | closed.close(); // nothing listens there now | |
| 126 | const url = await serveFeed(`http://127.0.0.1:${port}/stream`); | |
| 127 | const started = Date.now(); | |
| 128 | const v = await watch(url); | |
| 129 | await v.until(hasOffline); | |
| 130 | expect(Date.now() - started).toBeLessThan(3000); | |
| 131 | expect(feed!.state).toBe("offline"); | |
| 132 | expect(v.seen.toString("latin1")).toContain("Content-Type: image/png\r\n\r\n"); | |
| 133 | await v.cancel(); | |
| 134 | }); | |
| 135 | ||
| 136 | test("a source that accepts and never answers: the offline frame within 3 s", async () => { | |
| 137 | hang = createTcpServer(() => {}); | |
| 138 | const url = await serveFeed(`http://127.0.0.1:${await listening(hang)}/stream`); | |
| 139 | const started = Date.now(); | |
| 140 | const v = await watch(url); | |
| 141 | await v.until(hasOffline); | |
| 142 | expect(Date.now() - started).toBeLessThan(3000); | |
| 143 | await v.cancel(); | |
| 144 | }); | |
| 145 | ||
| 146 | test("a source that freezes mid-stream: viewers already watching get the offline frame within 3 s", async () => { | |
| 147 | camera = fakeCamera(); | |
| 148 | const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`); | |
| 149 | const v = await watch(url); | |
| 150 | await v.until(hasFrame); | |
| 151 | camera.freeze(); | |
| 152 | const started = Date.now(); | |
| 153 | await v.until(hasOffline); | |
| 154 | expect(Date.now() - started).toBeLessThan(3000); | |
| 155 | await v.cancel(); | |
| 156 | }); | |
| 157 | ||
| 158 | test("a source that comes back: live frames resume in the same response", async () => { | |
| 159 | const closed = createServer(); | |
| 160 | const port = await listening(closed); | |
| 161 | closed.close(); | |
| 162 | const url = await serveFeed(`http://127.0.0.1:${port}/stream`, { retryMs: 100 }); | |
| 163 | const v = await watch(url); | |
| 164 | await v.until(hasOffline); | |
| 165 | camera = fakeCamera(); | |
| 166 | await new Promise<void>((r) => camera!.server.listen(port, "127.0.0.1", () => r())); | |
| 167 | await v.until((s) => hasFrame(after(s, PNG))); | |
| 168 | expect(feed!.state).toBe("live"); | |
| 169 | await v.cancel(); | |
| 170 | }); | |
| 171 | ||
| 172 | test("no source: the offline frame at once, and again while anyone watches", async () => { | |
| 173 | const url = await serveFeed(null, { offlineEveryMs: 50 }); | |
| 174 | const v = await watch(url); | |
| 175 | await v.until(hasOffline, 500); | |
| 176 | await v.until((s) => s.indexOf(PNG) !== s.lastIndexOf(PNG), 500); | |
| 177 | await v.cancel(); | |
| 178 | }); | |
| 179 | ||
| 180 | test("the upstream connection closes once the last viewer has gone", async () => { | |
| 181 | camera = fakeCamera(); | |
| 182 | const url = await serveFeed(`http://127.0.0.1:${await listening(camera.server)}/stream`, { lingerMs: 50 }); | |
| 183 | const v = await watch(url); | |
| 184 | await v.until(hasFrame); | |
| 185 | expect(camera.clients.size).toBe(1); | |
| 186 | await v.cancel(); | |
| 187 | const deadline = Date.now() + 2000; | |
| 188 | while (camera.clients.size && Date.now() < deadline) await sleep(20); | |
| 189 | expect(camera.clients.size).toBe(0); | |
| 190 | expect(feed!.state).toBe("idle"); | |
| 191 | }); | |
| 192 | ||
| 193 | test("a source that is not MJPEG counts as offline", async () => { | |
| 194 | const page = createServer((_req, res) => res.writeHead(200, { "content-type": "text/html" }).end("<p>hello</p>")); | |
| 195 | const url = await serveFeed(`http://127.0.0.1:${await listening(page)}/`); | |
| 196 | const v = await watch(url); | |
| 197 | await v.until(hasOffline); | |
| 198 | expect(v.seen.toString()).not.toContain("hello"); | |
| 199 | await v.cancel(); | |
| 200 | page.closeAllConnections(); | |
| 201 | page.close(); | |
| 202 | }); | |
| 203 | }); |
Software/so101/src/camera-feed.ts added +230
| @@ -0,0 +1,230 @@ | ||
| 1 | /** | |
| 2 | * One camera, shared: a single upstream MJPEG connection fanned out to every viewer, part by part, as the bytes | |
| 3 | * arrive. No frame is gathered whole; a viewer who joins mid-frame starts at the next one, and a viewer who cannot | |
| 4 | * keep up skips frames rather than holding the others back. | |
| 5 | * | |
| 6 | * When there is no source, or it does not answer, or it stops sending, every viewer gets the "camera offline" | |
| 7 | * picture at once and again every few seconds, while the feed keeps trying the source; when it answers again the | |
| 8 | * live frames resume in the same response. A viewer never waits on a dead camera. | |
| 9 | */ | |
| 10 | ||
| 11 | import type { IncomingMessage, ServerResponse } from "node:http"; | |
| 12 | import { boundaryOf, MjpegParser } from "./mjpeg.ts"; | |
| 13 | import { OFFLINE_FRAME } from "./offline-frame.ts"; | |
| 14 | ||
| 15 | export interface FeedOptions { | |
| 16 | /** How long the source has to send its headers. */ | |
| 17 | connectTimeoutMs?: number; | |
| 18 | /** How long the source may go without a byte before it counts as gone. */ | |
| 19 | staleMs?: number; | |
| 20 | /** How soon an unreachable source is tried again, while anyone watches. */ | |
| 21 | retryMs?: number; | |
| 22 | /** How often the offline picture is sent again, so a pane is never left waiting. */ | |
| 23 | offlineEveryMs?: number; | |
| 24 | /** How long the upstream connection stays open after the last viewer leaves (a page reload keeps it). */ | |
| 25 | lingerMs?: number; | |
| 26 | /** Bytes queued for one viewer past which it skips frames. */ | |
| 27 | maxBacklog?: number; | |
| 28 | } | |
| 29 | ||
| 30 | export type FeedState = "idle" | "connecting" | "live" | "offline"; | |
| 31 | ||
| 32 | const BOUNDARY = "armframe"; | |
| 33 | const NEXT = Buffer.from(`\r\n--${BOUNDARY}\r\n`); | |
| 34 | const head = (type: string) => Buffer.from(`Content-Type: ${type}\r\n\r\n`); | |
| 35 | ||
| 36 | interface Viewer { | |
| 37 | res: ServerResponse; | |
| 38 | /** Mid-way through a live part; otherwise it stands right after a boundary. */ | |
| 39 | inPart: boolean; | |
| 40 | } | |
| 41 | ||
| 42 | export class CameraFeed { | |
| 43 | private readonly viewers = new Set<Viewer>(); | |
| 44 | private _state: FeedState; | |
| 45 | private attempt = 0; | |
| 46 | private controller: AbortController | null = null; | |
| 47 | private retry: ReturnType<typeof setTimeout> | undefined; | |
| 48 | private linger: ReturnType<typeof setTimeout> | undefined; | |
| 49 | private repeat: ReturnType<typeof setInterval> | undefined; | |
| 50 | private watchdog: ReturnType<typeof setInterval> | undefined; | |
| 51 | private lastByte = 0; | |
| 52 | private closed = false; | |
| 53 | private readonly o: Required<FeedOptions>; | |
| 54 | private readonly source: string | null; | |
| 55 | private readonly name: string; | |
| 56 | ||
| 57 | /** `source` is the camera's stream URL, or null when there is none (the arm is simulated). */ | |
| 58 | constructor(source: string | null, name = "camera", options: FeedOptions = {}) { | |
| 59 | this.source = source; | |
| 60 | this.name = name; | |
| 61 | this.o = { connectTimeoutMs: 1500, staleMs: 2000, retryMs: 2000, offlineEveryMs: 2000, lingerMs: 2000, maxBacklog: 1 << 20, ...options }; | |
| 62 | this._state = source ? "idle" : "offline"; | |
| 63 | } | |
| 64 | ||
| 65 | get state(): FeedState { | |
| 66 | return this._state; | |
| 67 | } | |
| 68 | ||
| 69 | /** How many viewers are watching right now. */ | |
| 70 | get watching(): number { | |
| 71 | return this.viewers.size; | |
| 72 | } | |
| 73 | ||
| 74 | /** Answer one viewer's GET (or HEAD) with the shared stream; it runs until the viewer leaves or the feed closes. */ | |
| 75 | serve(req: IncomingMessage, res: ServerResponse): void { | |
| 76 | res.writeHead(200, { | |
| 77 | "content-type": `multipart/x-mixed-replace;boundary=${BOUNDARY}`, | |
| 78 | "cache-control": "no-store, no-transform", | |
| 79 | "x-accel-buffering": "no", | |
| 80 | }); | |
| 81 | if (req.method === "HEAD" || this.closed) return void res.end(); | |
| 82 | res.write(`--${BOUNDARY}\r\n`); | |
| 83 | const viewer: Viewer = { res, inPart: false }; | |
| 84 | this.viewers.add(viewer); | |
| 85 | // Bun's node:http may only tell the socket that the viewer hung up, not the response: listen to both. | |
| 86 | res.on("close", () => this.leave(viewer)); | |
| 87 | res.socket?.once("close", () => this.leave(viewer)); | |
| 88 | clearTimeout(this.linger); | |
| 89 | if (this._state === "offline") { | |
| 90 | this.whole(viewer, "image/png", OFFLINE_FRAME); | |
| 91 | this.offline(); // keeps the repeat going, and the retry | |
| 92 | } else if (this._state === "idle") this.connect(); | |
| 93 | } | |
| 94 | ||
| 95 | close(): void { | |
| 96 | this.closed = true; | |
| 97 | this.stop(); | |
| 98 | for (const v of this.viewers) v.res.end(); | |
| 99 | this.viewers.clear(); | |
| 100 | } | |
| 101 | ||
| 102 | // ------------------------------------------------------------------ viewers | |
| 103 | private leave(viewer: Viewer) { | |
| 104 | if (!this.viewers.delete(viewer) || this.viewers.size) return; | |
| 105 | clearTimeout(this.linger); | |
| 106 | this.linger = setTimeout(() => { | |
| 107 | if (!this.viewers.size) this.stop(); | |
| 108 | }, this.o.lingerMs); | |
| 109 | this.linger.unref?.(); | |
| 110 | } | |
| 111 | ||
| 112 | private write(viewer: Viewer, bytes: Buffer) { | |
| 113 | if (!viewer.res.destroyed && !viewer.res.writableEnded) viewer.res.write(bytes); | |
| 114 | } | |
| 115 | ||
| 116 | private whole(viewer: Viewer, type: string, body: Buffer) { | |
| 117 | this.write(viewer, head(type)); | |
| 118 | this.write(viewer, body); | |
| 119 | this.write(viewer, NEXT); | |
| 120 | } | |
| 121 | ||
| 122 | /** Show the offline picture; `cut` closes the parts a failed source left half-sent, otherwise those viewers wait. */ | |
| 123 | private broadcastOffline(cut: boolean) { | |
| 124 | for (const v of this.viewers) { | |
| 125 | if (v.inPart && !cut) continue; | |
| 126 | if (v.inPart) this.write(v, NEXT); | |
| 127 | v.inPart = false; | |
| 128 | this.whole(v, "image/png", OFFLINE_FRAME); | |
| 129 | } | |
| 130 | } | |
| 131 | ||
| 132 | // ------------------------------------------------------------------ the source | |
| 133 | /** Let go of the source and every timer; with no source the feed is offline for good, otherwise idle until watched. */ | |
| 134 | private stop() { | |
| 135 | this.attempt++; | |
| 136 | this.controller?.abort(); | |
| 137 | this.controller = null; | |
| 138 | for (const t of [this.retry, this.linger]) clearTimeout(t); | |
| 139 | for (const t of [this.repeat, this.watchdog]) clearInterval(t); | |
| 140 | this.retry = this.linger = this.repeat = this.watchdog = undefined; | |
| 141 | this._state = this.source ? "idle" : "offline"; | |
| 142 | } | |
| 143 | ||
| 144 | private offline() { | |
| 145 | if (!this.repeat) { | |
| 146 | this.repeat = setInterval(() => this.broadcastOffline(false), this.o.offlineEveryMs); | |
| 147 | this.repeat.unref?.(); | |
| 148 | } | |
| 149 | if (this.source && !this.retry && !this.closed) { | |
| 150 | this.retry = setTimeout(() => { | |
| 151 | this.retry = undefined; | |
| 152 | if (this.viewers.size && this._state === "offline" && !this.controller) this.connect(); | |
| 153 | }, this.o.retryMs); | |
| 154 | this.retry.unref?.(); | |
| 155 | } | |
| 156 | } | |
| 157 | ||
| 158 | private fail(attempt: number, why: string) { | |
| 159 | if (attempt !== this.attempt || this.closed) return; | |
| 160 | this.attempt++; | |
| 161 | this.controller?.abort(); | |
| 162 | this.controller = null; | |
| 163 | clearInterval(this.watchdog); | |
| 164 | this.watchdog = undefined; | |
| 165 | if (this._state !== "offline") console.error(`${this.name} ${this.source}: ${why}; showing it offline`); | |
| 166 | this._state = "offline"; | |
| 167 | this.broadcastOffline(true); | |
| 168 | this.offline(); | |
| 169 | } | |
| 170 | ||
| 171 | private connect() { | |
| 172 | const attempt = ++this.attempt; | |
| 173 | const controller = new AbortController(); | |
| 174 | this.controller = controller; | |
| 175 | if (this._state !== "offline") this._state = "connecting"; | |
| 176 | const timer = setTimeout(() => this.fail(attempt, `no answer in ${this.o.connectTimeoutMs} ms`), this.o.connectTimeoutMs); | |
| 177 | void (async () => { | |
| 178 | let response: Response; | |
| 179 | try { | |
| 180 | response = await fetch(this.source!, { signal: controller.signal, cache: "no-store", redirect: "error", headers: { accept: "multipart/x-mixed-replace, image/*" } }); | |
| 181 | } catch (err) { | |
| 182 | clearTimeout(timer); | |
| 183 | const e = err as { cause?: { code?: string }; code?: string; message?: string }; | |
| 184 | return this.fail(attempt, e?.cause?.code ?? e?.code ?? e?.message ?? String(err)); | |
| 185 | } | |
| 186 | clearTimeout(timer); | |
| 187 | if (attempt !== this.attempt) return void response.body?.cancel().catch(() => {}); | |
| 188 | const boundary = boundaryOf(response.headers.get("content-type")); | |
| 189 | if (!response.ok || !boundary || !response.body) { | |
| 190 | void response.body?.cancel().catch(() => {}); | |
| 191 | return this.fail(attempt, response.ok ? `not an MJPEG stream (${response.headers.get("content-type")})` : `answered ${response.status}`); | |
| 192 | } | |
| 193 | const parser = new MjpegParser(boundary, { | |
| 194 | start: (type) => { | |
| 195 | for (const v of this.viewers) { | |
| 196 | v.inPart = v.res.writableLength <= this.o.maxBacklog; | |
| 197 | if (v.inPart) this.write(v, head(type)); | |
| 198 | } | |
| 199 | }, | |
| 200 | data: (bytes) => { | |
| 201 | for (const v of this.viewers) if (v.inPart) this.write(v, bytes); | |
| 202 | }, | |
| 203 | end: () => { | |
| 204 | if (this._state !== "live") { | |
| 205 | if (this._state === "offline") console.log(`${this.name} ${this.source}: live`); | |
| 206 | this._state = "live"; | |
| 207 | clearInterval(this.repeat); | |
| 208 | this.repeat = undefined; | |
| 209 | } | |
| 210 | for (const v of this.viewers) if (v.inPart) this.write(v, NEXT), (v.inPart = false); | |
| 211 | }, | |
| 212 | }); | |
| 213 | this.lastByte = Date.now(); | |
| 214 | this.watchdog = setInterval(() => { | |
| 215 | if (Date.now() - this.lastByte > this.o.staleMs) this.fail(attempt, `nothing for ${this.o.staleMs} ms`); | |
| 216 | }, Math.max(50, this.o.staleMs / 4)); | |
| 217 | this.watchdog.unref?.(); | |
| 218 | try { | |
| 219 | for await (const chunk of response.body as unknown as AsyncIterable<Uint8Array>) { | |
| 220 | if (attempt !== this.attempt) return; | |
| 221 | this.lastByte = Date.now(); | |
| 222 | parser.push(chunk); | |
| 223 | } | |
| 224 | this.fail(attempt, "the stream ended"); | |
| 225 | } catch (err) { | |
| 226 | this.fail(attempt, (err as Error)?.message ?? String(err)); | |
| 227 | } | |
| 228 | })(); | |
| 229 | } | |
| 230 | } |
Software/so101/src/mjpeg.test.ts added +73
| @@ -0,0 +1,73 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { boundaryOf, MjpegParser } from "./mjpeg.ts"; | |
| 3 | ||
| 4 | /** Parse `bytes` fed in pieces of `step`, and give back each part as [type, body] and the order of the sink's calls. */ | |
| 5 | function parse(bytes: Buffer, step = bytes.length, boundary = "frame") { | |
| 6 | const parts: [string, Buffer][] = []; | |
| 7 | const events: string[] = []; | |
| 8 | let body: Buffer[] = []; | |
| 9 | let type = ""; | |
| 10 | const parser = new MjpegParser(boundary, { | |
| 11 | start: (t) => (events.push("start"), (type = t), (body = [])), | |
| 12 | data: (c) => (events.push("data"), body.push(Buffer.from(c))), | |
| 13 | end: () => (events.push("end"), parts.push([type, Buffer.concat(body)])), | |
| 14 | }); | |
| 15 | for (let i = 0; i < bytes.length; i += step) parser.push(bytes.subarray(i, i + step)); | |
| 16 | return { parts, events }; | |
| 17 | } | |
| 18 | const part = (body: string, headers = "Content-Type: image/jpeg") => `--frame\r\n${headers}\r\n\r\n${body}\r\n`; | |
| 19 | ||
| 20 | describe("boundaryOf", () => { | |
| 21 | test("reads the boundary of a multipart/x-mixed-replace type, quoted or not", () => { | |
| 22 | expect(boundaryOf("multipart/x-mixed-replace;boundary=boundarydonotcross")).toBe("boundarydonotcross"); | |
| 23 | expect(boundaryOf('multipart/x-mixed-replace; boundary="a b"')).toBe("a b"); | |
| 24 | expect(boundaryOf("multipart/x-mixed-replace; boundary=--frame")).toBe("frame"); | |
| 25 | }); | |
| 26 | test("anything else is not a stream", () => { | |
| 27 | expect(boundaryOf(null)).toBeNull(); | |
| 28 | expect(boundaryOf("image/jpeg")).toBeNull(); | |
| 29 | expect(boundaryOf("multipart/x-mixed-replace")).toBeNull(); | |
| 30 | }); | |
| 31 | }); | |
| 32 | ||
| 33 | describe("MjpegParser", () => { | |
| 34 | const stream = Buffer.from(`preamble\r\n${part("one")}${part("two\r\n-x", "content-type: image/png")}${part("three", "X-Timestamp: 1")}--frame\r\n`); | |
| 35 | ||
| 36 | test("cuts a stream into parts, whatever size the chunks come in", () => { | |
| 37 | for (const step of [1, 2, 3, 7, 11, 64, stream.length]) { | |
| 38 | const { parts } = parse(stream, step); | |
| 39 | expect(parts.map(([t, b]) => [t, b.toString()])).toEqual([ | |
| 40 | ["image/jpeg", "one"], | |
| 41 | ["image/png", "two\r\n-x"], | |
| 42 | ["image/jpeg", "three"], | |
| 43 | ]); | |
| 44 | } | |
| 45 | }); | |
| 46 | ||
| 47 | test("hands a part's body on before the part is over", () => { | |
| 48 | const { events } = parse(Buffer.from(`--frame\r\n\r\n${"x".repeat(100)}`), 10); | |
| 49 | expect(events[0]).toBe("start"); | |
| 50 | expect(events.filter((e) => e === "data").length).toBeGreaterThan(5); | |
| 51 | expect(events).not.toContain("end"); | |
| 52 | }); | |
| 53 | ||
| 54 | test("a part with a Content-Length ends with its last byte, before the next boundary arrives", () => { | |
| 55 | const { parts, events } = parse(Buffer.from("--frame\r\nContent-Type: image/jpeg\r\nContent-Length: 4\r\n\r\nJPEG")); | |
| 56 | expect(events.at(-1)).toBe("end"); | |
| 57 | expect(parts[0]![1].toString()).toBe("JPEG"); | |
| 58 | }); | |
| 59 | ||
| 60 | test("a part without one holds back only what could be a boundary", () => { | |
| 61 | const { events } = parse(Buffer.from("--frame\r\n\r\nJPEG\r\n")); | |
| 62 | expect(events).toEqual(["start", "data"]); | |
| 63 | }); | |
| 64 | ||
| 65 | test("the close delimiter ends the stream; headers past 8 KiB are refused", () => { | |
| 66 | expect(parse(Buffer.from(`${part("a")}--frame--\r\n`)).parts.length).toBe(1); | |
| 67 | expect(() => parse(Buffer.from(`--frame\r\nX: ${"y".repeat(9000)}`))).toThrow(/too long/); | |
| 68 | }); | |
| 69 | ||
| 70 | test("a type that is not an image is read as JPEG", () => { | |
| 71 | expect(parse(Buffer.from(part("a", "Content-Type: text/html") + "--frame")).parts[0]![0]).toBe("image/jpeg"); | |
| 72 | }); | |
| 73 | }); |
Software/so101/src/mjpeg.ts added +111
| @@ -0,0 +1,111 @@ | ||
| 1 | /** | |
| 2 | * MJPEG framing: a multipart/x-mixed-replace body, cut into parts as the bytes arrive. | |
| 3 | * | |
| 4 | * A part's body is handed on in the chunks it came in, never gathered into a whole frame: the parser only holds | |
| 5 | * back a part's headers (bounded by MAX_HEAD) and, for a part without a Content-Length, the few bytes that could be | |
| 6 | * the start of the next boundary. A part with one (µStreamer sends it) ends the moment its last byte arrives, so a | |
| 7 | * frame reaches the viewer without waiting for the next. | |
| 8 | */ | |
| 9 | ||
| 10 | export interface PartSink { | |
| 11 | /** A part begins; `type` is its image type (image/jpeg when the source does not say). */ | |
| 12 | start(type: string): void; | |
| 13 | /** Bytes of the current part's body. */ | |
| 14 | data(chunk: Buffer): void; | |
| 15 | /** The current part is complete. */ | |
| 16 | end(): void; | |
| 17 | } | |
| 18 | ||
| 19 | const MAX_HEAD = 8192; | |
| 20 | const CRLF2 = Buffer.from("\r\n\r\n"); | |
| 21 | const IMAGE = /^image\/[a-z0-9.+-]{1,40}$/i; | |
| 22 | ||
| 23 | /** The boundary a multipart/x-mixed-replace content type names, or null when it is not one. */ | |
| 24 | export function boundaryOf(contentType: string | null): string | null { | |
| 25 | if (!contentType || !/^\s*multipart\/x-mixed-replace\s*;/i.test(contentType)) return null; | |
| 26 | const match = /;\s*boundary\s*=\s*(?:"([^"]{1,70})"|([^;\s]{1,70}))/i.exec(contentType); | |
| 27 | const boundary = match?.[1] ?? match?.[2]; | |
| 28 | return boundary?.replace(/^--/, "") || null; | |
| 29 | } | |
| 30 | ||
| 31 | export class MjpegParser { | |
| 32 | private state: "seek" | "head" | "body" = "seek"; | |
| 33 | private buf: Buffer = Buffer.alloc(0); | |
| 34 | private readonly delimiter: Buffer; | |
| 35 | private readonly bodyEnd: Buffer; | |
| 36 | private readonly sink: PartSink; | |
| 37 | /** Body bytes still due in a part that gave its Content-Length; -1 when it gave none. */ | |
| 38 | private remaining = -1; | |
| 39 | ||
| 40 | constructor(boundary: string, sink: PartSink) { | |
| 41 | this.sink = sink; | |
| 42 | this.delimiter = Buffer.from(`--${boundary}`); | |
| 43 | this.bodyEnd = Buffer.from(`\r\n--${boundary}`); | |
| 44 | } | |
| 45 | ||
| 46 | /** Feed the next bytes of the stream. Throws when a part's headers run past MAX_HEAD. */ | |
| 47 | push(chunk: Uint8Array): void { | |
| 48 | this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); | |
| 49 | for (;;) { | |
| 50 | if (this.state === "seek") { | |
| 51 | const at = this.buf.indexOf(this.delimiter); | |
| 52 | if (at < 0) return void (this.buf = this.buf.subarray(Math.max(0, this.buf.length - this.delimiter.length + 1))); | |
| 53 | this.buf = this.buf.subarray(at + this.delimiter.length); | |
| 54 | this.state = "head"; | |
| 55 | } else if (this.state === "head") { | |
| 56 | if (this.buf.length >= 2 && this.buf[0] === 0x2d && this.buf[1] === 0x2d) { | |
| 57 | // The close delimiter: the source says it is done. Anything after it is not ours. | |
| 58 | this.buf = this.buf.subarray(2); | |
| 59 | this.state = "seek"; | |
| 60 | continue; | |
| 61 | } | |
| 62 | const at = this.buf.indexOf(CRLF2); | |
| 63 | if (at < 0) { | |
| 64 | if (this.buf.length > MAX_HEAD) throw new Error("a part's headers are too long"); | |
| 65 | return; | |
| 66 | } | |
| 67 | let type = "image/jpeg"; | |
| 68 | this.remaining = -1; | |
| 69 | for (const line of this.buf.subarray(0, at).toString("latin1").split("\r\n")) { | |
| 70 | const t = /^content-type\s*:\s*(.*?)\s*$/i.exec(line); | |
| 71 | if (t && IMAGE.test(t[1]!)) type = t[1]!.toLowerCase(); | |
| 72 | const n = /^content-length\s*:\s*(\d{1,9})\s*$/i.exec(line); | |
| 73 | if (n) this.remaining = Number(n[1]); | |
| 74 | } | |
| 75 | this.buf = this.buf.subarray(at + CRLF2.length); | |
| 76 | this.state = "body"; | |
| 77 | this.sink.start(type); | |
| 78 | } else if (this.remaining >= 0) { | |
| 79 | const take = Math.min(this.remaining, this.buf.length); | |
| 80 | if (take > 0) this.sink.data(this.buf.subarray(0, take)); | |
| 81 | this.buf = this.buf.subarray(take); | |
| 82 | this.remaining -= take; | |
| 83 | if (this.remaining > 0) return; | |
| 84 | this.sink.end(); | |
| 85 | this.state = "seek"; | |
| 86 | } else { | |
| 87 | const at = this.buf.indexOf(this.bodyEnd); | |
| 88 | if (at < 0) { | |
| 89 | const keep = this.heldBack(); | |
| 90 | if (this.buf.length > keep) { | |
| 91 | this.sink.data(this.buf.subarray(0, this.buf.length - keep)); | |
| 92 | this.buf = this.buf.subarray(this.buf.length - keep); | |
| 93 | } | |
| 94 | return; | |
| 95 | } | |
| 96 | if (at > 0) this.sink.data(this.buf.subarray(0, at)); | |
| 97 | this.sink.end(); | |
| 98 | this.buf = this.buf.subarray(at + 2); // keep "--boundary" for the seek | |
| 99 | this.state = "seek"; | |
| 100 | } | |
| 101 | } | |
| 102 | } | |
| 103 | ||
| 104 | /** The longest tail of the buffer that could be the start of the next boundary. */ | |
| 105 | private heldBack(): number { | |
| 106 | for (let n = Math.min(this.bodyEnd.length - 1, this.buf.length); n > 0; n--) { | |
| 107 | if (this.buf.subarray(this.buf.length - n).equals(this.bodyEnd.subarray(0, n))) return n; | |
| 108 | } | |
| 109 | return 0; | |
| 110 | } | |
| 111 | } |
Software/so101/src/offline-frame.test.ts added +22
| @@ -0,0 +1,22 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { inflateSync } from "node:zlib"; | |
| 3 | import { OFFLINE_FRAME, OFFLINE_HEIGHT, OFFLINE_INK, OFFLINE_PAPER, OFFLINE_WIDTH } from "./offline-frame.ts"; | |
| 4 | ||
| 5 | test("the offline picture is a 640×360 RGB PNG with lettering on the page's paper", () => { | |
| 6 | expect([...OFFLINE_FRAME.subarray(0, 8)]).toEqual([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); | |
| 7 | expect(OFFLINE_FRAME.subarray(12, 16).toString()).toBe("IHDR"); | |
| 8 | expect(OFFLINE_FRAME.readUInt32BE(16)).toBe(OFFLINE_WIDTH); | |
| 9 | expect(OFFLINE_FRAME.readUInt32BE(20)).toBe(OFFLINE_HEIGHT); | |
| 10 | expect(OFFLINE_FRAME.length).toBeLessThan(8192); | |
| 11 | ||
| 12 | const idatLength = OFFLINE_FRAME.readUInt32BE(33); | |
| 13 | expect(OFFLINE_FRAME.subarray(37, 41).toString()).toBe("IDAT"); | |
| 14 | const raw = inflateSync(OFFLINE_FRAME.subarray(41, 41 + idatLength)); | |
| 15 | const stride = OFFLINE_WIDTH * 3 + 1; | |
| 16 | expect(raw.length).toBe(stride * OFFLINE_HEIGHT); | |
| 17 | const pixel = (x: number, y: number) => [...raw.subarray(y * stride + 1 + x * 3, y * stride + 4 + x * 3)]; | |
| 18 | expect(pixel(0, 0)).toEqual([...OFFLINE_PAPER]); | |
| 19 | let ink = 0; | |
| 20 | for (let x = 0; x < OFFLINE_WIDTH; x++) if (pixel(x, OFFLINE_HEIGHT / 2).join() === OFFLINE_INK.join()) ink++; | |
| 21 | expect(ink).toBeGreaterThan(50); // the words cross the middle row | |
| 22 | }); |
Software/so101/src/offline-frame.ts added +79
| @@ -0,0 +1,79 @@ | ||
| 1 | /** | |
| 2 | * The "camera offline" picture: a 640×360 PNG drawn once at start, so a camera pane always has something to show. | |
| 3 | * PNG, not JPEG, because a PNG is a few lines over node:zlib; browsers take either as a part of an MJPEG stream. | |
| 4 | */ | |
| 5 | ||
| 6 | import { deflateSync } from "node:zlib"; | |
| 7 | ||
| 8 | export const OFFLINE_WIDTH = 640; | |
| 9 | export const OFFLINE_HEIGHT = 360; | |
| 10 | /** The page's ink and paper (viewer.html's dark panes). */ | |
| 11 | export const OFFLINE_PAPER = [0x16, 0x19, 0x1e] as const; | |
| 12 | export const OFFLINE_INK = [0xe6, 0xe3, 0xdc] as const; | |
| 13 | const MUTED = [0x5c, 0x63, 0x6e] as const; | |
| 14 | ||
| 15 | /** 5×7 glyphs, only the letters the picture needs. */ | |
| 16 | const GLYPHS: Record<string, string[]> = { | |
| 17 | A: ["01110", "10001", "10001", "11111", "10001", "10001", "10001"], | |
| 18 | C: ["01110", "10001", "10000", "10000", "10000", "10001", "01110"], | |
| 19 | E: ["11111", "10000", "10000", "11110", "10000", "10000", "11111"], | |
| 20 | F: ["11111", "10000", "10000", "11110", "10000", "10000", "10000"], | |
| 21 | I: ["01110", "00100", "00100", "00100", "00100", "00100", "01110"], | |
| 22 | L: ["10000", "10000", "10000", "10000", "10000", "10000", "11111"], | |
| 23 | M: ["10001", "11011", "10101", "10101", "10001", "10001", "10001"], | |
| 24 | N: ["10001", "11001", "10101", "10011", "10001", "10001", "10001"], | |
| 25 | O: ["01110", "10001", "10001", "10001", "10001", "10001", "01110"], | |
| 26 | R: ["11110", "10001", "10001", "11110", "10100", "10010", "10001"], | |
| 27 | " ": ["00000", "00000", "00000", "00000", "00000", "00000", "00000"], | |
| 28 | }; | |
| 29 | ||
| 30 | const CRC = Array.from({ length: 256 }, (_, n) => { | |
| 31 | let c = n; | |
| 32 | for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; | |
| 33 | return c >>> 0; | |
| 34 | }); | |
| 35 | const crc32 = (bytes: Buffer) => { | |
| 36 | let c = 0xffffffff; | |
| 37 | for (const b of bytes) c = CRC[(c ^ b) & 0xff]! ^ (c >>> 8); | |
| 38 | return (c ^ 0xffffffff) >>> 0; | |
| 39 | }; | |
| 40 | const chunk = (type: string, data: Buffer) => { | |
| 41 | const body = Buffer.concat([Buffer.from(type, "latin1"), data]); | |
| 42 | const out = Buffer.alloc(body.length + 8); | |
| 43 | out.writeUInt32BE(data.length, 0); | |
| 44 | body.copy(out, 4); | |
| 45 | out.writeUInt32BE(crc32(body), body.length + 4); | |
| 46 | return out; | |
| 47 | }; | |
| 48 | ||
| 49 | function draw(text: string): Buffer { | |
| 50 | const w = OFFLINE_WIDTH; | |
| 51 | const h = OFFLINE_HEIGHT; | |
| 52 | const stride = w * 3 + 1; // a filter byte (0, none) starts each row | |
| 53 | const raw = Buffer.alloc(stride * h); | |
| 54 | const paint = (x: number, y: number, rgb: readonly number[]) => raw.set(rgb, y * stride + 1 + x * 3); | |
| 55 | for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) paint(x, y, OFFLINE_PAPER); | |
| 56 | ||
| 57 | const scale = 5; | |
| 58 | const advance = 6 * scale; | |
| 59 | const left = Math.round((w - (text.length * advance - scale)) / 2); | |
| 60 | const top = Math.round((h - 7 * scale) / 2); | |
| 61 | [...text].forEach((ch, i) => { | |
| 62 | GLYPHS[ch]!.forEach((row, gy) => { | |
| 63 | for (let gx = 0; gx < 5; gx++) { | |
| 64 | if (row[gx] !== "1") continue; | |
| 65 | for (let dy = 0; dy < scale; dy++) for (let dx = 0; dx < scale; dx++) paint(left + i * advance + gx * scale + dx, top + gy * scale + dy, OFFLINE_INK); | |
| 66 | } | |
| 67 | }); | |
| 68 | }); | |
| 69 | // A short rule under the words, in the muted tone of the page's tags. | |
| 70 | for (let y = top + 7 * scale + 22; y < top + 7 * scale + 24; y++) for (let x = w / 2 - 24; x < w / 2 + 24; x++) paint(x, y, MUTED); | |
| 71 | ||
| 72 | const ihdr = Buffer.alloc(13); | |
| 73 | ihdr.writeUInt32BE(w, 0); | |
| 74 | ihdr.writeUInt32BE(h, 4); | |
| 75 | ihdr.set([8, 2, 0, 0, 0], 8); // 8-bit RGB, deflate, no filter, no interlace | |
| 76 | return Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk("IHDR", ihdr), chunk("IDAT", deflateSync(raw, { level: 9 })), chunk("IEND", Buffer.alloc(0))]); | |
| 77 | } | |
| 78 | ||
| 79 | export const OFFLINE_FRAME: Buffer = draw("CAMERA OFFLINE"); |
Software/so101/src/relay.test.ts +45 −8
| @@ -7,6 +7,7 @@ import { join } from "node:path"; | ||
| 7 | 7 | import WebSocket from "ws"; |
| 8 | 8 | import { FeetechBus } from "./bus.ts"; |
| 9 | 9 | import { FakeBus } from "./fake-bus.ts"; |
| 10 | import { OFFLINE_FRAME } from "./offline-frame.ts"; | |
| 10 | 11 | import { parseRemote, sameOrigin, startRelay, type Remote, type RunningRelay } from "./relay.ts"; |
| 11 | 12 | import { startServer, type RunningServer } from "./server.ts"; |
| 12 | 13 | |
| @@ -22,6 +23,26 @@ async function arm(ids: number[]): Promise<RunningServer> { | ||
| 22 | 23 | } |
| 23 | 24 | const remoteAt = (port: number, camera?: number): Remote => parseRemote({ ARM_SERVER: `ws://127.0.0.1:${port}`, ...(camera ? { ARM_CAMERA_1: `http://127.0.0.1:${camera}/stream` } : {}) })!; |
| 24 | 25 | const JPEG = Buffer.from([0xff, 0xd8, 0x4a, 0x50, 0x45, 0x47, 0xff, 0xd9]); |
| 26 | const PNG = OFFLINE_FRAME.subarray(0, 8); | |
| 27 | /** Read a camera stream until `mark` shows up (or 3 s pass), then hang up. */ | |
| 28 | async function cameraShows(url: string, mark: Buffer): Promise<{ type: string | null; ms: number }> { | |
| 29 | const started = Date.now(); | |
| 30 | const hangUp = new AbortController(); | |
| 31 | const response = await fetch(url, { signal: hangUp.signal }); | |
| 32 | const reader = response.body!.getReader(); | |
| 33 | let seen = Buffer.alloc(0); | |
| 34 | try { | |
| 35 | while (!seen.includes(mark)) { | |
| 36 | if (Date.now() - started > 3000) throw new Error(`no ${mark.toString("hex")} in 3 s`); | |
| 37 | const { value, done } = await reader.read(); | |
| 38 | if (done) throw new Error("the stream ended"); | |
| 39 | seen = Buffer.concat([seen, value]); | |
| 40 | } | |
| 41 | } finally { | |
| 42 | hangUp.abort(); | |
| 43 | } | |
| 44 | return { type: response.headers.get("content-type"), ms: Date.now() - started }; | |
| 45 | } | |
| 25 | 46 | const joints = async (base: string) => ((await (await fetch(`${base}/status`)).json()) as { joints: unknown[] }).joints.length; |
| 26 | 47 | const link = async (base: string) => (await (await fetch(`${base}/link`)).json()) as { mode: string; host: string | null; problem: string | null }; |
| 27 | 48 | |
| @@ -48,6 +69,14 @@ describe("ARM_SERVER", () => { | ||
| 48 | 69 | expect(parseRemote({ ARM_SERVER: "ws://pi:8101", ARM_CAMERA_2: "https://cams.example/two/stream" })?.cameras[1]).toBe("https://cams.example/two/stream"); |
| 49 | 70 | }); |
| 50 | 71 | |
| 72 | test("ARM_CAMERA names camera 1: a URL, or a host:port whose /stream it is", () => { | |
| 73 | expect(parseRemote({ ARM_SERVER: "ws://100.89.95.68:8101" })?.cameras[0]).toBe("http://100.89.95.68:8080/stream"); | |
| 74 | expect(parseRemote({ ARM_SERVER: "ws://pi:8101", ARM_CAMERA: "cam.tailnet:9000" })?.cameras[0]).toBe("http://cam.tailnet:9000/stream"); | |
| 75 | expect(parseRemote({ ARM_SERVER: "ws://pi:8101", ARM_CAMERA: "http://cam:8080/live", ARM_CAMERA_1: "http://other:1/x" })?.cameras[0]).toBe("http://cam:8080/live"); | |
| 76 | expect(parseRemote({ ARM_SERVER: "ws://pi:8101", ARM_CAMERA_1: "http://other:1/x" })?.cameras[0]).toBe("http://other:1/x"); | |
| 77 | expect(() => parseRemote({ ARM_SERVER: "ws://pi:8101", ARM_CAMERA: "http://cam:8080/stream?key=1" })).toThrow(/ARM_CAMERA /); | |
| 78 | }); | |
| 79 | ||
| 51 | 80 | test("anything else is refused with a reason", () => { |
| 52 | 81 | expect(() => parseRemote({ ARM_SERVER: "ftp://pi:8101" })).toThrow(/must start with/); |
| 53 | 82 | expect(() => parseRemote({ ARM_SERVER: "ws://user:pw@pi:8101" })).toThrow(/nothing else/); |
| @@ -93,7 +122,11 @@ describe("relay", () => { | ||
| 93 | 122 | expect(await page.text()).toContain("<title>SO-101</title>"); |
| 94 | 123 | expect(page.headers.get("x-content-type-options")).toBe("nosniff"); |
| 95 | 124 | expect(page.headers.get("content-security-policy")).toContain("frame-ancestors 'none'"); |
| 96 | expect((await fetch(`${base}/camera/1/stream`)).status).toBe(404); | |
| 125 | // No camera to show: the offline picture, at once, to anyone. | |
| 126 | const shown = await cameraShows(`${base}/camera/1/stream`, PNG); | |
| 127 | expect(shown.type).toBe("multipart/x-mixed-replace;boundary=armframe"); | |
| 128 | expect(shown.ms).toBeLessThan(1000); | |
| 129 | expect((await fetch(`${base}/camera/1/state`)).status).toBe(404); | |
| 97 | 130 | }); |
| 98 | 131 | |
| 99 | 132 | test("set: status, poses and /follow go to the arm server; the camera's bytes come through", async () => { |
| @@ -101,7 +134,7 @@ describe("relay", () => { | ||
| 101 | 134 | camera = createServer((req, res) => { |
| 102 | 135 | if (req.url?.startsWith("/state")) return res.writeHead(200, { "content-type": "application/json" }).end('{"ok":true}'); |
| 103 | 136 | res.writeHead(200, { "content-type": "multipart/x-mixed-replace;boundary=frame" }); |
| 104 | res.write(Buffer.concat([Buffer.from("--frame\r\ncontent-type: image/jpeg\r\n\r\n"), JPEG])); // and the stream stays open, like µStreamer | |
| 137 | res.write(Buffer.concat([Buffer.from(`--frame\r\ncontent-type: image/jpeg\r\ncontent-length: ${JPEG.length}\r\n\r\n`), JPEG])); // and the stream stays open, like µStreamer | |
| 105 | 138 | }); |
| 106 | 139 | const cameraPort = await listening(camera); |
| 107 | 140 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(remote.port, cameraPort) }); |
| @@ -121,15 +154,19 @@ describe("relay", () => { | ||
| 121 | 154 | expect(status.driving).toBe("stream"); // the remote arm took the stream, not the simulation |
| 122 | 155 | ws.close(); |
| 123 | 156 | |
| 124 | const stream = await fetch(`${base}/camera/1/stream`); | |
| 125 | expect(stream.headers.get("content-type")).toBe("multipart/x-mixed-replace;boundary=frame"); | |
| 126 | const reader = stream.body!.getReader(); | |
| 127 | const { value } = await reader.read(); | |
| 128 | expect(Buffer.from(value!).includes(JPEG)).toBe(true); | |
| 129 | await reader.cancel(); | |
| 157 | expect((await cameraShows(`${base}/camera/1/stream`, JPEG)).type).toBe("multipart/x-mixed-replace;boundary=armframe"); | |
| 130 | 158 | expect(await (await fetch(`${base}/camera/1/state`)).json()).toEqual({ ok: true }); |
| 131 | 159 | }); |
| 132 | 160 | |
| 161 | test("set, with the camera down: the offline picture within 3 s while the arm stays live", async () => { | |
| 162 | remote = await arm([1, 2, 6]); | |
| 163 | hang = createTcpServer(() => {}); // a camera port that accepts and never answers | |
| 164 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(remote.port, await listening(hang)) }); | |
| 165 | const base = `http://127.0.0.1:${relay.port}`; | |
| 166 | expect((await cameraShows(`${base}/camera/1/stream`, PNG)).ms).toBeLessThan(3000); | |
| 167 | expect((await link(base)).mode).toBe("live"); | |
| 168 | }); | |
| 169 | ||
| 133 | 170 | test("a mutation from another site is refused before it reaches the arm", async () => { |
| 134 | 171 | remote = await arm([1, 2, 6]); |
| 135 | 172 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(remote.port) }); |
Software/so101/src/relay.ts +21 −5
| @@ -12,7 +12,9 @@ | ||
| 12 | 12 | * |
| 13 | 13 | * WS /live the lobby: the turn, the queue, the chat |
| 14 | 14 | * GET /link {"mode": "live"|"simulated", "host": "<host:port>"|null, "problem": "…"|null} |
| 15 | * GET /camera/<1|2>/<stream|state|snapshot> the arm machine's camera (µStreamer on :8080 and :8081 by default) | |
| 15 | * GET /camera/<1|2>/stream the arm machine's camera, one upstream connection shared by every viewer | |
| 16 | * (src/camera-feed.ts); the "camera offline" picture when there is none or it is down | |
| 17 | * GET /camera/<1|2>/<state|snapshot> passed through to the camera (µStreamer on :8080 and :8081 by default) | |
| 16 | 18 | * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time, |
| 17 | 19 | * turn or no turn: GET answers it or null, PUT {"joints": {...}} stores it |
| 18 | 20 | */ |
| @@ -22,6 +24,7 @@ import type { Duplex } from "node:stream"; | ||
| 22 | 24 | import { Readable } from "node:stream"; |
| 23 | 25 | import { pipeline } from "node:stream/promises"; |
| 24 | 26 | import WebSocket, { WebSocketServer } from "ws"; |
| 27 | import { CameraFeed, type FeedOptions } from "./camera-feed.ts"; | |
| 25 | 28 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 26 | 29 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| 27 | 30 | import { CalibrationError } from "./leader-map.js"; |
| @@ -62,6 +65,8 @@ export interface RelayOptions { | ||
| 62 | 65 | readTimeoutMs?: number; |
| 63 | 66 | /** Who may command the arm; without one, anyone who reaches the relay may (a run on one's own machine). */ |
| 64 | 67 | lobby?: Lobby | null; |
| 68 | /** Timings of the shared camera feeds. */ | |
| 69 | camera?: FeedOptions; | |
| 65 | 70 | /** Every driver's leader calibration; without one they are kept in memory. */ |
| 66 | 71 | leaders?: LeaderCalibrations; |
| 67 | 72 | } |
| @@ -96,7 +101,11 @@ const absolute = (text: string, what: string, schemes: Record<string, string>): | ||
| 96 | 101 | return url; |
| 97 | 102 | }; |
| 98 | 103 | |
| 99 | /** The arm server named by ARM_SERVER (and the cameras, ARM_CAMERA_1 / ARM_CAMERA_2), or null when unset. Throws on a bad value. */ | |
| 104 | /** | |
| 105 | * The arm server named by ARM_SERVER, or null when unset. Throws on a bad value. | |
| 106 | * The cameras default to ports 8080 and 8081 on its host; ARM_CAMERA (or ARM_CAMERA_1) and ARM_CAMERA_2 name others, | |
| 107 | * as a URL or host:port (a bare host:port means its /stream). | |
| 108 | */ | |
| 100 | 109 | export function parseRemote(env: Record<string, string | undefined>): Remote | null { |
| 101 | 110 | const text = env.ARM_SERVER?.trim(); |
| 102 | 111 | if (!text) return null; |
| @@ -104,8 +113,12 @@ export function parseRemote(env: Record<string, string | undefined>): Remote | n | ||
| 104 | 113 | const scheme = SCHEMES[url.protocol]!; |
| 105 | 114 | const path = url.pathname.replace(/\/+$/, ""); |
| 106 | 115 | const cameras = [1, 2].map((n) => { |
| 107 | const override = env[`ARM_CAMERA_${n}`]?.trim(); | |
| 108 | return override ? absolute(override, `ARM_CAMERA_${n}`, { "http:": "", "https:": "" }).href : `http://${url.hostname}:${8079 + n}/stream`; | |
| 116 | const name = n === 1 && env.ARM_CAMERA?.trim() ? "ARM_CAMERA" : `ARM_CAMERA_${n}`; | |
| 117 | const override = env[name]?.trim(); | |
| 118 | if (!override) return `http://${url.hostname}:${8079 + n}/stream`; | |
| 119 | const camera = absolute(override, name, { "http:": "", "https:": "" }); | |
| 120 | if (camera.pathname === "/") camera.pathname = "/stream"; | |
| 121 | return camera.href; | |
| 109 | 122 | }); |
| 110 | 123 | return { http: `${scheme}//${url.host}${path}`, ws: `${scheme === "https:" ? "wss:" : "ws:"}//${url.host}${path}`, host: url.host, cameras }; |
| 111 | 124 | } |
| @@ -137,6 +150,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 137 | 150 | const probeEveryMs = options.probeEveryMs ?? 5000; |
| 138 | 151 | const readTimeoutMs = options.readTimeoutMs ?? 2500; |
| 139 | 152 | const lobby = options.lobby ?? null; |
| 153 | const feeds = [1, 2].map((n) => new CameraFeed(remote?.cameras[n - 1] ?? null, `camera ${n}`, options.camera)); | |
| 140 | 154 | const leaders = options.leaders ?? new LeaderCalibrations(); |
| 141 | 155 | |
| 142 | 156 | // ------------------------------------------------------------------ the link |
| @@ -274,7 +288,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 274 | 288 | const camera = CAMERA.exec(url.pathname); |
| 275 | 289 | if (camera) { |
| 276 | 290 | if (!reading) return json(res, 405, { error: "cameras are read-only" }); |
| 277 | if (!remote || !live) return json(res, 404, { error: "no camera: the arm is simulated" }); | |
| 291 | if (camera[2] === "stream") return feeds[Number(camera[1]) - 1]!.serve(req, res); | |
| 292 | if (!remote) return json(res, 404, { error: "no camera: the arm is simulated" }); | |
| 278 | 293 | const stream = remote.cameras[Number(camera[1]) - 1]!; |
| 279 | 294 | const target = stream.replace(/\/[^/]*$/, `/${camera[2]}`) + url.search; |
| 280 | 295 | try { |
| @@ -396,6 +411,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 396 | 411 | new Promise<void>((resolve) => { |
| 397 | 412 | clearInterval(timer); |
| 398 | 413 | turnEnded?.(); |
| 414 | for (const feed of feeds) feed.close(); | |
| 399 | 415 | for (const c of inFlight) c.abort(); |
| 400 | 416 | for (const client of wss.clients) client.terminate(); |
| 401 | 417 | wss.close(); |
new above
offline
Low (DoS): the live path skips a viewer whose
writableLengthis overmaxBacklog, but the offline repeat writes the PNG to every viewer regardless. A client that connects and never reads gets about 1.6 KB every 2 s queued in memory for as long as the socket stays open, and many idle anonymous connections add up. Fix: skip it here the same way as the live path,if (v.res.writableLength > this.o.maxBacklog) continue;, and add a test with a viewer that never reads.