Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
The relay serves /camera/<id>/video from ARM_H264_<n> sources, flags h264 in /cameras, and allows the Pi's H.264 ports in egress #267
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/259-the-relay-serves-camera-id-video-from-ar → main 1 commit fork
8 files
changed +409
−5
Software/README.md +35
| @@ -218,6 +218,41 @@ again every 2 s and the live picture comes back in the same stream, without | ||
| 218 | 218 | a reload. The camera is independent of the arm server: a camera that is down |
| 219 | 219 | does not put the arm on the simulation, and the other way round. |
| 220 | 220 | |
| 221 | **Real video: H.264 from the Pi.** Where a camera has H.264 renditions, the | |
| 222 | page plays them instead of MJPEG, in a fraction of the bytes. It is opt-in per | |
| 223 | camera (1 and 2 only), set in the app's settings: | |
| 224 | ||
| 225 | ARM_H264_1=720p=http://<pi>:8082/h264,480p=http://<pi>:8083/h264,240p=http://<pi>:8084/h264 | |
| 226 | ||
| 227 | Each entry is `<rung>=<url>`. The ladder is fixed: **720p** (1280x720, 20 fps, | |
| 228 | ~1500 kbps), **480p** (854x480, 15 fps, ~600 kbps) and **240p** (426x240, 10 fps, | |
| 229 | ~200 kbps). List any subset: viewers are only ever offered, started on, pinned to or stepped to the rungs that have a source. A name outside the ladder or a URL that is not | |
| 230 | http(s) stops the app at start with a message. `parseVideo` in `src/relay.ts` | |
| 231 | reads the variable, and `bench/fake-serve.ts`, the entry `forge.json` runs, | |
| 232 | is where it is wired in. | |
| 233 | ||
| 234 | *What the viewer sees.* The relay opens a rendition from the Pi only while | |
| 235 | someone is on it and fans it out over the WebSocket `/camera/<id>/video`, one | |
| 236 | rendition per viewer. A viewer whose connection backs up steps down the ladder | |
| 237 | and, rather than queueing, skips to the next keyframe; the **quality selector** | |
| 238 | on the picture lists the configured rungs and pins one, or leaves it on Auto. The page decodes with | |
| 239 | WebCodecs, loading the player only when `/cameras` reports `"h264": true` for | |
| 240 | the camera. When the browser has no WebCodecs, or no picture arrives in time, | |
| 241 | the page falls back to MJPEG (`/camera/<id>/stream`) by itself. The video | |
| 242 | socket is open to guests, like `/stream`; the driver is given the best | |
| 243 | rendition. Control messages from the viewer are capped at 200 bytes. | |
| 244 | ||
| 245 | *Not in this pull.* The Pi's encoder side: something on the Pi must serve each | |
| 246 | rendition as an H.264 stream at the URL you name (hardware encoder, one port | |
| 247 | per rung). Without it the relay's fetch fails and viewers stay on MJPEG. | |
| 248 | ||
| 249 | *The forge's egress list.* The relay reaches the Pi through the forge's egress | |
| 250 | proxy (`proxyFor` in `src/egress.ts`, the same as the other camera paths), so every rendition's `host:port` must be in `forge.json`'s `egress`. | |
| 251 | It lists 100.89.95.68 on 8080, 8081, 8101 and 8082 to 8084 (the example ports above); | |
| 252 | if your encoder listens elsewhere, add those ports. | |
| 253 | ||
| 254 | *The terminal (`ssh`) shows no video, and has no quality choice:* both are page-only. | |
| 255 | ||
| 221 | 256 | **In the browser.** Behind the relay the page draws each camera from |
| 222 | 257 | `WS /camera/<id>/ws`: whole JPEG frames, drawn as they arrive. If no frame |
| 223 | 258 | comes for 2 s the page closes that socket and opens a new one, and it retries |
Software/so101/bench/fake-serve.ts +8 −2
| @@ -22,7 +22,7 @@ import { CameraHub } from "../src/cameras.ts"; | ||
| 22 | 22 | import { LeaderCalibrations } from "../src/leader-calibrations.ts"; |
| 23 | 23 | import { Lobby } from "../src/lobby.ts"; |
| 24 | 24 | import { Moderator, modelFromEnv } from "../src/moderator.ts"; |
| 25 | import { parseRemote, startRelay, type Remote } from "../src/relay.ts"; | |
| 25 | import { parseRemote, parseVideo, startRelay, type Remote } from "../src/relay.ts"; | |
| 26 | 26 | import { startServer } from "../src/server.ts"; |
| 27 | 27 | |
| 28 | 28 | const fake = new FakeBus([1, 2, 3, 4, 5, 6]); |
| @@ -59,6 +59,12 @@ const lobby = new Lobby({ | ||
| 59 | 59 | const cameras = new CameraHub({ file: join(dir, "cameras.json"), seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); |
| 60 | 60 | |
| 61 | 61 | const leaders = new LeaderCalibrations({ file: join(dir, "leader-calibrations.json") }); |
| 62 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders }); | |
| 62 | let video = {}; | |
| 63 | try { | |
| 64 | video = parseVideo(process.env); | |
| 65 | } catch (err) { | |
| 66 | console.error(`${(err as Error).message}; cameras stay MJPEG`); | |
| 67 | } | |
| 68 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video }); | |
| 63 | 69 | const { mode, host: armHost } = relay.link(); |
| 64 | 70 | console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`); |
Software/so101/src/cameras.test.ts +1 −1
| @@ -173,7 +173,7 @@ describe("through the relay", () => { | ||
| 173 | 173 | |
| 174 | 174 | const added = await fetch(`${base}/admin/cameras`, { method: "POST", headers: owner, body: JSON.stringify({ name: "Desk <b>", source: `127.0.0.1:${one.port}` }) }); |
| 175 | 175 | expect(added.status).toBe(201); |
| 176 | expect(await (await fetch(`${base}/cameras`)).json()).toEqual([{ id: 1, name: "Desk <b>", online: null }]); | |
| 176 | expect(await (await fetch(`${base}/cameras`)).json()).toEqual([{ id: 1, name: "Desk <b>", online: null, h264: false }]); | |
| 177 | 177 | const watcher = await fetch(`${base}/camera/1/stream`); |
| 178 | 178 | const reader = watcher.body!.getReader(); |
| 179 | 179 | await reader.read(); |
Software/so101/src/relay-video.test.ts added +93
| @@ -0,0 +1,93 @@ | ||
| 1 | import { afterEach, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 3 | import { createServer, type Server } from "node:http"; | |
| 4 | import { connect } from "node:net"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import WebSocket from "ws"; | |
| 8 | import { CameraHub } from "./cameras.ts"; | |
| 9 | import { clip } from "./h264-clip.ts"; | |
| 10 | import { Lobby } from "./lobby.ts"; | |
| 11 | import { startRelay, type RunningRelay } from "./relay.ts"; | |
| 12 | ||
| 13 | let dir = ""; | |
| 14 | let relay: RunningRelay | undefined; | |
| 15 | let lobby: Lobby | undefined; | |
| 16 | let pi: Server | undefined; | |
| 17 | afterEach(async () => { | |
| 18 | // Bun can keep a server-closed connection from ending (see relay.test.ts): cap the close. | |
| 19 | await Promise.race([relay?.close(), new Promise((r) => setTimeout(r, 500))]); | |
| 20 | lobby?.close(); | |
| 21 | pi?.closeAllConnections(); | |
| 22 | pi?.close(); | |
| 23 | if (dir) rmSync(dir, { recursive: true, force: true }); | |
| 24 | dir = ""; | |
| 25 | relay = lobby = pi = undefined; | |
| 26 | }); | |
| 27 | ||
| 28 | /** A relay with two cameras and a Pi that serves only the 720p rung of camera 1. */ | |
| 29 | async function boot() { | |
| 30 | dir = mkdtempSync(join(tmpdir(), "so101-relay-video-")); | |
| 31 | const pictures = clip(2, 20, 10, 500); | |
| 32 | pi = createServer((req, res) => { | |
| 33 | if (req.url !== "/720p") return void res.writeHead(404).end(); | |
| 34 | res.writeHead(200, { "content-type": "video/h264" }); | |
| 35 | let i = 0; | |
| 36 | const timer = setInterval(() => res.write(pictures[i++ % pictures.length]!), 50); | |
| 37 | res.on("close", () => clearInterval(timer)); | |
| 38 | }); | |
| 39 | await new Promise<void>((r) => pi!.listen(0, "127.0.0.1", r)); | |
| 40 | const piPort = (pi.address() as { port: number }).port; | |
| 41 | lobby = new Lobby({ identityKey: "k".repeat(64), adminFile: join(dir, "admin.json") }); | |
| 42 | const cameras = new CameraHub({ seed: [{ name: "One", source: "http://127.0.0.1:9/stream" }, { name: "Two", source: "http://127.0.0.1:9/stream" }] }); | |
| 43 | relay = await startRelay({ port: 0, simulation: "http://127.0.0.1:1", lobby, cameras, video: { 1: { "720p": `http://127.0.0.1:${piPort}/720p` } }, env: {} }); | |
| 44 | return `127.0.0.1:${relay.port}`; | |
| 45 | } | |
| 46 | ||
| 47 | /** The status line of the answer to a WebSocket upgrade request, read off a bare socket. */ | |
| 48 | const upgrade = (host: string, path: string, headers: Record<string, string> = {}) => | |
| 49 | new Promise<string>((resolve) => { | |
| 50 | const [hostname, port] = host.split(":"); | |
| 51 | const socket = connect(Number(port), hostname!, () => { | |
| 52 | const lines = [`GET ${path} HTTP/1.1`, `Host: ${host}`, "Upgrade: websocket", "Connection: Upgrade", "Sec-WebSocket-Version: 13", "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==", ...Object.entries(headers).map(([k, v]) => `${k}: ${v}`)]; | |
| 53 | socket.write(`${lines.join("\r\n")}\r\n\r\n`); | |
| 54 | }); | |
| 55 | let got = ""; | |
| 56 | socket.on("data", (data) => (got += String(data))); | |
| 57 | socket.on("close", () => resolve(got.split("\r\n")[0]!)); | |
| 58 | socket.on("error", () => resolve(got)); | |
| 59 | setTimeout(() => socket.destroy(), 1500); | |
| 60 | }); | |
| 61 | ||
| 62 | test("/cameras says which camera has H.264", async () => { | |
| 63 | const host = await boot(); | |
| 64 | const list = (await (await fetch(`http://${host}/cameras`)).json()) as { id: number; h264: boolean }[]; | |
| 65 | expect(list.map((c) => [c.id, c.h264])).toEqual([[1, true], [2, false]]); | |
| 66 | }); | |
| 67 | ||
| 68 | test("a guest is served the video of a camera that has a source, with only the rungs that have one, and gets pictures", async () => { | |
| 69 | const host = await boot(); | |
| 70 | const frames: Buffer[] = []; | |
| 71 | const ws = new WebSocket(`ws://${host}/camera/1/video`); | |
| 72 | const ladder = new Promise<{ name: string }[]>((resolve) => | |
| 73 | ws.on("message", (data, binary) => { | |
| 74 | if (binary) frames.push(data as Buffer); | |
| 75 | else resolve(JSON.parse(String(data)).ladder ?? []); | |
| 76 | }), | |
| 77 | ); | |
| 78 | expect((await ladder).map((r) => r.name)).toEqual(["720p"]); | |
| 79 | await new Promise((r) => setTimeout(r, 600)); | |
| 80 | ws.close(); | |
| 81 | expect(frames.length).toBeGreaterThan(3); | |
| 82 | expect(frames[0]![1]).toBe(0); | |
| 83 | }); | |
| 84 | ||
| 85 | // Bun closes a refused upgrade without its status line (relay.test.ts reads a refusal the same way: the socket never opens), | |
| 86 | // so a refusal here is "no 101"; the 404/403 split is in the relay's own order of checks. | |
| 87 | test("a camera with no source, and a cross-origin upgrade, are refused; the same origin is let in", async () => { | |
| 88 | const host = await boot(); | |
| 89 | expect(await upgrade(host, "/camera/2/video")).not.toContain("101"); | |
| 90 | expect(await upgrade(host, "/camera/99/video")).not.toContain("101"); | |
| 91 | expect(await upgrade(host, "/camera/1/video", { Origin: "https://evil.example" })).not.toContain("101"); | |
| 92 | expect(await upgrade(host, "/camera/1/video", { Origin: `http://${host}` })).toContain("101"); | |
| 93 | }); |
Software/so101/src/relay.ts +38 −2
| @@ -40,6 +40,7 @@ | ||
| 40 | 40 | * GET /status the arm's own /status, and which arm said it: "source": "live <host:port>", "reconnecting to the arm: |
| 41 | 41 | * <why>" (the last known status, with "reconnecting": true and "seenAt"; 503 before |
| 42 | 42 | * there is one), or "simulated: <why>" ("simulated: offline, <why>" after 60 s down) |
| 43 | * WS /camera/<id>/video the camera as H.264 access units, a rendition per viewer (src/video-feed.ts; sources from ARM_H264_<n>); 404 when it has none | |
| 43 | 44 | * GET /cameras the listed cameras: [{"id", "name", "online"}] |
| 44 | 45 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 45 | 46 | * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate |
| @@ -67,6 +68,8 @@ import { OFFLINE_FRAME } from "./offline-frame.ts"; | ||
| 67 | 68 | import { handleAdmin } from "./admin-routes.ts"; |
| 68 | 69 | import { parseFps, type FeedOptions } from "./camera-feed.ts"; |
| 69 | 70 | import { CameraError, CameraHub } from "./cameras.ts"; |
| 71 | import { VideoFeed, type VideoSources } from "./video-feed.ts"; | |
| 72 | import { LADDER } from "./video-fanout.ts"; | |
| 70 | 73 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 71 | 74 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| 72 | 75 | import { proxyFor, whyNot } from "./egress.ts"; |
| @@ -118,6 +121,8 @@ export interface RelayOptions { | ||
| 118 | 121 | heartbeatMs?: number; |
| 119 | 122 | /** Timings of the shared camera feeds. */ |
| 120 | 123 | camera?: FeedOptions; |
| 124 | /** The Pi's H.264 streams per camera id (src/video-feed.ts); a camera without one stays MJPEG. */ | |
| 125 | video?: Record<number, VideoSources>; | |
| 121 | 126 | /** Every driver's leader calibration; without one they are kept in memory. */ |
| 122 | 127 | leaders?: LeaderCalibrations; |
| 123 | 128 | /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */ |
| @@ -134,6 +139,7 @@ export interface RunningRelay { | ||
| 134 | 139 | |
| 135 | 140 | const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" }; |
| 136 | 141 | const MAX_BODY = 1_000_000; |
| 142 | const VIDEO = /^\/camera\/(\d{1,9})\/video$/; | |
| 137 | 143 | const CAMERA_WS = /^\/camera\/(\d{1,9})\/ws$/; |
| 138 | 144 | const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/; |
| 139 | 145 | const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/; |
| @@ -227,6 +233,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 227 | 233 | const heartbeatMs = options.heartbeatMs ?? 1000; |
| 228 | 234 | const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); |
| 229 | 235 | const leaders = options.leaders ?? new LeaderCalibrations(); |
| 236 | const videos = new Map<number, VideoFeed>(Object.entries(options.video ?? {}).map(([id, sources]) => [Number(id), new VideoFeed(sources, {}, options.env ?? process.env)])); | |
| 230 | 237 | |
| 231 | 238 | // ------------------------------------------------------------------ the link |
| 232 | 239 | let armLink: ArmLink | null = null; |
| @@ -543,7 +550,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 543 | 550 | const path = url.pathname + url.search; |
| 544 | 551 | const reading = req.method === "GET" || req.method === "HEAD"; |
| 545 | 552 | if (!reading && !sameOrigin(req)) return json(res, 403, { error: "only this page may change the arm" }); |
| 546 | if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online }))); | |
| 553 | if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online, h264: videos.get(id)?.available ?? false }))); | |
| 547 | 554 | if (req.method === "GET" && url.pathname === "/admin/camera-list") { |
| 548 | 555 | res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" }); |
| 549 | 556 | return res.end(CAMERA_PAGE); |
| @@ -772,6 +779,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 772 | 779 | }; |
| 773 | 780 | http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => { |
| 774 | 781 | const url = new URL(req.url ?? "/", "http://x"); |
| 782 | const video = VIDEO.exec(url.pathname); | |
| 775 | 783 | const cameraWs = CAMERA_WS.exec(url.pathname); |
| 776 | 784 | if (cameraWs) { |
| 777 | 785 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); |
| @@ -780,9 +788,16 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 780 | 788 | socket.on("error", () => socket.destroy()); |
| 781 | 789 | return wss.handleUpgrade(req, socket, head, (browser) => watchCamera(id, browser)); |
| 782 | 790 | } |
| 783 | if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); | |
| 791 | if (url.pathname !== "/follow" && url.pathname !== "/machine" && !video && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); | |
| 784 | 792 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); |
| 785 | 793 | socket.on("error", () => socket.destroy()); |
| 794 | if (video) { | |
| 795 | const feed = videos.get(Number(video[1])); | |
| 796 | if (!feed?.available) return refuse(socket, "404 Not Found"); | |
| 797 | const who = lobby?.who(req); // watching is open, like /stream; the driver's picture gets the best rendition | |
| 798 | const driver = !!who && !!lobby?.turns.isDriver(who.id); | |
| 799 | return wss.handleUpgrade(req, socket, head, (browser) => feed.serve(browser, driver)); | |
| 800 | } | |
| 786 | 801 | if (url.pathname === "/live") return lobby!.upgrade(req, socket, head); |
| 787 | 802 | if (url.pathname === "/machine") return machine.upgrade(req, socket, head); |
| 788 | 803 | let driver: string | null = null; |
| @@ -855,6 +870,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 855 | 870 | simLink?.close(); |
| 856 | 871 | machine.close(); |
| 857 | 872 | if (!options.cameras) cameras.close(); |
| 873 | for (const v of videos.values()) v.close(); | |
| 858 | 874 | for (const c of inFlight) c.abort(); |
| 859 | 875 | for (const client of wss.clients) client.terminate(); |
| 860 | 876 | wss.close(); |
| @@ -863,3 +879,23 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 863 | 879 | }), |
| 864 | 880 | }; |
| 865 | 881 | } |
| 882 | ||
| 883 | /** | |
| 884 | * ARM_H264_<n> = "720p=http://pi:8082/h264,480p=http://pi:8083/h264,240p=…" → camera n's rendition streams. | |
| 885 | * Throws on a name outside the ladder or a URL that is not http(s). | |
| 886 | */ | |
| 887 | export function parseVideo(env: Record<string, string | undefined>): Record<number, VideoSources> { | |
| 888 | const out: Record<number, VideoSources> = {}; | |
| 889 | for (const n of [1, 2]) { | |
| 890 | const text = env[`ARM_H264_${n}`]?.trim(); | |
| 891 | if (!text) continue; | |
| 892 | const sources: VideoSources = {}; | |
| 893 | for (const part of text.split(",")) { | |
| 894 | const [name, ...rest] = part.trim().split("="); | |
| 895 | if (!name || !LADDER.some((r) => r.name === name)) throw new Error(`ARM_H264_${n}: "${name}" is not a rung of the ladder (${LADDER.map((r) => r.name).join(", ")})`); | |
| 896 | sources[name] = absolute(rest.join("="), `ARM_H264_${n}`, { "http:": "", "https:": "" }).href; | |
| 897 | } | |
| 898 | out[n] = sources; | |
| 899 | } | |
| 900 | return out; | |
| 901 | } |
Software/so101/src/video-feed.test.ts added +119
| @@ -0,0 +1,119 @@ | ||
| 1 | import { afterEach, describe, expect, test } from "bun:test"; | |
| 2 | import { createServer, type Server } from "node:http"; | |
| 3 | import WebSocket, { WebSocketServer } from "ws"; | |
| 4 | import { clip } from "./h264-clip.ts"; | |
| 5 | import { parseVideo } from "./relay.ts"; | |
| 6 | import { HEADER, LADDER } from "./video-fanout.ts"; | |
| 7 | import { VideoFeed } from "./video-feed.ts"; | |
| 8 | ||
| 9 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 10 | const closers: (() => void)[] = []; | |
| 11 | afterEach(() => closers.splice(0).forEach((c) => c())); | |
| 12 | ||
| 13 | /** The Pi: each rung is an endless Annex-B stream of its own clip, 20 pictures a second, looping. */ | |
| 14 | async function pi(): Promise<string> { | |
| 15 | const clips = new Map(LADDER.map((r) => [`/${r.name}`, clip(2, 20, 10, r.kbps / 3)])); | |
| 16 | const server: Server = createServer((req, res) => { | |
| 17 | const pictures = clips.get(req.url ?? ""); | |
| 18 | if (!pictures) return void res.writeHead(404).end(); | |
| 19 | res.writeHead(200, { "content-type": "video/h264" }); | |
| 20 | let i = 0; | |
| 21 | const timer = setInterval(() => res.write(pictures[i++ % pictures.length]!), 50); | |
| 22 | res.on("close", () => clearInterval(timer)); | |
| 23 | }); | |
| 24 | await new Promise<void>((r) => server.listen(0, "127.0.0.1", r)); | |
| 25 | closers.push(() => (server.closeAllConnections(), server.close())); | |
| 26 | return `http://127.0.0.1:${(server.address() as { port: number }).port}`; | |
| 27 | } | |
| 28 | ||
| 29 | /** A relay-shaped server: every WebSocket is a viewer; `driver` is chosen by the path. */ | |
| 30 | async function relay(feed: VideoFeed): Promise<string> { | |
| 31 | const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" }); | |
| 32 | wss.on("connection", (ws, req) => feed.serve(ws, req.url === "/driver")); | |
| 33 | await new Promise((r) => wss.once("listening", r)); | |
| 34 | closers.push(() => (feed.close(), wss.close())); | |
| 35 | return `ws://127.0.0.1:${(wss.address() as { port: number }).port}`; | |
| 36 | } | |
| 37 | ||
| 38 | function watch(url: string) { | |
| 39 | const got = { frames: [] as { key: boolean; rung: number; sentAt: number; bytes: number }[], texts: [] as any[] }; | |
| 40 | const ws = new WebSocket(url); | |
| 41 | ws.on("message", (data, binary) => { | |
| 42 | const m = data as Buffer; | |
| 43 | if (!binary) return void got.texts.push(JSON.parse(String(data))); | |
| 44 | got.frames.push({ key: (m[0]! & 1) === 1, rung: m[1]!, sentAt: m.readDoubleBE(2), bytes: m.length }); | |
| 45 | }); | |
| 46 | closers.push(() => ws.close()); | |
| 47 | return { ws, got }; | |
| 48 | } | |
| 49 | ||
| 50 | describe("video feed", () => { | |
| 51 | test("a driver and a watcher are fed different renditions of the Pi's streams, from keyframes, and only those are pulled", async () => { | |
| 52 | const base = await pi(); | |
| 53 | const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` }); | |
| 54 | const url = await relay(feed); | |
| 55 | const driver = watch(`${url}/driver`); | |
| 56 | const watcher = watch(`${url}/watcher`); | |
| 57 | await sleep(700); | |
| 58 | expect(driver.got.frames.length).toBeGreaterThan(5); | |
| 59 | expect(driver.got.frames[0]!.key).toBe(true); | |
| 60 | expect(watcher.got.frames[0]!.key).toBe(true); | |
| 61 | expect(new Set(driver.got.frames.map((f) => f.rung))).toEqual(new Set([0])); | |
| 62 | expect(new Set(watcher.got.frames.map((f) => f.rung))).toEqual(new Set([1])); | |
| 63 | expect(driver.got.frames.at(-1)!.bytes).toBeGreaterThan(HEADER); | |
| 64 | expect(Math.abs(Date.now() - driver.got.frames.at(-1)!.sentAt)).toBeLessThan(500); | |
| 65 | expect(feed.fanout.wanted()).toEqual(new Set(["720p", "480p"])); // 240p is not pulled | |
| 66 | expect(driver.got.texts[0].ladder.map((r: { name: string }) => r.name)).toEqual(["720p", "480p", "240p"]); | |
| 67 | }); | |
| 68 | ||
| 69 | test("a viewer pinning a quality gets that rung, switched at a keyframe", async () => { | |
| 70 | const base = await pi(); | |
| 71 | const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` }); | |
| 72 | const { ws, got } = watch(`${await relay(feed)}/watcher`); | |
| 73 | await sleep(400); | |
| 74 | ws.send(JSON.stringify({ quality: "240p" })); | |
| 75 | await sleep(1500); | |
| 76 | const at = got.frames.findIndex((f) => f.rung === 2); | |
| 77 | expect(at).toBeGreaterThan(0); | |
| 78 | expect(got.frames[at]!.key).toBe(true); | |
| 79 | expect(got.frames.slice(at).every((f) => f.rung === 2)).toBe(true); | |
| 80 | }); | |
| 81 | ||
| 82 | test("a feed without sources is not available; a down source just sends nothing", async () => { | |
| 83 | expect(new VideoFeed({}).available).toBe(false); | |
| 84 | const feed = new VideoFeed({ "720p": "http://127.0.0.1:1/x" }); | |
| 85 | expect(feed.available).toBe(true); | |
| 86 | const { got } = watch(`${await relay(feed)}/driver`); | |
| 87 | await sleep(300); | |
| 88 | expect(got.frames).toHaveLength(0); | |
| 89 | expect(got.texts[0].rendition).toBe("720p"); | |
| 90 | }); | |
| 91 | }); | |
| 92 | ||
| 93 | describe("a subset of rungs", () => { | |
| 94 | test("with only 720p configured, a watcher, a pin and a step-down all stay on it and get pictures", async () => { | |
| 95 | const base = await pi(); | |
| 96 | const feed = new VideoFeed({ "720p": `${base}/720p` }); | |
| 97 | expect(feed.fanout.ladder.map((r) => r.name)).toEqual(["720p"]); | |
| 98 | const url = await relay(feed); | |
| 99 | const watcher = watch(`${url}/watcher`); | |
| 100 | await sleep(600); | |
| 101 | expect(watcher.got.texts[0].ladder.map((r: any) => r.name)).toEqual(["720p"]); | |
| 102 | expect(watcher.got.frames.length).toBeGreaterThan(3); | |
| 103 | expect(watcher.got.frames.every((f) => f.rung === 0)).toBe(true); | |
| 104 | watcher.ws.send('{"quality":"240p"}'); | |
| 105 | for (let i = 0; i < 5; i++) (watcher.ws.send('{"age":5000}'), await sleep(30)); | |
| 106 | const before = watcher.got.frames.length; | |
| 107 | await sleep(500); | |
| 108 | expect(watcher.got.frames.length).toBeGreaterThan(before); | |
| 109 | }); | |
| 110 | }); | |
| 111 | ||
| 112 | describe("ARM_H264_n", () => { | |
| 113 | test("names rungs of the ladder with http URLs", () => { | |
| 114 | expect(parseVideo({})).toEqual({}); | |
| 115 | expect(parseVideo({ ARM_H264_1: "720p=pi:8082/h264, 240p=http://pi:8083/h264" })).toEqual({ 1: { "720p": "http://pi:8082/h264", "240p": "http://pi:8083/h264" } }); | |
| 116 | expect(() => parseVideo({ ARM_H264_1: "4k=pi:1" })).toThrow(/ladder/); | |
| 117 | expect(() => parseVideo({ ARM_H264_1: "720p=ftp://pi/x" })).toThrow(); | |
| 118 | }); | |
| 119 | }); |
Software/so101/src/video-feed.ts added +112
| @@ -0,0 +1,112 @@ | ||
| 1 | /** | |
| 2 | * One camera's H.264 over WebSockets: a VideoFanout fed by the Pi's encoder streams. | |
| 3 | * | |
| 4 | * Each rendition is an HTTP stream of Annex-B H.264 on the Pi. It is pulled only while some viewer is on it (or | |
| 5 | * moving to it), so a camera nobody watches costs the Pi nothing, and the lower rungs cost nothing until a viewer | |
| 6 | * needs them. A source that is down is tried again every second while wanted; its viewers simply see no pictures, and | |
| 7 | * the page falls back to MJPEG when none come. | |
| 8 | */ | |
| 9 | ||
| 10 | import { proxyFor } from "./egress.ts"; | |
| 11 | import { AccessUnitSplitter } from "./h264.ts"; | |
| 12 | import { LADDER, VideoFanout, type FanoutOptions, type VideoSink } from "./video-fanout.ts"; | |
| 13 | ||
| 14 | export interface VideoSources { | |
| 15 | /** Rendition name → the URL of its Annex-B stream. */ | |
| 16 | [name: string]: string; | |
| 17 | } | |
| 18 | ||
| 19 | /** The wire the feed talks to a viewer over; a `ws` WebSocket fits. */ | |
| 20 | export interface VideoSocket { | |
| 21 | bufferedAmount: number; | |
| 22 | readyState: number; | |
| 23 | send(data: Uint8Array | string): void; | |
| 24 | on(event: "message", listener: (data: unknown, isBinary: boolean) => void): unknown; | |
| 25 | on(event: "close", listener: () => void): unknown; | |
| 26 | } | |
| 27 | ||
| 28 | const OPEN = 1; | |
| 29 | ||
| 30 | export class VideoFeed { | |
| 31 | readonly fanout: VideoFanout; | |
| 32 | private readonly sources: VideoSources; | |
| 33 | private readonly env: Record<string, string | undefined>; | |
| 34 | private readonly pulls = new Map<string, AbortController>(); | |
| 35 | private timer: ReturnType<typeof setInterval> | undefined; | |
| 36 | private closed = false; | |
| 37 | ||
| 38 | /** The ladder a viewer sees is only the rungs that have a source, so none can start, pin or step onto a rung nothing feeds. */ | |
| 39 | constructor(sources: VideoSources, options: FanoutOptions = {}, env: Record<string, string | undefined> = process.env) { | |
| 40 | this.sources = sources; | |
| 41 | this.env = env; | |
| 42 | this.fanout = new VideoFanout({ ...options, ladder: (options.ladder ?? LADDER).filter((r) => sources[r.name]) }); | |
| 43 | } | |
| 44 | ||
| 45 | /** Whether this feed has an encoder stream for any rung of the ladder. */ | |
| 46 | get available(): boolean { | |
| 47 | return this.fanout.ladder.length > 0; | |
| 48 | } | |
| 49 | ||
| 50 | /** Serve one viewer's WebSocket until it closes. */ | |
| 51 | serve(ws: VideoSocket, driver: boolean): void { | |
| 52 | if (this.closed || !this.available) return; | |
| 53 | const sink: VideoSink = { | |
| 54 | backlog: () => ws.bufferedAmount, | |
| 55 | send: (data) => ws.readyState === OPEN && ws.send(data), | |
| 56 | text: (message) => ws.readyState === OPEN && ws.send(message), | |
| 57 | }; | |
| 58 | const viewer = this.fanout.join(sink, driver); | |
| 59 | ws.on("message", (data, isBinary) => { | |
| 60 | if (!isBinary) viewer.message(String(data)); | |
| 61 | }); | |
| 62 | ws.on("close", () => { | |
| 63 | viewer.leave(); | |
| 64 | if (!this.fanout.watching) this.idle(); | |
| 65 | }); | |
| 66 | this.timer ??= setInterval(() => this.tick(), 1000); | |
| 67 | this.timer.unref?.(); | |
| 68 | this.tick(); | |
| 69 | } | |
| 70 | ||
| 71 | close(): void { | |
| 72 | this.closed = true; | |
| 73 | this.fanout.close(); | |
| 74 | this.idle(); | |
| 75 | } | |
| 76 | ||
| 77 | private idle() { | |
| 78 | clearInterval(this.timer); | |
| 79 | this.timer = undefined; | |
| 80 | for (const c of this.pulls.values()) c.abort(); | |
| 81 | this.pulls.clear(); | |
| 82 | } | |
| 83 | ||
| 84 | private tick() { | |
| 85 | this.fanout.adapt(); | |
| 86 | const wanted = this.fanout.wanted(); | |
| 87 | for (const [name, controller] of this.pulls) { | |
| 88 | if (!wanted.has(name)) (controller.abort(), this.pulls.delete(name)); | |
| 89 | } | |
| 90 | for (const name of wanted) if (this.sources[name] && !this.pulls.has(name)) this.pull(name, this.sources[name]!); | |
| 91 | } | |
| 92 | ||
| 93 | private pull(name: string, url: string) { | |
| 94 | const controller = new AbortController(); | |
| 95 | this.pulls.set(name, controller); | |
| 96 | void (async () => { | |
| 97 | try { | |
| 98 | const proxy = proxyFor(url, this.env); // inside a forge cell the egress proxy is the only way out | |
| 99 | const response = await fetch(url, { signal: controller.signal, cache: "no-store", redirect: "error", ...(proxy ? { proxy } : {}) }); | |
| 100 | if (!response.ok || !response.body) throw new Error(`answered ${response.status}`); | |
| 101 | const splitter = new AccessUnitSplitter(); | |
| 102 | for await (const chunk of response.body as unknown as AsyncIterable<Uint8Array>) { | |
| 103 | for (const unit of splitter.push(chunk)) this.fanout.push(name, unit); | |
| 104 | } | |
| 105 | } catch { | |
| 106 | // down or cut off: the next tick tries again while the rung is wanted | |
| 107 | } finally { | |
| 108 | if (this.pulls.get(name) === controller) this.pulls.delete(name); | |
| 109 | } | |
| 110 | })(); | |
| 111 | } | |
| 112 | } |
forge.json +3
| @@ -13,6 +13,9 @@ | ||
| 13 | 13 | "egress": [ |
| 14 | 14 | "100.89.95.68:8080", |
| 15 | 15 | "100.89.95.68:8081", |
| 16 | "100.89.95.68:8082", | |
| 17 | "100.89.95.68:8083", | |
| 18 | "100.89.95.68:8084", | |
| 16 | 19 | "100.89.95.68:8101" |
| 17 | 20 | ] |
| 18 | 21 | } |
new above
offline