Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
Real video: hardware H.264 from the Pi over the WebSocket, decoded with WebCodecs, smaller streams for slow viewers - a fraction of MJPEG's bytes #207
smith/arm:smith/70-real-video-hardware-h-264-from-the-pi-ov → main 6 commits fork 2 threads · 2 unresolved
17 files
changed +1131
−8
Software/README.md +35
| @@ -218,6 +218,41 @@ again every 2 s and the live picture comes back in the same stream, without | ||
| 218 | 218 | a reload. The camera is independent of the arm server: a camera that is down |
| 219 | 219 | does not put the arm on the simulation, and the other way round. |
| 220 | 220 | |
| 221 | **Real video: H.264 from the Pi.** Where a camera has H.264 renditions, the | |
| 222 | page plays them instead of MJPEG, in a fraction of the bytes. It is opt-in per | |
| 223 | camera (1 and 2 only), set in the app's settings: | |
| 224 | ||
| 225 | ARM_H264_1=720p=http://<pi>:8082/h264,480p=http://<pi>:8083/h264,240p=http://<pi>:8084/h264 | |
| 226 | ||
| 227 | Each entry is `<rung>=<url>`. The ladder is fixed: **720p** (1280x720, 20 fps, | |
| 228 | ~1500 kbps), **480p** (854x480, 15 fps, ~600 kbps) and **240p** (426x240, 10 fps, | |
| 229 | ~200 kbps). List any subset: viewers are only ever offered, started on, pinned to or stepped to the rungs that have a source. A name outside the ladder or a URL that is not | |
| 230 | http(s) stops the app at start with a message. `parseVideo` in `src/relay.ts` | |
| 231 | reads the variable, and `bench/fake-serve.ts`, the entry `forge.json` runs, | |
| 232 | is where it is wired in. | |
| 233 | ||
| 234 | *What the viewer sees.* The relay opens a rendition from the Pi only while | |
| 235 | someone is on it and fans it out over the WebSocket `/camera/<id>/video`, one | |
| 236 | rendition per viewer. A viewer whose connection backs up steps down the ladder | |
| 237 | and, rather than queueing, skips to the next keyframe; the **quality selector** | |
| 238 | on the picture lists the configured rungs and pins one, or leaves it on Auto. The page decodes with | |
| 239 | WebCodecs, loading the player only when `/cameras` reports `"h264": true` for | |
| 240 | the camera. When the browser has no WebCodecs, or no picture arrives in time, | |
| 241 | the page falls back to MJPEG (`/camera/<id>/stream`) by itself. The video | |
| 242 | socket is open to guests, like `/stream`; the driver is given the best | |
| 243 | rendition. Control messages from the viewer are capped at 200 bytes. | |
| 244 | ||
| 245 | *Not in this pull.* The Pi's encoder side: something on the Pi must serve each | |
| 246 | rendition as an H.264 stream at the URL you name (hardware encoder, one port | |
| 247 | per rung). Without it the relay's fetch fails and viewers stay on MJPEG. | |
| 248 | ||
| 249 | *The forge's egress list.* The relay reaches the Pi through the forge's egress | |
| 250 | proxy (`proxyFor` in `src/egress.ts`, the same as the other camera paths), so every rendition's `host:port` must be in `forge.json`'s `egress`. | |
| 251 | It lists 100.89.95.68 on 8080, 8101 and 8082 to 8084 (the example ports above); | |
| 252 | if your encoder listens elsewhere, add those ports. | |
| 253 | ||
| 254 | *The terminal (`ssh`) shows no video, and has no quality choice:* both are page-only. | |
| 255 | ||
| 221 | 256 | **In the browser.** Behind the relay the page draws each camera from |
| 222 | 257 | `WS /camera/<id>/ws`: whole JPEG frames, drawn as they arrive. If no frame |
| 223 | 258 | comes for 2 s the page closes that socket and opens a new one, and it retries |
Software/so101/bench/fake-serve.ts +8 −2
| @@ -22,7 +22,7 @@ import { CameraHub } from "../src/cameras.ts"; | ||
| 22 | 22 | import { LeaderCalibrations } from "../src/leader-calibrations.ts"; |
| 23 | 23 | import { Lobby } from "../src/lobby.ts"; |
| 24 | 24 | import { Moderator, modelFromEnv } from "../src/moderator.ts"; |
| 25 | import { parseRemote, startRelay, type Remote } from "../src/relay.ts"; | |
| 25 | import { parseRemote, parseVideo, startRelay, type Remote } from "../src/relay.ts"; | |
| 26 | 26 | import { startServer } from "../src/server.ts"; |
| 27 | 27 | |
| 28 | 28 | const fake = new FakeBus([1, 2, 3, 4, 5, 6]); |
| @@ -59,6 +59,12 @@ const lobby = new Lobby({ | ||
| 59 | 59 | const cameras = new CameraHub({ file: join(dir, "cameras.json"), seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); |
| 60 | 60 | |
| 61 | 61 | const leaders = new LeaderCalibrations({ file: join(dir, "leader-calibrations.json") }); |
| 62 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders }); | |
| 62 | let video = {}; | |
| 63 | try { | |
| 64 | video = parseVideo(process.env); | |
| 65 | } catch (err) { | |
| 66 | console.error(`${(err as Error).message}; cameras stay MJPEG`); | |
| 67 | } | |
| 68 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video }); | |
| 63 | 69 | const { mode, host: armHost } = relay.link(); |
| 64 | 70 | console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`); |
Software/so101/src/cameras.test.ts +1 −1
| @@ -173,7 +173,7 @@ describe("through the relay", () => { | ||
| 173 | 173 | |
| 174 | 174 | const added = await fetch(`${base}/admin/cameras`, { method: "POST", headers: owner, body: JSON.stringify({ name: "Desk <b>", source: `127.0.0.1:${one.port}` }) }); |
| 175 | 175 | expect(added.status).toBe(201); |
| 176 | expect(await (await fetch(`${base}/cameras`)).json()).toEqual([{ id: 1, name: "Desk <b>", online: null }]); | |
| 176 | expect(await (await fetch(`${base}/cameras`)).json()).toEqual([{ id: 1, name: "Desk <b>", online: null, h264: false }]); | |
| 177 | 177 | const watcher = await fetch(`${base}/camera/1/stream`); |
| 178 | 178 | const reader = watcher.body!.getReader(); |
| 179 | 179 | await reader.read(); |
Software/so101/src/h264-clip.ts added +16
| @@ -0,0 +1,16 @@ | ||
| 1 | /** A stand-in for the Pi's encoder in tests: a short Annex-B clip with the shape of the real one (SPS, PPS, IDR, then P pictures; no B-frames). */ | |
| 2 | ||
| 3 | const nal = (type: number, size: number, fill: number) => Buffer.concat([Buffer.from([0, 0, 0, 1, type | 0x60]), Buffer.alloc(size, fill)]); | |
| 4 | // SPS body: profile 66 (baseline), constraints 0xc0, level 31, then filler that contains no start code. | |
| 5 | const SPS = Buffer.concat([Buffer.from([0, 0, 0, 1, 0x67, 66, 0xc0, 31]), Buffer.alloc(6, 0x11)]); | |
| 6 | const PPS = nal(8, 3, 0x22); | |
| 7 | ||
| 8 | /** One picture: `key` pictures carry the parameter sets; `size` is the slice's payload in bytes. */ | |
| 9 | export function picture(key: boolean, size: number): Buffer { | |
| 10 | return key ? Buffer.concat([SPS, PPS, nal(5, size, 0x33)]) : nal(1, size, 0x44); | |
| 11 | } | |
| 12 | ||
| 13 | /** `seconds` of video at `fps` with a keyframe every `gop` pictures; keyframes are 8x a P picture's size. */ | |
| 14 | export function clip(seconds: number, fps: number, gop: number, deltaBytes: number): Buffer[] { | |
| 15 | return Array.from({ length: Math.round(seconds * fps) }, (_, i) => picture(i % gop === 0, i % gop === 0 ? deltaBytes * 8 : deltaBytes)); | |
| 16 | } |
Software/so101/src/h264.test.ts added +40
| @@ -0,0 +1,40 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { clip } from "./h264-clip.ts"; | |
| 3 | import { AccessUnitSplitter, codecOf } from "./h264.ts"; | |
| 4 | ||
| 5 | describe("access units", () => { | |
| 6 | const pictures = clip(1, 10, 5, 100); | |
| 7 | const stream = Buffer.concat(pictures); | |
| 8 | ||
| 9 | test("a stream cut anywhere comes out as the same pictures, key flags right", () => { | |
| 10 | for (const step of [1, 7, 333, stream.length]) { | |
| 11 | const splitter = new AccessUnitSplitter(); | |
| 12 | const units = []; | |
| 13 | for (let i = 0; i < stream.length; i += step) units.push(...splitter.push(stream.subarray(i, i + step))); | |
| 14 | // the last picture waits for the start of the next | |
| 15 | expect(units.length).toBe(pictures.length - 1); | |
| 16 | units.forEach((u, i) => { | |
| 17 | expect(u.data.equals(pictures[i]!)).toBe(true); | |
| 18 | expect(u.key).toBe(i % 5 === 0); | |
| 19 | }); | |
| 20 | } | |
| 21 | }); | |
| 22 | ||
| 23 | test("a picture is out as soon as the next one begins, not when it ends", () => { | |
| 24 | const splitter = new AccessUnitSplitter(); | |
| 25 | expect(splitter.push(pictures[0]!)).toEqual([]); | |
| 26 | expect(splitter.push(pictures[1]!.subarray(0, 5))).toHaveLength(1); | |
| 27 | }); | |
| 28 | ||
| 29 | test("reads the codec string from the SPS", () => { | |
| 30 | expect(codecOf(pictures[0]!)).toBe("avc1.42c01f"); | |
| 31 | expect(codecOf(pictures[1]!)).toBeNull(); | |
| 32 | }); | |
| 33 | ||
| 34 | test("refuses a stream that never ends a unit", () => { | |
| 35 | const splitter = new AccessUnitSplitter(); | |
| 36 | expect(() => { | |
| 37 | for (let i = 0; i < 6; i++) splitter.push(Buffer.alloc(1 << 20, 7)); | |
| 38 | }).toThrow(); | |
| 39 | }); | |
| 40 | }); |
Software/so101/src/h264.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** | |
| 2 | * H.264 in Annex-B (start-code) form, cut into access units as the bytes arrive. | |
| 3 | * | |
| 4 | * An access unit is one picture: optional parameter sets (SPS/PPS), then its slices. The encoder on the Pi sends one | |
| 5 | * slice per picture and no B-frames, so a picture ends the moment the next NAL that opens one begins: the unit | |
| 6 | * leaves as soon as the next start code and its header byte are seen, never waiting for the next picture to finish. | |
| 7 | */ | |
| 8 | ||
| 9 | export interface AccessUnit { | |
| 10 | /** An IDR picture: a viewer can start decoding here. */ | |
| 11 | key: boolean; | |
| 12 | /** The unit's bytes, start codes included. */ | |
| 13 | data: Buffer; | |
| 14 | } | |
| 15 | ||
| 16 | /** NAL types that open a new access unit once a picture's slice has been seen. */ | |
| 17 | const OPENS = new Set([1, 5, 6, 7, 8, 9]); | |
| 18 | /** No unit is larger than this; a stream that never finds a start code is cut off by throwing. */ | |
| 19 | export const MAX_UNIT = 4 << 20; | |
| 20 | ||
| 21 | export class AccessUnitSplitter { | |
| 22 | private buf: Buffer = Buffer.alloc(0); | |
| 23 | /** Where the current unit starts in `buf`. */ | |
| 24 | private from = 0; | |
| 25 | /** Where to look for the next start code. */ | |
| 26 | private scan = 0; | |
| 27 | private picture = false; | |
| 28 | private key = false; | |
| 29 | ||
| 30 | push(chunk: Uint8Array): AccessUnit[] { | |
| 31 | this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : Buffer.from(chunk); | |
| 32 | const out: AccessUnit[] = []; | |
| 33 | const b = this.buf; | |
| 34 | for (let i = this.scan; i + 3 < b.length; i++) { | |
| 35 | if (b[i + 2]! > 1) i += 2; // cannot be inside a start code | |
| 36 | else if (b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 1) { | |
| 37 | const type = b[i + 3]! & 0x1f; | |
| 38 | const start = i > this.from && b[i - 1] === 0 ? i - 1 : i; | |
| 39 | if (this.picture && OPENS.has(type) && start > this.from) { | |
| 40 | out.push({ key: this.key, data: Buffer.from(b.subarray(this.from, start)) }); | |
| 41 | this.from = start; | |
| 42 | this.picture = this.key = false; | |
| 43 | } | |
| 44 | if (type === 1 || type === 5) this.picture = true; | |
| 45 | if (type === 5) this.key = true; | |
| 46 | i += 3; | |
| 47 | this.scan = i + 1; | |
| 48 | } | |
| 49 | } | |
| 50 | this.scan = Math.max(this.scan, b.length - 3); | |
| 51 | if (this.from > 0) { | |
| 52 | this.buf = Buffer.from(b.subarray(this.from)); | |
| 53 | this.scan -= this.from; | |
| 54 | this.from = 0; | |
| 55 | } | |
| 56 | if (this.buf.length > MAX_UNIT) throw new Error("an H.264 unit is too long"); | |
| 57 | return out; | |
| 58 | } | |
| 59 | } | |
| 60 | ||
| 61 | /** The codec string WebCodecs wants ("avc1.PPCCLL"), from the first SPS in a unit, or null when it has none. */ | |
| 62 | export function codecOf(unit: Uint8Array): string | null { | |
| 63 | for (let i = 0; i + 6 < unit.length; i++) { | |
| 64 | if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3]! & 0x1f) === 7) { | |
| 65 | const hex = (n: number) => n.toString(16).padStart(2, "0"); | |
| 66 | return `avc1.${hex(unit[i + 4]!)}${hex(unit[i + 5]!)}${hex(unit[i + 6]!)}`; | |
| 67 | } | |
| 68 | } | |
| 69 | return null; | |
| 70 | } |
Software/so101/src/relay-video.test.ts added +93
| @@ -0,0 +1,93 @@ | ||
| 1 | import { afterEach, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 3 | import { createServer, type Server } from "node:http"; | |
| 4 | import { connect } from "node:net"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import WebSocket from "ws"; | |
| 8 | import { CameraHub } from "./cameras.ts"; | |
| 9 | import { clip } from "./h264-clip.ts"; | |
| 10 | import { Lobby } from "./lobby.ts"; | |
| 11 | import { startRelay, type RunningRelay } from "./relay.ts"; | |
| 12 | ||
| 13 | let dir = ""; | |
| 14 | let relay: RunningRelay | undefined; | |
| 15 | let lobby: Lobby | undefined; | |
| 16 | let pi: Server | undefined; | |
| 17 | afterEach(async () => { | |
| 18 | // Bun can keep a server-closed connection from ending (see relay.test.ts): cap the close. | |
| 19 | await Promise.race([relay?.close(), new Promise((r) => setTimeout(r, 500))]); | |
| 20 | lobby?.close(); | |
| 21 | pi?.closeAllConnections(); | |
| 22 | pi?.close(); | |
| 23 | if (dir) rmSync(dir, { recursive: true, force: true }); | |
| 24 | dir = ""; | |
| 25 | relay = lobby = pi = undefined; | |
| 26 | }); | |
| 27 | ||
| 28 | /** A relay with two cameras and a Pi that serves only the 720p rung of camera 1. */ | |
| 29 | async function boot() { | |
| 30 | dir = mkdtempSync(join(tmpdir(), "so101-relay-video-")); | |
| 31 | const pictures = clip(2, 20, 10, 500); | |
| 32 | pi = createServer((req, res) => { | |
| 33 | if (req.url !== "/720p") return void res.writeHead(404).end(); | |
| 34 | res.writeHead(200, { "content-type": "video/h264" }); | |
| 35 | let i = 0; | |
| 36 | const timer = setInterval(() => res.write(pictures[i++ % pictures.length]!), 50); | |
| 37 | res.on("close", () => clearInterval(timer)); | |
| 38 | }); | |
| 39 | await new Promise<void>((r) => pi!.listen(0, "127.0.0.1", r)); | |
| 40 | const piPort = (pi.address() as { port: number }).port; | |
| 41 | lobby = new Lobby({ identityKey: "k".repeat(64), adminFile: join(dir, "admin.json") }); | |
| 42 | const cameras = new CameraHub({ seed: [{ name: "One", source: "http://127.0.0.1:9/stream" }, { name: "Two", source: "http://127.0.0.1:9/stream" }] }); | |
| 43 | relay = await startRelay({ port: 0, simulation: "http://127.0.0.1:1", lobby, cameras, video: { 1: { "720p": `http://127.0.0.1:${piPort}/720p` } }, env: {} }); | |
| 44 | return `127.0.0.1:${relay.port}`; | |
| 45 | } | |
| 46 | ||
| 47 | /** The status line of the answer to a WebSocket upgrade request, read off a bare socket. */ | |
| 48 | const upgrade = (host: string, path: string, headers: Record<string, string> = {}) => | |
| 49 | new Promise<string>((resolve) => { | |
| 50 | const [hostname, port] = host.split(":"); | |
| 51 | const socket = connect(Number(port), hostname!, () => { | |
| 52 | const lines = [`GET ${path} HTTP/1.1`, `Host: ${host}`, "Upgrade: websocket", "Connection: Upgrade", "Sec-WebSocket-Version: 13", "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==", ...Object.entries(headers).map(([k, v]) => `${k}: ${v}`)]; | |
| 53 | socket.write(`${lines.join("\r\n")}\r\n\r\n`); | |
| 54 | }); | |
| 55 | let got = ""; | |
| 56 | socket.on("data", (data) => (got += String(data))); | |
| 57 | socket.on("close", () => resolve(got.split("\r\n")[0]!)); | |
| 58 | socket.on("error", () => resolve(got)); | |
| 59 | setTimeout(() => socket.destroy(), 1500); | |
| 60 | }); | |
| 61 | ||
| 62 | test("/cameras says which camera has H.264", async () => { | |
| 63 | const host = await boot(); | |
| 64 | const list = (await (await fetch(`http://${host}/cameras`)).json()) as { id: number; h264: boolean }[]; | |
| 65 | expect(list.map((c) => [c.id, c.h264])).toEqual([[1, true], [2, false]]); | |
| 66 | }); | |
| 67 | ||
| 68 | test("a guest is served the video of a camera that has a source, with only the rungs that have one, and gets pictures", async () => { | |
| 69 | const host = await boot(); | |
| 70 | const frames: Buffer[] = []; | |
| 71 | const ws = new WebSocket(`ws://${host}/camera/1/video`); | |
| 72 | const ladder = new Promise<{ name: string }[]>((resolve) => | |
| 73 | ws.on("message", (data, binary) => { | |
| 74 | if (binary) frames.push(data as Buffer); | |
| 75 | else resolve(JSON.parse(String(data)).ladder ?? []); | |
| 76 | }), | |
| 77 | ); | |
| 78 | expect((await ladder).map((r) => r.name)).toEqual(["720p"]); | |
| 79 | await new Promise((r) => setTimeout(r, 600)); | |
| 80 | ws.close(); | |
| 81 | expect(frames.length).toBeGreaterThan(3); | |
| 82 | expect(frames[0]![1]).toBe(0); | |
| 83 | }); | |
| 84 | ||
| 85 | // Bun closes a refused upgrade without its status line (relay.test.ts reads a refusal the same way: the socket never opens), | |
| 86 | // so a refusal here is "no 101"; the 404/403 split is in the relay's own order of checks. | |
| 87 | test("a camera with no source, and a cross-origin upgrade, are refused; the same origin is let in", async () => { | |
| 88 | const host = await boot(); | |
| 89 | expect(await upgrade(host, "/camera/2/video")).not.toContain("101"); | |
| 90 | expect(await upgrade(host, "/camera/99/video")).not.toContain("101"); | |
| 91 | expect(await upgrade(host, "/camera/1/video", { Origin: "https://evil.example" })).not.toContain("101"); | |
| 92 | expect(await upgrade(host, "/camera/1/video", { Origin: `http://${host}` })).toContain("101"); | |
| 93 | }); |
Software/so101/src/relay.ts +38 −2
| @@ -40,6 +40,7 @@ | ||
| 40 | 40 | * GET /status the arm's own /status, and which arm said it: "source": "live <host:port>", "reconnecting to the arm: |
| 41 | 41 | * <why>" (the last known status, with "reconnecting": true and "seenAt"; 503 before |
| 42 | 42 | * there is one), or "simulated: <why>" ("simulated: offline, <why>" after 60 s down) |
| 43 | * WS /camera/<id>/video the camera as H.264 access units, a rendition per viewer (src/video-feed.ts; sources from ARM_H264_<n>); 404 when it has none | |
| 43 | 44 | * GET /cameras the listed cameras: [{"id", "name", "online"}] |
| 44 | 45 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 45 | 46 | * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate |
| @@ -67,6 +68,8 @@ import { OFFLINE_FRAME } from "./offline-frame.ts"; | ||
| 67 | 68 | import { handleAdmin } from "./admin-routes.ts"; |
| 68 | 69 | import { parseFps, type FeedOptions } from "./camera-feed.ts"; |
| 69 | 70 | import { CameraError, CameraHub } from "./cameras.ts"; |
| 71 | import { VideoFeed, type VideoSources } from "./video-feed.ts"; | |
| 72 | import { LADDER } from "./video-fanout.ts"; | |
| 70 | 73 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 71 | 74 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| 72 | 75 | import { proxyFor, whyNot } from "./egress.ts"; |
| @@ -118,6 +121,8 @@ export interface RelayOptions { | ||
| 118 | 121 | heartbeatMs?: number; |
| 119 | 122 | /** Timings of the shared camera feeds. */ |
| 120 | 123 | camera?: FeedOptions; |
| 124 | /** The Pi's H.264 streams per camera id (src/video-feed.ts); a camera without one stays MJPEG. */ | |
| 125 | video?: Record<number, VideoSources>; | |
| 121 | 126 | /** Every driver's leader calibration; without one they are kept in memory. */ |
| 122 | 127 | leaders?: LeaderCalibrations; |
| 123 | 128 | /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */ |
| @@ -134,6 +139,7 @@ export interface RunningRelay { | ||
| 134 | 139 | |
| 135 | 140 | const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" }; |
| 136 | 141 | const MAX_BODY = 1_000_000; |
| 142 | const VIDEO = /^\/camera\/(\d{1,9})\/video$/; | |
| 137 | 143 | const CAMERA_WS = /^\/camera\/(\d{1,9})\/ws$/; |
| 138 | 144 | const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/; |
| 139 | 145 | const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/; |
| @@ -227,6 +233,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 227 | 233 | const heartbeatMs = options.heartbeatMs ?? 1000; |
| 228 | 234 | const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); |
| 229 | 235 | const leaders = options.leaders ?? new LeaderCalibrations(); |
| 236 | const videos = new Map<number, VideoFeed>(Object.entries(options.video ?? {}).map(([id, sources]) => [Number(id), new VideoFeed(sources, {}, options.env ?? process.env)])); | |
| 230 | 237 | |
| 231 | 238 | // ------------------------------------------------------------------ the link |
| 232 | 239 | let armLink: ArmLink | null = null; |
| @@ -543,7 +550,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 543 | 550 | const path = url.pathname + url.search; |
| 544 | 551 | const reading = req.method === "GET" || req.method === "HEAD"; |
| 545 | 552 | if (!reading && !sameOrigin(req)) return json(res, 403, { error: "only this page may change the arm" }); |
| 546 | if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online }))); | |
| 553 | if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online, h264: videos.get(id)?.available ?? false }))); | |
| 547 | 554 | if (req.method === "GET" && url.pathname === "/admin/camera-list") { |
| 548 | 555 | res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" }); |
| 549 | 556 | return res.end(CAMERA_PAGE); |
| @@ -772,6 +779,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 772 | 779 | }; |
| 773 | 780 | http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => { |
| 774 | 781 | const url = new URL(req.url ?? "/", "http://x"); |
| 782 | const video = VIDEO.exec(url.pathname); | |
| 775 | 783 | const cameraWs = CAMERA_WS.exec(url.pathname); |
| 776 | 784 | if (cameraWs) { |
| 777 | 785 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); |
| @@ -780,9 +788,16 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 780 | 788 | socket.on("error", () => socket.destroy()); |
| 781 | 789 | return wss.handleUpgrade(req, socket, head, (browser) => watchCamera(id, browser)); |
| 782 | 790 | } |
| 783 | if (url.pathname !== "/follow" && url.pathname !== "/machine" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); | |
| 791 | if (url.pathname !== "/follow" && url.pathname !== "/machine" && !video && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); | |
| 784 | 792 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); |
| 785 | 793 | socket.on("error", () => socket.destroy()); |
| 794 | if (video) { | |
| 795 | const feed = videos.get(Number(video[1])); | |
| 796 | if (!feed?.available) return refuse(socket, "404 Not Found"); | |
| 797 | const who = lobby?.who(req); // watching is open, like /stream; the driver's picture gets the best rendition | |
| 798 | const driver = !!who && !!lobby?.turns.isDriver(who.id); | |
| 799 | return wss.handleUpgrade(req, socket, head, (browser) => feed.serve(browser, driver)); | |
| 800 | } | |
| 786 | 801 | if (url.pathname === "/live") return lobby!.upgrade(req, socket, head); |
| 787 | 802 | if (url.pathname === "/machine") return machine.upgrade(req, socket, head); |
| 788 | 803 | let driver: string | null = null; |
| @@ -855,6 +870,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 855 | 870 | simLink?.close(); |
| 856 | 871 | machine.close(); |
| 857 | 872 | if (!options.cameras) cameras.close(); |
| 873 | for (const v of videos.values()) v.close(); | |
| 858 | 874 | for (const c of inFlight) c.abort(); |
| 859 | 875 | for (const client of wss.clients) client.terminate(); |
| 860 | 876 | wss.close(); |
| @@ -863,3 +879,23 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 863 | 879 | }), |
| 864 | 880 | }; |
| 865 | 881 | } |
| 882 | ||
| 883 | /** | |
| 884 | * ARM_H264_<n> = "720p=http://pi:8082/h264,480p=http://pi:8083/h264,240p=…" → camera n's rendition streams. | |
| 885 | * Throws on a name outside the ladder or a URL that is not http(s). | |
| 886 | */ | |
| 887 | export function parseVideo(env: Record<string, string | undefined>): Record<number, VideoSources> { | |
| 888 | const out: Record<number, VideoSources> = {}; | |
| 889 | for (const n of [1, 2]) { | |
| 890 | const text = env[`ARM_H264_${n}`]?.trim(); | |
| 891 | if (!text) continue; | |
| 892 | const sources: VideoSources = {}; | |
| 893 | for (const part of text.split(",")) { | |
| 894 | const [name, ...rest] = part.trim().split("="); | |
| 895 | if (!name || !LADDER.some((r) => r.name === name)) throw new Error(`ARM_H264_${n}: "${name}" is not a rung of the ladder (${LADDER.map((r) => r.name).join(", ")})`); | |
| 896 | sources[name] = absolute(rest.join("="), `ARM_H264_${n}`, { "http:": "", "https:": "" }).href; | |
| 897 | } | |
| 898 | out[n] = sources; | |
| 899 | } | |
| 900 | return out; | |
| 901 | } |
Software/so101/src/ui/video-pane.js added +23
| @@ -0,0 +1,23 @@ | ||
| 1 | // The H.264 picture of one camera pane: a canvas and a quality chip, driven by a VideoPlayer. Loaded only for a viewer who gets H.264. | |
| 2 | import { VideoPlayer } from "./video-player.js"; | |
| 3 | ||
| 4 | /** Returns false when the pane is gone. `fallback` runs when no picture comes. @param {any} c @param {{ el: Function, loc: { protocol: string, host: string }, ok: () => boolean, fallback: () => void }} d */ | |
| 5 | export function mountVideo(c, { el, loc, ok, fallback }) { | |
| 6 | if (!ok() || c.player || c.canvas) return; | |
| 7 | c.canvas = el("canvas", { ariaLabel: c.img.alt }); | |
| 8 | c.chip = el("select", { className: "quality", title: "Video quality" }); | |
| 9 | c.chip.onchange = () => c.player?.quality(c.chip.value); | |
Add ariaLabel: "Video quality" to this select. A title is not a reliable name for a screen reader, and the chip needs a thumb-sized target on a phone. |
||
| 10 | c.pane.append(c.canvas, c.chip); | |
| 11 | const player = c.player = new VideoPlayer({ | |
| 12 | url: `${loc.protocol === "https:" ? "wss:" : "ws:"}//${loc.host}/camera/${c.id}/video`, | |
| 13 | canvas: c.canvas, | |
| 14 | onStatus: (s) => { | |
| 15 | if (c.player !== player) return; | |
| 16 | if (s.state === "none") return fallback(); | |
| 17 | c.img.style.visibility = "hidden"; | |
| 18 | if (c.chip.options.length !== (s.ladder ?? []).length + 1) c.chip.replaceChildren(...["auto", ...(s.ladder ?? []).map((r) => r.name)].map((name) => el("option", { value: name, textContent: name }))); | |
| 19 | c.tag.textContent = `${c.name} · ${s.rendition ?? ""} · ${Math.round(s.age ?? 0)} ms`; | |
| 20 | }, | |
| 21 | }); | |
| 22 | player.start(); | |
| 23 | } |
Software/so101/src/ui/video-player.js added +131
| @@ -0,0 +1,131 @@ | ||
| 1 | // @ts-nocheck -- WebCodecs and the DOM are not in this project's lib; the tests drive the class with fakes | |
| 2 | /** | |
| 3 | * The camera as H.264 over the relay's WebSocket (/camera/<id>/video), decoded with WebCodecs into a canvas. | |
| 4 | * Plain ESM, served at /ui/ and imported by the tests. The wire format is documented in src/video-fanout.ts: | |
| 5 | * 10 header bytes (keyframe flag, rendition index, relay time float64) then one Annex-B access unit. | |
| 6 | * | |
| 7 | * Nothing is queued: a picture that arrives while the decoder is behind is dropped, and so is every one after it | |
| 8 | * until the next keyframe. `onStatus` hears "live", "none" (no picture in `waitMs`: use MJPEG) and the frame age. | |
| 9 | */ | |
| 10 | ||
| 11 | export const HEADER = 10; | |
| 12 | ||
| 13 | /** @param {ArrayBuffer | Uint8Array} data */ | |
| 14 | export function parseFrame(data) { | |
| 15 | const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); | |
| 16 | if (bytes.length <= HEADER) return null; | |
| 17 | const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); | |
| 18 | return { key: (bytes[0] & 1) === 1, rendition: bytes[1], sentAt: view.getFloat64(2), data: bytes.subarray(HEADER) }; | |
| 19 | } | |
| 20 | ||
| 21 | /** "avc1.PPCCLL" from the first SPS of an Annex-B unit, or null. @param {Uint8Array} unit */ | |
| 22 | export function codecOf(unit) { | |
| 23 | for (let i = 0; i + 6 < unit.length; i++) { | |
| 24 | if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3] & 0x1f) === 7) { | |
| 25 | return "avc1." + [4, 5, 6].map((n) => unit[i + n].toString(16).padStart(2, "0")).join(""); | |
| 26 | } | |
| 27 | } | |
| 28 | return null; | |
| 29 | } | |
| 30 | ||
| 31 | export const canDecode = () => typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function" && typeof WebSocket === "function"; | |
| 32 | ||
| 33 | export class VideoPlayer { | |
| 34 | /** | |
| 35 | * @param {{ url: string, canvas: { width: number, height: number, getContext(k: string): any }, onStatus?: (s: { state: string, age?: number, rendition?: string, ladder?: any[] }) => void, waitMs?: number, | |
| 36 | * deps?: { WebSocket?: any, VideoDecoder?: any, EncodedVideoChunk?: any, now?: () => number } }} o | |
| 37 | */ | |
| 38 | constructor(o) { | |
| 39 | this.o = o; | |
| 40 | const d = o.deps ?? {}; | |
| 41 | this.WS = d.WebSocket ?? WebSocket; | |
| 42 | this.Decoder = d.VideoDecoder ?? (typeof VideoDecoder === "undefined" ? undefined : VideoDecoder); | |
| 43 | this.Chunk = d.EncodedVideoChunk ?? (typeof EncodedVideoChunk === "undefined" ? undefined : EncodedVideoChunk); | |
| 44 | this.now = d.now ?? (() => Date.now()); | |
| 45 | this.ctx = o.canvas.getContext("2d"); | |
| 46 | this.decoder = null; | |
| 47 | this.configured = false; | |
| 48 | this.needKey = true; | |
| 49 | this.lowest = Infinity; | |
| 50 | this.age = 0; | |
| 51 | this.frames = 0; | |
| 52 | this.ladder = []; | |
| 53 | this.closed = false; | |
| 54 | this.ws = null; | |
| 55 | this.timers = []; | |
| 56 | } | |
| 57 | ||
| 58 | start() { | |
| 59 | const ws = (this.ws = new this.WS(this.o.url)); | |
| 60 | ws.binaryType = "arraybuffer"; | |
| 61 | ws.onmessage = (e) => (typeof e.data === "string" ? this.control(e.data) : this.frame(e.data)); | |
| 62 | ws.onclose = () => this.status("none"); | |
| 63 | this.timers.push(setTimeout(() => this.frames || this.status("none"), this.o.waitMs ?? 3000)); | |
| 64 | this.timers.push(setInterval(() => this.report(), 1000)); | |
| 65 | } | |
| 66 | ||
| 67 | /** Pin a rung by name, or "auto". @param {string} quality */ | |
| 68 | quality(quality) { | |
| 69 | if (this.ws?.readyState === 1) this.ws.send(JSON.stringify({ quality })); | |
| 70 | } | |
| 71 | ||
| 72 | stop() { | |
| 73 | this.closed = true; | |
| 74 | this.timers.forEach((t) => (clearTimeout(t), clearInterval(t))); | |
| 75 | try { this.ws?.close(); } catch {} | |
| 76 | try { this.decoder?.close(); } catch {} | |
| 77 | } | |
| 78 | ||
| 79 | status(state) { | |
| 80 | this.o.onStatus?.({ state, age: this.age, rendition: this.rendition, ladder: this.ladder }); | |
| 81 | } | |
| 82 | ||
| 83 | control(text) { | |
| 84 | let m; | |
| 85 | try { m = JSON.parse(text); } catch { return; } | |
| 86 | if (Array.isArray(m.ladder)) this.ladder = m.ladder; | |
| 87 | if (typeof m.rendition === "string") this.rendition = m.rendition; | |
| 88 | if (this.frames) this.status("live"); | |
| 89 | } | |
| 90 | ||
| 91 | report() { | |
| 92 | if (this.ws?.readyState === 1 && this.age > 0) this.ws.send(JSON.stringify({ age: Math.round(this.age) })); | |
| 93 | } | |
| 94 | ||
| 95 | frame(buffer) { | |
| 96 | const f = parseFrame(buffer); | |
| 97 | if (!f || this.closed) return; | |
| 98 | // Age is measured against the least delay seen, since the relay's clock and the page's differ: it is the queueing, not the distance. | |
| 99 | const delay = this.now() - f.sentAt; | |
| 100 | this.lowest = Math.min(this.lowest, delay); | |
| 101 | this.age = Math.max(0, delay - this.lowest); | |
| 102 | if (this.decoder && this.decoder.decodeQueueSize > 3) this.needKey = true; // behind: skip to the next keyframe | |
| 103 | if (this.needKey && !f.key) return; | |
| 104 | if (f.key && !this.configured && !this.configure(f.data)) return; | |
| 105 | this.needKey = false; | |
| 106 | try { | |
| 107 | this.decoder.decode(new this.Chunk({ type: f.key ? "key" : "delta", timestamp: Math.round(f.sentAt * 1000), data: f.data })); | |
| 108 | } catch { | |
| 109 | this.needKey = true; | |
| 110 | this.configured = false; | |
| 111 | } | |
| 112 | } | |
| 113 | ||
| 114 | configure(unit) { | |
| 115 | const codec = codecOf(unit); | |
| 116 | if (!codec || !this.Decoder) return false; | |
| 117 | try { this.decoder?.close(); } catch {} | |
| 118 | this.decoder = new this.Decoder({ | |
| 119 | output: (frame) => { | |
| 120 | const c = this.o.canvas; | |
| 121 | if (c.width !== frame.displayWidth || c.height !== frame.displayHeight) (c.width = frame.displayWidth), (c.height = frame.displayHeight); | |
| 122 | this.ctx.drawImage(frame, 0, 0); | |
| 123 | frame.close(); | |
| 124 | if (!this.frames++) this.status("live"); | |
| 125 | }, | |
| 126 | error: () => ((this.needKey = true), (this.configured = false)), | |
| 127 | }); | |
| 128 | this.decoder.configure({ codec, optimizeForLatency: true }); | |
| 129 | return (this.configured = true); | |
| 130 | } | |
| 131 | } |
Software/so101/src/ui/video-player.test.ts added +111
| @@ -0,0 +1,111 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { clip } from "../h264-clip.ts"; | |
| 3 | import { codecOf, parseFrame, VideoPlayer } from "./video-player.js"; | |
| 4 | ||
| 5 | const frame = (key: boolean, rung: number, sentAt: number, data: Uint8Array) => { | |
| 6 | const out = Buffer.alloc(10 + data.length); | |
| 7 | out[0] = key ? 1 : 0; | |
| 8 | out[1] = rung; | |
| 9 | out.writeDoubleBE(sentAt, 2); | |
| 10 | out.set(data, 10); | |
| 11 | return out; | |
| 12 | }; | |
| 13 | ||
| 14 | class FakeSocket { | |
| 15 | static last: FakeSocket; | |
| 16 | readyState = 1; | |
| 17 | sent: string[] = []; | |
| 18 | onmessage: ((e: { data: unknown }) => void) | null = null; | |
| 19 | onclose: (() => void) | null = null; | |
| 20 | url: string; | |
| 21 | constructor(url: string) { | |
| 22 | this.url = url; | |
| 23 | FakeSocket.last = this; | |
| 24 | } | |
| 25 | send(s: string) { | |
| 26 | this.sent.push(s); | |
| 27 | } | |
| 28 | close() {} | |
| 29 | } | |
| 30 | ||
| 31 | function setup() { | |
| 32 | const decoded: { type: string; timestamp: number; bytes: number }[] = []; | |
| 33 | const states: string[] = []; | |
| 34 | let configured: unknown; | |
| 35 | let output: (f: unknown) => void = () => {}; | |
| 36 | const drawn: unknown[] = []; | |
| 37 | class FakeDecoder { | |
| 38 | decodeQueueSize = 0; | |
| 39 | constructor(init: { output: (f: unknown) => void }) { | |
| 40 | output = init.output; | |
| 41 | } | |
| 42 | configure(c: unknown) { | |
| 43 | configured = c; | |
| 44 | } | |
| 45 | decode(chunk: { type: string; timestamp: number; data: Uint8Array }) { | |
| 46 | decoded.push({ type: chunk.type, timestamp: chunk.timestamp, bytes: chunk.data.length }); | |
| 47 | output({ displayWidth: 1280, displayHeight: 720, close() {} }); | |
| 48 | } | |
| 49 | close() {} | |
| 50 | } | |
| 51 | const canvas = { width: 0, height: 0, getContext: () => ({ drawImage: (f: unknown) => drawn.push(f) }) }; | |
| 52 | let now = 10_000; | |
| 53 | const player = new VideoPlayer({ url: "ws://x/video", canvas, onStatus: (s) => states.push(s.state), deps: { WebSocket: FakeSocket, VideoDecoder: FakeDecoder, EncodedVideoChunk: class { constructor(init: object) { Object.assign(this, init); } }, now: () => now } }); | |
| 54 | player.start(); | |
| 55 | const ws = FakeSocket.last; | |
| 56 | return { player, ws, decoded, states, drawn, canvas, get configured() { return configured; }, tick: (ms: number) => (now += ms), set: (n: number) => (now = n), decoder: () => player.decoder }; | |
| 57 | } | |
| 58 | ||
| 59 | describe("video player", () => { | |
| 60 | const pics = clip(1, 10, 5, 100); | |
| 61 | ||
| 62 | test("parses the wire frame and the codec", () => { | |
| 63 | const f = parseFrame(frame(true, 2, 1234.5, pics[0]!))!; | |
| 64 | expect(f).toMatchObject({ key: true, rendition: 2, sentAt: 1234.5 }); | |
| 65 | expect(f.data.length).toBe(pics[0]!.length); | |
| 66 | expect(parseFrame(new Uint8Array(10))).toBeNull(); | |
| 67 | expect(codecOf(pics[0]!)).toBe("avc1.42c01f"); | |
| 68 | }); | |
| 69 | ||
| 70 | test("starts at a keyframe, configures from the SPS, decodes to the canvas", () => { | |
| 71 | const t = setup(); | |
| 72 | t.ws.onmessage!({ data: JSON.stringify({ ladder: [{ name: "720p" }], rendition: "720p" }) }); | |
| 73 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); // joined mid-GOP: ignored | |
| 74 | expect(t.decoded).toHaveLength(0); | |
| 75 | t.ws.onmessage!({ data: frame(true, 0, 10_000, pics[0]!) }); | |
| 76 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); | |
| 77 | expect(t.configured).toEqual({ codec: "avc1.42c01f", optimizeForLatency: true }); | |
| 78 | expect(t.decoded.map((d) => d.type)).toEqual(["key", "delta"]); | |
| 79 | expect(t.drawn).toHaveLength(2); | |
| 80 | expect(t.canvas.width).toBe(1280); | |
| 81 | expect(t.states).toContain("live"); | |
| 82 | t.player.stop(); | |
| 83 | }); | |
| 84 | ||
| 85 | test("a decoder that falls behind skips to the next keyframe; age is the delay above the best seen", () => { | |
| 86 | const t = setup(); | |
| 87 | t.ws.onmessage!({ data: frame(true, 0, 9_950, pics[0]!) }); // 50 ms away | |
| 88 | t.ws.onmessage!({ data: frame(false, 0, 9_950, pics[1]!) }); | |
| 89 | t.decoder().decodeQueueSize = 9; | |
| 90 | t.ws.onmessage!({ data: frame(false, 0, 9_800, pics[2]!) }); // 200 ms old: 150 ms above the best | |
| 91 | expect(t.player.age).toBe(150); | |
| 92 | expect(t.decoded).toHaveLength(2); // dropped | |
| 93 | t.decoder().decodeQueueSize = 0; | |
| 94 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[3]!) }); // still no key | |
| 95 | expect(t.decoded).toHaveLength(2); | |
| 96 | t.ws.onmessage!({ data: frame(true, 0, 9_850, pics[5]!) }); // 150 ms away; the best seen is 0 by now | |
| 97 | expect(t.decoded).toHaveLength(3); | |
| 98 | t.player.report(); | |
| 99 | expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ age: 150 }); | |
| 100 | t.player.quality("240p"); | |
| 101 | expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ quality: "240p" }); | |
| 102 | t.player.stop(); | |
| 103 | }); | |
| 104 | ||
| 105 | test("says none when the socket closes before any picture", () => { | |
| 106 | const t = setup(); | |
| 107 | t.ws.onclose!(); | |
| 108 | expect(t.states).toEqual(["none"]); | |
| 109 | t.player.stop(); | |
| 110 | }); | |
| 111 | }); |
Software/so101/src/video-fanout.test.ts added +115
| @@ -0,0 +1,115 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { clip } from "./h264-clip.ts"; | |
| 3 | import { HEADER, LADDER, VideoFanout, type VideoSink } from "./video-fanout.ts"; | |
| 4 | ||
| 5 | function sink(): VideoSink & { sent: Buffer[]; texts: any[]; queued: number } { | |
| 6 | const s = { | |
| 7 | sent: [] as Buffer[], | |
| 8 | texts: [] as any[], | |
| 9 | queued: 0, | |
| 10 | backlog: () => s.queued, | |
| 11 | send: (d: Uint8Array) => void s.sent.push(Buffer.from(d)), | |
| 12 | text: (t: string) => void s.texts.push(JSON.parse(t)), | |
| 13 | }; | |
| 14 | return s; | |
| 15 | } | |
| 16 | const frames = (s: { sent: Buffer[] }) => s.sent.map((m) => ({ key: (m[0]! & 1) === 1, rung: m[1]! })); | |
| 17 | const pictures = clip(10, 10, 5, 100); | |
| 18 | // Each rung's own clip, sized by its bitrate, like the three encoders on the Pi. | |
| 19 | const clips = new Map(LADDER.map((r) => [r.name, clip(10, 10, 5, r.kbps / 3)])); | |
| 20 | /** Every rung gets the next pictures, one at a time. */ | |
| 21 | function play(f: VideoFanout, from: number, to: number) { | |
| 22 | for (let i = from; i < to; i++) for (const r of f.ladder) f.push(r.name, { key: i % 5 === 0, data: clips.get(r.name)![i]! }); | |
| 23 | } | |
| 24 | ||
| 25 | describe("renditions and viewers", () => { | |
| 26 | test("the driver gets the best rung and a watcher the middle one, each from a keyframe", () => { | |
| 27 | const f = new VideoFanout(); | |
| 28 | const driver = sink(), watcher = sink(); | |
| 29 | f.join(driver, true); | |
| 30 | f.join(watcher); | |
| 31 | expect(driver.texts[0].rendition).toBe("720p"); | |
| 32 | expect(watcher.texts[0].rendition).toBe("480p"); | |
| 33 | play(f, 2, 12); // joined mid-GOP: nothing until picture 5 | |
| 34 | expect(frames(driver)[0]).toEqual({ key: true, rung: 0 }); | |
| 35 | expect(frames(watcher)[0]).toEqual({ key: true, rung: 1 }); | |
| 36 | expect(driver.sent).toHaveLength(7); | |
| 37 | expect(driver.sent[0]!.length).toBe(HEADER + clips.get('720p')![5]!.length); | |
| 38 | expect(f.wanted()).toEqual(new Set(["720p", "480p"])); | |
| 39 | }); | |
| 40 | ||
| 41 | test("a slow viewer skips to the next keyframe instead of queueing, then steps down at a keyframe", () => { | |
| 42 | let t = 0; | |
| 43 | const f = new VideoFanout({ now: () => t, dropsToStepDown: 2 }); | |
| 44 | const slow = sink(), fast = sink(); | |
| 45 | const viewer = f.join(slow); | |
| 46 | f.join(fast); | |
| 47 | play(f, 0, 1); | |
| 48 | slow.queued = 1 << 20; // the link backs up | |
| 49 | play(f, 1, 5); // four pictures: dropped, none sent | |
| 50 | expect(slow.sent).toHaveLength(1); | |
| 51 | expect(viewer.stats().skipped).toBe(1); // the first miss; the rest wait for a key without counting | |
| 52 | slow.queued = 1 << 20; | |
| 53 | play(f, 5, 6); | |
| 54 | slow.queued = 0; | |
| 55 | play(f, 6, 10); // clear again but there is no keyframe: still waiting | |
| 56 | expect(slow.sent).toHaveLength(1); | |
| 57 | t += 1000; | |
| 58 | f.adapt(); | |
| 59 | play(f, 10, 11); // a keyframe: the viewer is back | |
| 60 | expect(slow.sent).toHaveLength(2); | |
| 61 | expect(fast.sent).toHaveLength(11); // the fast viewer never missed one | |
| 62 | }); | |
| 63 | ||
| 64 | test("two viewers at different simulated bandwidths end up on different rungs", () => { | |
| 65 | let t = 0; | |
| 66 | const f = new VideoFanout({ now: () => t, dropsToStepDown: 1, cleanMs: 5000, maxBacklog: 2000 }); | |
| 67 | const wifi = sink(), mobile = sink(); | |
| 68 | f.join(wifi); | |
| 69 | const m = f.join(mobile); | |
| 70 | // The mobile link carries 300 bytes per 100 ms; whatever it cannot carry sits in its backlog. | |
| 71 | const send = mobile.send; | |
| 72 | mobile.send = (d) => (send(d), (mobile.queued += d.length)); | |
| 73 | for (let second = 0; second < 30; second++) { | |
| 74 | for (let i = 0; i < 10; i++) { | |
| 75 | mobile.queued = Math.max(0, mobile.queued - 300); | |
| 76 | const n = (second * 10 + i) % 100; | |
| 77 | play(f, n % 10, (n % 10) + 1); | |
| 78 | t += 100; | |
| 79 | } | |
| 80 | f.adapt(); | |
| 81 | } | |
| 82 | const onRung = (s: { sent: Buffer[] }, rung: number) => frames(s).filter((x) => x.rung === rung).length; | |
| 83 | expect(mobile.texts.some((x) => x.rendition === "240p")).toBe(true); | |
| 84 | expect(onRung(mobile, 2)).toBeGreaterThan(onRung(mobile, 1) + onRung(mobile, 0)); | |
| 85 | expect(onRung(wifi, 0)).toBeGreaterThan(onRung(wifi, 2)); // clean for 5 s: stepped up to the best | |
| 86 | expect(wifi.texts.some((x) => x.rendition === "720p")).toBe(true); | |
| 87 | expect(m.stats().bytes * 3).toBeLessThan(wifi.sent.reduce((n, b) => n + b.length, 0)); | |
| 88 | }); | |
| 89 | ||
| 90 | test("a pinned quality holds against drops; auto releases it; junk is ignored", () => { | |
| 91 | const f = new VideoFanout(); | |
| 92 | const s = sink(); | |
| 93 | const v = f.join(s); | |
| 94 | v.message('{"quality":"720p"}'); | |
| 95 | v.message("not json"); | |
| 96 | v.message('{"quality":"4k"}'); | |
| 97 | v.message('{"quality":' + '9'.repeat(300) + "}"); | |
| 98 | play(f, 0, 1); | |
| 99 | expect(v.stats().rendition).toBe("720p"); | |
| 100 | for (let i = 0; i < 5; i++) (v.message('{"age":2000}'), f.adapt()); | |
| 101 | expect(f.wanted()).toEqual(new Set(["720p"])); | |
| 102 | v.message('{"quality":"auto"}'); | |
| 103 | for (let i = 0; i < 5; i++) (v.message('{"age":2000}'), f.adapt()); | |
| 104 | expect(f.wanted()).toEqual(new Set(["720p", "480p"])); // released: the drops step it down again | |
| 105 | }); | |
| 106 | ||
| 107 | test("a viewer that leaves is no longer fed or wanted", () => { | |
| 108 | const f = new VideoFanout(); | |
| 109 | const s = sink(); | |
| 110 | f.join(s).leave(); | |
| 111 | play(f, 0, 3); | |
| 112 | expect(s.sent).toHaveLength(0); | |
| 113 | expect(f.wanted().size).toBe(0); | |
| 114 | }); | |
| 115 | }); |
Software/so101/src/video-fanout.ts added +197
| @@ -0,0 +1,197 @@ | ||
| 1 | /** | |
| 2 | * One camera's H.264 renditions, fanned out to viewers over their WebSockets, a viewer on one rendition at a time. | |
| 3 | * | |
| 4 | * Latency before everything: nothing is queued for a viewer. A viewer whose socket is backed up drops pictures and | |
| 5 | * waits for the next keyframe, and that same backlog is how its link is measured: a viewer that dropped steps down | |
| 6 | * the ladder, one that went a while clean steps back up, each switch at a keyframe of the new rendition. A viewer can | |
| 7 | * pin one rendition. The driver is patient (it steps down only after more drops) so the best picture stays with it. | |
| 8 | * | |
| 9 | * A binary message is HEADER bytes then the access unit: byte 0 bit 0 = keyframe, byte 1 = the rendition's index in | |
| 10 | * the ladder, bytes 2..9 = the time the relay got the picture (ms, float64 big-endian). Text messages are JSON. | |
| 11 | */ | |
| 12 | ||
| 13 | import type { AccessUnit } from "./h264.ts"; | |
| 14 | ||
| 15 | export interface Rendition { | |
| 16 | name: string; | |
| 17 | width: number; | |
| 18 | height: number; | |
| 19 | fps: number; | |
| 20 | kbps: number; | |
| 21 | } | |
| 22 | ||
| 23 | /** The ladder the Pi encodes, best first. */ | |
| 24 | export const LADDER: readonly Rendition[] = [ | |
| 25 | { name: "720p", width: 1280, height: 720, fps: 20, kbps: 1500 }, | |
| 26 | { name: "480p", width: 854, height: 480, fps: 15, kbps: 600 }, | |
| 27 | { name: "240p", width: 426, height: 240, fps: 10, kbps: 200 }, | |
| 28 | ]; | |
| 29 | ||
| 30 | export const HEADER = 10; | |
| 31 | ||
| 32 | export interface VideoSink { | |
| 33 | /** Bytes accepted but not yet on the wire. */ | |
| 34 | backlog(): number; | |
| 35 | send(data: Uint8Array): void; | |
| 36 | text(message: string): void; | |
| 37 | } | |
| 38 | ||
| 39 | export interface FanoutOptions { | |
| 40 | ladder?: readonly Rendition[]; | |
| 41 | /** Backlog past which a picture is dropped for that viewer. */ | |
| 42 | maxBacklog?: number; | |
| 43 | /** Drops (in one window) that step an ordinary viewer down; the driver takes twice as many. */ | |
| 44 | dropsToStepDown?: number; | |
| 45 | /** How long a viewer must go without a drop before it steps up. */ | |
| 46 | cleanMs?: number; | |
| 47 | /** A frame age the viewer reports above this also counts as a drop. */ | |
| 48 | maxAgeMs?: number; | |
| 49 | now?: () => number; | |
| 50 | } | |
| 51 | ||
| 52 | interface Viewer { | |
| 53 | sink: VideoSink; | |
| 54 | driver: boolean; | |
| 55 | /** Index of the rendition the viewer is on, and the one it is moving to. */ | |
| 56 | current: number; | |
| 57 | target: number; | |
| 58 | pinned: number | null; | |
| 59 | /** Waiting for a keyframe on `current`. */ | |
| 60 | needKey: boolean; | |
| 61 | drops: number; | |
| 62 | lastDrop: number; | |
| 63 | lastStep: number; | |
| 64 | bytes: number; | |
| 65 | frames: number; | |
| 66 | skipped: number; | |
| 67 | } | |
| 68 | ||
| 69 | export interface ViewerStats { | |
| 70 | rendition: string; | |
| 71 | bytes: number; | |
| 72 | frames: number; | |
| 73 | skipped: number; | |
| 74 | } | |
| 75 | ||
| 76 | export class VideoFanout { | |
| 77 | readonly ladder: readonly Rendition[]; | |
| 78 | private readonly viewers = new Set<Viewer>(); | |
| 79 | private readonly o: Required<Omit<FanoutOptions, "ladder">>; | |
| 80 | ||
| 81 | constructor(options: FanoutOptions = {}) { | |
| 82 | this.ladder = options.ladder ?? LADDER; | |
| 83 | this.o = { maxBacklog: 64 << 10, dropsToStepDown: 3, cleanMs: 6000, maxAgeMs: 500, now: Date.now, ...options }; | |
| 84 | } | |
| 85 | ||
| 86 | get watching(): number { | |
| 87 | return this.viewers.size; | |
| 88 | } | |
| 89 | ||
| 90 | /** The renditions someone is on or moving to: the only ones worth pulling from the Pi. */ | |
| 91 | wanted(): Set<string> { | |
| 92 | const names = new Set<string>(); | |
| 93 | for (const v of this.viewers) names.add(this.ladder[v.current]!.name).add(this.ladder[v.target]!.name); | |
| 94 | return names; | |
| 95 | } | |
| 96 | ||
| 97 | /** A viewer joins on the best rendition (the driver) or the middle of the ladder (a watcher), and waits for a keyframe. */ | |
| 98 | join(sink: VideoSink, driver = false): { leave(): void; message(text: string): void; stats(): ViewerStats } { | |
| 99 | const start = driver ? 0 : Math.min(1, this.ladder.length - 1); | |
| 100 | const now = this.o.now(); | |
| 101 | const v: Viewer = { sink, driver, current: start, target: start, pinned: null, needKey: true, drops: 0, lastDrop: now, lastStep: now, bytes: 0, frames: 0, skipped: 0 }; | |
| 102 | this.viewers.add(v); | |
| 103 | sink.text(JSON.stringify({ ladder: this.ladder, rendition: this.ladder[start]!.name })); | |
| 104 | return { | |
| 105 | leave: () => void this.viewers.delete(v), | |
| 106 | message: (text) => this.message(v, text), | |
| 107 | stats: () => ({ rendition: this.ladder[v.current]!.name, bytes: v.bytes, frames: v.frames, skipped: v.skipped }), | |
| 108 | }; | |
| 109 | } | |
| 110 | ||
| 111 | /** A viewer's JSON: {"quality":"auto"|<name>} to pin one, {"age":ms} for the frame age it measures. Anything else is ignored. */ | |
| 112 | private message(v: Viewer, text: string) { | |
| 113 | if (text.length > 200) return; | |
| 114 | let m: { quality?: unknown; age?: unknown }; | |
| 115 | try { | |
| 116 | m = JSON.parse(text); | |
| 117 | } catch { | |
| 118 | return; | |
| 119 | } | |
| 120 | if (typeof m !== "object" || m === null) return; | |
| 121 | if (typeof m.quality === "string") { | |
| 122 | const at = this.ladder.findIndex((r) => r.name === m.quality); | |
| 123 | if (m.quality === "auto") v.pinned = null; | |
| 124 | else if (at >= 0) (v.pinned = at), (v.target = at); | |
| 125 | } | |
| 126 | if (typeof m.age === "number" && Number.isFinite(m.age) && m.age > this.o.maxAgeMs) this.drop(v); | |
| 127 | } | |
| 128 | ||
| 129 | /** The next picture of a rendition arrives from the Pi. */ | |
| 130 | push(rendition: string, unit: AccessUnit): void { | |
| 131 | const at = this.ladder.findIndex((r) => r.name === rendition); | |
| 132 | if (at < 0) return; | |
| 133 | const now = this.o.now(); | |
| 134 | const message = Buffer.allocUnsafe(HEADER + unit.data.length); | |
| 135 | message[0] = unit.key ? 1 : 0; | |
| 136 | message[1] = at; | |
| 137 | message.writeDoubleBE(now, 2); | |
| 138 | unit.data.copy(message, HEADER); | |
| 139 | for (const v of this.viewers) { | |
| 140 | if (v.target === at && v.current !== at && unit.key) { | |
| 141 | v.current = at; | |
| 142 | v.needKey = false; | |
| 143 | v.lastStep = now; | |
| 144 | } | |
| 145 | if (v.current !== at) continue; | |
| 146 | if (v.needKey && (!unit.key || v.target !== v.current)) continue; // waiting for a key, and for the rung it is moving to | |
| 147 | ||
| 148 | if (v.sink.backlog() > this.o.maxBacklog) { | |
| 149 | v.skipped++; | |
| 150 | v.needKey = true; | |
| 151 | this.drop(v); | |
| 152 | continue; | |
| 153 | } | |
| 154 | v.needKey = false; | |
| 155 | v.sink.send(message); | |
| 156 | v.bytes += message.length; | |
| 157 | v.frames++; | |
| 158 | } | |
| 159 | } | |
| 160 | ||
| 161 | private drop(v: Viewer) { | |
| 162 | const now = this.o.now(); | |
| 163 | v.drops++; | |
| 164 | v.lastDrop = now; | |
| 165 | v.needKey = true; | |
| 166 | } | |
| 167 | ||
| 168 | /** Run about once a second: steps viewers down after drops and up after a clean while. */ | |
| 169 | adapt(): void { | |
| 170 | const now = this.o.now(); | |
| 171 | for (const v of this.viewers) { | |
| 172 | if (v.pinned !== null) { | |
| 173 | v.drops = 0; | |
| 174 | continue; | |
| 175 | } | |
| 176 | const limit = this.o.dropsToStepDown * (v.driver ? 2 : 1); | |
| 177 | if (v.drops >= limit && v.target < this.ladder.length - 1) { | |
| 178 | v.target = v.current + 1; | |
| 179 | this.announce(v); | |
| 180 | } else if (v.drops === 0 && v.target === v.current && v.current > 0 && now - v.lastDrop >= this.o.cleanMs && now - v.lastStep >= this.o.cleanMs) { | |
| 181 | v.target = v.current - 1; | |
| 182 | this.announce(v); | |
| 183 | } | |
| 184 | if (v.drops >= limit) v.drops = 0; | |
| 185 | else if (now - v.lastDrop > 2000) v.drops = 0; | |
| 186 | } | |
| 187 | } | |
| 188 | ||
| 189 | private announce(v: Viewer) { | |
| 190 | v.lastStep = this.o.now(); | |
| 191 | v.sink.text(JSON.stringify({ rendition: this.ladder[v.target]!.name })); | |
| 192 | } | |
| 193 | ||
| 194 | close(): void { | |
| 195 | this.viewers.clear(); | |
| 196 | } | |
| 197 | } |
Software/so101/src/video-feed.test.ts added +119
| @@ -0,0 +1,119 @@ | ||
| 1 | import { afterEach, describe, expect, test } from "bun:test"; | |
| 2 | import { createServer, type Server } from "node:http"; | |
| 3 | import WebSocket, { WebSocketServer } from "ws"; | |
| 4 | import { clip } from "./h264-clip.ts"; | |
| 5 | import { parseVideo } from "./relay.ts"; | |
| 6 | import { HEADER, LADDER } from "./video-fanout.ts"; | |
| 7 | import { VideoFeed } from "./video-feed.ts"; | |
| 8 | ||
| 9 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 10 | const closers: (() => void)[] = []; | |
| 11 | afterEach(() => closers.splice(0).forEach((c) => c())); | |
| 12 | ||
| 13 | /** The Pi: each rung is an endless Annex-B stream of its own clip, 20 pictures a second, looping. */ | |
| 14 | async function pi(): Promise<string> { | |
| 15 | const clips = new Map(LADDER.map((r) => [`/${r.name}`, clip(2, 20, 10, r.kbps / 3)])); | |
| 16 | const server: Server = createServer((req, res) => { | |
| 17 | const pictures = clips.get(req.url ?? ""); | |
| 18 | if (!pictures) return void res.writeHead(404).end(); | |
| 19 | res.writeHead(200, { "content-type": "video/h264" }); | |
| 20 | let i = 0; | |
| 21 | const timer = setInterval(() => res.write(pictures[i++ % pictures.length]!), 50); | |
| 22 | res.on("close", () => clearInterval(timer)); | |
| 23 | }); | |
| 24 | await new Promise<void>((r) => server.listen(0, "127.0.0.1", r)); | |
| 25 | closers.push(() => (server.closeAllConnections(), server.close())); | |
| 26 | return `http://127.0.0.1:${(server.address() as { port: number }).port}`; | |
| 27 | } | |
| 28 | ||
| 29 | /** A relay-shaped server: every WebSocket is a viewer; `driver` is chosen by the path. */ | |
| 30 | async function relay(feed: VideoFeed): Promise<string> { | |
| 31 | const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" }); | |
| 32 | wss.on("connection", (ws, req) => feed.serve(ws, req.url === "/driver")); | |
| 33 | await new Promise((r) => wss.once("listening", r)); | |
| 34 | closers.push(() => (feed.close(), wss.close())); | |
| 35 | return `ws://127.0.0.1:${(wss.address() as { port: number }).port}`; | |
| 36 | } | |
| 37 | ||
| 38 | function watch(url: string) { | |
| 39 | const got = { frames: [] as { key: boolean; rung: number; sentAt: number; bytes: number }[], texts: [] as any[] }; | |
| 40 | const ws = new WebSocket(url); | |
| 41 | ws.on("message", (data, binary) => { | |
| 42 | const m = data as Buffer; | |
| 43 | if (!binary) return void got.texts.push(JSON.parse(String(data))); | |
| 44 | got.frames.push({ key: (m[0]! & 1) === 1, rung: m[1]!, sentAt: m.readDoubleBE(2), bytes: m.length }); | |
| 45 | }); | |
| 46 | closers.push(() => ws.close()); | |
| 47 | return { ws, got }; | |
| 48 | } | |
| 49 | ||
| 50 | describe("video feed", () => { | |
| 51 | test("a driver and a watcher are fed different renditions of the Pi's streams, from keyframes, and only those are pulled", async () => { | |
| 52 | const base = await pi(); | |
| 53 | const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` }); | |
| 54 | const url = await relay(feed); | |
| 55 | const driver = watch(`${url}/driver`); | |
| 56 | const watcher = watch(`${url}/watcher`); | |
| 57 | await sleep(700); | |
| 58 | expect(driver.got.frames.length).toBeGreaterThan(5); | |
| 59 | expect(driver.got.frames[0]!.key).toBe(true); | |
| 60 | expect(watcher.got.frames[0]!.key).toBe(true); | |
| 61 | expect(new Set(driver.got.frames.map((f) => f.rung))).toEqual(new Set([0])); | |
| 62 | expect(new Set(watcher.got.frames.map((f) => f.rung))).toEqual(new Set([1])); | |
| 63 | expect(driver.got.frames.at(-1)!.bytes).toBeGreaterThan(HEADER); | |
| 64 | expect(Math.abs(Date.now() - driver.got.frames.at(-1)!.sentAt)).toBeLessThan(500); | |
| 65 | expect(feed.fanout.wanted()).toEqual(new Set(["720p", "480p"])); // 240p is not pulled | |
| 66 | expect(driver.got.texts[0].ladder.map((r: { name: string }) => r.name)).toEqual(["720p", "480p", "240p"]); | |
| 67 | }); | |
| 68 | ||
| 69 | test("a viewer pinning a quality gets that rung, switched at a keyframe", async () => { | |
| 70 | const base = await pi(); | |
| 71 | const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` }); | |
| 72 | const { ws, got } = watch(`${await relay(feed)}/watcher`); | |
| 73 | await sleep(400); | |
| 74 | ws.send(JSON.stringify({ quality: "240p" })); | |
| 75 | await sleep(1500); | |
| 76 | const at = got.frames.findIndex((f) => f.rung === 2); | |
| 77 | expect(at).toBeGreaterThan(0); | |
| 78 | expect(got.frames[at]!.key).toBe(true); | |
| 79 | expect(got.frames.slice(at).every((f) => f.rung === 2)).toBe(true); | |
| 80 | }); | |
| 81 | ||
| 82 | test("a feed without sources is not available; a down source just sends nothing", async () => { | |
| 83 | expect(new VideoFeed({}).available).toBe(false); | |
| 84 | const feed = new VideoFeed({ "720p": "http://127.0.0.1:1/x" }); | |
| 85 | expect(feed.available).toBe(true); | |
| 86 | const { got } = watch(`${await relay(feed)}/driver`); | |
| 87 | await sleep(300); | |
| 88 | expect(got.frames).toHaveLength(0); | |
| 89 | expect(got.texts[0].rendition).toBe("720p"); | |
| 90 | }); | |
| 91 | }); | |
| 92 | ||
| 93 | describe("a subset of rungs", () => { | |
| 94 | test("with only 720p configured, a watcher, a pin and a step-down all stay on it and get pictures", async () => { | |
| 95 | const base = await pi(); | |
| 96 | const feed = new VideoFeed({ "720p": `${base}/720p` }); | |
| 97 | expect(feed.fanout.ladder.map((r) => r.name)).toEqual(["720p"]); | |
| 98 | const url = await relay(feed); | |
| 99 | const watcher = watch(`${url}/watcher`); | |
| 100 | await sleep(600); | |
| 101 | expect(watcher.got.texts[0].ladder.map((r: any) => r.name)).toEqual(["720p"]); | |
| 102 | expect(watcher.got.frames.length).toBeGreaterThan(3); | |
| 103 | expect(watcher.got.frames.every((f) => f.rung === 0)).toBe(true); | |
| 104 | watcher.ws.send('{"quality":"240p"}'); | |
| 105 | for (let i = 0; i < 5; i++) (watcher.ws.send('{"age":5000}'), await sleep(30)); | |
| 106 | const before = watcher.got.frames.length; | |
| 107 | await sleep(500); | |
| 108 | expect(watcher.got.frames.length).toBeGreaterThan(before); | |
| 109 | }); | |
| 110 | }); | |
| 111 | ||
| 112 | describe("ARM_H264_n", () => { | |
| 113 | test("names rungs of the ladder with http URLs", () => { | |
| 114 | expect(parseVideo({})).toEqual({}); | |
| 115 | expect(parseVideo({ ARM_H264_1: "720p=pi:8082/h264, 240p=http://pi:8083/h264" })).toEqual({ 1: { "720p": "http://pi:8082/h264", "240p": "http://pi:8083/h264" } }); | |
| 116 | expect(() => parseVideo({ ARM_H264_1: "4k=pi:1" })).toThrow(/ladder/); | |
| 117 | expect(() => parseVideo({ ARM_H264_1: "720p=ftp://pi/x" })).toThrow(); | |
| 118 | }); | |
| 119 | }); |
Software/so101/src/video-feed.ts added +112
| @@ -0,0 +1,112 @@ | ||
| 1 | /** | |
| 2 | * One camera's H.264 over WebSockets: a VideoFanout fed by the Pi's encoder streams. | |
| 3 | * | |
| 4 | * Each rendition is an HTTP stream of Annex-B H.264 on the Pi. It is pulled only while some viewer is on it (or | |
| 5 | * moving to it), so a camera nobody watches costs the Pi nothing, and the lower rungs cost nothing until a viewer | |
| 6 | * needs them. A source that is down is tried again every second while wanted; its viewers simply see no pictures, and | |
| 7 | * the page falls back to MJPEG when none come. | |
| 8 | */ | |
| 9 | ||
| 10 | import { proxyFor } from "./egress.ts"; | |
| 11 | import { AccessUnitSplitter } from "./h264.ts"; | |
| 12 | import { LADDER, VideoFanout, type FanoutOptions, type VideoSink } from "./video-fanout.ts"; | |
| 13 | ||
| 14 | export interface VideoSources { | |
| 15 | /** Rendition name → the URL of its Annex-B stream. */ | |
| 16 | [name: string]: string; | |
| 17 | } | |
| 18 | ||
| 19 | /** The wire the feed talks to a viewer over; a `ws` WebSocket fits. */ | |
| 20 | export interface VideoSocket { | |
| 21 | bufferedAmount: number; | |
| 22 | readyState: number; | |
| 23 | send(data: Uint8Array | string): void; | |
| 24 | on(event: "message", listener: (data: unknown, isBinary: boolean) => void): unknown; | |
| 25 | on(event: "close", listener: () => void): unknown; | |
| 26 | } | |
| 27 | ||
| 28 | const OPEN = 1; | |
| 29 | ||
| 30 | export class VideoFeed { | |
| 31 | readonly fanout: VideoFanout; | |
| 32 | private readonly sources: VideoSources; | |
| 33 | private readonly env: Record<string, string | undefined>; | |
| 34 | private readonly pulls = new Map<string, AbortController>(); | |
| 35 | private timer: ReturnType<typeof setInterval> | undefined; | |
| 36 | private closed = false; | |
| 37 | ||
| 38 | /** The ladder a viewer sees is only the rungs that have a source, so none can start, pin or step onto a rung nothing feeds. */ | |
| 39 | constructor(sources: VideoSources, options: FanoutOptions = {}, env: Record<string, string | undefined> = process.env) { | |
| 40 | this.sources = sources; | |
| 41 | this.env = env; | |
| 42 | this.fanout = new VideoFanout({ ...options, ladder: (options.ladder ?? LADDER).filter((r) => sources[r.name]) }); | |
| 43 | } | |
| 44 | ||
| 45 | /** Whether this feed has an encoder stream for any rung of the ladder. */ | |
| 46 | get available(): boolean { | |
| 47 | return this.fanout.ladder.length > 0; | |
| 48 | } | |
| 49 | ||
| 50 | /** Serve one viewer's WebSocket until it closes. */ | |
| 51 | serve(ws: VideoSocket, driver: boolean): void { | |
| 52 | if (this.closed || !this.available) return; | |
| 53 | const sink: VideoSink = { | |
| 54 | backlog: () => ws.bufferedAmount, | |
| 55 | send: (data) => ws.readyState === OPEN && ws.send(data), | |
| 56 | text: (message) => ws.readyState === OPEN && ws.send(message), | |
| 57 | }; | |
| 58 | const viewer = this.fanout.join(sink, driver); | |
| 59 | ws.on("message", (data, isBinary) => { | |
| 60 | if (!isBinary) viewer.message(String(data)); | |
| 61 | }); | |
| 62 | ws.on("close", () => { | |
| 63 | viewer.leave(); | |
| 64 | if (!this.fanout.watching) this.idle(); | |
| 65 | }); | |
| 66 | this.timer ??= setInterval(() => this.tick(), 1000); | |
| 67 | this.timer.unref?.(); | |
| 68 | this.tick(); | |
| 69 | } | |
| 70 | ||
| 71 | close(): void { | |
| 72 | this.closed = true; | |
| 73 | this.fanout.close(); | |
| 74 | this.idle(); | |
| 75 | } | |
| 76 | ||
| 77 | private idle() { | |
| 78 | clearInterval(this.timer); | |
| 79 | this.timer = undefined; | |
| 80 | for (const c of this.pulls.values()) c.abort(); | |
| 81 | this.pulls.clear(); | |
| 82 | } | |
| 83 | ||
| 84 | private tick() { | |
| 85 | this.fanout.adapt(); | |
| 86 | const wanted = this.fanout.wanted(); | |
| 87 | for (const [name, controller] of this.pulls) { | |
| 88 | if (!wanted.has(name)) (controller.abort(), this.pulls.delete(name)); | |
| 89 | } | |
| 90 | for (const name of wanted) if (this.sources[name] && !this.pulls.has(name)) this.pull(name, this.sources[name]!); | |
| 91 | } | |
| 92 | ||
| 93 | private pull(name: string, url: string) { | |
| 94 | const controller = new AbortController(); | |
| 95 | this.pulls.set(name, controller); | |
| 96 | void (async () => { | |
| 97 | try { | |
| 98 | const proxy = proxyFor(url, this.env); // inside a forge cell the egress proxy is the only way out | |
| 99 | const response = await fetch(url, { signal: controller.signal, cache: "no-store", redirect: "error", ...(proxy ? { proxy } : {}) }); | |
| 100 | if (!response.ok || !response.body) throw new Error(`answered ${response.status}`); | |
| 101 | const splitter = new AccessUnitSplitter(); | |
| 102 | for await (const chunk of response.body as unknown as AsyncIterable<Uint8Array>) { | |
| 103 | for (const unit of splitter.push(chunk)) this.fanout.push(name, unit); | |
| 104 | } | |
| 105 | } catch { | |
| 106 | // down or cut off: the next tick tries again while the rung is wanted | |
| 107 | } finally { | |
| 108 | if (this.pulls.get(name) === controller) this.pulls.delete(name); | |
| 109 | } | |
| 110 | })(); | |
| 111 | } | |
| 112 | } |
Software/so101/src/viewer.html +19 −3
| @@ -68,7 +68,9 @@ | ||
| 68 | 68 | #banner button { margin-left: 10px; min-height: 32px; padding: 2px 10px; } |
| 69 | 69 | .pane { flex: 1; min-width: 0; position: relative; background: #000; display: none; } |
| 70 | 70 | .pane.on { display: block; } |
| 71 | .pane img { width: 100%; height: 100%; object-fit: contain; display: block; } | |
| 71 | .pane img, .pane canvas { width: 100%; height: 100%; object-fit: contain; display: block; } | |
| 72 | .pane canvas { position: absolute; inset: 0; } | |
| 73 | .pane select.quality { position: absolute; right: calc(8px + var(, 0px)); bottom: 8px; font-size: 11px; background: rgba(0,0,0,.55); color: #c9d1d9; border: 0; border-radius: 4px; padding: 2px 4px; } | |
| 72 | 74 | .pane .tag { position: absolute; left: calc(8px + var()); bottom: 8px; font-size: 11px; color: #c9d1d9; background: rgba(0,0,0,.55); padding: 2px 6px; border-radius: 4px; pointer-events: none; } |
| 73 | 75 | urdf-viewer { display: block; width: 100%; height: 100%; } |
| 74 | 76 | #top { position: absolute; top: 6px; right: calc(6px + var()); left: calc(6px + var()); z-index: 5; display: flex; gap: 6px; align-items: center; justify-content: flex-end; flex-wrap: wrap; font-size: 12px; pointer-events: none; } |
| @@ -675,11 +677,23 @@ | ||
| 675 | 677 | try { video ??= await import("/ui/__VERSION__/video.js"); } catch { return; } |
| 676 | 678 | video.attach(c, epoch, { visible: () => paneOn(c) && !document.hidden, loc: location, maxFps: plan.fps, create: URL.createObjectURL, revoke: URL.revokeObjectURL, now: Date.now }); |
| 677 | 679 | } |
| 680 | // H.264 over the relay's WebSocket, decoded into a canvas, where the browser has WebCodecs and the camera offers it; MJPEG otherwise, and when no picture comes. | |
| 681 | function stopVideo(c) { | |
| 682 | c.player?.stop(); c.player = null; | |
| 683 | c.canvas?.remove(); c.canvas = null; c.chip?.remove(); c.chip = null; | |
| 684 | c.img.style.visibility = ""; | |
| 685 | } | |
| 686 | async function startVideo(c) { | |
| 687 | stopVideo(c); | |
| 688 | const { mountVideo } = await import("/ui/__VERSION__/video-pane.js"); // only a viewer who gets H.264 downloads it | |
| 689 | mountVideo(c, { el, loc: location, ok: () => paneOn(c), fallback: () => { c.h264Failed = true; stopVideo(c); if (paneOn(c)) startFeed(c); } }); | |
| 690 | } | |
| 678 | 691 | const showCam = (c, on) => { |
| 679 | 692 | c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on)); |
| 680 | clearTimeout(c.timer); stopFeed(c); | |
| 693 | stopVideo(c); clearTimeout(c.timer); stopFeed(c); | |
| 681 | 694 | if (!on || document.hidden) c.img.src = ""; |
| 682 | 695 | else if (plan.mode === "snapshot" && c.snapshot) snap(c); |
| 696 | else if (c.h264 && !c.h264Failed && relayed && typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function") { c.img.src = ""; startVideo(c); } | |
| 683 | 697 | else if (relayed) startFeed(c); |
| 684 | 698 | else c.img.src = streamUrl(c); |
| 685 | 699 | }; |
| @@ -736,7 +750,7 @@ | ||
| 736 | 750 | } |
| 737 | 751 | function label(c, info) { |
| 738 | 752 | c.online = info.online; |
| 739 | c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name; | |
| 753 | if (!c.player) c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name; | |
| 740 | 754 | c.btn.textContent = info.name; c.btn.title = info.online === false ? `${info.name}: offline` : info.name; |
| 741 | 755 | c.img.alt = info.name; |
| 742 | 756 | } |
| @@ -746,8 +760,10 @@ | ||
| 746 | 760 | let c = cams.find((x) => x.id === info.id); |
| 747 | 761 | if (!c) { |
| 748 | 762 | c = addCam({ ...info, url: relayed ? `/camera/${info.id}/stream` : info.url }); |
| 763 | c.h264 = info.h264 === true; c.name = info.name; | |
| 749 | 764 | if (viewsReady && wanted(c.pane.dataset.key)) showCam(c, true); |
| 750 | 765 | } |
| 766 | c.h264 = info.h264 === true; c.name = info.name; | |
| 751 | 767 | label(c, info); |
| 752 | 768 | } |
| 753 | 769 | showQuality(); |
forge.json +3
| @@ -13,6 +13,9 @@ | ||
| 13 | 13 | "egress": [ |
| 14 | 14 | "100.89.95.68:8080", |
| 15 | 15 | "100.89.95.68:8081", |
| 16 | "100.89.95.68:8082", | |
| 17 | "100.89.95.68:8083", | |
| 18 | "100.89.95.68:8084", | |
| 16 | 19 | "100.89.95.68:8101" |
| 17 | 20 | ] |
| 18 | 21 | } |
new above
offline
Dead in production: nothing outside h264.test.ts calls codecOf. Remove it and its test; the player carries its own copy.