Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
Real video: hardware H.264 from the Pi over the WebSocket, decoded with WebCodecs, smaller streams for slow viewers - a fraction of MJPEG's bytes #75
smith/arm:smith/70-real-video-hardware-h-264-from-the-pi-ov → main 2 commits fork
12 files
changed +966
−6
Software/so101/bench/fake-serve.ts +8 −2
| @@ -22,7 +22,7 @@ import { CameraHub } from "../src/cameras.ts"; | ||
| 22 | 22 | import { LeaderCalibrations } from "../src/leader-calibrations.ts"; |
| 23 | 23 | import { Lobby } from "../src/lobby.ts"; |
| 24 | 24 | import { Moderator, modelFromEnv } from "../src/moderator.ts"; |
| 25 | import { parseRemote, startRelay, type Remote } from "../src/relay.ts"; | |
| 25 | import { parseRemote, parseVideo, startRelay, type Remote } from "../src/relay.ts"; | |
| 26 | 26 | import { startServer } from "../src/server.ts"; |
| 27 | 27 | |
| 28 | 28 | const fake = new FakeBus([1, 2, 3, 4, 5, 6]); |
| @@ -59,6 +59,12 @@ const lobby = new Lobby({ | ||
| 59 | 59 | const cameras = new CameraHub({ file: join(dir, "cameras.json"), seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); |
| 60 | 60 | |
| 61 | 61 | const leaders = new LeaderCalibrations({ file: join(dir, "leader-calibrations.json") }); |
| 62 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders }); | |
| 62 | let video = {}; | |
| 63 | try { | |
| 64 | video = parseVideo(process.env); | |
| 65 | } catch (err) { | |
| 66 | console.error(`${(err as Error).message}; cameras stay MJPEG`); | |
| 67 | } | |
| 68 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video }); | |
| 63 | 69 | const { mode, host: armHost } = relay.link(); |
| 64 | 70 | console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`); |
Software/so101/src/h264-clip.ts added +16
| @@ -0,0 +1,16 @@ | ||
| 1 | /** A stand-in for the Pi's encoder in tests: a short Annex-B clip with the shape of the real one (SPS, PPS, IDR, then P pictures; no B-frames). */ | |
| 2 | ||
| 3 | const nal = (type: number, size: number, fill: number) => Buffer.concat([Buffer.from([0, 0, 0, 1, type | 0x60]), Buffer.alloc(size, fill)]); | |
| 4 | // SPS body: profile 66 (baseline), constraints 0xc0, level 31, then filler that contains no start code. | |
| 5 | const SPS = Buffer.concat([Buffer.from([0, 0, 0, 1, 0x67, 66, 0xc0, 31]), Buffer.alloc(6, 0x11)]); | |
| 6 | const PPS = nal(8, 3, 0x22); | |
| 7 | ||
| 8 | /** One picture: `key` pictures carry the parameter sets; `size` is the slice's payload in bytes. */ | |
| 9 | export function picture(key: boolean, size: number): Buffer { | |
| 10 | return key ? Buffer.concat([SPS, PPS, nal(5, size, 0x33)]) : nal(1, size, 0x44); | |
| 11 | } | |
| 12 | ||
| 13 | /** `seconds` of video at `fps` with a keyframe every `gop` pictures; keyframes are 8x a P picture's size. */ | |
| 14 | export function clip(seconds: number, fps: number, gop: number, deltaBytes: number): Buffer[] { | |
| 15 | return Array.from({ length: Math.round(seconds * fps) }, (_, i) => picture(i % gop === 0, i % gop === 0 ? deltaBytes * 8 : deltaBytes)); | |
| 16 | } |
Software/so101/src/h264.test.ts added +40
| @@ -0,0 +1,40 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { clip } from "./h264-clip.ts"; | |
| 3 | import { AccessUnitSplitter, codecOf } from "./h264.ts"; | |
| 4 | ||
| 5 | describe("access units", () => { | |
| 6 | const pictures = clip(1, 10, 5, 100); | |
| 7 | const stream = Buffer.concat(pictures); | |
| 8 | ||
| 9 | test("a stream cut anywhere comes out as the same pictures, key flags right", () => { | |
| 10 | for (const step of [1, 7, 333, stream.length]) { | |
| 11 | const splitter = new AccessUnitSplitter(); | |
| 12 | const units = []; | |
| 13 | for (let i = 0; i < stream.length; i += step) units.push(...splitter.push(stream.subarray(i, i + step))); | |
| 14 | // the last picture waits for the start of the next | |
| 15 | expect(units.length).toBe(pictures.length - 1); | |
| 16 | units.forEach((u, i) => { | |
| 17 | expect(u.data.equals(pictures[i]!)).toBe(true); | |
| 18 | expect(u.key).toBe(i % 5 === 0); | |
| 19 | }); | |
| 20 | } | |
| 21 | }); | |
| 22 | ||
| 23 | test("a picture is out as soon as the next one begins, not when it ends", () => { | |
| 24 | const splitter = new AccessUnitSplitter(); | |
| 25 | expect(splitter.push(pictures[0]!)).toEqual([]); | |
| 26 | expect(splitter.push(pictures[1]!.subarray(0, 5))).toHaveLength(1); | |
| 27 | }); | |
| 28 | ||
| 29 | test("reads the codec string from the SPS", () => { | |
| 30 | expect(codecOf(pictures[0]!)).toBe("avc1.42c01f"); | |
| 31 | expect(codecOf(pictures[1]!)).toBeNull(); | |
| 32 | }); | |
| 33 | ||
| 34 | test("refuses a stream that never ends a unit", () => { | |
| 35 | const splitter = new AccessUnitSplitter(); | |
| 36 | expect(() => { | |
| 37 | for (let i = 0; i < 6; i++) splitter.push(Buffer.alloc(1 << 20, 7)); | |
| 38 | }).toThrow(); | |
| 39 | }); | |
| 40 | }); |
Software/so101/src/h264.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** | |
| 2 | * H.264 in Annex-B (start-code) form, cut into access units as the bytes arrive. | |
| 3 | * | |
| 4 | * An access unit is one picture: optional parameter sets (SPS/PPS), then its slices. The encoder on the Pi sends one | |
| 5 | * slice per picture and no B-frames, so a picture ends the moment the next NAL that opens one begins: the unit | |
| 6 | * leaves as soon as the next start code and its header byte are seen, never waiting for the next picture to finish. | |
| 7 | */ | |
| 8 | ||
| 9 | export interface AccessUnit { | |
| 10 | /** An IDR picture: a viewer can start decoding here. */ | |
| 11 | key: boolean; | |
| 12 | /** The unit's bytes, start codes included. */ | |
| 13 | data: Buffer; | |
| 14 | } | |
| 15 | ||
| 16 | /** NAL types that open a new access unit once a picture's slice has been seen. */ | |
| 17 | const OPENS = new Set([1, 5, 6, 7, 8, 9]); | |
| 18 | /** No unit is larger than this; a stream that never finds a start code is cut off by throwing. */ | |
| 19 | export const MAX_UNIT = 4 << 20; | |
| 20 | ||
| 21 | export class AccessUnitSplitter { | |
| 22 | private buf: Buffer = Buffer.alloc(0); | |
| 23 | /** Where the current unit starts in `buf`. */ | |
| 24 | private from = 0; | |
| 25 | /** Where to look for the next start code. */ | |
| 26 | private scan = 0; | |
| 27 | private picture = false; | |
| 28 | private key = false; | |
| 29 | ||
| 30 | push(chunk: Uint8Array): AccessUnit[] { | |
| 31 | this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : Buffer.from(chunk); | |
| 32 | const out: AccessUnit[] = []; | |
| 33 | const b = this.buf; | |
| 34 | for (let i = this.scan; i + 3 < b.length; i++) { | |
| 35 | if (b[i + 2]! > 1) i += 2; // cannot be inside a start code | |
| 36 | else if (b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 1) { | |
| 37 | const type = b[i + 3]! & 0x1f; | |
| 38 | const start = i > this.from && b[i - 1] === 0 ? i - 1 : i; | |
| 39 | if (this.picture && OPENS.has(type) && start > this.from) { | |
| 40 | out.push({ key: this.key, data: Buffer.from(b.subarray(this.from, start)) }); | |
| 41 | this.from = start; | |
| 42 | this.picture = this.key = false; | |
| 43 | } | |
| 44 | if (type === 1 || type === 5) this.picture = true; | |
| 45 | if (type === 5) this.key = true; | |
| 46 | i += 3; | |
| 47 | this.scan = i + 1; | |
| 48 | } | |
| 49 | } | |
| 50 | this.scan = Math.max(this.scan, b.length - 3); | |
| 51 | if (this.from > 0) { | |
| 52 | this.buf = Buffer.from(b.subarray(this.from)); | |
| 53 | this.scan -= this.from; | |
| 54 | this.from = 0; | |
| 55 | } | |
| 56 | if (this.buf.length > MAX_UNIT) throw new Error("an H.264 unit is too long"); | |
| 57 | return out; | |
| 58 | } | |
| 59 | } | |
| 60 | ||
| 61 | /** The codec string WebCodecs wants ("avc1.PPCCLL"), from the first SPS in a unit, or null when it has none. */ | |
| 62 | export function codecOf(unit: Uint8Array): string | null { | |
| 63 | for (let i = 0; i + 6 < unit.length; i++) { | |
| 64 | if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3]! & 0x1f) === 7) { | |
| 65 | const hex = (n: number) => n.toString(16).padStart(2, "0"); | |
| 66 | return `avc1.${hex(unit[i + 4]!)}${hex(unit[i + 5]!)}${hex(unit[i + 6]!)}`; | |
| 67 | } | |
| 68 | } | |
| 69 | return null; | |
| 70 | } |
Software/so101/src/relay.ts +38 −2
| @@ -29,6 +29,7 @@ | ||
| 29 | 29 | * GET /status the arm's own /status, and which arm said it: "source": "live <host:port>", "reconnecting to the arm: |
| 30 | 30 | * <why>" (the last known status, with "reconnecting": true and "seenAt"; 503 before |
| 31 | 31 | * there is one), or "simulated: <why>" ("simulated: offline, <why>" after 60 s down) |
| 32 | * WS /camera/<id>/video the camera as H.264 access units, a rendition per viewer (src/video-feed.ts; sources from ARM_H264_<n>); 404 when it has none | |
| 32 | 33 | * GET /cameras the listed cameras: [{"id", "name", "online"}] |
| 33 | 34 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 34 | 35 | * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate |
| @@ -51,6 +52,8 @@ import { AdminError, changeFromForm } from "./admin.ts"; | ||
| 51 | 52 | import { ADMIN_CSP, DONE, renderAdmin, renderRefusal, type Done } from "./admin-page.ts"; |
| 52 | 53 | import { parseFps, type FeedOptions } from "./camera-feed.ts"; |
| 53 | 54 | import { CameraError, CameraHub } from "./cameras.ts"; |
| 55 | import { VideoFeed, type VideoSources } from "./video-feed.ts"; | |
| 56 | import { LADDER } from "./video-fanout.ts"; | |
| 54 | 57 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 55 | 58 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| 56 | 59 | import { proxyFor, whyNot } from "./egress.ts"; |
| @@ -100,6 +103,8 @@ export interface RelayOptions { | ||
| 100 | 103 | heartbeatMs?: number; |
| 101 | 104 | /** Timings of the shared camera feeds. */ |
| 102 | 105 | camera?: FeedOptions; |
| 106 | /** The Pi's H.264 streams per camera id (src/video-feed.ts); a camera without one stays MJPEG. */ | |
| 107 | video?: Record<number, VideoSources>; | |
| 103 | 108 | /** Every driver's leader calibration; without one they are kept in memory. */ |
| 104 | 109 | leaders?: LeaderCalibrations; |
| 105 | 110 | /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */ |
| @@ -116,6 +121,7 @@ export interface RunningRelay { | ||
| 116 | 121 | |
| 117 | 122 | const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" }; |
| 118 | 123 | const MAX_BODY = 1_000_000; |
| 124 | const VIDEO = /^\/camera\/(\d{1,9})\/video$/; | |
| 119 | 125 | const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/; |
| 120 | 126 | const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/; |
| 121 | 127 | const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8"); |
| @@ -209,6 +215,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 209 | 215 | const heartbeatMs = options.heartbeatMs ?? 1000; |
| 210 | 216 | const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] }); |
| 211 | 217 | const leaders = options.leaders ?? new LeaderCalibrations(); |
| 218 | const videos = new Map<number, VideoFeed>(Object.entries(options.video ?? {}).map(([id, sources]) => [Number(id), new VideoFeed(sources)])); | |
| 212 | 219 | |
| 213 | 220 | // ------------------------------------------------------------------ the link |
| 214 | 221 | let armLink: ArmLink | null = null; |
| @@ -539,7 +546,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 539 | 546 | const path = url.pathname + url.search; |
| 540 | 547 | const reading = req.method === "GET" || req.method === "HEAD"; |
| 541 | 548 | if (!reading && !sameOrigin(req)) return json(res, 403, { error: "only this page may change the arm" }); |
| 542 | if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online }))); | |
| 549 | if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online, h264: videos.get(id)?.available ?? false }))); | |
| 543 | 550 | if (req.method === "GET" && url.pathname === "/admin/camera-list") { |
| 544 | 551 | res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" }); |
| 545 | 552 | return res.end(CAMERA_PAGE); |
| @@ -699,9 +706,17 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 699 | 706 | }; |
| 700 | 707 | http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => { |
| 701 | 708 | const url = new URL(req.url ?? "/", "http://x"); |
| 702 | if (url.pathname !== "/follow" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); | |
| 709 | const video = VIDEO.exec(url.pathname); | |
| 710 | if (url.pathname !== "/follow" && !video && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found"); | |
| 703 | 711 | if (!sameOrigin(req)) return refuse(socket, "403 Forbidden"); |
| 704 | 712 | socket.on("error", () => socket.destroy()); |
| 713 | if (video) { | |
| 714 | const feed = videos.get(Number(video[1])); | |
| 715 | if (!feed?.available) return refuse(socket, "404 Not Found"); | |
| 716 | const who = lobby?.who(req); // watching is open, like /stream; the driver's picture gets the best rendition | |
| 717 | const driver = !!who && !!lobby?.turns.isDriver(who.id); | |
| 718 | return wss.handleUpgrade(req, socket, head, (browser) => feed.serve(browser, driver)); | |
| 719 | } | |
| 705 | 720 | if (url.pathname === "/live") return lobby!.upgrade(req, socket, head); |
| 706 | 721 | let driver: string | null = null; |
| 707 | 722 | if (lobby) { |
| @@ -770,6 +785,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 770 | 785 | arrived?.(); |
| 771 | 786 | armLink?.close(); |
| 772 | 787 | if (!options.cameras) cameras.close(); |
| 788 | for (const v of videos.values()) v.close(); | |
| 773 | 789 | for (const c of inFlight) c.abort(); |
| 774 | 790 | for (const client of wss.clients) client.terminate(); |
| 775 | 791 | wss.close(); |
| @@ -778,3 +794,23 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 778 | 794 | }), |
| 779 | 795 | }; |
| 780 | 796 | } |
| 797 | ||
| 798 | /** | |
| 799 | * ARM_H264_<n> = "720p=http://pi:8082/h264,480p=http://pi:8083/h264,240p=…" → camera n's rendition streams. | |
| 800 | * Throws on a name outside the ladder or a URL that is not http(s). | |
| 801 | */ | |
| 802 | export function parseVideo(env: Record<string, string | undefined>): Record<number, VideoSources> { | |
| 803 | const out: Record<number, VideoSources> = {}; | |
| 804 | for (const n of [1, 2]) { | |
| 805 | const text = env[`ARM_H264_${n}`]?.trim(); | |
| 806 | if (!text) continue; | |
| 807 | const sources: VideoSources = {}; | |
| 808 | for (const part of text.split(",")) { | |
| 809 | const [name, ...rest] = part.trim().split("="); | |
| 810 | if (!name || !LADDER.some((r) => r.name === name)) throw new Error(`ARM_H264_${n}: "${name}" is not a rung of the ladder (${LADDER.map((r) => r.name).join(", ")})`); | |
| 811 | sources[name] = absolute(rest.join("="), `ARM_H264_${n}`, { "http:": "", "https:": "" }).href; | |
| 812 | } | |
| 813 | out[n] = sources; | |
| 814 | } | |
| 815 | return out; | |
| 816 | } |
Software/so101/src/ui/video-player.js added +131
| @@ -0,0 +1,131 @@ | ||
| 1 | // @ts-nocheck -- WebCodecs and the DOM are not in this project's lib; the tests drive the class with fakes | |
| 2 | /** | |
| 3 | * The camera as H.264 over the relay's WebSocket (/camera/<id>/video), decoded with WebCodecs into a canvas. | |
| 4 | * Plain ESM, served at /ui/ and imported by the tests. The wire format is documented in src/video-fanout.ts: | |
| 5 | * 10 header bytes (keyframe flag, rendition index, relay time float64) then one Annex-B access unit. | |
| 6 | * | |
| 7 | * Nothing is queued: a picture that arrives while the decoder is behind is dropped, and so is every one after it | |
| 8 | * until the next keyframe. `onStatus` hears "live", "none" (no picture in `waitMs`: use MJPEG) and the frame age. | |
| 9 | */ | |
| 10 | ||
| 11 | export const HEADER = 10; | |
| 12 | ||
| 13 | /** @param {ArrayBuffer | Uint8Array} data */ | |
| 14 | export function parseFrame(data) { | |
| 15 | const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); | |
| 16 | if (bytes.length <= HEADER) return null; | |
| 17 | const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); | |
| 18 | return { key: (bytes[0] & 1) === 1, rendition: bytes[1], sentAt: view.getFloat64(2), data: bytes.subarray(HEADER) }; | |
| 19 | } | |
| 20 | ||
| 21 | /** "avc1.PPCCLL" from the first SPS of an Annex-B unit, or null. @param {Uint8Array} unit */ | |
| 22 | export function codecOf(unit) { | |
| 23 | for (let i = 0; i + 6 < unit.length; i++) { | |
| 24 | if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3] & 0x1f) === 7) { | |
| 25 | return "avc1." + [4, 5, 6].map((n) => unit[i + n].toString(16).padStart(2, "0")).join(""); | |
| 26 | } | |
| 27 | } | |
| 28 | return null; | |
| 29 | } | |
| 30 | ||
| 31 | export const canDecode = () => typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function" && typeof WebSocket === "function"; | |
| 32 | ||
| 33 | export class VideoPlayer { | |
| 34 | /** | |
| 35 | * @param {{ url: string, canvas: { width: number, height: number, getContext(k: string): any }, onStatus?: (s: { state: string, age?: number, rendition?: string, ladder?: any[] }) => void, waitMs?: number, | |
| 36 | * deps?: { WebSocket?: any, VideoDecoder?: any, EncodedVideoChunk?: any, now?: () => number } }} o | |
| 37 | */ | |
| 38 | constructor(o) { | |
| 39 | this.o = o; | |
| 40 | const d = o.deps ?? {}; | |
| 41 | this.WS = d.WebSocket ?? WebSocket; | |
| 42 | this.Decoder = d.VideoDecoder ?? (typeof VideoDecoder === "undefined" ? undefined : VideoDecoder); | |
| 43 | this.Chunk = d.EncodedVideoChunk ?? (typeof EncodedVideoChunk === "undefined" ? undefined : EncodedVideoChunk); | |
| 44 | this.now = d.now ?? (() => Date.now()); | |
| 45 | this.ctx = o.canvas.getContext("2d"); | |
| 46 | this.decoder = null; | |
| 47 | this.configured = false; | |
| 48 | this.needKey = true; | |
| 49 | this.lowest = Infinity; | |
| 50 | this.age = 0; | |
| 51 | this.frames = 0; | |
| 52 | this.ladder = []; | |
| 53 | this.closed = false; | |
| 54 | this.ws = null; | |
| 55 | this.timers = []; | |
| 56 | } | |
| 57 | ||
| 58 | start() { | |
| 59 | const ws = (this.ws = new this.WS(this.o.url)); | |
| 60 | ws.binaryType = "arraybuffer"; | |
| 61 | ws.onmessage = (e) => (typeof e.data === "string" ? this.control(e.data) : this.frame(e.data)); | |
| 62 | ws.onclose = () => this.status("none"); | |
| 63 | this.timers.push(setTimeout(() => this.frames || this.status("none"), this.o.waitMs ?? 3000)); | |
| 64 | this.timers.push(setInterval(() => this.report(), 1000)); | |
| 65 | } | |
| 66 | ||
| 67 | /** Pin a rung by name, or "auto". @param {string} quality */ | |
| 68 | quality(quality) { | |
| 69 | if (this.ws?.readyState === 1) this.ws.send(JSON.stringify({ quality })); | |
| 70 | } | |
| 71 | ||
| 72 | stop() { | |
| 73 | this.closed = true; | |
| 74 | this.timers.forEach((t) => (clearTimeout(t), clearInterval(t))); | |
| 75 | try { this.ws?.close(); } catch {} | |
| 76 | try { this.decoder?.close(); } catch {} | |
| 77 | } | |
| 78 | ||
| 79 | status(state) { | |
| 80 | this.o.onStatus?.({ state, age: this.age, rendition: this.rendition, ladder: this.ladder }); | |
| 81 | } | |
| 82 | ||
| 83 | control(text) { | |
| 84 | let m; | |
| 85 | try { m = JSON.parse(text); } catch { return; } | |
| 86 | if (Array.isArray(m.ladder)) this.ladder = m.ladder; | |
| 87 | if (typeof m.rendition === "string") this.rendition = m.rendition; | |
| 88 | if (this.frames) this.status("live"); | |
| 89 | } | |
| 90 | ||
| 91 | report() { | |
| 92 | if (this.ws?.readyState === 1 && this.age > 0) this.ws.send(JSON.stringify({ age: Math.round(this.age) })); | |
| 93 | } | |
| 94 | ||
| 95 | frame(buffer) { | |
| 96 | const f = parseFrame(buffer); | |
| 97 | if (!f || this.closed) return; | |
| 98 | // Age is measured against the least delay seen, since the relay's clock and the page's differ: it is the queueing, not the distance. | |
| 99 | const delay = this.now() - f.sentAt; | |
| 100 | this.lowest = Math.min(this.lowest, delay); | |
| 101 | this.age = Math.max(0, delay - this.lowest); | |
| 102 | if (this.decoder && this.decoder.decodeQueueSize > 3) this.needKey = true; // behind: skip to the next keyframe | |
| 103 | if (this.needKey && !f.key) return; | |
| 104 | if (f.key && !this.configured && !this.configure(f.data)) return; | |
| 105 | this.needKey = false; | |
| 106 | try { | |
| 107 | this.decoder.decode(new this.Chunk({ type: f.key ? "key" : "delta", timestamp: Math.round(f.sentAt * 1000), data: f.data })); | |
| 108 | } catch { | |
| 109 | this.needKey = true; | |
| 110 | this.configured = false; | |
| 111 | } | |
| 112 | } | |
| 113 | ||
| 114 | configure(unit) { | |
| 115 | const codec = codecOf(unit); | |
| 116 | if (!codec || !this.Decoder) return false; | |
| 117 | try { this.decoder?.close(); } catch {} | |
| 118 | this.decoder = new this.Decoder({ | |
| 119 | output: (frame) => { | |
| 120 | const c = this.o.canvas; | |
| 121 | if (c.width !== frame.displayWidth || c.height !== frame.displayHeight) (c.width = frame.displayWidth), (c.height = frame.displayHeight); | |
| 122 | this.ctx.drawImage(frame, 0, 0); | |
| 123 | frame.close(); | |
| 124 | if (!this.frames++) this.status("live"); | |
| 125 | }, | |
| 126 | error: () => ((this.needKey = true), (this.configured = false)), | |
| 127 | }); | |
| 128 | this.decoder.configure({ codec, optimizeForLatency: true }); | |
| 129 | return (this.configured = true); | |
| 130 | } | |
| 131 | } |
Software/so101/src/ui/video-player.test.ts added +111
| @@ -0,0 +1,111 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { clip } from "../h264-clip.ts"; | |
| 3 | import { codecOf, parseFrame, VideoPlayer } from "./video-player.js"; | |
| 4 | ||
| 5 | const frame = (key: boolean, rung: number, sentAt: number, data: Uint8Array) => { | |
| 6 | const out = Buffer.alloc(10 + data.length); | |
| 7 | out[0] = key ? 1 : 0; | |
| 8 | out[1] = rung; | |
| 9 | out.writeDoubleBE(sentAt, 2); | |
| 10 | out.set(data, 10); | |
| 11 | return out; | |
| 12 | }; | |
| 13 | ||
| 14 | class FakeSocket { | |
| 15 | static last: FakeSocket; | |
| 16 | readyState = 1; | |
| 17 | sent: string[] = []; | |
| 18 | onmessage: ((e: { data: unknown }) => void) | null = null; | |
| 19 | onclose: (() => void) | null = null; | |
| 20 | url: string; | |
| 21 | constructor(url: string) { | |
| 22 | this.url = url; | |
| 23 | FakeSocket.last = this; | |
| 24 | } | |
| 25 | send(s: string) { | |
| 26 | this.sent.push(s); | |
| 27 | } | |
| 28 | close() {} | |
| 29 | } | |
| 30 | ||
| 31 | function setup() { | |
| 32 | const decoded: { type: string; timestamp: number; bytes: number }[] = []; | |
| 33 | const states: string[] = []; | |
| 34 | let configured: unknown; | |
| 35 | let output: (f: unknown) => void = () => {}; | |
| 36 | const drawn: unknown[] = []; | |
| 37 | class FakeDecoder { | |
| 38 | decodeQueueSize = 0; | |
| 39 | constructor(init: { output: (f: unknown) => void }) { | |
| 40 | output = init.output; | |
| 41 | } | |
| 42 | configure(c: unknown) { | |
| 43 | configured = c; | |
| 44 | } | |
| 45 | decode(chunk: { type: string; timestamp: number; data: Uint8Array }) { | |
| 46 | decoded.push({ type: chunk.type, timestamp: chunk.timestamp, bytes: chunk.data.length }); | |
| 47 | output({ displayWidth: 1280, displayHeight: 720, close() {} }); | |
| 48 | } | |
| 49 | close() {} | |
| 50 | } | |
| 51 | const canvas = { width: 0, height: 0, getContext: () => ({ drawImage: (f: unknown) => drawn.push(f) }) }; | |
| 52 | let now = 10_000; | |
| 53 | const player = new VideoPlayer({ url: "ws://x/video", canvas, onStatus: (s) => states.push(s.state), deps: { WebSocket: FakeSocket, VideoDecoder: FakeDecoder, EncodedVideoChunk: class { constructor(init: object) { Object.assign(this, init); } }, now: () => now } }); | |
| 54 | player.start(); | |
| 55 | const ws = FakeSocket.last; | |
| 56 | return { player, ws, decoded, states, drawn, canvas, get configured() { return configured; }, tick: (ms: number) => (now += ms), set: (n: number) => (now = n), decoder: () => player.decoder }; | |
| 57 | } | |
| 58 | ||
| 59 | describe("video player", () => { | |
| 60 | const pics = clip(1, 10, 5, 100); | |
| 61 | ||
| 62 | test("parses the wire frame and the codec", () => { | |
| 63 | const f = parseFrame(frame(true, 2, 1234.5, pics[0]!))!; | |
| 64 | expect(f).toMatchObject({ key: true, rendition: 2, sentAt: 1234.5 }); | |
| 65 | expect(f.data.length).toBe(pics[0]!.length); | |
| 66 | expect(parseFrame(new Uint8Array(10))).toBeNull(); | |
| 67 | expect(codecOf(pics[0]!)).toBe("avc1.42c01f"); | |
| 68 | }); | |
| 69 | ||
| 70 | test("starts at a keyframe, configures from the SPS, decodes to the canvas", () => { | |
| 71 | const t = setup(); | |
| 72 | t.ws.onmessage!({ data: JSON.stringify({ ladder: [{ name: "720p" }], rendition: "720p" }) }); | |
| 73 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); // joined mid-GOP: ignored | |
| 74 | expect(t.decoded).toHaveLength(0); | |
| 75 | t.ws.onmessage!({ data: frame(true, 0, 10_000, pics[0]!) }); | |
| 76 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); | |
| 77 | expect(t.configured).toEqual({ codec: "avc1.42c01f", optimizeForLatency: true }); | |
| 78 | expect(t.decoded.map((d) => d.type)).toEqual(["key", "delta"]); | |
| 79 | expect(t.drawn).toHaveLength(2); | |
| 80 | expect(t.canvas.width).toBe(1280); | |
| 81 | expect(t.states).toContain("live"); | |
| 82 | t.player.stop(); | |
| 83 | }); | |
| 84 | ||
| 85 | test("a decoder that falls behind skips to the next keyframe; age is the delay above the best seen", () => { | |
| 86 | const t = setup(); | |
| 87 | t.ws.onmessage!({ data: frame(true, 0, 9_950, pics[0]!) }); // 50 ms away | |
| 88 | t.ws.onmessage!({ data: frame(false, 0, 9_950, pics[1]!) }); | |
| 89 | t.decoder().decodeQueueSize = 9; | |
| 90 | t.ws.onmessage!({ data: frame(false, 0, 9_800, pics[2]!) }); // 200 ms old: 150 ms above the best | |
| 91 | expect(t.player.age).toBe(150); | |
| 92 | expect(t.decoded).toHaveLength(2); // dropped | |
| 93 | t.decoder().decodeQueueSize = 0; | |
| 94 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[3]!) }); // still no key | |
| 95 | expect(t.decoded).toHaveLength(2); | |
| 96 | t.ws.onmessage!({ data: frame(true, 0, 9_850, pics[5]!) }); // 150 ms away; the best seen is 0 by now | |
| 97 | expect(t.decoded).toHaveLength(3); | |
| 98 | t.player.report(); | |
| 99 | expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ age: 150 }); | |
| 100 | t.player.quality("240p"); | |
| 101 | expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ quality: "240p" }); | |
| 102 | t.player.stop(); | |
| 103 | }); | |
| 104 | ||
| 105 | test("says none when the socket closes before any picture", () => { | |
| 106 | const t = setup(); | |
| 107 | t.ws.onclose!(); | |
| 108 | expect(t.states).toEqual(["none"]); | |
| 109 | t.player.stop(); | |
| 110 | }); | |
| 111 | }); |
Software/so101/src/video-fanout.test.ts added +113
| @@ -0,0 +1,113 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { clip } from "./h264-clip.ts"; | |
| 3 | import { HEADER, LADDER, VideoFanout, type VideoSink } from "./video-fanout.ts"; | |
| 4 | ||
| 5 | function sink(): VideoSink & { sent: Buffer[]; texts: any[]; queued: number } { | |
| 6 | const s = { | |
| 7 | sent: [] as Buffer[], | |
| 8 | texts: [] as any[], | |
| 9 | queued: 0, | |
| 10 | backlog: () => s.queued, | |
| 11 | send: (d: Uint8Array) => void s.sent.push(Buffer.from(d)), | |
| 12 | text: (t: string) => void s.texts.push(JSON.parse(t)), | |
| 13 | }; | |
| 14 | return s; | |
| 15 | } | |
| 16 | const frames = (s: { sent: Buffer[] }) => s.sent.map((m) => ({ key: (m[0]! & 1) === 1, rung: m[1]! })); | |
| 17 | const pictures = clip(10, 10, 5, 100); | |
| 18 | // Each rung's own clip, sized by its bitrate, like the three encoders on the Pi. | |
| 19 | const clips = new Map(LADDER.map((r) => [r.name, clip(10, 10, 5, r.kbps / 3)])); | |
| 20 | /** Every rung gets the next pictures, one at a time. */ | |
| 21 | function play(f: VideoFanout, from: number, to: number) { | |
| 22 | for (let i = from; i < to; i++) for (const r of f.ladder) f.push(r.name, { key: i % 5 === 0, data: clips.get(r.name)![i]! }); | |
| 23 | } | |
| 24 | ||
| 25 | describe("renditions and viewers", () => { | |
| 26 | test("the driver gets the best rung and a watcher the middle one, each from a keyframe", () => { | |
| 27 | const f = new VideoFanout(); | |
| 28 | const driver = sink(), watcher = sink(); | |
| 29 | f.join(driver, true); | |
| 30 | f.join(watcher); | |
| 31 | expect(driver.texts[0].rendition).toBe("720p"); | |
| 32 | expect(watcher.texts[0].rendition).toBe("480p"); | |
| 33 | play(f, 2, 12); // joined mid-GOP: nothing until picture 5 | |
| 34 | expect(frames(driver)[0]).toEqual({ key: true, rung: 0 }); | |
| 35 | expect(frames(watcher)[0]).toEqual({ key: true, rung: 1 }); | |
| 36 | expect(driver.sent).toHaveLength(7); | |
| 37 | expect(driver.sent[0]!.length).toBe(HEADER + clips.get('720p')![5]!.length); | |
| 38 | expect(f.wanted()).toEqual(new Set(["720p", "480p"])); | |
| 39 | }); | |
| 40 | ||
| 41 | test("a slow viewer skips to the next keyframe instead of queueing, then steps down at a keyframe", () => { | |
| 42 | let t = 0; | |
| 43 | const f = new VideoFanout({ now: () => t, dropsToStepDown: 2 }); | |
| 44 | const slow = sink(), fast = sink(); | |
| 45 | const viewer = f.join(slow); | |
| 46 | f.join(fast); | |
| 47 | play(f, 0, 1); | |
| 48 | slow.queued = 1 << 20; // the link backs up | |
| 49 | play(f, 1, 5); // four pictures: dropped, none sent | |
| 50 | expect(slow.sent).toHaveLength(1); | |
| 51 | expect(viewer.stats().skipped).toBe(1); // the first miss; the rest wait for a key without counting | |
| 52 | slow.queued = 1 << 20; | |
| 53 | play(f, 5, 6); | |
| 54 | slow.queued = 0; | |
| 55 | play(f, 6, 10); // clear again but there is no keyframe: still waiting | |
| 56 | expect(slow.sent).toHaveLength(1); | |
| 57 | t += 1000; | |
| 58 | f.adapt(); | |
| 59 | play(f, 10, 11); // a keyframe: the viewer is back | |
| 60 | expect(slow.sent).toHaveLength(2); | |
| 61 | expect(fast.sent).toHaveLength(11); // the fast viewer never missed one | |
| 62 | }); | |
| 63 | ||
| 64 | test("two viewers at different simulated bandwidths end up on different rungs", () => { | |
| 65 | let t = 0; | |
| 66 | const f = new VideoFanout({ now: () => t, dropsToStepDown: 1, cleanMs: 5000, maxBacklog: 2000 }); | |
| 67 | const wifi = sink(), mobile = sink(); | |
| 68 | f.join(wifi); | |
| 69 | const m = f.join(mobile); | |
| 70 | // The mobile link carries 300 bytes per 100 ms; whatever it cannot carry sits in its backlog. | |
| 71 | const send = mobile.send; | |
| 72 | mobile.send = (d) => (send(d), (mobile.queued += d.length)); | |
| 73 | for (let second = 0; second < 30; second++) { | |
| 74 | for (let i = 0; i < 10; i++) { | |
| 75 | mobile.queued = Math.max(0, mobile.queued - 300); | |
| 76 | const n = (second * 10 + i) % 100; | |
| 77 | play(f, n % 10, (n % 10) + 1); | |
| 78 | t += 100; | |
| 79 | } | |
| 80 | f.adapt(); | |
| 81 | } | |
| 82 | const onRung = (s: { sent: Buffer[] }, rung: number) => frames(s).filter((x) => x.rung === rung).length; | |
| 83 | expect(mobile.texts.some((x) => x.rendition === "240p")).toBe(true); | |
| 84 | expect(onRung(mobile, 2)).toBeGreaterThan(onRung(mobile, 1) + onRung(mobile, 0)); | |
| 85 | expect(onRung(wifi, 0)).toBeGreaterThan(onRung(wifi, 2)); // clean for 5 s: stepped up to the best | |
| 86 | expect(wifi.texts.some((x) => x.rendition === "720p")).toBe(true); | |
| 87 | expect(m.stats().bytes * 3).toBeLessThan(wifi.sent.reduce((n, b) => n + b.length, 0)); | |
| 88 | }); | |
| 89 | ||
| 90 | test("a pinned quality holds against drops; auto releases it; junk is ignored", () => { | |
| 91 | const f = new VideoFanout(); | |
| 92 | const s = sink(); | |
| 93 | const v = f.join(s); | |
| 94 | v.message('{"quality":"720p"}'); | |
| 95 | v.message("not json"); | |
| 96 | v.message('{"quality":"4k"}'); | |
| 97 | v.message('{"quality":' + '9'.repeat(300) + "}"); | |
| 98 | play(f, 0, 1); | |
| 99 | expect(v.stats().rendition).toBe("720p"); | |
| 100 | for (let i = 0; i < 5; i++) (v.message('{"age":2000}'), f.adapt()); | |
| 101 | expect(f.wanted()).toEqual(new Set(["720p"])); | |
| 102 | v.message('{"quality":"auto"}'); | |
| 103 | }); | |
| 104 | ||
| 105 | test("a viewer that leaves is no longer fed or wanted", () => { | |
| 106 | const f = new VideoFanout(); | |
| 107 | const s = sink(); | |
| 108 | f.join(s).leave(); | |
| 109 | play(f, 0, 3); | |
| 110 | expect(s.sent).toHaveLength(0); | |
| 111 | expect(f.wanted().size).toBe(0); | |
| 112 | }); | |
| 113 | }); |
Software/so101/src/video-fanout.ts added +197
| @@ -0,0 +1,197 @@ | ||
| 1 | /** | |
| 2 | * One camera's H.264 renditions, fanned out to viewers over their WebSockets, a viewer on one rendition at a time. | |
| 3 | * | |
| 4 | * Latency before everything: nothing is queued for a viewer. A viewer whose socket is backed up drops pictures and | |
| 5 | * waits for the next keyframe, and that same backlog is how its link is measured: a viewer that dropped steps down | |
| 6 | * the ladder, one that went a while clean steps back up, each switch at a keyframe of the new rendition. A viewer can | |
| 7 | * pin one rendition. The driver is patient (it steps down only after more drops) so the best picture stays with it. | |
| 8 | * | |
| 9 | * A binary message is HEADER bytes then the access unit: byte 0 bit 0 = keyframe, byte 1 = the rendition's index in | |
| 10 | * the ladder, bytes 2..9 = the time the relay got the picture (ms, float64 big-endian). Text messages are JSON. | |
| 11 | */ | |
| 12 | ||
| 13 | import type { AccessUnit } from "./h264.ts"; | |
| 14 | ||
| 15 | export interface Rendition { | |
| 16 | name: string; | |
| 17 | width: number; | |
| 18 | height: number; | |
| 19 | fps: number; | |
| 20 | kbps: number; | |
| 21 | } | |
| 22 | ||
| 23 | /** The ladder the Pi encodes, best first. */ | |
| 24 | export const LADDER: readonly Rendition[] = [ | |
| 25 | { name: "720p", width: 1280, height: 720, fps: 20, kbps: 1500 }, | |
| 26 | { name: "480p", width: 854, height: 480, fps: 15, kbps: 600 }, | |
| 27 | { name: "240p", width: 426, height: 240, fps: 10, kbps: 200 }, | |
| 28 | ]; | |
| 29 | ||
| 30 | export const HEADER = 10; | |
| 31 | ||
| 32 | export interface VideoSink { | |
| 33 | /** Bytes accepted but not yet on the wire. */ | |
| 34 | backlog(): number; | |
| 35 | send(data: Uint8Array): void; | |
| 36 | text(message: string): void; | |
| 37 | } | |
| 38 | ||
| 39 | export interface FanoutOptions { | |
| 40 | ladder?: readonly Rendition[]; | |
| 41 | /** Backlog past which a picture is dropped for that viewer. */ | |
| 42 | maxBacklog?: number; | |
| 43 | /** Drops (in one window) that step an ordinary viewer down; the driver takes twice as many. */ | |
| 44 | dropsToStepDown?: number; | |
| 45 | /** How long a viewer must go without a drop before it steps up. */ | |
| 46 | cleanMs?: number; | |
| 47 | /** A frame age the viewer reports above this also counts as a drop. */ | |
| 48 | maxAgeMs?: number; | |
| 49 | now?: () => number; | |
| 50 | } | |
| 51 | ||
| 52 | interface Viewer { | |
| 53 | sink: VideoSink; | |
| 54 | driver: boolean; | |
| 55 | /** Index of the rendition the viewer is on, and the one it is moving to. */ | |
| 56 | current: number; | |
| 57 | target: number; | |
| 58 | pinned: number | null; | |
| 59 | /** Waiting for a keyframe on `current`. */ | |
| 60 | needKey: boolean; | |
| 61 | drops: number; | |
| 62 | lastDrop: number; | |
| 63 | lastStep: number; | |
| 64 | bytes: number; | |
| 65 | frames: number; | |
| 66 | skipped: number; | |
| 67 | } | |
| 68 | ||
| 69 | export interface ViewerStats { | |
| 70 | rendition: string; | |
| 71 | bytes: number; | |
| 72 | frames: number; | |
| 73 | skipped: number; | |
| 74 | } | |
| 75 | ||
| 76 | export class VideoFanout { | |
| 77 | readonly ladder: readonly Rendition[]; | |
| 78 | private readonly viewers = new Set<Viewer>(); | |
| 79 | private readonly o: Required<Omit<FanoutOptions, "ladder">>; | |
| 80 | ||
| 81 | constructor(options: FanoutOptions = {}) { | |
| 82 | this.ladder = options.ladder ?? LADDER; | |
| 83 | this.o = { maxBacklog: 64 << 10, dropsToStepDown: 3, cleanMs: 6000, maxAgeMs: 500, now: Date.now, ...options }; | |
| 84 | } | |
| 85 | ||
| 86 | get watching(): number { | |
| 87 | return this.viewers.size; | |
| 88 | } | |
| 89 | ||
| 90 | /** The renditions someone is on or moving to: the only ones worth pulling from the Pi. */ | |
| 91 | wanted(): Set<string> { | |
| 92 | const names = new Set<string>(); | |
| 93 | for (const v of this.viewers) names.add(this.ladder[v.current]!.name).add(this.ladder[v.target]!.name); | |
| 94 | return names; | |
| 95 | } | |
| 96 | ||
| 97 | /** A viewer joins on the best rendition (the driver) or the middle of the ladder (a watcher), and waits for a keyframe. */ | |
| 98 | join(sink: VideoSink, driver = false): { leave(): void; message(text: string): void; stats(): ViewerStats } { | |
| 99 | const start = driver ? 0 : Math.min(1, this.ladder.length - 1); | |
| 100 | const now = this.o.now(); | |
| 101 | const v: Viewer = { sink, driver, current: start, target: start, pinned: null, needKey: true, drops: 0, lastDrop: now, lastStep: now, bytes: 0, frames: 0, skipped: 0 }; | |
| 102 | this.viewers.add(v); | |
| 103 | sink.text(JSON.stringify({ ladder: this.ladder, rendition: this.ladder[start]!.name })); | |
| 104 | return { | |
| 105 | leave: () => void this.viewers.delete(v), | |
| 106 | message: (text) => this.message(v, text), | |
| 107 | stats: () => ({ rendition: this.ladder[v.current]!.name, bytes: v.bytes, frames: v.frames, skipped: v.skipped }), | |
| 108 | }; | |
| 109 | } | |
| 110 | ||
| 111 | /** A viewer's JSON: {"quality":"auto"|<name>} to pin one, {"age":ms} for the frame age it measures. Anything else is ignored. */ | |
| 112 | private message(v: Viewer, text: string) { | |
| 113 | if (text.length > 200) return; | |
| 114 | let m: { quality?: unknown; age?: unknown }; | |
| 115 | try { | |
| 116 | m = JSON.parse(text); | |
| 117 | } catch { | |
| 118 | return; | |
| 119 | } | |
| 120 | if (typeof m !== "object" || m === null) return; | |
| 121 | if (typeof m.quality === "string") { | |
| 122 | const at = this.ladder.findIndex((r) => r.name === m.quality); | |
| 123 | if (m.quality === "auto") v.pinned = null; | |
| 124 | else if (at >= 0) (v.pinned = at), (v.target = at); | |
| 125 | } | |
| 126 | if (typeof m.age === "number" && Number.isFinite(m.age) && m.age > this.o.maxAgeMs) this.drop(v); | |
| 127 | } | |
| 128 | ||
| 129 | /** The next picture of a rendition arrives from the Pi. */ | |
| 130 | push(rendition: string, unit: AccessUnit): void { | |
| 131 | const at = this.ladder.findIndex((r) => r.name === rendition); | |
| 132 | if (at < 0) return; | |
| 133 | const now = this.o.now(); | |
| 134 | const message = Buffer.allocUnsafe(HEADER + unit.data.length); | |
| 135 | message[0] = unit.key ? 1 : 0; | |
| 136 | message[1] = at; | |
| 137 | message.writeDoubleBE(now, 2); | |
| 138 | unit.data.copy(message, HEADER); | |
| 139 | for (const v of this.viewers) { | |
| 140 | if (v.target === at && v.current !== at && unit.key) { | |
| 141 | v.current = at; | |
| 142 | v.needKey = false; | |
| 143 | v.lastStep = now; | |
| 144 | } | |
| 145 | if (v.current !== at) continue; | |
| 146 | if (v.needKey && (!unit.key || v.target !== v.current)) continue; // waiting for a key, and for the rung it is moving to | |
| 147 | ||
| 148 | if (v.sink.backlog() > this.o.maxBacklog) { | |
| 149 | v.skipped++; | |
| 150 | v.needKey = true; | |
| 151 | this.drop(v); | |
| 152 | continue; | |
| 153 | } | |
| 154 | v.needKey = false; | |
| 155 | v.sink.send(message); | |
| 156 | v.bytes += message.length; | |
| 157 | v.frames++; | |
| 158 | } | |
| 159 | } | |
| 160 | ||
| 161 | private drop(v: Viewer) { | |
| 162 | const now = this.o.now(); | |
| 163 | v.drops++; | |
| 164 | v.lastDrop = now; | |
| 165 | v.needKey = true; | |
| 166 | } | |
| 167 | ||
| 168 | /** Run about once a second: steps viewers down after drops and up after a clean while. */ | |
| 169 | adapt(): void { | |
| 170 | const now = this.o.now(); | |
| 171 | for (const v of this.viewers) { | |
| 172 | if (v.pinned !== null) { | |
| 173 | v.drops = 0; | |
| 174 | continue; | |
| 175 | } | |
| 176 | const limit = this.o.dropsToStepDown * (v.driver ? 2 : 1); | |
| 177 | if (v.drops >= limit && v.target < this.ladder.length - 1) { | |
| 178 | v.target = v.current + 1 > this.ladder.length - 1 ? v.current : v.current + 1; | |
| 179 | this.announce(v); | |
| 180 | } else if (v.drops === 0 && v.target === v.current && v.current > 0 && now - v.lastDrop >= this.o.cleanMs && now - v.lastStep >= this.o.cleanMs) { | |
| 181 | v.target = v.current - 1; | |
| 182 | this.announce(v); | |
| 183 | } | |
| 184 | if (v.drops >= limit) v.drops = 0; | |
| 185 | else if (now - v.lastDrop > 2000) v.drops = 0; | |
| 186 | } | |
| 187 | } | |
| 188 | ||
| 189 | private announce(v: Viewer) { | |
| 190 | v.lastStep = this.o.now(); | |
| 191 | v.sink.text(JSON.stringify({ rendition: this.ladder[v.target]!.name })); | |
| 192 | } | |
| 193 | ||
| 194 | close(): void { | |
| 195 | this.viewers.clear(); | |
| 196 | } | |
| 197 | } |
Software/so101/src/video-feed.test.ts added +100
| @@ -0,0 +1,100 @@ | ||
| 1 | import { afterEach, describe, expect, test } from "bun:test"; | |
| 2 | import { createServer, type Server } from "node:http"; | |
| 3 | import WebSocket, { WebSocketServer } from "ws"; | |
| 4 | import { clip } from "./h264-clip.ts"; | |
| 5 | import { parseVideo } from "./relay.ts"; | |
| 6 | import { HEADER, LADDER } from "./video-fanout.ts"; | |
| 7 | import { VideoFeed } from "./video-feed.ts"; | |
| 8 | ||
| 9 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 10 | const closers: (() => void)[] = []; | |
| 11 | afterEach(() => closers.splice(0).forEach((c) => c())); | |
| 12 | ||
| 13 | /** The Pi: each rung is an endless Annex-B stream of its own clip, 20 pictures a second, looping. */ | |
| 14 | async function pi(): Promise<string> { | |
| 15 | const clips = new Map(LADDER.map((r) => [`/${r.name}`, clip(2, 20, 10, r.kbps / 3)])); | |
| 16 | const server: Server = createServer((req, res) => { | |
| 17 | const pictures = clips.get(req.url ?? ""); | |
| 18 | if (!pictures) return void res.writeHead(404).end(); | |
| 19 | res.writeHead(200, { "content-type": "video/h264" }); | |
| 20 | let i = 0; | |
| 21 | const timer = setInterval(() => res.write(pictures[i++ % pictures.length]!), 50); | |
| 22 | res.on("close", () => clearInterval(timer)); | |
| 23 | }); | |
| 24 | await new Promise<void>((r) => server.listen(0, "127.0.0.1", r)); | |
| 25 | closers.push(() => (server.closeAllConnections(), server.close())); | |
| 26 | return `http://127.0.0.1:${(server.address() as { port: number }).port}`; | |
| 27 | } | |
| 28 | ||
| 29 | /** A relay-shaped server: every WebSocket is a viewer; `driver` is chosen by the path. */ | |
| 30 | async function relay(feed: VideoFeed): Promise<string> { | |
| 31 | const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" }); | |
| 32 | wss.on("connection", (ws, req) => feed.serve(ws, req.url === "/driver")); | |
| 33 | await new Promise((r) => wss.once("listening", r)); | |
| 34 | closers.push(() => (feed.close(), wss.close())); | |
| 35 | return `ws://127.0.0.1:${(wss.address() as { port: number }).port}`; | |
| 36 | } | |
| 37 | ||
| 38 | function watch(url: string) { | |
| 39 | const got = { frames: [] as { key: boolean; rung: number; sentAt: number; bytes: number }[], texts: [] as any[] }; | |
| 40 | const ws = new WebSocket(url); | |
| 41 | ws.on("message", (data, binary) => { | |
| 42 | const m = data as Buffer; | |
| 43 | if (!binary) return void got.texts.push(JSON.parse(String(data))); | |
| 44 | got.frames.push({ key: (m[0]! & 1) === 1, rung: m[1]!, sentAt: m.readDoubleBE(2), bytes: m.length }); | |
| 45 | }); | |
| 46 | closers.push(() => ws.close()); | |
| 47 | return { ws, got }; | |
| 48 | } | |
| 49 | ||
| 50 | describe("video feed", () => { | |
| 51 | test("a driver and a watcher are fed different renditions of the Pi's streams, from keyframes, and only those are pulled", async () => { | |
| 52 | const base = await pi(); | |
| 53 | const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` }); | |
| 54 | const url = await relay(feed); | |
| 55 | const driver = watch(`${url}/driver`); | |
| 56 | const watcher = watch(`${url}/watcher`); | |
| 57 | await sleep(700); | |
| 58 | expect(driver.got.frames.length).toBeGreaterThan(5); | |
| 59 | expect(driver.got.frames[0]!.key).toBe(true); | |
| 60 | expect(watcher.got.frames[0]!.key).toBe(true); | |
| 61 | expect(new Set(driver.got.frames.map((f) => f.rung))).toEqual(new Set([0])); | |
| 62 | expect(new Set(watcher.got.frames.map((f) => f.rung))).toEqual(new Set([1])); | |
| 63 | expect(driver.got.frames.at(-1)!.bytes).toBeGreaterThan(HEADER); | |
| 64 | expect(Math.abs(Date.now() - driver.got.frames.at(-1)!.sentAt)).toBeLessThan(500); | |
| 65 | expect(feed.fanout.wanted()).toEqual(new Set(["720p", "480p"])); // 240p is not pulled | |
| 66 | expect(driver.got.texts[0].ladder.map((r: { name: string }) => r.name)).toEqual(["720p", "480p", "240p"]); | |
| 67 | }); | |
| 68 | ||
| 69 | test("a viewer pinning a quality gets that rung, switched at a keyframe", async () => { | |
| 70 | const base = await pi(); | |
| 71 | const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` }); | |
| 72 | const { ws, got } = watch(`${await relay(feed)}/watcher`); | |
| 73 | await sleep(400); | |
| 74 | ws.send(JSON.stringify({ quality: "240p" })); | |
| 75 | await sleep(1500); | |
| 76 | const at = got.frames.findIndex((f) => f.rung === 2); | |
| 77 | expect(at).toBeGreaterThan(0); | |
| 78 | expect(got.frames[at]!.key).toBe(true); | |
| 79 | expect(got.frames.slice(at).every((f) => f.rung === 2)).toBe(true); | |
| 80 | }); | |
| 81 | ||
| 82 | test("a feed without sources is not available; a down source just sends nothing", async () => { | |
| 83 | expect(new VideoFeed({}).available).toBe(false); | |
| 84 | const feed = new VideoFeed({ "720p": "http://127.0.0.1:1/x" }); | |
| 85 | expect(feed.available).toBe(true); | |
| 86 | const { got } = watch(`${await relay(feed)}/driver`); | |
| 87 | await sleep(300); | |
| 88 | expect(got.frames).toHaveLength(0); | |
| 89 | expect(got.texts[0].rendition).toBe("720p"); | |
| 90 | }); | |
| 91 | }); | |
| 92 | ||
| 93 | describe("ARM_H264_n", () => { | |
| 94 | test("names rungs of the ladder with http URLs", () => { | |
| 95 | expect(parseVideo({})).toEqual({}); | |
| 96 | expect(parseVideo({ ARM_H264_1: "720p=pi:8082/h264, 240p=http://pi:8083/h264" })).toEqual({ 1: { "720p": "http://pi:8082/h264", "240p": "http://pi:8083/h264" } }); | |
| 97 | expect(() => parseVideo({ ARM_H264_1: "4k=pi:1" })).toThrow(/ladder/); | |
| 98 | expect(() => parseVideo({ ARM_H264_1: "720p=ftp://pi/x" })).toThrow(); | |
| 99 | }); | |
| 100 | }); |
Software/so101/src/video-feed.ts added +107
| @@ -0,0 +1,107 @@ | ||
| 1 | /** | |
| 2 | * One camera's H.264 over WebSockets: a VideoFanout fed by the Pi's encoder streams. | |
| 3 | * | |
| 4 | * Each rendition is an HTTP stream of Annex-B H.264 on the Pi. It is pulled only while some viewer is on it (or | |
| 5 | * moving to it), so a camera nobody watches costs the Pi nothing, and the lower rungs cost nothing until a viewer | |
| 6 | * needs them. A source that is down is tried again every second while wanted; its viewers simply see no pictures, and | |
| 7 | * the page falls back to MJPEG when none come. | |
| 8 | */ | |
| 9 | ||
| 10 | import { AccessUnitSplitter } from "./h264.ts"; | |
| 11 | import { VideoFanout, type FanoutOptions, type VideoSink } from "./video-fanout.ts"; | |
| 12 | ||
| 13 | export interface VideoSources { | |
| 14 | /** Rendition name → the URL of its Annex-B stream. */ | |
| 15 | [name: string]: string; | |
| 16 | } | |
| 17 | ||
| 18 | /** The wire the feed talks to a viewer over; a `ws` WebSocket fits. */ | |
| 19 | export interface VideoSocket { | |
| 20 | bufferedAmount: number; | |
| 21 | readyState: number; | |
| 22 | send(data: Uint8Array | string): void; | |
| 23 | on(event: "message", listener: (data: unknown, isBinary: boolean) => void): unknown; | |
| 24 | on(event: "close", listener: () => void): unknown; | |
| 25 | } | |
| 26 | ||
| 27 | const OPEN = 1; | |
| 28 | ||
| 29 | export class VideoFeed { | |
| 30 | readonly fanout: VideoFanout; | |
| 31 | private readonly sources: VideoSources; | |
| 32 | private readonly pulls = new Map<string, AbortController>(); | |
| 33 | private timer: ReturnType<typeof setInterval> | undefined; | |
| 34 | private closed = false; | |
| 35 | ||
| 36 | constructor(sources: VideoSources, options: FanoutOptions = {}) { | |
| 37 | this.sources = sources; | |
| 38 | this.fanout = new VideoFanout(options); | |
| 39 | } | |
| 40 | ||
| 41 | /** Whether this feed has an encoder stream for any rung of the ladder. */ | |
| 42 | get available(): boolean { | |
| 43 | return this.fanout.ladder.some((r) => this.sources[r.name]); | |
| 44 | } | |
| 45 | ||
| 46 | /** Serve one viewer's WebSocket until it closes. */ | |
| 47 | serve(ws: VideoSocket, driver: boolean): void { | |
| 48 | if (this.closed) return; | |
| 49 | const sink: VideoSink = { | |
| 50 | backlog: () => ws.bufferedAmount, | |
| 51 | send: (data) => ws.readyState === OPEN && ws.send(data), | |
| 52 | text: (message) => ws.readyState === OPEN && ws.send(message), | |
| 53 | }; | |
| 54 | const viewer = this.fanout.join(sink, driver); | |
| 55 | ws.on("message", (data, isBinary) => { | |
| 56 | if (!isBinary) viewer.message(String(data)); | |
| 57 | }); | |
| 58 | ws.on("close", () => { | |
| 59 | viewer.leave(); | |
| 60 | if (!this.fanout.watching) this.idle(); | |
| 61 | }); | |
| 62 | this.timer ??= setInterval(() => this.tick(), 1000); | |
| 63 | this.timer.unref?.(); | |
| 64 | this.tick(); | |
| 65 | } | |
| 66 | ||
| 67 | close(): void { | |
| 68 | this.closed = true; | |
| 69 | this.fanout.close(); | |
| 70 | this.idle(); | |
| 71 | } | |
| 72 | ||
| 73 | private idle() { | |
| 74 | clearInterval(this.timer); | |
| 75 | this.timer = undefined; | |
| 76 | for (const c of this.pulls.values()) c.abort(); | |
| 77 | this.pulls.clear(); | |
| 78 | } | |
| 79 | ||
| 80 | private tick() { | |
| 81 | this.fanout.adapt(); | |
| 82 | const wanted = this.fanout.wanted(); | |
| 83 | for (const [name, controller] of this.pulls) { | |
| 84 | if (!wanted.has(name)) (controller.abort(), this.pulls.delete(name)); | |
| 85 | } | |
| 86 | for (const name of wanted) if (this.sources[name] && !this.pulls.has(name)) this.pull(name, this.sources[name]!); | |
| 87 | } | |
| 88 | ||
| 89 | private pull(name: string, url: string) { | |
| 90 | const controller = new AbortController(); | |
| 91 | this.pulls.set(name, controller); | |
| 92 | void (async () => { | |
| 93 | try { | |
| 94 | const response = await fetch(url, { signal: controller.signal, cache: "no-store", redirect: "error" }); | |
| 95 | if (!response.ok || !response.body) throw new Error(`answered ${response.status}`); | |
| 96 | const splitter = new AccessUnitSplitter(); | |
| 97 | for await (const chunk of response.body as unknown as AsyncIterable<Uint8Array>) { | |
| 98 | for (const unit of splitter.push(chunk)) this.fanout.push(name, unit); | |
| 99 | } | |
| 100 | } catch { | |
| 101 | // down or cut off: the next tick tries again while the rung is wanted | |
| 102 | } finally { | |
| 103 | if (this.pulls.get(name) === controller) this.pulls.delete(name); | |
| 104 | } | |
| 105 | })(); | |
| 106 | } | |
| 107 | } |
Software/so101/src/viewer.html +35 −2
| @@ -65,7 +65,9 @@ | ||
| 65 | 65 | #banner button { margin-left: 10px; min-height: 32px; padding: 2px 10px; } |
| 66 | 66 | .pane { flex: 1; min-width: 0; position: relative; background: #000; display: none; } |
| 67 | 67 | .pane.on { display: block; } |
| 68 | .pane img { width: 100%; height: 100%; object-fit: contain; display: block; } | |
| 68 | .pane img, .pane canvas { width: 100%; height: 100%; object-fit: contain; display: block; } | |
| 69 | .pane canvas { position: absolute; inset: 0; } | |
| 70 | .pane select.quality { position: absolute; right: calc(8px + var(, 0px)); bottom: 8px; font-size: 11px; background: rgba(0,0,0,.55); color: #c9d1d9; border: 0; border-radius: 4px; padding: 2px 4px; } | |
| 69 | 71 | .pane .tag { position: absolute; left: calc(8px + var()); bottom: 8px; font-size: 11px; color: #c9d1d9; background: rgba(0,0,0,.55); padding: 2px 6px; border-radius: 4px; pointer-events: none; } |
| 70 | 72 | urdf-viewer { display: block; width: 100%; height: 100%; } |
| 71 | 73 | #top { position: absolute; top: 6px; right: calc(6px + var()); left: calc(6px + var()); z-index: 5; display: flex; gap: 6px; align-items: center; justify-content: flex-end; flex-wrap: wrap; font-size: 12px; pointer-events: none; } |
| @@ -640,11 +642,40 @@ | ||
| 640 | 642 | } catch { c.live = false; } |
| 641 | 643 | if (plan.mode === "snapshot") c.timer = setTimeout(() => snap(c), Math.max(0, plan.everyMs - (performance.now() - started))); |
| 642 | 644 | } |
| 645 | // H.264 over the relay's WebSocket, decoded into a canvas, where the browser has WebCodecs and the camera offers it; MJPEG otherwise, and when no picture comes. | |
| 646 | function stopVideo(c) { | |
| 647 | c.player?.stop(); c.player = null; | |
| 648 | c.canvas?.remove(); c.canvas = null; c.chip?.remove(); c.chip = null; | |
| 649 | c.img.style.visibility = ""; | |
| 650 | } | |
| 651 | async function startVideo(c) { | |
| 652 | stopVideo(c); | |
| 653 | const { VideoPlayer } = await import("/ui/__VERSION__/video-player.js"); // only a viewer who gets H.264 downloads it | |
| 654 | if (!paneOn(c) || c.player || c.canvas) return; | |
| 655 | c.canvas = el("canvas", { ariaLabel: c.img.alt }); | |
| 656 | c.chip = el("select", { className: "quality", title: "Video quality" }); | |
| 657 | c.chip.onchange = () => c.player?.quality(c.chip.value); | |
| 658 | c.pane.append(c.canvas, c.chip); | |
| 659 | const player = c.player = new VideoPlayer({ | |
| 660 | url: `${location.protocol === "https:" ? "wss:" : "ws:"}//${location.host}/camera/${c.id}/video`, | |
| 661 | canvas: c.canvas, | |
| 662 | onStatus: (s) => { | |
| 663 | if (c.player !== player) return; | |
| 664 | if (s.state === "none") { c.h264Failed = true; stopVideo(c); if (paneOn(c)) c.img.src = streamUrl(c); return; } | |
| 665 | c.img.style.visibility = "hidden"; | |
| 666 | if (c.chip.options.length !== s.ladder.length + 1) c.chip.replaceChildren(new Option("auto", "auto"), ...s.ladder.map((r) => new Option(r.name, r.name))); | |
| 667 | c.tag.textContent = `${c.name} · ${s.rendition ?? ""} · ${Math.round(s.age)} ms`; | |
| 668 | }, | |
| 669 | }); | |
| 670 | player.start(); | |
| 671 | } | |
| 643 | 672 | const showCam = (c, on) => { |
| 644 | 673 | c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on)); |
| 645 | 674 | clearTimeout(c.timer); |
| 675 | stopVideo(c); | |
| 646 | 676 | if (!on || document.hidden) c.img.src = ""; |
| 647 | 677 | else if (plan.mode === "snapshot" && c.snapshot) snap(c); |
| 678 | else if (c.h264 && !c.h264Failed && relayed && typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function") { c.img.src = ""; startVideo(c); } | |
| 648 | 679 | else c.img.src = streamUrl(c); |
| 649 | 680 | }; |
| 650 | 681 | function setPane(key, on) { |
| @@ -700,7 +731,7 @@ | ||
| 700 | 731 | } |
| 701 | 732 | function label(c, info) { |
| 702 | 733 | c.online = info.online; |
| 703 | c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name; | |
| 734 | if (!c.player) c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name; | |
| 704 | 735 | c.btn.textContent = info.name; c.btn.title = info.online === false ? `${info.name}: offline` : info.name; |
| 705 | 736 | c.img.alt = info.name; |
| 706 | 737 | } |
| @@ -710,8 +741,10 @@ | ||
| 710 | 741 | let c = cams.find((x) => x.id === info.id); |
| 711 | 742 | if (!c) { |
| 712 | 743 | c = addCam({ ...info, url: relayed ? `/camera/${info.id}/stream` : info.url }); |
| 744 | c.h264 = info.h264 === true; c.name = info.name; | |
| 713 | 745 | if (viewsReady && wanted(c.pane.dataset.key)) showCam(c, true); |
| 714 | 746 | } |
| 747 | c.h264 = info.h264 === true; c.name = info.name; | |
| 715 | 748 | label(c, info); |
| 716 | 749 | } |
| 717 | 750 | showQuality(); |
new above
offline