Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
src/ui/video-player.js decodes H.264 access units with WebCodecs into a canvas and reports when it cannot - module and test only #416
vesper/arm:vesper/258-src-ui-video-player-js-decodes-h-264-acc → main 3 commits fork
2 files
changed +296
−0
Software/so101/src/ui/video-player.js added +131
| @@ -0,0 +1,131 @@ | ||
| 1 | // @ts-nocheck -- WebCodecs and the DOM are not in this project's lib; the tests drive the class with fakes | |
| 2 | /** | |
| 3 | * The camera as H.264 over the relay's WebSocket (/camera/<id>/video), decoded with WebCodecs into a canvas. | |
| 4 | * Plain ESM, served at /ui/ and imported by the tests. The wire format is documented in src/video-fanout.ts: | |
| 5 | * 10 header bytes (keyframe flag, rendition index, relay time float64) then one Annex-B access unit. | |
| 6 | * | |
| 7 | * Nothing is queued: a picture that arrives while the decoder is behind is dropped, and so is every one after it | |
| 8 | * until the next keyframe. `onStatus` hears "live", "none" (no picture in `waitMs`: use MJPEG) and the frame age. | |
| 9 | */ | |
| 10 | ||
| 11 | export const HEADER = 10; | |
| 12 | ||
| 13 | /** @param {ArrayBuffer | Uint8Array} data */ | |
| 14 | export function parseFrame(data) { | |
| 15 | const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); | |
| 16 | if (bytes.length <= HEADER) return null; | |
| 17 | const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); | |
| 18 | return { key: (bytes[0] & 1) === 1, rendition: bytes[1], sentAt: view.getFloat64(2), data: bytes.subarray(HEADER) }; | |
| 19 | } | |
| 20 | ||
| 21 | /** "avc1.PPCCLL" from the first SPS of an Annex-B unit, or null. @param {Uint8Array} unit */ | |
| 22 | export function codecOf(unit) { | |
| 23 | for (let i = 0; i + 6 < unit.length; i++) { | |
| 24 | if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3] & 0x1f) === 7) { | |
| 25 | return "avc1." + [4, 5, 6].map((n) => unit[i + n].toString(16).padStart(2, "0")).join(""); | |
| 26 | } | |
| 27 | } | |
| 28 | return null; | |
| 29 | } | |
| 30 | ||
| 31 | export const canDecode = () => typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function" && typeof WebSocket === "function"; | |
| 32 | ||
| 33 | export class VideoPlayer { | |
| 34 | /** | |
| 35 | * @param {{ url: string, canvas: { width: number, height: number, getContext(k: string): any }, onStatus?: (s: { state: string, age?: number, rendition?: string, ladder?: any[] }) => void, waitMs?: number, | |
| 36 | * deps?: { WebSocket?: any, VideoDecoder?: any, EncodedVideoChunk?: any, now?: () => number } }} o | |
| 37 | */ | |
| 38 | constructor(o) { | |
| 39 | this.o = o; | |
| 40 | const d = o.deps ?? {}; | |
| 41 | this.WS = d.WebSocket ?? WebSocket; | |
| 42 | this.Decoder = d.VideoDecoder ?? (typeof VideoDecoder === "undefined" ? undefined : VideoDecoder); | |
| 43 | this.Chunk = d.EncodedVideoChunk ?? (typeof EncodedVideoChunk === "undefined" ? undefined : EncodedVideoChunk); | |
| 44 | this.now = d.now ?? (() => Date.now()); | |
| 45 | this.ctx = o.canvas.getContext("2d"); | |
| 46 | this.decoder = null; | |
| 47 | this.configured = false; | |
| 48 | this.needKey = true; | |
| 49 | this.lowest = Infinity; | |
| 50 | this.age = 0; | |
| 51 | this.frames = 0; | |
| 52 | this.ladder = []; | |
| 53 | this.closed = false; | |
| 54 | this.ws = null; | |
| 55 | this.timers = []; | |
| 56 | } | |
| 57 | ||
| 58 | start() { | |
| 59 | const ws = (this.ws = new this.WS(this.o.url)); | |
| 60 | ws.binaryType = "arraybuffer"; | |
| 61 | ws.onmessage = (e) => (typeof e.data === "string" ? this.control(e.data) : this.frame(e.data)); | |
| 62 | ws.onclose = () => this.status("none"); | |
| 63 | this.timers.push(setTimeout(() => this.frames || this.status("none"), this.o.waitMs ?? 3000)); | |
| 64 | this.timers.push(setInterval(() => this.report(), 1000)); | |
| 65 | } | |
| 66 | ||
| 67 | /** Pin a rung by name, or "auto". @param {string} quality */ | |
| 68 | quality(quality) { | |
| 69 | if (this.ws?.readyState === 1) this.ws.send(JSON.stringify({ quality })); | |
| 70 | } | |
| 71 | ||
| 72 | stop() { | |
| 73 | this.closed = true; | |
| 74 | this.timers.forEach((t) => (clearTimeout(t), clearInterval(t))); | |
| 75 | try { this.ws?.close(); } catch {} | |
| 76 | try { this.decoder?.close(); } catch {} | |
| 77 | } | |
| 78 | ||
| 79 | status(state) { | |
| 80 | this.o.onStatus?.({ state, age: this.age, rendition: this.rendition, ladder: this.ladder }); | |
| 81 | } | |
| 82 | ||
| 83 | control(text) { | |
| 84 | let m; | |
| 85 | try { m = JSON.parse(text); } catch { return; } | |
| 86 | if (Array.isArray(m.ladder)) this.ladder = m.ladder; | |
| 87 | if (typeof m.rendition === "string") this.rendition = m.rendition; | |
| 88 | if (this.frames) this.status("live"); | |
| 89 | } | |
| 90 | ||
| 91 | report() { | |
| 92 | if (this.ws?.readyState === 1 && this.age > 0) this.ws.send(JSON.stringify({ age: Math.round(this.age) })); | |
| 93 | } | |
| 94 | ||
| 95 | frame(buffer) { | |
| 96 | const f = parseFrame(buffer); | |
| 97 | if (!f || this.closed) return; | |
| 98 | // Age is measured against the least delay seen, since the relay's clock and the page's differ: it is the queueing, not the distance. | |
| 99 | const delay = this.now() - f.sentAt; | |
| 100 | this.lowest = Math.min(this.lowest, delay); | |
| 101 | this.age = Math.max(0, delay - this.lowest); | |
| 102 | if (this.decoder && this.decoder.decodeQueueSize > 3) this.needKey = true; // behind: skip to the next keyframe | |
| 103 | if (this.needKey && !f.key) return; | |
| 104 | if (f.key && !this.configured && !this.configure(f.data)) return; | |
| 105 | this.needKey = false; | |
| 106 | try { | |
| 107 | this.decoder.decode(new this.Chunk({ type: f.key ? "key" : "delta", timestamp: Math.round(f.sentAt * 1000), data: f.data })); | |
| 108 | } catch { | |
| 109 | this.needKey = true; | |
| 110 | this.configured = false; | |
| 111 | } | |
| 112 | } | |
| 113 | ||
| 114 | configure(unit) { | |
| 115 | const codec = codecOf(unit); | |
| 116 | if (!codec || !this.Decoder) return false; | |
| 117 | try { this.decoder?.close(); } catch {} | |
| 118 | this.decoder = new this.Decoder({ | |
| 119 | output: (frame) => { | |
| 120 | const c = this.o.canvas; | |
| 121 | if (c.width !== frame.displayWidth || c.height !== frame.displayHeight) (c.width = frame.displayWidth), (c.height = frame.displayHeight); | |
| 122 | this.ctx.drawImage(frame, 0, 0); | |
| 123 | frame.close(); | |
| 124 | if (!this.frames++) this.status("live"); | |
| 125 | }, | |
| 126 | error: () => ((this.needKey = true), (this.configured = false)), | |
| 127 | }); | |
| 128 | this.decoder.configure({ codec, optimizeForLatency: true }); | |
| 129 | return (this.configured = true); | |
| 130 | } | |
| 131 | } |
Software/so101/src/ui/video-player.test.ts added +165
| @@ -0,0 +1,165 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { canDecode, codecOf, parseFrame, VideoPlayer } from "./video-player.js"; | |
| 3 | ||
| 4 | // A short Annex-B clip with the shape of the Pi's: SPS (baseline, constraints 0xc0, level 31), PPS and IDR on keyframes, P pictures between. | |
| 5 | const nal = (type: number, size: number, fill: number) => Buffer.concat([Buffer.from([0, 0, 0, 1, type | 0x60]), Buffer.alloc(size, fill)]); | |
| 6 | const SPS = Buffer.concat([Buffer.from([0, 0, 0, 1, 0x67, 66, 0xc0, 31]), Buffer.alloc(6, 0x11)]); | |
| 7 | const picture = (key: boolean, size: number) => (key ? Buffer.concat([SPS, nal(8, 3, 0x22), nal(5, size, 0x33)]) : nal(1, size, 0x44)); | |
| 8 | const clip = (count: number, gop: number, bytes: number) => Array.from({ length: count }, (_, i) => picture(i % gop === 0, i % gop === 0 ? bytes * 8 : bytes)); | |
| 9 | ||
| 10 | const frame = (key: boolean, rung: number, sentAt: number, data: Uint8Array) => { | |
| 11 | const out = Buffer.alloc(10 + data.length); | |
| 12 | out[0] = key ? 1 : 0; | |
| 13 | out[1] = rung; | |
| 14 | out.writeDoubleBE(sentAt, 2); | |
| 15 | out.set(data, 10); | |
| 16 | return out; | |
| 17 | }; | |
| 18 | ||
| 19 | class FakeSocket { | |
| 20 | static last: FakeSocket; | |
| 21 | readyState = 1; | |
| 22 | sent: string[] = []; | |
| 23 | onmessage: ((e: { data: unknown }) => void) | null = null; | |
| 24 | onclose: (() => void) | null = null; | |
| 25 | url: string; | |
| 26 | constructor(url: string) { | |
| 27 | this.url = url; | |
| 28 | FakeSocket.last = this; | |
| 29 | } | |
| 30 | send(s: string) { | |
| 31 | this.sent.push(s); | |
| 32 | } | |
| 33 | close() {} | |
| 34 | } | |
| 35 | ||
| 36 | function setup() { | |
| 37 | const decoded: { type: string; timestamp: number; bytes: number }[] = []; | |
| 38 | const states: string[] = []; | |
| 39 | let configured: unknown; | |
| 40 | let output: (f: unknown) => void = () => {}; | |
| 41 | const drawn: unknown[] = []; | |
| 42 | class FakeDecoder { | |
| 43 | decodeQueueSize = 0; | |
| 44 | constructor(init: { output: (f: unknown) => void }) { | |
| 45 | output = init.output; | |
| 46 | } | |
| 47 | configure(c: unknown) { | |
| 48 | configured = c; | |
| 49 | } | |
| 50 | decode(chunk: { type: string; timestamp: number; data: Uint8Array }) { | |
| 51 | decoded.push({ type: chunk.type, timestamp: chunk.timestamp, bytes: chunk.data.length }); | |
| 52 | output({ displayWidth: 1280, displayHeight: 720, close() {} }); | |
| 53 | } | |
| 54 | close() {} | |
| 55 | } | |
| 56 | const canvas = { width: 0, height: 0, getContext: () => ({ drawImage: (f: unknown) => drawn.push(f) }) }; | |
| 57 | let now = 10_000; | |
| 58 | const player = new VideoPlayer({ url: "ws://x/video", canvas, onStatus: (s) => states.push(s.state), deps: { WebSocket: FakeSocket, VideoDecoder: FakeDecoder, EncodedVideoChunk: class { constructor(init: object) { Object.assign(this, init); } }, now: () => now } }); | |
| 59 | player.start(); | |
| 60 | const ws = FakeSocket.last; | |
| 61 | return { player, ws, decoded, states, drawn, canvas, get configured() { return configured; }, tick: (ms: number) => (now += ms), set: (n: number) => (now = n), decoder: () => player.decoder }; | |
| 62 | } | |
| 63 | ||
| 64 | describe("video player", () => { | |
| 65 | const pics = clip(10, 5, 100); | |
| 66 | ||
| 67 | test("parses the wire frame and the codec", () => { | |
| 68 | const f = parseFrame(frame(true, 2, 1234.5, pics[0]!))!; | |
| 69 | expect(f).toMatchObject({ key: true, rendition: 2, sentAt: 1234.5 }); | |
| 70 | expect(f.data.length).toBe(pics[0]!.length); | |
| 71 | expect(parseFrame(new Uint8Array(10))).toBeNull(); | |
| 72 | expect(codecOf(pics[0]!)).toBe("avc1.42c01f"); | |
| 73 | expect(codecOf(pics[1]!)).toBeNull(); // a P picture carries no SPS | |
| 74 | expect(codecOf(new Uint8Array([0, 0, 1, 0x67, 66]))).toBeNull(); // truncated SPS: no read past the end | |
| 75 | }); | |
| 76 | ||
| 77 | test("reports no support where WebCodecs is missing", () => { | |
| 78 | const saved = { VideoDecoder: (globalThis as any).VideoDecoder, EncodedVideoChunk: (globalThis as any).EncodedVideoChunk }; | |
| 79 | try { | |
| 80 | delete (globalThis as any).VideoDecoder; | |
| 81 | expect(canDecode()).toBe(false); | |
| 82 | (globalThis as any).VideoDecoder = class {}; | |
| 83 | (globalThis as any).EncodedVideoChunk = class {}; | |
| 84 | expect(canDecode()).toBe(typeof WebSocket === "function"); | |
| 85 | } finally { | |
| 86 | Object.assign(globalThis, saved); | |
| 87 | if (!saved.VideoDecoder) delete (globalThis as any).VideoDecoder; | |
| 88 | if (!saved.EncodedVideoChunk) delete (globalThis as any).EncodedVideoChunk; | |
| 89 | } | |
| 90 | }); | |
| 91 | ||
| 92 | test("a keyframe without an SPS is not decoded, and no decoder at all never configures", () => { | |
| 93 | const t = setup(); | |
| 94 | t.ws.onmessage!({ data: frame(true, 0, 10_000, nal(5, 50, 0x33)) }); | |
| 95 | expect(t.decoded).toHaveLength(0); | |
| 96 | expect(t.configured).toBeUndefined(); | |
| 97 | t.player.stop(); | |
| 98 | const states: string[] = []; | |
| 99 | const bare = new VideoPlayer({ url: "ws://x/video", canvas: { width: 0, height: 0, getContext: () => ({}) }, onStatus: (s) => states.push(s.state), deps: { WebSocket: FakeSocket, VideoDecoder: undefined, now: () => 0 } }); | |
| 100 | (bare as any).Decoder = undefined; | |
| 101 | bare.start(); | |
| 102 | FakeSocket.last.onmessage!({ data: frame(true, 0, 0, pics[0]!) }); | |
| 103 | expect(bare.decoder).toBeNull(); | |
| 104 | FakeSocket.last.onclose!(); | |
| 105 | expect(states).toEqual(["none"]); | |
| 106 | bare.stop(); | |
| 107 | }); | |
| 108 | ||
| 109 | test("says none when no picture arrives within waitMs, and stays quiet once one has", async () => { | |
| 110 | const quiet: string[] = []; | |
| 111 | const idle = new VideoPlayer({ url: "ws://x/video", canvas: { width: 0, height: 0, getContext: () => ({}) }, waitMs: 5, onStatus: (s) => quiet.push(s.state), deps: { WebSocket: FakeSocket, now: () => 0 } }); | |
| 112 | idle.start(); | |
| 113 | await Bun.sleep(20); | |
| 114 | expect(quiet).toEqual(["none"]); | |
| 115 | idle.stop(); | |
| 116 | ||
| 117 | const t = setup(); | |
| 118 | t.ws.onmessage!({ data: frame(true, 0, 10_000, pics[0]!) }); | |
| 119 | await Bun.sleep(20); | |
| 120 | expect(t.states).toEqual(["live"]); | |
| 121 | t.player.stop(); | |
| 122 | }); | |
| 123 | ||
| 124 | test("starts at a keyframe, configures from the SPS, decodes to the canvas", () => { | |
| 125 | const t = setup(); | |
| 126 | t.ws.onmessage!({ data: JSON.stringify({ ladder: [{ name: "720p" }], rendition: "720p" }) }); | |
| 127 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); // joined mid-GOP: ignored | |
| 128 | expect(t.decoded).toHaveLength(0); | |
| 129 | t.ws.onmessage!({ data: frame(true, 0, 10_000, pics[0]!) }); | |
| 130 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); | |
| 131 | expect(t.configured).toEqual({ codec: "avc1.42c01f", optimizeForLatency: true }); | |
| 132 | expect(t.decoded.map((d) => d.type)).toEqual(["key", "delta"]); | |
| 133 | expect(t.drawn).toHaveLength(2); | |
| 134 | expect(t.canvas.width).toBe(1280); | |
| 135 | expect(t.states).toContain("live"); | |
| 136 | t.player.stop(); | |
| 137 | }); | |
| 138 | ||
| 139 | test("a decoder that falls behind skips to the next keyframe; age is the delay above the best seen", () => { | |
| 140 | const t = setup(); | |
| 141 | t.ws.onmessage!({ data: frame(true, 0, 9_950, pics[0]!) }); // 50 ms away | |
| 142 | t.ws.onmessage!({ data: frame(false, 0, 9_950, pics[1]!) }); | |
| 143 | t.decoder().decodeQueueSize = 9; | |
| 144 | t.ws.onmessage!({ data: frame(false, 0, 9_800, pics[2]!) }); // 200 ms old: 150 ms above the best | |
| 145 | expect(t.player.age).toBe(150); | |
| 146 | expect(t.decoded).toHaveLength(2); // dropped | |
| 147 | t.decoder().decodeQueueSize = 0; | |
| 148 | t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[3]!) }); // still no key | |
| 149 | expect(t.decoded).toHaveLength(2); | |
| 150 | t.ws.onmessage!({ data: frame(true, 0, 9_850, pics[5]!) }); // 150 ms away; the best seen is 0 by now | |
| 151 | expect(t.decoded).toHaveLength(3); | |
| 152 | t.player.report(); | |
| 153 | expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ age: 150 }); | |
| 154 | t.player.quality("240p"); | |
| 155 | expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ quality: "240p" }); | |
| 156 | t.player.stop(); | |
| 157 | }); | |
| 158 | ||
| 159 | test("says none when the socket closes before any picture", () => { | |
| 160 | const t = setup(); | |
| 161 | t.ws.onclose!(); | |
| 162 | expect(t.states).toEqual(["none"]); | |
| 163 | t.player.stop(); | |
| 164 | }); | |
| 165 | }); |
new above
offline