Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

src/ui/video-player.js decodes H.264 access units with WebCodecs into a canvas and reports when it cannot - module and test only #389

Closed opened this
Closed without merging

vesper/arm:vesper/258-src-ui-video-player-js-decodes-h-264-acc → main 3 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
2 files changed +296 −0
Software/so101/src/ui/video-player.js added +131
@@ -0,0 +1,131 @@
1 // @ts-nocheck -- WebCodecs and the DOM are not in this project's lib; the tests drive the class with fakes
2 /**
3 * The camera as H.264 over the relay's WebSocket (/camera/<id>/video), decoded with WebCodecs into a canvas.
4 * Plain ESM, served at /ui/ and imported by the tests. The wire format is documented in src/video-fanout.ts:
5 * 10 header bytes (keyframe flag, rendition index, relay time float64) then one Annex-B access unit.
6 *
7 * Nothing is queued: a picture that arrives while the decoder is behind is dropped, and so is every one after it
8 * until the next keyframe. `onStatus` hears "live", "none" (no picture in `waitMs`: use MJPEG) and the frame age.
9 */
10
11 export const HEADER = 10;
12
13 /** @param {ArrayBuffer | Uint8Array} data */
14 export function parseFrame(data) {
15 const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
16 if (bytes.length <= HEADER) return null;
17 const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
18 return { key: (bytes[0] & 1) === 1, rendition: bytes[1], sentAt: view.getFloat64(2), data: bytes.subarray(HEADER) };
19 }
20
21 /** "avc1.PPCCLL" from the first SPS of an Annex-B unit, or null. @param {Uint8Array} unit */
22 export function codecOf(unit) {
23 for (let i = 0; i + 6 < unit.length; i++) {
24 if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3] & 0x1f) === 7) {
25 return "avc1." + [4, 5, 6].map((n) => unit[i + n].toString(16).padStart(2, "0")).join("");
26 }
27 }
28 return null;
29 }
30
31 export const canDecode = () => typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function" && typeof WebSocket === "function";
32
33 export class VideoPlayer {
34 /**
35 * @param {{ url: string, canvas: { width: number, height: number, getContext(k: string): any }, onStatus?: (s: { state: string, age?: number, rendition?: string, ladder?: any[] }) => void, waitMs?: number,
36 * deps?: { WebSocket?: any, VideoDecoder?: any, EncodedVideoChunk?: any, now?: () => number } }} o
37 */
38 constructor(o) {
39 this.o = o;
40 const d = o.deps ?? {};
41 this.WS = d.WebSocket ?? WebSocket;
42 this.Decoder = d.VideoDecoder ?? (typeof VideoDecoder === "undefined" ? undefined : VideoDecoder);
43 this.Chunk = d.EncodedVideoChunk ?? (typeof EncodedVideoChunk === "undefined" ? undefined : EncodedVideoChunk);
44 this.now = d.now ?? (() => Date.now());
45 this.ctx = o.canvas.getContext("2d");
46 this.decoder = null;
47 this.configured = false;
48 this.needKey = true;
49 this.lowest = Infinity;
50 this.age = 0;
51 this.frames = 0;
52 this.ladder = [];
53 this.closed = false;
54 this.ws = null;
55 this.timers = [];
56 }
57
58 start() {
59 const ws = (this.ws = new this.WS(this.o.url));
60 ws.binaryType = "arraybuffer";
61 ws.onmessage = (e) => (typeof e.data === "string" ? this.control(e.data) : this.frame(e.data));
62 ws.onclose = () => this.status("none");
63 this.timers.push(setTimeout(() => this.frames || this.status("none"), this.o.waitMs ?? 3000));
64 this.timers.push(setInterval(() => this.report(), 1000));
65 }
66
67 /** Pin a rung by name, or "auto". @param {string} quality */
68 quality(quality) {
69 if (this.ws?.readyState === 1) this.ws.send(JSON.stringify({ quality }));
70 }
71
72 stop() {
73 this.closed = true;
74 this.timers.forEach((t) => (clearTimeout(t), clearInterval(t)));
75 try { this.ws?.close(); } catch {}
76 try { this.decoder?.close(); } catch {}
77 }
78
79 status(state) {
80 this.o.onStatus?.({ state, age: this.age, rendition: this.rendition, ladder: this.ladder });
81 }
82
83 control(text) {
84 let m;
85 try { m = JSON.parse(text); } catch { return; }
86 if (Array.isArray(m.ladder)) this.ladder = m.ladder;
87 if (typeof m.rendition === "string") this.rendition = m.rendition;
88 if (this.frames) this.status("live");
89 }
90
91 report() {
92 if (this.ws?.readyState === 1 && this.age > 0) this.ws.send(JSON.stringify({ age: Math.round(this.age) }));
93 }
94
95 frame(buffer) {
96 const f = parseFrame(buffer);
97 if (!f || this.closed) return;
98 // Age is measured against the least delay seen, since the relay's clock and the page's differ: it is the queueing, not the distance.
99 const delay = this.now() - f.sentAt;
100 this.lowest = Math.min(this.lowest, delay);
101 this.age = Math.max(0, delay - this.lowest);
102 if (this.decoder && this.decoder.decodeQueueSize > 3) this.needKey = true; // behind: skip to the next keyframe
103 if (this.needKey && !f.key) return;
104 if (f.key && !this.configured && !this.configure(f.data)) return;
105 this.needKey = false;
106 try {
107 this.decoder.decode(new this.Chunk({ type: f.key ? "key" : "delta", timestamp: Math.round(f.sentAt * 1000), data: f.data }));
108 } catch {
109 this.needKey = true;
110 this.configured = false;
111 }
112 }
113
114 configure(unit) {
115 const codec = codecOf(unit);
116 if (!codec || !this.Decoder) return false;
117 try { this.decoder?.close(); } catch {}
118 this.decoder = new this.Decoder({
119 output: (frame) => {
120 const c = this.o.canvas;
121 if (c.width !== frame.displayWidth || c.height !== frame.displayHeight) (c.width = frame.displayWidth), (c.height = frame.displayHeight);
122 this.ctx.drawImage(frame, 0, 0);
123 frame.close();
124 if (!this.frames++) this.status("live");
125 },
126 error: () => ((this.needKey = true), (this.configured = false)),
127 });
128 this.decoder.configure({ codec, optimizeForLatency: true });
129 return (this.configured = true);
130 }
131 }
Software/so101/src/ui/video-player.test.ts added +165
@@ -0,0 +1,165 @@
1 import { describe, expect, test } from "bun:test";
2 import { canDecode, codecOf, parseFrame, VideoPlayer } from "./video-player.js";
3
4 // A short Annex-B clip with the shape of the Pi's: SPS (baseline, constraints 0xc0, level 31), PPS and IDR on keyframes, P pictures between.
5 const nal = (type: number, size: number, fill: number) => Buffer.concat([Buffer.from([0, 0, 0, 1, type | 0x60]), Buffer.alloc(size, fill)]);
6 const SPS = Buffer.concat([Buffer.from([0, 0, 0, 1, 0x67, 66, 0xc0, 31]), Buffer.alloc(6, 0x11)]);
7 const picture = (key: boolean, size: number) => (key ? Buffer.concat([SPS, nal(8, 3, 0x22), nal(5, size, 0x33)]) : nal(1, size, 0x44));
8 const clip = (count: number, gop: number, bytes: number) => Array.from({ length: count }, (_, i) => picture(i % gop === 0, i % gop === 0 ? bytes * 8 : bytes));
9
10 const frame = (key: boolean, rung: number, sentAt: number, data: Uint8Array) => {
11 const out = Buffer.alloc(10 + data.length);
12 out[0] = key ? 1 : 0;
13 out[1] = rung;
14 out.writeDoubleBE(sentAt, 2);
15 out.set(data, 10);
16 return out;
17 };
18
19 class FakeSocket {
20 static last: FakeSocket;
21 readyState = 1;
22 sent: string[] = [];
23 onmessage: ((e: { data: unknown }) => void) | null = null;
24 onclose: (() => void) | null = null;
25 url: string;
26 constructor(url: string) {
27 this.url = url;
28 FakeSocket.last = this;
29 }
30 send(s: string) {
31 this.sent.push(s);
32 }
33 close() {}
34 }
35
36 function setup() {
37 const decoded: { type: string; timestamp: number; bytes: number }[] = [];
38 const states: string[] = [];
39 let configured: unknown;
40 let output: (f: unknown) => void = () => {};
41 const drawn: unknown[] = [];
42 class FakeDecoder {
43 decodeQueueSize = 0;
44 constructor(init: { output: (f: unknown) => void }) {
45 output = init.output;
46 }
47 configure(c: unknown) {
48 configured = c;
49 }
50 decode(chunk: { type: string; timestamp: number; data: Uint8Array }) {
51 decoded.push({ type: chunk.type, timestamp: chunk.timestamp, bytes: chunk.data.length });
52 output({ displayWidth: 1280, displayHeight: 720, close() {} });
53 }
54 close() {}
55 }
56 const canvas = { width: 0, height: 0, getContext: () => ({ drawImage: (f: unknown) => drawn.push(f) }) };
57 let now = 10_000;
58 const player = new VideoPlayer({ url: "ws://x/video", canvas, onStatus: (s) => states.push(s.state), deps: { WebSocket: FakeSocket, VideoDecoder: FakeDecoder, EncodedVideoChunk: class { constructor(init: object) { Object.assign(this, init); } }, now: () => now } });
59 player.start();
60 const ws = FakeSocket.last;
61 return { player, ws, decoded, states, drawn, canvas, get configured() { return configured; }, tick: (ms: number) => (now += ms), set: (n: number) => (now = n), decoder: () => player.decoder };
62 }
63
64 describe("video player", () => {
65 const pics = clip(10, 5, 100);
66
67 test("parses the wire frame and the codec", () => {
68 const f = parseFrame(frame(true, 2, 1234.5, pics[0]!))!;
69 expect(f).toMatchObject({ key: true, rendition: 2, sentAt: 1234.5 });
70 expect(f.data.length).toBe(pics[0]!.length);
71 expect(parseFrame(new Uint8Array(10))).toBeNull();
72 expect(codecOf(pics[0]!)).toBe("avc1.42c01f");
73 expect(codecOf(pics[1]!)).toBeNull(); // a P picture carries no SPS
74 expect(codecOf(new Uint8Array([0, 0, 1, 0x67, 66]))).toBeNull(); // truncated SPS: no read past the end
75 });
76
77 test("reports no support where WebCodecs is missing", () => {
78 const saved = { VideoDecoder: (globalThis as any).VideoDecoder, EncodedVideoChunk: (globalThis as any).EncodedVideoChunk };
79 try {
80 delete (globalThis as any).VideoDecoder;
81 expect(canDecode()).toBe(false);
82 (globalThis as any).VideoDecoder = class {};
83 (globalThis as any).EncodedVideoChunk = class {};
84 expect(canDecode()).toBe(typeof WebSocket === "function");
85 } finally {
86 Object.assign(globalThis, saved);
87 if (!saved.VideoDecoder) delete (globalThis as any).VideoDecoder;
88 if (!saved.EncodedVideoChunk) delete (globalThis as any).EncodedVideoChunk;
89 }
90 });
91
92 test("a keyframe without an SPS is not decoded, and no decoder at all never configures", () => {
93 const t = setup();
94 t.ws.onmessage!({ data: frame(true, 0, 10_000, nal(5, 50, 0x33)) });
95 expect(t.decoded).toHaveLength(0);
96 expect(t.configured).toBeUndefined();
97 t.player.stop();
98 const states: string[] = [];
99 const bare = new VideoPlayer({ url: "ws://x/video", canvas: { width: 0, height: 0, getContext: () => ({}) }, onStatus: (s) => states.push(s.state), deps: { WebSocket: FakeSocket, VideoDecoder: undefined, now: () => 0 } });
100 (bare as any).Decoder = undefined;
101 bare.start();
102 FakeSocket.last.onmessage!({ data: frame(true, 0, 0, pics[0]!) });
103 expect(bare.decoder).toBeNull();
104 FakeSocket.last.onclose!();
105 expect(states).toEqual(["none"]);
106 bare.stop();
107 });
108
109 test("says none when no picture arrives within waitMs, and stays quiet once one has", async () => {
110 const quiet: string[] = [];
111 const idle = new VideoPlayer({ url: "ws://x/video", canvas: { width: 0, height: 0, getContext: () => ({}) }, waitMs: 5, onStatus: (s) => quiet.push(s.state), deps: { WebSocket: FakeSocket, now: () => 0 } });
112 idle.start();
113 await Bun.sleep(20);
114 expect(quiet).toEqual(["none"]);
115 idle.stop();
116
117 const t = setup();
118 t.ws.onmessage!({ data: frame(true, 0, 10_000, pics[0]!) });
119 await Bun.sleep(20);
120 expect(t.states).toEqual(["live"]);
121 t.player.stop();
122 });
123
124 test("starts at a keyframe, configures from the SPS, decodes to the canvas", () => {
125 const t = setup();
126 t.ws.onmessage!({ data: JSON.stringify({ ladder: [{ name: "720p" }], rendition: "720p" }) });
127 t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); // joined mid-GOP: ignored
128 expect(t.decoded).toHaveLength(0);
129 t.ws.onmessage!({ data: frame(true, 0, 10_000, pics[0]!) });
130 t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) });
131 expect(t.configured).toEqual({ codec: "avc1.42c01f", optimizeForLatency: true });
132 expect(t.decoded.map((d) => d.type)).toEqual(["key", "delta"]);
133 expect(t.drawn).toHaveLength(2);
134 expect(t.canvas.width).toBe(1280);
135 expect(t.states).toContain("live");
136 t.player.stop();
137 });
138
139 test("a decoder that falls behind skips to the next keyframe; age is the delay above the best seen", () => {
140 const t = setup();
141 t.ws.onmessage!({ data: frame(true, 0, 9_950, pics[0]!) }); // 50 ms away
142 t.ws.onmessage!({ data: frame(false, 0, 9_950, pics[1]!) });
143 t.decoder().decodeQueueSize = 9;
144 t.ws.onmessage!({ data: frame(false, 0, 9_800, pics[2]!) }); // 200 ms old: 150 ms above the best
145 expect(t.player.age).toBe(150);
146 expect(t.decoded).toHaveLength(2); // dropped
147 t.decoder().decodeQueueSize = 0;
148 t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[3]!) }); // still no key
149 expect(t.decoded).toHaveLength(2);
150 t.ws.onmessage!({ data: frame(true, 0, 9_850, pics[5]!) }); // 150 ms away; the best seen is 0 by now
151 expect(t.decoded).toHaveLength(3);
152 t.player.report();
153 expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ age: 150 });
154 t.player.quality("240p");
155 expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ quality: "240p" });
156 t.player.stop();
157 });
158
159 test("says none when the socket closes before any picture", () => {
160 const t = setup();
161 t.ws.onclose!();
162 expect(t.states).toEqual(["none"]);
163 t.player.stop();
164 });
165 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs