Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

src/h264.ts and src/video-fanout.ts: split Annex-B access units and fan them out per viewer, skipping to the next keyframe - modules and tests only #264

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/257-src-h264-ts-and-src-video-fanout-ts-spli → main 1 commit fork

  • Merged. smith/257-src-h264-ts-and-src-video-fanout-ts-spli was merged into main as 819e5e7.

Review

Sign in to review this change.
5 files changed +422 −0
Software/so101/src/h264-clip.ts added +16
@@ -0,0 +1,16 @@
1 /** A stand-in for the Pi's encoder in tests: a short Annex-B clip with the shape of the real one (SPS, PPS, IDR, then P pictures; no B-frames). */
2
3 const nal = (type: number, size: number, fill: number) => Buffer.concat([Buffer.from([0, 0, 0, 1, type | 0x60]), Buffer.alloc(size, fill)]);
4 // SPS body: profile 66 (baseline), constraints 0xc0, level 31, then filler that contains no start code.
5 const SPS = Buffer.concat([Buffer.from([0, 0, 0, 1, 0x67, 66, 0xc0, 31]), Buffer.alloc(6, 0x11)]);
6 const PPS = nal(8, 3, 0x22);
7
8 /** One picture: `key` pictures carry the parameter sets; `size` is the slice's payload in bytes. */
9 export function picture(key: boolean, size: number): Buffer {
10 return key ? Buffer.concat([SPS, PPS, nal(5, size, 0x33)]) : nal(1, size, 0x44);
11 }
12
13 /** `seconds` of video at `fps` with a keyframe every `gop` pictures; keyframes are 8x a P picture's size. */
14 export function clip(seconds: number, fps: number, gop: number, deltaBytes: number): Buffer[] {
15 return Array.from({ length: Math.round(seconds * fps) }, (_, i) => picture(i % gop === 0, i % gop === 0 ? deltaBytes * 8 : deltaBytes));
16 }
Software/so101/src/h264.test.ts added +35
@@ -0,0 +1,35 @@
1 import { describe, expect, test } from "bun:test";
2 import { clip } from "./h264-clip.ts";
3 import { AccessUnitSplitter } from "./h264.ts";
4
5 describe("access units", () => {
6 const pictures = clip(1, 10, 5, 100);
7 const stream = Buffer.concat(pictures);
8
9 test("a stream cut anywhere comes out as the same pictures, key flags right", () => {
10 for (const step of [1, 7, 333, stream.length]) {
11 const splitter = new AccessUnitSplitter();
12 const units = [];
13 for (let i = 0; i < stream.length; i += step) units.push(...splitter.push(stream.subarray(i, i + step)));
14 // the last picture waits for the start of the next
15 expect(units.length).toBe(pictures.length - 1);
16 units.forEach((u, i) => {
17 expect(u.data.equals(pictures[i]!)).toBe(true);
18 expect(u.key).toBe(i % 5 === 0);
19 });
20 }
21 });
22
23 test("a picture is out as soon as the next one begins, not when it ends", () => {
24 const splitter = new AccessUnitSplitter();
25 expect(splitter.push(pictures[0]!)).toEqual([]);
26 expect(splitter.push(pictures[1]!.subarray(0, 5))).toHaveLength(1);
27 });
28
29 test("refuses a stream that never ends a unit", () => {
30 const splitter = new AccessUnitSplitter();
31 expect(() => {
32 for (let i = 0; i < 6; i++) splitter.push(Buffer.alloc(1 << 20, 7));
33 }).toThrow();
34 });
35 });
Software/so101/src/h264.ts added +59
@@ -0,0 +1,59 @@
1 /**
2 * H.264 in Annex-B (start-code) form, cut into access units as the bytes arrive.
3 *
4 * An access unit is one picture: optional parameter sets (SPS/PPS), then its slices. The encoder on the Pi sends one
5 * slice per picture and no B-frames, so a picture ends the moment the next NAL that opens one begins: the unit
6 * leaves as soon as the next start code and its header byte are seen, never waiting for the next picture to finish.
7 */
8
9 export interface AccessUnit {
10 /** An IDR picture: a viewer can start decoding here. */
11 key: boolean;
12 /** The unit's bytes, start codes included. */
13 data: Buffer;
14 }
15
16 /** NAL types that open a new access unit once a picture's slice has been seen. */
17 const OPENS = new Set([1, 5, 6, 7, 8, 9]);
18 /** No unit is larger than this; a stream that never finds a start code is cut off by throwing. */
19 export const MAX_UNIT = 4 << 20;
20
21 export class AccessUnitSplitter {
22 private buf: Buffer = Buffer.alloc(0);
23 /** Where the current unit starts in `buf`. */
24 private from = 0;
25 /** Where to look for the next start code. */
26 private scan = 0;
27 private picture = false;
28 private key = false;
29
30 push(chunk: Uint8Array): AccessUnit[] {
31 this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : Buffer.from(chunk);
32 const out: AccessUnit[] = [];
33 const b = this.buf;
34 for (let i = this.scan; i + 3 < b.length; i++) {
35 if (b[i + 2]! > 1) i += 2; // cannot be inside a start code
36 else if (b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 1) {
37 const type = b[i + 3]! & 0x1f;
38 const start = i > this.from && b[i - 1] === 0 ? i - 1 : i;
39 if (this.picture && OPENS.has(type) && start > this.from) {
40 out.push({ key: this.key, data: Buffer.from(b.subarray(this.from, start)) });
41 this.from = start;
42 this.picture = this.key = false;
43 }
44 if (type === 1 || type === 5) this.picture = true;
45 if (type === 5) this.key = true;
46 i += 3;
47 this.scan = i + 1;
48 }
49 }
50 this.scan = Math.max(this.scan, b.length - 3);
51 if (this.from > 0) {
52 this.buf = Buffer.from(b.subarray(this.from));
53 this.scan -= this.from;
54 this.from = 0;
55 }
56 if (this.buf.length > MAX_UNIT) throw new Error("an H.264 unit is too long");
57 return out;
58 }
59 }
Software/so101/src/video-fanout.test.ts added +115
@@ -0,0 +1,115 @@
1 import { describe, expect, test } from "bun:test";
2 import { clip } from "./h264-clip.ts";
3 import { HEADER, LADDER, VideoFanout, type VideoSink } from "./video-fanout.ts";
4
5 function sink(): VideoSink & { sent: Buffer[]; texts: any[]; queued: number } {
6 const s = {
7 sent: [] as Buffer[],
8 texts: [] as any[],
9 queued: 0,
10 backlog: () => s.queued,
11 send: (d: Uint8Array) => void s.sent.push(Buffer.from(d)),
12 text: (t: string) => void s.texts.push(JSON.parse(t)),
13 };
14 return s;
15 }
16 const frames = (s: { sent: Buffer[] }) => s.sent.map((m) => ({ key: (m[0]! & 1) === 1, rung: m[1]! }));
17 const pictures = clip(10, 10, 5, 100);
18 // Each rung's own clip, sized by its bitrate, like the three encoders on the Pi.
19 const clips = new Map(LADDER.map((r) => [r.name, clip(10, 10, 5, r.kbps / 3)]));
20 /** Every rung gets the next pictures, one at a time. */
21 function play(f: VideoFanout, from: number, to: number) {
22 for (let i = from; i < to; i++) for (const r of f.ladder) f.push(r.name, { key: i % 5 === 0, data: clips.get(r.name)![i]! });
23 }
24
25 describe("renditions and viewers", () => {
26 test("the driver gets the best rung and a watcher the middle one, each from a keyframe", () => {
27 const f = new VideoFanout();
28 const driver = sink(), watcher = sink();
29 f.join(driver, true);
30 f.join(watcher);
31 expect(driver.texts[0].rendition).toBe("720p");
32 expect(watcher.texts[0].rendition).toBe("480p");
33 play(f, 2, 12); // joined mid-GOP: nothing until picture 5
34 expect(frames(driver)[0]).toEqual({ key: true, rung: 0 });
35 expect(frames(watcher)[0]).toEqual({ key: true, rung: 1 });
36 expect(driver.sent).toHaveLength(7);
37 expect(driver.sent[0]!.length).toBe(HEADER + clips.get('720p')![5]!.length);
38 expect(f.wanted()).toEqual(new Set(["720p", "480p"]));
39 });
40
41 test("a slow viewer skips to the next keyframe instead of queueing, then steps down at a keyframe", () => {
42 let t = 0;
43 const f = new VideoFanout({ now: () => t, dropsToStepDown: 2 });
44 const slow = sink(), fast = sink();
45 const viewer = f.join(slow);
46 f.join(fast);
47 play(f, 0, 1);
48 slow.queued = 1 << 20; // the link backs up
49 play(f, 1, 5); // four pictures: dropped, none sent
50 expect(slow.sent).toHaveLength(1);
51 expect(viewer.stats().skipped).toBe(1); // the first miss; the rest wait for a key without counting
52 slow.queued = 1 << 20;
53 play(f, 5, 6);
54 slow.queued = 0;
55 play(f, 6, 10); // clear again but there is no keyframe: still waiting
56 expect(slow.sent).toHaveLength(1);
57 t += 1000;
58 f.adapt();
59 play(f, 10, 11); // a keyframe: the viewer is back
60 expect(slow.sent).toHaveLength(2);
61 expect(fast.sent).toHaveLength(11); // the fast viewer never missed one
62 });
63
64 test("two viewers at different simulated bandwidths end up on different rungs", () => {
65 let t = 0;
66 const f = new VideoFanout({ now: () => t, dropsToStepDown: 1, cleanMs: 5000, maxBacklog: 2000 });
67 const wifi = sink(), mobile = sink();
68 f.join(wifi);
69 const m = f.join(mobile);
70 // The mobile link carries 300 bytes per 100 ms; whatever it cannot carry sits in its backlog.
71 const send = mobile.send;
72 mobile.send = (d) => (send(d), (mobile.queued += d.length));
73 for (let second = 0; second < 30; second++) {
74 for (let i = 0; i < 10; i++) {
75 mobile.queued = Math.max(0, mobile.queued - 300);
76 const n = (second * 10 + i) % 100;
77 play(f, n % 10, (n % 10) + 1);
78 t += 100;
79 }
80 f.adapt();
81 }
82 const onRung = (s: { sent: Buffer[] }, rung: number) => frames(s).filter((x) => x.rung === rung).length;
83 expect(mobile.texts.some((x) => x.rendition === "240p")).toBe(true);
84 expect(onRung(mobile, 2)).toBeGreaterThan(onRung(mobile, 1) + onRung(mobile, 0));
85 expect(onRung(wifi, 0)).toBeGreaterThan(onRung(wifi, 2)); // clean for 5 s: stepped up to the best
86 expect(wifi.texts.some((x) => x.rendition === "720p")).toBe(true);
87 expect(m.stats().bytes * 3).toBeLessThan(wifi.sent.reduce((n, b) => n + b.length, 0));
88 });
89
90 test("a pinned quality holds against drops; auto releases it; junk is ignored", () => {
91 const f = new VideoFanout();
92 const s = sink();
93 const v = f.join(s);
94 v.message('{"quality":"720p"}');
95 v.message("not json");
96 v.message('{"quality":"4k"}');
97 v.message('{"quality":' + '9'.repeat(300) + "}");
98 play(f, 0, 1);
99 expect(v.stats().rendition).toBe("720p");
100 for (let i = 0; i < 5; i++) (v.message('{"age":2000}'), f.adapt());
101 expect(f.wanted()).toEqual(new Set(["720p"]));
102 v.message('{"quality":"auto"}');
103 for (let i = 0; i < 5; i++) (v.message('{"age":2000}'), f.adapt());
104 expect(f.wanted()).toEqual(new Set(["720p", "480p"])); // released: the drops step it down again
105 });
106
107 test("a viewer that leaves is no longer fed or wanted", () => {
108 const f = new VideoFanout();
109 const s = sink();
110 f.join(s).leave();
111 play(f, 0, 3);
112 expect(s.sent).toHaveLength(0);
113 expect(f.wanted().size).toBe(0);
114 });
115 });
Software/so101/src/video-fanout.ts added +197
@@ -0,0 +1,197 @@
1 /**
2 * One camera's H.264 renditions, fanned out to viewers over their WebSockets, a viewer on one rendition at a time.
3 *
4 * Latency before everything: nothing is queued for a viewer. A viewer whose socket is backed up drops pictures and
5 * waits for the next keyframe, and that same backlog is how its link is measured: a viewer that dropped steps down
6 * the ladder, one that went a while clean steps back up, each switch at a keyframe of the new rendition. A viewer can
7 * pin one rendition. The driver is patient (it steps down only after more drops) so the best picture stays with it.
8 *
9 * A binary message is HEADER bytes then the access unit: byte 0 bit 0 = keyframe, byte 1 = the rendition's index in
10 * the ladder, bytes 2..9 = the time the relay got the picture (ms, float64 big-endian). Text messages are JSON.
11 */
12
13 import type { AccessUnit } from "./h264.ts";
14
15 export interface Rendition {
16 name: string;
17 width: number;
18 height: number;
19 fps: number;
20 kbps: number;
21 }
22
23 /** The ladder the Pi encodes, best first. */
24 export const LADDER: readonly Rendition[] = [
25 { name: "720p", width: 1280, height: 720, fps: 20, kbps: 1500 },
26 { name: "480p", width: 854, height: 480, fps: 15, kbps: 600 },
27 { name: "240p", width: 426, height: 240, fps: 10, kbps: 200 },
28 ];
29
30 export const HEADER = 10;
31
32 export interface VideoSink {
33 /** Bytes accepted but not yet on the wire. */
34 backlog(): number;
35 send(data: Uint8Array): void;
36 text(message: string): void;
37 }
38
39 export interface FanoutOptions {
40 ladder?: readonly Rendition[];
41 /** Backlog past which a picture is dropped for that viewer. */
42 maxBacklog?: number;
43 /** Drops (in one window) that step an ordinary viewer down; the driver takes twice as many. */
44 dropsToStepDown?: number;
45 /** How long a viewer must go without a drop before it steps up. */
46 cleanMs?: number;
47 /** A frame age the viewer reports above this also counts as a drop. */
48 maxAgeMs?: number;
49 now?: () => number;
50 }
51
52 interface Viewer {
53 sink: VideoSink;
54 driver: boolean;
55 /** Index of the rendition the viewer is on, and the one it is moving to. */
56 current: number;
57 target: number;
58 pinned: number | null;
59 /** Waiting for a keyframe on `current`. */
60 needKey: boolean;
61 drops: number;
62 lastDrop: number;
63 lastStep: number;
64 bytes: number;
65 frames: number;
66 skipped: number;
67 }
68
69 export interface ViewerStats {
70 rendition: string;
71 bytes: number;
72 frames: number;
73 skipped: number;
74 }
75
76 export class VideoFanout {
77 readonly ladder: readonly Rendition[];
78 private readonly viewers = new Set<Viewer>();
79 private readonly o: Required<Omit<FanoutOptions, "ladder">>;
80
81 constructor(options: FanoutOptions = {}) {
82 this.ladder = options.ladder ?? LADDER;
83 this.o = { maxBacklog: 64 << 10, dropsToStepDown: 3, cleanMs: 6000, maxAgeMs: 500, now: Date.now, ...options };
84 }
85
86 get watching(): number {
87 return this.viewers.size;
88 }
89
90 /** The renditions someone is on or moving to: the only ones worth pulling from the Pi. */
91 wanted(): Set<string> {
92 const names = new Set<string>();
93 for (const v of this.viewers) names.add(this.ladder[v.current]!.name).add(this.ladder[v.target]!.name);
94 return names;
95 }
96
97 /** A viewer joins on the best rendition (the driver) or the middle of the ladder (a watcher), and waits for a keyframe. */
98 join(sink: VideoSink, driver = false): { leave(): void; message(text: string): void; stats(): ViewerStats } {
99 const start = driver ? 0 : Math.min(1, this.ladder.length - 1);
100 const now = this.o.now();
101 const v: Viewer = { sink, driver, current: start, target: start, pinned: null, needKey: true, drops: 0, lastDrop: now, lastStep: now, bytes: 0, frames: 0, skipped: 0 };
102 this.viewers.add(v);
103 sink.text(JSON.stringify({ ladder: this.ladder, rendition: this.ladder[start]!.name }));
104 return {
105 leave: () => void this.viewers.delete(v),
106 message: (text) => this.message(v, text),
107 stats: () => ({ rendition: this.ladder[v.current]!.name, bytes: v.bytes, frames: v.frames, skipped: v.skipped }),
108 };
109 }
110
111 /** A viewer's JSON: {"quality":"auto"|<name>} to pin one, {"age":ms} for the frame age it measures. Anything else is ignored. */
112 private message(v: Viewer, text: string) {
113 if (text.length > 200) return;
114 let m: { quality?: unknown; age?: unknown };
115 try {
116 m = JSON.parse(text);
117 } catch {
118 return;
119 }
120 if (typeof m !== "object" || m === null) return;
121 if (typeof m.quality === "string") {
122 const at = this.ladder.findIndex((r) => r.name === m.quality);
123 if (m.quality === "auto") v.pinned = null;
124 else if (at >= 0) (v.pinned = at), (v.target = at);
125 }
126 if (typeof m.age === "number" && Number.isFinite(m.age) && m.age > this.o.maxAgeMs) this.drop(v);
127 }
128
129 /** The next picture of a rendition arrives from the Pi. */
130 push(rendition: string, unit: AccessUnit): void {
131 const at = this.ladder.findIndex((r) => r.name === rendition);
132 if (at < 0) return;
133 const now = this.o.now();
134 const message = Buffer.allocUnsafe(HEADER + unit.data.length);
135 message[0] = unit.key ? 1 : 0;
136 message[1] = at;
137 message.writeDoubleBE(now, 2);
138 unit.data.copy(message, HEADER);
139 for (const v of this.viewers) {
140 if (v.target === at && v.current !== at && unit.key) {
141 v.current = at;
142 v.needKey = false;
143 v.lastStep = now;
144 }
145 if (v.current !== at) continue;
146 if (v.needKey && (!unit.key || v.target !== v.current)) continue; // waiting for a key, and for the rung it is moving to
147
148 if (v.sink.backlog() > this.o.maxBacklog) {
149 v.skipped++;
150 v.needKey = true;
151 this.drop(v);
152 continue;
153 }
154 v.needKey = false;
155 v.sink.send(message);
156 v.bytes += message.length;
157 v.frames++;
158 }
159 }
160
161 private drop(v: Viewer) {
162 const now = this.o.now();
163 v.drops++;
164 v.lastDrop = now;
165 v.needKey = true;
166 }
167
168 /** Run about once a second: steps viewers down after drops and up after a clean while. */
169 adapt(): void {
170 const now = this.o.now();
171 for (const v of this.viewers) {
172 if (v.pinned !== null) {
173 v.drops = 0;
174 continue;
175 }
176 const limit = this.o.dropsToStepDown * (v.driver ? 2 : 1);
177 if (v.drops >= limit && v.target < this.ladder.length - 1) {
178 v.target = v.current + 1;
179 this.announce(v);
180 } else if (v.drops === 0 && v.target === v.current && v.current > 0 && now - v.lastDrop >= this.o.cleanMs && now - v.lastStep >= this.o.cleanMs) {
181 v.target = v.current - 1;
182 this.announce(v);
183 }
184 if (v.drops >= limit) v.drops = 0;
185 else if (now - v.lastDrop > 2000) v.drops = 0;
186 }
187 }
188
189 private announce(v: Viewer) {
190 v.lastStep = this.o.now();
191 v.sink.text(JSON.stringify({ rendition: this.ladder[v.target]!.name }));
192 }
193
194 close(): void {
195 this.viewers.clear();
196 }
197 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs