Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

Real video: hardware H.264 from the Pi over the WebSocket, decoded with WebCodecs, smaller streams for slow viewers - a fraction of MJPEG's bytes #75

Parked opened this in answer to
Closed without merging

smith/arm:smith/70-real-video-hardware-h-264-from-the-pi-ov → main 2 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
12 files changed +966 −6
Software/so101/bench/fake-serve.ts +8 −2
@@ -22,7 +22,7 @@ import { CameraHub } from "../src/cameras.ts";
2222 import { LeaderCalibrations } from "../src/leader-calibrations.ts";
2323 import { Lobby } from "../src/lobby.ts";
2424 import { Moderator, modelFromEnv } from "../src/moderator.ts";
25 import { parseRemote, startRelay, type Remote } from "../src/relay.ts";
25 import { parseRemote, parseVideo, startRelay, type Remote } from "../src/relay.ts";
2626 import { startServer } from "../src/server.ts";
2727
2828 const fake = new FakeBus([1, 2, 3, 4, 5, 6]);
@@ -59,6 +59,12 @@ const lobby = new Lobby({
5959 const cameras = new CameraHub({ file: join(dir, "cameras.json"), seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
6060
6161 const leaders = new LeaderCalibrations({ file: join(dir, "leader-calibrations.json") });
62 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders });
62 let video = {};
63 try {
64 video = parseVideo(process.env);
65 } catch (err) {
66 console.error(`${(err as Error).message}; cameras stay MJPEG`);
67 }
68 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video });
6369 const { mode, host: armHost } = relay.link();
6470 console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`);
Software/so101/src/h264-clip.ts added +16
@@ -0,0 +1,16 @@
1 /** A stand-in for the Pi's encoder in tests: a short Annex-B clip with the shape of the real one (SPS, PPS, IDR, then P pictures; no B-frames). */
2
3 const nal = (type: number, size: number, fill: number) => Buffer.concat([Buffer.from([0, 0, 0, 1, type | 0x60]), Buffer.alloc(size, fill)]);
4 // SPS body: profile 66 (baseline), constraints 0xc0, level 31, then filler that contains no start code.
5 const SPS = Buffer.concat([Buffer.from([0, 0, 0, 1, 0x67, 66, 0xc0, 31]), Buffer.alloc(6, 0x11)]);
6 const PPS = nal(8, 3, 0x22);
7
8 /** One picture: `key` pictures carry the parameter sets; `size` is the slice's payload in bytes. */
9 export function picture(key: boolean, size: number): Buffer {
10 return key ? Buffer.concat([SPS, PPS, nal(5, size, 0x33)]) : nal(1, size, 0x44);
11 }
12
13 /** `seconds` of video at `fps` with a keyframe every `gop` pictures; keyframes are 8x a P picture's size. */
14 export function clip(seconds: number, fps: number, gop: number, deltaBytes: number): Buffer[] {
15 return Array.from({ length: Math.round(seconds * fps) }, (_, i) => picture(i % gop === 0, i % gop === 0 ? deltaBytes * 8 : deltaBytes));
16 }
Software/so101/src/h264.test.ts added +40
@@ -0,0 +1,40 @@
1 import { describe, expect, test } from "bun:test";
2 import { clip } from "./h264-clip.ts";
3 import { AccessUnitSplitter, codecOf } from "./h264.ts";
4
5 describe("access units", () => {
6 const pictures = clip(1, 10, 5, 100);
7 const stream = Buffer.concat(pictures);
8
9 test("a stream cut anywhere comes out as the same pictures, key flags right", () => {
10 for (const step of [1, 7, 333, stream.length]) {
11 const splitter = new AccessUnitSplitter();
12 const units = [];
13 for (let i = 0; i < stream.length; i += step) units.push(...splitter.push(stream.subarray(i, i + step)));
14 // the last picture waits for the start of the next
15 expect(units.length).toBe(pictures.length - 1);
16 units.forEach((u, i) => {
17 expect(u.data.equals(pictures[i]!)).toBe(true);
18 expect(u.key).toBe(i % 5 === 0);
19 });
20 }
21 });
22
23 test("a picture is out as soon as the next one begins, not when it ends", () => {
24 const splitter = new AccessUnitSplitter();
25 expect(splitter.push(pictures[0]!)).toEqual([]);
26 expect(splitter.push(pictures[1]!.subarray(0, 5))).toHaveLength(1);
27 });
28
29 test("reads the codec string from the SPS", () => {
30 expect(codecOf(pictures[0]!)).toBe("avc1.42c01f");
31 expect(codecOf(pictures[1]!)).toBeNull();
32 });
33
34 test("refuses a stream that never ends a unit", () => {
35 const splitter = new AccessUnitSplitter();
36 expect(() => {
37 for (let i = 0; i < 6; i++) splitter.push(Buffer.alloc(1 << 20, 7));
38 }).toThrow();
39 });
40 });
Software/so101/src/h264.ts added +70
@@ -0,0 +1,70 @@
1 /**
2 * H.264 in Annex-B (start-code) form, cut into access units as the bytes arrive.
3 *
4 * An access unit is one picture: optional parameter sets (SPS/PPS), then its slices. The encoder on the Pi sends one
5 * slice per picture and no B-frames, so a picture ends the moment the next NAL that opens one begins: the unit
6 * leaves as soon as the next start code and its header byte are seen, never waiting for the next picture to finish.
7 */
8
9 export interface AccessUnit {
10 /** An IDR picture: a viewer can start decoding here. */
11 key: boolean;
12 /** The unit's bytes, start codes included. */
13 data: Buffer;
14 }
15
16 /** NAL types that open a new access unit once a picture's slice has been seen. */
17 const OPENS = new Set([1, 5, 6, 7, 8, 9]);
18 /** No unit is larger than this; a stream that never finds a start code is cut off by throwing. */
19 export const MAX_UNIT = 4 << 20;
20
21 export class AccessUnitSplitter {
22 private buf: Buffer = Buffer.alloc(0);
23 /** Where the current unit starts in `buf`. */
24 private from = 0;
25 /** Where to look for the next start code. */
26 private scan = 0;
27 private picture = false;
28 private key = false;
29
30 push(chunk: Uint8Array): AccessUnit[] {
31 this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : Buffer.from(chunk);
32 const out: AccessUnit[] = [];
33 const b = this.buf;
34 for (let i = this.scan; i + 3 < b.length; i++) {
35 if (b[i + 2]! > 1) i += 2; // cannot be inside a start code
36 else if (b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 1) {
37 const type = b[i + 3]! & 0x1f;
38 const start = i > this.from && b[i - 1] === 0 ? i - 1 : i;
39 if (this.picture && OPENS.has(type) && start > this.from) {
40 out.push({ key: this.key, data: Buffer.from(b.subarray(this.from, start)) });
41 this.from = start;
42 this.picture = this.key = false;
43 }
44 if (type === 1 || type === 5) this.picture = true;
45 if (type === 5) this.key = true;
46 i += 3;
47 this.scan = i + 1;
48 }
49 }
50 this.scan = Math.max(this.scan, b.length - 3);
51 if (this.from > 0) {
52 this.buf = Buffer.from(b.subarray(this.from));
53 this.scan -= this.from;
54 this.from = 0;
55 }
56 if (this.buf.length > MAX_UNIT) throw new Error("an H.264 unit is too long");
57 return out;
58 }
59 }
60
61 /** The codec string WebCodecs wants ("avc1.PPCCLL"), from the first SPS in a unit, or null when it has none. */
62 export function codecOf(unit: Uint8Array): string | null {
63 for (let i = 0; i + 6 < unit.length; i++) {
64 if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3]! & 0x1f) === 7) {
65 const hex = (n: number) => n.toString(16).padStart(2, "0");
66 return `avc1.${hex(unit[i + 4]!)}${hex(unit[i + 5]!)}${hex(unit[i + 6]!)}`;
67 }
68 }
69 return null;
70 }
Software/so101/src/relay.ts +38 −2
@@ -29,6 +29,7 @@
2929 * GET /status the arm's own /status, and which arm said it: "source": "live <host:port>", "reconnecting to the arm:
3030 * <why>" (the last known status, with "reconnecting": true and "seenAt"; 503 before
3131 * there is one), or "simulated: <why>" ("simulated: offline, <why>" after 60 s down)
32 * WS /camera/<id>/video the camera as H.264 access units, a rendition per viewer (src/video-feed.ts; sources from ARM_H264_<n>); 404 when it has none
3233 * GET /cameras the listed cameras: [{"id", "name", "online"}]
3334 * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts);
3435 * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate
@@ -51,6 +52,8 @@ import { AdminError, changeFromForm } from "./admin.ts";
5152 import { ADMIN_CSP, DONE, renderAdmin, renderRefusal, type Done } from "./admin-page.ts";
5253 import { parseFps, type FeedOptions } from "./camera-feed.ts";
5354 import { CameraError, CameraHub } from "./cameras.ts";
55 import { VideoFeed, type VideoSources } from "./video-feed.ts";
56 import { LADDER } from "./video-fanout.ts";
5457 import { LOCAL as LOCAL_PERSON } from "./identity.ts";
5558 import { LeaderCalibrations } from "./leader-calibrations.ts";
5659 import { proxyFor, whyNot } from "./egress.ts";
@@ -100,6 +103,8 @@ export interface RelayOptions {
100103 heartbeatMs?: number;
101104 /** Timings of the shared camera feeds. */
102105 camera?: FeedOptions;
106 /** The Pi's H.264 streams per camera id (src/video-feed.ts); a camera without one stays MJPEG. */
107 video?: Record<number, VideoSources>;
103108 /** Every driver's leader calibration; without one they are kept in memory. */
104109 leaders?: LeaderCalibrations;
105110 /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */
@@ -116,6 +121,7 @@ export interface RunningRelay {
116121
117122 const SCHEMES: Record<string, "http:" | "https:"> = { "http:": "http:", "https:": "https:", "ws:": "http:", "wss:": "https:" };
118123 const MAX_BODY = 1_000_000;
124 const VIDEO = /^\/camera\/(\d{1,9})\/video$/;
119125 const CAMERA = /^\/camera\/(\d{1,9})\/(stream|state|snapshot)$/;
120126 const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9}))?$/;
121127 const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8");
@@ -209,6 +215,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
209215 const heartbeatMs = options.heartbeatMs ?? 1000;
210216 const cameras = options.cameras ?? new CameraHub({ feed: options.camera, seed: remote ? [{ name: "Camera 1", source: remote.cameras[0]! }] : [] });
211217 const leaders = options.leaders ?? new LeaderCalibrations();
218 const videos = new Map<number, VideoFeed>(Object.entries(options.video ?? {}).map(([id, sources]) => [Number(id), new VideoFeed(sources)]));
212219
213220 // ------------------------------------------------------------------ the link
214221 let armLink: ArmLink | null = null;
@@ -539,7 +546,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
539546 const path = url.pathname + url.search;
540547 const reading = req.method === "GET" || req.method === "HEAD";
541548 if (!reading && !sameOrigin(req)) return json(res, 403, { error: "only this page may change the arm" });
542 if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online })));
549 if (req.method === "GET" && url.pathname === "/cameras") return json(res, 200, cameras.list().map(({ id, name, online }) => ({ id, name, online, h264: videos.get(id)?.available ?? false })));
543550 if (req.method === "GET" && url.pathname === "/admin/camera-list") {
544551 res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" });
545552 return res.end(CAMERA_PAGE);
@@ -699,9 +706,17 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
699706 };
700707 http.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => {
701708 const url = new URL(req.url ?? "/", "http://x");
702 if (url.pathname !== "/follow" && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found");
709 const video = VIDEO.exec(url.pathname);
710 if (url.pathname !== "/follow" && !video && !(url.pathname === "/live" && lobby)) return refuse(socket, "404 Not Found");
703711 if (!sameOrigin(req)) return refuse(socket, "403 Forbidden");
704712 socket.on("error", () => socket.destroy());
713 if (video) {
714 const feed = videos.get(Number(video[1]));
715 if (!feed?.available) return refuse(socket, "404 Not Found");
716 const who = lobby?.who(req); // watching is open, like /stream; the driver's picture gets the best rendition
717 const driver = !!who && !!lobby?.turns.isDriver(who.id);
718 return wss.handleUpgrade(req, socket, head, (browser) => feed.serve(browser, driver));
719 }
705720 if (url.pathname === "/live") return lobby!.upgrade(req, socket, head);
706721 let driver: string | null = null;
707722 if (lobby) {
@@ -770,6 +785,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
770785 arrived?.();
771786 armLink?.close();
772787 if (!options.cameras) cameras.close();
788 for (const v of videos.values()) v.close();
773789 for (const c of inFlight) c.abort();
774790 for (const client of wss.clients) client.terminate();
775791 wss.close();
@@ -778,3 +794,23 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
778794 }),
779795 };
780796 }
797
798 /**
799 * ARM_H264_<n> = "720p=http://pi:8082/h264,480p=http://pi:8083/h264,240p=…" → camera n's rendition streams.
800 * Throws on a name outside the ladder or a URL that is not http(s).
801 */
802 export function parseVideo(env: Record<string, string | undefined>): Record<number, VideoSources> {
803 const out: Record<number, VideoSources> = {};
804 for (const n of [1, 2]) {
805 const text = env[`ARM_H264_${n}`]?.trim();
806 if (!text) continue;
807 const sources: VideoSources = {};
808 for (const part of text.split(",")) {
809 const [name, ...rest] = part.trim().split("=");
810 if (!name || !LADDER.some((r) => r.name === name)) throw new Error(`ARM_H264_${n}: "${name}" is not a rung of the ladder (${LADDER.map((r) => r.name).join(", ")})`);
811 sources[name] = absolute(rest.join("="), `ARM_H264_${n}`, { "http:": "", "https:": "" }).href;
812 }
813 out[n] = sources;
814 }
815 return out;
816 }
Software/so101/src/ui/video-player.js added +131
@@ -0,0 +1,131 @@
1 // @ts-nocheck -- WebCodecs and the DOM are not in this project's lib; the tests drive the class with fakes
2 /**
3 * The camera as H.264 over the relay's WebSocket (/camera/<id>/video), decoded with WebCodecs into a canvas.
4 * Plain ESM, served at /ui/ and imported by the tests. The wire format is documented in src/video-fanout.ts:
5 * 10 header bytes (keyframe flag, rendition index, relay time float64) then one Annex-B access unit.
6 *
7 * Nothing is queued: a picture that arrives while the decoder is behind is dropped, and so is every one after it
8 * until the next keyframe. `onStatus` hears "live", "none" (no picture in `waitMs`: use MJPEG) and the frame age.
9 */
10
11 export const HEADER = 10;
12
13 /** @param {ArrayBuffer | Uint8Array} data */
14 export function parseFrame(data) {
15 const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
16 if (bytes.length <= HEADER) return null;
17 const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
18 return { key: (bytes[0] & 1) === 1, rendition: bytes[1], sentAt: view.getFloat64(2), data: bytes.subarray(HEADER) };
19 }
20
21 /** "avc1.PPCCLL" from the first SPS of an Annex-B unit, or null. @param {Uint8Array} unit */
22 export function codecOf(unit) {
23 for (let i = 0; i + 6 < unit.length; i++) {
24 if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3] & 0x1f) === 7) {
25 return "avc1." + [4, 5, 6].map((n) => unit[i + n].toString(16).padStart(2, "0")).join("");
26 }
27 }
28 return null;
29 }
30
31 export const canDecode = () => typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function" && typeof WebSocket === "function";
32
33 export class VideoPlayer {
34 /**
35 * @param {{ url: string, canvas: { width: number, height: number, getContext(k: string): any }, onStatus?: (s: { state: string, age?: number, rendition?: string, ladder?: any[] }) => void, waitMs?: number,
36 * deps?: { WebSocket?: any, VideoDecoder?: any, EncodedVideoChunk?: any, now?: () => number } }} o
37 */
38 constructor(o) {
39 this.o = o;
40 const d = o.deps ?? {};
41 this.WS = d.WebSocket ?? WebSocket;
42 this.Decoder = d.VideoDecoder ?? (typeof VideoDecoder === "undefined" ? undefined : VideoDecoder);
43 this.Chunk = d.EncodedVideoChunk ?? (typeof EncodedVideoChunk === "undefined" ? undefined : EncodedVideoChunk);
44 this.now = d.now ?? (() => Date.now());
45 this.ctx = o.canvas.getContext("2d");
46 this.decoder = null;
47 this.configured = false;
48 this.needKey = true;
49 this.lowest = Infinity;
50 this.age = 0;
51 this.frames = 0;
52 this.ladder = [];
53 this.closed = false;
54 this.ws = null;
55 this.timers = [];
56 }
57
58 start() {
59 const ws = (this.ws = new this.WS(this.o.url));
60 ws.binaryType = "arraybuffer";
61 ws.onmessage = (e) => (typeof e.data === "string" ? this.control(e.data) : this.frame(e.data));
62 ws.onclose = () => this.status("none");
63 this.timers.push(setTimeout(() => this.frames || this.status("none"), this.o.waitMs ?? 3000));
64 this.timers.push(setInterval(() => this.report(), 1000));
65 }
66
67 /** Pin a rung by name, or "auto". @param {string} quality */
68 quality(quality) {
69 if (this.ws?.readyState === 1) this.ws.send(JSON.stringify({ quality }));
70 }
71
72 stop() {
73 this.closed = true;
74 this.timers.forEach((t) => (clearTimeout(t), clearInterval(t)));
75 try { this.ws?.close(); } catch {}
76 try { this.decoder?.close(); } catch {}
77 }
78
79 status(state) {
80 this.o.onStatus?.({ state, age: this.age, rendition: this.rendition, ladder: this.ladder });
81 }
82
83 control(text) {
84 let m;
85 try { m = JSON.parse(text); } catch { return; }
86 if (Array.isArray(m.ladder)) this.ladder = m.ladder;
87 if (typeof m.rendition === "string") this.rendition = m.rendition;
88 if (this.frames) this.status("live");
89 }
90
91 report() {
92 if (this.ws?.readyState === 1 && this.age > 0) this.ws.send(JSON.stringify({ age: Math.round(this.age) }));
93 }
94
95 frame(buffer) {
96 const f = parseFrame(buffer);
97 if (!f || this.closed) return;
98 // Age is measured against the least delay seen, since the relay's clock and the page's differ: it is the queueing, not the distance.
99 const delay = this.now() - f.sentAt;
100 this.lowest = Math.min(this.lowest, delay);
101 this.age = Math.max(0, delay - this.lowest);
102 if (this.decoder && this.decoder.decodeQueueSize > 3) this.needKey = true; // behind: skip to the next keyframe
103 if (this.needKey && !f.key) return;
104 if (f.key && !this.configured && !this.configure(f.data)) return;
105 this.needKey = false;
106 try {
107 this.decoder.decode(new this.Chunk({ type: f.key ? "key" : "delta", timestamp: Math.round(f.sentAt * 1000), data: f.data }));
108 } catch {
109 this.needKey = true;
110 this.configured = false;
111 }
112 }
113
114 configure(unit) {
115 const codec = codecOf(unit);
116 if (!codec || !this.Decoder) return false;
117 try { this.decoder?.close(); } catch {}
118 this.decoder = new this.Decoder({
119 output: (frame) => {
120 const c = this.o.canvas;
121 if (c.width !== frame.displayWidth || c.height !== frame.displayHeight) (c.width = frame.displayWidth), (c.height = frame.displayHeight);
122 this.ctx.drawImage(frame, 0, 0);
123 frame.close();
124 if (!this.frames++) this.status("live");
125 },
126 error: () => ((this.needKey = true), (this.configured = false)),
127 });
128 this.decoder.configure({ codec, optimizeForLatency: true });
129 return (this.configured = true);
130 }
131 }
Software/so101/src/ui/video-player.test.ts added +111
@@ -0,0 +1,111 @@
1 import { describe, expect, test } from "bun:test";
2 import { clip } from "../h264-clip.ts";
3 import { codecOf, parseFrame, VideoPlayer } from "./video-player.js";
4
5 const frame = (key: boolean, rung: number, sentAt: number, data: Uint8Array) => {
6 const out = Buffer.alloc(10 + data.length);
7 out[0] = key ? 1 : 0;
8 out[1] = rung;
9 out.writeDoubleBE(sentAt, 2);
10 out.set(data, 10);
11 return out;
12 };
13
14 class FakeSocket {
15 static last: FakeSocket;
16 readyState = 1;
17 sent: string[] = [];
18 onmessage: ((e: { data: unknown }) => void) | null = null;
19 onclose: (() => void) | null = null;
20 url: string;
21 constructor(url: string) {
22 this.url = url;
23 FakeSocket.last = this;
24 }
25 send(s: string) {
26 this.sent.push(s);
27 }
28 close() {}
29 }
30
31 function setup() {
32 const decoded: { type: string; timestamp: number; bytes: number }[] = [];
33 const states: string[] = [];
34 let configured: unknown;
35 let output: (f: unknown) => void = () => {};
36 const drawn: unknown[] = [];
37 class FakeDecoder {
38 decodeQueueSize = 0;
39 constructor(init: { output: (f: unknown) => void }) {
40 output = init.output;
41 }
42 configure(c: unknown) {
43 configured = c;
44 }
45 decode(chunk: { type: string; timestamp: number; data: Uint8Array }) {
46 decoded.push({ type: chunk.type, timestamp: chunk.timestamp, bytes: chunk.data.length });
47 output({ displayWidth: 1280, displayHeight: 720, close() {} });
48 }
49 close() {}
50 }
51 const canvas = { width: 0, height: 0, getContext: () => ({ drawImage: (f: unknown) => drawn.push(f) }) };
52 let now = 10_000;
53 const player = new VideoPlayer({ url: "ws://x/video", canvas, onStatus: (s) => states.push(s.state), deps: { WebSocket: FakeSocket, VideoDecoder: FakeDecoder, EncodedVideoChunk: class { constructor(init: object) { Object.assign(this, init); } }, now: () => now } });
54 player.start();
55 const ws = FakeSocket.last;
56 return { player, ws, decoded, states, drawn, canvas, get configured() { return configured; }, tick: (ms: number) => (now += ms), set: (n: number) => (now = n), decoder: () => player.decoder };
57 }
58
59 describe("video player", () => {
60 const pics = clip(1, 10, 5, 100);
61
62 test("parses the wire frame and the codec", () => {
63 const f = parseFrame(frame(true, 2, 1234.5, pics[0]!))!;
64 expect(f).toMatchObject({ key: true, rendition: 2, sentAt: 1234.5 });
65 expect(f.data.length).toBe(pics[0]!.length);
66 expect(parseFrame(new Uint8Array(10))).toBeNull();
67 expect(codecOf(pics[0]!)).toBe("avc1.42c01f");
68 });
69
70 test("starts at a keyframe, configures from the SPS, decodes to the canvas", () => {
71 const t = setup();
72 t.ws.onmessage!({ data: JSON.stringify({ ladder: [{ name: "720p" }], rendition: "720p" }) });
73 t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) }); // joined mid-GOP: ignored
74 expect(t.decoded).toHaveLength(0);
75 t.ws.onmessage!({ data: frame(true, 0, 10_000, pics[0]!) });
76 t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[1]!) });
77 expect(t.configured).toEqual({ codec: "avc1.42c01f", optimizeForLatency: true });
78 expect(t.decoded.map((d) => d.type)).toEqual(["key", "delta"]);
79 expect(t.drawn).toHaveLength(2);
80 expect(t.canvas.width).toBe(1280);
81 expect(t.states).toContain("live");
82 t.player.stop();
83 });
84
85 test("a decoder that falls behind skips to the next keyframe; age is the delay above the best seen", () => {
86 const t = setup();
87 t.ws.onmessage!({ data: frame(true, 0, 9_950, pics[0]!) }); // 50 ms away
88 t.ws.onmessage!({ data: frame(false, 0, 9_950, pics[1]!) });
89 t.decoder().decodeQueueSize = 9;
90 t.ws.onmessage!({ data: frame(false, 0, 9_800, pics[2]!) }); // 200 ms old: 150 ms above the best
91 expect(t.player.age).toBe(150);
92 expect(t.decoded).toHaveLength(2); // dropped
93 t.decoder().decodeQueueSize = 0;
94 t.ws.onmessage!({ data: frame(false, 0, 10_000, pics[3]!) }); // still no key
95 expect(t.decoded).toHaveLength(2);
96 t.ws.onmessage!({ data: frame(true, 0, 9_850, pics[5]!) }); // 150 ms away; the best seen is 0 by now
97 expect(t.decoded).toHaveLength(3);
98 t.player.report();
99 expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ age: 150 });
100 t.player.quality("240p");
101 expect(JSON.parse(t.ws.sent.at(-1)!)).toEqual({ quality: "240p" });
102 t.player.stop();
103 });
104
105 test("says none when the socket closes before any picture", () => {
106 const t = setup();
107 t.ws.onclose!();
108 expect(t.states).toEqual(["none"]);
109 t.player.stop();
110 });
111 });
Software/so101/src/video-fanout.test.ts added +113
@@ -0,0 +1,113 @@
1 import { describe, expect, test } from "bun:test";
2 import { clip } from "./h264-clip.ts";
3 import { HEADER, LADDER, VideoFanout, type VideoSink } from "./video-fanout.ts";
4
5 function sink(): VideoSink & { sent: Buffer[]; texts: any[]; queued: number } {
6 const s = {
7 sent: [] as Buffer[],
8 texts: [] as any[],
9 queued: 0,
10 backlog: () => s.queued,
11 send: (d: Uint8Array) => void s.sent.push(Buffer.from(d)),
12 text: (t: string) => void s.texts.push(JSON.parse(t)),
13 };
14 return s;
15 }
16 const frames = (s: { sent: Buffer[] }) => s.sent.map((m) => ({ key: (m[0]! & 1) === 1, rung: m[1]! }));
17 const pictures = clip(10, 10, 5, 100);
18 // Each rung's own clip, sized by its bitrate, like the three encoders on the Pi.
19 const clips = new Map(LADDER.map((r) => [r.name, clip(10, 10, 5, r.kbps / 3)]));
20 /** Every rung gets the next pictures, one at a time. */
21 function play(f: VideoFanout, from: number, to: number) {
22 for (let i = from; i < to; i++) for (const r of f.ladder) f.push(r.name, { key: i % 5 === 0, data: clips.get(r.name)![i]! });
23 }
24
25 describe("renditions and viewers", () => {
26 test("the driver gets the best rung and a watcher the middle one, each from a keyframe", () => {
27 const f = new VideoFanout();
28 const driver = sink(), watcher = sink();
29 f.join(driver, true);
30 f.join(watcher);
31 expect(driver.texts[0].rendition).toBe("720p");
32 expect(watcher.texts[0].rendition).toBe("480p");
33 play(f, 2, 12); // joined mid-GOP: nothing until picture 5
34 expect(frames(driver)[0]).toEqual({ key: true, rung: 0 });
35 expect(frames(watcher)[0]).toEqual({ key: true, rung: 1 });
36 expect(driver.sent).toHaveLength(7);
37 expect(driver.sent[0]!.length).toBe(HEADER + clips.get('720p')![5]!.length);
38 expect(f.wanted()).toEqual(new Set(["720p", "480p"]));
39 });
40
41 test("a slow viewer skips to the next keyframe instead of queueing, then steps down at a keyframe", () => {
42 let t = 0;
43 const f = new VideoFanout({ now: () => t, dropsToStepDown: 2 });
44 const slow = sink(), fast = sink();
45 const viewer = f.join(slow);
46 f.join(fast);
47 play(f, 0, 1);
48 slow.queued = 1 << 20; // the link backs up
49 play(f, 1, 5); // four pictures: dropped, none sent
50 expect(slow.sent).toHaveLength(1);
51 expect(viewer.stats().skipped).toBe(1); // the first miss; the rest wait for a key without counting
52 slow.queued = 1 << 20;
53 play(f, 5, 6);
54 slow.queued = 0;
55 play(f, 6, 10); // clear again but there is no keyframe: still waiting
56 expect(slow.sent).toHaveLength(1);
57 t += 1000;
58 f.adapt();
59 play(f, 10, 11); // a keyframe: the viewer is back
60 expect(slow.sent).toHaveLength(2);
61 expect(fast.sent).toHaveLength(11); // the fast viewer never missed one
62 });
63
64 test("two viewers at different simulated bandwidths end up on different rungs", () => {
65 let t = 0;
66 const f = new VideoFanout({ now: () => t, dropsToStepDown: 1, cleanMs: 5000, maxBacklog: 2000 });
67 const wifi = sink(), mobile = sink();
68 f.join(wifi);
69 const m = f.join(mobile);
70 // The mobile link carries 300 bytes per 100 ms; whatever it cannot carry sits in its backlog.
71 const send = mobile.send;
72 mobile.send = (d) => (send(d), (mobile.queued += d.length));
73 for (let second = 0; second < 30; second++) {
74 for (let i = 0; i < 10; i++) {
75 mobile.queued = Math.max(0, mobile.queued - 300);
76 const n = (second * 10 + i) % 100;
77 play(f, n % 10, (n % 10) + 1);
78 t += 100;
79 }
80 f.adapt();
81 }
82 const onRung = (s: { sent: Buffer[] }, rung: number) => frames(s).filter((x) => x.rung === rung).length;
83 expect(mobile.texts.some((x) => x.rendition === "240p")).toBe(true);
84 expect(onRung(mobile, 2)).toBeGreaterThan(onRung(mobile, 1) + onRung(mobile, 0));
85 expect(onRung(wifi, 0)).toBeGreaterThan(onRung(wifi, 2)); // clean for 5 s: stepped up to the best
86 expect(wifi.texts.some((x) => x.rendition === "720p")).toBe(true);
87 expect(m.stats().bytes * 3).toBeLessThan(wifi.sent.reduce((n, b) => n + b.length, 0));
88 });
89
90 test("a pinned quality holds against drops; auto releases it; junk is ignored", () => {
91 const f = new VideoFanout();
92 const s = sink();
93 const v = f.join(s);
94 v.message('{"quality":"720p"}');
95 v.message("not json");
96 v.message('{"quality":"4k"}');
97 v.message('{"quality":' + '9'.repeat(300) + "}");
98 play(f, 0, 1);
99 expect(v.stats().rendition).toBe("720p");
100 for (let i = 0; i < 5; i++) (v.message('{"age":2000}'), f.adapt());
101 expect(f.wanted()).toEqual(new Set(["720p"]));
102 v.message('{"quality":"auto"}');
103 });
104
105 test("a viewer that leaves is no longer fed or wanted", () => {
106 const f = new VideoFanout();
107 const s = sink();
108 f.join(s).leave();
109 play(f, 0, 3);
110 expect(s.sent).toHaveLength(0);
111 expect(f.wanted().size).toBe(0);
112 });
113 });
Software/so101/src/video-fanout.ts added +197
@@ -0,0 +1,197 @@
1 /**
2 * One camera's H.264 renditions, fanned out to viewers over their WebSockets, a viewer on one rendition at a time.
3 *
4 * Latency before everything: nothing is queued for a viewer. A viewer whose socket is backed up drops pictures and
5 * waits for the next keyframe, and that same backlog is how its link is measured: a viewer that dropped steps down
6 * the ladder, one that went a while clean steps back up, each switch at a keyframe of the new rendition. A viewer can
7 * pin one rendition. The driver is patient (it steps down only after more drops) so the best picture stays with it.
8 *
9 * A binary message is HEADER bytes then the access unit: byte 0 bit 0 = keyframe, byte 1 = the rendition's index in
10 * the ladder, bytes 2..9 = the time the relay got the picture (ms, float64 big-endian). Text messages are JSON.
11 */
12
13 import type { AccessUnit } from "./h264.ts";
14
15 export interface Rendition {
16 name: string;
17 width: number;
18 height: number;
19 fps: number;
20 kbps: number;
21 }
22
23 /** The ladder the Pi encodes, best first. */
24 export const LADDER: readonly Rendition[] = [
25 { name: "720p", width: 1280, height: 720, fps: 20, kbps: 1500 },
26 { name: "480p", width: 854, height: 480, fps: 15, kbps: 600 },
27 { name: "240p", width: 426, height: 240, fps: 10, kbps: 200 },
28 ];
29
30 export const HEADER = 10;
31
32 export interface VideoSink {
33 /** Bytes accepted but not yet on the wire. */
34 backlog(): number;
35 send(data: Uint8Array): void;
36 text(message: string): void;
37 }
38
39 export interface FanoutOptions {
40 ladder?: readonly Rendition[];
41 /** Backlog past which a picture is dropped for that viewer. */
42 maxBacklog?: number;
43 /** Drops (in one window) that step an ordinary viewer down; the driver takes twice as many. */
44 dropsToStepDown?: number;
45 /** How long a viewer must go without a drop before it steps up. */
46 cleanMs?: number;
47 /** A frame age the viewer reports above this also counts as a drop. */
48 maxAgeMs?: number;
49 now?: () => number;
50 }
51
52 interface Viewer {
53 sink: VideoSink;
54 driver: boolean;
55 /** Index of the rendition the viewer is on, and the one it is moving to. */
56 current: number;
57 target: number;
58 pinned: number | null;
59 /** Waiting for a keyframe on `current`. */
60 needKey: boolean;
61 drops: number;
62 lastDrop: number;
63 lastStep: number;
64 bytes: number;
65 frames: number;
66 skipped: number;
67 }
68
69 export interface ViewerStats {
70 rendition: string;
71 bytes: number;
72 frames: number;
73 skipped: number;
74 }
75
76 export class VideoFanout {
77 readonly ladder: readonly Rendition[];
78 private readonly viewers = new Set<Viewer>();
79 private readonly o: Required<Omit<FanoutOptions, "ladder">>;
80
81 constructor(options: FanoutOptions = {}) {
82 this.ladder = options.ladder ?? LADDER;
83 this.o = { maxBacklog: 64 << 10, dropsToStepDown: 3, cleanMs: 6000, maxAgeMs: 500, now: Date.now, ...options };
84 }
85
86 get watching(): number {
87 return this.viewers.size;
88 }
89
90 /** The renditions someone is on or moving to: the only ones worth pulling from the Pi. */
91 wanted(): Set<string> {
92 const names = new Set<string>();
93 for (const v of this.viewers) names.add(this.ladder[v.current]!.name).add(this.ladder[v.target]!.name);
94 return names;
95 }
96
97 /** A viewer joins on the best rendition (the driver) or the middle of the ladder (a watcher), and waits for a keyframe. */
98 join(sink: VideoSink, driver = false): { leave(): void; message(text: string): void; stats(): ViewerStats } {
99 const start = driver ? 0 : Math.min(1, this.ladder.length - 1);
100 const now = this.o.now();
101 const v: Viewer = { sink, driver, current: start, target: start, pinned: null, needKey: true, drops: 0, lastDrop: now, lastStep: now, bytes: 0, frames: 0, skipped: 0 };
102 this.viewers.add(v);
103 sink.text(JSON.stringify({ ladder: this.ladder, rendition: this.ladder[start]!.name }));
104 return {
105 leave: () => void this.viewers.delete(v),
106 message: (text) => this.message(v, text),
107 stats: () => ({ rendition: this.ladder[v.current]!.name, bytes: v.bytes, frames: v.frames, skipped: v.skipped }),
108 };
109 }
110
111 /** A viewer's JSON: {"quality":"auto"|<name>} to pin one, {"age":ms} for the frame age it measures. Anything else is ignored. */
112 private message(v: Viewer, text: string) {
113 if (text.length > 200) return;
114 let m: { quality?: unknown; age?: unknown };
115 try {
116 m = JSON.parse(text);
117 } catch {
118 return;
119 }
120 if (typeof m !== "object" || m === null) return;
121 if (typeof m.quality === "string") {
122 const at = this.ladder.findIndex((r) => r.name === m.quality);
123 if (m.quality === "auto") v.pinned = null;
124 else if (at >= 0) (v.pinned = at), (v.target = at);
125 }
126 if (typeof m.age === "number" && Number.isFinite(m.age) && m.age > this.o.maxAgeMs) this.drop(v);
127 }
128
129 /** The next picture of a rendition arrives from the Pi. */
130 push(rendition: string, unit: AccessUnit): void {
131 const at = this.ladder.findIndex((r) => r.name === rendition);
132 if (at < 0) return;
133 const now = this.o.now();
134 const message = Buffer.allocUnsafe(HEADER + unit.data.length);
135 message[0] = unit.key ? 1 : 0;
136 message[1] = at;
137 message.writeDoubleBE(now, 2);
138 unit.data.copy(message, HEADER);
139 for (const v of this.viewers) {
140 if (v.target === at && v.current !== at && unit.key) {
141 v.current = at;
142 v.needKey = false;
143 v.lastStep = now;
144 }
145 if (v.current !== at) continue;
146 if (v.needKey && (!unit.key || v.target !== v.current)) continue; // waiting for a key, and for the rung it is moving to
147
148 if (v.sink.backlog() > this.o.maxBacklog) {
149 v.skipped++;
150 v.needKey = true;
151 this.drop(v);
152 continue;
153 }
154 v.needKey = false;
155 v.sink.send(message);
156 v.bytes += message.length;
157 v.frames++;
158 }
159 }
160
161 private drop(v: Viewer) {
162 const now = this.o.now();
163 v.drops++;
164 v.lastDrop = now;
165 v.needKey = true;
166 }
167
168 /** Run about once a second: steps viewers down after drops and up after a clean while. */
169 adapt(): void {
170 const now = this.o.now();
171 for (const v of this.viewers) {
172 if (v.pinned !== null) {
173 v.drops = 0;
174 continue;
175 }
176 const limit = this.o.dropsToStepDown * (v.driver ? 2 : 1);
177 if (v.drops >= limit && v.target < this.ladder.length - 1) {
178 v.target = v.current + 1 > this.ladder.length - 1 ? v.current : v.current + 1;
179 this.announce(v);
180 } else if (v.drops === 0 && v.target === v.current && v.current > 0 && now - v.lastDrop >= this.o.cleanMs && now - v.lastStep >= this.o.cleanMs) {
181 v.target = v.current - 1;
182 this.announce(v);
183 }
184 if (v.drops >= limit) v.drops = 0;
185 else if (now - v.lastDrop > 2000) v.drops = 0;
186 }
187 }
188
189 private announce(v: Viewer) {
190 v.lastStep = this.o.now();
191 v.sink.text(JSON.stringify({ rendition: this.ladder[v.target]!.name }));
192 }
193
194 close(): void {
195 this.viewers.clear();
196 }
197 }
Software/so101/src/video-feed.test.ts added +100
@@ -0,0 +1,100 @@
1 import { afterEach, describe, expect, test } from "bun:test";
2 import { createServer, type Server } from "node:http";
3 import WebSocket, { WebSocketServer } from "ws";
4 import { clip } from "./h264-clip.ts";
5 import { parseVideo } from "./relay.ts";
6 import { HEADER, LADDER } from "./video-fanout.ts";
7 import { VideoFeed } from "./video-feed.ts";
8
9 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
10 const closers: (() => void)[] = [];
11 afterEach(() => closers.splice(0).forEach((c) => c()));
12
13 /** The Pi: each rung is an endless Annex-B stream of its own clip, 20 pictures a second, looping. */
14 async function pi(): Promise<string> {
15 const clips = new Map(LADDER.map((r) => [`/${r.name}`, clip(2, 20, 10, r.kbps / 3)]));
16 const server: Server = createServer((req, res) => {
17 const pictures = clips.get(req.url ?? "");
18 if (!pictures) return void res.writeHead(404).end();
19 res.writeHead(200, { "content-type": "video/h264" });
20 let i = 0;
21 const timer = setInterval(() => res.write(pictures[i++ % pictures.length]!), 50);
22 res.on("close", () => clearInterval(timer));
23 });
24 await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
25 closers.push(() => (server.closeAllConnections(), server.close()));
26 return `http://127.0.0.1:${(server.address() as { port: number }).port}`;
27 }
28
29 /** A relay-shaped server: every WebSocket is a viewer; `driver` is chosen by the path. */
30 async function relay(feed: VideoFeed): Promise<string> {
31 const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" });
32 wss.on("connection", (ws, req) => feed.serve(ws, req.url === "/driver"));
33 await new Promise((r) => wss.once("listening", r));
34 closers.push(() => (feed.close(), wss.close()));
35 return `ws://127.0.0.1:${(wss.address() as { port: number }).port}`;
36 }
37
38 function watch(url: string) {
39 const got = { frames: [] as { key: boolean; rung: number; sentAt: number; bytes: number }[], texts: [] as any[] };
40 const ws = new WebSocket(url);
41 ws.on("message", (data, binary) => {
42 const m = data as Buffer;
43 if (!binary) return void got.texts.push(JSON.parse(String(data)));
44 got.frames.push({ key: (m[0]! & 1) === 1, rung: m[1]!, sentAt: m.readDoubleBE(2), bytes: m.length });
45 });
46 closers.push(() => ws.close());
47 return { ws, got };
48 }
49
50 describe("video feed", () => {
51 test("a driver and a watcher are fed different renditions of the Pi's streams, from keyframes, and only those are pulled", async () => {
52 const base = await pi();
53 const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` });
54 const url = await relay(feed);
55 const driver = watch(`${url}/driver`);
56 const watcher = watch(`${url}/watcher`);
57 await sleep(700);
58 expect(driver.got.frames.length).toBeGreaterThan(5);
59 expect(driver.got.frames[0]!.key).toBe(true);
60 expect(watcher.got.frames[0]!.key).toBe(true);
61 expect(new Set(driver.got.frames.map((f) => f.rung))).toEqual(new Set([0]));
62 expect(new Set(watcher.got.frames.map((f) => f.rung))).toEqual(new Set([1]));
63 expect(driver.got.frames.at(-1)!.bytes).toBeGreaterThan(HEADER);
64 expect(Math.abs(Date.now() - driver.got.frames.at(-1)!.sentAt)).toBeLessThan(500);
65 expect(feed.fanout.wanted()).toEqual(new Set(["720p", "480p"])); // 240p is not pulled
66 expect(driver.got.texts[0].ladder.map((r: { name: string }) => r.name)).toEqual(["720p", "480p", "240p"]);
67 });
68
69 test("a viewer pinning a quality gets that rung, switched at a keyframe", async () => {
70 const base = await pi();
71 const feed = new VideoFeed({ "720p": `${base}/720p`, "480p": `${base}/480p`, "240p": `${base}/240p` });
72 const { ws, got } = watch(`${await relay(feed)}/watcher`);
73 await sleep(400);
74 ws.send(JSON.stringify({ quality: "240p" }));
75 await sleep(1500);
76 const at = got.frames.findIndex((f) => f.rung === 2);
77 expect(at).toBeGreaterThan(0);
78 expect(got.frames[at]!.key).toBe(true);
79 expect(got.frames.slice(at).every((f) => f.rung === 2)).toBe(true);
80 });
81
82 test("a feed without sources is not available; a down source just sends nothing", async () => {
83 expect(new VideoFeed({}).available).toBe(false);
84 const feed = new VideoFeed({ "720p": "http://127.0.0.1:1/x" });
85 expect(feed.available).toBe(true);
86 const { got } = watch(`${await relay(feed)}/driver`);
87 await sleep(300);
88 expect(got.frames).toHaveLength(0);
89 expect(got.texts[0].rendition).toBe("720p");
90 });
91 });
92
93 describe("ARM_H264_n", () => {
94 test("names rungs of the ladder with http URLs", () => {
95 expect(parseVideo({})).toEqual({});
96 expect(parseVideo({ ARM_H264_1: "720p=pi:8082/h264, 240p=http://pi:8083/h264" })).toEqual({ 1: { "720p": "http://pi:8082/h264", "240p": "http://pi:8083/h264" } });
97 expect(() => parseVideo({ ARM_H264_1: "4k=pi:1" })).toThrow(/ladder/);
98 expect(() => parseVideo({ ARM_H264_1: "720p=ftp://pi/x" })).toThrow();
99 });
100 });
Software/so101/src/video-feed.ts added +107
@@ -0,0 +1,107 @@
1 /**
2 * One camera's H.264 over WebSockets: a VideoFanout fed by the Pi's encoder streams.
3 *
4 * Each rendition is an HTTP stream of Annex-B H.264 on the Pi. It is pulled only while some viewer is on it (or
5 * moving to it), so a camera nobody watches costs the Pi nothing, and the lower rungs cost nothing until a viewer
6 * needs them. A source that is down is tried again every second while wanted; its viewers simply see no pictures, and
7 * the page falls back to MJPEG when none come.
8 */
9
10 import { AccessUnitSplitter } from "./h264.ts";
11 import { VideoFanout, type FanoutOptions, type VideoSink } from "./video-fanout.ts";
12
13 export interface VideoSources {
14 /** Rendition name → the URL of its Annex-B stream. */
15 [name: string]: string;
16 }
17
18 /** The wire the feed talks to a viewer over; a `ws` WebSocket fits. */
19 export interface VideoSocket {
20 bufferedAmount: number;
21 readyState: number;
22 send(data: Uint8Array | string): void;
23 on(event: "message", listener: (data: unknown, isBinary: boolean) => void): unknown;
24 on(event: "close", listener: () => void): unknown;
25 }
26
27 const OPEN = 1;
28
29 export class VideoFeed {
30 readonly fanout: VideoFanout;
31 private readonly sources: VideoSources;
32 private readonly pulls = new Map<string, AbortController>();
33 private timer: ReturnType<typeof setInterval> | undefined;
34 private closed = false;
35
36 constructor(sources: VideoSources, options: FanoutOptions = {}) {
37 this.sources = sources;
38 this.fanout = new VideoFanout(options);
39 }
40
41 /** Whether this feed has an encoder stream for any rung of the ladder. */
42 get available(): boolean {
43 return this.fanout.ladder.some((r) => this.sources[r.name]);
44 }
45
46 /** Serve one viewer's WebSocket until it closes. */
47 serve(ws: VideoSocket, driver: boolean): void {
48 if (this.closed) return;
49 const sink: VideoSink = {
50 backlog: () => ws.bufferedAmount,
51 send: (data) => ws.readyState === OPEN && ws.send(data),
52 text: (message) => ws.readyState === OPEN && ws.send(message),
53 };
54 const viewer = this.fanout.join(sink, driver);
55 ws.on("message", (data, isBinary) => {
56 if (!isBinary) viewer.message(String(data));
57 });
58 ws.on("close", () => {
59 viewer.leave();
60 if (!this.fanout.watching) this.idle();
61 });
62 this.timer ??= setInterval(() => this.tick(), 1000);
63 this.timer.unref?.();
64 this.tick();
65 }
66
67 close(): void {
68 this.closed = true;
69 this.fanout.close();
70 this.idle();
71 }
72
73 private idle() {
74 clearInterval(this.timer);
75 this.timer = undefined;
76 for (const c of this.pulls.values()) c.abort();
77 this.pulls.clear();
78 }
79
80 private tick() {
81 this.fanout.adapt();
82 const wanted = this.fanout.wanted();
83 for (const [name, controller] of this.pulls) {
84 if (!wanted.has(name)) (controller.abort(), this.pulls.delete(name));
85 }
86 for (const name of wanted) if (this.sources[name] && !this.pulls.has(name)) this.pull(name, this.sources[name]!);
87 }
88
89 private pull(name: string, url: string) {
90 const controller = new AbortController();
91 this.pulls.set(name, controller);
92 void (async () => {
93 try {
94 const response = await fetch(url, { signal: controller.signal, cache: "no-store", redirect: "error" });
95 if (!response.ok || !response.body) throw new Error(`answered ${response.status}`);
96 const splitter = new AccessUnitSplitter();
97 for await (const chunk of response.body as unknown as AsyncIterable<Uint8Array>) {
98 for (const unit of splitter.push(chunk)) this.fanout.push(name, unit);
99 }
100 } catch {
101 // down or cut off: the next tick tries again while the rung is wanted
102 } finally {
103 if (this.pulls.get(name) === controller) this.pulls.delete(name);
104 }
105 })();
106 }
107 }
Software/so101/src/viewer.html +35 −2
@@ -65,7 +65,9 @@
6565 #banner button { margin-left: 10px; min-height: 32px; padding: 2px 10px; }
6666 .pane { flex: 1; min-width: 0; position: relative; background: #000; display: none; }
6767 .pane.on { display: block; }
68 .pane img { width: 100%; height: 100%; object-fit: contain; display: block; }
68 .pane img, .pane canvas { width: 100%; height: 100%; object-fit: contain; display: block; }
69 .pane canvas { position: absolute; inset: 0; }
70 .pane select.quality { position: absolute; right: calc(8px + var(--safe-r, 0px)); bottom: 8px; font-size: 11px; background: rgba(0,0,0,.55); color: #c9d1d9; border: 0; border-radius: 4px; padding: 2px 4px; }
6971 .pane .tag { position: absolute; left: calc(8px + var(--safe-l)); bottom: 8px; font-size: 11px; color: #c9d1d9; background: rgba(0,0,0,.55); padding: 2px 6px; border-radius: 4px; pointer-events: none; }
7072 urdf-viewer { display: block; width: 100%; height: 100%; }
7173 #top { position: absolute; top: 6px; right: calc(6px + var(--safe-r)); left: calc(6px + var(--safe-l)); z-index: 5; display: flex; gap: 6px; align-items: center; justify-content: flex-end; flex-wrap: wrap; font-size: 12px; pointer-events: none; }
@@ -640,11 +642,40 @@
640642 } catch { c.live = false; }
641643 if (plan.mode === "snapshot") c.timer = setTimeout(() => snap(c), Math.max(0, plan.everyMs - (performance.now() - started)));
642644 }
645 // H.264 over the relay's WebSocket, decoded into a canvas, where the browser has WebCodecs and the camera offers it; MJPEG otherwise, and when no picture comes.
646 function stopVideo(c) {
647 c.player?.stop(); c.player = null;
648 c.canvas?.remove(); c.canvas = null; c.chip?.remove(); c.chip = null;
649 c.img.style.visibility = "";
650 }
651 async function startVideo(c) {
652 stopVideo(c);
653 const { VideoPlayer } = await import("/ui/__VERSION__/video-player.js"); // only a viewer who gets H.264 downloads it
654 if (!paneOn(c) || c.player || c.canvas) return;
655 c.canvas = el("canvas", { ariaLabel: c.img.alt });
656 c.chip = el("select", { className: "quality", title: "Video quality" });
657 c.chip.onchange = () => c.player?.quality(c.chip.value);
658 c.pane.append(c.canvas, c.chip);
659 const player = c.player = new VideoPlayer({
660 url: `${location.protocol === "https:" ? "wss:" : "ws:"}//${location.host}/camera/${c.id}/video`,
661 canvas: c.canvas,
662 onStatus: (s) => {
663 if (c.player !== player) return;
664 if (s.state === "none") { c.h264Failed = true; stopVideo(c); if (paneOn(c)) c.img.src = streamUrl(c); return; }
665 c.img.style.visibility = "hidden";
666 if (c.chip.options.length !== s.ladder.length + 1) c.chip.replaceChildren(new Option("auto", "auto"), ...s.ladder.map((r) => new Option(r.name, r.name)));
667 c.tag.textContent = `${c.name} · ${s.rendition ?? ""} · ${Math.round(s.age)} ms`;
668 },
669 });
670 player.start();
671 }
643672 const showCam = (c, on) => {
644673 c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on));
645674 clearTimeout(c.timer);
675 stopVideo(c);
646676 if (!on || document.hidden) c.img.src = "";
647677 else if (plan.mode === "snapshot" && c.snapshot) snap(c);
678 else if (c.h264 && !c.h264Failed && relayed && typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function") { c.img.src = ""; startVideo(c); }
648679 else c.img.src = streamUrl(c);
649680 };
650681 function setPane(key, on) {
@@ -700,7 +731,7 @@
700731 }
701732 function label(c, info) {
702733 c.online = info.online;
703 c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name;
734 if (!c.player) c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name;
704735 c.btn.textContent = info.name; c.btn.title = info.online === false ? `${info.name}: offline` : info.name;
705736 c.img.alt = info.name;
706737 }
@@ -710,8 +741,10 @@
710741 let c = cams.find((x) => x.id === info.id);
711742 if (!c) {
712743 c = addCam({ ...info, url: relayed ? `/camera/${info.id}/stream` : info.url });
744 c.h264 = info.h264 === true; c.name = info.name;
713745 if (viewsReady && wanted(c.pane.dataset.key)) showCam(c, true);
714746 }
747 c.h264 = info.h264 === true; c.name = info.name;
715748 label(c, info);
716749 }
717750 showQuality();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs