Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

The page plays a camera's H.264 through video-pane.js when it offers it and WebCodecs exists, and falls back to the frame feed when no picture comes within 3 s #422

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/383-the-page-plays-a-camera-s-h-264-through → main 2 commits fork

  • Merged. smith/383-the-page-plays-a-camera-s-h-264-through was merged into main as 5ed77fd.

Review

Sign in to review this change.
4 files changed +133 −6
Software/so101/src/server.test.ts +12
@@ -171,6 +171,18 @@ describe("viewer page", () => {
171171 expect(script).toMatch(/import\("\/ui\/[0-9a-f]{12}\/metrics-panel\.js"\)/);
172172 });
173173
174 test("video-pane.js and video-player.js load only for a camera flagged h264, and a silent H.264 feed falls back to the frames", async () => {
175 const page = await (await fetch(`${base}/`)).text();
176 const script = page.slice(page.indexOf('<script type="module">'));
177 expect(page).not.toMatch(/<link rel="modulepreload" href="[^"]*video-(pane|player)\.js">/);
178 expect(script).not.toMatch(/^\s*import [^;]*? from "[^"]*video-(pane|player)\.js";/m);
179 const lines = script.split("\n").filter((l) => /import\("[^"]*video-(pane|player)\.js"\)/.test(l));
180 expect(lines.length).toBe(1);
181 expect(script).toContain("c.h264 === true && !c.noH264 && await playH264(c, epoch)");
182 expect(script).toContain("c.noH264 = true; startFeed(c)");
183 expect(script).toContain("pane.playH264(c,");
184 });
185
174186 test("names its modules by their content, serves them for good, and serves nothing else from src/ui", async () => {
175187 const page = await (await fetch(`${base}/`)).text();
176188 expect(page).not.toContain("__VERSION__");
Software/so101/src/ui/video-pane.js +38
@@ -26,3 +26,41 @@ export function qualitySelect(doc, options, onChange) {
2626 select.addEventListener("change", () => onChange(select.value));
2727 return select;
2828 }
29
30 /** Which of a camera's two elements shows: the canvas while H.264 plays, else the img; the quality select goes with the canvas. @param {any} c @param {boolean} on */
31 export function showH264(c, on) {
32 c.img.hidden = on;
33 c.canvas.hidden = !on;
34 if (c.qsel) c.qsel.hidden = !on;
35 }
36
37 /**
38 * Starts the H.264 feed for camera `c` (its pane, img, canvas) if it can play: true when a player was started, false when the frames should be used.
39 * `d.fallback(c)` is called when the player hears no picture; the select is rebuilt with each player's ladder and always speaks to the current one.
40 * @param {any} c @param {{ Player: any, canDecode: () => boolean, doc: any, loc: { protocol: string, host: string }, fallback: (c: any) => void }} d
41 */
42 export function playH264(c, d) {
43 if (choose(c, d.canDecode()) !== "h264") return false;
44 const feed = { h264: true, live: false, player: null, ladder: "", close() { feed.player.stop(); } };
45 feed.player = new d.Player({
46 url: `${d.loc.protocol === "https:" ? "wss" : "ws"}://${d.loc.host}/camera/${c.id}/video`,
47 canvas: c.canvas,
48 onStatus: (st) => {
49 if (c.feed !== feed) return;
50 if (st.state === "live") { feed.live = true; showH264(c, true); }
51 else if (st.state === "none") return d.fallback(c);
52 const names = (st.ladder ?? []).map((l) => String(l.name));
53 if (names.length && names.join() !== feed.ladder) {
54 feed.ladder = names.join();
55 c.qsel?.remove();
56 c.qsel = qualitySelect(d.doc, [{ value: "auto", label: "Auto" }, ...names.map((n) => ({ value: n, label: n }))], (v) => c.feed?.player?.quality(v));
57 c.qsel.className = "q";
58 c.qsel.hidden = !feed.live;
59 c.pane.append(c.qsel);
60 }
61 },
62 });
63 c.feed = feed;
64 feed.player.start();
65 return true;
66 }
Software/so101/src/ui/video-pane.test.ts +65 −1
@@ -1,5 +1,5 @@
11 import { describe, expect, test } from "bun:test";
2 import { choose, qualitySelect } from "./video-pane.js";
2 import { choose, playH264, qualitySelect, showH264 } from "./video-pane.js";
33
44 function fakeDoc() {
55 const el = (tag: string) => {
@@ -36,4 +36,68 @@ describe("the video pane", () => {
3636 s.listeners.change();
3737 expect(heard).toEqual(["sd"]);
3838 });
39
40 describe("playH264", () => {
41 class FakePlayer {
42 static all: FakePlayer[] = [];
43 started = false; stopped = false; pinned: string[] = [];
44 constructor(public o: any) { FakePlayer.all.push(this); }
45 start() { this.started = true; }
46 stop() { this.stopped = true; }
47 quality(q: string) { this.pinned.push(q); }
48 }
49 const node = () => ({ hidden: false, className: "", removed: false, remove() { this.removed = true; } });
50 function setup() {
51 FakePlayer.all = [];
52 const doc = { createElement: (tag: string) => ({ ...fakeDoc().createElement(tag), hidden: false, className: "", removed: false, remove() { this.removed = true; } }) };
53 const c: any = { id: 7, h264: true, img: node(), canvas: node(), qsel: undefined, feed: null, pane: { kids: [] as any[], append(k: any) { this.kids.push(k); } } };
54 const fallbacks: any[] = [];
55 const d = { Player: FakePlayer, canDecode: () => true, doc, loc: { protocol: "https:", host: "h" }, fallback: (x: any) => fallbacks.push(x) };
56 return { c, d, fallbacks };
57 }
58 const ladder = [{ name: "720p" }, { name: "480p" }];
59
60 test("does not start without support or a flag, and connects to /camera/<id>/video", () => {
61 const { c, d } = setup();
62 expect(playH264({ ...c, h264: false }, d)).toBe(false);
63 expect(playH264(c, { ...d, canDecode: () => false })).toBe(false);
64 expect(FakePlayer.all.length).toBe(0);
65 expect(playH264(c, d)).toBe(true);
66 expect(FakePlayer.all[0]!.o.url).toBe("wss://h/camera/7/video");
67 expect(FakePlayer.all[0]!.started).toBe(true);
68 });
69
70 test("live shows the canvas and the select; none hands back to the frames", () => {
71 const { c, d, fallbacks } = setup();
72 playH264(c, d);
73 const p = FakePlayer.all[0]!;
74 p.o.onStatus({ state: "live", ladder });
75 expect([c.img.hidden, c.canvas.hidden, c.qsel.hidden]).toEqual([true, false, false]);
76 expect(c.qsel.children.map((o: any) => o.value)).toEqual(["auto", "720p", "480p"]);
77 p.o.onStatus({ state: "none" });
78 expect(fallbacks).toEqual([c]);
79 showH264(c, false);
80 expect([c.img.hidden, c.canvas.hidden, c.qsel.hidden]).toEqual([false, true, true]);
81 });
82
83 test("after hide and show the select speaks to the new player, not the stopped one", () => {
84 const { c, d } = setup();
85 playH264(c, d);
86 const first = FakePlayer.all[0]!;
87 first.o.onStatus({ state: "live", ladder });
88 const old = c.qsel;
89 c.feed.close(); c.feed = null; showH264(c, false); // showCam: stop, hide
90 expect(first.stopped).toBe(true);
91 playH264(c, d);
92 const second = FakePlayer.all[1]!;
93 second.o.onStatus({ state: "live", ladder: [{ name: "360p" }] });
94 expect(old.removed).toBe(true);
95 expect(c.qsel).not.toBe(old);
96 c.qsel.value = "360p"; c.qsel.listeners.change();
97 expect(second.pinned).toEqual(["360p"]);
98 expect(first.pinned).toEqual([]);
99 first.o.onStatus({ state: "live", ladder }); // a stale player's status is ignored
100 expect(c.qsel.children.map((o: any) => o.value)).toEqual(["auto", "360p"]);
101 });
102 });
39103 });
Software/so101/src/viewer.html +18 −5
@@ -69,7 +69,9 @@
6969 #banner button { margin-left: 10px; min-height: 32px; padding: 2px 10px; }
7070 .pane { flex: 1; min-width: 0; position: relative; background: #000; display: none; }
7171 .pane.on { display: block; }
72 .pane img { width: 100%; height: 100%; object-fit: contain; display: block; }
72 .pane img, .pane canvas { width: 100%; height: 100%; object-fit: contain; display: block; }
73 .pane canvas[hidden], .pane .q[hidden] { display: none; }
74 .pane .q { position: absolute; right: calc(8px + var(--safe-r, 0px)); bottom: 8px; }
7375 .pane .tag { position: absolute; left: calc(8px + var(--safe-l)); bottom: 8px; font-size: 11px; color: #c9d1d9; background: rgba(0,0,0,.55); padding: 2px 6px; border-radius: 4px; pointer-events: none; }
7476 urdf-viewer { display: block; width: 100%; height: 100%; }
7577 #top { position: absolute; top: 6px; right: calc(6px + var(--safe-r)); left: calc(6px + var(--safe-l)); z-index: 5; display: flex; gap: 6px; align-items: center; justify-content: flex-end; flex-wrap: wrap; font-size: 12px; pointer-events: none; }
@@ -626,14 +628,24 @@
626628 // Behind the relay the camera is frames over WS /camera/<id>/ws (src/ui/video.js, imported when first needed); on the machine itself, the plain <img> stream.
627629 let video = null;
628630 const stopFeed = (c) => { c.epoch = (c.epoch ?? 0) + 1; c.feed?.close?.(); c.feed = null; };
631 // A camera the relay flags h264 plays through ui/video-pane.js and video-player.js, only where WebCodecs exists; no picture in 3 s and it goes back to the frames.
632 async function playH264(c, epoch) {
633 let pane, player;
634 try { [pane, player] = await Promise.all([import("/ui/__VERSION__/video-pane.js"), import("/ui/__VERSION__/video-player.js")]); } catch { return false; }
635 if (c.epoch !== epoch) return true;
636 c.showH264 = pane.showH264;
637 return pane.playH264(c, { Player: player.VideoPlayer, canDecode: player.canDecode, doc: document, loc: location, fallback: (c) => { stopFeed(c); pane.showH264(c, false); c.noH264 = true; startFeed(c); } });
638 }
629639 async function startFeed(c) {
630640 const epoch = c.epoch;
641 if (c.h264 === true && !c.noH264 && await playH264(c, epoch)) return;
642 if (c.epoch !== epoch) return;
631643 try { video ??= await import("/ui/__VERSION__/video.js"); } catch { return; }
632644 video.attach(c, epoch, { visible: () => paneOn(c) && !document.hidden, loc: location, maxFps: plan.fps, create: URL.createObjectURL, revoke: URL.revokeObjectURL, now: Date.now });
633645 }
634646 const showCam = (c, on) => {
635647 c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on));
636 clearTimeout(c.timer); stopFeed(c);
648 c.showH264?.(c, false); clearTimeout(c.timer); stopFeed(c);
637649 if (!on || document.hidden) c.img.src = "";
638650 else if (plan.mode === "snapshot" && c.snapshot) snap(c);
639651 else if (relayed) startFeed(c);
@@ -678,7 +690,8 @@
678690 c.snapshot = /\/stream(?=$|\?)/.test(c.url) ? c.url.replace(/\/stream(?=$|\?)/, "/snapshot") : null;
679691 c.img = el("img", { alt: info.name, decoding: "async", crossOrigin: "anonymous" });
680692 c.tag = el("span", { className: "tag" });
681 c.pane = el("div", { className: "pane" }, c.img, c.tag); c.pane.dataset.key = `cam:${info.id}`;
693 c.h264 = info.h264 === true; c.canvas = el("canvas", { hidden: true });
694 c.pane = el("div", { className: "pane" }, c.img, c.canvas, c.tag); c.pane.dataset.key = `cam:${info.id}`;
682695 c.btn = el("button", { type: "button" }); c.btn.setAttribute("aria-pressed", "false");
683696 c.img.onerror = () => { if (!relayed && paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); };
684697 c.btn.onclick = () => pick(c.pane.dataset.key);
@@ -694,7 +707,7 @@
694707 fallback();
695708 }
696709 function label(c, info) {
697 c.online = info.online;
710 c.online = info.online; c.h264 = info.h264 === true;
698711 c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name;
699712 c.btn.textContent = info.name; c.btn.title = info.online === false ? `${info.name}: offline` : info.name;
700713 c.img.alt = info.name;
@@ -748,7 +761,7 @@
748761 document.addEventListener("visibilitychange", () => { for (const c of cams) if (paneOn(c)) showCam(c, true); });
749762 setInterval(() => {
750763 for (const c of cams) {
751 if (c.feed && paneOn(c)) { c.live = video.isLive(c, Date.now()); continue; }
764 if (c.feed && paneOn(c)) { c.live = c.feed.h264 ? c.feed.live : video.isLive(c, Date.now()); continue; }
752765 if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue;
753766 let changed = false;
754767 try {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs