Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
The page plays a camera's H.264 through video-pane.js when it offers it and WebCodecs exists, and falls back to the frame feed when no picture comes within 3 s #422
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/383-the-page-plays-a-camera-s-h-264-through → main 2 commits fork
4 files
changed +133
−6
Software/so101/src/server.test.ts +12
| @@ -171,6 +171,18 @@ describe("viewer page", () => { | ||
| 171 | 171 | expect(script).toMatch(/import\("\/ui\/[0-9a-f]{12}\/metrics-panel\.js"\)/); |
| 172 | 172 | }); |
| 173 | 173 | |
| 174 | test("video-pane.js and video-player.js load only for a camera flagged h264, and a silent H.264 feed falls back to the frames", async () => { | |
| 175 | const page = await (await fetch(`${base}/`)).text(); | |
| 176 | const script = page.slice(page.indexOf('<script type="module">')); | |
| 177 | expect(page).not.toMatch(/<link rel="modulepreload" href="[^"]*video-(pane|player)\.js">/); | |
| 178 | expect(script).not.toMatch(/^\s*import [^;]*? from "[^"]*video-(pane|player)\.js";/m); | |
| 179 | const lines = script.split("\n").filter((l) => /import\("[^"]*video-(pane|player)\.js"\)/.test(l)); | |
| 180 | expect(lines.length).toBe(1); | |
| 181 | expect(script).toContain("c.h264 === true && !c.noH264 && await playH264(c, epoch)"); | |
| 182 | expect(script).toContain("c.noH264 = true; startFeed(c)"); | |
| 183 | expect(script).toContain("pane.playH264(c,"); | |
| 184 | }); | |
| 185 | ||
| 174 | 186 | test("names its modules by their content, serves them for good, and serves nothing else from src/ui", async () => { |
| 175 | 187 | const page = await (await fetch(`${base}/`)).text(); |
| 176 | 188 | expect(page).not.toContain("__VERSION__"); |
Software/so101/src/ui/video-pane.js +38
| @@ -26,3 +26,41 @@ export function qualitySelect(doc, options, onChange) { | ||
| 26 | 26 | select.addEventListener("change", () => onChange(select.value)); |
| 27 | 27 | return select; |
| 28 | 28 | } |
| 29 | ||
| 30 | /** Which of a camera's two elements shows: the canvas while H.264 plays, else the img; the quality select goes with the canvas. @param {any} c @param {boolean} on */ | |
| 31 | export function showH264(c, on) { | |
| 32 | c.img.hidden = on; | |
| 33 | c.canvas.hidden = !on; | |
| 34 | if (c.qsel) c.qsel.hidden = !on; | |
| 35 | } | |
| 36 | ||
| 37 | /** | |
| 38 | * Starts the H.264 feed for camera `c` (its pane, img, canvas) if it can play: true when a player was started, false when the frames should be used. | |
| 39 | * `d.fallback(c)` is called when the player hears no picture; the select is rebuilt with each player's ladder and always speaks to the current one. | |
| 40 | * @param {any} c @param {{ Player: any, canDecode: () => boolean, doc: any, loc: { protocol: string, host: string }, fallback: (c: any) => void }} d | |
| 41 | */ | |
| 42 | export function playH264(c, d) { | |
| 43 | if (choose(c, d.canDecode()) !== "h264") return false; | |
| 44 | const feed = { h264: true, live: false, player: null, ladder: "", close() { feed.player.stop(); } }; | |
| 45 | feed.player = new d.Player({ | |
| 46 | url: `${d.loc.protocol === "https:" ? "wss" : "ws"}://${d.loc.host}/camera/${c.id}/video`, | |
| 47 | canvas: c.canvas, | |
| 48 | onStatus: (st) => { | |
| 49 | if (c.feed !== feed) return; | |
| 50 | if (st.state === "live") { feed.live = true; showH264(c, true); } | |
| 51 | else if (st.state === "none") return d.fallback(c); | |
| 52 | const names = (st.ladder ?? []).map((l) => String(l.name)); | |
| 53 | if (names.length && names.join() !== feed.ladder) { | |
| 54 | feed.ladder = names.join(); | |
| 55 | c.qsel?.remove(); | |
| 56 | c.qsel = qualitySelect(d.doc, [{ value: "auto", label: "Auto" }, ...names.map((n) => ({ value: n, label: n }))], (v) => c.feed?.player?.quality(v)); | |
| 57 | c.qsel.className = "q"; | |
| 58 | c.qsel.hidden = !feed.live; | |
| 59 | c.pane.append(c.qsel); | |
| 60 | } | |
| 61 | }, | |
| 62 | }); | |
| 63 | c.feed = feed; | |
| 64 | feed.player.start(); | |
| 65 | return true; | |
| 66 | } |
Software/so101/src/ui/video-pane.test.ts +65 −1
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | import { describe, expect, test } from "bun:test"; |
| 2 | import { choose, qualitySelect } from "./video-pane.js"; | |
| 2 | import { choose, playH264, qualitySelect, showH264 } from "./video-pane.js"; | |
| 3 | 3 | |
| 4 | 4 | function fakeDoc() { |
| 5 | 5 | const el = (tag: string) => { |
| @@ -36,4 +36,68 @@ describe("the video pane", () => { | ||
| 36 | 36 | s.listeners.change(); |
| 37 | 37 | expect(heard).toEqual(["sd"]); |
| 38 | 38 | }); |
| 39 | ||
| 40 | describe("playH264", () => { | |
| 41 | class FakePlayer { | |
| 42 | static all: FakePlayer[] = []; | |
| 43 | started = false; stopped = false; pinned: string[] = []; | |
| 44 | constructor(public o: any) { FakePlayer.all.push(this); } | |
| 45 | start() { this.started = true; } | |
| 46 | stop() { this.stopped = true; } | |
| 47 | quality(q: string) { this.pinned.push(q); } | |
| 48 | } | |
| 49 | const node = () => ({ hidden: false, className: "", removed: false, remove() { this.removed = true; } }); | |
| 50 | function setup() { | |
| 51 | FakePlayer.all = []; | |
| 52 | const doc = { createElement: (tag: string) => ({ ...fakeDoc().createElement(tag), hidden: false, className: "", removed: false, remove() { this.removed = true; } }) }; | |
| 53 | const c: any = { id: 7, h264: true, img: node(), canvas: node(), qsel: undefined, feed: null, pane: { kids: [] as any[], append(k: any) { this.kids.push(k); } } }; | |
| 54 | const fallbacks: any[] = []; | |
| 55 | const d = { Player: FakePlayer, canDecode: () => true, doc, loc: { protocol: "https:", host: "h" }, fallback: (x: any) => fallbacks.push(x) }; | |
| 56 | return { c, d, fallbacks }; | |
| 57 | } | |
| 58 | const ladder = [{ name: "720p" }, { name: "480p" }]; | |
| 59 | ||
| 60 | test("does not start without support or a flag, and connects to /camera/<id>/video", () => { | |
| 61 | const { c, d } = setup(); | |
| 62 | expect(playH264({ ...c, h264: false }, d)).toBe(false); | |
| 63 | expect(playH264(c, { ...d, canDecode: () => false })).toBe(false); | |
| 64 | expect(FakePlayer.all.length).toBe(0); | |
| 65 | expect(playH264(c, d)).toBe(true); | |
| 66 | expect(FakePlayer.all[0]!.o.url).toBe("wss://h/camera/7/video"); | |
| 67 | expect(FakePlayer.all[0]!.started).toBe(true); | |
| 68 | }); | |
| 69 | ||
| 70 | test("live shows the canvas and the select; none hands back to the frames", () => { | |
| 71 | const { c, d, fallbacks } = setup(); | |
| 72 | playH264(c, d); | |
| 73 | const p = FakePlayer.all[0]!; | |
| 74 | p.o.onStatus({ state: "live", ladder }); | |
| 75 | expect([c.img.hidden, c.canvas.hidden, c.qsel.hidden]).toEqual([true, false, false]); | |
| 76 | expect(c.qsel.children.map((o: any) => o.value)).toEqual(["auto", "720p", "480p"]); | |
| 77 | p.o.onStatus({ state: "none" }); | |
| 78 | expect(fallbacks).toEqual([c]); | |
| 79 | showH264(c, false); | |
| 80 | expect([c.img.hidden, c.canvas.hidden, c.qsel.hidden]).toEqual([false, true, true]); | |
| 81 | }); | |
| 82 | ||
| 83 | test("after hide and show the select speaks to the new player, not the stopped one", () => { | |
| 84 | const { c, d } = setup(); | |
| 85 | playH264(c, d); | |
| 86 | const first = FakePlayer.all[0]!; | |
| 87 | first.o.onStatus({ state: "live", ladder }); | |
| 88 | const old = c.qsel; | |
| 89 | c.feed.close(); c.feed = null; showH264(c, false); // showCam: stop, hide | |
| 90 | expect(first.stopped).toBe(true); | |
| 91 | playH264(c, d); | |
| 92 | const second = FakePlayer.all[1]!; | |
| 93 | second.o.onStatus({ state: "live", ladder: [{ name: "360p" }] }); | |
| 94 | expect(old.removed).toBe(true); | |
| 95 | expect(c.qsel).not.toBe(old); | |
| 96 | c.qsel.value = "360p"; c.qsel.listeners.change(); | |
| 97 | expect(second.pinned).toEqual(["360p"]); | |
| 98 | expect(first.pinned).toEqual([]); | |
| 99 | first.o.onStatus({ state: "live", ladder }); // a stale player's status is ignored | |
| 100 | expect(c.qsel.children.map((o: any) => o.value)).toEqual(["auto", "360p"]); | |
| 101 | }); | |
| 102 | }); | |
| 39 | 103 | }); |
Software/so101/src/viewer.html +18 −5
| @@ -69,7 +69,9 @@ | ||
| 69 | 69 | #banner button { margin-left: 10px; min-height: 32px; padding: 2px 10px; } |
| 70 | 70 | .pane { flex: 1; min-width: 0; position: relative; background: #000; display: none; } |
| 71 | 71 | .pane.on { display: block; } |
| 72 | .pane img { width: 100%; height: 100%; object-fit: contain; display: block; } | |
| 72 | .pane img, .pane canvas { width: 100%; height: 100%; object-fit: contain; display: block; } | |
| 73 | .pane canvas[hidden], .pane .q[hidden] { display: none; } | |
| 74 | .pane .q { position: absolute; right: calc(8px + var(, 0px)); bottom: 8px; } | |
| 73 | 75 | .pane .tag { position: absolute; left: calc(8px + var()); bottom: 8px; font-size: 11px; color: #c9d1d9; background: rgba(0,0,0,.55); padding: 2px 6px; border-radius: 4px; pointer-events: none; } |
| 74 | 76 | urdf-viewer { display: block; width: 100%; height: 100%; } |
| 75 | 77 | #top { position: absolute; top: 6px; right: calc(6px + var()); left: calc(6px + var()); z-index: 5; display: flex; gap: 6px; align-items: center; justify-content: flex-end; flex-wrap: wrap; font-size: 12px; pointer-events: none; } |
| @@ -626,14 +628,24 @@ | ||
| 626 | 628 | // Behind the relay the camera is frames over WS /camera/<id>/ws (src/ui/video.js, imported when first needed); on the machine itself, the plain <img> stream. |
| 627 | 629 | let video = null; |
| 628 | 630 | const stopFeed = (c) => { c.epoch = (c.epoch ?? 0) + 1; c.feed?.close?.(); c.feed = null; }; |
| 631 | // A camera the relay flags h264 plays through ui/video-pane.js and video-player.js, only where WebCodecs exists; no picture in 3 s and it goes back to the frames. | |
| 632 | async function playH264(c, epoch) { | |
| 633 | let pane, player; | |
| 634 | try { [pane, player] = await Promise.all([import("/ui/__VERSION__/video-pane.js"), import("/ui/__VERSION__/video-player.js")]); } catch { return false; } | |
| 635 | if (c.epoch !== epoch) return true; | |
| 636 | c.showH264 = pane.showH264; | |
| 637 | return pane.playH264(c, { Player: player.VideoPlayer, canDecode: player.canDecode, doc: document, loc: location, fallback: (c) => { stopFeed(c); pane.showH264(c, false); c.noH264 = true; startFeed(c); } }); | |
| 638 | } | |
| 629 | 639 | async function startFeed(c) { |
| 630 | 640 | const epoch = c.epoch; |
| 641 | if (c.h264 === true && !c.noH264 && await playH264(c, epoch)) return; | |
| 642 | if (c.epoch !== epoch) return; | |
| 631 | 643 | try { video ??= await import("/ui/__VERSION__/video.js"); } catch { return; } |
| 632 | 644 | video.attach(c, epoch, { visible: () => paneOn(c) && !document.hidden, loc: location, maxFps: plan.fps, create: URL.createObjectURL, revoke: URL.revokeObjectURL, now: Date.now }); |
| 633 | 645 | } |
| 634 | 646 | const showCam = (c, on) => { |
| 635 | 647 | c.pane.classList.toggle("on", on); c.btn.classList.toggle("on", on); c.btn.setAttribute("aria-pressed", String(on)); |
| 636 | clearTimeout(c.timer); stopFeed(c); | |
| 648 | c.showH264?.(c, false); clearTimeout(c.timer); stopFeed(c); | |
| 637 | 649 | if (!on || document.hidden) c.img.src = ""; |
| 638 | 650 | else if (plan.mode === "snapshot" && c.snapshot) snap(c); |
| 639 | 651 | else if (relayed) startFeed(c); |
| @@ -678,7 +690,8 @@ | ||
| 678 | 690 | c.snapshot = /\/stream(?=$|\?)/.test(c.url) ? c.url.replace(/\/stream(?=$|\?)/, "/snapshot") : null; |
| 679 | 691 | c.img = el("img", { alt: info.name, decoding: "async", crossOrigin: "anonymous" }); |
| 680 | 692 | c.tag = el("span", { className: "tag" }); |
| 681 | c.pane = el("div", { className: "pane" }, c.img, c.tag); c.pane.dataset.key = `cam:${info.id}`; | |
| 693 | c.h264 = info.h264 === true; c.canvas = el("canvas", { hidden: true }); | |
| 694 | c.pane = el("div", { className: "pane" }, c.img, c.canvas, c.tag); c.pane.dataset.key = `cam:${info.id}`; | |
| 682 | 695 | c.btn = el("button", { type: "button" }); c.btn.setAttribute("aria-pressed", "false"); |
| 683 | 696 | c.img.onerror = () => { if (!relayed && paneOn(c) && plan.mode === "stream" && c.img.getAttribute("src")) setTimeout(() => { if (paneOn(c) && plan.mode === "stream" && !document.hidden) c.img.src = streamUrl(c); }, 3000); }; |
| 684 | 697 | c.btn.onclick = () => pick(c.pane.dataset.key); |
| @@ -694,7 +707,7 @@ | ||
| 694 | 707 | fallback(); |
| 695 | 708 | } |
| 696 | 709 | function label(c, info) { |
| 697 | c.online = info.online; | |
| 710 | c.online = info.online; c.h264 = info.h264 === true; | |
| 698 | 711 | c.tag.textContent = info.online === false ? `${info.name} · offline` : info.name; |
| 699 | 712 | c.btn.textContent = info.name; c.btn.title = info.online === false ? `${info.name}: offline` : info.name; |
| 700 | 713 | c.img.alt = info.name; |
| @@ -748,7 +761,7 @@ | ||
| 748 | 761 | document.addEventListener("visibilitychange", () => { for (const c of cams) if (paneOn(c)) showCam(c, true); }); |
| 749 | 762 | setInterval(() => { |
| 750 | 763 | for (const c of cams) { |
| 751 | if (c.feed && paneOn(c)) { c.live = video.isLive(c, Date.now()); continue; } | |
| 764 | if (c.feed && paneOn(c)) { c.live = c.feed.h264 ? c.feed.live : video.isLive(c, Date.now()); continue; } | |
| 752 | 765 | if (!paneOn(c) || (plan.mode === "snapshot" && c.snapshot)) continue; |
| 753 | 766 | let changed = false; |
| 754 | 767 | try { |
new above
offline