Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
Each viewer reports command round trip, camera fps and frame age, and the owner sees them live under Admin -> Health #302
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/89-each-viewer-reports-command-round-trip-c → main 5 commits fork 3 threads · 3 unresolved
18 files
changed +551
−21
Software/README.md +10
| @@ -528,6 +528,10 @@ watches. A line you type is said in the chat; commands start with `/`: | ||
| 528 | 528 | command in and out and every clamp, line by line as it happens. `/machine` |
| 529 | 529 | again prints the joints again; `/machine off` stops the stream. Anyone may, |
| 530 | 530 | guests too: it is read-only. |
| 531 | - `/health`, owner or moderator: Admin → Health as lines, one per open page | |
| 532 | (serial, name, round trip p50/p95, sent/s, dropped/s, camera fps, frame age, | |
| 533 | stalls, reconnects), from the same `GET /admin/health`. A guest or a plain | |
| 534 | user is told it is for the owner and moderators and sees no rows. | |
| 531 | 535 | - `/who` (who drives, who waits), `/help`, `/quit`. |
| 532 | 536 | |
| 533 | 537 | The terminal is a client of the running server, like the page: it opens |
| @@ -903,6 +907,12 @@ security headers). It carries the round trip to the arm server, which `src/arm-l | ||
| 903 | 907 | `pi_rtt_ms{quantile="0.95"}`, `pi_rtt_ms_sum` and `pi_rtt_ms_count`, over the last 256 samples. It is empty until the |
| 904 | 908 | first round trip is measured. |
| 905 | 909 | |
| 910 | Each open page also reports its own link every 5 s over `/live` (`src/viewer-reports.ts`), and the relay adds one | |
| 911 | `viewer_*{viewer="v1"}` gauge per viewer: `viewer_rtt_p50`, `viewer_rtt_p95` (command to pose, ms), `viewer_sent_per_s`, | |
| 912 | `viewer_dropped_per_s` (command targets folded into a newer one in the same 33 ms slot before they went out, `src/ui/frames.js`), `viewer_cam_fps`, `viewer_frame_age_ms`, `viewer_stalls` and `viewer_reconnects`. A viewer is a | |
| 913 | serial number, never a name; the series go when the socket closes. The owner sees the same, with names, under Admin → | |
| 914 | Health (`GET /admin/health`), and in the terminal with `/health`. | |
| 915 | ||
| 906 | 916 | |
| 907 | 917 | curl https://arm--forge-trevato-apps.open-platform.sh/metrics.prom |
| 908 | 918 |
Software/so101/src/admin-page.ts +40 −1
| @@ -9,6 +9,7 @@ import { ANNOUNCEMENT_CHARS, type AdminSettings, type LogEntry } from "./admin.t | ||
| 9 | 9 | import type { Removed } from "./chat.ts"; |
| 10 | 10 | import type { Person, Who } from "./identity.ts"; |
| 11 | 11 | import type { Snapshot } from "./turns.ts"; |
| 12 | import type { ViewerRow } from "./viewer-reports.ts"; | |
| 12 | 13 | |
| 13 | 14 | export const esc = (text: string): string => text.replace(/[&<>"']/g, (c) => `&#${c.charCodeAt(0)};`); |
| 14 | 15 | |
| @@ -46,12 +47,43 @@ textarea { min-height: 5.5rem; resize: vertical; } | ||
| 46 | 47 | ol { list-style: none; margin: 0; padding: 0; display: grid; gap: .5rem; font-size: .95rem; } |
| 47 | 48 | ol li { display: grid; grid-template-columns: 7.5rem 1fr; overflow-wrap: anywhere; gap: 1rem; padding-bottom: .5rem; border-bottom: 1px solid var(--line); } |
| 48 | 49 | time { color: var(--dim); font-variant-numeric: tabular-nums; } |
| 50 | table { width: 100%; border-collapse: collapse; font-size: .85rem; font-variant-numeric: tabular-nums; } | |
| 51 | th, td { text-align: right; padding: .3rem .5rem; border-bottom: 1px solid var(--line); } | |
| 52 | th:first-child, td:first-child { text-align: left; } | |
| 53 | th { color: var(--dim); font-weight: 600; } | |
| 54 | .scroll { overflow-x: auto; } | |
| 49 | 55 | .inline { display: inline; margin: 0; } |
| 50 | 56 | .empty { color: var(--dim); } |
| 51 | 57 | @media (prefers-reduced-motion: reduce) { * { transition: none !important; } } |
| 52 | 58 | `; |
| 53 | 59 | |
| 54 | export const ADMIN_CSP = `default-src 'none'; style-src 'sha256-${createHash("sha256").update(STYLE).digest("base64")}'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`; | |
| 60 | /** The Health table's columns: the row key and its heading. */ | |
| 61 | const HEALTH_COLUMNS: readonly (readonly [keyof ViewerRow, string])[] = [ | |
| 62 | ["name", "Viewer"], ["rttP50", "RTT p50 ms"], ["rttP95", "RTT p95 ms"], ["sentPerS", "Sent/s"], ["droppedPerS", "Dropped/s"], | |
| 63 | ["camFps", "Cam fps"], ["frameAgeMs", "Frame age ms"], ["stalls", "Stalls"], ["reconnects", "Reconnects"], | |
| 64 | ]; | |
| 65 | ||
| 66 | /** Refreshes the Health table from /admin/health every 2 s; cells are set as text, so nothing in a row is read as markup. */ | |
| 67 | const HEALTH_SCRIPT = ` | |
| 68 | const body = document.getElementById("health-rows"); | |
| 69 | const cols = ${JSON.stringify(HEALTH_COLUMNS.map(([k]) => k))}; | |
| 70 | async function tick() { | |
| 71 | try { | |
| 72 | const r = await fetch("/admin/health", { cache: "no-store", credentials: "same-origin" }); | |
| 73 | if (!r.ok) return; | |
| 74 | const rows = await r.json(); | |
| 75 | body.replaceChildren(...rows.map((row) => { | |
| 76 | const tr = document.createElement("tr"); | |
| 77 | for (const k of cols) { const td = document.createElement("td"); td.textContent = String(row[k]); tr.append(td); } | |
| 78 | return tr; | |
| 79 | })); | |
| 80 | document.getElementById("health-none").hidden = rows.length > 0; | |
| 81 | } catch {} | |
| 82 | } | |
| 83 | setInterval(tick, 2000); | |
| 84 | `; | |
| 85 | ||
| 86 | export const ADMIN_CSP = `default-src 'none'; style-src 'sha256-${createHash("sha256").update(STYLE).digest("base64")}'; script-src 'sha256-${createHash("sha256").update(HEALTH_SCRIPT).digest("base64")}'; connect-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`; | |
| 55 | 87 | |
| 56 | 88 | /** The fixed lines a redirect back may ask for; nothing from the request is echoed. */ |
| 57 | 89 | export const DONE = { saved: "Saved.", paused: "Paused: nobody new gets a turn.", resumed: "Open again: the next in line has control.", ended: "Turn ended.", taken: "You have control.", restored: "Line restored." } as const; |
| @@ -63,6 +95,8 @@ export interface AdminPage { | ||
| 63 | 95 | canEdit: boolean; |
| 64 | 96 | settings: AdminSettings; |
| 65 | 97 | snapshot: Snapshot; |
| 98 | /** What each open page last reported about its link. */ | |
| 99 | health?: readonly ViewerRow[]; | |
| 66 | 100 | log: readonly LogEntry[]; |
| 67 | 101 | /** Chat lines a person removed, newest first. */ |
| 68 | 102 | removed?: readonly Removed[]; |
| @@ -83,6 +117,7 @@ const page = (title: string, body: string) => `<!doctype html> | ||
| 83 | 117 | <main> |
| 84 | 118 | ${body} |
| 85 | 119 | </main> |
| 120 | <script>${HEALTH_SCRIPT}</script> | |
| 86 | 121 | </body> |
| 87 | 122 | </html> |
| 88 | 123 | `; |
| @@ -102,6 +137,9 @@ const time = (at: number) => { | ||
| 102 | 137 | const entries = (list: readonly LogEntry[], none: string): string => |
| 103 | 138 | list.length ? `<ol>${[...list].reverse().map((e) => `<li>${time(e.at)}<span><b>${esc(e.by.name)}</b> ${esc(e.what)}</span></li>`).join("")}</ol>` : `<p class="empty">${none}</p>`; |
| 104 | 139 | |
| 140 | const health = (rows: readonly ViewerRow[]): string => | |
| 141 | `<div class="scroll"><table aria-label="Viewer health"><caption>Each open page's link, as it last reported; refreshed every 2 seconds.</caption><thead><tr>${HEALTH_COLUMNS.map(([, h]) => `<th scope="col">${h}</th>`).join("")}</tr></thead><tbody id="health-rows">${rows.map((r) => `<tr>${HEALTH_COLUMNS.map(([k]) => `<td>${esc(String(r[k]))}</td>`).join("")}</tr>`).join("")}</tbody></table></div><p class="empty" id="health-none"${rows.length ? " hidden" : ""}>No viewer has reported yet.</p>`; | |
| 142 | ||
| 105 | 143 | const field = (name: keyof AdminSettings, label: string, hint: string, value: number, min: number, max: number, off: string) => |
| 106 | 144 | `<label>${label}<small>${hint}</small><input name="${name}" type="number" inputmode="numeric" required min="${min}" max="${max}" step="1" value="${value}"${off}></label>`; |
| 107 | 145 | |
| @@ -155,6 +193,7 @@ ${field("slowModeSeconds", "Slow mode", "seconds between a person's chat message | ||
| 155 | 193 | ${p.canEdit ? `<div class="save"><button class="go">Save settings</button></div>` : ""} |
| 156 | 194 | </form> |
| 157 | 195 | </section> |
| 196 | <section aria-labelledby="health"><h2 id="health">Health</h2>${health(p.health ?? [])}</section> | |
| 158 | 197 | <section aria-labelledby="mods"><h2 id="mods">Moderators</h2>${mods}</section> |
| 159 | 198 | <section aria-labelledby="moderation"><h2 id="moderation">Moderation</h2>${actions}</section> |
| 160 | 199 | <section aria-labelledby="removed"><h2 id="removed">Removed lines</h2>${removedLines(p.removed ?? [], p.canEdit)}</section> |
Software/so101/src/admin-routes.ts +11 −1
| @@ -33,8 +33,18 @@ export function serveAdminPage(lobby: Lobby, req: IncomingMessage, res: ServerRe | ||
| 33 | 33 | res.end(JSON.stringify({ error: "no such route" })); |
| 34 | 34 | return null; |
| 35 | 35 | } |
| 36 | if (url.pathname === "/admin/health") { | |
Health is readable only on the page and over HTTP; add the equivalent read to src/tty.ts with the same gate, and document it in the README. |
||
| 37 | if (req.method !== "GET") { | |
| 38 | res.writeHead(405, { "content-type": "application/json", "cache-control": "no-store", allow: "GET" }); | |
| 39 | res.end(JSON.stringify({ error: "GET" })); | |
| 40 | } else { | |
| 41 | res.writeHead(200, { "content-type": "application/json", "cache-control": "no-store" }); | |
| 42 | res.end(JSON.stringify(lobby.reports.rows())); | |
| 43 | } | |
| 44 | return null; | |
| 45 | } | |
| 36 | 46 | const show = (status: number, done: Done | null, error: string | null) => |
| 37 | html(status, renderAdmin({ who, canEdit, settings: lobby.admin.settings, snapshot: lobby.turns.snapshot(), log: lobby.admin.log, removed: lobby.chat.removed, done, error })); | |
| 47 | html(status, renderAdmin({ who, canEdit, settings: lobby.admin.settings, snapshot: lobby.turns.snapshot(), log: lobby.admin.log, removed: lobby.chat.removed, health: lobby.reports.rows(), done, error })); | |
| 38 | 48 | if (req.method === "GET" || req.method === "HEAD") { |
| 39 | 49 | const done = url.searchParams.get("done"); |
| 40 | 50 | show(200, done && Object.hasOwn(DONE, done) ? (done as Done) : null, null); |
Software/so101/src/lobby.test.ts +3 −2
| @@ -762,7 +762,7 @@ describe("lobby", () => { | ||
| 762 | 762 | const ended = await form(port, { action: "end" }, OWNER); |
| 763 | 763 | expect(ended.status).toBe(303); |
| 764 | 764 | expect(ended.headers.get("location")).toBe("/admin?done=ended"); |
| 765 | await alice.next(drivingIs("bob")); | |
| 765 | await alice.first(drivingIs("bob")); | |
| 766 | 766 | |
| 767 | 767 | const saved = await form(port, { action: "save", turnSeconds: "90", idleSeconds: "20", maxSpeedPercent: "40", maxQueue: "3", announcement: '<script>alert("x")</script>' }, OWNER); |
| 768 | 768 | expect(saved.status).toBe(303); |
| @@ -777,7 +777,8 @@ describe("lobby", () => { | ||
| 777 | 777 | expect(ok.status).toBe(200); |
| 778 | 778 | expect(ok.headers.get("content-security-policy")).toMatch(/^default-src 'none'; style-src 'sha256-[A-Za-z0-9+/=]+'/); |
| 779 | 779 | const html = await ok.text(); |
| 780 | expect(html).not.toContain("<script"); | |
| 780 | expect(html.match(/<script/g)).toHaveLength(1); // the Health refresh, pinned by hash in the CSP; the announcement's is escaped | |
| 781 | expect(html).not.toContain("<script>alert"); | |
| 781 | 782 | expect(html).toContain("<script>alert("x")</script>"); |
| 782 | 783 | expect(html).toContain('name="maxSpeedPercent" type="number" inputmode="numeric" required min="5" max="100" step="1" value="40"'); |
| 783 | 784 | expect(html).toContain("<b>trevato</b> ended alice's turn"); |
Software/so101/src/lobby.ts +12
| @@ -21,6 +21,7 @@ | ||
| 21 | 21 | * {"type": "dismiss", "id": 12} owner: dismiss a mute suggestion |
| 22 | 22 | * {"type": "ban", "id": "<user id>", "on": true|false} owner: bars their account, name and address; they still watch |
| 23 | 23 | * {"type": "words", "words": ["…"]} owner: the chat's word filter; a line with a listed word is refused |
| 24 | * {"type": "report", …} anyone: the page's own timings (src/viewer-reports.ts), every 5 s | |
| 24 | 25 | * {"type": "pong"} the answer to {"type": "ping"} |
| 25 | 26 | * {"type": "history", "before": 12} anyone: up to HISTORY_PAGE lines older than that one |
| 26 | 27 | * Server → client: hello (who you are, and whether you may moderate: delete a line), state (the turn, the line, pause, the announcement and the speed cap), |
| @@ -41,6 +42,7 @@ import { Moderator } from "./moderator.ts"; | ||
| 41 | 42 | import { WordFilter } from "./word-filter.ts"; |
| 42 | 43 | import { identify, LOCAL, type Person, type Who } from "./identity.ts"; |
| 43 | 44 | import { Turns, type TurnEvent } from "./turns.ts"; |
| 45 | import { parseReport, ViewerReports } from "./viewer-reports.ts"; | |
| 44 | 46 | |
| 45 | 47 | /** How long a mute lasts, by the name the page and the terminal send; null is for good. */ |
| 46 | 48 | export const MUTE_FOR: Record<string, number | null> = { "10m": 600_000, "1h": 3_600_000, "24h": 86_400_000, forever: null }; |
| @@ -95,6 +97,8 @@ export class Lobby { | ||
| 95 | 97 | readonly accounts: Accounts; |
| 96 | 98 | readonly admin: Admin; |
| 97 | 99 | readonly filter: WordFilter; |
| 100 | /** What each open page last reported about its own link. */ | |
| 101 | readonly reports: ViewerReports; | |
| 98 | 102 | readonly local: boolean; |
| 99 | 103 | readonly moderator: Moderator | undefined; |
| 100 | 104 | private readonly key: string; |
| @@ -119,6 +123,7 @@ export class Lobby { | ||
| 119 | 123 | this.chat = options.chat ?? new Chat({ file: options.chatFile }); |
| 120 | 124 | this.accounts = options.accounts ?? new Accounts(); |
| 121 | 125 | this.filter = new WordFilter(options.wordsFile); |
| 126 | this.reports = new ViewerReports(this.now); | |
| 122 | 127 | this.silentMs = options.silentMs ?? 15_000; |
| 123 | 128 | this.moderator = options.moderator; |
| 124 | 129 | if (this.moderator) { |
| @@ -293,6 +298,7 @@ export class Lobby { | ||
| 293 | 298 | }); |
| 294 | 299 | ws.on("close", () => { |
| 295 | 300 | this.seats.delete(seat); |
| 301 | this.reports.drop(seat); | |
| 296 | 302 | if (who) this.turns.disconnect(who.id); |
| 297 | 303 | }); |
| 298 | 304 | ws.on("error", () => ws.terminate()); |
| @@ -303,6 +309,12 @@ export class Lobby { | ||
| 303 | 309 | const m = message as Record<string, unknown>; |
| 304 | 310 | const who = seat.who; |
| 305 | 311 | if (m.type === "pong") return; |
| 312 | if (m.type === "report") { | |
| 313 | const report = parseReport(m); | |
| 314 | if (!report) return this.send(seat, { type: "error", error: "that report is not numbers in range" }); | |
| 315 | this.reports.set(seat, who?.name ?? "watcher", report); | |
| 316 | return; | |
| 317 | } | |
| 306 | 318 | if (m.type === "history") { |
| 307 | 319 | if (!Number.isSafeInteger(m.before)) return this.send(seat, { type: "error", error: "history before which line?" }); |
| 308 | 320 | return this.send(seat, { type: "history", ...this.page(m.before as number, HISTORY_PAGE) }); |
Software/so101/src/relay.ts +2 −2
| @@ -564,7 +564,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 564 | 564 | } |
| 565 | 565 | const cameraAdmin = ADMIN_CAMERA.exec(url.pathname); |
| 566 | 566 | if (cameraAdmin) return await administer(req, res, cameraAdmin[1] === undefined ? null : Number(cameraAdmin[1])); |
| 567 | if (lobby && url.pathname === "/admin") return await handleAdmin(lobby, req, res, url, readBody); | |
| 567 | if (lobby && (url.pathname === "/admin" || url.pathname === "/admin/health")) return await handleAdmin(lobby, req, res, url, readBody); | |
| 568 | 568 | if (url.pathname === LEADER) return await leaderCalibration(req, res); |
| 569 | 569 | if (url.pathname === "/join") { |
| 570 | 570 | if (req.method === "POST") return await join(req, res); |
| @@ -588,7 +588,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 588 | 588 | // Public on purpose, like /link: timing aggregates only (README, "Metrics"). |
| 589 | 589 | if (req.method === "GET" && url.pathname === "/metrics.prom") { |
| 590 | 590 | res.writeHead(200, { "content-type": "text/plain; version=0.0.4; charset=utf-8", "cache-control": "no-store" }); |
| 591 | return res.end(metrics.prometheus()); | |
| 591 | return res.end(metrics.prometheus() + (lobby?.reports.prometheus() ?? "")); | |
| 592 | 592 | } |
| 593 | 593 | if (req.method === "GET" && url.pathname === "/link") return json(res, 200, link()); |
| 594 | 594 | if (req.method === "GET" && url.pathname === "/status") return await status(res, path); |
Software/so101/src/terminal.test.ts +48 −2
| @@ -12,7 +12,7 @@ import { identify, sign } from "./identity.ts"; | ||
| 12 | 12 | import { Lobby } from "./lobby.ts"; |
| 13 | 13 | import { startRelay, type RunningRelay } from "./relay.ts"; |
| 14 | 14 | import { startServer, type RunningServer } from "./server.ts"; |
| 15 | import { Door, emptyRoom, hear, liveUrl, machineEntry, machineLines, machineUrl, plain, poster, terminalHeaders, typed } from "./terminal.ts"; | |
| 15 | import { Door, emptyRoom, hear, liveUrl, machineEntry, machineLines, machineUrl, healthLines, plain, poster, reader, terminalHeaders, typed } from "./terminal.ts"; | |
| 16 | 16 | import { jointRows } from "./ui/machine.js"; |
| 17 | 17 | import { Turns } from "./turns.ts"; |
| 18 | 18 | |
| @@ -144,6 +144,36 @@ describe("terminal table", () => { | ||
| 144 | 144 | expect(refused).toEqual({ status: 403, error: "admins only" }); |
| 145 | 145 | }); |
| 146 | 146 | |
| 147 | test("/health asks the reader, prints a line per viewer as text, or the refusal; without a reader it says so", async () => { | |
| 148 | expect(typed(emptyRoom(), "/health")).toEqual({ health: true }); | |
| 149 | const row = { viewer: "v1", name: "al\x1b[31mice", rttP50: 40, rttP95: 90, sentPerS: 30, droppedPerS: 2, camFps: 14.5, frameAgeMs: 120, stalls: 1, reconnects: 2 }; | |
| 150 | expect(healthLines([row])).toEqual(["v1 al[31mice · rtt p50 40 p95 90 ms · sent 30/s · dropped 2/s · cam 14.5 fps · frame age 120 ms · stalls 1 · reconnects 2"]); | |
| 151 | expect(healthLines([])).toEqual(["no viewer has reported yet"]); | |
| 152 | const printed: string[] = []; | |
| 153 | const ok = new Door("ws://127.0.0.1:1/live", {}, (l) => printed.push(l), null, async () => ({ status: 200, error: null, rows: [row] })); | |
| 154 | ok.input("/health"); | |
| 155 | const no = new Door("ws://127.0.0.1:1/live", {}, (l) => printed.push(l), null, async () => ({ status: 404, error: "health is for the owner and moderators", rows: [] })); | |
| 156 | no.input("/health"); | |
| 157 | new Door("ws://127.0.0.1:1/live", {}, (l) => printed.push(l)).input("/health"); | |
| 158 | await Bun.sleep(50); | |
| 159 | expect(printed).toHaveLength(3); | |
| 160 | expect(printed).toContain("error: health is for the owner and moderators"); | |
| 161 | expect(printed).toContain("error: health is not available here"); | |
| 162 | expect(printed.some((l) => l.startsWith("v1 al"))).toBe(true); | |
| 163 | }); | |
| 164 | ||
| 165 | test("the reader GETs /admin/health signed, and a 404 reads as a refusal", async () => { | |
| 166 | const calls: Array<{ url: string; signed: boolean }> = []; | |
| 167 | const answer = (status: number, body: unknown) => async (url: unknown, init: RequestInit = {}) => { | |
| 168 | calls.push({ url: String(url), signed: Boolean((init.headers as Record<string, string>)["x-forge-signature"]) }); | |
| 169 | return new Response(JSON.stringify(body), { status }); | |
| 170 | }; | |
| 171 | const env = { PORT: "9", FORGE_APP_IDENTITY_KEY: "k".repeat(32), FORGE_APP_USER_ID: "u-owner", FORGE_APP_USER: "owner", FORGE_APP_USER_ROLE: "admin" }; | |
| 172 | expect(await reader(env, answer(200, [{ viewer: "v1" }]) as never)()).toEqual({ status: 200, error: null, rows: [{ viewer: "v1" }] }); | |
| 173 | expect(await reader(env, answer(404, { error: "no such route" }) as never)()).toMatchObject({ status: 404, error: "health is for the owner and moderators", rows: [] }); | |
| 174 | expect(calls).toEqual([{ url: "http://127.0.0.1:9/admin/health", signed: true }, { url: "http://127.0.0.1:9/admin/health", signed: true }]); | |
| 175 | }); | |
| 176 | ||
| 147 | 177 | test("the door prints the offset the server holds, or its refusal", async () => { |
| 148 | 178 | const printed: string[] = []; |
| 149 | 179 | const asked: unknown[] = []; |
| @@ -206,7 +236,7 @@ describe("terminal door", () => { | ||
| 206 | 236 | /** A terminal as the forge would start it for this person. */ |
| 207 | 237 | const terminal = async (port: number, env: Record<string, string>) => { |
| 208 | 238 | const printed: string[] = []; |
| 209 | const door = new Door(liveUrl({ ...env, PORT: String(port) }), terminalHeaders(env), (line) => printed.push(line), poster({ ...env, PORT: String(port) })); | |
| 239 | const door = new Door(liveUrl({ ...env, PORT: String(port) }), terminalHeaders(env), (line) => printed.push(line), poster({ ...env, PORT: String(port) }), reader({ ...env, PORT: String(port) })); | |
| 210 | 240 | await door.open(); |
| 211 | 241 | open.push(door); |
| 212 | 242 | const printedLine = (match: RegExp) => waitFor(() => printed.find((l) => match.test(l)), `line matching ${match}; printed ${JSON.stringify(printed)}`); |
| @@ -238,6 +268,22 @@ describe("terminal door", () => { | ||
| 238 | 268 | await alice.printedLine(/^bob: hello terminal$/); |
| 239 | 269 | }); |
| 240 | 270 | |
| 271 | test("/health at the terminal: the owner sees each viewer's report, a guest and a plain user see nothing", async () => { | |
| 272 | const port = await start(); | |
| 273 | const page = await viewer(port, web("u-bob", "bob")); | |
| 274 | page.send({ type: "report", rttP50: 40, rttP95: 90, sentPerS: 30, droppedPerS: 2, camFps: 14.5, frameAgeMs: 120, stalls: 1, reconnects: 2 }); | |
| 275 | await waitFor(() => (lobby!.reports.rows().length ? true : undefined), "the report"); | |
| 276 | const owner = await terminal(port, ttyEnv("u-o", "trevato", "admin")); | |
| 277 | owner.door.input("/health"); | |
| 278 | await owner.printedLine(/^v\d+ bob · rtt p50 40 p95 90 ms · sent 30\/s · dropped 2\/s · cam 14\.5 fps · frame age 120 ms · stalls 1 · reconnects 2$/); | |
| 279 | for (const env of [ttyEnv("u-alice", "alice", "user"), {}]) { | |
| 280 | const other = await terminal(port, env as Record<string, string>); | |
| 281 | other.door.input("/health"); | |
| 282 | await other.printedLine(/^error: health is for the owner and moderators$/); | |
| 283 | expect(other.printed.some((l) => /bob/.test(l))).toBe(false); | |
| 284 | } | |
| 285 | }); | |
| 286 | ||
| 241 | 287 | test("/machine at the terminal: the joints as the page shows them, then every command and clamp as it happens", async () => { |
| 242 | 288 | const port = await start(); |
| 243 | 289 | const owner = await terminal(port, ttyEnv("u-o", "trevato", "admin")); |
Software/so101/src/terminal.ts +44 −4
| @@ -26,6 +26,7 @@ | ||
| 26 | 26 | * /restore #<n> owner: put a removed line back for everyone (Lobby.restoreLine; it is logged) |
| 27 | 27 | * /table here admin: put the table under the arm's lowest link now (POST /settings/table-here) |
| 28 | 28 | * /table <mm> admin: set the table offset, -100..100 mm (PUT /settings tableOffsetMm) |
| 29 | * /health owner or moderator: each open page's link as it last reported it (GET /admin/health, the page's Admin -> Health) | |
| 29 | 30 | * /machine [off] the Machine panel (/machine, src/machine-feed.ts): the joints, the link and the server as they are, |
| 30 | 31 | * then every command in and out as it happens, in the page's own words (src/ui/machine.js); off stops it |
| 31 | 32 | * /who /help /quit |
| @@ -88,6 +89,33 @@ export function poster(env: Env, fetcher: typeof fetch = fetch): Post { | ||
| 88 | 89 | }; |
| 89 | 90 | } |
| 90 | 91 | |
| 92 | /** What GET /admin/health answered: the rows, or why not (a guest is told there is no such route). */ | |
| 93 | export type Health = { status: number; error: string | null; rows: unknown[] }; | |
| 94 | export type Read = () => Promise<Health>; | |
| 95 | ||
| 96 | /** GET the owner's Health rows as the page does, signed afresh; the server decides who may. */ | |
| 97 | export function reader(env: Env, fetcher: typeof fetch = fetch): Read { | |
| 98 | return async () => { | |
| 99 | try { | |
| 100 | const r = await fetcher(`${appUrl(env)}/admin/health`, { headers: terminalHeaders(env), signal: AbortSignal.timeout(5000) }); | |
| 101 | const body = (await r.json().catch(() => null)) as unknown; | |
| 102 | if (r.ok && Array.isArray(body)) return { status: r.status, error: null, rows: body }; | |
| 103 | return { status: r.status, error: r.status === 404 ? "health is for the owner and moderators" : plain((body as { error?: unknown } | null)?.error ?? `status ${r.status}`), rows: [] }; | |
| 104 | } catch (err) { | |
| 105 | return { status: 0, error: plain((err as Error).message), rows: [] }; | |
| 106 | } | |
| 107 | }; | |
| 108 | } | |
| 109 | ||
| 110 | /** One line per viewer, the page's columns in order; text from the room is made safe. */ | |
| 111 | export function healthLines(rows: unknown[]): string[] { | |
| 112 | if (!rows.length) return ["no viewer has reported yet"]; | |
| 113 | return rows.map((r) => { | |
| 114 | const x = r as Record<string, unknown>; | |
| 115 | return plain(`${x.viewer} ${x.name} · rtt p50 ${x.rttP50} p95 ${x.rttP95} ms · sent ${x.sentPerS}/s · dropped ${x.droppedPerS}/s · cam ${x.camFps} fps · frame age ${x.frameAgeMs} ms · stalls ${x.stalls} · reconnects ${x.reconnects}`); | |
| 116 | }); | |
| 117 | } | |
| 118 | ||
| 91 | 119 | /** Text from the room, safe to write to a terminal: no escapes, no control characters, no line breaks. */ |
| 92 | 120 | export const plain = (text: unknown): string => String(text ?? "").replace(/\r\n?|\n|\t/g, " ").replace(UNPRINTABLE, ""); |
| 93 | 121 | |
| @@ -139,7 +167,7 @@ const said = (room: Room, m: unknown): string => { | ||
| 139 | 167 | return from ? `${tag}${plain(from.name)}: ${plain(message?.text)}` : `· ${plain(message?.text)}`; |
| 140 | 168 | }; |
| 141 | 169 | |
| 142 | export const HELP = "type to chat · /join /leave /ready /extend · /stop /resume (driver, owner) · /end [who] /table here|<mm> (owner) · /mute <who> /unmute <who> (owner, moderator) · /delete <who|#id> (owner, moderator) · /removed /restore #<id> (owner) · /machine [off] · /who · /quit"; | |
| 170 | export const HELP = "type to chat · /join /leave /ready /extend · /stop /resume (driver, owner) · /end [who] /table here|<mm> (owner) · /mute <who> /unmute <who> (owner, moderator) · /delete <who|#id> (owner, moderator) · /removed /restore #<id> (owner) · /health (owner, moderator) · /machine [off] · /who · /quit"; | |
| 143 | 171 | |
| 144 | 172 | /** The Machine feed's socket beside the lobby's: same app, same port. */ |
| 145 | 173 | export const machineUrl = (live: string): string => live.replace(/\/live$/, "/machine"); |
| @@ -254,7 +282,7 @@ function target(room: Room, word: string): string | null { | ||
| 254 | 282 | return ID.test(word) ? word : null; |
| 255 | 283 | } |
| 256 | 284 | |
| 257 | export type Typed = { send: Frame } | { post: "/stop" | "/resume" } | { table: "here" | number } | { print: string } | { machine: boolean } | { quit: true } | null; | |
| 285 | export type Typed = { send: Frame } | { post: "/stop" | "/resume" } | { table: "here" | number } | { print: string } | { machine: boolean } | { health: true } | { quit: true } | null; | |
| 258 | 286 | |
| 259 | 287 | /** What a typed line asks for. Validation the server repeats anyway; this only saves a round trip on a typo. */ |
| 260 | 288 | export function typed(room: Room, input: string): Typed { |
| @@ -304,6 +332,8 @@ export function typed(room: Room, input: string): Typed { | ||
| 304 | 332 | if (/^-?\d{1,3}$/.test(arg) && Math.abs(Number(arg)) <= 100) return { table: Number(arg) }; |
| 305 | 333 | return { print: "usage: /table here | /table <mm, -100 to 100>" }; |
| 306 | 334 | } |
| 335 | case "health": | |
| 336 | return { health: true }; | |
| 307 | 337 | case "machine": |
| 308 | 338 | if (arg && arg.toLowerCase() !== "off" && arg.toLowerCase() !== "on") return { print: "usage: /machine [off]" }; |
| 309 | 339 | return { machine: arg.toLowerCase() !== "off" }; |
| @@ -329,9 +359,11 @@ export class Door { | ||
| 329 | 359 | private readonly headers: Record<string, string>; |
| 330 | 360 | private readonly print: (line: string) => void; |
| 331 | 361 | private readonly post: Post | null; |
| 362 | private readonly read: Read | null; | |
| 332 | 363 | |
| 333 | /** `post` reaches the app's /stop and /resume; without it, the door says it cannot. */ | |
| 334 | constructor(url: string, headers: Record<string, string>, print: (line: string) => void, post: Post | null = null) { | |
| 364 | /** `post` reaches the app's /stop and /resume, `read` its /admin/health; without them, the door says it cannot. */ | |
| 365 | constructor(url: string, headers: Record<string, string>, print: (line: string) => void, post: Post | null = null, read: Read | null = null) { | |
| 366 | this.read = read; | |
| 335 | 367 | this.url = url; |
| 336 | 368 | this.headers = headers; |
| 337 | 369 | this.print = print; |
| @@ -371,6 +403,7 @@ export class Door { | ||
| 371 | 403 | if ("print" in t) this.print(t.print); |
| 372 | 404 | else if ("post" in t) void this.ask(t.post); |
| 373 | 405 | else if ("table" in t) void this.table(t.table); |
| 406 | else if ("health" in t) void this.health(); | |
| 374 | 407 | else if ("machine" in t) this.watchMachine(t.machine); |
| 375 | 408 | else this.send(t.send); |
| 376 | 409 | return true; |
| @@ -385,6 +418,13 @@ export class Door { | ||
| 385 | 418 | else if (path === "/resume" && this.room.held === null) this.print("the arm was not stopped"); |
| 386 | 419 | } |
| 387 | 420 | |
| 421 | /** The server decides who may; a refusal is printed, else a line per viewer. */ | |
| 422 | private async health(): Promise<void> { | |
| 423 | if (!this.read) return this.print("error: health is not available here"); | |
| 424 | const r = await this.read(); | |
| 425 | for (const line of r.error ? [`error: ${r.error}`] : healthLines(r.rows)) this.print(line); | |
| 426 | } | |
| 427 | ||
| 388 | 428 | /** The server decides who may; a refusal is printed, success says the offset the server now holds. */ |
| 389 | 429 | private async table(what: "here" | number): Promise<void> { |
| 390 | 430 | if (!this.post) return this.print("error: the table is not available here"); |
Software/so101/src/tty.ts +2 −2
| @@ -4,7 +4,7 @@ | ||
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | import { createInterface, type Interface } from "node:readline"; |
| 7 | import { Door, HELP, liveUrl, poster, terminalHeaders } from "./terminal.ts"; | |
| 7 | import { Door, HELP, liveUrl, poster, reader, terminalHeaders } from "./terminal.ts"; | |
| 8 | 8 | |
| 9 | 9 | const env = process.env; |
| 10 | 10 | const where = env.FORGE_APP_ENV === "preview" ? `preview of #${env.FORGE_APP_PULL ?? "?"}` : (env.FORGE_APP_ENV ?? "local"); |
| @@ -17,7 +17,7 @@ const print = (line: string) => { | ||
| 17 | 17 | rl?.prompt(true); |
| 18 | 18 | }; |
| 19 | 19 | |
| 20 | const door = new Door(liveUrl(env), terminalHeaders(env), print, poster(env)); | |
| 20 | const door = new Door(liveUrl(env), terminalHeaders(env), print, poster(env), reader(env)); | |
| 21 | 21 | print(`\x1b[1mSO-101 arm\x1b[0m · ${where} · ${HELP}`); |
| 22 | 22 | try { |
| 23 | 23 | await door.open(() => { |
Software/so101/src/ui/frames.js +4 −1
| @@ -4,6 +4,9 @@ | ||
| 4 | 4 | * now (a finger let go). Each is stamped with `seq` and `t`, when its newest target was given (src/command-gate.ts). |
| 5 | 5 | * @typedef {Record<number, number>} Frame |
| 6 | 6 | */ |
| 7 | /** Tells the page something happened, for the health meter (src/ui/health-meter.js) to count. */ | |
| 8 | const tell = (/** @type {string} */ what) => /** @type {any} */ (globalThis).dispatchEvent?.(new Event(what)); | |
| 9 | ||
| 7 | 10 | export class Frames { |
| 8 | 11 | /** |
| 9 | 12 | * @param {(data: string) => void} write |
| @@ -28,7 +31,7 @@ export class Frames { | ||
| 28 | 31 | send(frame) { |
| 29 | 32 | this.pending = { ...(this.pending ?? {}), ...frame }; |
| 30 | 33 | this.at = this.now(); |
| 31 | if (this.timer !== null) return; | |
| 34 | if (this.timer !== null) return tell("cmd-drop"); // folded into the slot already waiting: the older target never goes out | |
| 32 | 35 | this.timer = this.later(() => { |
| 33 | 36 | this.timer = null; |
| 34 | 37 | const data = this.pending; this.pending = null; |
Software/so101/src/ui/frames.test.ts +19
| @@ -31,6 +31,25 @@ describe("Frames", () => { | ||
| 31 | 31 | expect(wire.length).toBe(1); |
| 32 | 32 | }); |
| 33 | 33 | |
| 34 | test("a send folded into a waiting slot tells the page a command target was dropped; the first of a slot does not", () => { | |
| 35 | const heard: string[] = []; | |
| 36 | const on = () => heard.push("cmd-drop"); | |
| 37 | globalThis.addEventListener("cmd-drop", on); | |
| 38 | try { | |
| 39 | const { frames, slot } = rig(); | |
| 40 | frames.send({ 1: 2000 }); | |
| 41 | expect(heard).toEqual([]); | |
| 42 | frames.send({ 1: 2010 }); | |
| 43 | frames.send({ 2: 1990 }); | |
| 44 | expect(heard).toEqual(["cmd-drop", "cmd-drop"]); | |
| 45 | slot(); | |
| 46 | frames.send({ 1: 2020 }); | |
| 47 | expect(heard.length).toBe(2); | |
| 48 | } finally { | |
| 49 | globalThis.removeEventListener("cmd-drop", on); | |
| 50 | } | |
| 51 | }); | |
| 52 | ||
| 34 | 53 | test("a flush goes out now, carrying what was pending under it, and the slot it cancels sends nothing", () => { |
| 35 | 54 | const { frames, wire, slot, waiting } = rig(); |
| 36 | 55 | frames.send({ 1: 2012, 3: 1988 }); |
Software/so101/src/ui/health-meter.js added +72
| @@ -0,0 +1,72 @@ | ||
| 1 | // @ts-check | |
| 2 | /** | |
| 3 | * What this page measures about its own link, for the report it sends over /live every REPORT_MS (src/viewer-reports.ts): | |
| 4 | * command -> pose round trip (a command sent, the next pose that arrives after it), commands sent and dropped, camera | |
| 5 | * frames received, the age of the newest frame, and camera stalls and reconnects. `report` returns the numbers for the | |
| 6 | * window since the last report and starts the next; stalls and reconnects are totals since the page opened. | |
| 7 | */ | |
| 8 | ||
| 9 | export const REPORT_MS = 5000; | |
| 10 | /** Round trips kept per window; a window of 5 s holds far fewer. */ | |
| 11 | const KEEP = 256; | |
| 12 | ||
| 13 | /** Nearest-rank percentile of an ascending array. @param {number[]} sorted @param {number} p */ | |
| 14 | const pct = (sorted, p) => (sorted.length ? sorted[Math.max(0, Math.ceil(p * sorted.length) - 1)] ?? 0 : 0); | |
| 15 | ||
| 16 | export class HealthMeter { | |
| 17 | /** @param {() => number} [now] */ | |
| 18 | constructor(now = () => performance.now()) { | |
| 19 | this.now = now; | |
| 20 | this.since = now(); | |
| 21 | /** @type {number[]} */ this.trips = []; | |
| 22 | /** @type {number | null} */ this.waiting = null; | |
| 23 | this.sent = 0; this.dropped = 0; this.frames = 0; | |
| 24 | /** @type {number | null} */ this.lastFrame = null; | |
| 25 | this.stalls = 0; this.reconnects = 0; | |
| 26 | } | |
| 27 | ||
| 28 | /** A command went out. The round trip is timed from the first one nothing has answered yet. */ | |
| 29 | command() { this.sent++; this.waiting ??= this.now(); } | |
| 30 | /** A command target was folded into a newer one before it went out (src/ui/frames.js). */ | |
| 31 | drop() { this.dropped++; } | |
| 32 | /** A pose arrived: it answers the command waiting, if any. */ | |
| 33 | pose() { | |
| 34 | if (this.waiting === null) return; | |
| 35 | if (this.trips.length < KEEP) this.trips.push(this.now() - this.waiting); | |
| 36 | this.waiting = null; | |
| 37 | } | |
drop() is never called, so droppedPerS is always 0. Wire it to the real drop point or remove the field end to end. |
||
| 38 | frame() { this.frames++; this.lastFrame = this.now(); } | |
| 39 | stall() { this.stalls++; } | |
| 40 | reconnect() { this.reconnects++; } | |
| 41 | ||
| 42 | /** The window's numbers as the /live frame, and a fresh window. */ | |
| 43 | report() { | |
| 44 | const t = this.now(); | |
| 45 | const secs = Math.max((t - this.since) / 1000, 0.001); | |
| 46 | const sorted = [...this.trips].sort((a, b) => a - b); | |
| 47 | const r = { | |
| 48 | type: "report", | |
| 49 | rttP50: Math.round(pct(sorted, 0.5)), rttP95: Math.round(pct(sorted, 0.95)), | |
| 50 | sentPerS: +(this.sent / secs).toFixed(1), droppedPerS: +(this.dropped / secs).toFixed(1), | |
| 51 | camFps: +(this.frames / secs).toFixed(1), | |
| 52 | frameAgeMs: this.lastFrame === null ? 0 : Math.round(t - this.lastFrame), | |
| 53 | stalls: this.stalls, reconnects: this.reconnects, | |
| 54 | }; | |
| 55 | this.since = t; this.trips = []; this.sent = this.dropped = this.frames = 0; | |
| 56 | return r; | |
| 57 | } | |
| 58 | } | |
| 59 | ||
| 60 | /** | |
| 61 | * A meter whose report goes out through `say` every REPORT_MS, counting each command `frames` writes. | |
| 62 | * @param {(m: Record<string, unknown>) => void} say | |
| 63 | * @param {{ write: (data: string) => void }} frames | |
| 64 | */ | |
| 65 | export function start(say, frames) { | |
| 66 | const meter = new HealthMeter(); | |
| 67 | const write = frames.write; | |
| 68 | frames.write = (data) => { write(data); meter.command(); }; | |
| 69 | setInterval(() => say(meter.report()), REPORT_MS); | |
| 70 | for (const [event, count] of /** @type {const} */ ([["live-reconnect", "reconnect"], ["cam-reconnect", "reconnect"], ["cam-frame", "frame"], ["cam-stall", "stall"], ["pose", "pose"], ["cmd-drop", "drop"]])) globalThis.addEventListener?.(event, () => meter[count]()); | |
| 71 | return meter; | |
| 72 | } |
Software/so101/src/ui/health-meter.test.ts added +32
| @@ -0,0 +1,32 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { Frames } from "./frames.js"; | |
| 3 | import { HealthMeter, start } from "./health-meter.js"; | |
| 4 | ||
| 5 | test("a window's round trips, rates, frame age and totals", () => { | |
| 6 | let t = 0; | |
| 7 | const m = new HealthMeter(() => t); | |
| 8 | t = 100; m.command(); m.command(); // the second does not restart the timing | |
| 9 | t = 160; m.pose(); | |
| 10 | t = 200; m.command(); | |
| 11 | t = 400; m.pose(); | |
| 12 | m.drop(); | |
| 13 | for (let i = 0; i < 20; i++) m.frame(); | |
| 14 | m.stall(); m.reconnect(); m.reconnect(); | |
| 15 | t = 5000; | |
| 16 | expect(m.report()).toEqual({ type: "report", rttP50: 60, rttP95: 200, sentPerS: 0.6, droppedPerS: 0.2, camFps: 4, frameAgeMs: 4600, stalls: 1, reconnects: 2 }); | |
| 17 | t = 10000; | |
| 18 | expect(m.report()).toMatchObject({ rttP50: 0, sentPerS: 0, camFps: 0, stalls: 1, reconnects: 2 }); | |
| 19 | }); | |
| 20 | ||
| 21 | test("start counts the commands frames writes and the targets it folds away, so droppedPerS is real", () => { | |
| 22 | const reports: Record<string, any>[] = []; | |
| 23 | const frames = new Frames(() => {}, { later: () => 1, cancel: () => {} }); | |
| 24 | const meter = start((m) => reports.push(m), frames); | |
| 25 | frames.send({ 1: 2000 }); | |
| 26 | frames.send({ 1: 2010 }); | |
| 27 | frames.send({ 1: 2020 }); | |
| 28 | frames.flush({ 1: 2030 }); | |
| 29 | expect(meter.dropped).toBe(2); | |
| 30 | expect(meter.sent).toBe(1); | |
| 31 | expect(meter.report().droppedPerS).toBeGreaterThan(0); | |
| 32 | }); |
Software/so101/src/ui/lobby-panel.js +5 −1
| @@ -25,6 +25,9 @@ export function queueEntry(el, x, mine) { | ||
| 25 | 25 | /** @param {number} attempt 1 for the first retry @returns {number} ms to wait: 250, 500, 1000, then 2000 at most */ |
| 26 | 26 | export const backoff = (attempt) => Math.min(250 * 2 ** Math.max(0, attempt - 1), 2000); |
| 27 | 27 | |
| 28 | /** Tell the page's health meter (src/ui/health-meter.js) what happened. @param {string} what */ | |
| 29 | const tell = (what) => /** @type {any} */ (globalThis).dispatchEvent?.(new Event(what)); | |
| 30 | ||
| 28 | 31 | /** |
| 29 | 32 | * @param {any} page what the page owns: $, el, relayed, tab(), showTab, msg, applyLock, applyStatus, noStatus, words, admin |
| 30 | 33 | */ |
| @@ -250,7 +253,7 @@ export function createRoom(page) { | ||
| 250 | 253 | if (lobby.retry) { lobby.retry = 0; $("reconnect").classList.remove("on"); } // the first message: it is back |
| 251 | 254 | const m = JSON.parse(e.data); |
| 252 | 255 | if (m.type === "ping") return say({ type: "pong" }); |
| 253 | if (m.type === "status") { lobby.statusSeen = true; return m.status ? page.applyStatus(m.status) : page.noStatus(); } | |
| 256 | if (m.type === "status") { tell("pose"); lobby.statusSeen = true; return m.status ? page.applyStatus(m.status) : page.noStatus(); } | |
| 254 | 257 | if (m.type === "hello") lobby.hello = m; |
| 255 | 258 | else if (m.type === "state") { lobby.offset = m.now - Date.now(); lobby.state = m; } |
| 256 | 259 | else if (m.type === "chat") return addChat(m.messages, true), showEarlier(m.more); |
| @@ -274,6 +277,7 @@ export function createRoom(page) { | ||
| 274 | 277 | return lobbyConnect(); |
| 275 | 278 | } |
| 276 | 279 | lobby.attempt++; |
| 280 | tell("live-reconnect"); | |
| 277 | 281 | $("reconnect").classList.add("on"); |
| 278 | 282 | page.noStatus(); // never the last "simulated" while waiting |
| 279 | 283 | page.applyLock(); // no hello, so no turn: the drive controls are inert and no motion is sent |
Software/so101/src/ui/video.js +10 −5
| @@ -19,7 +19,7 @@ export function socketUrl(loc, id) { | ||
| 19 | 19 | /** |
| 20 | 20 | * Keep one camera's socket open and hand every frame (a Blob of JPEG or PNG) to `onFrame`. |
| 21 | 21 | * @param {{ |
| 22 | * url: string, onFrame: (frame: Blob) => void, maxFps?: number, | |
| 22 | * url: string, onFrame: (frame: Blob) => void, maxFps?: number, onStall?: () => void, onReconnect?: () => void, | |
| 23 | 23 | * open?: (url: string) => WebSocket, now?: () => number, |
| 24 | 24 | * after?: (fn: () => void, ms: number) => any, cancel?: (timer: any) => void, |
| 25 | 25 | * }} options the last four are for the tests |
| @@ -44,6 +44,8 @@ export function openFeed(options) { | ||
| 44 | 44 | stop(); |
| 45 | 45 | if (!closed) watchdog = after(connect, ms); |
| 46 | 46 | }; |
| 47 | /** The watchdog fired: no frame for QUIET_MS. */ | |
| 48 | const stalled = () => { options.onStall?.(); again(0); }; | |
| 47 | 49 | function connect() { |
| 48 | 50 | watchdog = null; |
| 49 | 51 | if (closed) return; |
| @@ -53,14 +55,14 @@ export function openFeed(options) { | ||
| 53 | 55 | w.binaryType = /** @type {any} */ ("blob"); // Bun's types know only arraybuffer and nodebuffer; browsers default to blob anyway |
| 54 | 56 | w.onmessage = (e) => { |
| 55 | 57 | if (ws !== w || !(e.data instanceof Blob)) return; |
| 56 | cancel(watchdog); watchdog = after(() => again(0), QUIET_MS); | |
| 58 | cancel(watchdog); watchdog = after(stalled, QUIET_MS); | |
| 57 | 59 | const t = now(); |
| 58 | 60 | if (t - shown < gap) return; // held to fewer frames on a slow link |
| 59 | 61 | shown = t; |
| 60 | 62 | onFrame(e.data); |
| 61 | 63 | }; |
| 62 | w.onclose = w.onerror = () => { if (ws === w) again(RETRY_MS); }; | |
| 63 | watchdog = after(() => again(0), QUIET_MS); // the first frame is due too | |
| 64 | w.onclose = w.onerror = () => { if (ws === w) { options.onReconnect?.(); again(RETRY_MS); } }; | |
| 65 | watchdog = after(stalled, QUIET_MS); // the first frame is due too | |
| 64 | 66 | } |
| 65 | 67 | connect(); |
| 66 | 68 | return { close() { closed = true; stop(); } }; |
| @@ -83,6 +85,9 @@ export function drawFrame(cam, frame, env) { | ||
| 83 | 85 | /** Whether a socket-drawn camera is live: a frame lately, and the relay not saying it is offline. @param {{ lastFrame?: number, online?: boolean | null }} cam */ |
| 84 | 86 | export const isLive = (cam, /** @type {number} */ now) => now - (cam.lastFrame ?? -Infinity) < QUIET_MS + 500 && cam.online !== false; |
| 85 | 87 | |
| 88 | /** Tell the page's health meter (src/ui/health-meter.js) what the camera did. @param {string} what */ | |
| 89 | const tell = (what) => /** @type {any} */ (globalThis).dispatchEvent?.(new Event(what)); | |
| 90 | ||
| 86 | 91 | /** |
| 87 | 92 | * Start a camera's socket, unless the page moved on while this module was loading: `epoch` is the camera's count when |
| 88 | 93 | * the load began, and the page counts up whenever it shows or hides the camera again. |
| @@ -92,5 +97,5 @@ export const isLive = (cam, /** @type {number} */ now) => now - (cam.lastFrame ? | ||
| 92 | 97 | */ |
| 93 | 98 | export function attach(cam, epoch, env) { |
| 94 | 99 | if (cam.epoch !== epoch || !env.visible()) return; |
| 95 | cam.feed = openFeed({ url: socketUrl(env.loc, cam.id), maxFps: env.maxFps, onFrame: (frame) => drawFrame(cam, frame, env) }); | |
| 100 | cam.feed = openFeed({ url: socketUrl(env.loc, cam.id), maxFps: env.maxFps, onFrame: (frame) => { tell("cam-frame"); drawFrame(cam, frame, env); }, onStall: () => tell("cam-stall"), onReconnect: () => tell("cam-reconnect") }); | |
| 96 | 101 | } |
Software/so101/src/viewer-reports.test.ts added +137
| @@ -0,0 +1,137 @@ | ||
| 1 | import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import WebSocket from "ws"; | |
| 6 | import { FeetechBus } from "./bus.ts"; | |
| 7 | import { FakeBus } from "./fake-bus.ts"; | |
| 8 | import { sign } from "./identity.ts"; | |
| 9 | import { Lobby } from "./lobby.ts"; | |
| 10 | import { startRelay, type RunningRelay } from "./relay.ts"; | |
| 11 | import { startServer, type RunningServer } from "./server.ts"; | |
| 12 | import { MAX_VIEWERS, MIN_GAP_MS, parseReport, ViewerReports } from "./viewer-reports.ts"; | |
| 13 | import { renderAdmin } from "./admin-page.ts"; | |
| 14 | import { Admin } from "./admin.ts"; | |
| 15 | import { Turns } from "./turns.ts"; | |
| 16 | ||
| 17 | const GOOD = { rttP50: 40, rttP95: 90, sentPerS: 30, droppedPerS: 2, camFps: 14.5, frameAgeMs: 120, stalls: 1, reconnects: 2 }; | |
| 18 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 19 | ||
| 20 | describe("parseReport", () => { | |
| 21 | test("takes the eight numbers and nothing else", () => { | |
| 22 | expect(parseReport({ type: "report", ...GOOD, extra: "<b>" })).toEqual(GOOD); | |
| 23 | }); | |
| 24 | test("refuses a missing, mistyped, negative, infinite or out-of-range field", () => { | |
| 25 | const { rttP50: _, ...missing } = GOOD; | |
| 26 | expect(parseReport(missing)).toBeNull(); | |
| 27 | expect(parseReport({ ...GOOD, camFps: "30" })).toBeNull(); | |
| 28 | expect(parseReport({ ...GOOD, camFps: -1 })).toBeNull(); | |
| 29 | expect(parseReport({ ...GOOD, rttP95: Infinity })).toBeNull(); | |
| 30 | expect(parseReport({ ...GOOD, rttP95: NaN })).toBeNull(); | |
| 31 | expect(parseReport({ ...GOOD, stalls: 1e12 })).toBeNull(); | |
| 32 | }); | |
| 33 | }); | |
| 34 | ||
| 35 | describe("ViewerReports", () => { | |
| 36 | test("reports the injected values per viewer, in rows and in Prometheus text", () => { | |
| 37 | let t = 1000; | |
| 38 | const r = new ViewerReports(() => t); | |
| 39 | const a = {}, b = {}; | |
| 40 | expect(r.set(a, "alice", GOOD)).toBe(true); | |
| 41 | t += 500; | |
| 42 | expect(r.set(b, "bob", { ...GOOD, camFps: 5 })).toBe(true); | |
| 43 | expect(r.rows()).toEqual([ | |
| 44 | { viewer: "v1", name: "alice", ageMs: 500, ...GOOD }, | |
| 45 | { viewer: "v2", name: "bob", ageMs: 0, ...GOOD, camFps: 5 }, | |
| 46 | ]); | |
| 47 | const text = r.prometheus(); | |
| 48 | expect(text).toContain('viewer_rtt_p95{viewer="v1"} 90'); | |
| 49 | expect(text).toContain('viewer_cam_fps{viewer="v2"} 5'); | |
| 50 | expect(text).toContain('viewer_frame_age_ms{viewer="v1"} 120'); | |
| 51 | expect(text).not.toContain("alice"); | |
| 52 | r.drop(a); | |
| 53 | expect(r.rows().map((x) => x.viewer)).toEqual(["v2"]); | |
| 54 | expect(new ViewerReports().prometheus()).toBe(""); | |
| 55 | }); | |
| 56 | test("refuses a socket that reports faster than the gap, and keeps the first", () => { | |
| 57 | let t = 0; | |
| 58 | const r = new ViewerReports(() => t); | |
| 59 | const k = {}; | |
| 60 | expect(r.set(k, "a", GOOD)).toBe(true); | |
| 61 | t += MIN_GAP_MS - 1; | |
| 62 | expect(r.set(k, "a", { ...GOOD, camFps: 1 })).toBe(false); | |
| 63 | expect(r.rows()[0]!.camFps).toBe(14.5); | |
| 64 | t += 1; | |
| 65 | expect(r.set(k, "a", { ...GOOD, camFps: 1 })).toBe(true); | |
| 66 | expect(r.rows()[0]!.viewer).toBe("v1"); | |
| 67 | }); | |
| 68 | test("keeps at most MAX_VIEWERS", () => { | |
| 69 | const r = new ViewerReports(); | |
| 70 | for (let i = 0; i < MAX_VIEWERS; i++) expect(r.set({}, "x", GOOD)).toBe(true); | |
| 71 | expect(r.set({}, "x", GOOD)).toBe(false); | |
| 72 | }); | |
| 73 | }); | |
| 74 | ||
| 75 | describe("the Health section", () => { | |
| 76 | test("escapes names and shows the numbers", () => { | |
| 77 | const rows = new ViewerReports(); | |
| 78 | rows.set({}, '<img src=x onerror=alert(1)>', GOOD); | |
| 79 | const html = renderAdmin({ | |
| 80 | who: { id: "o", name: "owner", owner: true }, canEdit: true, settings: new Admin().settings, | |
| 81 | snapshot: new Turns().snapshot(), log: [], health: rows.rows(), | |
| 82 | }); | |
| 83 | expect(html).toContain("Health"); | |
| 84 | expect(html).toContain("<img src=x onerror=alert(1)>"); | |
| 85 | expect(html).not.toContain("<img src=x"); | |
| 86 | expect(html).toContain("<td>14.5</td>"); | |
| 87 | }); | |
| 88 | }); | |
| 89 | ||
| 90 | const KEY = "test-identity-key"; | |
| 91 | const as = (id: string, name: string, role = "user") => { | |
| 92 | const identity = `${id}|${name}|${role}|${Date.now() + 3_600_000}`; | |
| 93 | return { "x-forge-identity": identity, "x-forge-signature": sign(identity, KEY) }; | |
| 94 | }; | |
| 95 | ||
| 96 | describe("over /live", () => { | |
| 97 | let simulation: RunningServer; | |
| 98 | let relay: RunningRelay; | |
| 99 | let lobby: Lobby; | |
| 100 | const sockets: WebSocket[] = []; | |
| 101 | beforeAll(async () => { | |
| 102 | simulation = await startServer(new FeetechBus(new FakeBus([1, 2, 6])), { host: "127.0.0.1", port: 0, ids: [1, 2, 6], posesFile: join(mkdtempSync(join(tmpdir(), "so101-vr-")), "poses.json"), watchdogMs: 60 }); | |
| 103 | lobby = new Lobby({ identityKey: KEY }); | |
| 104 | relay = await startRelay({ port: 0, simulation: `http://127.0.0.1:${simulation.port}`, lobby }); | |
| 105 | }); | |
| 106 | afterEach(() => { for (const s of sockets.splice(0)) s.terminate(); }); | |
| 107 | afterAll(async () => { await Promise.race([relay.close(), sleep(500)]); lobby.close(); await Promise.race([simulation.close(), sleep(500)]); }); | |
| 108 | const open = async (headers = {}) => { | |
| 109 | const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/live`, { headers }); | |
| 110 | sockets.push(ws); | |
| 111 | const heard: any[] = []; | |
| 112 | ws.on("message", (d) => heard.push(JSON.parse(String(d)))); | |
| 113 | await new Promise((r, j) => (ws.once("open", r), ws.once("error", j))); | |
| 114 | return { ws, heard }; | |
| 115 | }; | |
| 116 | ||
| 117 | test("a report shows under /admin/health and /metrics.prom; forged and oversized ones do not", async () => { | |
| 118 | const owner = as("u-owner", "trevato", "admin"); | |
| 119 | const v = await open(as("u-alice", "alice")); | |
| 120 | v.ws.send(JSON.stringify({ type: "report", ...GOOD, camFps: -5 })); | |
| 121 | v.ws.send("{" + "x".repeat(5000)); | |
| 122 | await sleep(100); | |
| 123 | expect(v.heard.some((m) => m.type === "error")).toBe(true); | |
| 124 | expect(lobby.reports.rows()).toEqual([]); | |
| 125 | const w = await open(as("u-bob", "bob")); | |
| 126 | w.ws.send(JSON.stringify({ type: "report", ...GOOD, reconnects: 7 })); | |
| 127 | await sleep(100); | |
| 128 | const health = await fetch(`http://127.0.0.1:${relay.port}/admin/health`, { headers: owner }); | |
| 129 | expect(((await health.json()) as any[]).map((r) => [r.name, r.reconnects])).toEqual([["bob", 7]]); | |
| 130 | const prom = await (await fetch(`http://127.0.0.1:${relay.port}/metrics.prom`)).text(); | |
| 131 | expect(prom).toContain('viewer_reconnects{viewer="v1"} 7'); | |
| 132 | expect((await fetch(`http://127.0.0.1:${relay.port}/admin/health`)).status).toBe(404); | |
| 133 | w.ws.close(); | |
| 134 | await sleep(100); | |
| 135 | expect(lobby.reports.rows()).toEqual([]); | |
| 136 | }); | |
| 137 | }); |
Software/so101/src/viewer-reports.ts added +99
| @@ -0,0 +1,99 @@ | ||
| 1 | /** | |
| 2 | * What each viewer's page measures and sends over /live every few seconds: | |
| 3 | * {"type": "report", "rttP50": ms, "rttP95": ms, "sentPerS": n, "droppedPerS": n, "camFps": n, "frameAgeMs": ms, "stalls": n, "reconnects": n} | |
| 4 | * `parseReport` takes only those eight keys and only finite numbers in range (null for anything else, so a forged report | |
| 5 | * is refused, never clamped); the rest of the frame is ignored. `ViewerReports` keeps the latest report of each open | |
| 6 | * socket, drops it when the socket closes, refuses a socket that reports faster than MIN_GAP_MS, and renders the lot as | |
| 7 | * rows for the admin page and as Prometheus lines. A viewer is named by a serial number for the process, never by user | |
| 8 | * text, so the series stay few and the output needs no escaping. Pure state: the clock is passed in. | |
| 9 | */ | |
| 10 | ||
| 11 | export const MIN_GAP_MS = 1000; | |
| 12 | /** Viewers kept at once; a socket past this is not recorded. */ | |
| 13 | export const MAX_VIEWERS = 200; | |
| 14 | ||
| 15 | const FIELDS = { | |
| 16 | rttP50: { max: 600_000, help: "command to pose round trip p50, ms" }, | |
| 17 | rttP95: { max: 600_000, help: "command to pose round trip p95, ms" }, | |
| 18 | sentPerS: { max: 10_000, help: "commands sent per second" }, | |
| 19 | droppedPerS: { max: 10_000, help: "commands dropped per second" }, | |
| 20 | camFps: { max: 1_000, help: "camera frames received per second" }, | |
| 21 | frameAgeMs: { max: 3_600_000, help: "age of the newest camera frame, ms" }, | |
| 22 | stalls: { max: 1_000_000, help: "camera stalls since the page opened" }, | |
| 23 | reconnects: { max: 1_000_000, help: "reconnects since the page opened" }, | |
| 24 | } as const; | |
| 25 | ||
| 26 | export type ReportField = keyof typeof FIELDS; | |
| 27 | export type Report = Record<ReportField, number>; | |
| 28 | export const REPORT_FIELDS = Object.keys(FIELDS) as ReportField[]; | |
| 29 | ||
| 30 | /** The report in a frame, or null when it is not exactly numbers within range for every field. */ | |
| 31 | export function parseReport(m: Record<string, unknown>): Report | null { | |
| 32 | const out = {} as Report; | |
| 33 | for (const k of REPORT_FIELDS) { | |
| 34 | const v = m[k]; | |
| 35 | if (typeof v !== "number" || !Number.isFinite(v) || v < 0 || v > FIELDS[k].max) return null; | |
| 36 | out[k] = Math.round(v * 100) / 100; | |
| 37 | } | |
| 38 | return out; | |
| 39 | } | |
| 40 | ||
| 41 | export interface ViewerRow extends Report { | |
| 42 | /** Serial number of the socket, `v1`, `v2`… */ | |
| 43 | viewer: string; | |
| 44 | name: string; | |
| 45 | /** Milliseconds since the report arrived. */ | |
| 46 | ageMs: number; | |
| 47 | } | |
| 48 | ||
| 49 | interface Entry { | |
| 50 | viewer: string; | |
| 51 | name: string; | |
| 52 | at: number; | |
| 53 | report: Report; | |
| 54 | } | |
| 55 | ||
| 56 | export class ViewerReports { | |
| 57 | private readonly entries = new Map<object, Entry>(); | |
| 58 | private readonly last = new Map<object, number>(); | |
| 59 | private serial = 0; | |
| 60 | private readonly now: () => number; | |
| 61 | ||
| 62 | constructor(now: () => number = Date.now) { | |
| 63 | this.now = now; | |
| 64 | } | |
| 65 | ||
| 66 | /** Record `report` for the socket `key`; false when it came too soon after the last or there is no room. */ | |
| 67 | set(key: object, name: string, report: Report): boolean { | |
| 68 | const t = this.now(); | |
| 69 | const before = this.last.get(key); | |
| 70 | if (before !== undefined && t - before < MIN_GAP_MS) return false; | |
| 71 | const known = this.entries.get(key); | |
| 72 | if (!known && this.entries.size >= MAX_VIEWERS) return false; | |
| 73 | this.last.set(key, t); | |
| 74 | this.entries.set(key, { viewer: known?.viewer ?? `v${++this.serial}`, name, at: t, report }); | |
| 75 | return true; | |
| 76 | } | |
| 77 | ||
| 78 | drop(key: object): void { | |
| 79 | this.entries.delete(key); | |
| 80 | this.last.delete(key); | |
| 81 | } | |
| 82 | ||
| 83 | rows(): ViewerRow[] { | |
| 84 | const t = this.now(); | |
| 85 | return [...this.entries.values()].map((e) => ({ viewer: e.viewer, name: e.name, ageMs: t - e.at, ...e.report })); | |
| 86 | } | |
| 87 | ||
| 88 | prometheus(): string { | |
| 89 | const rows = this.rows(); | |
| 90 | const out: string[] = []; | |
| 91 | for (const k of REPORT_FIELDS) { | |
| 92 | if (!rows.length) break; | |
| 93 | const name = `viewer_${k.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)}`; | |
| 94 | out.push(`# HELP ${name} ${FIELDS[k].help}`, `# TYPE ${name} gauge`); | |
| 95 | for (const r of rows) out.push(`${name}{viewer="${r.viewer}"} ${r[k]}`); | |
| 96 | } | |
| 97 | return out.length ? out.join("\n") + "\n" : ""; | |
| 98 | } | |
| 99 | } |
Software/so101/src/viewer.html +1
| @@ -1517,6 +1517,7 @@ | ||
| 1517 | 1517 | admin: (saved) => fillAdmin(saved), |
| 1518 | 1518 | }); |
| 1519 | 1519 | const { lobby, mayDrive, isOwner, say, serverNow, lobbyConnect, chatBox, renderTurn, button } = room; |
| 1520 | if (relayed) import("/ui/__VERSION__/health-meter.js").then((m) => m.start(say, stream.frames)); | |
| 1520 | 1521 | let drove = false; |
| 1521 | 1522 | function explainLock() { |
| 1522 | 1523 | const me = lobby.hello?.you, down = linkNow === "offline", st = typeof status?.source === "string" && status.source.startsWith("simulated"); |
new above
offline
The script is in the shared page() wrapper, so it runs on pages with no #health-rows and keeps polling. Emit it only with the Health section and skip the tick when document.hidden.