Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
Per-hop latency on aligned clocks: browser→relay, relay→Pi, read-back→browser, in /precision #470
smith/arm:smith/458-per-hop-latency-on-aligned-clocks-browse → main 1 commit fork
11 files
changed +421
−16
Software/so101/bench/precision.ts +17 −1
| @@ -15,6 +15,7 @@ import { FakeBus } from "../src/fake-bus.ts"; | ||
| 15 | 15 | import type { Card } from "../src/precision.ts"; |
| 16 | 16 | import type { Scorecard } from "../src/precision-history.ts"; |
| 17 | 17 | import { startServer } from "../src/server.ts"; |
| 18 | import { HopLink } from "../src/ui/hop-link.js"; | |
| 18 | 19 | |
| 19 | 20 | /** One frame of the script: when (ms from the start) and the joints it names. */ |
| 20 | 21 | export interface Step { |
| @@ -58,12 +59,27 @@ export async function bench(script: readonly Step[] = gesture()): Promise<Card> | ||
| 58 | 59 | try { |
| 59 | 60 | const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`); |
| 60 | 61 | await new Promise((r, j) => (ws.once("open", r), ws.once("error", j))); |
| 62 | // The bench is the relay and the page both, on one machine: its clocks are one clock, aligned by the same ping the relay | |
| 63 | // uses, so the three hops are loopback (src/hops.ts). | |
| 64 | let seq = 0; | |
| 65 | const link = new HopLink((text) => ws.send(text)); | |
| 66 | ws.on("message", (data) => void link.message(String(data))); | |
| 67 | for (let n = 0; n < 4; n++) { | |
| 68 | link.ping(); | |
| 69 | await new Promise((r) => setTimeout(r, 20)); | |
| 70 | } | |
| 71 | const readbacks = setInterval(async () => { | |
| 72 | const { readAt } = (await (await fetch(`http://127.0.0.1:${server.port}/status`)).json()) as { readAt: number }; | |
| 73 | if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ readback: Math.max(0, Date.now() - readAt) })); | |
| 74 | }, 100); | |
| 61 | 75 | const start = performance.now(); |
| 62 | 76 | for (const step of script) { |
| 63 | 77 | const wait = step.at - (performance.now() - start); |
| 64 | 78 | if (wait > 0) await new Promise((r) => setTimeout(r, wait)); |
| 65 | ws.send(JSON.stringify(step.joints)); | |
| 79 | const t = Date.now(); | |
| 80 | ws.send(JSON.stringify({ ...step.joints, seq: ++seq, t, up: Date.now() - t, rt: Date.now() + (link.offset ?? 0) })); | |
| 66 | 81 | } |
| 82 | clearInterval(readbacks); | |
| 67 | 83 | await new Promise((r) => setTimeout(r, 300)); |
| 68 | 84 | ws.close(1000); |
| 69 | 85 | await new Promise((r) => ws.once("close", r)); |
Software/so101/src/hops.test.ts added +101
| @@ -0,0 +1,101 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { control, frameHops, hopMs, pong } from "./hops.ts"; | |
| 3 | import { emptyRecords, summarize, Tracker } from "./precision.ts"; | |
| 4 | import { ClockSync, exchange } from "./ui/clock-sync.js"; | |
| 5 | import { HopLink } from "./ui/hop-link.js"; | |
| 6 | ||
| 7 | describe("clock alignment", () => { | |
| 8 | test("one exchange: the offset is the answerer's clock minus the asker's, the delay the round trip less the answerer's wait", () => { | |
| 9 | // answerer 500 ahead; 20 ms out, 30 back, 5 ms spent answering | |
| 10 | expect(exchange(1000, 1520, 1525, 1055)).toEqual({ offset: 500 + (20 - 30) / 2, delay: 50 }); | |
| 11 | // symmetric legs: exact | |
| 12 | expect(exchange(0, 510, 510, 20)).toEqual({ offset: 500, delay: 20 }); | |
| 13 | }); | |
| 14 | ||
| 15 | test("the offset comes from the lowest-delay of the last few exchanges", () => { | |
| 16 | const sync = new ClockSync(); | |
| 17 | expect(sync.offset).toBeNull(); | |
| 18 | sync.add(0, 700, 700, 400); // a queued answer: delay 400, offset off by 200 | |
| 19 | sync.add(1000, 1510, 1510, 1020); // delay 20, offset 500 | |
| 20 | sync.add(2000, 2600, 2600, 2300); // delay 300 | |
| 21 | expect(sync.offset).toBe(500); | |
| 22 | }); | |
| 23 | ||
| 24 | test("an old exchange is forgotten, a malformed or time-reversed one ignored", () => { | |
| 25 | const sync = new ClockSync(); | |
| 26 | sync.add(0, 510, 510, 20); | |
| 27 | for (let i = 1; i <= 8; i++) sync.add(i * 100, i * 100 + 800, i * 100 + 800, i * 100 + 100); // delay 100, offset 750 | |
| 28 | expect(sync.offset).toBe(750); | |
| 29 | expect(sync.add(0, Number.NaN, 0, 0)).toBe(false); | |
| 30 | expect(sync.add(100, 0, 0, 50)).toBe(false); | |
| 31 | }); | |
| 32 | }); | |
| 33 | ||
| 34 | describe("hop wire", () => { | |
| 35 | test("control messages are told from frames", () => { | |
| 36 | expect(control('{"ping":3}')).toEqual({ kind: "ping", id: 3 }); | |
| 37 | expect(control('{"offset":-12.5}')).toEqual({ kind: "offset", ms: -12.5 }); | |
| 38 | expect(control('{"readback":42}')).toEqual({ kind: "readback", ms: 42 }); | |
| 39 | expect(control(pong(3, 10, 11))).toEqual({ kind: "pong" }); | |
| 40 | expect(control('{"1":2048,"seq":1,"t":5}')).toBeNull(); | |
| 41 | expect(control('{"readback":"fast"}')).toBeNull(); | |
| 42 | expect(control('{"readback":1e9}')).toBeNull(); | |
| 43 | expect(control('{"ping":')).toBeNull(); | |
| 44 | }); | |
| 45 | ||
| 46 | test("a hop is a plausible time, a small negative counting as zero", () => { | |
| 47 | expect(hopMs(12.4)).toBe(12); | |
| 48 | expect(hopMs(-3)).toBe(0); | |
| 49 | expect(hopMs(Number.NaN)).toBeNull(); | |
| 50 | expect(hopMs(10 ** 7)).toBeNull(); | |
| 51 | expect(hopMs("5")).toBeNull(); | |
| 52 | }); | |
| 53 | ||
| 54 | test("a frame's hops: browser→relay as the relay measured it, relay→Pi from its stamp against the receive time", () => { | |
| 55 | expect(frameHops('{"1":2048,"up":14,"rt":1000}', 1007)).toEqual({ up: 14, relayToPi: 7 }); | |
| 56 | expect(frameHops('{"1":2048,"seq":1,"t":5}', 1007)).toEqual({ up: null, relayToPi: null }); // from the CLI leader: nothing to measure | |
| 57 | expect(frameHops('{"1":2048,"rt":"x","up":-1}', 1007)).toEqual({ up: 0, relayToPi: null }); | |
| 58 | }); | |
| 59 | }); | |
| 60 | ||
| 61 | describe("the page's link", () => { | |
| 62 | test("pings, aligns its clock with the relay's and tells the relay; a status's age is on the relay's clock", () => { | |
| 63 | let now = 10_000; | |
| 64 | const sent: string[] = []; | |
| 65 | const link = new HopLink((t) => sent.push(t), { now: () => now, later: () => 1, cancel: () => {} }); | |
| 66 | link.start(); | |
| 67 | expect(JSON.parse(sent[0]!)).toEqual({ ping: 1, t0: 10_000 }); | |
| 68 | now = 10_020; // relay 500 ahead, 10 ms each way | |
| 69 | expect(link.message(pong(1, 10_510, 10_510))).toBe(true); | |
| 70 | expect(link.offset).toBe(500); | |
| 71 | expect(JSON.parse(sent[1]!)).toEqual({ offset: 500 }); | |
| 72 | expect(link.message('{"type":"status"}')).toBe(false); | |
| 73 | // read by the arm at relay time 10_530 and here at page 10_050 (= relay 10_550): 20 ms | |
| 74 | now = 10_050; | |
| 75 | link.readback(10_530); | |
| 76 | expect(JSON.parse(sent.at(-1)!)).toEqual({ readback: 20 }); | |
| 77 | const before = sent.length; | |
| 78 | link.readback(undefined); | |
| 79 | expect(sent.length).toBe(before); | |
| 80 | }); | |
| 81 | }); | |
| 82 | ||
| 83 | describe("hops in the scorecard", () => { | |
| 84 | test("each hop has its p50 and p95, and the headline latency sums them with the bus hop", () => { | |
| 85 | const tracker = new Tracker(0); | |
| 86 | for (const ms of [4, 5, 6, 7, 40]) tracker.hop("browser→relay", 100, ms); | |
| 87 | for (const ms of [2, 3]) tracker.hop("relay→Pi", 100, ms); | |
| 88 | tracker.hop("read-back→browser", 100, 30); | |
| 89 | tracker.hop("read-back→browser", 10, 90); // before the span asked for | |
| 90 | const card = summarize(tracker.records, 50); | |
| 91 | expect(card.hops["browser→relay"]).toMatchObject({ n: 5, p50: 6, p95: 40, max: 40 }); | |
| 92 | expect(card.hops["relay→Pi"]).toMatchObject({ n: 2, p50: 2, p95: 3 }); | |
| 93 | expect(card.hops["read-back→browser"]).toMatchObject({ n: 1, p50: 30, p95: 30 }); | |
| 94 | expect(card.hops["bus→move"]!.p95).toBeNull(); | |
| 95 | expect(card.latencyP95Ms).toBe(40 + 3 + 30); | |
| 96 | }); | |
| 97 | ||
| 98 | test("with no hop measured there is no latency", () => { | |
| 99 | expect(summarize(emptyRecords()).latencyP95Ms).toBeNull(); | |
| 100 | }); | |
| 101 | }); |
Software/so101/src/hops.ts added +55
| @@ -0,0 +1,55 @@ | ||
| 1 | /** | |
| 2 | * The latency hops a driver's command and the arm's answer make, each timed on clocks aligned with an NTP-style ping | |
| 3 | * (src/ui/clock-sync.js) and recorded on the arm server, whose scorecard they are in (src/precision.ts): | |
| 4 | * | |
| 5 | * browser→relay the frame's `t` against the relay's receive time; the page tells the relay its offset | |
| 6 | * relay→Pi the relay's send time against the arm server's receive time; the relay stamps `rt` (arm clock) | |
| 7 | * read-back→browser the arm server's read time of a status against the page's receive time; the page reports it | |
| 8 | * | |
| 9 | * Messages on a /follow stream that are not frames (a frame is only servo ids, seq, t, epoch, and the relay's up, rt): | |
| 10 | * {"ping": n} answered {"pong": n, "t1": receive, "t2": send}, by the relay to the page and the arm to the relay | |
| 11 | * {"offset": ms} the page's clock aligned: relay minus page | |
| 12 | * {"readback": ms} how old a status was when the page got it (passed on to the arm by the relay) | |
| 13 | * The arm server's command gate still accepts a frame with none of this, as from the CLI leader. | |
| 14 | */ | |
| 15 | ||
| 16 | export const HOPS = ["browser→relay", "relay→Pi", "read-back→browser"] as const; | |
| 17 | /** The most a hop can honestly be, ms: anything beyond is a broken clock, not a slow network. */ | |
| 18 | const MAX_HOP_MS = 60_000; | |
| 19 | ||
| 20 | export type Control = { kind: "ping"; id: number } | { kind: "offset"; ms: number } | { kind: "readback"; ms: number } | { kind: "pong" }; | |
| 21 | ||
| 22 | /** A hop's time, ms, when it is a plausible one: a small negative (alignment is only so good) counts as 0. */ | |
| 23 | export function hopMs(value: unknown): number | null { | |
| 24 | if (typeof value !== "number" || !Number.isFinite(value) || value > MAX_HOP_MS || value < -MAX_HOP_MS) return null; | |
| 25 | return Math.max(0, Math.round(value)); | |
| 26 | } | |
| 27 | ||
| 28 | /** What a stream's text is when it is one of the control messages above, else null (a frame, or nothing). */ | |
| 29 | export function control(text: string): Control | null { | |
| 30 | if (!/^\{"(ping|pong|offset|readback)"/.test(text)) return null; | |
| 31 | let m: { ping?: unknown; pong?: unknown; offset?: unknown; readback?: unknown }; | |
| 32 | try { | |
| 33 | m = JSON.parse(text); | |
| 34 | } catch { | |
| 35 | return null; | |
| 36 | } | |
| 37 | if (Number.isSafeInteger(m.ping)) return { kind: "ping", id: m.ping as number }; | |
| 38 | if ("pong" in m) return { kind: "pong" }; | |
| 39 | if (typeof m.offset === "number" && Number.isFinite(m.offset)) return { kind: "offset", ms: m.offset }; | |
| 40 | const ms = hopMs(m.readback); | |
| 41 | return ms === null ? null : { kind: "readback", ms }; | |
| 42 | } | |
| 43 | ||
| 44 | export const pong = (id: number, received: number, now = Date.now()) => JSON.stringify({ pong: id, t1: received, t2: now }); | |
| 45 | ||
| 46 | /** The hops a frame carries to the arm server: how long it took the page to the relay (`up`), and the relay's send stamp (`rt`). */ | |
| 47 | export function frameHops(text: string, received: number): { up: number | null; relayToPi: number | null } { | |
| 48 | if (!text.includes('"rt"') && !text.includes('"up"')) return { up: null, relayToPi: null }; | |
| 49 | try { | |
| 50 | const { up, rt } = JSON.parse(text) as { up?: unknown; rt?: unknown }; | |
| 51 | return { up: hopMs(up), relayToPi: typeof rt === "number" ? hopMs(received - rt) : null }; | |
| 52 | } catch { | |
| 53 | return { up: null, relayToPi: null }; | |
| 54 | } | |
| 55 | } |
Software/so101/src/precision-panel.ts +1 −1
| @@ -47,7 +47,7 @@ export function readPrecision(body: unknown): PrecisionPanel | null { | ||
| 47 | 47 | { label: "Tracking error, p95", unit: "°", value: pick(window.trackingP95Deg, last.trackingP95Deg), trend: rows.map((r) => num(r.trackingP95Deg)) }, |
| 48 | 48 | { label: "Time to arrive, p95", unit: "ms", value: pick(window.arriveP95Ms, last.arriveP95Ms), trend: rows.map((r) => num(r.arriveP95Ms)) }, |
| 49 | 49 | { label: "Snap-backs per 100 releases", unit: "", value: pick(window.snapBacksPer100Releases, rate(last)), trend: rows.map(rate) }, |
| 50 | { label: "Latency, p95, bus→motion", unit: "ms", value: pick(window.latencyP95Ms, last.latencyP95Ms), trend: rows.map((r) => num(r.latencyP95Ms)) }, | |
| 50 | { label: "Latency, p95, every hop summed", unit: "ms", value: pick(window.latencyP95Ms, last.latencyP95Ms), trend: rows.map((r) => num(r.latencyP95Ms)) }, | |
| 51 | 51 | ], |
| 52 | 52 | }; |
| 53 | 53 | } |
Software/so101/src/precision.ts +18 −4
| @@ -7,12 +7,14 @@ | ||
| 7 | 7 | * ARRIVE_TICKS; then, for SETTLE_MS: the overshoot past the target, the settling time (until the |
| 8 | 8 | * last entry into the band) and the oscillations (each exit and re-entry is one) |
| 9 | 9 | * snap-backs each time the arm is kept from the driver's target without the driver doing it, with its cause |
| 10 | * latency, by hop for now the arm server's own: a Goal_Position write to the first read-back past MOVE_TICKS | |
| 10 | * latency, by hop browser→relay, relay→Pi and read-back→browser on aligned clocks (src/hops.ts), and the arm | |
| 11 | * server's own: a Goal_Position write to the first read-back past MOVE_TICKS | |
| 11 | 12 | * |
| 12 | 13 | * One `Tracker` per drive session; `summarize` folds any span of its records into the same numbers, which is how the |
| 13 | 14 | * rolling window and the stored session rows are made. |
| 14 | 15 | */ |
| 15 | 16 | |
| 17 | import { HOPS } from "./hops.ts"; | |
| 16 | 18 | import { JOINTS } from "./sts3215.ts"; |
| 17 | 19 | |
| 18 | 20 | /** Ticks from the target that count as arrived. */ |
| @@ -39,9 +41,11 @@ export interface Records { | ||
| 39 | 41 | snaps: Array<{ at: number; cause: Cause; id: number | null; sizeDeg: number | null }>; |
| 40 | 42 | releases: number[]; |
| 41 | 43 | oscillations: Array<readonly [number, number]>; |
| 44 | /** Latency samples by hop name (HOPS): [at, ms]. */ | |
| 45 | hops: Record<string, Array<readonly [number, number]>>; | |
| 42 | 46 | } |
| 43 | 47 | |
| 44 | export const emptyRecords = (): Records => ({ joints: new Map(), snaps: [], releases: [], oscillations: [] }); | |
| 48 | export const emptyRecords = (): Records => ({ joints: new Map(), snaps: [], releases: [], oscillations: [], hops: {} }); | |
| 45 | 49 | |
| 46 | 50 | /** Nearest-rank percentile of unsorted values; null for none. */ |
| 47 | 51 | export function percentile(values: readonly number[], p: number): number | null { |
| @@ -82,7 +86,7 @@ export interface Card { | ||
| 82 | 86 | trackingP95Deg: number | null; |
| 83 | 87 | arriveP95Ms: number | null; |
| 84 | 88 | snapBacksPer100Releases: number | null; |
| 85 | /** End to end: the sum of every measured hop's p95. Only the arm server's own hop is measured so far. */ | |
| 89 | /** End to end: the sum of every measured hop's p95. */ | |
| 86 | 90 | latencyP95Ms: number | null; |
| 87 | 91 | /** The joint with the highest p95 tracking error. */ |
| 88 | 92 | worstJoint: string | null; |
| @@ -123,7 +127,7 @@ export function summarize(records: Records, since = -Infinity): Card { | ||
| 123 | 127 | for (const s of snaps) byCause[s.cause]++; |
| 124 | 128 | const releases = records.releases.filter((at) => at >= since).length; |
| 125 | 129 | const worst = joints.filter((j) => j.errorDeg.p95 !== null).sort((a, b) => b.errorDeg.p95! - a.errorDeg.p95!)[0]; |
| 126 | const hops = { "bus→move": spread(allBus, 0) }; | |
| 130 | const hops: Record<string, Spread> = { ...Object.fromEntries(HOPS.map((hop) => [hop, spread(after(records.hops[hop] ?? []), 0)])), "bus→move": spread(allBus, 0) }; | |
| 127 | 131 | const measured = Object.values(hops).filter((h) => h.p95 !== null); |
| 128 | 132 | return { |
| 129 | 133 | trackingP95Deg: round(percentile(allErrors.map((t) => t / TICKS_PER_DEGREE), 95)), |
| @@ -149,6 +153,7 @@ export function trim(records: Records, since: number): void { | ||
| 149 | 153 | keep(records.snaps, (s) => s.at); |
| 150 | 154 | keep(records.releases, (at) => at); |
| 151 | 155 | keep(records.oscillations, ([at]) => at); |
| 156 | for (const list of Object.values(records.hops)) keep(list, ([at]) => at); | |
| 152 | 157 | } |
| 153 | 158 | |
| 154 | 159 | interface Joint { |
| @@ -271,6 +276,15 @@ export class Tracker { | ||
| 271 | 276 | for (const records of [this.records, ...this.]) records.snaps.push(entry); |
| 272 | 277 | } |
| 273 | 278 | |
| 279 | /** A hop of a command's or a status's way took `ms`, on aligned clocks (src/hops.ts). */ | |
| 280 | hop(name: (typeof HOPS)[number], at: number, ms: number): void { | |
| 281 | for (const records of [this.records, ...this.]) { | |
| 282 | const list = (records.hops[name] ??= []); | |
| 283 | list.push([at, ms]); | |
| 284 | if (list.length > KEEP) list.shift(); | |
| 285 | } | |
| 286 | } | |
| 287 | ||
| 274 | 288 | /** The driver let go: a dead-man hold, or the end of the stream. */ |
| 275 | 289 | release(at: number): void { |
| 276 | 290 | for (const records of [this.records, ...this.]) records.releases.push(at); |
Software/so101/src/relay.test.ts +43 −1
| @@ -13,6 +13,7 @@ import { FakeBus } from "./fake-bus.ts"; | ||
| 13 | 13 | import { OFFLINE_FRAME } from "./offline-frame.ts"; |
| 14 | 14 | import { parseRemote, sameOrigin, startRelay, type Remote, type RunningRelay } from "./relay.ts"; |
| 15 | 15 | import { startServer, type RunningServer } from "./server.ts"; |
| 16 | import { control, pong } from "./hops.ts"; | |
| 16 | 17 | /** A frame as the arm sees it, without the STOP epoch the relay stamps on it. */ |
| 17 | 18 | const unstamped = (frame: string) => JSON.stringify(Object.fromEntries(Object.entries(JSON.parse(frame) as object).filter(([key]) => key !== "epoch"))); |
| 18 | 19 | |
| @@ -408,6 +409,46 @@ describe("relay", () => { | ||
| 408 | 409 | expect((await fetch(`${base}/move`, { method: "POST", body: '{"targets":{"1":2000}}' })).status).toBe(200); |
| 409 | 410 | }); |
| 410 | 411 | |
| 412 | test("the relay aligns clocks with the page and the arm server, and stamps what it forwards with the hops it measured", async () => { | |
| 413 | const seen: string[] = []; | |
| 414 | const wss = new WebSocketServer({ noServer: true }); | |
| 415 | camera = createServer((_req, res) => res.writeHead(404).end()); | |
| 416 | camera.on("upgrade", (req, socket, head) => | |
| 417 | wss.handleUpgrade(req, socket, head, (ws) => | |
| 418 | ws.on("message", (d) => { | |
| 419 | const text = String(d); | |
| 420 | const m = control(text); | |
| 421 | if (m?.kind === "ping") ws.send(pong(m.id, Date.now() + 1000, Date.now() + 1000)); // an arm server whose clock is a second ahead | |
| 422 | else seen.push(text); | |
| 423 | }), | |
| 424 | ), | |
| 425 | ); | |
| 426 | const pump = setInterval(() => wss.clients.forEach((c) => c.send('{"joints":[]}')), 10); | |
| 427 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(await listening(camera)), link: FAST }); | |
| 428 | const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/follow`); | |
| 429 | const pongs: string[] = []; | |
| 430 | ws.on("message", (d) => pongs.push(String(d))); | |
| 431 | await new Promise((r, j) => (ws.once("open", r), ws.once("error", j))); | |
| 432 | ws.send(JSON.stringify({ ping: 7 })); | |
| 433 | await sleep(60); | |
| 434 | const answer = pongs.find((t) => control(t)?.kind === "pong"); // the arm server's own chatter is passed on too | |
| 435 | expect(JSON.parse(answer!)).toMatchObject({ pong: 7 }); | |
| 436 | expect(pongs.filter((t) => control(t)?.kind === "pong")).toHaveLength(1); // the arm server's pong to the relay is not | |
| 437 | ws.send(JSON.stringify({ offset: -50 })); // the page's clock is 50 ms ahead of the relay's | |
| 438 | ws.send(JSON.stringify({ readback: 33 })); | |
| 439 | ws.send(stamper(() => Date.now() + 50 - 12)({ 1: 2050 })); // sent 12 ms ago on the page's clock | |
| 440 | await sleep(60); | |
| 441 | const [readback, frame] = [seen.find((t) => t.includes("readback")), seen.find((t) => t.includes('"1"'))]; | |
| 442 | expect(JSON.parse(readback!)).toEqual({ readback: 33 }); | |
| 443 | const sent = JSON.parse(frame!) as { up: number; rt: number }; | |
| 444 | expect(sent.up).toBeGreaterThanOrEqual(12); | |
| 445 | expect(sent.up).toBeLessThan(40); | |
| 446 | expect(sent.rt - Date.now()).toBeGreaterThan(900); // the relay's send time on the arm server's clock | |
| 447 | expect(sent.rt - Date.now()).toBeLessThan(1100); | |
| 448 | clearInterval(pump); | |
| 449 | ws.close(1000); | |
| 450 | }); | |
| 451 | ||
| 411 | 452 | test("a stream on the real arm goes quiet while the link reconnects, and flows again once it is back", async () => { |
| 412 | 453 | // An arm server whose /watch can be silenced while its /follow stays up: what the relay passes on is counted. |
| 413 | 454 | let watching = true; |
| @@ -416,7 +457,8 @@ describe("relay", () => { | ||
| 416 | 457 | camera = createServer((_req, res) => res.writeHead(404).end()); |
| 417 | 458 | camera.on("upgrade", (req, socket, head) => |
| 418 | 459 | wss.handleUpgrade(req, socket, head, (ws) => { |
| 419 | if (req.url === "/follow") ws.on("message", (d) => got.push(String(d))); | |
| 460 | // The relay also pings the arm server to align clocks (src/hops.ts): not a frame. | |
| 461 | if (req.url === "/follow") ws.on("message", (d) => void (control(String(d)) || got.push(String(d)))); | |
| 420 | 462 | }), |
| 421 | 463 | ); |
| 422 | 464 | const pump = setInterval(() => watching && wss.clients.forEach((c) => c.send('{"joints":[]}')), 10); |
Software/so101/src/relay.ts +38 −7
| @@ -70,6 +70,8 @@ import { CameraSocket } from "./camera-socket.ts"; | ||
| 70 | 70 | import { OFFLINE_FRAME } from "./offline-frame.ts"; |
| 71 | 71 | import { ServoBaseline } from "./servo-baseline.ts"; |
| 72 | 72 | import { handleAdmin, handleLift } from "./admin-routes.ts"; |
| 73 | import { control, hopMs, pong } from "./hops.ts"; | |
| 74 | import { HopLink } from "./ui/hop-link.js"; | |
| 73 | 75 | import { readPrecision } from "./precision-panel.ts"; |
| 74 | 76 | import { parseFps, type FeedOptions } from "./camera-feed.ts"; |
| 75 | 77 | import { CameraError, CameraHub } from "./cameras.ts"; |
| @@ -408,6 +410,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 408 | 410 | |
| 409 | 411 | /** STOP's hold, for every stream through here, and each driver's clock (their stamps, seen from here). */ |
| 410 | 412 | const gate = new CommandGate(options.holdFile); |
| 413 | /** The arm server's clock minus this relay's, from the last stream's pings; null before any (src/hops.ts). */ | |
| 414 | let armOffset: number | null = null; | |
| 411 | 415 | const clocks = new Map<string, Clock>(); |
| 412 | 416 | /** |
| 413 | 417 | * The arm's /status, saying which arm it is: from the link while it is live, the last one it had while reconnecting, |
| @@ -477,15 +481,20 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 477 | 481 | if (lobby.watching === 0) return void (pushed = ""); // the next page to open the room hears it at once |
| 478 | 482 | pushing = true; |
| 479 | 483 | let text: string; |
| 484 | let stamped: string; | |
| 480 | 485 | try { |
| 481 | 486 | const { code, body } = await statusOf(); |
| 482 | text = JSON.stringify(code === 200 ? { type: "status", status: body } : { type: "status", status: null, error: body.error ?? `status ${code}` }); | |
| 487 | // The arm's read time is stamped on this relay's clock for the page's read-back→browser hop (src/hops.ts); a stamp | |
| 488 | // that changed alone is no news, so what is compared leaves it out. | |
| 489 | const { readAt, ...rest } = body; | |
| 490 | text = JSON.stringify(code === 200 ? { type: "status", status: rest } : { type: "status", status: null, error: body.error ?? `status ${code}` }); | |
| 491 | stamped = code === 200 && armOffset !== null && typeof readAt === "number" ? JSON.stringify({ type: "status", status: { ...rest, readAtRelay: readAt - armOffset } }) : text; | |
| 483 | 492 | } catch (err) { |
| 484 | text = JSON.stringify({ type: "status", status: null, error: reason(err) }); | |
| 493 | text = stamped = JSON.stringify({ type: "status", status: null, error: reason(err) }); | |
| 485 | 494 | } finally { |
| 486 | 495 | pushing = false; |
| 487 | 496 | } |
| 488 | if (text !== pushed) lobby.publish(text); | |
| 497 | if (text !== pushed) lobby.publish(stamped); | |
| 489 | 498 | pushed = text; |
| 490 | 499 | }; |
| 491 | 500 | |
| @@ -782,7 +791,19 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 782 | 791 | armSides.delete(browser); |
| 783 | 792 | admission.close(); |
| 784 | 793 | }); |
| 785 | arm.on("message", (data, binary) => browser.readyState === WebSocket.OPEN && browser.send(data, { binary })); | |
| 794 | // The arm server's answers to the relay's own pings align the two clocks (src/hops.ts); anything else goes to the page. | |
| 795 | const armClock = new HopLink((text) => arm.readyState === WebSocket.OPEN && arm.send(text)); | |
| 796 | armClock.start(); | |
| 797 | arm.on("message", (data, binary) => { | |
| 798 | if (!binary && armClock.message(String(data))) { | |
| 799 | if (armClock.offset !== null) armOffset = armClock.offset; | |
| 800 | return; | |
| 801 | } | |
| 802 | if (browser.readyState === WebSocket.OPEN) browser.send(data, { binary }); | |
| 803 | }); | |
| 804 | arm.on("close", () => armClock.stop()); | |
| 805 | /** The relay's clock minus the page's, as the page last said it; null until it does. */ | |
| 806 | let pageOffset: number | null = null; | |
| 786 | 807 | const cap = new FrameCap(); |
| 787 | 808 | let seeded = false; |
| 788 | 809 | let asking = false; |
| @@ -807,7 +828,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 807 | 828 | const values = (text: string) => perServo(parseFrame(text) ?? []); |
| 808 | 829 | const rate = new FrameRate(); |
| 809 | 830 | /** A frame as it goes to the arm server: carrying the STOP epoch it was sent under. */ |
| 810 | const stamped = (frame: string) => JSON.stringify({ ...(JSON.parse(frame) as object), epoch }); | |
| 831 | const stamped = (frame: string, up: number | null) => JSON.stringify({ ...(JSON.parse(frame) as object), epoch, ...(up === null ? {} : { up }), ...(armOffset === null ? {} : { rt: Date.now() + armOffset }) }); | |
| 811 | 832 | /** Set when this relay ends the stream for breaking the limits: the arm then holds, as for a stream closed on purpose. */ |
| 812 | 833 | let overLimit = false; |
| 813 | 834 | browser.on("message", (raw) => { |
| @@ -826,6 +847,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 826 | 847 | return browser.close(FLOOD_CLOSE, "too many frames: the stream was closed"); |
| 827 | 848 | } |
| 828 | 849 | if (pace === "drop") return void metrics.count("follow_frames_dropped"); |
| 850 | const received = Date.now(); | |
| 851 | const message = control(String(raw)); | |
| 852 | if (message) { | |
| 853 | if (message.kind === "ping") browser.send(pong(message.id, received)); | |
| 854 | else if (message.kind === "offset") pageOffset = message.ms; | |
| 855 | else if (message.kind === "readback" && arm.readyState === WebSocket.OPEN) arm.send(JSON.stringify({ readback: message.ms })); | |
| 856 | return; | |
| 857 | } | |
| 829 | 858 | if (driver !== null) { |
| 830 | 859 | // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on. |
| 831 | 860 | if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void machine.note("clamp", "not passed on", { why: "the turn ended" }); |
| @@ -836,12 +865,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 836 | 865 | if (arm.readyState !== WebSocket.OPEN) return; |
| 837 | 866 | if (real && mode() !== "live") return void machine.note("clamp", "not passed on", { why: RECONNECTING }); |
| 838 | 867 | const data = JSON.stringify(Object.fromEntries(verdict.joints)); |
| 868 | // The page's stamp on the aligned clock: how long the frame took to get here. | |
| 869 | const up = pageOffset === null ? null : hopMs(received - ((JSON.parse(String(raw)) as { t: number }).t + pageOffset)); | |
| 839 | 870 | const percent = lobby?.maxSpeedPercent ?? 100; |
| 840 | 871 | const text = String(data); |
| 841 | 872 | if (percent >= 100) { |
| 842 | 873 | if (lobby) cap.cap(text, percent); // keeps each joint's last target, so a cap set mid-stream starts from there |
| 843 | 874 | machine.note("sent", "follow frame", { values: values(text) }); |
| 844 | arm.send(stamped(data)); | |
| 875 | arm.send(stamped(data, up)); | |
| 845 | 876 | return; |
| 846 | 877 | } |
| 847 | 878 | if (!seeded) seed(); |
| @@ -849,7 +880,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 849 | 880 | if (capped !== text) machine.note("clamp", "speed cap", { percent, asked: values(text), sent: capped === null ? {} : values(capped) }); |
| 850 | 881 | if (capped === null) return; |
| 851 | 882 | machine.note("sent", "follow frame", { values: values(capped) }); |
| 852 | arm.send(stamped(capped)); | |
| 883 | arm.send(stamped(capped, up)); | |
| 853 | 884 | }); |
| 854 | 885 | arm.on("close", (code, why) => browser.close(sendable(code) ? code : 1011, why.length ? why.toString() : "the arm server went away")); |
| 855 | 886 | // A browser that closed the stream on purpose leaves the arm holding; one that vanished leaves it parking. |
Software/so101/src/server.ts +17 −2
| @@ -31,7 +31,7 @@ | ||
| 31 | 31 | * PUT /settings change any of those |
| 32 | 32 | * POST /settings/table-here put the table (tableOffsetMm, -100..100) under the arm's lowest link now, the margin below it |
| 33 | 33 | * GET /metrics jams, blocks, stalls, peak load and temperature per joint, recent events |
| 34 | * GET /precision the precision scorecard (src/precision.ts): tracking error, time to arrive, snap-backs by cause, | |
| 34 | * GET /precision the precision scorecard (src/precision.ts, hops in src/hops.ts): tracking error, time to arrive, snap-backs by cause, | |
| 35 | 35 | * latency by hop, for the session being driven, the last 10 minutes, and the stored sessions and hours; |
| 36 | 36 | * ?hours=1-720 hourly rows (48 by default) |
| 37 | 37 | * GET /model.json the arm model: joints, the model's floor (floorZ) and the enforced table (tableZ), where the URDF is served |
| @@ -73,6 +73,7 @@ import { clamp, type Frame } from "./mapping.ts"; | ||
| 73 | 73 | import { MachineLog, perServo } from "./machine-log.ts"; |
| 74 | 74 | import { Metrics } from "./metrics.ts"; |
| 75 | 75 | import { ARRIVE_TICKS, type Cause, type Tracker } from "./precision.ts"; |
| 76 | import { control, frameHops, pong } from "./hops.ts"; | |
| 76 | 77 | import { PrecisionHistory } from "./precision-history.ts"; |
| 77 | 78 | import { MotionError, moveToPose, playTrajectory, recordTrajectory, STALL_SPAN_TICKS } from "./motion.ts"; |
| 78 | 79 | import { assertPoseName, parsePoseFile, serializePoseFile, type PoseFile } from "./poses.ts"; |
| @@ -559,6 +560,7 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}): | ||
| 559 | 560 | |
| 560 | 561 | const status = async () => { |
| 561 | 562 | const positions = await bus.syncRead(ids, STS3215.Present_Position); |
| 563 | const readAt = Date.now(); | |
| 562 | 564 | const loads = await bus.syncRead(ids, STS3215.Present_Load); |
| 563 | 565 | const temps = await bus.syncRead(ids, STS3215.Present_Temperature); |
| 564 | 566 | const torque = await bus.syncRead(ids, STS3215.Torque_Enable); |
| @@ -584,6 +586,8 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}): | ||
| 584 | 586 | approach(zone); |
| 585 | 587 | return { |
| 586 | 588 | joints, |
| 589 | /** When the positions were read, ms on this server's clock: the relay aligns it for the read-back→browser hop (src/hops.ts). */ | |
| 590 | readAt, | |
| 587 | 591 | busy: driver !== null, |
| 588 | 592 | driving: driver, |
| 589 | 593 | recording: recording?.name ?? null, |
| @@ -1312,7 +1316,15 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}): | ||
| 1312 | 1316 | return end(ws, FLOOD_CLOSE, "too many frames: the stream was closed"); |
| 1313 | 1317 | } |
| 1314 | 1318 | if (pace === "drop") return void stats.gated++; |
| 1315 | const verdict = admission.admit(String(data)); | |
| 1319 | const text = String(data); | |
| 1320 | const received = Date.now(); | |
| 1321 | const message = control(text); | |
| 1322 | if (message) { | |
| 1323 | if (message.kind === "ping") ws.send(pong(message.id, received)); | |
| 1324 | else if (message.kind === "readback") tracker.hop("read-back→browser", received, message.ms); | |
| 1325 | return; | |
| 1326 | } | |
| 1327 | const verdict = admission.admit(text); | |
| 1316 | 1328 | if (!verdict.ok) { |
| 1317 | 1329 | if (verdict.drop !== "malformed") { |
| 1318 | 1330 | stats.gated++; |
| @@ -1322,6 +1334,9 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}): | ||
| 1322 | 1334 | return; |
| 1323 | 1335 | } |
| 1324 | 1336 | staling = false; |
| 1337 | const { up, relayToPi } = frameHops(text, received); | |
| 1338 | if (up !== null) tracker.hop("browser→relay", received, up); | |
| 1339 | if (relayToPi !== null) tracker.hop("relay→Pi", received, relayToPi); | |
| 1325 | 1340 | const frame = verdict.joints; |
| 1326 | 1341 | clearTimeout(timer); |
| 1327 | 1342 | timer = setTimeout(() => void hold(), holdAfterMs); |
Software/so101/src/ui/clock-sync.js added +39
| @@ -0,0 +1,39 @@ | ||
| 1 | // @ts-check | |
| 2 | /** | |
| 3 | * Two clocks aligned with an NTP-style exchange: the asker stamps t0 on sending, the answerer t1 on receiving and t2 on | |
| 4 | * answering, the asker t3 on receiving the answer. The answerer's clock minus the asker's is ((t1-t0)+(t2-t3))/2, off by | |
| 5 | * at most half the round trip's delay, (t3-t0)-(t2-t1); so the offset is taken from the quickest of the last few | |
| 6 | * exchanges, the one a queue spoiled least. Pure: stamps in, offsets out. | |
| 7 | */ | |
| 8 | ||
| 9 | /** How many exchanges are kept. */ | |
| 10 | export const KEEP = 8; | |
| 11 | ||
| 12 | /** The offset (answerer minus asker) and the delay of one exchange, ms. @returns {{ offset: number, delay: number }} */ | |
| 13 | export function exchange(/** @type {number} */ t0, /** @type {number} */ t1, /** @type {number} */ t2, /** @type {number} */ t3) { | |
| 14 | return { offset: (t1 - t0 + (t2 - t3)) / 2, delay: t3 - t0 - (t2 - t1) }; | |
| 15 | } | |
| 16 | ||
| 17 | export class ClockSync { | |
| 18 | constructor() { | |
| 19 | /** @type {Array<{ offset: number, delay: number }>} */ | |
| 20 | this.recent = []; | |
| 21 | } | |
| 22 | ||
| 23 | /** Takes an exchange in; a malformed one, or one whose delay is negative (a clock stepped meanwhile), is ignored. */ | |
| 24 | add(/** @type {number} */ t0, /** @type {number} */ t1, /** @type {number} */ t2, /** @type {number} */ t3) { | |
| 25 | if (![t0, t1, t2, t3].every(Number.isFinite)) return false; | |
| 26 | const e = exchange(t0, t1, t2, t3); | |
| 27 | if (e.delay < 0) return false; | |
| 28 | this.recent.push(e); | |
| 29 | if (this.recent.length > KEEP) this.recent.shift(); | |
| 30 | return true; | |
| 31 | } | |
| 32 | ||
| 33 | /** The answerer's clock minus the asker's, from the lowest-delay exchange; null before any. @returns {number | null} */ | |
| 34 | get offset() { | |
| 35 | let best = null; | |
| 36 | for (const e of this.recent) if (!best || e.delay < best.delay) best = e; | |
| 37 | return best ? best.offset : null; | |
| 38 | } | |
| 39 | } |
Software/so101/src/ui/hop-link.js added +84
| @@ -0,0 +1,84 @@ | ||
| 1 | // @ts-check | |
| 2 | /** | |
| 3 | * The page's end of the latency hops (src/hops.ts): it pings the relay down the /follow stream, aligns its clock with the | |
| 4 | * relay's (src/ui/clock-sync.js) and tells the relay the offset, so the relay can say how long a frame took to get there; | |
| 5 | * and it says how old a status was, on the relay's clock, when it arrived. | |
| 6 | */ | |
| 7 | import { ClockSync } from "./clock-sync.js"; | |
| 8 | ||
| 9 | /** Pings at the start of a stream, ms apart, then one every PING_EVERY_MS. */ | |
| 10 | const BURST = 4; | |
| 11 | const BURST_MS = 100; | |
| 12 | const PING_EVERY_MS = 5000; | |
| 13 | ||
| 14 | export class HopLink { | |
| 15 | /** | |
| 16 | * @param {(text: string) => void} send | |
| 17 | * @param {{ now?: () => number, later?: (fn: () => void, ms: number) => unknown, cancel?: (h: unknown) => void }} [options] | |
| 18 | */ | |
| 19 | constructor(send, options = {}) { | |
| 20 | this.send = send; | |
| 21 | this.now = options.now ?? (() => Date.now()); | |
| 22 | this.later = options.later ?? ((fn, ms) => setInterval(fn, ms)); | |
| 23 | this.cancel = options.cancel ?? ((h) => clearInterval(/** @type {any} */ (h))); | |
| 24 | this.sync = new ClockSync(); | |
| 25 | /** @type {Map<number, number>} t0 of each ping out, by its id */ | |
| 26 | this.out = new Map(); | |
| 27 | this.n = 0; | |
| 28 | /** @type {unknown} */ | |
| 29 | this.timer = null; | |
| 30 | this.sent = 0; | |
| 31 | } | |
| 32 | ||
| 33 | /** Starts pinging; call when the stream opens. */ | |
| 34 | start() { | |
| 35 | this.stop(); | |
| 36 | this.sync = new ClockSync(); | |
| 37 | this.sent = 0; | |
| 38 | this.ping(); | |
| 39 | this.timer = this.later(() => { | |
| 40 | this.ping(); | |
| 41 | if (++this.sent === BURST && this.timer !== null) { | |
| 42 | this.cancel(this.timer); | |
| 43 | this.timer = this.later(() => this.ping(), PING_EVERY_MS); | |
| 44 | } | |
| 45 | }, BURST_MS); | |
| 46 | } | |
| 47 | ||
| 48 | stop() { | |
| 49 | if (this.timer !== null) this.cancel(this.timer); | |
| 50 | this.timer = null; | |
| 51 | this.out.clear(); | |
| 52 | } | |
| 53 | ||
| 54 | ping() { | |
| 55 | const t0 = this.now(); | |
| 56 | this.out.set(++this.n, t0); | |
| 57 | if (this.out.size > 16) this.out.delete(/** @type {number} */ (this.out.keys().next().value)); | |
| 58 | this.send(JSON.stringify({ ping: this.n, t0 })); | |
| 59 | } | |
| 60 | ||
| 61 | /** A message from the relay: answers whether it was a pong (and so is not for anyone else). */ | |
| 62 | message(/** @type {string} */ text) { | |
| 63 | if (!text.startsWith('{"pong"')) return false; | |
| 64 | let m; | |
| 65 | try { m = JSON.parse(text); } catch { return true; } | |
| 66 | const t0 = this.out.get(m?.pong); | |
| 67 | if (t0 === undefined) return true; | |
| 68 | this.out.delete(m.pong); | |
| 69 | if (this.sync.add(t0, m.t1, m.t2, this.now())) this.send(JSON.stringify({ offset: this.sync.offset })); | |
| 70 | return true; | |
| 71 | } | |
| 72 | ||
| 73 | /** The relay's clock minus this page's, ms; null before the first answer. */ | |
| 74 | get offset() { | |
| 75 | return this.sync.offset; | |
| 76 | } | |
| 77 | ||
| 78 | /** Tells the relay how old a status was on arrival; `readAtRelay` is when the arm read it, on the relay's clock. */ | |
| 79 | readback(/** @type {unknown} */ readAtRelay) { | |
| 80 | const off = this.offset; | |
| 81 | if (off === null || typeof readAtRelay !== "number" || !Number.isFinite(readAtRelay)) return; | |
| 82 | this.send(JSON.stringify({ readback: Math.max(0, Math.round(this.now() + off - readAtRelay)) })); | |
| 83 | } | |
| 84 | } |
Software/so101/src/viewer.html +8
| @@ -20,6 +20,7 @@ | ||
| 20 | 20 | <link rel="modulepreload" href="/ui/__VERSION__/camera-plan.js"> |
| 21 | 21 | <link rel="modulepreload" href="/ui/__VERSION__/drive.js"> |
| 22 | 22 | <link rel="modulepreload" href="/ui/__VERSION__/frames.js"> |
| 23 | <link rel="modulepreload" href="/ui/__VERSION__/hop-link.js"> | |
| 23 | 24 | <link rel="stylesheet" href="/ui/__VERSION__/tabs.css"> |
| 24 | 25 | <style> |
| 25 | 26 | /* Phone portrait first: the view on top, a status strip with the E-stop under it, a sheet of tabs below that. |
| @@ -493,6 +494,7 @@ | ||
| 493 | 494 | import { cameraPlan, QUALITIES, samePlan } from "/ui/__VERSION__/camera-plan.js"; |
| 494 | 495 | import { bindTray } from "/ui/__VERSION__/drive.js"; |
| 495 | 496 | import { Frames } from "/ui/__VERSION__/frames.js"; |
| 497 | import { HopLink } from "/ui/__VERSION__/hop-link.js"; | |
| 496 | 498 | import { holdText, warnText } from "/ui/__VERSION__/hold-line.js"; |
| 497 | 499 | import { noteSink } from "/ui/__VERSION__/zone-notes.js"; |
| 498 | 500 | import { createRoom } from "/ui/__VERSION__/lobby-panel.js"; |
| @@ -807,6 +809,8 @@ | ||
| 807 | 809 | if (ws?.readyState === 1) ws.send(data); |
| 808 | 810 | else if (ws?.readyState === 0) stream.waiting = { data, at: Date.now() }; |
| 809 | 811 | }), |
| 812 | /** Aligns this clock with the relay's over the stream, for the latency hops in /precision (src/hops.ts). */ | |
| 813 | hops: new HopLink((data) => { if (stream.ws?.readyState === 1) stream.ws.send(data); }), | |
| 810 | 814 | /** STOP: drop what is pending, close, send nothing until a release or a resume. */ |
| 811 | 815 | halt() { |
| 812 | 816 | clearTimeout(this.closeTimer); |
| @@ -825,12 +829,15 @@ | ||
| 825 | 829 | const ws = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/follow`); |
| 826 | 830 | this.waiting = null; |
| 827 | 831 | ws.onopen = () => { |
| 832 | if (this.ws === ws) this.hops.start(); | |
| 828 | 833 | const w = this.waiting; |
| 829 | 834 | this.waiting = null; |
| 830 | 835 | if (this.ws === ws && w && Date.now() - w.at <= 200) ws.send(w.data); |
| 831 | 836 | }; |
| 837 | ws.onmessage = (e) => { if (typeof e.data === "string") this.hops.message(e.data); }; | |
| 832 | 838 | ws.onclose = (e) => { |
| 833 | 839 | if (this.ws !== ws) return; |
| 840 | this.hops.stop(); | |
| 834 | 841 | this.ws = null; |
| 835 | 842 | if (e.code === 1013 || e.code === 1000 || e.code >= 4000) { this.halted = true; msg(e.reason || "the arm is busy", true); } |
| 836 | 843 | }; |
| @@ -1499,6 +1506,7 @@ | ||
| 1499 | 1506 | let started = false, early = null; |
| 1500 | 1507 | function applyStatus(d) { |
| 1501 | 1508 | if (!started) return void (early = d); |
| 1509 | if (stream.ws?.readyState === 1) stream.hops.readback(d.readAtRelay); | |
| 1502 | 1510 | { |
| 1503 | 1511 | const was = $("why").textContent; |
| 1504 | 1512 | status = d; |
new above
offline