Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

Per-hop latency on aligned clocks: browser→relay, relay→Pi, read-back→browser, in /precision #470

Parked opened this
Closed without merging

smith/arm:smith/458-per-hop-latency-on-aligned-clocks-browse → main 1 commit fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
11 files changed +421 −16
Software/so101/bench/precision.ts +17 −1
@@ -15,6 +15,7 @@ import { FakeBus } from "../src/fake-bus.ts";
1515 import type { Card } from "../src/precision.ts";
1616 import type { Scorecard } from "../src/precision-history.ts";
1717 import { startServer } from "../src/server.ts";
18 import { HopLink } from "../src/ui/hop-link.js";
1819
1920 /** One frame of the script: when (ms from the start) and the joints it names. */
2021 export interface Step {
@@ -58,12 +59,27 @@ export async function bench(script: readonly Step[] = gesture()): Promise<Card>
5859 try {
5960 const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`);
6061 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
62 // The bench is the relay and the page both, on one machine: its clocks are one clock, aligned by the same ping the relay
63 // uses, so the three hops are loopback (src/hops.ts).
64 let seq = 0;
65 const link = new HopLink((text) => ws.send(text));
66 ws.on("message", (data) => void link.message(String(data)));
67 for (let n = 0; n < 4; n++) {
68 link.ping();
69 await new Promise((r) => setTimeout(r, 20));
70 }
71 const readbacks = setInterval(async () => {
72 const { readAt } = (await (await fetch(`http://127.0.0.1:${server.port}/status`)).json()) as { readAt: number };
73 if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ readback: Math.max(0, Date.now() - readAt) }));
74 }, 100);
6175 const start = performance.now();
6276 for (const step of script) {
6377 const wait = step.at - (performance.now() - start);
6478 if (wait > 0) await new Promise((r) => setTimeout(r, wait));
65 ws.send(JSON.stringify(step.joints));
79 const t = Date.now();
80 ws.send(JSON.stringify({ ...step.joints, seq: ++seq, t, up: Date.now() - t, rt: Date.now() + (link.offset ?? 0) }));
6681 }
82 clearInterval(readbacks);
6783 await new Promise((r) => setTimeout(r, 300));
6884 ws.close(1000);
6985 await new Promise((r) => ws.once("close", r));
Software/so101/src/hops.test.ts added +101
@@ -0,0 +1,101 @@
1 import { describe, expect, test } from "bun:test";
2 import { control, frameHops, hopMs, pong } from "./hops.ts";
3 import { emptyRecords, summarize, Tracker } from "./precision.ts";
4 import { ClockSync, exchange } from "./ui/clock-sync.js";
5 import { HopLink } from "./ui/hop-link.js";
6
7 describe("clock alignment", () => {
8 test("one exchange: the offset is the answerer's clock minus the asker's, the delay the round trip less the answerer's wait", () => {
9 // answerer 500 ahead; 20 ms out, 30 back, 5 ms spent answering
10 expect(exchange(1000, 1520, 1525, 1055)).toEqual({ offset: 500 + (20 - 30) / 2, delay: 50 });
11 // symmetric legs: exact
12 expect(exchange(0, 510, 510, 20)).toEqual({ offset: 500, delay: 20 });
13 });
14
15 test("the offset comes from the lowest-delay of the last few exchanges", () => {
16 const sync = new ClockSync();
17 expect(sync.offset).toBeNull();
18 sync.add(0, 700, 700, 400); // a queued answer: delay 400, offset off by 200
19 sync.add(1000, 1510, 1510, 1020); // delay 20, offset 500
20 sync.add(2000, 2600, 2600, 2300); // delay 300
21 expect(sync.offset).toBe(500);
22 });
23
24 test("an old exchange is forgotten, a malformed or time-reversed one ignored", () => {
25 const sync = new ClockSync();
26 sync.add(0, 510, 510, 20);
27 for (let i = 1; i <= 8; i++) sync.add(i * 100, i * 100 + 800, i * 100 + 800, i * 100 + 100); // delay 100, offset 750
28 expect(sync.offset).toBe(750);
29 expect(sync.add(0, Number.NaN, 0, 0)).toBe(false);
30 expect(sync.add(100, 0, 0, 50)).toBe(false);
31 });
32 });
33
34 describe("hop wire", () => {
35 test("control messages are told from frames", () => {
36 expect(control('{"ping":3}')).toEqual({ kind: "ping", id: 3 });
37 expect(control('{"offset":-12.5}')).toEqual({ kind: "offset", ms: -12.5 });
38 expect(control('{"readback":42}')).toEqual({ kind: "readback", ms: 42 });
39 expect(control(pong(3, 10, 11))).toEqual({ kind: "pong" });
40 expect(control('{"1":2048,"seq":1,"t":5}')).toBeNull();
41 expect(control('{"readback":"fast"}')).toBeNull();
42 expect(control('{"readback":1e9}')).toBeNull();
43 expect(control('{"ping":')).toBeNull();
44 });
45
46 test("a hop is a plausible time, a small negative counting as zero", () => {
47 expect(hopMs(12.4)).toBe(12);
48 expect(hopMs(-3)).toBe(0);
49 expect(hopMs(Number.NaN)).toBeNull();
50 expect(hopMs(10 ** 7)).toBeNull();
51 expect(hopMs("5")).toBeNull();
52 });
53
54 test("a frame's hops: browser→relay as the relay measured it, relay→Pi from its stamp against the receive time", () => {
55 expect(frameHops('{"1":2048,"up":14,"rt":1000}', 1007)).toEqual({ up: 14, relayToPi: 7 });
56 expect(frameHops('{"1":2048,"seq":1,"t":5}', 1007)).toEqual({ up: null, relayToPi: null }); // from the CLI leader: nothing to measure
57 expect(frameHops('{"1":2048,"rt":"x","up":-1}', 1007)).toEqual({ up: 0, relayToPi: null });
58 });
59 });
60
61 describe("the page's link", () => {
62 test("pings, aligns its clock with the relay's and tells the relay; a status's age is on the relay's clock", () => {
63 let now = 10_000;
64 const sent: string[] = [];
65 const link = new HopLink((t) => sent.push(t), { now: () => now, later: () => 1, cancel: () => {} });
66 link.start();
67 expect(JSON.parse(sent[0]!)).toEqual({ ping: 1, t0: 10_000 });
68 now = 10_020; // relay 500 ahead, 10 ms each way
69 expect(link.message(pong(1, 10_510, 10_510))).toBe(true);
70 expect(link.offset).toBe(500);
71 expect(JSON.parse(sent[1]!)).toEqual({ offset: 500 });
72 expect(link.message('{"type":"status"}')).toBe(false);
73 // read by the arm at relay time 10_530 and here at page 10_050 (= relay 10_550): 20 ms
74 now = 10_050;
75 link.readback(10_530);
76 expect(JSON.parse(sent.at(-1)!)).toEqual({ readback: 20 });
77 const before = sent.length;
78 link.readback(undefined);
79 expect(sent.length).toBe(before);
80 });
81 });
82
83 describe("hops in the scorecard", () => {
84 test("each hop has its p50 and p95, and the headline latency sums them with the bus hop", () => {
85 const tracker = new Tracker(0);
86 for (const ms of [4, 5, 6, 7, 40]) tracker.hop("browser→relay", 100, ms);
87 for (const ms of [2, 3]) tracker.hop("relay→Pi", 100, ms);
88 tracker.hop("read-back→browser", 100, 30);
89 tracker.hop("read-back→browser", 10, 90); // before the span asked for
90 const card = summarize(tracker.records, 50);
91 expect(card.hops["browser→relay"]).toMatchObject({ n: 5, p50: 6, p95: 40, max: 40 });
92 expect(card.hops["relay→Pi"]).toMatchObject({ n: 2, p50: 2, p95: 3 });
93 expect(card.hops["read-back→browser"]).toMatchObject({ n: 1, p50: 30, p95: 30 });
94 expect(card.hops["bus→move"]!.p95).toBeNull();
95 expect(card.latencyP95Ms).toBe(40 + 3 + 30);
96 });
97
98 test("with no hop measured there is no latency", () => {
99 expect(summarize(emptyRecords()).latencyP95Ms).toBeNull();
100 });
101 });
Software/so101/src/hops.ts added +55
@@ -0,0 +1,55 @@
1 /**
2 * The latency hops a driver's command and the arm's answer make, each timed on clocks aligned with an NTP-style ping
3 * (src/ui/clock-sync.js) and recorded on the arm server, whose scorecard they are in (src/precision.ts):
4 *
5 * browser→relay the frame's `t` against the relay's receive time; the page tells the relay its offset
6 * relay→Pi the relay's send time against the arm server's receive time; the relay stamps `rt` (arm clock)
7 * read-back→browser the arm server's read time of a status against the page's receive time; the page reports it
8 *
9 * Messages on a /follow stream that are not frames (a frame is only servo ids, seq, t, epoch, and the relay's up, rt):
10 * {"ping": n} answered {"pong": n, "t1": receive, "t2": send}, by the relay to the page and the arm to the relay
11 * {"offset": ms} the page's clock aligned: relay minus page
12 * {"readback": ms} how old a status was when the page got it (passed on to the arm by the relay)
13 * The arm server's command gate still accepts a frame with none of this, as from the CLI leader.
14 */
15
16 export const HOPS = ["browser→relay", "relay→Pi", "read-back→browser"] as const;
17 /** The most a hop can honestly be, ms: anything beyond is a broken clock, not a slow network. */
18 const MAX_HOP_MS = 60_000;
19
20 export type Control = { kind: "ping"; id: number } | { kind: "offset"; ms: number } | { kind: "readback"; ms: number } | { kind: "pong" };
21
22 /** A hop's time, ms, when it is a plausible one: a small negative (alignment is only so good) counts as 0. */
23 export function hopMs(value: unknown): number | null {
24 if (typeof value !== "number" || !Number.isFinite(value) || value > MAX_HOP_MS || value < -MAX_HOP_MS) return null;
25 return Math.max(0, Math.round(value));
26 }
27
28 /** What a stream's text is when it is one of the control messages above, else null (a frame, or nothing). */
29 export function control(text: string): Control | null {
30 if (!/^\{"(ping|pong|offset|readback)"/.test(text)) return null;
31 let m: { ping?: unknown; pong?: unknown; offset?: unknown; readback?: unknown };
32 try {
33 m = JSON.parse(text);
34 } catch {
35 return null;
36 }
37 if (Number.isSafeInteger(m.ping)) return { kind: "ping", id: m.ping as number };
38 if ("pong" in m) return { kind: "pong" };
39 if (typeof m.offset === "number" && Number.isFinite(m.offset)) return { kind: "offset", ms: m.offset };
40 const ms = hopMs(m.readback);
41 return ms === null ? null : { kind: "readback", ms };
42 }
43
44 export const pong = (id: number, received: number, now = Date.now()) => JSON.stringify({ pong: id, t1: received, t2: now });
45
46 /** The hops a frame carries to the arm server: how long it took the page to the relay (`up`), and the relay's send stamp (`rt`). */
47 export function frameHops(text: string, received: number): { up: number | null; relayToPi: number | null } {
48 if (!text.includes('"rt"') && !text.includes('"up"')) return { up: null, relayToPi: null };
49 try {
50 const { up, rt } = JSON.parse(text) as { up?: unknown; rt?: unknown };
51 return { up: hopMs(up), relayToPi: typeof rt === "number" ? hopMs(received - rt) : null };
52 } catch {
53 return { up: null, relayToPi: null };
54 }
55 }
Software/so101/src/precision-panel.ts +1 −1
@@ -47,7 +47,7 @@ export function readPrecision(body: unknown): PrecisionPanel | null {
4747 { label: "Tracking error, p95", unit: "°", value: pick(window.trackingP95Deg, last.trackingP95Deg), trend: rows.map((r) => num(r.trackingP95Deg)) },
4848 { label: "Time to arrive, p95", unit: "ms", value: pick(window.arriveP95Ms, last.arriveP95Ms), trend: rows.map((r) => num(r.arriveP95Ms)) },
4949 { label: "Snap-backs per 100 releases", unit: "", value: pick(window.snapBacksPer100Releases, rate(last)), trend: rows.map(rate) },
50 { label: "Latency, p95, bus→motion", unit: "ms", value: pick(window.latencyP95Ms, last.latencyP95Ms), trend: rows.map((r) => num(r.latencyP95Ms)) },
50 { label: "Latency, p95, every hop summed", unit: "ms", value: pick(window.latencyP95Ms, last.latencyP95Ms), trend: rows.map((r) => num(r.latencyP95Ms)) },
5151 ],
5252 };
5353 }
Software/so101/src/precision.ts +18 −4
@@ -7,12 +7,14 @@
77 * ARRIVE_TICKS; then, for SETTLE_MS: the overshoot past the target, the settling time (until the
88 * last entry into the band) and the oscillations (each exit and re-entry is one)
99 * snap-backs each time the arm is kept from the driver's target without the driver doing it, with its cause
10 * latency, by hop for now the arm server's own: a Goal_Position write to the first read-back past MOVE_TICKS
10 * latency, by hop browser→relay, relay→Pi and read-back→browser on aligned clocks (src/hops.ts), and the arm
11 * server's own: a Goal_Position write to the first read-back past MOVE_TICKS
1112 *
1213 * One `Tracker` per drive session; `summarize` folds any span of its records into the same numbers, which is how the
1314 * rolling window and the stored session rows are made.
1415 */
1516
17 import { HOPS } from "./hops.ts";
1618 import { JOINTS } from "./sts3215.ts";
1719
1820 /** Ticks from the target that count as arrived. */
@@ -39,9 +41,11 @@ export interface Records {
3941 snaps: Array<{ at: number; cause: Cause; id: number | null; sizeDeg: number | null }>;
4042 releases: number[];
4143 oscillations: Array<readonly [number, number]>;
44 /** Latency samples by hop name (HOPS): [at, ms]. */
45 hops: Record<string, Array<readonly [number, number]>>;
4246 }
4347
44 export const emptyRecords = (): Records => ({ joints: new Map(), snaps: [], releases: [], oscillations: [] });
48 export const emptyRecords = (): Records => ({ joints: new Map(), snaps: [], releases: [], oscillations: [], hops: {} });
4549
4650 /** Nearest-rank percentile of unsorted values; null for none. */
4751 export function percentile(values: readonly number[], p: number): number | null {
@@ -82,7 +86,7 @@ export interface Card {
8286 trackingP95Deg: number | null;
8387 arriveP95Ms: number | null;
8488 snapBacksPer100Releases: number | null;
85 /** End to end: the sum of every measured hop's p95. Only the arm server's own hop is measured so far. */
89 /** End to end: the sum of every measured hop's p95. */
8690 latencyP95Ms: number | null;
8791 /** The joint with the highest p95 tracking error. */
8892 worstJoint: string | null;
@@ -123,7 +127,7 @@ export function summarize(records: Records, since = -Infinity): Card {
123127 for (const s of snaps) byCause[s.cause]++;
124128 const releases = records.releases.filter((at) => at >= since).length;
125129 const worst = joints.filter((j) => j.errorDeg.p95 !== null).sort((a, b) => b.errorDeg.p95! - a.errorDeg.p95!)[0];
126 const hops = { "bus→move": spread(allBus, 0) };
130 const hops: Record<string, Spread> = { ...Object.fromEntries(HOPS.map((hop) => [hop, spread(after(records.hops[hop] ?? []), 0)])), "bus→move": spread(allBus, 0) };
127131 const measured = Object.values(hops).filter((h) => h.p95 !== null);
128132 return {
129133 trackingP95Deg: round(percentile(allErrors.map((t) => t / TICKS_PER_DEGREE), 95)),
@@ -149,6 +153,7 @@ export function trim(records: Records, since: number): void {
149153 keep(records.snaps, (s) => s.at);
150154 keep(records.releases, (at) => at);
151155 keep(records.oscillations, ([at]) => at);
156 for (const list of Object.values(records.hops)) keep(list, ([at]) => at);
152157 }
153158
154159 interface Joint {
@@ -271,6 +276,15 @@ export class Tracker {
271276 for (const records of [this.records, ...this.#sinks]) records.snaps.push(entry);
272277 }
273278
279 /** A hop of a command's or a status's way took `ms`, on aligned clocks (src/hops.ts). */
280 hop(name: (typeof HOPS)[number], at: number, ms: number): void {
281 for (const records of [this.records, ...this.#sinks]) {
282 const list = (records.hops[name] ??= []);
283 list.push([at, ms]);
284 if (list.length > KEEP) list.shift();
285 }
286 }
287
274288 /** The driver let go: a dead-man hold, or the end of the stream. */
275289 release(at: number): void {
276290 for (const records of [this.records, ...this.#sinks]) records.releases.push(at);
Software/so101/src/relay.test.ts +43 −1
@@ -13,6 +13,7 @@ import { FakeBus } from "./fake-bus.ts";
1313 import { OFFLINE_FRAME } from "./offline-frame.ts";
1414 import { parseRemote, sameOrigin, startRelay, type Remote, type RunningRelay } from "./relay.ts";
1515 import { startServer, type RunningServer } from "./server.ts";
16 import { control, pong } from "./hops.ts";
1617 /** A frame as the arm sees it, without the STOP epoch the relay stamps on it. */
1718 const unstamped = (frame: string) => JSON.stringify(Object.fromEntries(Object.entries(JSON.parse(frame) as object).filter(([key]) => key !== "epoch")));
1819
@@ -408,6 +409,46 @@ describe("relay", () => {
408409 expect((await fetch(`${base}/move`, { method: "POST", body: '{"targets":{"1":2000}}' })).status).toBe(200);
409410 });
410411
412 test("the relay aligns clocks with the page and the arm server, and stamps what it forwards with the hops it measured", async () => {
413 const seen: string[] = [];
414 const wss = new WebSocketServer({ noServer: true });
415 camera = createServer((_req, res) => res.writeHead(404).end());
416 camera.on("upgrade", (req, socket, head) =>
417 wss.handleUpgrade(req, socket, head, (ws) =>
418 ws.on("message", (d) => {
419 const text = String(d);
420 const m = control(text);
421 if (m?.kind === "ping") ws.send(pong(m.id, Date.now() + 1000, Date.now() + 1000)); // an arm server whose clock is a second ahead
422 else seen.push(text);
423 }),
424 ),
425 );
426 const pump = setInterval(() => wss.clients.forEach((c) => c.send('{"joints":[]}')), 10);
427 relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(await listening(camera)), link: FAST });
428 const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/follow`);
429 const pongs: string[] = [];
430 ws.on("message", (d) => pongs.push(String(d)));
431 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
432 ws.send(JSON.stringify({ ping: 7 }));
433 await sleep(60);
434 const answer = pongs.find((t) => control(t)?.kind === "pong"); // the arm server's own chatter is passed on too
435 expect(JSON.parse(answer!)).toMatchObject({ pong: 7 });
436 expect(pongs.filter((t) => control(t)?.kind === "pong")).toHaveLength(1); // the arm server's pong to the relay is not
437 ws.send(JSON.stringify({ offset: -50 })); // the page's clock is 50 ms ahead of the relay's
438 ws.send(JSON.stringify({ readback: 33 }));
439 ws.send(stamper(() => Date.now() + 50 - 12)({ 1: 2050 })); // sent 12 ms ago on the page's clock
440 await sleep(60);
441 const [readback, frame] = [seen.find((t) => t.includes("readback")), seen.find((t) => t.includes('"1"'))];
442 expect(JSON.parse(readback!)).toEqual({ readback: 33 });
443 const sent = JSON.parse(frame!) as { up: number; rt: number };
444 expect(sent.up).toBeGreaterThanOrEqual(12);
445 expect(sent.up).toBeLessThan(40);
446 expect(sent.rt - Date.now()).toBeGreaterThan(900); // the relay's send time on the arm server's clock
447 expect(sent.rt - Date.now()).toBeLessThan(1100);
448 clearInterval(pump);
449 ws.close(1000);
450 });
451
411452 test("a stream on the real arm goes quiet while the link reconnects, and flows again once it is back", async () => {
412453 // An arm server whose /watch can be silenced while its /follow stays up: what the relay passes on is counted.
413454 let watching = true;
@@ -416,7 +457,8 @@ describe("relay", () => {
416457 camera = createServer((_req, res) => res.writeHead(404).end());
417458 camera.on("upgrade", (req, socket, head) =>
418459 wss.handleUpgrade(req, socket, head, (ws) => {
419 if (req.url === "/follow") ws.on("message", (d) => got.push(String(d)));
460 // The relay also pings the arm server to align clocks (src/hops.ts): not a frame.
461 if (req.url === "/follow") ws.on("message", (d) => void (control(String(d)) || got.push(String(d))));
420462 }),
421463 );
422464 const pump = setInterval(() => watching && wss.clients.forEach((c) => c.send('{"joints":[]}')), 10);
Software/so101/src/relay.ts +38 −7
@@ -70,6 +70,8 @@ import { CameraSocket } from "./camera-socket.ts";
7070 import { OFFLINE_FRAME } from "./offline-frame.ts";
7171 import { ServoBaseline } from "./servo-baseline.ts";
7272 import { handleAdmin, handleLift } from "./admin-routes.ts";
73 import { control, hopMs, pong } from "./hops.ts";
74 import { HopLink } from "./ui/hop-link.js";
7375 import { readPrecision } from "./precision-panel.ts";
7476 import { parseFps, type FeedOptions } from "./camera-feed.ts";
7577 import { CameraError, CameraHub } from "./cameras.ts";
@@ -408,6 +410,8 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
408410
409411 /** STOP's hold, for every stream through here, and each driver's clock (their stamps, seen from here). */
410412 const gate = new CommandGate(options.holdFile);
413 /** The arm server's clock minus this relay's, from the last stream's pings; null before any (src/hops.ts). */
414 let armOffset: number | null = null;
411415 const clocks = new Map<string, Clock>();
412416 /**
413417 * The arm's /status, saying which arm it is: from the link while it is live, the last one it had while reconnecting,
@@ -477,15 +481,20 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
477481 if (lobby.watching === 0) return void (pushed = ""); // the next page to open the room hears it at once
478482 pushing = true;
479483 let text: string;
484 let stamped: string;
480485 try {
481486 const { code, body } = await statusOf();
482 text = JSON.stringify(code === 200 ? { type: "status", status: body } : { type: "status", status: null, error: body.error ?? `status ${code}` });
487 // The arm's read time is stamped on this relay's clock for the page's read-back→browser hop (src/hops.ts); a stamp
488 // that changed alone is no news, so what is compared leaves it out.
489 const { readAt, ...rest } = body;
490 text = JSON.stringify(code === 200 ? { type: "status", status: rest } : { type: "status", status: null, error: body.error ?? `status ${code}` });
491 stamped = code === 200 && armOffset !== null && typeof readAt === "number" ? JSON.stringify({ type: "status", status: { ...rest, readAtRelay: readAt - armOffset } }) : text;
483492 } catch (err) {
484 text = JSON.stringify({ type: "status", status: null, error: reason(err) });
493 text = stamped = JSON.stringify({ type: "status", status: null, error: reason(err) });
485494 } finally {
486495 pushing = false;
487496 }
488 if (text !== pushed) lobby.publish(text);
497 if (text !== pushed) lobby.publish(stamped);
489498 pushed = text;
490499 };
491500
@@ -782,7 +791,19 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
782791 armSides.delete(browser);
783792 admission.close();
784793 });
785 arm.on("message", (data, binary) => browser.readyState === WebSocket.OPEN && browser.send(data, { binary }));
794 // The arm server's answers to the relay's own pings align the two clocks (src/hops.ts); anything else goes to the page.
795 const armClock = new HopLink((text) => arm.readyState === WebSocket.OPEN && arm.send(text));
796 armClock.start();
797 arm.on("message", (data, binary) => {
798 if (!binary && armClock.message(String(data))) {
799 if (armClock.offset !== null) armOffset = armClock.offset;
800 return;
801 }
802 if (browser.readyState === WebSocket.OPEN) browser.send(data, { binary });
803 });
804 arm.on("close", () => armClock.stop());
805 /** The relay's clock minus the page's, as the page last said it; null until it does. */
806 let pageOffset: number | null = null;
786807 const cap = new FrameCap();
787808 let seeded = false;
788809 let asking = false;
@@ -807,7 +828,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
807828 const values = (text: string) => perServo(parseFrame(text) ?? []);
808829 const rate = new FrameRate();
809830 /** A frame as it goes to the arm server: carrying the STOP epoch it was sent under. */
810 const stamped = (frame: string) => JSON.stringify({ ...(JSON.parse(frame) as object), epoch });
831 const stamped = (frame: string, up: number | null) => JSON.stringify({ ...(JSON.parse(frame) as object), epoch, ...(up === null ? {} : { up }), ...(armOffset === null ? {} : { rt: Date.now() + armOffset }) });
811832 /** Set when this relay ends the stream for breaking the limits: the arm then holds, as for a stream closed on purpose. */
812833 let overLimit = false;
813834 browser.on("message", (raw) => {
@@ -826,6 +847,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
826847 return browser.close(FLOOD_CLOSE, "too many frames: the stream was closed");
827848 }
828849 if (pace === "drop") return void metrics.count("follow_frames_dropped");
850 const received = Date.now();
851 const message = control(String(raw));
852 if (message) {
853 if (message.kind === "ping") browser.send(pong(message.id, received));
854 else if (message.kind === "offset") pageOffset = message.ms;
855 else if (message.kind === "readback" && arm.readyState === WebSocket.OPEN) arm.send(JSON.stringify({ readback: message.ms }));
856 return;
857 }
829858 if (driver !== null) {
830859 // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on.
831860 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void machine.note("clamp", "not passed on", { why: "the turn ended" });
@@ -836,12 +865,14 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
836865 if (arm.readyState !== WebSocket.OPEN) return;
837866 if (real && mode() !== "live") return void machine.note("clamp", "not passed on", { why: RECONNECTING });
838867 const data = JSON.stringify(Object.fromEntries(verdict.joints));
868 // The page's stamp on the aligned clock: how long the frame took to get here.
869 const up = pageOffset === null ? null : hopMs(received - ((JSON.parse(String(raw)) as { t: number }).t + pageOffset));
839870 const percent = lobby?.maxSpeedPercent ?? 100;
840871 const text = String(data);
841872 if (percent >= 100) {
842873 if (lobby) cap.cap(text, percent); // keeps each joint's last target, so a cap set mid-stream starts from there
843874 machine.note("sent", "follow frame", { values: values(text) });
844 arm.send(stamped(data));
875 arm.send(stamped(data, up));
845876 return;
846877 }
847878 if (!seeded) seed();
@@ -849,7 +880,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
849880 if (capped !== text) machine.note("clamp", "speed cap", { percent, asked: values(text), sent: capped === null ? {} : values(capped) });
850881 if (capped === null) return;
851882 machine.note("sent", "follow frame", { values: values(capped) });
852 arm.send(stamped(capped));
883 arm.send(stamped(capped, up));
853884 });
854885 arm.on("close", (code, why) => browser.close(sendable(code) ? code : 1011, why.length ? why.toString() : "the arm server went away"));
855886 // A browser that closed the stream on purpose leaves the arm holding; one that vanished leaves it parking.
Software/so101/src/server.ts +17 −2
@@ -31,7 +31,7 @@
3131 * PUT /settings change any of those
3232 * POST /settings/table-here put the table (tableOffsetMm, -100..100) under the arm's lowest link now, the margin below it
3333 * GET /metrics jams, blocks, stalls, peak load and temperature per joint, recent events
34 * GET /precision the precision scorecard (src/precision.ts): tracking error, time to arrive, snap-backs by cause,
34 * GET /precision the precision scorecard (src/precision.ts, hops in src/hops.ts): tracking error, time to arrive, snap-backs by cause,
3535 * latency by hop, for the session being driven, the last 10 minutes, and the stored sessions and hours;
3636 * ?hours=1-720 hourly rows (48 by default)
3737 * GET /model.json the arm model: joints, the model's floor (floorZ) and the enforced table (tableZ), where the URDF is served
@@ -73,6 +73,7 @@ import { clamp, type Frame } from "./mapping.ts";
7373 import { MachineLog, perServo } from "./machine-log.ts";
7474 import { Metrics } from "./metrics.ts";
7575 import { ARRIVE_TICKS, type Cause, type Tracker } from "./precision.ts";
76 import { control, frameHops, pong } from "./hops.ts";
7677 import { PrecisionHistory } from "./precision-history.ts";
7778 import { MotionError, moveToPose, playTrajectory, recordTrajectory, STALL_SPAN_TICKS } from "./motion.ts";
7879 import { assertPoseName, parsePoseFile, serializePoseFile, type PoseFile } from "./poses.ts";
@@ -559,6 +560,7 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
559560
560561 const status = async () => {
561562 const positions = await bus.syncRead(ids, STS3215.Present_Position);
563 const readAt = Date.now();
562564 const loads = await bus.syncRead(ids, STS3215.Present_Load);
563565 const temps = await bus.syncRead(ids, STS3215.Present_Temperature);
564566 const torque = await bus.syncRead(ids, STS3215.Torque_Enable);
@@ -584,6 +586,8 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
584586 approach(zone);
585587 return {
586588 joints,
589 /** When the positions were read, ms on this server's clock: the relay aligns it for the read-back→browser hop (src/hops.ts). */
590 readAt,
587591 busy: driver !== null,
588592 driving: driver,
589593 recording: recording?.name ?? null,
@@ -1312,7 +1316,15 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
13121316 return end(ws, FLOOD_CLOSE, "too many frames: the stream was closed");
13131317 }
13141318 if (pace === "drop") return void stats.gated++;
1315 const verdict = admission.admit(String(data));
1319 const text = String(data);
1320 const received = Date.now();
1321 const message = control(text);
1322 if (message) {
1323 if (message.kind === "ping") ws.send(pong(message.id, received));
1324 else if (message.kind === "readback") tracker.hop("read-back→browser", received, message.ms);
1325 return;
1326 }
1327 const verdict = admission.admit(text);
13161328 if (!verdict.ok) {
13171329 if (verdict.drop !== "malformed") {
13181330 stats.gated++;
@@ -1322,6 +1334,9 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
13221334 return;
13231335 }
13241336 staling = false;
1337 const { up, relayToPi } = frameHops(text, received);
1338 if (up !== null) tracker.hop("browser→relay", received, up);
1339 if (relayToPi !== null) tracker.hop("relay→Pi", received, relayToPi);
13251340 const frame = verdict.joints;
13261341 clearTimeout(timer);
13271342 timer = setTimeout(() => void hold(), holdAfterMs);
Software/so101/src/ui/clock-sync.js added +39
@@ -0,0 +1,39 @@
1 // @ts-check
2 /**
3 * Two clocks aligned with an NTP-style exchange: the asker stamps t0 on sending, the answerer t1 on receiving and t2 on
4 * answering, the asker t3 on receiving the answer. The answerer's clock minus the asker's is ((t1-t0)+(t2-t3))/2, off by
5 * at most half the round trip's delay, (t3-t0)-(t2-t1); so the offset is taken from the quickest of the last few
6 * exchanges, the one a queue spoiled least. Pure: stamps in, offsets out.
7 */
8
9 /** How many exchanges are kept. */
10 export const KEEP = 8;
11
12 /** The offset (answerer minus asker) and the delay of one exchange, ms. @returns {{ offset: number, delay: number }} */
13 export function exchange(/** @type {number} */ t0, /** @type {number} */ t1, /** @type {number} */ t2, /** @type {number} */ t3) {
14 return { offset: (t1 - t0 + (t2 - t3)) / 2, delay: t3 - t0 - (t2 - t1) };
15 }
16
17 export class ClockSync {
18 constructor() {
19 /** @type {Array<{ offset: number, delay: number }>} */
20 this.recent = [];
21 }
22
23 /** Takes an exchange in; a malformed one, or one whose delay is negative (a clock stepped meanwhile), is ignored. */
24 add(/** @type {number} */ t0, /** @type {number} */ t1, /** @type {number} */ t2, /** @type {number} */ t3) {
25 if (![t0, t1, t2, t3].every(Number.isFinite)) return false;
26 const e = exchange(t0, t1, t2, t3);
27 if (e.delay < 0) return false;
28 this.recent.push(e);
29 if (this.recent.length > KEEP) this.recent.shift();
30 return true;
31 }
32
33 /** The answerer's clock minus the asker's, from the lowest-delay exchange; null before any. @returns {number | null} */
34 get offset() {
35 let best = null;
36 for (const e of this.recent) if (!best || e.delay < best.delay) best = e;
37 return best ? best.offset : null;
38 }
39 }
Software/so101/src/viewer.html +8
@@ -20,6 +20,7 @@
2020 <link rel="modulepreload" href="/ui/__VERSION__/camera-plan.js">
2121 <link rel="modulepreload" href="/ui/__VERSION__/drive.js">
2222 <link rel="modulepreload" href="/ui/__VERSION__/frames.js">
23 <link rel="modulepreload" href="/ui/__VERSION__/hop-link.js">
2324 <link rel="stylesheet" href="/ui/__VERSION__/tabs.css">
2425 <style>
2526 /* Phone portrait first: the view on top, a status strip with the E-stop under it, a sheet of tabs below that.
@@ -493,6 +494,7 @@
493494 import { cameraPlan, QUALITIES, samePlan } from "/ui/__VERSION__/camera-plan.js";
494495 import { bindTray } from "/ui/__VERSION__/drive.js";
495496 import { Frames } from "/ui/__VERSION__/frames.js";
497 import { HopLink } from "/ui/__VERSION__/hop-link.js";
496498 import { holdText, warnText } from "/ui/__VERSION__/hold-line.js";
497499 import { noteSink } from "/ui/__VERSION__/zone-notes.js";
498500 import { createRoom } from "/ui/__VERSION__/lobby-panel.js";
@@ -807,6 +809,8 @@
807809 if (ws?.readyState === 1) ws.send(data);
808810 else if (ws?.readyState === 0) stream.waiting = { data, at: Date.now() };
809811 }),
812 /** Aligns this clock with the relay's over the stream, for the latency hops in /precision (src/hops.ts). */
813 hops: new HopLink((data) => { if (stream.ws?.readyState === 1) stream.ws.send(data); }),
810814 /** STOP: drop what is pending, close, send nothing until a release or a resume. */
811815 halt() {
812816 clearTimeout(this.closeTimer);
@@ -825,12 +829,15 @@
825829 const ws = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/follow`);
826830 this.waiting = null;
827831 ws.onopen = () => {
832 if (this.ws === ws) this.hops.start();
828833 const w = this.waiting;
829834 this.waiting = null;
830835 if (this.ws === ws && w && Date.now() - w.at <= 200) ws.send(w.data);
831836 };
837 ws.onmessage = (e) => { if (typeof e.data === "string") this.hops.message(e.data); };
832838 ws.onclose = (e) => {
833839 if (this.ws !== ws) return;
840 this.hops.stop();
834841 this.ws = null;
835842 if (e.code === 1013 || e.code === 1000 || e.code >= 4000) { this.halted = true; msg(e.reason || "the arm is busy", true); }
836843 };
@@ -1499,6 +1506,7 @@
14991506 let started = false, early = null;
15001507 function applyStatus(d) {
15011508 if (!started) return void (early = d);
1509 if (stream.ws?.readyState === 1) stream.hops.readback(d.readAtRelay);
15021510 {
15031511 const was = $("why").textContent;
15041512 status = d;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs