Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
On a phone, one finger on a link of the 3D model drags that joint live; two fingers orbit and pinch zoom and send nothing #349
vesper/arm:vesper/275-on-a-phone-one-finger-on-a-link-of-the-3 → main 1 commit fork
6 files
changed +352
−14
Software/README.md +4 −1
| @@ -164,7 +164,10 @@ of silence (see **Safety limits** below). There is no authentication: keep it | ||
| 164 | 164 | on a LAN or behind Tailscale. |
| 165 | 165 | |
| 166 | 166 | `GET /` is the browser app: up to two camera panes, a live 3D model of the |
| 167 | arm (its own URDF and meshes, drag a joint and the arm follows), a slider and | |
| 167 | arm (its own URDF and meshes, drag a joint and the arm follows; on a phone one | |
| 168 | finger on a link drags that joint, two fingers turn the view and pinch zooms | |
| 169 | without moving the arm; the target is drawn solid, the real arm as a faint | |
| 170 | ghost until it gets there), a slider and | |
| 168 | 171 | hold-to-jog buttons per joint that stream targets like a leader does, a |
| 169 | 172 | button per saved pose plus save and delete, hold and release, record and |
| 170 | 173 | play, one STOP button (pressed, not released; space or Esc) with a separate |
Software/so101/src/ui/ghost.js added +45
| @@ -0,0 +1,45 @@ | ||
| 1 | /** | |
| 2 | * The real arm as a faint ghost beside the solid model, which shows the target. A copy of the loaded robot sharing its | |
| 3 | * meshes' geometry, drawn in one translucent material, never hit by a pick, and shown only while the two poses differ. | |
| 4 | * Served at /ui/ and imported by the tests. | |
| 5 | */ | |
| 6 | ||
| 7 | /** Radians two poses may differ by and still be drawn as one. */ | |
| 8 | export const APART = 0.02; | |
| 9 | ||
| 10 | /** | |
| 11 | * Whether any joint of `real` stands more than `eps` from the same joint of `shown`. | |
| 12 | * @param {Record<string, number>} shown @param {Record<string, number>} real @param {number} [eps] | |
| 13 | */ | |
| 14 | export function apart(shown, real, eps = APART) { | |
| 15 | for (const [name, angle] of Object.entries(real)) { | |
| 16 | const at = shown[name]; | |
| 17 | if (Number.isFinite(angle) && typeof at === "number" && Number.isFinite(at) && Math.abs(angle - at) > eps) return true; | |
| 18 | } | |
| 19 | return false; | |
| 20 | } | |
| 21 | ||
| 22 | /** | |
| 23 | * A ghost of `robot` (a URDFRobot), added beside it. `set` takes the real pose in radians by joint name. | |
| 24 | * @param {any} robot @param {any} THREE | |
| 25 | */ | |
| 26 | export function makeGhost(robot, THREE) { | |
| 27 | const ghost = robot.clone(); | |
| 28 | const look = new THREE.MeshBasicMaterial({ color: 0x9ecbff, transparent: true, opacity: 0.22, depthWrite: false }); | |
| 29 | ghost.traverse((/** @type {any} */ o) => { | |
| 30 | o.raycast = () => {}; | |
| 31 | if (o.isMesh) { o.material = look; o.castShadow = o.receiveShadow = false; o.renderOrder = 1; } | |
| 32 | }); | |
| 33 | for (const joint of Object.values(ghost.joints)) /** @type {any} */ (joint).ignoreLimits = true; | |
| 34 | ghost.visible = false; | |
| 35 | robot.parent.add(ghost); | |
| 36 | return { | |
| 37 | /** @param {Record<string, number>} real */ | |
| 38 | set(real) { | |
| 39 | ghost.position.copy(robot.position); ghost.quaternion.copy(robot.quaternion); | |
| 40 | for (const [name, angle] of Object.entries(real)) ghost.joints[name]?.setJointValue(angle); | |
| 41 | ghost.visible = apart(Object.fromEntries(Object.entries(robot.joints).map(([n, j]) => [n, /** @type {any} */ (j).angle])), real); | |
| 42 | }, | |
| 43 | hide() { ghost.visible = false; }, | |
| 44 | }; | |
| 45 | } |
Software/so101/src/ui/ghost.test.ts added +52
| @@ -0,0 +1,52 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { apart, makeGhost } from "./ghost.js"; | |
| 3 | ||
| 4 | describe("apart", () => { | |
| 5 | test("the ghost shows only while the target and the real pose differ", () => { | |
| 6 | expect(apart({ elbow_flex: 0.5 }, { elbow_flex: 0.5 })).toBe(false); | |
| 7 | expect(apart({ elbow_flex: 0.5 }, { elbow_flex: 0.51 })).toBe(false); | |
| 8 | expect(apart({ elbow_flex: 0.5, wrist_flex: 0 }, { elbow_flex: 0.5, wrist_flex: 0.3 })).toBe(true); | |
| 9 | expect(apart({ elbow_flex: 0.5 }, { gripper: 1, elbow_flex: Number.NaN })).toBe(false); // unknown joints and no reading are not a difference | |
| 10 | }); | |
| 11 | }); | |
| 12 | ||
| 13 | /** Just enough of a URDFRobot: two joints and a mesh, cloned the way three.js clones. */ | |
| 14 | function fakeRobot() { | |
| 15 | const make = (angle: number): any => { | |
| 16 | const joint = (a: number) => ({ angle: a, ignoreLimits: false, setJointValue(v: number) { this.angle = v; } }); | |
| 17 | const mesh: any = { isMesh: true, material: "solid", castShadow: true, receiveShadow: true, raycast: "hit" }; | |
| 18 | const robot: any = { | |
| 19 | joints: { elbow_flex: joint(angle), wrist_flex: joint(0) }, children: [mesh], visible: true, raycast: "hit", | |
| 20 | position: { copy() {} }, quaternion: { copy() {} }, | |
| 21 | traverse(fn: (o: any) => void) { fn(robot); for (const c of robot.children) fn(c); }, | |
| 22 | clone: () => make(angle), | |
| 23 | }; | |
| 24 | return robot; | |
| 25 | }; | |
| 26 | const robot = make(0.5); | |
| 27 | const added: any[] = []; | |
| 28 | robot.parent = { add: (o: any) => added.push(o) }; | |
| 29 | return { robot, added }; | |
| 30 | } | |
| 31 | const THREE = { MeshBasicMaterial: class { opts: any; constructor(opts: any) { this.opts = opts; } } }; | |
| 32 | ||
| 33 | describe("makeGhost", () => { | |
| 34 | test("a faint copy beside the solid model: translucent, never picked, at the real pose", () => { | |
| 35 | const { robot, added } = fakeRobot(); | |
| 36 | const ghost = makeGhost(robot, THREE); | |
| 37 | const copy = added[0]; | |
| 38 | expect(copy).not.toBe(robot); | |
| 39 | expect(copy.children[0].material.opts).toMatchObject({ transparent: true, depthWrite: false }); | |
| 40 | expect(copy.children[0].material.opts.opacity).toBeLessThan(0.5); | |
| 41 | expect(robot.children[0].material).toBe("solid"); // the target stays solid | |
| 42 | expect(typeof copy.raycast).toBe("function"); expect(copy.children[0].raycast()).toBeUndefined(); | |
| 43 | expect(robot.children[0].raycast).toBe("hit"); | |
| 44 | ghost.set({ elbow_flex: 1.2 }); | |
| 45 | expect(copy.joints.elbow_flex.angle).toBe(1.2); | |
| 46 | expect(copy.visible).toBe(true); | |
| 47 | ghost.set({ elbow_flex: 0.5 }); | |
| 48 | expect(copy.visible).toBe(false); // the arm reached the target: one model | |
| 49 | ghost.set({ elbow_flex: 1.2 }); ghost.hide(); | |
| 50 | expect(copy.visible).toBe(false); | |
| 51 | }); | |
| 52 | }); |
Software/so101/src/ui/model-touch.js added +107
| @@ -0,0 +1,107 @@ | ||
| 1 | /** | |
| 2 | * Fingers on the 3D model. One finger that lands on a link drags that joint; one that lands beside the arm turns the | |
| 3 | * view. Two fingers turn the view by their midpoint and zoom by their spread, and never move the arm: a second finger | |
| 4 | * ends a drag where the first one is. Plain data in (fingers as { x, y } client pixels), calls out; served at /ui/ and | |
| 5 | * imported by the tests. | |
| 6 | */ | |
| 7 | ||
| 8 | /** | |
| 9 | * @typedef {{ x: number, y: number }} Finger | |
| 10 | * @typedef {{ | |
| 11 | * grab(x: number, y: number): boolean, | |
| 12 | * drag(x: number, y: number): void, | |
| 13 | * drop(x: number, y: number): void, | |
| 14 | * view(dx: number, dy: number, zoom: number): void, | |
| 15 | * }} Hooks grab: take the joint under the finger, true when there was one; drag/drop: move it, let go of it; | |
| 16 | * view: turn the view by a drag of dx, dy pixels and zoom by a factor (above 1 is closer) | |
| 17 | */ | |
| 18 | ||
| 19 | const mid = (/** @type {Finger} */ a, /** @type {Finger} */ b) => ({ x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 }); | |
| 20 | const spread = (/** @type {Finger} */ a, /** @type {Finger} */ b) => Math.hypot(a.x - b.x, a.y - b.y); | |
| 21 | ||
| 22 | export class ModelTouch { | |
| 23 | /** @param {Hooks} hooks */ | |
| 24 | constructor(hooks) { | |
| 25 | this.hooks = hooks; | |
| 26 | /** @type {"idle" | "drag" | "turn" | "view"} drag: one finger on a joint; turn: one finger turning the view; view: two fingers */ | |
| 27 | this.mode = "idle"; | |
| 28 | /** @type {Finger} */ | |
| 29 | this.last = { x: 0, y: 0 }; | |
| 30 | this.gap = 0; | |
| 31 | } | |
| 32 | ||
| 33 | /** A finger landed. @param {Finger[]} fingers all the fingers now on the model */ | |
| 34 | down(fingers) { | |
| 35 | const [a, b] = fingers; | |
| 36 | if (a && b) this.(a, b); | |
| 37 | else if (a && this.mode === "idle") { | |
| 38 | this.last = a; | |
| 39 | this.mode = this.hooks.grab(a.x, a.y) ? "drag" : "turn"; | |
| 40 | } | |
| 41 | } | |
| 42 | ||
| 43 | /** Fingers moved. @param {Finger[]} fingers */ | |
| 44 | move(fingers) { | |
| 45 | const [a, b] = fingers; | |
| 46 | if (!a) return; | |
| 47 | if (this.mode === "view" && b) { | |
| 48 | const m = mid(a, b), d = spread(a, b); | |
| 49 | this.hooks.view(m.x - this.last.x, m.y - this.last.y, this.gap > 0 && d > 0 ? d / this.gap : 1); | |
| 50 | this.last = m; this.gap = d; | |
| 51 | } else if (this.mode === "drag") { | |
| 52 | this.last = a; | |
| 53 | this.hooks.drag(a.x, a.y); | |
| 54 | } else if (this.mode === "turn") { | |
| 55 | this.hooks.view(a.x - this.last.x, a.y - this.last.y, 1); | |
| 56 | this.last = a; | |
| 57 | } | |
| 58 | } | |
| 59 | ||
| 60 | /** Fingers lifted (or the browser cancelled them). @param {Finger[]} fingers the ones still down */ | |
| 61 | up(fingers) { | |
| 62 | const [a, b] = fingers; | |
| 63 | if (!a) { | |
| 64 | if (this.mode === "drag") this.hooks.drop(this.last.x, this.last.y); | |
| 65 | this.mode = "idle"; | |
| 66 | } else if (b) this.(a, b); | |
| 67 | else if (this.mode === "view") { this.mode = "turn"; this.last = a; } // one finger left: it turns the view, it never grabs | |
| 68 | } | |
| 69 | ||
| 70 | /** Two fingers down: a drag lets go where its finger was, and the view follows them. @param {Finger} a @param {Finger} b */ | |
| 71 | (a, b) { | |
| 72 | if (this.mode === "drag") this.hooks.drop(this.last.x, this.last.y); | |
| 73 | this.mode = "view"; | |
| 74 | this.last = mid(a, b); | |
| 75 | this.gap = spread(a, b); | |
| 76 | } | |
| 77 | } | |
| 78 | ||
| 79 | /** | |
| 80 | * Touches on `canvas` go to a ModelTouch, and touch pointers stop at `guard` (an ancestor, in the capture phase) so | |
| 81 | * the orbit controls under it see only the mouse. Returns the ModelTouch. | |
| 82 | * @param {any} canvas @param {any} guard @param {Hooks} hooks | |
| 83 | */ | |
| 84 | export function bindModelTouch(canvas, guard, hooks) { | |
| 85 | const touch = new ModelTouch(hooks); | |
| 86 | const at = (/** @type {any} */ list) => Array.from(list, (/** @type {any} */ t) => ({ x: t.clientX, y: t.clientY })); | |
| 87 | for (const type of ["pointerdown", "pointermove", "pointerup", "pointercancel"]) guard.addEventListener(type, (/** @type {any} */ e) => { if (e.pointerType === "touch" && e.target === canvas) e.stopPropagation(); }, true); | |
| 88 | // Not passive: a cancelled touchstart also cancels the emulated mouse events a tap would fire afterwards. | |
| 89 | canvas.addEventListener("touchstart", (/** @type {any} */ e) => { e.preventDefault(); touch.down(at(e.touches)); }, { passive: false }); | |
| 90 | canvas.addEventListener("touchmove", (/** @type {any} */ e) => { e.preventDefault(); touch.move(at(e.touches)); }, { passive: false }); | |
| 91 | for (const type of ["touchend", "touchcancel"]) canvas.addEventListener(type, (/** @type {any} */ e) => touch.up(at(e.touches)), { passive: true }); | |
| 92 | return touch; | |
| 93 | } | |
| 94 | ||
| 95 | /** | |
| 96 | * The camera's offset from what it looks at, turned by a drag of dx, dy pixels on a view `height` pixels tall (a full | |
| 97 | * height is a full turn, as the orbit controls do it) and brought `zoom` times closer, kept within [near, far]. +Y is up. | |
| 98 | * @param {{ x: number, y: number, z: number }} offset @param {number} dx @param {number} dy @param {number} height | |
| 99 | * @param {number} zoom @param {number} near @param {number} far | |
| 100 | */ | |
| 101 | export function orbit(offset, dx, dy, height, zoom, near, far) { | |
| 102 | const r = Math.hypot(offset.x, offset.y, offset.z) || near; | |
| 103 | const theta = Math.atan2(offset.x, offset.z) - (2 * Math.PI * dx) / height; | |
| 104 | const phi = Math.min(Math.PI - 1e-3, Math.max(1e-3, Math.acos(Math.min(1, Math.max(-1, offset.y / r))) - (2 * Math.PI * dy) / height)); | |
| 105 | const to = Math.min(far, Math.max(near, r / (zoom > 0 && Number.isFinite(zoom) ? zoom : 1))); | |
| 106 | return { x: to * Math.sin(phi) * Math.sin(theta), y: to * Math.cos(phi), z: to * Math.sin(phi) * Math.cos(theta) }; | |
| 107 | } |
Software/so101/src/ui/model-touch.test.ts added +116
| @@ -0,0 +1,116 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { Frames } from "./frames.js"; | |
| 3 | import { bindModelTouch, ModelTouch, orbit } from "./model-touch.js"; | |
| 4 | ||
| 5 | /** | |
| 6 | * The page's wiring with a fake manipulator: the elbow (servo 3) is the link under x 100-200, and dragging it maps the | |
| 7 | * finger's y to ticks. Targets go out through the real latest-wins Frames, one slot at a time. | |
| 8 | */ | |
| 9 | function rig() { | |
| 10 | const wire: Record<string, number>[] = []; | |
| 11 | const slots: (() => void)[] = []; | |
| 12 | const frames = new Frames((data) => wire.push(JSON.parse(data)), { later: (fn) => slots.push(fn), cancel: () => {}, now: () => 0 }); | |
| 13 | const views: [number, number, number][] = []; | |
| 14 | let held: number | null = null; | |
| 15 | const touch = new ModelTouch({ | |
| 16 | grab: (x) => { held = x >= 100 && x <= 200 ? 3 : null; return held !== null; }, | |
| 17 | drag: (_, y) => { if (held !== null) frames.send({ [held]: 2048 + Math.round(y) }); }, | |
| 18 | drop: () => { held = null; }, | |
| 19 | view: (dx, dy, zoom) => views.push([dx, dy, zoom]), | |
| 20 | }); | |
| 21 | const tick = () => { for (const fn of slots.splice(0)) fn(); }; | |
| 22 | return { touch, wire, views, tick, held: () => held }; | |
| 23 | } | |
| 24 | const joints = (frame: Record<string, number>) => Object.keys(frame).filter((k) => k !== "seq" && k !== "t"); | |
| 25 | ||
| 26 | describe("one finger", () => { | |
| 27 | test("on a link, a drag streams that joint only, newest target winning each slot, and moves no view", () => { | |
| 28 | const r = rig(); | |
| 29 | r.touch.down([{ x: 150, y: 0 }]); | |
| 30 | for (let y = 1; y <= 30; y++) { r.touch.move([{ x: 150, y }]); if (y % 10 === 0) r.tick(); } | |
| 31 | r.touch.up([]); | |
| 32 | expect(r.wire.length).toBe(3); | |
| 33 | for (const f of r.wire) expect(joints(f)).toEqual(["3"]); | |
| 34 | expect(r.wire.map((f) => f[3])).toEqual([2058, 2068, 2078]); | |
| 35 | expect(r.views).toEqual([]); | |
| 36 | expect(r.held()).toBeNull(); | |
| 37 | }); | |
| 38 | test("beside the arm, it turns the view and sends nothing", () => { | |
| 39 | const r = rig(); | |
| 40 | r.touch.down([{ x: 10, y: 10 }]); | |
| 41 | r.touch.move([{ x: 20, y: 15 }]); | |
| 42 | r.touch.up([]); r.tick(); | |
| 43 | expect(r.wire).toEqual([]); | |
| 44 | expect(r.views).toEqual([[10, 5, 1]]); | |
| 45 | }); | |
| 46 | }); | |
| 47 | ||
| 48 | describe("two fingers", () => { | |
| 49 | test("a two-finger drag orbits and a pinch zooms, and neither sends a command", () => { | |
| 50 | const r = rig(); | |
| 51 | r.touch.down([{ x: 150, y: 0 }]); // the first finger lands on the elbow… | |
| 52 | r.touch.down([{ x: 150, y: 0 }, { x: 250, y: 0 }]); // …the second makes it a view gesture before anything moved | |
| 53 | r.touch.move([{ x: 160, y: 10 }, { x: 260, y: 10 }]); // both slide: an orbit by the midpoint | |
| 54 | r.touch.move([{ x: 110, y: 10 }, { x: 310, y: 10 }]); // they spread: twice as far apart is twice as close | |
| 55 | r.touch.up([{ x: 310, y: 10 }]); | |
| 56 | r.touch.move([{ x: 320, y: 10 }]); // the finger left behind turns the view, it never grabs | |
| 57 | r.touch.up([]); r.tick(); | |
| 58 | expect(r.wire).toEqual([]); | |
| 59 | expect(r.views).toEqual([[10, 10, 1], [0, 0, 2], [10, 0, 1]]); | |
| 60 | }); | |
| 61 | test("a second finger mid-drag lets go of the joint: nothing more is sent for it", () => { | |
| 62 | const r = rig(); | |
| 63 | r.touch.down([{ x: 150, y: 0 }]); | |
| 64 | r.touch.move([{ x: 150, y: 5 }]); r.tick(); | |
| 65 | expect(r.wire.length).toBe(1); | |
| 66 | r.touch.down([{ x: 150, y: 5 }, { x: 250, y: 5 }]); | |
| 67 | expect(r.held()).toBeNull(); | |
| 68 | r.touch.move([{ x: 150, y: 40 }, { x: 250, y: 40 }]); | |
| 69 | r.touch.up([{ x: 150, y: 60 }]); | |
| 70 | r.touch.move([{ x: 150, y: 80 }]); | |
| 71 | r.touch.up([]); r.tick(); | |
| 72 | expect(r.wire.length).toBe(1); | |
| 73 | }); | |
| 74 | }); | |
| 75 | ||
| 76 | describe("bindModelTouch", () => { | |
| 77 | test("touches reach the gesture, touch pointers stop before the orbit controls, the mouse does not", () => { | |
| 78 | const canvas = new EventTarget(), guard = new EventTarget(); | |
| 79 | const grabbed: number[] = []; | |
| 80 | bindModelTouch(canvas, guard, { grab: (x) => (grabbed.push(x), false), drag() {}, drop() {}, view() {} }); | |
| 81 | const ev = (type: string, touches: { clientX: number, clientY: number }[]) => Object.assign(new Event(type, { cancelable: true }), { touches }); | |
| 82 | const start = ev("touchstart", [{ clientX: 7, clientY: 8 }]); | |
| 83 | canvas.dispatchEvent(start); | |
| 84 | expect(grabbed).toEqual([7]); | |
| 85 | expect(start.defaultPrevented).toBe(true); // no emulated mouse events after the tap | |
| 86 | const stopped = (pointerType: string) => { | |
| 87 | let stop = false; | |
| 88 | const e = Object.assign(new Event("pointerdown"), { pointerType }); | |
| 89 | Object.defineProperty(e, "target", { value: canvas }); | |
| 90 | e.stopPropagation = () => { stop = true; }; | |
| 91 | guard.dispatchEvent(e); | |
| 92 | return stop; | |
| 93 | }; | |
| 94 | expect(stopped("touch")).toBe(true); | |
| 95 | expect(stopped("mouse")).toBe(false); | |
| 96 | }); | |
| 97 | }); | |
| 98 | ||
| 99 | describe("orbit", () => { | |
| 100 | const len = (o: { x: number, y: number, z: number }) => Math.hypot(o.x, o.y, o.z); | |
| 101 | test("a drag across the whole height is a full turn; a pinch to twice the spread halves the distance", () => { | |
| 102 | const o = { x: 0, y: 0, z: 1 }; | |
| 103 | const turned = orbit(o, 100, 0, 100, 1, 0.25, 50); | |
| 104 | expect(turned.x).toBeCloseTo(0); expect(turned.y).toBeCloseTo(0); expect(turned.z).toBeCloseTo(1); | |
| 105 | expect(orbit(o, 25, 0, 100, 1, 0.25, 50).x).toBeCloseTo(-1); | |
| 106 | expect(len(orbit(o, 0, 0, 100, 2, 0.25, 50))).toBeCloseTo(0.5); | |
| 107 | }); | |
| 108 | test("never zooms past its limits nor flips over the pole", () => { | |
| 109 | const o = { x: 0.5, y: 0.38, z: 0.62 }; | |
| 110 | expect(len(orbit(o, 0, 0, 100, 1000, 0.25, 50))).toBeCloseTo(0.25); | |
| 111 | expect(len(orbit(o, 0, 0, 100, 0.0001, 0.25, 50))).toBeCloseTo(50); | |
| 112 | const top = orbit(o, 0, 1000, 100, 1, 0.25, 50); | |
| 113 | expect(top.y).toBeGreaterThan(0.99 * len(o)); | |
| 114 | expect(len(orbit(o, 0, 0, 100, Number.NaN, 0.25, 50))).toBeCloseTo(len(o)); | |
| 115 | }); | |
| 116 | }); |
Software/so101/src/viewer.html +28 −13
| @@ -271,7 +271,7 @@ | ||
| 271 | 271 | <div id="shell"> |
| 272 | 272 | <div id="app"> |
| 273 | 273 | <div id="stage"> |
| 274 | <div class="pane" id="pane3" data-key="pane3"><urdf-viewer id="model" up="Z" ignore-limits display-shadow ambient-color="#b9c0c8"></urdf-viewer><span class="tag" id="tag3">3D · drag a joint to move it · drag around it to orbit</span></div> | |
| 274 | <div class="pane" id="pane3" data-key="pane3"><urdf-viewer id="model" up="Z" ignore-limits display-shadow ambient-color="#b9c0c8"></urdf-viewer><span class="tag" id="tag3">3D · drag a joint to move it · drag beside it or with two fingers to turn · pinch to zoom</span></div> | |
| 275 | 275 | </div> |
| 276 | 276 | <div id="top"> |
| 277 | 277 | <span class="seg" role="group" aria-label="Views"><button id="v3" aria-pressed="false">3D</button></span> |
| @@ -1000,7 +1000,7 @@ | ||
| 1000 | 1000 | $("leaderDrive").textContent = on ? "■ Teleop off" : "Teleop"; |
| 1001 | 1001 | $("leaderDrive").setAttribute("aria-pressed", String(on)); |
| 1002 | 1002 | $("teleopLive").hidden = !leaderStreaming(); |
| 1003 | $("tag3").textContent = leaderStreaming() ? "3D · the arm, following your leader" : leader ? "3D · mirroring your leader arm" : "3D · drag a joint to move it · drag around it to orbit"; | |
| 1003 | $("tag3").textContent = leaderStreaming() ? "3D · the arm, following your leader" : leader ? "3D · mirroring your leader arm" : "3D · drag a joint to move it · drag beside it or with two fingers to turn · pinch to zoom"; | |
| 1004 | 1004 | renderLag(); |
| 1005 | 1005 | } |
| 1006 | 1006 | /** Leader→follower lag as this browser measures it (src/teleop.ts): the follower's reported pose against the leader readings sent. */ |
| @@ -1126,25 +1126,35 @@ | ||
| 1126 | 1126 | // ---------------------------------------------------------------- the 3D model |
| 1127 | 1127 | // three.js is most of what the page downloads, so it comes after the first view: the room and the camera work without it. |
| 1128 | 1128 | const viewer = $("model"); |
| 1129 | let THREE = null, model3D = null, dragging = null, table = null, grid = null, marker = null, tablePlaneZ = null; | |
| 1129 | let THREE = null, model3D = null, dragging = null, table = null, grid = null, marker = null, tablePlaneZ = null, makeGhost = null, ghost = null; | |
| 1130 | 1130 | function load3D() { |
| 1131 | 1131 | if (!modelInfo?.ok) return Promise.resolve(false); |
| 1132 | 1132 | return (model3D ??= (async () => { |
| 1133 | const [three, { STLLoader }, { default: URDFManipulator }, plane] = await Promise.all([ | |
| 1133 | const [three, { STLLoader }, { default: URDFManipulator }, plane, fingers, ghosts] = await Promise.all([ | |
| 1134 | 1134 | import("three"), import("three/examples/jsm/loaders/STLLoader.js"), import("urdf-loader/manipulator"), import("/ui/__VERSION__/table.js"), |
| 1135 | import("/ui/__VERSION__/model-touch.js"), import("/ui/__VERSION__/ghost.js"), | |
| 1135 | 1136 | ]); |
| 1136 | THREE = three; tablePlaneZ = plane.tablePlaneZ; | |
| 1137 | THREE = three; tablePlaneZ = plane.tablePlaneZ; makeGhost = ghosts.makeGhost; | |
| 1137 | 1138 | if (!customElements.get("urdf-viewer")) customElements.define("urdf-viewer", URDFManipulator); |
| 1138 | 1139 | viewer.loadMeshFunc = (path, manager, material, done) => { |
| 1139 | 1140 | new STLLoader(manager).load(path, (geometry) => done(new THREE.Mesh(geometry, material ?? new THREE.MeshPhongMaterial({ color: 0x9aa4ae, shininess: 30 }))), undefined, (err) => done(null, err)); |
| 1140 | 1141 | }; |
| 1141 | 1142 | const canvas = viewer.renderer.domElement; |
| 1142 | 1143 | canvas.style.touchAction = "none"; |
| 1143 | const asMouse = (type, t) => canvas.dispatchEvent(new MouseEvent(type, { clientX: t.clientX, clientY: t.clientY, bubbles: true })); | |
| 1144 | canvas.addEventListener("touchstart", (e) => { if (e.touches.length === 1) asMouse("mousedown", e.touches[0]); }, { passive: true }); | |
| 1145 | canvas.addEventListener("touchmove", (e) => { if (e.touches.length === 1 && dragging) { e.preventDefault(); asMouse("mousemove", e.touches[0]); } }, { passive: false }); | |
| 1146 | canvas.addEventListener("touchend", (e) => { if (dragging) asMouse("mouseup", e.changedTouches[0]); }, { passive: true }); | |
| 1147 | canvas.addEventListener("touchcancel", (e) => { if (dragging) asMouse("mouseup", e.changedTouches[0]); }, { passive: true }); | |
| 1144 | // Fingers (src/ui/model-touch.js): one on a link drags its joint through the manipulator's mouse handlers; | |
| 1145 | // two, or one beside the arm, turn and zoom the view and send nothing. | |
| 1146 | const asMouse = (type, x, y) => canvas.dispatchEvent(new MouseEvent(type, { clientX: x, clientY: y, bubbles: true })); | |
| 1147 | fingers.bindModelTouch(canvas, $("pane3"), { | |
| 1148 | grab: (x, y) => { if (viewer.hasAttribute("disable-dragging")) return false; asMouse("mousedown", x, y); return dragging !== null; }, | |
| 1149 | drag: (x, y) => asMouse("mousemove", x, y), | |
| 1150 | drop: (x, y) => { asMouse("mouseup", x, y); asMouse("mousemove", -1e5, -1e5); }, // and off the link, so its highlight goes | |
| 1151 | view: (dx, dy, zoom) => { | |
| 1152 | const c = viewer.camera, t = viewer.controls.target; | |
| 1153 | const o = fingers.orbit(c.position.clone().sub(t), dx, dy, canvas.clientHeight || 1, zoom, viewer.controls.minDistance, viewer.controls.maxDistance); | |
| 1154 | c.position.set(t.x + o.x, t.y + o.y, t.z + o.z); | |
| 1155 | viewer.controls.update(); viewer.redraw(); | |
| 1156 | }, | |
| 1157 | }); | |
| 1148 | 1158 | applyLock(); |
| 1149 | 1159 | viewer.urdf = modelInfo.urdf; |
| 1150 | 1160 | return true; |
| @@ -1159,6 +1169,7 @@ | ||
| 1159 | 1169 | for (const o of [table, grid, marker]) o.raycast = () => {}; |
| 1160 | 1170 | viewer.world.add(table, grid, marker); |
| 1161 | 1171 | viewer.plane.visible = false; // the element's own shadow plane sits under the lowest part, which would be below the table |
| 1172 | ghost = makeGhost(viewer.robot, THREE); // the real arm, faint, while the solid model shows the target (src/ui/ghost.js) | |
| 1162 | 1173 | if (status) applyModel(status); |
| 1163 | 1174 | viewer.camera.position.set(0.5, 0.38, 0.62); |
| 1164 | 1175 | viewer.controls.target.set(0.08, 0.1, 0); |
| @@ -1183,13 +1194,17 @@ | ||
| 1183 | 1194 | } |
| 1184 | 1195 | function applyModel(d) { |
| 1185 | 1196 | placeTable(d); |
| 1186 | if (!viewer.robot || calib) return; // while calibrating, the model shows the pose to copy | |
| 1187 | for (const j of leaderMirrored() ? [] : d.joints) { | |
| 1197 | if (!viewer.robot || calib) return void ghost?.hide(); // while calibrating, the model shows the pose to copy | |
| 1198 | const real = {}; | |
| 1199 | for (const j of d.joints) { | |
| 1188 | 1200 | const name = NAMES[j.id]; |
| 1189 | if (!name || j.position == null || name === dragging || !limits[j.id]) continue; | |
| 1201 | if (!name || j.position == null || !limits[j.id]) continue; | |
| 1202 | real[name] = toRad(j.id, j.position); | |
| 1203 | if (name === dragging || leaderMirrored()) continue; | |
| 1190 | 1204 | const want = intent.target(j.id, j.position, performance.now()); |
| 1191 | 1205 | viewer.setJointValue(name, toRad(j.id, want ?? j.position)); |
| 1192 | 1206 | } |
| 1207 | ghost?.set(real); | |
| 1193 | 1208 | // A stream frame the zone refuses shows as the status's `blocked` in enforce mode; the next clear status ends it. |
| 1194 | 1209 | if (d.blocked && d.zone === "enforce") refused({ error: `${sayViolation(d.blocked)}; back the leader off` }); |
| 1195 | 1210 | else if (!d.blocked && $("refusal").dataset.stream) accepted(); |
new above
offline