Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

bench/limits-sweep.ts drives a fixed per-joint sweep and prints the /precision scorecard before and after, against the fake bus or a given arm URL #487

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/478-bench-limits-sweep-ts-drives-a-fixed-per → main 3 commits fork

  • Merged. smith/478-bench-limits-sweep-ts-drives-a-fixed-per was merged into main as 89c7699.

Review

Sign in to review this change.
5 files changed +162 −8
Software/README.md +3
@@ -1089,6 +1089,9 @@ Where to read it:
10891089 `/metrics`): unauthenticated, for the crew's scripts on that network. It holds timing and position aggregates
10901090 only: no names, tokens or commands.
10911091
1092 `bun Software/so101/bench/limits-sweep.ts [url]` drives each joint in turn through a fixed set of targets inside its bounds (the others held still) and prints that joint's p95 error, time to arrive, overshoot and snap-backs from `/precision`; with no url it runs on the fake bus, with a url it refuses to start unless nobody is driving and stops at once on STOP or any refusal.
1093 The sweep's targets are fixed; the measured numbers move a little with the machine's timers.
1094
10921095 `bun bench/precision-report.ts <url>` (the arm server's URL) reads the hourly roll-up from `GET /precision` and prints
10931096 the weekly comment for #439: each headline number this week against last, the worst joint and the open experiment. It
10941097 only reads; the crew posts it. The experiment is the line below, which the crew edits:
Software/so101/bench/limits-sweep.test.ts added +38
@@ -0,0 +1,38 @@
1 import { expect, test } from "bun:test";
2 import { fakeArm } from "./precision.ts";
3 import { sweep, targets } from "./limits-sweep.ts";
4
5 test("targets stay inside the bounds and start and end at the middle", () => {
6 expect(targets(1000, 3000)).toEqual([2000, 2250, 2000, 1750, 2000]);
7 expect(targets(1000, 1100).every((t) => t >= 1000 && t <= 1100)).toBe(true);
8 });
9
10 test("on the fake bus there is one line per joint, the same every run", async () => {
11 const run = async () => {
12 const arm = await fakeArm();
13 try {
14 return await sweep(arm.url, 350);
15 } finally {
16 await arm.close();
17 }
18 };
19 const a = await run();
20 expect(a).toHaveLength(6);
21 expect(a[0]).toStartWith("shoulder_pan");
22 expect(a.every((l) => l.includes("snap-backs 0"))).toBe(true);
23 // The targets and the layout are fixed; the measured numbers move a little with the machine's timers, so compare the shape.
24 const shape = (ls: string[]) => ls.map((l) => l.replace(/[\d.]+|—/g, "#").replace(/ +/g, " "));
25 expect(shape(await run())).toEqual(shape(a));
26 }, 30_000);
27
28 test("it refuses to start while someone drives", async () => {
29 const arm = await fakeArm();
30 try {
31 const ws = new (await import("ws")).default(`${arm.url.replace("http", "ws")}/follow`);
32 await new Promise((r) => ws.once("open", r));
33 await expect(sweep(arm.url, 50)).rejects.toThrow("is driving the arm");
34 ws.close(1000);
35 } finally {
36 await arm.close();
37 }
38 }, 10_000);
Software/so101/bench/limits-sweep.ts added +99
@@ -0,0 +1,99 @@
1 /**
2 * A fixed sweep of each joint in turn, then that joint's line from /precision.
3 * bun bench/limits-sweep.ts on the fake bus with the real Governor, as bench/precision.ts does
4 * bun bench/limits-sweep.ts <url> on a given arm server (http://host:8101); it refuses to start unless /status shows
5 * nobody driving, and stops at once on STOP, an E-stop, a closed stream or any refusal
6 * The targets, per joint, as ticks from the middle of the joint's bounds (lo..hi from /status), the others held still:
7 * 0, +250, 0, -250, 0, each held 1.5 s. A step is under the server's 400-tick maxStep; one that would leave the bounds
8 * is pulled back inside. The scorecard window is the server's own (48 h), so on a live arm the line also holds earlier
9 * sessions of that joint; on the fake bus it is this run alone.
10 */
11 import WebSocket from "ws";
12 import type { Card, JointCard } from "../src/precision.ts";
13 import type { Scorecard } from "../src/precision-history.ts";
14 import { fakeArm } from "./precision.ts";
15
16 export const OFFSETS = [0, 250, 0, -250, 0] as const;
17 const HOLD_MS = 1500;
18
19 interface Status {
20 joints: Array<{ id: number; joint: string | null; lo: number | null; hi: number | null }>;
21 driving: string | null;
22 held: unknown;
23 estop: unknown;
24 }
25
26 const get = async <T>(url: string, path: string): Promise<T> => {
27 const res = await fetch(url + path);
28 if (!res.ok) throw new Error(`${path}: ${res.status}`);
29 return (await res.json()) as T;
30 };
31 const card = async (url: string): Promise<Card> => (await get<Scorecard>(url, "/precision")).window;
32 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
33 const stopped = (s: Status) => (s.estop ? "emergency stop" : s.held ? "STOP" : null);
34
35 /** The joint's targets: the offsets from the middle of its bounds, kept inside them. */
36 export function targets(lo: number, hi: number): number[] {
37 const mid = Math.round((lo + hi) / 2);
38 return OFFSETS.map((o) => Math.min(hi, Math.max(lo, mid + o)));
39 }
40
41 const fmt = (v: number | null, unit = "") => (v === null ? "—" : `${v}${unit}`);
42 export function line(name: string, j: JointCard | undefined, snaps: number): string {
43 return `${name.padEnd(14)} p95 error ${fmt(j?.errorDeg.p95 ?? null, "°").padEnd(8)} arrive ${fmt(j?.arriveMs.p95 ?? null, " ms").padEnd(8)} overshoot ${fmt(j?.overshootTicks.max ?? null, " ticks").padEnd(10)} snap-backs ${snaps}`;
44 }
45
46 /** Drives every joint through its targets and answers the lines, one per joint. Throws, naming why, where it must stop. */
47 export async function sweep(url: string, hold = HOLD_MS, out: (s: string) => void = () => {}): Promise<string[]> {
48 const status = await get<Status>(url, "/status");
49 if (status.driving) throw new Error(`refusing to start: ${status.driving} is driving the arm`);
50 const why = stopped(status);
51 if (why) throw new Error(`refusing to start: ${why} is in force`);
52 const before = await card(url);
53 out(`before: p95 error ${fmt(before.trackingP95Deg, "°")}, arrive ${fmt(before.arriveP95Ms, " ms")}, snap-backs ${before.snapBacks.total}`);
54 const lines: string[] = [];
55 let total = before.snapBacks.total;
56 for (const joint of status.joints) {
57 if (joint.lo === null || joint.hi === null) throw new Error(`joint ${joint.id} has no bounds`);
58 const ws = new WebSocket(`${url.replace(/^http/, "ws")}/follow`);
59 let closed: number | null = null;
60 ws.on("close", (code) => (closed = code));
61 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j), ws.once("close", (c) => j(new Error(`the stream was refused (${c})`)))));
62 try {
63 for (const t of targets(joint.lo, joint.hi)) {
64 ws.send(JSON.stringify({ [joint.id]: t }));
65 await sleep(hold);
66 const now = await get<Status>(url, "/status");
67 const halt = stopped(now) ?? (closed !== null ? `the stream closed (${closed})` : null);
68 if (halt) throw new Error(`stopped at ${joint.joint ?? joint.id}: ${halt}`);
69 const snaps = (await card(url)).snapBacks.total;
70 if (snaps > total) throw new Error(`stopped at ${joint.joint ?? joint.id}: the arm refused or held back a target (${snaps - total} snap-back)`);
71 }
72 } finally {
73 ws.close(1000);
74 await new Promise((r) => (ws.readyState === WebSocket.CLOSED ? r(null) : ws.once("close", r)));
75 }
76 await sleep(150);
77 const after = await card(url);
78 const l = line(joint.joint ?? String(joint.id), after.joints.find((j) => j.id === joint.id), after.snapBacks.total - total);
79 total = after.snapBacks.total;
80 out(l);
81 lines.push(l);
82 }
83 const end = await card(url);
84 out(`after: p95 error ${fmt(end.trackingP95Deg, "°")}, arrive ${fmt(end.arriveP95Ms, " ms")}, snap-backs ${end.snapBacks.total}`);
85 return lines;
86 }
87
88 if (import.meta.main) {
89 const given = process.argv[2];
90 const arm = given ? { url: given.replace(/\/$/, ""), close: async () => {} } : await fakeArm();
91 try {
92 await sweep(arm.url, HOLD_MS, console.log);
93 } catch (err) {
94 console.error((err as Error).message);
95 process.exitCode = 1;
96 } finally {
97 await arm.close();
98 }
99 }
Software/so101/bench/precision.ts +17 −6
@@ -49,14 +49,26 @@ export function gesture(): Step[] {
4949 return steps;
5050 }
5151
52 /** Runs the script against a fresh arm server and answers its scorecard: the session just driven. */
53 export async function bench(script: readonly Step[] = gesture()): Promise<Card> {
52 /** An arm server on the fake bus with the real Governor and servos that follow at SERVO_TICKS_S: its base url, and how to end it. */
53 export async function fakeArm(): Promise<{ url: string; close: () => Promise<void> }> {
5454 const fake = new FakeBus([1, 2, 3, 4, 5, 6], { speedTicksS: SERVO_TICKS_S });
5555 for (const [id, [min, max]] of Object.entries(RANGES)) fake.servos.get(Number(id))!.set([min & 0xff, min >> 8, max & 0xff, max >> 8], 9);
5656 const dir = mkdtempSync(join(tmpdir(), "so101-precision-bench-"));
5757 const server = await startServer(new FeetechBus(fake), { host: "127.0.0.1", port: 0, posesFile: join(dir, "poses.json"), modelDir: "/nowhere", commit: "bench", jamSampleMs: 20 });
58 return {
59 url: `http://127.0.0.1:${server.port}`,
60 close: async () => {
61 await Promise.race([server.close(), new Promise((r) => setTimeout(r, 500))]);
62 rmSync(dir, { recursive: true, force: true });
63 },
64 };
65 }
66
67 /** Runs the script against a fresh arm server and answers its scorecard: the session just driven. */
68 export async function bench(script: readonly Step[] = gesture()): Promise<Card> {
69 const arm = await fakeArm();
5870 try {
59 const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`);
71 const ws = new WebSocket(`${arm.url.replace("http", "ws")}/follow`);
6072 await new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
6173 const start = performance.now();
6274 for (const step of script) {
@@ -68,11 +80,10 @@ export async function bench(script: readonly Step[] = gesture()): Promise<Card>
6880 ws.close(1000);
6981 await new Promise((r) => ws.once("close", r));
7082 await new Promise((r) => setTimeout(r, 100));
71 const card = (await (await fetch(`http://127.0.0.1:${server.port}/precision`)).json()) as Scorecard;
83 const card = (await (await fetch(`${arm.url}/precision`)).json()) as Scorecard;
7284 return card.window;
7385 } finally {
74 await Promise.race([server.close(), new Promise((r) => setTimeout(r, 500))]);
75 rmSync(dir, { recursive: true, force: true });
86 await arm.close();
7687 }
7788 }
7889
Software/so101/src/release.test.ts +5 −2
@@ -12,11 +12,12 @@ import { Frames } from "./ui/frames.js";
1212 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
1313 const START = 2048;
1414 const DEG40 = Math.round((40 * 4096) / 360);
15 let speed = 2000;
1516 let fake: FakeBus;
1617 let server: RunningServer;
1718
1819 beforeEach(async () => {
19 fake = new FakeBus([1], { speedTicksS: 2000 });
20 fake = new FakeBus([1], { speedTicksS: speed });
2021 fake.servos.get(1)!.set([0x00, 0x00, 0xff, 0x0f], 9); // limits 0-4095
2122 server = await startServer(new FeetechBus(fake), { host: "127.0.0.1", port: 0, ids: [1], posesFile: join(mkdtempSync(join(tmpdir(), "so101-release-")), "poses.json"), modelDir: "/nowhere" });
2223 });
@@ -39,6 +40,7 @@ const drag = async (ws: WebSocket, to: number) => {
3940
4041 describe("a release lands on the target", () => {
4142 test("a 40° drag released at once: the arm arrives, no goal is parked short, the goals only go toward the target", async () => {
43 speed = 800; // slow enough that a loaded machine's slow drag still ends with the arm on its way
4244 const ws = await open();
4345 await sleep(100); // armed
4446 const target = START + DEG40;
@@ -46,7 +48,8 @@ describe("a release lands on the target", () => {
4648 const mark = fake.writes.length;
4749 ws.send(JSON.stringify({ 1: target, type: "release" }));
4850 ws.close(1000);
49 await sleep(1500);
51 await sleep(2500);
52 speed = 2000;
5053 const g = goals(mark);
5154 expect(Math.abs((await present()) - target)).toBeLessThanOrEqual(12);
5255 expect(Math.abs(g.at(-1)! - target)).toBeLessThanOrEqual(12); // the hold writes present, which is there

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs