Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

bench/jam-rate.ts <url> reports jams per hour from the arm's /metrics, so #509's before-and-after can be measured on the real arm #542

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/537-bench-jam-rate-ts-url-reports-jams-per-h → main 1 commit fork

  • Merged. smith/537-bench-jam-rate-ts-url-reports-jams-per-h was merged into main as 59e8d63.

Review

Sign in to review this change.
3 files changed +151 −0
Software/README.md +2
@@ -1105,6 +1105,8 @@ Where to read it:
11051105 `bun Software/so101/bench/limits-sweep.ts [url]` drives each joint in turn through a fixed set of targets inside its bounds (the others held still) and prints that joint's p95 error, time to arrive, overshoot and snap-backs from `/precision`; with no url it runs on the fake bus, with a url it refuses to start unless nobody is driving and stops at once on STOP or any refusal.
11061106 The sweep's targets are fixed; the measured numbers move a little with the machine's timers.
11071107
1108 `bun run Software/so101/bench/jam-rate.ts <url> [--minutes N] [--events]` reads `/metrics` at the start and end of the window (default 60 min) and prints jams per hour, by joint, and each jam's load, ms and progress; `--events` reports from the recent events alone. It only reads.
1109
11081110 `bun bench/precision-report.ts <url>` (the arm server's URL) reads the hourly roll-up from `GET /precision` and prints
11091111 the weekly comment for #439: each headline number this week against last, the worst joint and the open experiment. It
11101112 only reads; the crew posts it. The experiment is the line below, which the crew edits:
Software/so101/bench/jam-rate.test.ts added +61
@@ -0,0 +1,61 @@
1 import { afterEach, expect, test } from "bun:test";
2 import { mkdtempSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import WebSocket from "ws";
6 import { FeetechBus } from "../src/bus.ts";
7 import { FakeBus } from "../src/fake-bus.ts";
8 import { startServer, type RunningServer } from "../src/server.ts";
9 import { format, fromEvents, measure } from "./jam-rate.ts";
10
11 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
12 const START = 2048;
13 let server: RunningServer;
14 afterEach(() => Promise.race([server.close(), sleep(500)]));
15
16 const boot = async (stuck: number[]) => {
17 server = await startServer(new FeetechBus(new FakeBus([1, 2], { dynamics: {}, stuck: new Set(stuck) })), {
18 host: "127.0.0.1",
19 port: 0,
20 ids: [1, 2],
21 posesFile: join(mkdtempSync(join(tmpdir(), "so101-jamrate-")), "poses.json"),
22 modelDir: "/nowhere",
23 jamSampleMs: 100,
24 safety: { maxSpeedDegS: 180, maxAccelDegS2: 1080 },
25 });
26 const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`);
27 await new Promise((r) => ws.once("open", r));
28 await sleep(100);
29 return ws;
30 };
31
32 /** Drive the arm for `ms` while the report "waits", as a person at the arm would. */
33 const drive = (ws: WebSocket, ms: number) => async () => {
34 for (let t = 0; t < ms; t += 20) ws.send(JSON.stringify({ 1: START + 300, 2: START + 300 })), await sleep(20);
35 };
36
37 test("a free move is no jam", async () => {
38 const ws = await boot([]);
39 const r = await measure(`http://127.0.0.1:${server.port}`, 60, drive(ws, 1000));
40 expect(r.jams).toBe(0);
41 expect(r.perHour).toBe(0);
42 expect(format(r, "x")).toContain("by joint: none");
43 ws.close();
44 });
45
46 test("one stuck-servo episode is one jam, by joint, with its load, ms and progress", async () => {
47 const ws = await boot([2]);
48 const r = await measure(`http://127.0.0.1:${server.port}`, 30, drive(ws, 1500));
49 expect(r.jams).toBe(1);
50 expect(r.perHour).toBe(2);
51 expect(r.byJoint).toEqual({ shoulder_lift: 1 });
52 const text = format(r, "x");
53 expect(text).toContain("shoulder_lift load 90");
54 expect(text).toMatch(/ms \d+ progress/);
55 // --events: the same jam, from the recent events alone
56 const m = (await (await fetch(`http://127.0.0.1:${server.port}/metrics`)).json()) as Parameters<typeof fromEvents>[0];
57 const e = fromEvents(m, Date.now());
58 expect(e.jams).toBe(1);
59 expect(e.hours).toBeGreaterThan(0);
60 ws.close();
61 }, 10_000);
Software/so101/bench/jam-rate.ts added +88
@@ -0,0 +1,88 @@
1 /**
2 * Jams per hour on the real arm, from GET /metrics. It only reads; it never moves the arm.
3 * bun run Software/so101/bench/jam-rate.ts <url> [--minutes N] [--events]
4 * It reads /metrics at the start and the end of the window (default 60 minutes) and counts the jams between.
5 * With --events it reports from the recent events alone, with no waiting, and says how many hours they span.
6 */
7 import type { MetricEvent } from "../src/metrics.ts";
8
9 interface Metrics {
10 counters: { jam: number };
11 recent: MetricEvent[];
12 }
13
14 export interface Report {
15 hours: number;
16 jams: number;
17 perHour: number;
18 byJoint: Record<string, number>;
19 events: MetricEvent[];
20 }
21
22 const key = (e: MetricEvent) => JSON.stringify(e);
23 const isJam = (e: MetricEvent) => e.kind === "jam";
24
25 function build(jams: MetricEvent[], count: number, hours: number): Report {
26 const byJoint: Record<string, number> = {};
27 for (const e of jams) {
28 const j = String(e.detail.joint ?? e.detail.id ?? "unknown");
29 byJoint[j] = (byJoint[j] ?? 0) + 1;
30 }
31 return { hours, jams: count, perHour: hours > 0 ? count / hours : 0, byJoint, events: jams };
32 }
33
34 /** The jams that happened between two readings of /metrics, over `hours`. */
35 export function between(start: Metrics, end: Metrics, hours: number): Report {
36 const seen = new Set(start.recent.map(key));
37 return build(end.recent.filter((e) => isJam(e) && !seen.has(key(e))), end.counters.jam - start.counters.jam, hours);
38 }
39
40 /** The jams in the recent events, over the time they span (the oldest event of any kind up to `now`). */
41 export function fromEvents(m: Metrics, now: number): Report {
42 const first = m.recent.length ? Date.parse(m.recent[0]!.t) : now;
43 const hours = Math.max(now - first, 60_000) / 3600_000;
44 const jams = m.recent.filter(isJam);
45 return build(jams, jams.length, hours);
46 }
47
48 export function format(r: Report, how: string): string {
49 const lines = [`${how}: ${r.jams} jams in ${r.hours.toFixed(2)} h = ${r.perHour.toFixed(1)} jams per hour`];
50 const joints = Object.entries(r.byJoint).sort((a, b) => b[1] - a[1]);
51 lines.push(`by joint: ${joints.length ? joints.map(([j, n]) => `${j} ${n}`).join(", ") : "none"}`);
52 for (const e of r.events) {
53 const d = e.detail;
54 lines.push(` ${e.t} ${d.joint ?? d.id} load ${d.load} ms ${d.ms} progress ${d.progress}`);
55 }
56 return lines.join("\n");
57 }
58
59 async function read(url: string): Promise<Metrics> {
60 const res = await fetch(new URL("/metrics", url), { signal: AbortSignal.timeout(10_000) });
61 if (!res.ok) throw new Error(`GET /metrics answered ${res.status}`);
62 return (await res.json()) as Metrics;
63 }
64
65 /** Read, wait out the window, read again. `wait` is the clock, so a test need not sit through it. */
66 export async function measure(url: string, minutes: number, wait: (ms: number) => Promise<void> = (ms) => new Promise((r) => setTimeout(r, ms))): Promise<Report> {
67 const start = await read(url);
68 await wait(minutes * 60_000);
69 return between(start, await read(url), minutes / 60);
70 }
71
72 if (import.meta.main) {
73 const args = process.argv.slice(2);
74 const url = args.find((a, i) => !a.startsWith("--") && args[i - 1] !== "--minutes");
75 const mi = args.indexOf("--minutes");
76 const minutes = mi >= 0 ? Number(args[mi + 1]) : 60;
77 if (!url || !(minutes > 0)) {
78 console.error("usage: bun run Software/so101/bench/jam-rate.ts <url> [--minutes N] [--events]");
79 process.exit(2);
80 }
81 try {
82 if (args.includes("--events")) console.log(format(fromEvents(await read(url), Date.now()), "recent events"));
83 else console.log(format(await measure(url, minutes), `${minutes} min window`));
84 } catch (e) {
85 console.error(e instanceof Error ? e.message : String(e));
86 process.exit(1);
87 }
88 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs