Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
bench/jam-rate.ts <url> reports jams per hour from the arm's /metrics, so #509's before-and-after can be measured on the real arm #542
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/537-bench-jam-rate-ts-url-reports-jams-per-h → main 1 commit fork
3 files
changed +151
−0
Software/README.md +2
| @@ -1105,6 +1105,8 @@ Where to read it: | ||
| 1105 | 1105 | `bun Software/so101/bench/limits-sweep.ts [url]` drives each joint in turn through a fixed set of targets inside its bounds (the others held still) and prints that joint's p95 error, time to arrive, overshoot and snap-backs from `/precision`; with no url it runs on the fake bus, with a url it refuses to start unless nobody is driving and stops at once on STOP or any refusal. |
| 1106 | 1106 | The sweep's targets are fixed; the measured numbers move a little with the machine's timers. |
| 1107 | 1107 | |
| 1108 | `bun run Software/so101/bench/jam-rate.ts <url> [--minutes N] [--events]` reads `/metrics` at the start and end of the window (default 60 min) and prints jams per hour, by joint, and each jam's load, ms and progress; `--events` reports from the recent events alone. It only reads. | |
| 1109 | ||
| 1108 | 1110 | `bun bench/precision-report.ts <url>` (the arm server's URL) reads the hourly roll-up from `GET /precision` and prints |
| 1109 | 1111 | the weekly comment for #439: each headline number this week against last, the worst joint and the open experiment. It |
| 1110 | 1112 | only reads; the crew posts it. The experiment is the line below, which the crew edits: |
Software/so101/bench/jam-rate.test.ts added +61
| @@ -0,0 +1,61 @@ | ||
| 1 | import { afterEach, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import WebSocket from "ws"; | |
| 6 | import { FeetechBus } from "../src/bus.ts"; | |
| 7 | import { FakeBus } from "../src/fake-bus.ts"; | |
| 8 | import { startServer, type RunningServer } from "../src/server.ts"; | |
| 9 | import { format, fromEvents, measure } from "./jam-rate.ts"; | |
| 10 | ||
| 11 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 12 | const START = 2048; | |
| 13 | let server: RunningServer; | |
| 14 | afterEach(() => Promise.race([server.close(), sleep(500)])); | |
| 15 | ||
| 16 | const boot = async (stuck: number[]) => { | |
| 17 | server = await startServer(new FeetechBus(new FakeBus([1, 2], { dynamics: {}, stuck: new Set(stuck) })), { | |
| 18 | host: "127.0.0.1", | |
| 19 | port: 0, | |
| 20 | ids: [1, 2], | |
| 21 | posesFile: join(mkdtempSync(join(tmpdir(), "so101-jamrate-")), "poses.json"), | |
| 22 | modelDir: "/nowhere", | |
| 23 | jamSampleMs: 100, | |
| 24 | safety: { maxSpeedDegS: 180, maxAccelDegS2: 1080 }, | |
| 25 | }); | |
| 26 | const ws = new WebSocket(`ws://127.0.0.1:${server.port}/follow`); | |
| 27 | await new Promise((r) => ws.once("open", r)); | |
| 28 | await sleep(100); | |
| 29 | return ws; | |
| 30 | }; | |
| 31 | ||
| 32 | /** Drive the arm for `ms` while the report "waits", as a person at the arm would. */ | |
| 33 | const drive = (ws: WebSocket, ms: number) => async () => { | |
| 34 | for (let t = 0; t < ms; t += 20) ws.send(JSON.stringify({ 1: START + 300, 2: START + 300 })), await sleep(20); | |
| 35 | }; | |
| 36 | ||
| 37 | test("a free move is no jam", async () => { | |
| 38 | const ws = await boot([]); | |
| 39 | const r = await measure(`http://127.0.0.1:${server.port}`, 60, drive(ws, 1000)); | |
| 40 | expect(r.jams).toBe(0); | |
| 41 | expect(r.perHour).toBe(0); | |
| 42 | expect(format(r, "x")).toContain("by joint: none"); | |
| 43 | ws.close(); | |
| 44 | }); | |
| 45 | ||
| 46 | test("one stuck-servo episode is one jam, by joint, with its load, ms and progress", async () => { | |
| 47 | const ws = await boot([2]); | |
| 48 | const r = await measure(`http://127.0.0.1:${server.port}`, 30, drive(ws, 1500)); | |
| 49 | expect(r.jams).toBe(1); | |
| 50 | expect(r.perHour).toBe(2); | |
| 51 | expect(r.byJoint).toEqual({ shoulder_lift: 1 }); | |
| 52 | const text = format(r, "x"); | |
| 53 | expect(text).toContain("shoulder_lift load 90"); | |
| 54 | expect(text).toMatch(/ms \d+ progress/); | |
| 55 | // --events: the same jam, from the recent events alone | |
| 56 | const m = (await (await fetch(`http://127.0.0.1:${server.port}/metrics`)).json()) as Parameters<typeof fromEvents>[0]; | |
| 57 | const e = fromEvents(m, Date.now()); | |
| 58 | expect(e.jams).toBe(1); | |
| 59 | expect(e.hours).toBeGreaterThan(0); | |
| 60 | ws.close(); | |
| 61 | }, 10_000); |
Software/so101/bench/jam-rate.ts added +88
| @@ -0,0 +1,88 @@ | ||
| 1 | /** | |
| 2 | * Jams per hour on the real arm, from GET /metrics. It only reads; it never moves the arm. | |
| 3 | * bun run Software/so101/bench/jam-rate.ts <url> [--minutes N] [--events] | |
| 4 | * It reads /metrics at the start and the end of the window (default 60 minutes) and counts the jams between. | |
| 5 | * With --events it reports from the recent events alone, with no waiting, and says how many hours they span. | |
| 6 | */ | |
| 7 | import type { MetricEvent } from "../src/metrics.ts"; | |
| 8 | ||
| 9 | interface Metrics { | |
| 10 | counters: { jam: number }; | |
| 11 | recent: MetricEvent[]; | |
| 12 | } | |
| 13 | ||
| 14 | export interface Report { | |
| 15 | hours: number; | |
| 16 | jams: number; | |
| 17 | perHour: number; | |
| 18 | byJoint: Record<string, number>; | |
| 19 | events: MetricEvent[]; | |
| 20 | } | |
| 21 | ||
| 22 | const key = (e: MetricEvent) => JSON.stringify(e); | |
| 23 | const isJam = (e: MetricEvent) => e.kind === "jam"; | |
| 24 | ||
| 25 | function build(jams: MetricEvent[], count: number, hours: number): Report { | |
| 26 | const byJoint: Record<string, number> = {}; | |
| 27 | for (const e of jams) { | |
| 28 | const j = String(e.detail.joint ?? e.detail.id ?? "unknown"); | |
| 29 | byJoint[j] = (byJoint[j] ?? 0) + 1; | |
| 30 | } | |
| 31 | return { hours, jams: count, perHour: hours > 0 ? count / hours : 0, byJoint, events: jams }; | |
| 32 | } | |
| 33 | ||
| 34 | /** The jams that happened between two readings of /metrics, over `hours`. */ | |
| 35 | export function between(start: Metrics, end: Metrics, hours: number): Report { | |
| 36 | const seen = new Set(start.recent.map(key)); | |
| 37 | return build(end.recent.filter((e) => isJam(e) && !seen.has(key(e))), end.counters.jam - start.counters.jam, hours); | |
| 38 | } | |
| 39 | ||
| 40 | /** The jams in the recent events, over the time they span (the oldest event of any kind up to `now`). */ | |
| 41 | export function fromEvents(m: Metrics, now: number): Report { | |
| 42 | const first = m.recent.length ? Date.parse(m.recent[0]!.t) : now; | |
| 43 | const hours = Math.max(now - first, 60_000) / 3600_000; | |
| 44 | const jams = m.recent.filter(isJam); | |
| 45 | return build(jams, jams.length, hours); | |
| 46 | } | |
| 47 | ||
| 48 | export function format(r: Report, how: string): string { | |
| 49 | const lines = [`${how}: ${r.jams} jams in ${r.hours.toFixed(2)} h = ${r.perHour.toFixed(1)} jams per hour`]; | |
| 50 | const joints = Object.entries(r.byJoint).sort((a, b) => b[1] - a[1]); | |
| 51 | lines.push(`by joint: ${joints.length ? joints.map(([j, n]) => `${j} ${n}`).join(", ") : "none"}`); | |
| 52 | for (const e of r.events) { | |
| 53 | const d = e.detail; | |
| 54 | lines.push(` ${e.t} ${d.joint ?? d.id} load ${d.load} ms ${d.ms} progress ${d.progress}`); | |
| 55 | } | |
| 56 | return lines.join("\n"); | |
| 57 | } | |
| 58 | ||
| 59 | async function read(url: string): Promise<Metrics> { | |
| 60 | const res = await fetch(new URL("/metrics", url), { signal: AbortSignal.timeout(10_000) }); | |
| 61 | if (!res.ok) throw new Error(`GET /metrics answered ${res.status}`); | |
| 62 | return (await res.json()) as Metrics; | |
| 63 | } | |
| 64 | ||
| 65 | /** Read, wait out the window, read again. `wait` is the clock, so a test need not sit through it. */ | |
| 66 | export async function measure(url: string, minutes: number, wait: (ms: number) => Promise<void> = (ms) => new Promise((r) => setTimeout(r, ms))): Promise<Report> { | |
| 67 | const start = await read(url); | |
| 68 | await wait(minutes * 60_000); | |
| 69 | return between(start, await read(url), minutes / 60); | |
| 70 | } | |
| 71 | ||
| 72 | if (import.meta.main) { | |
| 73 | const args = process.argv.slice(2); | |
| 74 | const url = args.find((a, i) => !a.startsWith("--") && args[i - 1] !== "--minutes"); | |
| 75 | const mi = args.indexOf("--minutes"); | |
| 76 | const minutes = mi >= 0 ? Number(args[mi + 1]) : 60; | |
| 77 | if (!url || !(minutes > 0)) { | |
| 78 | console.error("usage: bun run Software/so101/bench/jam-rate.ts <url> [--minutes N] [--events]"); | |
| 79 | process.exit(2); | |
| 80 | } | |
| 81 | try { | |
| 82 | if (args.includes("--events")) console.log(format(fromEvents(await read(url), Date.now()), "recent events")); | |
| 83 | else console.log(format(await measure(url, minutes), `${minutes} min window`)); | |
| 84 | } catch (e) { | |
| 85 | console.error(e instanceof Error ? e.message : String(e)); | |
| 86 | process.exit(1); | |
| 87 | } | |
| 88 | } |
new above
offline