Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
TOP: the hosted app connects to the real arm and camera - it still serves the simulation with ARM_SERVER set and both addresses granted #43
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
quill/arm:quill/40-top-the-hosted-app-connects-to-the-real → main 2 commits fork 2 threads · 2 unresolved
5 files
changed +157
−5
Software/README.md +10
| @@ -207,6 +207,16 @@ there, so if the Pi moves, change it in `forge.json` (and in `ARM_SERVER`). | ||
| 207 | 207 | The hosted app currently runs `bench/fake-serve.ts`, so nothing uses this |
| 208 | 208 | egress until `ARM_SERVER` is set. |
| 209 | 209 | |
| 210 | Inside the forge's cell the only way out is the forge's egress proxy | |
| 211 | (`HTTP_PROXY`). Requests and camera streams find it on their own. The | |
| 212 | `/follow` WebSocket is handed it explicitly (`src/egress.ts`), because Bun's | |
| 213 | WebSocket client doesn't read `HTTP_PROXY`. When the proxy refuses a | |
| 214 | destination, or can't reach it itself, the top bar and `/link` give its | |
| 215 | reason, e.g. **the forge's egress proxy answered 502: 100.89.95.68:8101 did | |
| 216 | not answer**. That means the forge's machine has no route to the Pi, which | |
| 217 | is not something the app can fix. Every `/status` says which arm answered: | |
| 218 | `"source": "live <host:port>"` or `"source": "simulated: <why>"`. | |
| 219 | ||
| 210 | 220 | With `ARM_SERVER` unset, the app runs the simulated arm and says |
| 211 | 221 | **Simulated**. When it is set but the server doesn't answer `/status` within |
| 212 | 222 | 1.5 s, the top bar says **Simulated · \<host\> unreachable (why)**, and the |
Software/so101/src/egress.test.ts added +22
| @@ -0,0 +1,22 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { proxyFor, whyNot } from "./egress.ts"; | |
| 3 | ||
| 4 | describe("egress", () => { | |
| 5 | test("a WebSocket goes through the proxy the cell names, unless NO_PROXY says otherwise", () => { | |
| 6 | const cell = { HTTP_PROXY: "http://127.0.0.1:3128", NO_PROXY: "localhost,127.0.0.1,::1" }; | |
| 7 | expect(proxyFor("ws://100.89.95.68:8101/follow", cell)).toBe("http://127.0.0.1:3128"); | |
| 8 | expect(proxyFor("ws://127.0.0.1:41000/follow", cell)).toBeNull(); | |
| 9 | expect(proxyFor("ws://localhost:41000/follow", cell)).toBeNull(); | |
| 10 | expect(proxyFor("ws://pi.tailnet:8101/follow", { http_proxy: "http://p:1", no_proxy: ".tailnet" })).toBeNull(); | |
| 11 | expect(proxyFor("ws://pi.tailnet:8101/follow", { http_proxy: "http://p:1", no_proxy: "*" })).toBeNull(); | |
| 12 | expect(proxyFor("wss://arm.example/follow", { HTTPS_PROXY: "http://s:2", HTTP_PROXY: "http://p:1" })).toBe("http://s:2"); | |
| 13 | expect(proxyFor("ws://arm.example/follow", { HTTPS_PROXY: "http://s:2" })).toBeNull(); | |
| 14 | expect(proxyFor("ws://100.89.95.68:8101/follow", {})).toBeNull(); | |
| 15 | }); | |
| 16 | ||
| 17 | test("the forge's egress proxy is named, with its reason; anyone else's answer is only its status", async () => { | |
| 18 | const refused = new Response("100.89.95.68:8101 did not answer\n", { status: 502, headers: { "x-forge-egress": "refused" } }); | |
| 19 | expect(await whyNot(refused, "/status")).toBe("the forge's egress proxy answered 502: 100.89.95.68:8101 did not answer"); | |
| 20 | expect(await whyNot(new Response('{"error":"bus"}', { status: 500 }), "/status")).toBe("/status answered 500"); | |
| 21 | }); | |
| 22 | }); |
Software/so101/src/egress.ts added +37
| @@ -0,0 +1,37 @@ | ||
| 1 | /** | |
| 2 | * The way out of the process, when there is one to name. | |
| 3 | * | |
| 4 | * Inside a forge cell there is no route anywhere but the forge's egress proxy, named by HTTP_PROXY (and HTTPS_PROXY). | |
| 5 | * Bun's fetch follows those on its own; its WebSocket client does not, so a WebSocket to the arm server is opened with | |
| 6 | * the proxy this names, as a CONNECT tunnel. Away from the forge nothing is set and connections go straight out. | |
| 7 | */ | |
| 8 | ||
| 9 | /** Whether `host` (a name or address, no port) is one NO_PROXY says to reach directly. */ | |
| 10 | function bypassed(host: string, noProxy: string): boolean { | |
| 11 | const name = host.toLowerCase().replace(/^\[|\]$/g, ""); | |
| 12 | return noProxy | |
| 13 | .split(",") | |
| 14 | .map((e) => e.trim().toLowerCase().replace(/:\d+$/, "")) | |
| 15 | .some((e) => e === "*" || (e !== "" && (name === e.replace(/^\./, "") || name.endsWith(e.startsWith(".") ? e : `.${e}`)))); | |
| 16 | } | |
| 17 | ||
| 18 | /** The proxy to reach `target` (an http, https, ws or wss URL) through, or null to connect directly. */ | |
| 19 | export function proxyFor(target: string, env: Record<string, string | undefined>): string | null { | |
| 20 | const url = new URL(target); | |
| 21 | const secure = url.protocol === "https:" || url.protocol === "wss:"; | |
| 22 | const proxy = (secure ? (env.HTTPS_PROXY ?? env.https_proxy) : undefined) ?? env.HTTP_PROXY ?? env.http_proxy; | |
| 23 | if (!proxy?.trim()) return null; | |
| 24 | if (bypassed(url.hostname, env.NO_PROXY ?? env.no_proxy ?? "")) return null; | |
| 25 | return proxy.trim(); | |
| 26 | } | |
| 27 | ||
| 28 | /** | |
| 29 | * Why a URL's answer is not a good one, for the page and the log. The forge's egress proxy marks its own answers | |
| 30 | * (`X-Forge-Egress`, a line of text saying why), so a refusal or a dial that failed there is told apart from an arm | |
| 31 | * server that answered badly. | |
| 32 | */ | |
| 33 | export async function whyNot(response: Response, what: string): Promise<string> { | |
| 34 | const text = (await response.text().catch(() => "")).trim().split("\n")[0]!.slice(0, 200); | |
| 35 | if (response.headers.has("x-forge-egress")) return `the forge's egress proxy answered ${response.status}${text ? `: ${text}` : ""}`; | |
| 36 | return `${what} answered ${response.status}`; | |
| 37 | } |
Software/so101/src/relay.test.ts +56 −1
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; |
| 2 | 2 | import { mkdtempSync } from "node:fs"; |
| 3 | 3 | import { createServer, type IncomingMessage, type Server } from "node:http"; |
| 4 | import { createServer as createTcpServer, type Server as TcpServer } from "node:net"; | |
| 4 | import { connect, createServer as createTcpServer, type Server as TcpServer } from "node:net"; | |
| 5 | 5 | import { tmpdir } from "node:os"; |
| 6 | 6 | import { join } from "node:path"; |
| 7 | 7 | import { gunzipSync } from "node:zlib"; |
| @@ -45,6 +45,7 @@ async function cameraShows(url: string, mark: Buffer): Promise<{ type: string | | ||
| 45 | 45 | return { type: response.headers.get("content-type"), ms: Date.now() - started }; |
| 46 | 46 | } |
| 47 | 47 | const joints = async (base: string) => ((await (await fetch(`${base}/status`)).json()) as { joints: unknown[] }).joints.length; |
| 48 | const source = async (base: string) => ((await (await fetch(`${base}/status`)).json()) as { source: string }).source; | |
| 48 | 49 | const link = async (base: string) => (await (await fetch(`${base}/link`)).json()) as { mode: string; host: string | null; problem: string | null }; |
| 49 | 50 | |
| 50 | 51 | let simulation: RunningServer; |
| @@ -118,6 +119,7 @@ describe("relay", () => { | ||
| 118 | 119 | const base = `http://127.0.0.1:${relay.port}`; |
| 119 | 120 | expect(await link(base)).toEqual({ mode: "simulated", host: null, problem: null }); |
| 120 | 121 | expect(await joints(base)).toBe(6); |
| 122 | expect(await source(base)).toBe("simulated: ARM_SERVER is not set"); | |
| 121 | 123 | const page = await fetch(`${base}/`); |
| 122 | 124 | expect(page.status).toBe(200); |
| 123 | 125 | expect(await page.text()).toContain("<title>SO-101</title>"); |
| @@ -162,6 +164,7 @@ describe("relay", () => { | ||
| 162 | 164 | const base = `http://127.0.0.1:${relay.port}`; |
| 163 | 165 | expect(await link(base)).toEqual({ mode: "live", host: `127.0.0.1:${remote.port}`, problem: null }); |
| 164 | 166 | expect(await joints(base)).toBe(3); |
| 167 | expect(await source(base)).toBe(`live 127.0.0.1:${remote.port}`); | |
| 165 | 168 | |
| 166 | 169 | const saved = await fetch(`${base}/pose/save`, { method: "POST", body: JSON.stringify({ name: "real" }) }); |
| 167 | 170 | expect(saved.status).toBe(200); |
| @@ -207,6 +210,58 @@ describe("relay", () => { | ||
| 207 | 210 | expect(l.mode).toBe("simulated"); |
| 208 | 211 | expect(l.host).toBe(`127.0.0.1:${port}`); |
| 209 | 212 | expect(l.problem).toContain("unreachable"); |
| 213 | expect(await source(base)).toBe(`simulated: ${l.problem}`); | |
| 214 | }); | |
| 215 | ||
| 216 | test("the forge's egress proxy cannot reach the arm: its reason is the one the page and /status give", async () => { | |
| 217 | camera = createServer((_req, res) => res.writeHead(502, { "content-type": "text/plain", "x-forge-egress": "refused" }).end("100.89.95.68:8101 did not answer\n")); | |
| 218 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(await listening(camera)) }); | |
| 219 | const base = `http://127.0.0.1:${relay.port}`; | |
| 220 | const l = await link(base); | |
| 221 | expect(l.problem).toContain("the forge's egress proxy answered 502: 100.89.95.68:8101 did not answer"); | |
| 222 | expect(await source(base)).toBe(`simulated: ${l.problem}`); | |
| 223 | expect(await joints(base)).toBe(6); | |
| 224 | }); | |
| 225 | ||
| 226 | test("inside a cell, /follow reaches the arm server through the egress proxy", async () => { | |
| 227 | remote = await arm([1, 2, 6]); | |
| 228 | const tunnelled: string[] = []; | |
| 229 | // A CONNECT proxy, the way the forge's egress tunnels: answer 200, then pass bytes both ways. | |
| 230 | hang = createTcpServer((client) => { | |
| 231 | client.once("data", (head: Buffer) => { | |
| 232 | const [, target] = head.toString().split("\r\n")[0]!.split(" "); | |
| 233 | tunnelled.push(target!); | |
| 234 | const [host, port] = target!.split(":"); | |
| 235 | const upstream = connect(Number(port), host!, () => { | |
| 236 | client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); | |
| 237 | const rest = head.subarray(head.indexOf("\r\n\r\n") + 4); | |
| 238 | if (rest.length) upstream.write(rest); | |
| 239 | client.pipe(upstream).pipe(client); | |
| 240 | }); | |
| 241 | upstream.on("error", () => client.destroy()); | |
| 242 | client.on("error", () => upstream.destroy()); | |
| 243 | }); | |
| 244 | }); | |
| 245 | const proxyPort = await listening(hang); | |
| 246 | // This arm is on loopback, which a cell's NO_PROXY names, and Bun's WebSocket honours NO_PROXY even with a proxy | |
| 247 | // given; so for this test the environment names no proxy at all, and only the relay is told of this one. | |
| 248 | const names = ["NO_PROXY", "no_proxy", "HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"]; | |
| 249 | const saved = Object.fromEntries(names.map((n) => [n, process.env[n]])); | |
| 250 | for (const n of names) process.env[n] = ""; // blanked, not deleted: Bun keeps reading a deleted one from where it started | |
| 251 | try { | |
| 252 | relay = await startRelay({ port: 0, simulation: simBase, remote: remoteAt(remote.port), env: { HTTP_PROXY: `http://127.0.0.1:${proxyPort}` } }); | |
| 253 | const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/follow`); | |
| 254 | await new Promise((r, j) => (ws.once("open", r), ws.once("error", j))); | |
| 255 | await sleep(50); | |
| 256 | expect(tunnelled).toEqual([`127.0.0.1:${remote.port}`]); | |
| 257 | expect(((await (await fetch(`http://127.0.0.1:${remote.port}/status`)).json()) as { driving: string | null }).driving).toBe("stream"); | |
| 258 | ws.close(); | |
| 259 | } finally { | |
| 260 | for (const n of names) { | |
| 261 | if (saved[n] === undefined) delete process.env[n]; | |
| 262 | else process.env[n] = saved[n]; | |
| 263 | } | |
| 264 | } | |
| 210 | 265 | }); |
| 211 | 266 | |
| 212 | 267 | test("an arm server that goes away mid-session falls back on the next request, /follow included", async () => { |
Software/so101/src/relay.ts +32 −4
| @@ -5,6 +5,8 @@ | ||
| 5 | 5 | * the arm API, the WebSocket /follow and the cameras go to that `so101 serve`, as long as it answers; when it does |
| 6 | 6 | * not, the relay says so on /link and answers from the simulation instead, so the page never hangs on a dead link. |
| 7 | 7 | * The page itself, the 3D model and the vendored scripts always come from here. |
| 8 | * Inside a forge cell the only way out is the egress proxy (src/egress.ts): fetch finds it on its own, the WebSocket | |
| 9 | * to the arm server is told it, and when the proxy refuses or cannot dial, its reason is the one /link and /status give. | |
| 8 | 10 | * |
| 9 | 11 | * With a lobby (src/lobby.ts), anyone may watch but only the person whose turn it is may command the arm: every |
| 10 | 12 | * request that is not a read, and the /follow stream, is refused for anyone else before it reaches an arm. When a |
| @@ -17,6 +19,7 @@ | ||
| 17 | 19 | * sets the HttpOnly session cookie and names you: {"you": {"id", "name", "owner"}} |
| 18 | 20 | * GET /admin, POST /admin the owner's page: pause, end a turn, take control, the settings, the log |
| 19 | 21 | * GET /link {"mode": "live"|"simulated", "host": "<host:port>"|null, "problem": "…"|null} |
| 22 | * GET /status the arm's own /status, and which arm said it: "source": "live <host:port>" or "simulated: <why>" | |
| 20 | 23 | * GET /cameras the listed cameras: [{"id", "name", "online"}] |
| 21 | 24 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 22 | 25 | * the "camera offline" picture when its source is down |
| @@ -40,6 +43,7 @@ import type { FeedOptions } from "./camera-feed.ts"; | ||
| 40 | 43 | import { CameraError, CameraHub } from "./cameras.ts"; |
| 41 | 44 | import { LOCAL as LOCAL_PERSON } from "./identity.ts"; |
| 42 | 45 | import { LeaderCalibrations } from "./leader-calibrations.ts"; |
| 46 | import { proxyFor, whyNot } from "./egress.ts"; | |
| 43 | 47 | import { CalibrationError } from "./leader-map.js"; |
| 44 | 48 | import type { Lobby } from "./lobby.ts"; |
| 45 | 49 | import { capBody, FrameCap } from "./speed-cap.ts"; |
| @@ -85,6 +89,8 @@ export interface RelayOptions { | ||
| 85 | 89 | camera?: FeedOptions; |
| 86 | 90 | /** Every driver's leader calibration; without one they are kept in memory. */ |
| 87 | 91 | leaders?: LeaderCalibrations; |
| 92 | /** Where HTTP_PROXY and NO_PROXY are read from; process.env by default. */ | |
| 93 | env?: Record<string, string | undefined>; | |
| 88 | 94 | } |
| 89 | 95 | |
| 90 | 96 | export interface RunningRelay { |
| @@ -197,9 +203,10 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 197 | 203 | (probing ??= (async () => { |
| 198 | 204 | try { |
| 199 | 205 | const r = await fetch(`${remote!.http}/status`, { signal: AbortSignal.timeout(probeTimeoutMs), cache: "no-store" }); |
| 200 | await r.body?.cancel(); | |
| 201 | if (r.ok) up(); | |
| 202 | else down(`/status answered ${r.status}`); | |
| 206 | if (r.ok) { | |
| 207 | await r.body?.cancel(); | |
| 208 | up(); | |
| 209 | } else down(await whyNot(r, "/status")); | |
| 203 | 210 | } catch (err) { |
| 204 | 211 | down((err as Error)?.name === "TimeoutError" ? `no answer in ${probeTimeoutMs} ms` : reason(err)); |
| 205 | 212 | } finally { |
| @@ -207,6 +214,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 207 | 214 | } |
| 208 | 215 | })()); |
| 209 | 216 | const link = (): Link => ({ mode: live ? "live" : "simulated", host: remote?.host ?? null, problem }); |
| 217 | const source = () => (live && remote ? `live ${remote.host}` : `simulated: ${problem ?? "ARM_SERVER is not set"}`); | |
| 210 | 218 | |
| 211 | 219 | // ------------------------------------------------------------------ HTTP |
| 212 | 220 | const json = (res: ServerResponse, status: number, body: unknown) => { |
| @@ -267,6 +275,22 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 267 | 275 | } |
| 268 | 276 | }; |
| 269 | 277 | |
| 278 | /** The arm's /status, from the live arm while it answers and the simulation otherwise, saying which it is. */ | |
| 279 | const status = async (res: ServerResponse, path: string) => { | |
| 280 | let answer: Response | undefined; | |
| 281 | if (live && remote) { | |
| 282 | try { | |
| 283 | answer = await fetch(remote.http + path, { signal: AbortSignal.timeout(readTimeoutMs), cache: "no-store" }); | |
| 284 | if (!answer.ok) down(await whyNot(answer, "/status")); | |
| 285 | } catch (err) { | |
| 286 | down((err as Error)?.name === "TimeoutError" ? `no answer in ${readTimeoutMs} ms` : reason(err)); | |
| 287 | } | |
| 288 | } | |
| 289 | if (!live || !answer?.ok) answer = await fetch(simulation + path, { cache: "no-store" }); | |
When the live answer is not ok and the simulation answers instead, the upstream response body is never cancelled unless whyNot consumed it. Cancel it to release the connection. |
||
| 290 | const body = (await answer.json()) as Record<string, unknown>; | |
| 291 | return json(res, answer.status, { ...body, source: source() }); | |
| 292 | }; | |
| 293 | ||
| 270 | 294 | /** A static answer, gzipped: whole, so the length is known; kept when the browser may cache it too. */ |
| 271 | 295 | const gzipped = new GzipCache(); |
| 272 | 296 | const packed = async (res: ServerResponse, { response }: Awaited<ReturnType<typeof ask>>, path: string) => { |
| @@ -440,6 +464,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 440 | 464 | if (refused) return json(res, refused.status, { error: refused.error }); |
| 441 | 465 | } |
| 442 | 466 | if (req.method === "GET" && url.pathname === "/link") return json(res, 200, link()); |
| 467 | if (req.method === "GET" && url.pathname === "/status") return await status(res, path); | |
| 443 | 468 | |
| 444 | 469 | const camera = CAMERA.exec(url.pathname); |
| 445 | 470 | if (camera) { |
| @@ -477,9 +502,12 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 477 | 502 | |
| 478 | 503 | // ------------------------------------------------------------------ WebSocket /follow |
| 479 | 504 | const wss = new WebSocketServer({ noServer: true }); |
| 505 | const env = options.env ?? process.env; | |
| 480 | 506 | const open = (url: string, timeoutMs: number) => |
| 481 | 507 | new Promise<WebSocket>((resolve, reject) => { |
| 482 | const ws = new WebSocket(url); | |
| 508 | // Bun's WebSocket client takes a proxy (a CONNECT tunnel) but does not look for one in the environment. | |
| 509 | const proxy = proxyFor(url, env); | |
| 510 | const ws = proxy ? new WebSocket(url, { proxy } as WebSocket.ClientOptions) : new WebSocket(url); | |
| 483 | 511 | const timer = setTimeout(() => { |
| 484 | 512 | ws.terminate(); |
| 485 | 513 | reject(new Error(`no answer in ${timeoutMs} ms`)); |
new above
offline
NO_PROXY matching handles names, suffixes and *, but not CIDR ranges or IP prefixes. That is fine for the loopback list a cell uses, so just keep it in mind if a cell ever lists a range.