Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

/follow: size and rate checks #78

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

quill/arm:quill/36-follow-size-and-rate-checks → main 1 commit fork 1 thread · 1 unresolved

  • Merged. quill/36-follow-size-and-rate-checks was merged into main as ff3968f.

Review

Sign in to review this change.
6 files changed +314 −9
Software/README.md +4 −1
@@ -134,7 +134,10 @@ The API, all JSON (the full list with bodies is at the top of
134134 `/estop`, `/estop/clear`, `/record/start`, `/record/stop`,
135135 `/recording/delete`, `/play`;
136136 `PUT /calibration/camera|handeye`; the WebSocket `/follow` taking
137 `{"1": ticks, ..., "seq": n, "t": ms}` frames; and the read-only WebSocket `/watch`, which sends
137 `{"1": ticks, ..., "seq": n, "t": ms}` frames (at most 1 KiB each, or the
138 stream closes with 1009; past 200 a second they are dropped, and a stream that
139 keeps sending twice that is closed with 1008; either way the arm holds, on the
140 arm server and on the relay alike: `so101/src/frame-limit.ts`); and the read-only WebSocket `/watch`, which sends
138141 the `/status` body every 250 ms, one bus read shared by every watcher, and
139142 between them the machine log: every register written on the bus, every change
140143 of driver, hold and E-stop, and every safety clamp, as it happens. Each joint
Software/so101/src/frame-limit.test.ts added +196
@@ -0,0 +1,196 @@
1 import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
2 import { mkdtempSync } from "node:fs";
3 import { createServer, type Server } from "node:http";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import WebSocket, { WebSocketServer } from "ws";
7 import { FeetechBus } from "./bus.ts";
8 import { stamper } from "./command-gate.ts";
9 import { FakeBus } from "./fake-bus.ts";
10 import { BURST, FLOOD_CLOSE, FLOOD_MS, FrameRate, frameBytes, MAX_FRAME_BYTES, TOO_BIG_CLOSE } from "./frame-limit.ts";
11 import { LOST_CLOSE, parseRemote, startRelay, type RunningRelay } from "./relay.ts";
12 import { startServer, type RunningServer } from "./server.ts";
13
14 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
15 const shut = (s: { close(): Promise<void> } | undefined) => (s ? Promise.race([s.close(), sleep(500)]) : undefined);
16 const closed = (ws: WebSocket) => new Promise<number>((r) => ws.once("close", (code) => r(code)));
17 const opened = (ws: WebSocket) => new Promise((r, j) => (ws.once("open", r), ws.once("error", j)));
18 /** A frame of joint 1 padded with an unknown key until it is `bytes` long. */
19 const padded = (bytes: number) => {
20 const head = '{"1":2048,"pad":"';
21 return head + "x".repeat(bytes - head.length - 2) + '"}';
22 };
23
24 describe("frame size", () => {
25 test("counts the bytes of every shape ws hands over", () => {
26 expect(frameBytes("é")).toBe(2);
27 expect(frameBytes(Buffer.alloc(7))).toBe(7);
28 expect(frameBytes(new ArrayBuffer(5))).toBe(5);
29 expect(frameBytes([Buffer.alloc(3), Buffer.alloc(4)])).toBe(7);
30 expect(padded(MAX_FRAME_BYTES).length).toBe(MAX_FRAME_BYTES);
31 });
32 });
33
34 describe("frame rate", () => {
35 test("a burst goes through, the rest of it is dropped, and the bucket refills with time", () => {
36 const rate = new FrameRate(200, 50, 2000);
37 const verdicts = Array.from({ length: 60 }, () => rate.take(1000));
38 expect(verdicts.filter((v) => v === "ok").length).toBe(50);
39 expect(verdicts.slice(50).every((v) => v === "drop")).toBe(true);
40 expect(rate.take(1005)).toBe("ok"); // 5 ms at 200 a second is one frame
41 expect(rate.take(1005)).toBe("drop");
42 });
43
44 test("a leader at 100 frames a second is never held back", () => {
45 const rate = new FrameRate();
46 for (let i = 0; i < 1000; i++) expect(rate.take(i * 10)).toBe("ok");
47 });
48
49 test("a sender at twice the rate or more for FLOOD_MS is a flood; a burst now and then is not", () => {
50 let rate = new FrameRate(200, 50, 2000);
51 let verdict = rate.take(0);
52 for (let t = 0; t < 1900; t += 2) for (let i = 0; i < 2; i++) verdict = rate.take(t); // 1000 a second
53 expect(verdict).toBe("flood");
54 rate = new FrameRate(200, 50, 2000);
55 for (let t = 0; t < 60_000; t += 1000) for (let i = 0; i < 150; i++) expect(rate.take(t)).not.toBe("flood"); // a buffer flushed every second
56 rate = new FrameRate(200, 50, 2000);
57 for (let t = 0; t < 60_000; t += 3) expect(rate.take(t)).not.toBe("flood"); // 333 a second: dropped down to the rate, never closed
58 });
59 });
60
61 describe("the arm server's /follow", () => {
62 let srv: RunningServer;
63 let url: string;
64 beforeAll(async () => {
65 srv = await startServer(new FeetechBus(new FakeBus([1, 2, 6])), { host: "127.0.0.1", port: 0, ids: [1, 2, 6], posesFile: join(mkdtempSync(join(tmpdir(), "so101-limit-")), "poses.json"), watchdogMs: 5000 });
66 url = `http://127.0.0.1:${srv.port}`;
67 });
68 afterAll(() => shut(srv));
69 const open = async () => {
70 const ws = new WebSocket(`ws://127.0.0.1:${srv.port}/follow`);
71 await opened(ws);
72 return ws;
73 };
74 const idle = async () => {
75 for (let i = 0; i < 100 && ((await (await fetch(`${url}/status`)).json()) as { driving: unknown }).driving !== null; i++) await sleep(10);
76 };
77 const lastStream = async () => {
78 const m = (await (await fetch(`${url}/metrics`)).json()) as { recent: Array<{ kind: string; detail: Record<string, unknown> }> };
79 return m.recent.filter((e) => e.kind === "stream").at(-1)!.detail;
80 };
81
82 test("a frame at the limit is a frame; one byte over closes the stream with 1009, and the arm holds", async () => {
83 const ws = await open();
84 ws.send(padded(MAX_FRAME_BYTES));
85 await sleep(60);
86 expect(ws.readyState).toBe(WebSocket.OPEN);
87 const code = closed(ws);
88 ws.send(padded(MAX_FRAME_BYTES + 1));
89 expect(await code).toBe(TOO_BIG_CLOSE);
90 await idle();
91 expect((await lastStream()).lost).toBeUndefined(); // held, not parked
92 });
93
94 test("frames past the rate are dropped, and are not merged into what the arm is sent", async () => {
95 const ws = await open();
96 for (let i = 0; i < BURST + 30; i++) ws.send(JSON.stringify({ "2": 2000 + (i < BURST ? 0 : 50) }));
97 await sleep(80);
98 const s = (await (await fetch(`${url}/status`)).json()) as { stream: { gated: number; merged: number } };
99 expect(s.stream.gated).toBeGreaterThanOrEqual(25);
100 expect(s.stream.merged).toBeLessThanOrEqual(BURST);
101 ws.close();
102 await idle();
103 });
104
105 test("a stream that floods is closed with 1008", async () => {
106 const ws = await open();
107 const code = closed(ws);
108 const flood = setInterval(() => {
109 for (let i = 0; i < 20 && ws.readyState === WebSocket.OPEN; i++) ws.send('{"2":2000}');
110 }, 5);
111 try {
112 expect(await Promise.race([code, sleep(FLOOD_MS + 2000).then(() => 0)])).toBe(FLOOD_CLOSE);
113 } finally {
114 clearInterval(flood);
115 }
116 await idle();
117 expect((await lastStream()).lost).toBeUndefined();
118 });
119 });
120
121 describe("the relay's /follow", () => {
122 let camera: Server;
123 let relay: RunningRelay | undefined;
124 let simulation: RunningServer;
125 afterEach(async () => {
126 await shut(relay);
127 relay = undefined;
128 camera.closeAllConnections();
129 await Promise.race([new Promise((r) => camera.close(r)), sleep(500)]);
130 });
131 beforeAll(async () => {
132 simulation = await startServer(new FeetechBus(new FakeBus([1, 2, 3, 4, 5, 6])), { host: "127.0.0.1", port: 0, ids: [1, 2, 3, 4, 5, 6], posesFile: join(mkdtempSync(join(tmpdir(), "so101-limit-")), "poses.json"), watchEveryMs: 10 });
133 });
134 afterAll(() => shut(simulation));
135
136 /** An arm server that counts what the relay passes on, and how the relay closed the stream toward it. */
137 const fakeArm = async () => {
138 const got: string[] = [];
139 const ended: number[] = [];
140 const wss = new WebSocketServer({ noServer: true });
141 camera = createServer((_req, res) => res.writeHead(404).end());
142 camera.on("upgrade", (req, socket, head) =>
143 wss.handleUpgrade(req, socket, head, (ws) => {
144 if (req.url !== "/follow") return;
145 ws.on("message", (d) => got.push(String(d)));
146 ws.on("close", (code) => ended.push(code));
147 }),
148 );
149 const pump = setInterval(() => wss.clients.forEach((c) => c.send('{"joints":[]}')), 10);
150 const port = await new Promise<number>((r) => camera.listen(0, "127.0.0.1", () => r((camera.address() as { port: number }).port)));
151 relay = await startRelay({
152 port: 0,
153 simulation: `http://127.0.0.1:${simulation.port}`,
154 remote: parseRemote({ ARM_SERVER: `ws://127.0.0.1:${port}` })!,
155 link: { backoffMs: [5, 10, 20, 40], staleMs: 60, offlineAfterMs: 600, connectTimeoutMs: 50, pollEveryMs: 10 },
156 });
157 await sleep(60);
158 const ws = new WebSocket(`ws://127.0.0.1:${relay.port}/follow`);
159 await opened(ws);
160 return { ws, got, ended, stop: () => (clearInterval(pump), wss.clients.forEach((c) => c.terminate())) };
161 };
162
163 test("an oversized frame is not passed on: the stream closes with 1009 and the arm holds", async () => {
164 const arm = await fakeArm();
165 try {
166 const stamp = stamper();
167 arm.ws.send(stamp({ 1: 2050 }));
168 await sleep(30);
169 expect(arm.got).toEqual(['{"1":2050}']);
170 const code = closed(arm.ws);
171 arm.ws.send(JSON.stringify({ 1: 2060, seq: 99, t: Date.now(), pad: "x".repeat(MAX_FRAME_BYTES) }));
172 expect(await code).toBe(TOO_BIG_CLOSE);
173 await sleep(30);
174 expect(arm.got).toEqual(['{"1":2050}']);
175 expect(arm.ended).toEqual([1000]); // not LOST_CLOSE: the arm server holds rather than parks
176 expect(arm.ended).not.toContain(LOST_CLOSE);
177 } finally {
178 arm.stop();
179 }
180 });
181
182 test("frames past the rate are not passed on", async () => {
183 const arm = await fakeArm();
184 try {
185 const stamp = stamper();
186 for (let i = 0; i < BURST + 40; i++) arm.ws.send(stamp({ 1: 2000 + i }));
187 await sleep(60);
188 expect(arm.got.length).toBeGreaterThanOrEqual(BURST);
189 expect(arm.got.length).toBeLessThan(BURST + 20);
190 expect(arm.ws.readyState).toBe(WebSocket.OPEN);
191 arm.ws.close();
192 } finally {
193 arm.stop();
194 }
195 });
196 });
Software/so101/src/frame-limit.ts added +69
@@ -0,0 +1,69 @@
1 /**
2 * How big and how fast a /follow frame may be, checked before it is parsed: on the arm server and again on the relay.
3 *
4 * A frame is six joints and a stamp, about a hundred bytes; anything over MAX_FRAME_BYTES is not a frame, and the
5 * stream is closed with TOO_BIG_CLOSE (1009, "message too big"). The `ws` option `maxPayload` says the same, but Bun's
6 * `ws` does not enforce it, so the size is counted here.
7 *
8 * A leader sends 50 to 100 frames a second; a page's sliders and sticks fewer. A `FrameRate` lets RATE_PER_S through
9 * on average, BURST at once after a quiet spell, and drops the rest: commands are latest-wins targets, so the next
10 * frame let through carries the hand on. A sender that has more dropped in FLOOD_MS than the rate lets through in it
11 * (twice the rate, kept up) is not driving but flooding, and its stream is closed with FLOOD_CLOSE (1008, "policy
12 * violation"). Either close leaves the arm holding, not parking: the server ended the stream, it was not lost.
13 * Pure: bytes and times in, verdicts out.
14 */
15
16 import type { RawData } from "ws";
17
18 export const MAX_FRAME_BYTES = 1024;
19 export const RATE_PER_S = 200;
20 export const BURST = 50;
21 export const FLOOD_MS = 2000;
22 export const TOO_BIG_CLOSE = 1009;
23 export const FLOOD_CLOSE = 1008;
24
25 /** The bytes in a message as `ws` hands it over: a Buffer, an ArrayBuffer, or fragments. */
26 export function frameBytes(data: RawData | string): number {
27 if (typeof data === "string") return Buffer.byteLength(data);
28 if (Array.isArray(data)) return data.reduce((n, b) => n + b.byteLength, 0);
29 return data.byteLength;
30 }
31
32 export type RateVerdict = "ok" | "drop" | "flood";
33
34 /** A token bucket, one per stream, and a count of what it dropped lately. */
35 export class FrameRate {
36 private tokens: number;
37 private at: number | null = null;
38 /** Frames dropped since `windowAt`; a window is FLOOD_MS long. */
39 private drops = 0;
40 private windowAt = 0;
41 private readonly perMs: number;
42 private readonly burst: number;
43 private readonly floodMs: number;
44 /** Drops in one window that make a flood: twice the rate, kept up for the whole of it. */
45 private readonly floodDrops: number;
46
47 constructor(perS = RATE_PER_S, burst = BURST, floodMs = FLOOD_MS) {
48 this.perMs = perS / 1000;
49 this.burst = burst;
50 this.floodMs = floodMs;
51 this.floodDrops = perS * (floodMs / 1000);
52 this.tokens = burst;
53 }
54
55 /** Whether a frame arriving `now` may go on, is dropped, or ends the stream. */
56 take(now = Date.now()): RateVerdict {
57 if (this.at !== null) this.tokens = Math.min(this.burst, this.tokens + (now - this.at) * this.perMs);
58 this.at = now;
59 if (this.tokens >= 1) {
60 this.tokens -= 1;
61 return "ok";
62 }
63 if (now - this.windowAt >= this.floodMs) {
64 this.windowAt = now;
65 this.drops = 0;
66 }
67 return ++this.drops > this.floodDrops ? "flood" : "drop";
68 }

This closes at more than floodDrops drops in a window, which is about 3x the rate sent (200 allowed plus 200 dropped per second is only 400 sent), not 2x as the header and README say. Align the wording or the count.

69 }
Software/so101/src/relay.ts +22 −3
@@ -21,7 +21,9 @@
2121 * while held, a stream is refused (HELD_CLOSE) and no frame goes through; /resume opens it once the arm server agrees,
2222 * and only a frame sent after that moves the arm. Every frame from the page carries "seq" and "t": one without them,
2323 * out of order, older than 200 ms, or sent before the resume is dropped here, never kept. What goes on to the arm
24 * server is the joints alone. A turn that ends stops what drives without holding ("hold": false).
24 * server is the joints alone. A turn that ends stops what drives without holding ("hold": false). Before any of that, a
25 * frame over 1 KiB closes the stream, and frames past 200 a second are dropped and, kept up, close it (src/frame-limit.ts):
26 * the arm holds then.
2527 *
2628 * A /follow stream whose browser goes away without closing it (or stops answering pings) is closed toward the arm
2729 * with LOST_CLOSE: the arm server parks the arm then, where a stream closed on purpose only holds.
@@ -70,6 +72,7 @@ import { CalibrationError } from "./leader-map.js";
7072 import type { Lobby } from "./lobby.ts";
7173 import { capBody, FrameCap } from "./speed-cap.ts";
7274 import { Clock, CommandGate, HELD_CLOSE } from "./command-gate.ts";
75 import { FLOOD_CLOSE, FrameRate, frameBytes, MAX_FRAME_BYTES, TOO_BIG_CLOSE } from "./frame-limit.ts";
7376
7477 export interface Remote {
7578 /** The arm server's HTTP base, no trailing slash. */
@@ -681,7 +684,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
681684 });
682685
683686 // ------------------------------------------------------------------ WebSocket /follow
684 const wss = new WebSocketServer({ noServer: true });
687 const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_BYTES });
685688 const env = options.env ?? process.env;
686689 const open = (url: string, timeoutMs: number) =>
687690 new Promise<WebSocket>((resolve, reject) => {
@@ -742,7 +745,23 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
742745 seed();
743746 /** A frame as the panel shows it: servo id → ticks. */
744747 const values = (text: string) => perServo(parseFrame(text) ?? []);
748 const rate = new FrameRate();
749 /** Set when this relay ends the stream for breaking the limits: the arm then holds, as for a stream closed on purpose. */
750 let overLimit = false;
745751 browser.on("message", (raw) => {
752 if (overLimit) return;
753 if (frameBytes(raw) > MAX_FRAME_BYTES) {
754 overLimit = true;
755 machine.note("clamp", "not passed on", { why: "frame too large", bytes: frameBytes(raw) });
756 return browser.close(TOO_BIG_CLOSE, `a frame is at most ${MAX_FRAME_BYTES} bytes`);
757 }
758 const pace = rate.take();
759 if (pace === "flood") {
760 overLimit = true;
761 machine.note("clamp", "not passed on", { why: "too many frames" });
762 return browser.close(FLOOD_CLOSE, "too many frames: the stream was closed");
763 }
764 if (pace === "drop") return;
746765 if (driver !== null) {
747766 // The turn may have ended while this frame was on its way, before the close reaches the page: it is not passed on.
748767 if (lobby && !lobby.turns.isDriver(driver) && !lobby.local) return void machine.note("clamp", "not passed on", { why: "the turn ended" });
@@ -770,7 +789,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
770789 });
771790 arm.on("close", (code, why) => browser.close(sendable(code) ? code : 1011, why.length ? why.toString() : "the arm server went away"));
772791 // A browser that closed the stream on purpose leaves the arm holding; one that vanished leaves it parking.
773 browser.on("close", (code) => (code === 1000 || code === 1005 || code === HELD_CLOSE ? arm.close(1000) : arm.close(LOST_CLOSE, "the driver's connection was lost")));
792 browser.on("close", (code) => (overLimit || code === 1000 || code === 1005 || code === HELD_CLOSE ? arm.close(1000) : arm.close(LOST_CLOSE, "the driver's connection was lost")));
774793 let answered = true;
775794 browser.on("pong", () => (answered = true));
776795 const heartbeat = setInterval(() => {
Software/so101/src/server.test.ts +2 −2
@@ -763,10 +763,10 @@ describe("the safety envelope", () => {
763763
764764 test("frames faster than the command rate are merged, the latest winning", async () => {
765765 const ws = await open();
766 for (let i = 0; i < 100; i++) ws.send(JSON.stringify({ "2": 2000 + (i % 7) }));
766 for (let i = 0; i < 50; i++) ws.send(JSON.stringify({ "2": 2000 + (i % 7) })); // the most a burst may hold (src/frame-limit.ts)
767767 await sleep(80);
768768 const s = await status();
769 expect(s.stream!.merged).toBeGreaterThanOrEqual(90);
769 expect(s.stream!.merged).toBeGreaterThanOrEqual(44);
770770 expect(s.stream!.frames).toBeLessThanOrEqual(5);
771771 ws.close();
772772 await idle();
Software/so101/src/server.ts +21 −3
@@ -35,7 +35,9 @@
3535 * GET /app/<file> the modules the page imports itself (the leader arm over Web Serial), as JavaScript
3636 * GET /printables the sheets to print for the vision quest (board, cube markers, gripper marker); GET /printables/<file>
3737 * WS /follow client sends {"1": ticks, "2": ticks, ...} frames; the server drives the arm. A frame may carry
38 * "seq" and "t" (src/command-gate.ts): then a stale, replayed or pre-resume one is dropped
38 * "seq" and "t" (src/command-gate.ts): then a stale, replayed or pre-resume one is dropped.
39 * A frame over 1 KiB closes the stream; frames past 200 a second are dropped, and a stream that
40 * keeps that up is closed (src/frame-limit.ts)
3941 * WS /watch read-only: the server sends the /status body every watchEveryMs, one bus read shared by every watcher;
4042 * the hosted app's relay keeps one open, so viewers see the arm without each of them asking the Pi.
4143 * Between them, {"machine": [entries], "run": "<id>"}: every register written on the bus, every
@@ -58,6 +60,7 @@ import { fileURLToPath } from "node:url";
5860 import { WebSocketServer, type WebSocket } from "ws";
5961 import { browserModule } from "./browser-modules.ts";
6062 import { Clock, CommandGate, HELD_CLOSE } from "./command-gate.ts";
63 import { FLOOD_CLOSE, FrameRate, frameBytes, MAX_FRAME_BYTES, TOO_BIG_CLOSE } from "./frame-limit.ts";
6164 import type { FeetechBus } from "./bus.ts";
6265 import { ArmModel, gripperTicksToRadians, ticksToRadians, type Violation } from "./kinematics.ts";
6366 import { clamp, type Frame } from "./mapping.ts";
@@ -956,9 +959,11 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
956959 // Frames are merged down to the command rate; each joint's target is pulled inside its bounds, judged by the zone,
957960 // and handed to the governor, which moves the goals toward it within the speed and acceleration limits.
958961 // Silence makes the arm hold (the dead-man); a connection that is lost rather than closed makes it park.
959 const wss = new WebSocketServer({ noServer: true, maxPayload: 4096 });
962 const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_FRAME_BYTES });
960963 /** Each stream's halt: what it has queued or under way stops at once, not when the close handshake comes back. */
961964 const halts = new Map<WebSocket, () => void>();
965 /** Streams this server is ending for breaking the limits: whatever the sender answers, the arm holds rather than parks. */
966 const overLimit = new WeakSet<WebSocket>();
962967 const end = (ws: WebSocket, code: number, why: string) => {
963968 halts.get(ws)?.();
964969 ws.close(code, why);
@@ -1206,8 +1211,21 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
12061211 lastTaken = Date.now();
12071212 follow(frame);
12081213 };
1214 const rate = new FrameRate();
12091215 ws.on("message", (data) => {
12101216 if (closed) return;
1217 if (frameBytes(data) > MAX_FRAME_BYTES) {
1218 machine.note("clamp", "frame too large", { bytes: frameBytes(data), max: MAX_FRAME_BYTES });
1219 overLimit.add(ws);
1220 return end(ws, TOO_BIG_CLOSE, `a frame is at most ${MAX_FRAME_BYTES} bytes`);
1221 }
1222 const pace = rate.take();
1223 if (pace === "flood") {
1224 machine.note("clamp", "too many frames", { source: "stream" });
1225 overLimit.add(ws);
1226 return end(ws, FLOOD_CLOSE, "too many frames: the stream was closed");
1227 }
1228 if (pace === "drop") return void stats.gated++;
12111229 const verdict = admission.admit(String(data));
12121230 if (!verdict.ok) {
12131231 if (verdict.drop !== "malformed") stats.gated++;
@@ -1238,7 +1256,7 @@ export async function startServer(bus: FeetechBus, options: ServerOptions = {}):
12381256 halts.delete(ws);
12391257 admission.close();
12401258 clearInterval(heartbeat);
1241 const lost = !CLEAN_CLOSE.has(code) && code !== ESTOP_CLOSE;
1259 const lost = !CLEAN_CLOSE.has(code) && code !== ESTOP_CLOSE && !overLimit.has(ws);
12421260 void run(async () => {
12431261 try {
12441262 if (armed) {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs