Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

The relay keeps the turns and the hold state under FORGE_APP_DATA, so a restart restores the queue, the driver and a held arm #296

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/252-the-relay-keeps-the-turns-and-the-hold-s → main 4 commits fork 3 threads · 3 unresolved

  • Merged. smith/252-the-relay-keeps-the-turns-and-the-hold-s was merged into main as 5cc704e.

Review

Sign in to review this change.
7 files changed +167 −4
Software/README.md +9
@@ -743,6 +743,15 @@ it survives a restart. Behind the relay, `by` is the signed-in name; the
743743 page can't set it. It is kept as one line of at most 64 characters, and
744744 pages show it as text, never as HTML.
745745
746 **A relay restart keeps the line and the hold.** The relay writes the queue, the
747 current driver (with when their turn ends) and a STOP hold to `turns.json` and
748 `hold.json` under `FORGE_APP_DATA/so101`, and restores them on start. Nobody
749 who was connected is still connected, but their place and turn time are kept. A
750 turn that ended while the relay was down is not brought back, and the next in
751 line is called. A held arm stays held after a restart: only `POST /resume` lifts
752 it, and a `hold.json` that cannot be read counts as a hold "by restart". The
753 arm server's own E-stop is kept by the arm server (`estop.json`, above).
754
746755 > **The roles hold only behind the relay.** The arm server has no sign-in.
747756 > Anyone who reaches its port can call `POST /estop/clear` and `PUT /settings`,
748757 > and can put any name in `by`. Admin-only clearing and settings, and the
Software/so101/bench/fake-serve.ts +2 −1
@@ -52,6 +52,7 @@ const lobby = new Lobby({
5252 local,
5353 chatFile: join(dir, "chat.json"),
5454 adminFile: join(dir, "admin.json"),
55 turnsFile: join(dir, "turns.json"),
5556 wordsFile: join(dir, "words.json"),
5657 accounts: new Accounts({ file: join(dir, "accounts.sqlite") }),
5758 moderator: new Moderator({ file: join(dir, "crew.json"), model: modelFromEnv(process.env) }),
@@ -65,6 +66,6 @@ try {
6566 } catch (err) {
6667 console.error(`${(err as Error).message}; cameras stay MJPEG`);
6768 }
68 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video, summaryEveryMs: 60_000 });
69 const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video, summaryEveryMs: 60_000, holdFile: join(dir, "hold.json") });
6970 const { mode, host: armHost } = relay.link();
7071 console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`);
Software/so101/src/command-gate.test.ts +48 −1
@@ -1,4 +1,7 @@
1 import { describe, expect, test } from "bun:test";
1 import { afterEach, describe, expect, test } from "bun:test";
2 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
25 import { Clock, CommandGate, MAX_AGE_MS, parseCommand, stamper } from "./command-gate.ts";
36
47 const frame = (seq: number, t: number, joints: Record<string, number> = { 1: 2000 }) => JSON.stringify({ ...joints, seq, t });
@@ -94,3 +97,47 @@ describe("stamper", () => {
9497 expect(JSON.parse(stamp({ 6: 2500 }))).toEqual({ 6: 2500, seq: 2, t: 80 });
9598 });
9699 });
100
101 describe("a hold kept in a file", () => {
102 let dir = "";
103 afterEach(() => {
104 if (dir) rmSync(dir, { recursive: true, force: true });
105 dir = "";
106 });
107 const file = () => join((dir ||= mkdtempSync(join(tmpdir(), "so101-hold-"))), "sub", "hold.json");
108
109 test("a hold comes back on a new gate, and only a resume lifts it for good", () => {
110 const f = file();
111 const gate = new CommandGate(f);
112 gate.hold("ann", 1000);
113 expect(new CommandGate(f).held).toEqual({ by: "ann", at: new Date(1000).toISOString() });
114 expect(gate.resume(2000)).toBe(true);
115 expect(new CommandGate(f).held).toBeNull();
116 });
117
118 test("a missing file is no hold", () => {
119 expect(new CommandGate(file()).held).toBeNull();
120 });
121
122 test("a file that cannot be read stays held, by restart", () => {
123 const f = file();
124 new CommandGate(f).hold("ann"); // makes the directory
125 for (const bad of ["not json {", '{"held":{"by":5,"at":"x"}}', '{"held":"yes"}', "[]"]) {
126 writeFileSync(f, bad);
127 expect(new CommandGate(f).held?.by).toBe("restart");
128 }
129 });
130
131 test("a file that exists but cannot be read stays held, by restart", () => {
132 const f = file();
133 mkdirSync(f, { recursive: true }); // reading a directory fails with EISDIR, not ENOENT
134 expect(new CommandGate(f).held?.by).toBe("restart");
135 });
136
137 test("a file that says nothing is held is no hold", () => {
138 const f = file();
139 new CommandGate(f).hold("ann");
140 writeFileSync(f, '{"held":null}');
141 expect(new CommandGate(f).held).toBeNull();
142 });
143 });
Software/so101/src/command-gate.ts +40
@@ -18,6 +18,8 @@
1818 * Pure: text and times in, verdicts out.
1919 */
2020
21 import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
22 import { dirname } from "node:path";
2123 import { parseFrame } from "./safety.ts";
2224
2325 export const MAX_AGE_MS = 200;
@@ -78,17 +80,54 @@ export class Clock {
7880 }
7981 }
8082
83 /** The kept hold; a missing file is no hold, an unreadable one stays held, by "restart" (the safe side). */
84 function loadHeld(file: string): Held | null {
85 let text: string;
86 try {
87 text = readFileSync(file, "utf8");
88 } catch (error) {
89 if ((error as NodeJS.ErrnoException)?.code === "ENOENT") return null;
outdated

A read error other than ENOENT logs and then returns null, which lifts a STOP hold on restart. Return { by: "restart", at: now } here for any code but ENOENT.

90 console.error(`hold: cannot read ${file}, staying held`);
91 return { by: "restart", at: new Date().toISOString() };
92 }
93 try {
94 const h = (JSON.parse(text) as { held?: Partial<Held> | null } | null)?.held;
95 if (h === null) return null; // only an explicit "not held" frees the arm
96 if (h && typeof h.by === "string" && typeof h.at === "string") return { by: h.by.slice(0, 80), at: h.at.slice(0, 40) };
97 } catch {}
outdated

This fallback is the safety-critical branch (a corrupt or malformed file stays held). It needs a test that would fail without it.

Only an explicit null frees the arm, which is the right side to fail on. Make sure the admin page shows a hold by "restart" so a person knows to resume.

98 return { by: "restart", at: new Date().toISOString() };
99 }
100
81101 export class CommandGate {
82102 /** Who stopped the arm, and when; null while it may move. */
83103 held: Held | null = null;
84104 /** When the arm was last resumed, on this clock, ms; frames sent before it never count. */
85105 private resumedAt = 0;
86106 private readonly streams = new Set<GateStream>();
107 private readonly file: string | undefined;
108
109 /** With a `file`, a hold is kept there and restored on construction: only a resume lifts it, never a restart. */
110 constructor(file?: string) {
111 this.file = file;
112 if (file) this.held = loadHeld(file);
113 }
114
115 private save(): void {
116 if (!this.file) return;
117 try {
118 mkdirSync(dirname(this.file), { recursive: true });
119 writeFileSync(`${this.file}.tmp`, JSON.stringify({ held: this.held }));
120 renameSync(`${this.file}.tmp`, this.file);
121 } catch (error) {
122 console.error(`hold: cannot save ${this.file}: ${error instanceof Error ? error.message : error}`);
123 }
124 }
87125
88126 /** Stop: from now on every frame is dropped until `resume`. Answers whether it was moving. */
89127 hold(by: string, now = Date.now()): boolean {
90128 if (this.held) return false;
91129 this.held = { by, at: new Date(now).toISOString() };
130 this.save();
92131 return true;
93132 }
94133
@@ -97,6 +136,7 @@ export class CommandGate {
97136 if (!this.held) return false;
98137 this.held = null;
99138 this.resumedAt = now;
outdated

The save() in resume() is what stops a lifted hold coming back after a restart. Nothing tests it.

139 this.save();
100140 for (const s of this.streams) s.floor();
101141 return true;
102142 }
Software/so101/src/lobby.ts +3 −1
@@ -59,6 +59,8 @@ export interface LobbyOptions {
5959 wordsFile?: string;
6060 /** The admin settings' file; none keeps them in memory. */
6161 adminFile?: string;
62 /** The line and the current turn's file; none keeps them in memory. */
63 turnsFile?: string;
6264 /** Given turns keep their own limits until the owner changes one; made here, they start from the admin settings. */
6365 turns?: Turns;
6466 chat?: Chat;
@@ -111,7 +113,7 @@ export class Lobby {
111113 this.local = Boolean(options.local);
112114 this.admin = options.admin ?? new Admin({ file: options.adminFile });
113115 const a = this.admin.settings;
114 this.turns = options.turns ?? new Turns({ turnMs: a.turnSeconds * 1000, idleMs: a.idleSeconds * 1000, maxQueue: a.maxQueue });
116 this.turns = options.turns ?? new Turns({ turnMs: a.turnSeconds * 1000, idleMs: a.idleSeconds * 1000, maxQueue: a.maxQueue, file: options.turnsFile });
115117 this.turns.setPaused(a.paused);
116118 this.chat = options.chat ?? new Chat({ file: options.chatFile });
117119 this.accounts = options.accounts ?? new Accounts();
Software/so101/src/relay-restart.test.ts added +62
@@ -0,0 +1,62 @@
1 import { afterAll, afterEach, expect, test } from "bun:test";
2 import { mkdtempSync, rmSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { sign } from "./identity.ts";
6 import { Lobby } from "./lobby.ts";
7 import { startRelay, type RunningRelay } from "./relay.ts";
8
9 const KEY = "k".repeat(64);
10 const later = Date.now() + 600_000;
11 const as = (name: string, role: "user" | "admin") => {
12 const identity = `id-${name}|${name}|${role}|${later}`;
13 return { "x-forge-identity": identity, "x-forge-signature": sign(identity, KEY) };
14 };
15
16 // The arm server stand-in: a status that says nothing is held, and a stop it accepts.
17 const sim = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: (req) => (new URL(req.url).pathname === "/status" ? Response.json({ joints: [], held: null }) : Response.json({})) });
18
19 let dir = "";
20 let relay: RunningRelay | undefined;
21 let lobby: Lobby | undefined;
22 afterAll(() => void sim.stop(true));
23 afterEach(async () => {
24 await relay?.close();
25 lobby?.close();
26 if (dir) rmSync(dir, { recursive: true, force: true });
27 dir = "";
28 relay = lobby = undefined;
29 });
30
31 async function boot() {
32 dir ||= mkdtempSync(join(tmpdir(), "so101-relay-restart-"));
33 lobby = new Lobby({ identityKey: KEY, adminFile: join(dir, "admin.json"), turnsFile: join(dir, "turns.json") });
34 relay = await startRelay({ port: 0, simulation: `http://127.0.0.1:${sim.port}`, lobby, holdFile: join(dir, "hold.json") });
35 return relay.port;
36 }
37
38 async function stop() {
39 await relay!.close();
40 lobby!.close();
41 }
42
43 const held = async (port: number) => ((await (await fetch(`http://127.0.0.1:${port}/status`, { headers: { connection: "close" } })).json()) as { held: { by: string } | null }).held;
44
45 test("the queue, the driver and a held arm survive a relay restart on the same data dir", async () => {
46 let port = await boot();
47 const person = (n: string) => ({ id: `id-${n}`, name: n, owner: false });
48 expect(lobby!.turns.join(person("ann"))).toBe("driving");
49 expect(lobby!.turns.join(person("bob"))).toBe("queued");
50 expect(lobby!.turns.join(person("cy"))).toBe("queued");
51 await fetch(`http://127.0.0.1:${port}/stop`, { method: "POST", headers: { ...as("trevato", "admin"), connection: "close" } }); // the arm server is not there; the gate shuts first
52 const before = lobby!.turns.snapshot();
53
54 await stop();
55 port = await boot();
56
57 const after = lobby!.turns.snapshot();
58 expect(after.driver?.id).toBe("id-ann");
59 expect(after.driver?.endsAt).toBe(before.driver?.endsAt ?? 0);
60 expect(after.queue.map((p) => p.id)).toEqual(["id-bob", "id-cy"]);
61 expect((await held(port))?.by).toBe("trevato");
62 });
Software/so101/src/relay.ts +3 −1
@@ -124,6 +124,8 @@ export interface RelayOptions {
124124 camera?: FeedOptions;
125125 /** The Pi's H.264 streams per camera id (src/video-feed.ts); a camera without one stays MJPEG. */
126126 video?: Record<number, VideoSources>;
127 /** Keep STOP's hold here, and restore it on start; without one it is in memory. */
128 holdFile?: string;
127129 /** Every driver's leader calibration; without one they are kept in memory. */
128130 leaders?: LeaderCalibrations;
129131 /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */
@@ -370,7 +372,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
370372 };
371373
372374 /** STOP's hold, for every stream through here, and each driver's clock (their stamps, seen from here). */
373 const gate = new CommandGate();
375 const gate = new CommandGate(options.holdFile);
374376 const clocks = new Map<string, Clock>();
375377 /**
376378 * The arm's /status, saying which arm it is: from the link while it is live, the last one it had while reconnecting,

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs