Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
The relay keeps the turns and the hold state under FORGE_APP_DATA, so a restart restores the queue, the driver and a held arm #296
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/252-the-relay-keeps-the-turns-and-the-hold-s → main 4 commits fork 3 threads · 3 unresolved
Software/README.md +9
| @@ -743,6 +743,15 @@ it survives a restart. Behind the relay, `by` is the signed-in name; the | ||
| 743 | 743 | page can't set it. It is kept as one line of at most 64 characters, and |
| 744 | 744 | pages show it as text, never as HTML. |
| 745 | 745 | |
| 746 | **A relay restart keeps the line and the hold.** The relay writes the queue, the | |
| 747 | current driver (with when their turn ends) and a STOP hold to `turns.json` and | |
| 748 | `hold.json` under `FORGE_APP_DATA/so101`, and restores them on start. Nobody | |
| 749 | who was connected is still connected, but their place and turn time are kept. A | |
| 750 | turn that ended while the relay was down is not brought back, and the next in | |
| 751 | line is called. A held arm stays held after a restart: only `POST /resume` lifts | |
| 752 | it, and a `hold.json` that cannot be read counts as a hold "by restart". The | |
| 753 | arm server's own E-stop is kept by the arm server (`estop.json`, above). | |
| 754 | ||
| 746 | 755 | > **The roles hold only behind the relay.** The arm server has no sign-in. |
| 747 | 756 | > Anyone who reaches its port can call `POST /estop/clear` and `PUT /settings`, |
| 748 | 757 | > and can put any name in `by`. Admin-only clearing and settings, and the |
Software/so101/bench/fake-serve.ts +2 −1
| @@ -52,6 +52,7 @@ const lobby = new Lobby({ | ||
| 52 | 52 | local, |
| 53 | 53 | chatFile: join(dir, "chat.json"), |
| 54 | 54 | adminFile: join(dir, "admin.json"), |
| 55 | turnsFile: join(dir, "turns.json"), | |
| 55 | 56 | wordsFile: join(dir, "words.json"), |
| 56 | 57 | accounts: new Accounts({ file: join(dir, "accounts.sqlite") }), |
| 57 | 58 | moderator: new Moderator({ file: join(dir, "crew.json"), model: modelFromEnv(process.env) }), |
| @@ -65,6 +66,6 @@ try { | ||
| 65 | 66 | } catch (err) { |
| 66 | 67 | console.error(`${(err as Error).message}; cameras stay MJPEG`); |
| 67 | 68 | } |
| 68 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video, summaryEveryMs: 60_000 }); | |
| 69 | const relay = await startRelay({ host, port, simulation: `http://127.0.0.1:${simulation.port}`, remote, problem, lobby, cameras, leaders, video, summaryEveryMs: 60_000, holdFile: join(dir, "hold.json") }); | |
| 69 | 70 | const { mode, host: armHost } = relay.link(); |
| 70 | 71 | console.log(`arm on http://${host}:${relay.port} (${mode === "simulated" ? "simulated" : `arm ${armHost}: ${mode}`}; data in ${dir}${local ? "; local: every visitor drives as the owner" : ""})`); |
Software/so101/src/command-gate.test.ts +48 −1
| @@ -1,4 +1,7 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 1 | import { afterEach, describe, expect, test } from "bun:test"; | |
| 2 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 2 | 5 | import { Clock, CommandGate, MAX_AGE_MS, parseCommand, stamper } from "./command-gate.ts"; |
| 3 | 6 | |
| 4 | 7 | const frame = (seq: number, t: number, joints: Record<string, number> = { 1: 2000 }) => JSON.stringify({ ...joints, seq, t }); |
| @@ -94,3 +97,47 @@ describe("stamper", () => { | ||
| 94 | 97 | expect(JSON.parse(stamp({ 6: 2500 }))).toEqual({ 6: 2500, seq: 2, t: 80 }); |
| 95 | 98 | }); |
| 96 | 99 | }); |
| 100 | ||
| 101 | describe("a hold kept in a file", () => { | |
| 102 | let dir = ""; | |
| 103 | afterEach(() => { | |
| 104 | if (dir) rmSync(dir, { recursive: true, force: true }); | |
| 105 | dir = ""; | |
| 106 | }); | |
| 107 | const file = () => join((dir ||= mkdtempSync(join(tmpdir(), "so101-hold-"))), "sub", "hold.json"); | |
| 108 | ||
| 109 | test("a hold comes back on a new gate, and only a resume lifts it for good", () => { | |
| 110 | const f = file(); | |
| 111 | const gate = new CommandGate(f); | |
| 112 | gate.hold("ann", 1000); | |
| 113 | expect(new CommandGate(f).held).toEqual({ by: "ann", at: new Date(1000).toISOString() }); | |
| 114 | expect(gate.resume(2000)).toBe(true); | |
| 115 | expect(new CommandGate(f).held).toBeNull(); | |
| 116 | }); | |
| 117 | ||
| 118 | test("a missing file is no hold", () => { | |
| 119 | expect(new CommandGate(file()).held).toBeNull(); | |
| 120 | }); | |
| 121 | ||
| 122 | test("a file that cannot be read stays held, by restart", () => { | |
| 123 | const f = file(); | |
| 124 | new CommandGate(f).hold("ann"); // makes the directory | |
| 125 | for (const bad of ["not json {", '{"held":{"by":5,"at":"x"}}', '{"held":"yes"}', "[]"]) { | |
| 126 | writeFileSync(f, bad); | |
| 127 | expect(new CommandGate(f).held?.by).toBe("restart"); | |
| 128 | } | |
| 129 | }); | |
| 130 | ||
| 131 | test("a file that exists but cannot be read stays held, by restart", () => { | |
| 132 | const f = file(); | |
| 133 | mkdirSync(f, { recursive: true }); // reading a directory fails with EISDIR, not ENOENT | |
| 134 | expect(new CommandGate(f).held?.by).toBe("restart"); | |
| 135 | }); | |
| 136 | ||
| 137 | test("a file that says nothing is held is no hold", () => { | |
| 138 | const f = file(); | |
| 139 | new CommandGate(f).hold("ann"); | |
| 140 | writeFileSync(f, '{"held":null}'); | |
| 141 | expect(new CommandGate(f).held).toBeNull(); | |
| 142 | }); | |
| 143 | }); |
Software/so101/src/command-gate.ts +40
| @@ -18,6 +18,8 @@ | ||
| 18 | 18 | * Pure: text and times in, verdicts out. |
| 19 | 19 | */ |
| 20 | 20 | |
| 21 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; | |
| 22 | import { dirname } from "node:path"; | |
| 21 | 23 | import { parseFrame } from "./safety.ts"; |
| 22 | 24 | |
| 23 | 25 | export const MAX_AGE_MS = 200; |
| @@ -78,17 +80,54 @@ export class Clock { | ||
| 78 | 80 | } |
| 79 | 81 | } |
| 80 | 82 | |
| 83 | /** The kept hold; a missing file is no hold, an unreadable one stays held, by "restart" (the safe side). */ | |
| 84 | function loadHeld(file: string): Held | null { | |
| 85 | let text: string; | |
| 86 | try { | |
| 87 | text = readFileSync(file, "utf8"); | |
| 88 | } catch (error) { | |
| 89 | if ((error as NodeJS.ErrnoException)?.code === "ENOENT") return null; | |
| 90 | console.error(`hold: cannot read ${file}, staying held`); | |
| 91 | return { by: "restart", at: new Date().toISOString() }; | |
| 92 | } | |
| 93 | try { | |
| 94 | const h = (JSON.parse(text) as { held?: Partial<Held> | null } | null)?.held; | |
| 95 | if (h === null) return null; // only an explicit "not held" frees the arm | |
| 96 | if (h && typeof h.by === "string" && typeof h.at === "string") return { by: h.by.slice(0, 80), at: h.at.slice(0, 40) }; | |
| 97 | } catch {} | |
outdated
This fallback is the safety-critical branch (a corrupt or malformed file stays held). It needs a test that would fail without it. Only an explicit null frees the arm, which is the right side to fail on. Make sure the admin page shows a hold by "restart" so a person knows to resume. |
||
| 98 | return { by: "restart", at: new Date().toISOString() }; | |
| 99 | } | |
| 100 | ||
| 81 | 101 | export class CommandGate { |
| 82 | 102 | /** Who stopped the arm, and when; null while it may move. */ |
| 83 | 103 | held: Held | null = null; |
| 84 | 104 | /** When the arm was last resumed, on this clock, ms; frames sent before it never count. */ |
| 85 | 105 | private resumedAt = 0; |
| 86 | 106 | private readonly streams = new Set<GateStream>(); |
| 107 | private readonly file: string | undefined; | |
| 108 | ||
| 109 | /** With a `file`, a hold is kept there and restored on construction: only a resume lifts it, never a restart. */ | |
| 110 | constructor(file?: string) { | |
| 111 | this.file = file; | |
| 112 | if (file) this.held = loadHeld(file); | |
| 113 | } | |
| 114 | ||
| 115 | private save(): void { | |
| 116 | if (!this.file) return; | |
| 117 | try { | |
| 118 | mkdirSync(dirname(this.file), { recursive: true }); | |
| 119 | writeFileSync(`${this.file}.tmp`, JSON.stringify({ held: this.held })); | |
| 120 | renameSync(`${this.file}.tmp`, this.file); | |
| 121 | } catch (error) { | |
| 122 | console.error(`hold: cannot save ${this.file}: ${error instanceof Error ? error.message : error}`); | |
| 123 | } | |
| 124 | } | |
| 87 | 125 | |
| 88 | 126 | /** Stop: from now on every frame is dropped until `resume`. Answers whether it was moving. */ |
| 89 | 127 | hold(by: string, now = Date.now()): boolean { |
| 90 | 128 | if (this.held) return false; |
| 91 | 129 | this.held = { by, at: new Date(now).toISOString() }; |
| 130 | this.save(); | |
| 92 | 131 | return true; |
| 93 | 132 | } |
| 94 | 133 | |
| @@ -97,6 +136,7 @@ export class CommandGate { | ||
| 97 | 136 | if (!this.held) return false; |
| 98 | 137 | this.held = null; |
| 99 | 138 | this.resumedAt = now; |
outdated
The save() in resume() is what stops a lifted hold coming back after a restart. Nothing tests it. |
||
| 139 | this.save(); | |
| 100 | 140 | for (const s of this.streams) s.floor(); |
| 101 | 141 | return true; |
| 102 | 142 | } |
Software/so101/src/lobby.ts +3 −1
| @@ -59,6 +59,8 @@ export interface LobbyOptions { | ||
| 59 | 59 | wordsFile?: string; |
| 60 | 60 | /** The admin settings' file; none keeps them in memory. */ |
| 61 | 61 | adminFile?: string; |
| 62 | /** The line and the current turn's file; none keeps them in memory. */ | |
| 63 | turnsFile?: string; | |
| 62 | 64 | /** Given turns keep their own limits until the owner changes one; made here, they start from the admin settings. */ |
| 63 | 65 | turns?: Turns; |
| 64 | 66 | chat?: Chat; |
| @@ -111,7 +113,7 @@ export class Lobby { | ||
| 111 | 113 | this.local = Boolean(options.local); |
| 112 | 114 | this.admin = options.admin ?? new Admin({ file: options.adminFile }); |
| 113 | 115 | const a = this.admin.settings; |
| 114 | this.turns = options.turns ?? new Turns({ turnMs: a.turnSeconds * 1000, idleMs: a.idleSeconds * 1000, maxQueue: a.maxQueue }); | |
| 116 | this.turns = options.turns ?? new Turns({ turnMs: a.turnSeconds * 1000, idleMs: a.idleSeconds * 1000, maxQueue: a.maxQueue, file: options.turnsFile }); | |
| 115 | 117 | this.turns.setPaused(a.paused); |
| 116 | 118 | this.chat = options.chat ?? new Chat({ file: options.chatFile }); |
| 117 | 119 | this.accounts = options.accounts ?? new Accounts(); |
Software/so101/src/relay-restart.test.ts added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | import { afterAll, afterEach, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { sign } from "./identity.ts"; | |
| 6 | import { Lobby } from "./lobby.ts"; | |
| 7 | import { startRelay, type RunningRelay } from "./relay.ts"; | |
| 8 | ||
| 9 | const KEY = "k".repeat(64); | |
| 10 | const later = Date.now() + 600_000; | |
| 11 | const as = (name: string, role: "user" | "admin") => { | |
| 12 | const identity = `id-${name}|${name}|${role}|${later}`; | |
| 13 | return { "x-forge-identity": identity, "x-forge-signature": sign(identity, KEY) }; | |
| 14 | }; | |
| 15 | ||
| 16 | // The arm server stand-in: a status that says nothing is held, and a stop it accepts. | |
| 17 | const sim = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: (req) => (new URL(req.url).pathname === "/status" ? Response.json({ joints: [], held: null }) : Response.json({})) }); | |
| 18 | ||
| 19 | let dir = ""; | |
| 20 | let relay: RunningRelay | undefined; | |
| 21 | let lobby: Lobby | undefined; | |
| 22 | afterAll(() => void sim.stop(true)); | |
| 23 | afterEach(async () => { | |
| 24 | await relay?.close(); | |
| 25 | lobby?.close(); | |
| 26 | if (dir) rmSync(dir, { recursive: true, force: true }); | |
| 27 | dir = ""; | |
| 28 | relay = lobby = undefined; | |
| 29 | }); | |
| 30 | ||
| 31 | async function boot() { | |
| 32 | dir ||= mkdtempSync(join(tmpdir(), "so101-relay-restart-")); | |
| 33 | lobby = new Lobby({ identityKey: KEY, adminFile: join(dir, "admin.json"), turnsFile: join(dir, "turns.json") }); | |
| 34 | relay = await startRelay({ port: 0, simulation: `http://127.0.0.1:${sim.port}`, lobby, holdFile: join(dir, "hold.json") }); | |
| 35 | return relay.port; | |
| 36 | } | |
| 37 | ||
| 38 | async function stop() { | |
| 39 | await relay!.close(); | |
| 40 | lobby!.close(); | |
| 41 | } | |
| 42 | ||
| 43 | const held = async (port: number) => ((await (await fetch(`http://127.0.0.1:${port}/status`, { headers: { connection: "close" } })).json()) as { held: { by: string } | null }).held; | |
| 44 | ||
| 45 | test("the queue, the driver and a held arm survive a relay restart on the same data dir", async () => { | |
| 46 | let port = await boot(); | |
| 47 | const person = (n: string) => ({ id: `id-${n}`, name: n, owner: false }); | |
| 48 | expect(lobby!.turns.join(person("ann"))).toBe("driving"); | |
| 49 | expect(lobby!.turns.join(person("bob"))).toBe("queued"); | |
| 50 | expect(lobby!.turns.join(person("cy"))).toBe("queued"); | |
| 51 | await fetch(`http://127.0.0.1:${port}/stop`, { method: "POST", headers: { ...as("trevato", "admin"), connection: "close" } }); // the arm server is not there; the gate shuts first | |
| 52 | const before = lobby!.turns.snapshot(); | |
| 53 | ||
| 54 | await stop(); | |
| 55 | port = await boot(); | |
| 56 | ||
| 57 | const after = lobby!.turns.snapshot(); | |
| 58 | expect(after.driver?.id).toBe("id-ann"); | |
| 59 | expect(after.driver?.endsAt).toBe(before.driver?.endsAt ?? 0); | |
| 60 | expect(after.queue.map((p) => p.id)).toEqual(["id-bob", "id-cy"]); | |
| 61 | expect((await held(port))?.by).toBe("trevato"); | |
| 62 | }); |
Software/so101/src/relay.ts +3 −1
| @@ -124,6 +124,8 @@ export interface RelayOptions { | ||
| 124 | 124 | camera?: FeedOptions; |
| 125 | 125 | /** The Pi's H.264 streams per camera id (src/video-feed.ts); a camera without one stays MJPEG. */ |
| 126 | 126 | video?: Record<number, VideoSources>; |
| 127 | /** Keep STOP's hold here, and restore it on start; without one it is in memory. */ | |
| 128 | holdFile?: string; | |
| 127 | 129 | /** Every driver's leader calibration; without one they are kept in memory. */ |
| 128 | 130 | leaders?: LeaderCalibrations; |
| 129 | 131 | /** How often the arm's status is read for the pages on /live (and sent to them when it changed). */ |
| @@ -370,7 +372,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 370 | 372 | }; |
| 371 | 373 | |
| 372 | 374 | /** STOP's hold, for every stream through here, and each driver's clock (their stamps, seen from here). */ |
| 373 | const gate = new CommandGate(); | |
| 375 | const gate = new CommandGate(options.holdFile); | |
| 374 | 376 | const clocks = new Map<string, Clock>(); |
| 375 | 377 | /** |
| 376 | 378 | * The arm's /status, saying which arm it is: from the link while it is live, the last one it had while reconnecting, |
A read error other than ENOENT logs and then returns null, which lifts a STOP hold on restart. Return { by: "restart", at: now } here for any code but ENOENT.