Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

When a turn ends, the relay fetches its trace from GET /precision/trace and stores it with the turn in facts (capped, gone at 90 days) #553

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/547-when-a-turn-ends-the-relay-fetches-its-t → main 2 commits fork 1 thread · 1 unresolved

  • Merged. smith/547-when-a-turn-ends-the-relay-fetches-its-t was merged into main as 27e9c27.

Review

Sign in to review this change.
4 files changed +132 −7
Software/so101/src/facts.test.ts +75 −1
@@ -3,7 +3,7 @@ import { mkdtempSync } from "node:fs";
33 import { tmpdir } from "node:os";
44 import { join } from "node:path";
55 import { ArmEdges } from "./arm-edges.ts";
6 import { Facts, KEEP_MS, MAX_VISITS_PER_HOUR, parseCard, whoIs } from "./facts.ts";
6 import { Facts, KEEP_MS, MAX_TRACE_BYTES, MAX_VISITS_PER_HOUR, parseCard, parseSessionId, whoIs } from "./facts.ts";
77 import type { Snapshot } from "./turns.ts";
88
99 const person = (id: string, name = id) => ({ id, name, owner: false });
@@ -167,6 +167,80 @@ test("the relay writes app start and a turn's row with its card", async () => {
167167 }
168168 });
169169
170 const trace = (n: number) => ({ session: 7, t0: 1, hz: 5, joints: { shoulder_pan: { intent: Array(n).fill(2048), actual: Array(n).fill(2000) } } });
171
172 test("a trace round-trips with its turn, is capped, and goes with the turn at 90 days", () => {
173 let t = 1000;
174 const facts = new Facts({ commit: "t", now: () => t });
175 facts.turn({ kind: "turn", ended: null, started: person("a") } as never, snap());
176 const id = facts.turn({ kind: "turn", ended: { who: person("a"), reason: "done" }, started: null } as never, snap())!;
177 expect(facts.traceFor(id)).toBeNull();
178 facts.trace(id, trace(3));
179 expect(facts.traceFor(id)).toEqual(trace(3));
180 facts.trace(id, "not a trace");
181 expect(facts.traceFor(id)).toEqual(trace(3));
182 facts.trace(id, trace(200_000));
183 const big = facts.traceFor(id) as ReturnType<typeof trace> & { truncated: boolean };
184 expect(JSON.stringify(big).length).toBeLessThanOrEqual(MAX_TRACE_BYTES);
185 expect(big.truncated).toBe(true);
186 expect(big.joints.shoulder_pan.intent.length).toBe(big.joints.shoulder_pan.actual.length);
187 t = 1000 + KEEP_MS + 1;
188 facts.prune();
189 expect(facts.traceFor(id)).toBeNull();
190 });
191
192 test("the session of a turn is the newest stored one that ended since it began", () => {
193 const at = new Date(5000).toISOString();
194 expect(parseSessionId({ sessions: [{ id: 3, at }] }, 4000)).toBe(3);
195 expect(parseSessionId({ sessions: [{ id: 3, at }] }, 6000)).toBeNull();
196 expect(parseSessionId({ sessions: [{ at }] }, 0)).toBeNull();
197 expect(parseSessionId({}, 0)).toBeNull();
198 });
199
200 async function traceRelay(route: "trace" | "404") {
201 const facts = new Facts({ commit: "t" });
202 const { Lobby } = await import("./lobby.ts");
203 const { startRelay } = await import("./relay.ts");
204 const sim = Bun.serve({
205 port: 0,
206 hostname: "127.0.0.1",
207 fetch: (req) => {
208 const path = new URL(req.url).pathname;
209 if (path === "/status") return Response.json({ joints: [], held: null });
210 if (path === "/precision") return Response.json({ session: { releases: 1 }, sessions: [{ id: 9, at: new Date(Date.now() + 5000).toISOString() }] });
211 return route === "trace" ? Response.json(trace(4)) : Response.json({ error: "no" }, { status: 404 });
212 },
213 });
214 const lobby = new Lobby({ identityKey: "k".repeat(64) });
215 const relay = await startRelay({ port: 0, simulation: `http://127.0.0.1:${sim.port}`, lobby, facts });
216 lobby.turns.join({ id: "id-ann", name: "ann", owner: false });
217 lobby.turns.end("id-ann");
218 for (let i = 0; i < 100 && facts.turns()[0]?.releases == null; i++) await Bun.sleep(50);
219 await Bun.sleep(200);
220 const done = async () => {
221 await relay.close();
222 lobby.close();
223 sim.stop(true);
224 };
225 return { facts, done };
226 }
227
228 test("the relay stores the turn's trace; an arm that 404s stores nothing and the row is still there", async () => {
229 const ok = await traceRelay("trace");
230 try {
231 expect(ok.facts.traceFor(ok.facts.turns()[0]!.id)).toEqual(trace(4));
232 } finally {
233 await ok.done();
234 }
235 const none = await traceRelay("404");
236 try {
237 expect(none.facts.turns()[0]).toMatchObject({ releases: 1 });
238 expect(none.facts.traceFor(none.facts.turns()[0]!.id)).toBeNull();
239 } finally {
240 await none.done();
241 }
242 });
243
170244 test("the minute sample stores the arm's pi temperature and under-voltage flag", async () => {
171245 const facts = new Facts();
172246 const { stop } = await relayWith(facts, undefined, { pi: { tempC: 61.5, underVoltage: true }, sampleEveryMs: 30 });
Software/so101/src/facts.ts +43 −2
@@ -47,6 +47,32 @@ export function parseCard(body: unknown): TurnCard | null {
4747 return { releases: count(card.releases), snapBacks, jams: snapBacks.jam ?? 0, arriveP95Ms: typeof p95 === "number" && Number.isFinite(p95) ? p95 : null };
4848 }
4949
50 /** The newest stored session of the arm's GET /precision that began at or after `sinceMs`: the id its trace is read by. */
51 export function parseSessionId(body: unknown, sinceMs: number): number | null {
52 const rows = (body as { sessions?: unknown } | null)?.sessions;
53 const last = Array.isArray(rows) ? (rows.at(-1) as { id?: unknown; at?: unknown; seconds?: unknown } | null) : null;
54 if (!last || typeof last.id !== "number" || !Number.isSafeInteger(last.id) || last.id < 0) return null;
55 const end = typeof last.at === "string" ? Date.parse(last.at) : NaN;
56 return end >= sinceMs ? last.id : null;
57 }
58
59 /** The most a stored trace may take, as JSON; a longer one keeps its first samples. */
60 export const MAX_TRACE_BYTES = 256 * 1024;
61
62 /** A trace as the arm served it, cut to fit MAX_TRACE_BYTES; null when it is not one. */
63 export function capTrace(body: unknown): string | null {
64 const t = body && typeof body === "object" ? (body as { joints?: unknown }) : null;
65 if (!t || !t.joints || typeof t.joints !== "object" || Array.isArray(t.joints)) return null;
66 let text = JSON.stringify(body);
67 let keep = Math.max(...Object.values(t.joints as Record<string, { intent?: unknown[] }>).map((j) => (Array.isArray(j?.intent) ? j.intent.length : 0)), 0);
68 while (text.length > MAX_TRACE_BYTES && keep > 0) {
69 keep = Math.floor(keep * Math.min(0.9, MAX_TRACE_BYTES / text.length));
70 const joints = Object.fromEntries(Object.entries(t.joints as Record<string, { intent?: unknown[]; actual?: unknown[] }>).map(([k, j]) => [k, { intent: (j.intent ?? []).slice(0, keep), actual: (j.actual ?? []).slice(0, keep) }]));
71 text = JSON.stringify({ ...(body as object), joints, truncated: true });
72 }
73 return text.length <= MAX_TRACE_BYTES ? text : null;
74 }
75
5076 export interface Sample {
5177 watchers: number;
5278 drivers: number;
@@ -105,7 +131,8 @@ export class Facts {
105131 const have = (this.#db.query("PRAGMA table_info(samples)").all() as Array<{ name: string }>).map((c) => c.name);
106132 if (!have.includes("pi_temp")) this.#db.run("ALTER TABLE samples ADD COLUMN pi_temp REAL");
107133 if (!have.includes("pi_undervolt")) this.#db.run("ALTER TABLE samples ADD COLUMN pi_undervolt INTEGER");
108 for (const t of ["turns", "events", "samples"]) this.#db.run(`CREATE INDEX IF NOT EXISTS ${t}_at ON ${t} (at)`);
134 this.#db.run("CREATE TABLE IF NOT EXISTS traces (turn_id INTEGER PRIMARY KEY, at INTEGER NOT NULL, body TEXT NOT NULL)");
135 for (const t of ["turns", "events", "samples", "traces"]) this.#db.run(`CREATE INDEX IF NOT EXISTS ${t}_at ON ${t} (at)`);
109136 this.#attempt(() => this.prune());
110137 }
111138
@@ -183,11 +210,25 @@ export class Facts {
183210 this.#attempt(() => void this.#db.query("UPDATE turns SET releases = ?, snap_backs = ?, jams = ?, arrive_p95_ms = ? WHERE id = ?").run(card.releases, JSON.stringify(card.snapBacks), card.jams, card.arriveP95Ms, turnId));
184211 }
185212
213 /** The arm's trace for a turn, stored with it (capped); a body that is not a trace stores nothing. */
214 trace(turnId: number, body: unknown): void {
215 this.#attempt(() => {
216 const text = capTrace(body);
217 if (text) this.#db.query("INSERT OR REPLACE INTO traces (turn_id, at, body) VALUES (?, ?, ?)").run(turnId, this.#now(), text);
218 });
219 }
220
221 /** The trace kept for a turn, or null when there is none (the arm had no route, or it is gone at 90 days). */
222 traceFor(turnId: number): unknown | null {
223 const r = this.#attempt(() => this.#db.query("SELECT body FROM traces WHERE turn_id = ?").get(turnId) as { body: string } | null);
224 return r ? JSON.parse(r.body) : null;
225 }
226
186227 /** Drop what is older than 90 days. */
187228 prune(): void {
188229 const before = this.#now() - KEEP_MS;
189230 this.#pruned = this.#now();
190 for (const t of ["turns", "events", "samples"]) this.#db.query(`DELETE FROM ${t} WHERE at < ?`).run(before);
231 for (const t of ["turns", "events", "samples", "traces"]) this.#db.query(`DELETE FROM ${t} WHERE at < ?`).run(before);
191232 }
192233
193234 turns(): TurnRow[] {
Software/so101/src/precision-history.ts +5 −2
@@ -87,6 +87,8 @@ export interface Row {
8787 snapBacks: number;
8888 releases: number;
8989 worstJoint: string | null;
90 /** A stored session's id, the one /precision/trace takes; the hourly rows have none. */
91 id?: number;
9092 }
9193
9294 /** One P experiment as kept: the run's own numbers, tagged with the commit that ran it. */
@@ -125,7 +127,7 @@ type Column = (typeof HEADLINE)[number][1];
125127 const RETIRED = "tracking_p95_deg REAL, tracking_max_deg REAL, latency_p95_ms REAL";
126128 const SELECT = `at, commit_id, sessions, seconds, ${HEADLINE.map(([c]) => c).join(", ")}, worst_joint`;
127129
128 type Stored = Record<string, unknown> & { at: string; commit_id: string; sessions: number; seconds: number; worst_joint: string | null };
130 type Stored = Record<string, unknown> & { id?: number; at: string; commit_id: string; sessions: number; seconds: number; worst_joint: string | null };
129131 const fromStored = (r: Stored): Row => ({
130132 at: r.at,
131133 commit: r.commit_id,
@@ -133,6 +135,7 @@ const fromStored = (r: Stored): Row => ({
133135 seconds: r.seconds,
134136 ...(Object.fromEntries(HEADLINE.map(([c, k, fold]) => [k, (r[c] as number | null) ?? (fold === "sum" ? 0 : null)])) as Record<Column, number>),
135137 worstJoint: r.worst_joint,
138 ...(r.id === undefined ? {} : { id: r.id }),
136139 });
137140
138141 /** A card's headline, as the columns hold it. */
@@ -242,7 +245,7 @@ export class PrecisionHistory {
242245
243246 /** The stored rows, oldest first. */
244247 rows(table: "sessions" | "hours", limit = 48): Row[] {
245 const sql = table === "sessions" ? `SELECT ${SELECT} FROM sessions ORDER BY id DESC LIMIT ?` : `SELECT ${SELECT} FROM hours ORDER BY at DESC LIMIT ?`;
248 const sql = table === "sessions" ? `SELECT id, ${SELECT} FROM sessions ORDER BY id DESC LIMIT ?` : `SELECT ${SELECT} FROM hours ORDER BY at DESC LIMIT ?`;
246249 const rows = this.#db.query(sql).all(limit) as Stored[];
247250 return rows.reverse().map(fromStored);
248251 }
Software/so101/src/relay.ts +9 −2
@@ -64,7 +64,7 @@ import { LiveMetrics } from "./live-metrics.ts";
6464 import { wallMs } from "./clock.ts";
6565 import { ArmLink, type LinkTimings } from "./arm-link.ts";
6666 import { ArmEdges } from "./arm-edges.ts";
67 import { parseCard, parsePiHealth, type Facts } from "./facts.ts";
67 import { parseCard, parsePiHealth, parseSessionId, type Facts } from "./facts.ts";
6868 import { MachineFeed } from "./machine-feed.ts";
6969 import { refusalBatch } from "./refusals.ts";
7070 import { BROWSER_HOP, MOTION_HOP, PAGE_HOP, parseSnapBack } from "./precision.ts";
@@ -1136,8 +1136,15 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
11361136 setTimeout(() => {
11371137 armPrecision(null)
11381138 .then(async (answer) => {
1139 const card = answer.ok ? parseCard(await answer.json()) : null;
1139 const body = answer.ok ? await answer.json() : null;
1140 const card = parseCard(body);
11401141 if (card) facts.card(row, card);
1142 // The turn's trace, once: the arm's newest stored session that began with the turn; an arm without the route stores nothing.
1143 const startedAt = Date.parse(facts.turns().find((t) => t.id === row)?.startedAt ?? "");
1144 const session = parseSessionId(body, startedAt);

facts.turns().find(...) scans the listing to recover a start time you already had when the turn ended; pass it in, or add a Facts lookup by id.

1145 if (session === null) return;
1146 const traced = await fetch(`${remote && onArm() ? remote.http : simulation}/precision/trace?session=${session}`, { cache: "no-store", signal: AbortSignal.timeout(2000) });
1147 if (traced.ok) facts.trace(row, await traced.json());
11411148 })
11421149 .catch(() => undefined); // the row stays without its card
11431150 }, 1000).unref?.();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs