Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

GET /data?from&to&step[&compare=1] and ?turn=<id> serve the facts as series for owner and moderators (src/data-series.ts, src/data-api.ts) #550

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/529-get-data-from-to-step-compare-1-and-turn → main 4 commits fork 1 thread · 1 unresolved

  • Merged. smith/529-get-data-from-to-step-compare-1-and-turn was merged into main as cc68622.

Review

Sign in to review this change.
7 files changed +593 −20
Software/so101/README.md +11
@@ -70,6 +70,17 @@ The relay (`src/relay.ts`, through `src/facts.ts`) keeps 90 days of what happene
7070
7171 "Who" is an account id or a 16-hex hash of the browser id, never a name or address.
7272
73 ## Reading the facts: GET /data
74
75 `GET /data` on the hosted app answers the kept facts as JSON (src/data-api.ts, src/data-series.ts; how each number is made is at the top of the latter). It is for the owner and moderators; anonymous visitors and plain users get 403. There is no terminal command for it.
76
77 - `from`, `to`: ms since the epoch, `from` before `to`; default the last 24 hours.
78 - `step`: seconds, one of 60, 300, 900, 1800, 3600, 10800, 21600, 86400; default about 96 steps over the range, at most 400.
79 - `compare=1`: adds `previous`, the same length of time just before, on the same steps (`compare=0` is the default).
80 - `turn=<id>`: instead of a range, one turn: `{ turn, events, samples }` from a minute before to a minute after, with the arm's events only (STOP, jam, block, reconnect, app start) and no one named. An unknown id is 404.
81
82 A bad parameter is a 400 whose `error` says what to send; POST is 405. Shape of a range answer (trimmed): `{ from, to, step, t: [step starts], overview: { visitors, sessions, drivers, minutes, snapBacksPer100, arriveP95Ms, jams, uptime }, series: { sessions: [..], snapBacksPer100: [..], watchers: [..], uptime: [..], ... one array per step }, funnel, turnLengths, reasons, versions, live: { watchers, driving, queue }, oldest, commit, previous? }`.
83
7384 ## Faster limits
7485
7586 A person runs this at the arm while nobody drives: `bun bench/limits-search.ts http://<arm>:8101` (add `--joint 2` for one joint; with no URL it runs on the fake bus). From today's 90 °/s and 360 °/s² it steps each joint's speed and acceleration up within the safety ranges (the gripper is skipped), sweeps at each step, and keeps the last step that did not make p95 error or overshoot worse, writing them through `PUT /settings`. It prints its command line, then a table of limits and p95 error, overshoot and time to arrive, before → after. It refuses to start while anyone drives or a STOP or E-stop is in force, and if one appears mid-run it stops and puts the starting limits back.
Software/so101/src/data-api.test.ts added +80
@@ -0,0 +1,80 @@
1 import { afterEach, expect, test } from "bun:test";
2 import { mkdtempSync, rmSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { Facts, whoIs } from "./facts.ts";
6 import { sign } from "./identity.ts";
7 import { Lobby } from "./lobby.ts";
8 import { startRelay, type RunningRelay } from "./relay.ts";
9
10 const KEY = "k".repeat(64);
11 const later = Date.now() + 600_000;
12 const as = (name: string, role: "user" | "admin") => {
13 const identity = `id-${name}|${name}|${role}|${later}`;
14 return { "x-forge-identity": identity, "x-forge-signature": sign(identity, KEY) };
15 };
16 const owner = as("trevato", "admin");
17
18 let dir = "";
19 let relay: RunningRelay | undefined;
20 let lobby: Lobby | undefined;
21 afterEach(async () => {
22 await relay?.close();
23 lobby?.close();
24 if (dir) rmSync(dir, { recursive: true, force: true });
25 dir = "";
26 relay = lobby = undefined;
27 });
28
29 async function boot() {
30 dir = mkdtempSync(join(tmpdir(), "so101-data-api-"));
31 const facts = new Facts({ file: join(dir, "facts.sqlite"), commit: "abc1234def56" });
32 lobby = new Lobby({ identityKey: KEY, adminFile: join(dir, "admin.json"), countsFile: join(dir, "visitors.json") });
33 relay = await startRelay({ port: 0, simulation: "http://127.0.0.1:1", lobby, facts });
34 return { base: `http://127.0.0.1:${relay.port}`, facts };
35 }
36
37 test("GET /data: anonymous and plain users get 403; the owner and a moderator get the series", async () => {
38 const { base, facts } = await boot();
39 expect((await fetch(`${base}/data`)).status).toBe(403);
40 expect((await fetch(`${base}/data`, { headers: as("eve", "user") })).status).toBe(403);
41 await fetch(`${base}/admin`, { method: "POST", headers: { ...owner, "content-type": "application/x-www-form-urlencoded" }, body: new URLSearchParams({ action: "mod-add", name: "bob" }), redirect: "manual" });
42
43 const ada = { id: "a-ada", name: "ada", owner: false };
44 const snapshot = { driver: null, queue: [] } as never;
45 facts.turn({ kind: "turn", ended: null, started: ada }, snapshot);
46 const id = facts.turn({ kind: "turn", ended: { who: ada, reason: "time is up" }, started: null }, snapshot)!;
47 facts.event("app start");
48 facts.event("STOP");
49 for (const who of [owner, as("bob", "user")]) {
50 const res = await fetch(`${base}/data?compare=1`, { headers: who });
51 expect(res.status).toBe(200);
52 expect(res.headers.get("cache-control")).toBe("no-store");
53 expect(res.headers.get("x-content-type-options")).toBe("nosniff");
54 const body = (await res.json()) as Record<string, any>;
55 expect(body.step).toBe(900);
56 expect(body.t).toHaveLength(96);
57 expect(body.overview.sessions).toBe(1);
58 expect(body.commit).toBe("abc1234def56");
59 expect(body.versions[0]).toMatchObject({ commit: "abc1234def56" });
60 expect(body.previous.overview.sessions).toBe(0);
61 expect(body.live).toEqual({ watchers: 0, driving: 0, queue: 0 });
62 }
63
64 const one = (await (await fetch(`${base}/data?turn=${id}`, { headers: owner })).json()) as Record<string, any>;
65 expect(one.turn).toMatchObject({ id, who: whoIs("a-ada"), ended: "time is up" });
66 expect(one.events.map((e: { kind: string }) => e.kind)).toContain("STOP");
67 expect(one.events.every((e: Record<string, unknown>) => !("who" in e))).toBe(true);
68 expect((await fetch(`${base}/data?turn=${id}`)).status).toBe(403);
69 });
70
71 test("GET /data refuses a bad range or turn with a 400 that says what to send; POST is 405", async () => {
72 const { base } = await boot();
73 for (const q of ["from=yesterday", "from=10&to=5", "step=7", "compare=yes", "turn=-1"]) {
74 const res = await fetch(`${base}/data?${q}`, { headers: owner });
75 expect(res.status).toBe(400);
76 expect(((await res.json()) as { error: string }).error).toBeTruthy();
77 }
78 expect((await fetch(`${base}/data?turn=999`, { headers: owner })).status).toBe(404);
79 expect((await fetch(`${base}/data`, { method: "POST", headers: owner })).status).not.toBe(200);
80 });
Software/so101/src/data-api.ts added +52
@@ -0,0 +1,52 @@
1 /**
2 * GET /data: the Data view's one read, for the owner and moderators (anyone else: 403).
3 * ?from=<ms>&to=<ms>&step=<s>[&compare=1] the series over the range, per step (src/data-series.ts); compare=1 adds
4 * `previous`, the same length of time just before, on the same steps
5 * ?turn=<id> one turn: its row, and the events and samples from a minute before to a minute after
6 * Every parameter is checked here; a bad one is a 400 that says what to send.
7 */
8
9 import type { IncomingMessage, ServerResponse } from "node:http";
10 import type { Facts } from "./facts.ts";
11 import type { Lobby } from "./lobby.ts";
12 import { aggregate, parseRange } from "./data-series.ts";
13
14 const MINUTE = 60_000;
15 /** Shown on a turn's timeline: what happened to the arm, not who opened the page or spoke. */
16 const ARM_EVENTS = new Set(["STOP", "jam", "block", "reconnect", "app start"]);
17
18 function answer(res: ServerResponse, status: number, body: unknown, head = false): void {
19 res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" });
20 res.end(head ? undefined : JSON.stringify(body));
21 }
22 /** `now` is one ms ahead so a row written in this very millisecond is inside the range, which ends before `to`. */
23 export function serveData(facts: Facts | undefined, lobby: Lobby | undefined, req: IncomingMessage, res: ServerResponse, url: URL, now = Date.now() + 1): void {
24 if (!lobby || !facts) return answer(res, 404, { error: "no data is kept here" });
25 if (!lobby.who(req) || !lobby.moderates(req)) return answer(res, 403, { error: "the data is the owner's and the moderators'" });
26 if (req.method !== "GET" && req.method !== "HEAD") {
27 res.setHeader("allow", "GET, HEAD");
28 return answer(res, 405, { error: "GET" });
29 }

The 403 is correct. Please document this door in the README (parameters, who may call it, response keys).

30 const head = req.method === "HEAD";
31 const turn = url.searchParams.get("turn");
32 if (turn !== null) {
33 if (!/^\d{1,12}$/.test(turn)) return answer(res, 400, { error: "turn is a turn's id" });
34 const row = facts.turnById(Number(turn));
35 if (!row) return answer(res, 404, { error: "no such turn" });
36 const start = Date.parse(row.startedAt);
37 const [from, to] = [start - MINUTE, start + row.seconds * 1000 + MINUTE];
38 return answer(res, 200, { turn: row, events: facts.events(from, to).filter((e) => ARM_EVENTS.has(e.kind)).map(({ who: _who, ...e }) => e), samples: facts.samples(from, to) }, head);
39 }
40 const range = parseRange(url.searchParams, now);
41 if ("error" in range) return answer(res, 400, range);
42 const compare = url.searchParams.get("compare");
43 if (compare !== null && compare !== "1" && compare !== "0") return answer(res, 400, { error: "compare is 1 or 0" });
44 const oldest = facts.oldest();
45 const over = (from: number, to: number) => aggregate({ from, to, step: range.step, now, oldest, turns: facts.turns(from, to), events: facts.events(from, to), samples: facts.samples(from, to) });
46 const body = over(range.from, range.to);
47 const span = range.to - range.from;
48 const previous = compare === "1" ? over(range.from - span, range.from) : null;
49 const snapshot = lobby.turns.snapshot();
50 const live = { watchers: lobby.watching, driving: snapshot.driver ? 1 : 0, queue: snapshot.queue.length };
51 return answer(res, 200, { ...body, live, oldest, commit: facts.commit, previous: previous && { overview: previous.overview, series: previous.series, funnel: previous.funnel } }, head);
52 }
Software/so101/src/data-series.test.ts added +111
@@ -0,0 +1,111 @@
1 import { describe, expect, test } from "bun:test";
2 import { aggregate, autoStep, histogram, MAX_POINTS, parseRange, percentile, type Input } from "./data-series.ts";
3 import type { EventRow, SampleRow, TurnRow } from "./facts.ts";
4
5 const H = 3_600_000;
6 const T0 = 1_700_000_000_000 - (1_700_000_000_000 % H);
7
8 const turn = (over: Partial<TurnRow> & { start?: number }): TurnRow => {
9 const { start = T0, ...rest } = over;
10 return { id: 1, at: new Date(start).toISOString(), commit: "abc1234", who: "a-1", startedAt: new Date(start).toISOString(), seconds: 60, ended: "time is up", waitSeconds: 0, releases: null, snapBacks: null, jams: null, arriveP95Ms: null, ...rest };
11 };
12 const ev = (at: number, kind: EventRow["kind"], who: string | null = null, commit = "abc1234"): EventRow => ({ at, kind, who, commit });
13 const sample = (at: number, over: Partial<SampleRow> = {}): SampleRow => ({ at, commit: "abc1234", watchers: 0, drivers: 0, queue: 0, linkLive: true, fps: {}, ...over });
14 const input = (over: Partial<Input>): Input => ({ from: T0, to: T0 + 4 * H, step: 3600, now: T0 + 4 * H, oldest: T0, turns: [], events: [], samples: [], ...over });
15
16 describe("percentile and histogram", () => {
17 test("nearest rank: p95 of 1..20 is 19, p50 of 1..4 is 2, of nothing null", () => {
18 expect(percentile(Array.from({ length: 20 }, (_, i) => i + 1), 95)).toBe(19);
19 expect(percentile([4, 1, 3, 2], 50)).toBe(2);
20 expect(percentile([7], 95)).toBe(7);
21 expect(percentile([], 95)).toBeNull();
22 });
23
24 test("a histogram counts below the first edge, between, and at or past the last", () => {
25 expect(histogram([0, 9.9, 10, 29, 30, 500], [10, 30])).toEqual([2, 2, 2]);
26 });
27
28 test("autoStep keeps about 96 points: an hour by the minute, a day by 15 min, a week by 3 h", () => {
29 expect(autoStep(H)).toBe(60);
30 expect(autoStep(24 * H)).toBe(900);
31 expect(autoStep(7 * 24 * H)).toBe(3 * 3600);
32 expect(autoStep(90 * 24 * H)).toBe(86_400);
33 });
34 });
35
36 describe("aggregate", () => {
37 test("snap-backs per 100 releases are summed then divided, never an average of ratios", () => {
38 const a = aggregate(input({ turns: [turn({ id: 1, releases: 10, snapBacks: { jam: 5 }, jams: 5 }), turn({ id: 2, start: T0 + 60_000, releases: 90, snapBacks: {}, jams: 0 })] }));
39 // the mean of 50 and 0 would be 25; the truth is 5 in 100
40 expect(a.overview.snapBacksPer100).toBe(5);
41 expect(a.series.snapBacksPer100).toEqual([5, null, null, null]);
42 expect(a.series.byCause.jam).toEqual([5, 0, 0, 0]);
43 });
44
45 test("steps: each turn lands in the step it began; sessions, drivers and minutes add up", () => {
46 const a = aggregate(input({ turns: [turn({ id: 1, who: "a-1", seconds: 30 }), turn({ id: 2, who: "a-1", start: T0 + 2 * H, seconds: 90 }), turn({ id: 3, who: "b", start: T0 + 2 * H + 1, seconds: 5 })] }));
47 expect(a.t).toEqual([T0, T0 + H, T0 + 2 * H, T0 + 3 * H]);
48 expect(a.series.sessions).toEqual([1, 0, 2, 0]);
49 expect(a.series.drivers).toEqual([1, 0, 2, 0]);
50 expect(a.overview).toMatchObject({ sessions: 3, drivers: 2, minutes: 2.1 });
51 expect(a.turnLengths.counts).toEqual([1, 0, 1, 1, 0, 0]);
52 });
53
54 test("a step's p95 time to arrive is the p95 of its turns' p95s, so one bad turn shows", () => {
55 const turns = Array.from({ length: 20 }, (_, i) => turn({ id: i, start: T0 + i, releases: 1, snapBacks: {}, arriveP95Ms: i === 19 ? 2000 : 100 }));
56 const a = aggregate(input({ turns }));
57 expect(a.series.arriveP95Ms[0]).toBe(100);
58 expect(a.overview.arriveP95Ms).toBe(100);
59 const worse = aggregate(input({ turns: turns.map((t, i) => (i === 18 ? { ...t, arriveP95Ms: 1500 } : t)) }));
60 expect(worse.overview.arriveP95Ms).toBe(1500);
61 });
62
63 test("visitors are distinct people per step, anonymous against account; the funnel counts how far each got", () => {
64 const events = [ev(T0, "visit", "0123456789abcdef"), ev(T0 + 1, "visit", "0123456789abcdef"), ev(T0 + 2, "visit", "a-ada"), ev(T0 + 3, "visit", "a-bob"), ev(T0 + H, "visit", "0123456789abcdef"), ev(T0 + 5, "name set", "a-ada"), ev(T0 + 6, "joined line", "a-ada"), ev(T0 + 7, "chat", "a-ada"), ev(T0 + 8, "chat", "a-bob")];
65 const a = aggregate(input({ events, turns: [turn({ who: "a-ada", seconds: 5 }), turn({ id: 2, who: "a-bob", seconds: 40 })] }));
66 expect(a.series.visitorsAnon).toEqual([1, 1, 0, 0]);
67 expect(a.series.visitorsNamed).toEqual([2, 0, 0, 0]);
68 expect(a.overview.visitors).toBe(3);
69 expect(a.funnel).toEqual({ visitors: 3, named: 2, queued: 1, drove: 2, under10s: 1 });
70 expect(a.series.chat).toEqual([2, 0, 0, 0]);
71 });
72
73 test("uptime is live minutes over minutes passed since the record began; no samples is null", () => {
74 const samples = Array.from({ length: 60 }, (_, i) => sample(T0 + i * 60_000, { linkLive: i < 45 }));
75 const a = aggregate(input({ samples, now: T0 + H }));
76 expect(a.series.uptime[0]).toBe(0.75);
77 expect(a.series.uptime[1]).toBeNull(); // in the future
78 expect(a.overview.uptime).toBe(0.75);
79 expect(aggregate(input({})).overview.uptime).toBeNull();
80 });
81
82 test("room samples show each step's peak; camera fps their mean", () => {
83 const a = aggregate(input({ samples: [sample(T0, { watchers: 3, fps: { front: 20 } }), sample(T0 + 60_000, { watchers: 7, fps: { front: 30 } })] }));
84 expect(a.series.watchers).toEqual([7, null, null, null]);
85 expect(a.series.fps.front).toEqual([25, null, null, null]);
86 });
87
88 test("event counts per step, and one version marker per change of commit", () => {
89 const events = [ev(T0, "app start", null, "aaa1111"), ev(T0 + 10, "app start", null, "aaa1111"), ev(T0 + 11, "STOP"), ev(T0 + 12, "jam"), ev(T0 + H, "app start", null, "bbb2222")];
90 const a = aggregate(input({ events }));
91 expect(a.versions).toEqual([{ at: T0, commit: "aaa1111" }, { at: T0 + H, commit: "bbb2222" }]);
92 expect(a.series.restarts).toEqual([2, 1, 0, 0]);
93 expect(a.series.stops).toEqual([1, 0, 0, 0]);
94 expect(a.overview.jams).toBe(1);
95 });
96 });
97
98 describe("parseRange", () => {
99 const now = T0 + 24 * H;
100 const p = (q: string) => parseRange(new URLSearchParams(q), now);
101 test("defaults to the last day at its auto step", () => {
102 expect(p("")).toEqual({ from: now - 24 * H, to: now, step: 900 });
103 });
104 test("refuses what is not a range", () => {
105 expect(p("from=abc")).toHaveProperty("error");
106 expect(p(`from=${now}&to=${now - 1}`)).toHaveProperty("error");
107 expect(p("step=7")).toHaveProperty("error");
108 expect(p(`from=${now - (MAX_POINTS + 1) * 60_000}&to=${now}&step=60`)).toHaveProperty("error");
109 expect(p(`from=${now - H}&to=${now}&step=300`)).toEqual({ from: now - H, to: now, step: 300 });
110 });
111 });
Software/so101/src/data-series.ts added +283
@@ -0,0 +1,283 @@
1 /**
2 * The Data view's numbers: src/facts.ts's rows over a time range, cut into steps. Plain data in, plain data out.
3 *
4 * What each number is, so a chart never claims more than was measured:
5 * - visitors: distinct people who opened the page (an account id, or the hash of a browser); named = an account.
6 * - a session is a turn, placed in the step it began; drivers are distinct people who had one; minutes are the turns' lengths.
7 * - snap-backs per 100 releases: both summed over the turns the arm measured, then divided (never an average of ratios).
8 * - p95 time to arrive: each turn keeps its own p95 (the arm's src/precision.ts); a step's number is the p95 of its
9 * turns' p95s (nearest rank), so one bad turn shows. Waits and turn lengths: nearest-rank p50 and p95 of the turns.
10 * - watchers, drivers now, queue: the minute samples; a step shows the most seen. Camera fps: their mean.
11 * - uptime: the minutes the arm link was sampled live over the minutes that have passed since the range (or the
12 * record) began, in a step and in all.
13 * - counts of chat, STOP, jam, block and reconnect events are what happened in the step, nothing smoothed.
14 */
15
16 import type { EventRow, SampleRow, TurnRow } from "./facts.ts";
17
18 /** The steps a request may name, in seconds, smallest first. */
19 export const STEPS = [60, 300, 900, 1800, 3600, 3 * 3600, 6 * 3600, 86_400] as const;
20 /** The most steps one answer carries. */
21 export const MAX_POINTS = 400;
22 const TARGET_POINTS = 96;
23 const MINUTE = 60_000;
24
25 /** The step that cuts a range into about TARGET_POINTS, never more than MAX_POINTS. */
26 export function autoStep(ms: number): number {
27 for (const s of STEPS) if (ms / (s * 1000) <= TARGET_POINTS) return s;
28 return STEPS[STEPS.length - 1]!;
29 }
30
31 /** Nearest-rank percentile; null for no values. */
32 export function percentile(values: readonly number[], p: number): number | null {
33 if (values.length === 0) return null;
34 const sorted = [...values].sort((a, b) => a - b);
35 const rank = Math.max(1, Math.ceil((p / 100) * sorted.length));
36 return sorted[rank - 1]!;
37 }
38
39 const round = (v: number | null, digits = 1): number | null => (v === null || !Number.isFinite(v) ? null : Number(v.toFixed(digits)));
40 const sum = (values: readonly number[]): number => values.reduce((a, b) => a + b, 0);
41 const mean = (values: readonly number[]): number | null => (values.length ? sum(values) / values.length : null);
42
43 export const TURN_BINS = [10, 30, 60, 120, 300] as const;
44 /** Counts per bin: below the first edge, between each pair, and at or past the last. */
45 export function histogram(values: readonly number[], edges: readonly number[]): number[] {
46 const out = new Array<number>(edges.length + 1).fill(0);
47 for (const v of values) {
48 let i = 0;
49 while (i < edges.length && v >= edges[i]!) i++;
50 out[i]!++;
51 }
52 return out;
53 }
54
55 export interface Overview {
56 visitors: number;
57 sessions: number;
58 drivers: number;
59 minutes: number;
60 snapBacksPer100: number | null;
61 arriveP95Ms: number | null;
62 jams: number;
63 /** 0–1; null with no samples. */
64 uptime: number | null;
65 }
66
67 export interface Series {
68 visitorsAnon: number[];
69 visitorsNamed: number[];
70 sessions: number[];
71 drivers: number[];
72 minutes: number[];
73 releases: number[];
74 snapBacks: number[];
75 snapBacksPer100: Array<number | null>;
76 byCause: Record<string, number[]>;
77 arriveP95Ms: Array<number | null>;
78 watchers: Array<number | null>;
79 driving: Array<number | null>;
80 queue: Array<number | null>;
81 waitP50S: Array<number | null>;
82 waitP95S: Array<number | null>;
83 turnP50S: Array<number | null>;
84 chat: number[];
85 stops: number[];
86 jams: number[];
87 blocks: number[];
88 reconnects: number[];
89 restarts: number[];
90 uptime: Array<number | null>;
91 fps: Record<string, Array<number | null>>;
92 }
93
94 export interface Marker {
95 at: number;
96 commit: string;
97 }
98
99 export interface Aggregate {
100 from: number;
101 to: number;
102 step: number;
103 /** Each step's start, ms. */
104 t: number[];
105 overview: Overview;
106 series: Series;
107 /** Who got how far: opened the page, set a name, joined the line, drove, drove under 10 s (each distinct people). */
108 funnel: { visitors: number; named: number; queued: number; drove: number; under10s: number };
109 turnLengths: { edgesS: readonly number[]; counts: number[] };
110 /** How turns ended, by the reason kept. */
111 reasons: Record<string, number>;
112 /** Each time the app started on a commit other than the one before. */
113 versions: Marker[];
114 }
115
116 export interface Input {
117 from: number;
118 to: number;
119 /** Seconds. */
120 step: number;
121 /** Now: no step past it is uptime's fault. */
122 now: number;
123 /** The first moment anything was recorded; uptime counts from there. */
124 oldest: number | null;
125 turns: readonly TurnRow[];
126 events: readonly EventRow[];
127 samples: readonly SampleRow[];
128 }
129
130 export function aggregate(input: Input): Aggregate {
131 const { from, to, now } = input;
132 const stepMs = input.step * 1000;
133 const n = Math.max(1, Math.ceil((to - from) / stepMs));
134 const t = Array.from({ length: n }, (_, i) => from + i * stepMs);
135 const at = (ms: number) => Math.min(n - 1, Math.max(0, Math.floor((ms - from) / stepMs)));
136 const zeros = () => new Array<number>(n).fill(0);
137 const lists = () => Array.from({ length: n }, () => [] as number[]);
138 const pct = (l: number[][], p: number, digits = 1) => l.map((v) => round(percentile(v, p), digits));
139
140 // ---- the room's events
141 const anon = Array.from({ length: n }, () => new Set<string>());
142 const named = Array.from({ length: n }, () => new Set<string>());
143 const people = { visitors: new Set<string>(), named: new Set<string>(), queued: new Set<string>() };
144 const count = { chat: zeros(), STOP: zeros(), jam: zeros(), block: zeros(), reconnect: zeros(), "app start": zeros() };
145 const versions: Marker[] = [];
146 for (const e of input.events) {
147 const i = at(e.at);
148 if (e.kind === "visit" && e.who) {
149 people.visitors.add(e.who);
150 (e.who.startsWith("a-") ? named : anon)[i]!.add(e.who);
151 } else if (e.kind === "name set" && e.who) people.named.add(e.who);
152 else if (e.kind === "joined line" && e.who) people.queued.add(e.who);
153 else if (e.kind in count) count[e.kind as keyof typeof count][i]!++;
154 if (e.kind === "app start" && e.commit !== versions[versions.length - 1]?.commit) versions.push({ at: e.at, commit: e.commit });
155 }
156 for (const s of named) for (const who of s) people.named.add(who);
157
158 // ---- turns
159 const sessions = zeros(), minutes = zeros(), releases = zeros(), snapBacks = zeros();
160 const driverSets = Array.from({ length: n }, () => new Set<string>());
161 const byCause: Record<string, number[]> = {};
162 const arrive = lists(), waits = lists(), lengths = lists();
163 const drove = new Map<string, number>();
164 const reasons: Record<string, number> = {};
165 const arrives: number[] = [];
166 for (const turn of input.turns) {
167 const i = at(Date.parse(turn.startedAt));
168 sessions[i]!++;
169 driverSets[i]!.add(turn.who);
170 minutes[i]! += turn.seconds / 60;
171 lengths[i]!.push(turn.seconds);
172 waits[i]!.push(turn.waitSeconds);
173 drove.set(turn.who, Math.max(drove.get(turn.who) ?? 0, turn.seconds));
174 reasons[turn.ended] = (reasons[turn.ended] ?? 0) + 1;
175 if (turn.releases === null) continue;
176 releases[i]! += turn.releases;
177 for (const [cause, c] of Object.entries(turn.snapBacks ?? {})) {
178 snapBacks[i]! += c;
179 (byCause[cause] ??= zeros())[i]! += c;
180 }
181 if (turn.arriveP95Ms !== null) {
182 arrive[i]!.push(turn.arriveP95Ms);
183 arrives.push(turn.arriveP95Ms);
184 }
185 }
186
187 // ---- the minute samples
188 const sampled = { watchers: lists(), drivers: lists(), queue: lists() };
189 const live = zeros();
190 const sampledLink = zeros();
191 const fpsOf: Record<string, number[][]> = {};
192 for (const s of input.samples) {
193 const i = at(s.at);
194 sampled.watchers[i]!.push(s.watchers);
195 sampled.drivers[i]!.push(s.drivers);
196 sampled.queue[i]!.push(s.queue);
197 sampledLink[i]!++;
198 if (s.linkLive) live[i]!++;
199 for (const [camera, v] of Object.entries(s.fps)) (fpsOf[camera] ??= lists())[i]!.push(v);
200 }
201 const peak = (l: number[][]) => l.map((v) => (v.length ? Math.max(...v) : null));
202 const since = Math.max(from, input.oldest ?? from);
203 const passed = t.map((start) => Math.max(0, (Math.min(start + stepMs, now, to) - Math.max(start, since)) / MINUTE));
204 const uptime = passed.map((p, i) => (p >= 1 ? round(Math.min(1, live[i]! / Math.floor(p)), 3) : null));
205 const allPassed = Math.floor(sum(passed));
206
207 const allReleases = sum(releases);
208 const overview: Overview = {
209 visitors: people.visitors.size,
210 sessions: input.turns.length,
211 drivers: drove.size,
212 minutes: round(sum(minutes)) ?? 0,
213 snapBacksPer100: allReleases > 0 ? round((100 * sum(snapBacks)) / allReleases) : null,
214 arriveP95Ms: round(percentile(arrives, 95), 0),
215 jams: sum(count.jam),
216 uptime: allPassed >= 1 && sum(sampledLink) > 0 ? round(Math.min(1, sum(live) / allPassed), 3) : null,
217 };
218
219 return {
220 from,
221 to,
222 step: input.step,
223 t,
224 overview,
225 series: {
226 visitorsAnon: anon.map((s) => s.size),
227 visitorsNamed: named.map((s) => s.size),
228 sessions,
229 drivers: driverSets.map((s) => s.size),
230 minutes: minutes.map((m) => round(m) ?? 0),
231 releases,
232 snapBacks,
233 snapBacksPer100: releases.map((r, i) => (r > 0 ? round((100 * snapBacks[i]!) / r) : null)),
234 byCause,
235 arriveP95Ms: pct(arrive, 95, 0),
236 watchers: peak(sampled.watchers),
237 driving: peak(sampled.drivers),
238 queue: peak(sampled.queue),
239 waitP50S: pct(waits, 50),
240 waitP95S: pct(waits, 95),
241 turnP50S: pct(lengths, 50),
242 chat: count.chat,
243 stops: count.STOP,
244 jams: count.jam,
245 blocks: count.block,
246 reconnects: count.reconnect,
247 restarts: count["app start"],
248 uptime,
249 fps: Object.fromEntries(Object.entries(fpsOf).map(([camera, l]) => [camera, l.map((v) => round(mean(v)))])),
250 },
251 funnel: {
252 visitors: people.visitors.size,
253 named: people.named.size,
254 queued: people.queued.size,
255 drove: drove.size,
256 under10s: [...drove.values()].filter((s) => s < 10).length,
257 },
258 turnLengths: { edgesS: TURN_BINS, counts: histogram(input.turns.map((x) => x.seconds), TURN_BINS) },
259 reasons,
260 versions,
261 };
262 }
263
264 /** A request's `from`, `to` and `step`, checked: ms since the epoch, from before to, a listed step; else why not. */
265 export function parseRange(params: URLSearchParams, now: number): { from: number; to: number; step: number } | { error: string } {
266 const read = (name: string) => {
267 const v = params.get(name);
268 if (v === null || v === "") return undefined;
269 return /^\d{1,15}$/.test(v) ? Number(v) : NaN;
270 };
271 const to = read("to") ?? now;
272 const from = read("from") ?? to - 86_400_000;
273 if (Number.isNaN(to) || Number.isNaN(from)) return { error: "from and to are ms since the epoch" };
274 if (from >= to) return { error: "from must be before to" };
275 const span = to - from;
276 const named = read("step");
277 const choices = `step is one of ${STEPS.join(", ")} seconds`;
278 if (Number.isNaN(named)) return { error: choices };
279 const step = named ?? autoStep(span);
280 if (!(STEPS as readonly number[]).includes(step)) return { error: choices };
281 if (span / (step * 1000) > MAX_POINTS) return { error: `at most ${MAX_POINTS} steps: choose a longer step` };
282 return { from, to, step };
283 }
Software/so101/src/facts.ts +53 −20
@@ -83,6 +83,40 @@ export interface TurnRow {
8383 arriveP95Ms: number | null;
8484 }
8585
86 export interface EventRow {
87 at: number;
88 commit: string;
89 kind: EventKind;
90 who: string | null;
91 }
92
93 export interface SampleRow {
94 at: number;
95 commit: string;
96 watchers: number;
97 drivers: number;
98 queue: number;
99 linkLive: boolean;
100 fps: Record<string, number>;
101 piTemp?: number;
102 piUndervolt?: boolean;
103 }
104
105 const rowToTurn = (r: Record<string, unknown>): TurnRow => ({
106 id: r.id as number,
107 at: new Date(r.at as number).toISOString(),
108 commit: r.commit_id as string,
109 who: r.who as string,
110 startedAt: new Date(r.started as number).toISOString(),
111 seconds: r.seconds as number,
112 ended: r.ended as string,
113 waitSeconds: r.wait_seconds as number,
114 releases: r.releases as number | null,
115 snapBacks: r.snap_backs ? (JSON.parse(r.snap_backs as string) as Partial<Record<Cause, number>>) : null,
116 jams: r.jams as number | null,
117 arriveP95Ms: r.arrive_p95_ms as number | null,
118 });
119
86120 export class Facts {
87121 readonly commit: string;
88122 readonly #db: Db;
@@ -190,30 +224,29 @@ export class Facts {
190224 for (const t of ["turns", "events", "samples"]) this.#db.query(`DELETE FROM ${t} WHERE at < ?`).run(before);
191225 }
192226
193 turns(): TurnRow[] {
194 const rows = this.#db.query("SELECT * FROM turns ORDER BY id").all() as Array<Record<string, unknown>>;
195 return rows.map((r) => ({
196 id: r.id as number,
197 at: new Date(r.at as number).toISOString(),
198 commit: r.commit_id as string,
199 who: r.who as string,
200 startedAt: new Date(r.started as number).toISOString(),
201 seconds: r.seconds as number,
202 ended: r.ended as string,
203 waitSeconds: r.wait_seconds as number,
204 releases: r.releases as number | null,
205 snapBacks: r.snap_backs ? (JSON.parse(r.snap_backs as string) as Partial<Record<Cause, number>>) : null,
206 jams: r.jams as number | null,
207 arriveP95Ms: r.arrive_p95_ms as number | null,
208 }));
227 /** Turns that began in [from, to), or all of them. */
228 turns(from = -Number.MAX_SAFE_INTEGER, to = Number.MAX_SAFE_INTEGER): TurnRow[] {
229 const rows = this.#db.query("SELECT * FROM turns WHERE started >= ? AND started < ? ORDER BY id").all(from, to) as Array<Record<string, unknown>>;
230 return rows.map(rowToTurn);
209231 }
210232
211 events(): Array<{ at: number; commit: string; kind: EventKind; who: string | null }> {
212 return (this.#db.query("SELECT at, commit_id, kind, who FROM events ORDER BY at, rowid").all() as Array<{ at: number; commit_id: string; kind: EventKind; who: string | null }>).map((r) => ({ at: r.at, commit: r.commit_id, kind: r.kind, who: r.who }));
233 turnById(id: number): TurnRow | null {
234 const row = this.#db.query("SELECT * FROM turns WHERE id = ?").get(id) as Record<string, unknown> | null;
235 return row ? rowToTurn(row) : null;
213236 }
214237
215 samples(): Array<{ at: number; commit: string; watchers: number; drivers: number; queue: number; linkLive: boolean; fps: Record<string, number>; piTemp?: number; piUndervolt?: boolean }> {
216 return (this.#db.query("SELECT * FROM samples ORDER BY at, rowid").all() as Array<Record<string, unknown>>).map((r) => ({
238 /** The first moment anything was kept, ms; null for an empty file. */
239 oldest(): number | null {
240 const row = this.#db.query("SELECT MIN(at) AS at FROM (SELECT MIN(at) AS at FROM events UNION ALL SELECT MIN(at) FROM samples UNION ALL SELECT MIN(at) FROM turns)").get() as { at: number | null };
241 return row.at;
242 }
243
244 events(from = -Number.MAX_SAFE_INTEGER, to = Number.MAX_SAFE_INTEGER): EventRow[] {
245 return (this.#db.query("SELECT at, commit_id, kind, who FROM events WHERE at >= ? AND at < ? ORDER BY at, rowid").all(from, to) as Array<{ at: number; commit_id: string; kind: EventKind; who: string | null }>).map((r) => ({ at: r.at, commit: r.commit_id, kind: r.kind, who: r.who }));
246 }
247
248 samples(from = -Number.MAX_SAFE_INTEGER, to = Number.MAX_SAFE_INTEGER): SampleRow[] {
249 return (this.#db.query("SELECT * FROM samples WHERE at >= ? AND at < ? ORDER BY at, rowid").all(from, to) as Array<Record<string, unknown>>).map((r) => ({
217250 at: r.at as number,
218251 commit: r.commit_id as string,
219252 watchers: r.watchers as number,
Software/so101/src/relay.ts +3
@@ -48,6 +48,7 @@
4848 * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts);
4949 * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate
5050 * WS /camera/<id>/ws the same feed as whole binary frames (JPEG; the offline PNG when its source is down), latest frame wins
51 * GET /data?from&to&step[&compare=1], ?turn=<id> the facts as series, for the owner and moderators (403 to anyone else)
5152 * GET /admin/camera-list the owner's page for the camera list
5253 * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds
5354 * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time,
@@ -78,6 +79,7 @@ import { CameraSocket } from "./camera-socket.ts";
7879 import { OFFLINE_FRAME } from "./offline-frame.ts";
7980 import { ServoBaseline } from "./servo-baseline.ts";
8081 import type { PiSync } from "./pi-deploy.ts";
82 import { serveData } from "./data-api.ts";
8183 import { handleAdmin, handleLift, type TuneP } from "./admin-routes.ts";
8284 import { readPrecision } from "./precision-panel.ts";
8385 import { readTuning } from "./tuning-panel.ts";
@@ -772,6 +774,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
772774 if (lobby && url.pathname === "/admin/lift") return await handleLift(lobby, req, res, readBody);
773775 if (lobby && (url.pathname === "/admin" || url.pathname === "/admin/health")) return await handleAdmin(lobby, req, res, url, readBody, askPrecision, askServos, videoCards, options.piSync ? () => options.piSync!.versions() : undefined, tuneP);
774776 if (url.pathname === "/admin/versions") return versionsRead(req, res);
777 if (url.pathname === "/data") return serveData(facts ?? undefined, lobby ?? undefined, req, res, url);
775778 if (url.pathname === "/precision/trace") return await traceRead(req, res, url);
776779 if (url.pathname === "/precision") return await precisionRead(req, res, url);
777780 if (url.pathname === LEADER) return await leaderCalibration(req, res);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs