Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
workbenchthis machine
GET /data?from&to&step[&compare=1] and ?turn=<id> serve the facts as series for owner and moderators (src/data-series.ts, src/data-api.ts) #550
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/529-get-data-from-to-step-compare-1-and-turn → main 4 commits fork 1 thread · 1 unresolved
7 files
changed +593
−20
Software/so101/README.md +11
| @@ -70,6 +70,17 @@ The relay (`src/relay.ts`, through `src/facts.ts`) keeps 90 days of what happene | ||
| 70 | 70 | |
| 71 | 71 | "Who" is an account id or a 16-hex hash of the browser id, never a name or address. |
| 72 | 72 | |
| 73 | ## Reading the facts: GET /data | |
| 74 | ||
| 75 | `GET /data` on the hosted app answers the kept facts as JSON (src/data-api.ts, src/data-series.ts; how each number is made is at the top of the latter). It is for the owner and moderators; anonymous visitors and plain users get 403. There is no terminal command for it. | |
| 76 | ||
| 77 | - `from`, `to`: ms since the epoch, `from` before `to`; default the last 24 hours. | |
| 78 | - `step`: seconds, one of 60, 300, 900, 1800, 3600, 10800, 21600, 86400; default about 96 steps over the range, at most 400. | |
| 79 | - `compare=1`: adds `previous`, the same length of time just before, on the same steps (`compare=0` is the default). | |
| 80 | - `turn=<id>`: instead of a range, one turn: `{ turn, events, samples }` from a minute before to a minute after, with the arm's events only (STOP, jam, block, reconnect, app start) and no one named. An unknown id is 404. | |
| 81 | ||
| 82 | A bad parameter is a 400 whose `error` says what to send; POST is 405. Shape of a range answer (trimmed): `{ from, to, step, t: [step starts], overview: { visitors, sessions, drivers, minutes, snapBacksPer100, arriveP95Ms, jams, uptime }, series: { sessions: [..], snapBacksPer100: [..], watchers: [..], uptime: [..], ... one array per step }, funnel, turnLengths, reasons, versions, live: { watchers, driving, queue }, oldest, commit, previous? }`. | |
| 83 | ||
| 73 | 84 | ## Faster limits |
| 74 | 85 | |
| 75 | 86 | A person runs this at the arm while nobody drives: `bun bench/limits-search.ts http://<arm>:8101` (add `--joint 2` for one joint; with no URL it runs on the fake bus). From today's 90 °/s and 360 °/s² it steps each joint's speed and acceleration up within the safety ranges (the gripper is skipped), sweeps at each step, and keeps the last step that did not make p95 error or overshoot worse, writing them through `PUT /settings`. It prints its command line, then a table of limits and p95 error, overshoot and time to arrive, before → after. It refuses to start while anyone drives or a STOP or E-stop is in force, and if one appears mid-run it stops and puts the starting limits back. |
Software/so101/src/data-api.test.ts added +80
| @@ -0,0 +1,80 @@ | ||
| 1 | import { afterEach, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { Facts, whoIs } from "./facts.ts"; | |
| 6 | import { sign } from "./identity.ts"; | |
| 7 | import { Lobby } from "./lobby.ts"; | |
| 8 | import { startRelay, type RunningRelay } from "./relay.ts"; | |
| 9 | ||
| 10 | const KEY = "k".repeat(64); | |
| 11 | const later = Date.now() + 600_000; | |
| 12 | const as = (name: string, role: "user" | "admin") => { | |
| 13 | const identity = `id-${name}|${name}|${role}|${later}`; | |
| 14 | return { "x-forge-identity": identity, "x-forge-signature": sign(identity, KEY) }; | |
| 15 | }; | |
| 16 | const owner = as("trevato", "admin"); | |
| 17 | ||
| 18 | let dir = ""; | |
| 19 | let relay: RunningRelay | undefined; | |
| 20 | let lobby: Lobby | undefined; | |
| 21 | afterEach(async () => { | |
| 22 | await relay?.close(); | |
| 23 | lobby?.close(); | |
| 24 | if (dir) rmSync(dir, { recursive: true, force: true }); | |
| 25 | dir = ""; | |
| 26 | relay = lobby = undefined; | |
| 27 | }); | |
| 28 | ||
| 29 | async function boot() { | |
| 30 | dir = mkdtempSync(join(tmpdir(), "so101-data-api-")); | |
| 31 | const facts = new Facts({ file: join(dir, "facts.sqlite"), commit: "abc1234def56" }); | |
| 32 | lobby = new Lobby({ identityKey: KEY, adminFile: join(dir, "admin.json"), countsFile: join(dir, "visitors.json") }); | |
| 33 | relay = await startRelay({ port: 0, simulation: "http://127.0.0.1:1", lobby, facts }); | |
| 34 | return { base: `http://127.0.0.1:${relay.port}`, facts }; | |
| 35 | } | |
| 36 | ||
| 37 | test("GET /data: anonymous and plain users get 403; the owner and a moderator get the series", async () => { | |
| 38 | const { base, facts } = await boot(); | |
| 39 | expect((await fetch(`${base}/data`)).status).toBe(403); | |
| 40 | expect((await fetch(`${base}/data`, { headers: as("eve", "user") })).status).toBe(403); | |
| 41 | await fetch(`${base}/admin`, { method: "POST", headers: { ...owner, "content-type": "application/x-www-form-urlencoded" }, body: new URLSearchParams({ action: "mod-add", name: "bob" }), redirect: "manual" }); | |
| 42 | ||
| 43 | const ada = { id: "a-ada", name: "ada", owner: false }; | |
| 44 | const snapshot = { driver: null, queue: [] } as never; | |
| 45 | facts.turn({ kind: "turn", ended: null, started: ada }, snapshot); | |
| 46 | const id = facts.turn({ kind: "turn", ended: { who: ada, reason: "time is up" }, started: null }, snapshot)!; | |
| 47 | facts.event("app start"); | |
| 48 | facts.event("STOP"); | |
| 49 | for (const who of [owner, as("bob", "user")]) { | |
| 50 | const res = await fetch(`${base}/data?compare=1`, { headers: who }); | |
| 51 | expect(res.status).toBe(200); | |
| 52 | expect(res.headers.get("cache-control")).toBe("no-store"); | |
| 53 | expect(res.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 54 | const body = (await res.json()) as Record<string, any>; | |
| 55 | expect(body.step).toBe(900); | |
| 56 | expect(body.t).toHaveLength(96); | |
| 57 | expect(body.overview.sessions).toBe(1); | |
| 58 | expect(body.commit).toBe("abc1234def56"); | |
| 59 | expect(body.versions[0]).toMatchObject({ commit: "abc1234def56" }); | |
| 60 | expect(body.previous.overview.sessions).toBe(0); | |
| 61 | expect(body.live).toEqual({ watchers: 0, driving: 0, queue: 0 }); | |
| 62 | } | |
| 63 | ||
| 64 | const one = (await (await fetch(`${base}/data?turn=${id}`, { headers: owner })).json()) as Record<string, any>; | |
| 65 | expect(one.turn).toMatchObject({ id, who: whoIs("a-ada"), ended: "time is up" }); | |
| 66 | expect(one.events.map((e: { kind: string }) => e.kind)).toContain("STOP"); | |
| 67 | expect(one.events.every((e: Record<string, unknown>) => !("who" in e))).toBe(true); | |
| 68 | expect((await fetch(`${base}/data?turn=${id}`)).status).toBe(403); | |
| 69 | }); | |
| 70 | ||
| 71 | test("GET /data refuses a bad range or turn with a 400 that says what to send; POST is 405", async () => { | |
| 72 | const { base } = await boot(); | |
| 73 | for (const q of ["from=yesterday", "from=10&to=5", "step=7", "compare=yes", "turn=-1"]) { | |
| 74 | const res = await fetch(`${base}/data?${q}`, { headers: owner }); | |
| 75 | expect(res.status).toBe(400); | |
| 76 | expect(((await res.json()) as { error: string }).error).toBeTruthy(); | |
| 77 | } | |
| 78 | expect((await fetch(`${base}/data?turn=999`, { headers: owner })).status).toBe(404); | |
| 79 | expect((await fetch(`${base}/data`, { method: "POST", headers: owner })).status).not.toBe(200); | |
| 80 | }); |
Software/so101/src/data-api.ts added +52
| @@ -0,0 +1,52 @@ | ||
| 1 | /** | |
| 2 | * GET /data: the Data view's one read, for the owner and moderators (anyone else: 403). | |
| 3 | * ?from=<ms>&to=<ms>&step=<s>[&compare=1] the series over the range, per step (src/data-series.ts); compare=1 adds | |
| 4 | * `previous`, the same length of time just before, on the same steps | |
| 5 | * ?turn=<id> one turn: its row, and the events and samples from a minute before to a minute after | |
| 6 | * Every parameter is checked here; a bad one is a 400 that says what to send. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { IncomingMessage, ServerResponse } from "node:http"; | |
| 10 | import type { Facts } from "./facts.ts"; | |
| 11 | import type { Lobby } from "./lobby.ts"; | |
| 12 | import { aggregate, parseRange } from "./data-series.ts"; | |
| 13 | ||
| 14 | const MINUTE = 60_000; | |
| 15 | /** Shown on a turn's timeline: what happened to the arm, not who opened the page or spoke. */ | |
| 16 | const ARM_EVENTS = new Set(["STOP", "jam", "block", "reconnect", "app start"]); | |
| 17 | ||
| 18 | function answer(res: ServerResponse, status: number, body: unknown, head = false): void { | |
| 19 | res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); | |
| 20 | res.end(head ? undefined : JSON.stringify(body)); | |
| 21 | } | |
| 22 | /** `now` is one ms ahead so a row written in this very millisecond is inside the range, which ends before `to`. */ | |
| 23 | export function serveData(facts: Facts | undefined, lobby: Lobby | undefined, req: IncomingMessage, res: ServerResponse, url: URL, now = Date.now() + 1): void { | |
| 24 | if (!lobby || !facts) return answer(res, 404, { error: "no data is kept here" }); | |
| 25 | if (!lobby.who(req) || !lobby.moderates(req)) return answer(res, 403, { error: "the data is the owner's and the moderators'" }); | |
| 26 | if (req.method !== "GET" && req.method !== "HEAD") { | |
| 27 | res.setHeader("allow", "GET, HEAD"); | |
| 28 | return answer(res, 405, { error: "GET" }); | |
| 29 | } | |
| 30 | const head = req.method === "HEAD"; | |
| 31 | const turn = url.searchParams.get("turn"); | |
| 32 | if (turn !== null) { | |
| 33 | if (!/^\d{1,12}$/.test(turn)) return answer(res, 400, { error: "turn is a turn's id" }); | |
| 34 | const row = facts.turnById(Number(turn)); | |
| 35 | if (!row) return answer(res, 404, { error: "no such turn" }); | |
| 36 | const start = Date.parse(row.startedAt); | |
| 37 | const [from, to] = [start - MINUTE, start + row.seconds * 1000 + MINUTE]; | |
| 38 | return answer(res, 200, { turn: row, events: facts.events(from, to).filter((e) => ARM_EVENTS.has(e.kind)).map(({ who: _who, ...e }) => e), samples: facts.samples(from, to) }, head); | |
| 39 | } | |
| 40 | const range = parseRange(url.searchParams, now); | |
| 41 | if ("error" in range) return answer(res, 400, range); | |
| 42 | const compare = url.searchParams.get("compare"); | |
| 43 | if (compare !== null && compare !== "1" && compare !== "0") return answer(res, 400, { error: "compare is 1 or 0" }); | |
| 44 | const oldest = facts.oldest(); | |
| 45 | const over = (from: number, to: number) => aggregate({ from, to, step: range.step, now, oldest, turns: facts.turns(from, to), events: facts.events(from, to), samples: facts.samples(from, to) }); | |
| 46 | const body = over(range.from, range.to); | |
| 47 | const span = range.to - range.from; | |
| 48 | const previous = compare === "1" ? over(range.from - span, range.from) : null; | |
| 49 | const snapshot = lobby.turns.snapshot(); | |
| 50 | const live = { watchers: lobby.watching, driving: snapshot.driver ? 1 : 0, queue: snapshot.queue.length }; | |
| 51 | return answer(res, 200, { ...body, live, oldest, commit: facts.commit, previous: previous && { overview: previous.overview, series: previous.series, funnel: previous.funnel } }, head); | |
| 52 | } |
Software/so101/src/data-series.test.ts added +111
| @@ -0,0 +1,111 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { aggregate, autoStep, histogram, MAX_POINTS, parseRange, percentile, type Input } from "./data-series.ts"; | |
| 3 | import type { EventRow, SampleRow, TurnRow } from "./facts.ts"; | |
| 4 | ||
| 5 | const H = 3_600_000; | |
| 6 | const T0 = 1_700_000_000_000 - (1_700_000_000_000 % H); | |
| 7 | ||
| 8 | const turn = (over: Partial<TurnRow> & { start?: number }): TurnRow => { | |
| 9 | const { start = T0, ...rest } = over; | |
| 10 | return { id: 1, at: new Date(start).toISOString(), commit: "abc1234", who: "a-1", startedAt: new Date(start).toISOString(), seconds: 60, ended: "time is up", waitSeconds: 0, releases: null, snapBacks: null, jams: null, arriveP95Ms: null, ...rest }; | |
| 11 | }; | |
| 12 | const ev = (at: number, kind: EventRow["kind"], who: string | null = null, commit = "abc1234"): EventRow => ({ at, kind, who, commit }); | |
| 13 | const sample = (at: number, over: Partial<SampleRow> = {}): SampleRow => ({ at, commit: "abc1234", watchers: 0, drivers: 0, queue: 0, linkLive: true, fps: {}, ...over }); | |
| 14 | const input = (over: Partial<Input>): Input => ({ from: T0, to: T0 + 4 * H, step: 3600, now: T0 + 4 * H, oldest: T0, turns: [], events: [], samples: [], ...over }); | |
| 15 | ||
| 16 | describe("percentile and histogram", () => { | |
| 17 | test("nearest rank: p95 of 1..20 is 19, p50 of 1..4 is 2, of nothing null", () => { | |
| 18 | expect(percentile(Array.from({ length: 20 }, (_, i) => i + 1), 95)).toBe(19); | |
| 19 | expect(percentile([4, 1, 3, 2], 50)).toBe(2); | |
| 20 | expect(percentile([7], 95)).toBe(7); | |
| 21 | expect(percentile([], 95)).toBeNull(); | |
| 22 | }); | |
| 23 | ||
| 24 | test("a histogram counts below the first edge, between, and at or past the last", () => { | |
| 25 | expect(histogram([0, 9.9, 10, 29, 30, 500], [10, 30])).toEqual([2, 2, 2]); | |
| 26 | }); | |
| 27 | ||
| 28 | test("autoStep keeps about 96 points: an hour by the minute, a day by 15 min, a week by 3 h", () => { | |
| 29 | expect(autoStep(H)).toBe(60); | |
| 30 | expect(autoStep(24 * H)).toBe(900); | |
| 31 | expect(autoStep(7 * 24 * H)).toBe(3 * 3600); | |
| 32 | expect(autoStep(90 * 24 * H)).toBe(86_400); | |
| 33 | }); | |
| 34 | }); | |
| 35 | ||
| 36 | describe("aggregate", () => { | |
| 37 | test("snap-backs per 100 releases are summed then divided, never an average of ratios", () => { | |
| 38 | const a = aggregate(input({ turns: [turn({ id: 1, releases: 10, snapBacks: { jam: 5 }, jams: 5 }), turn({ id: 2, start: T0 + 60_000, releases: 90, snapBacks: {}, jams: 0 })] })); | |
| 39 | // the mean of 50 and 0 would be 25; the truth is 5 in 100 | |
| 40 | expect(a.overview.snapBacksPer100).toBe(5); | |
| 41 | expect(a.series.snapBacksPer100).toEqual([5, null, null, null]); | |
| 42 | expect(a.series.byCause.jam).toEqual([5, 0, 0, 0]); | |
| 43 | }); | |
| 44 | ||
| 45 | test("steps: each turn lands in the step it began; sessions, drivers and minutes add up", () => { | |
| 46 | const a = aggregate(input({ turns: [turn({ id: 1, who: "a-1", seconds: 30 }), turn({ id: 2, who: "a-1", start: T0 + 2 * H, seconds: 90 }), turn({ id: 3, who: "b", start: T0 + 2 * H + 1, seconds: 5 })] })); | |
| 47 | expect(a.t).toEqual([T0, T0 + H, T0 + 2 * H, T0 + 3 * H]); | |
| 48 | expect(a.series.sessions).toEqual([1, 0, 2, 0]); | |
| 49 | expect(a.series.drivers).toEqual([1, 0, 2, 0]); | |
| 50 | expect(a.overview).toMatchObject({ sessions: 3, drivers: 2, minutes: 2.1 }); | |
| 51 | expect(a.turnLengths.counts).toEqual([1, 0, 1, 1, 0, 0]); | |
| 52 | }); | |
| 53 | ||
| 54 | test("a step's p95 time to arrive is the p95 of its turns' p95s, so one bad turn shows", () => { | |
| 55 | const turns = Array.from({ length: 20 }, (_, i) => turn({ id: i, start: T0 + i, releases: 1, snapBacks: {}, arriveP95Ms: i === 19 ? 2000 : 100 })); | |
| 56 | const a = aggregate(input({ turns })); | |
| 57 | expect(a.series.arriveP95Ms[0]).toBe(100); | |
| 58 | expect(a.overview.arriveP95Ms).toBe(100); | |
| 59 | const worse = aggregate(input({ turns: turns.map((t, i) => (i === 18 ? { ...t, arriveP95Ms: 1500 } : t)) })); | |
| 60 | expect(worse.overview.arriveP95Ms).toBe(1500); | |
| 61 | }); | |
| 62 | ||
| 63 | test("visitors are distinct people per step, anonymous against account; the funnel counts how far each got", () => { | |
| 64 | const events = [ev(T0, "visit", "0123456789abcdef"), ev(T0 + 1, "visit", "0123456789abcdef"), ev(T0 + 2, "visit", "a-ada"), ev(T0 + 3, "visit", "a-bob"), ev(T0 + H, "visit", "0123456789abcdef"), ev(T0 + 5, "name set", "a-ada"), ev(T0 + 6, "joined line", "a-ada"), ev(T0 + 7, "chat", "a-ada"), ev(T0 + 8, "chat", "a-bob")]; | |
| 65 | const a = aggregate(input({ events, turns: [turn({ who: "a-ada", seconds: 5 }), turn({ id: 2, who: "a-bob", seconds: 40 })] })); | |
| 66 | expect(a.series.visitorsAnon).toEqual([1, 1, 0, 0]); | |
| 67 | expect(a.series.visitorsNamed).toEqual([2, 0, 0, 0]); | |
| 68 | expect(a.overview.visitors).toBe(3); | |
| 69 | expect(a.funnel).toEqual({ visitors: 3, named: 2, queued: 1, drove: 2, under10s: 1 }); | |
| 70 | expect(a.series.chat).toEqual([2, 0, 0, 0]); | |
| 71 | }); | |
| 72 | ||
| 73 | test("uptime is live minutes over minutes passed since the record began; no samples is null", () => { | |
| 74 | const samples = Array.from({ length: 60 }, (_, i) => sample(T0 + i * 60_000, { linkLive: i < 45 })); | |
| 75 | const a = aggregate(input({ samples, now: T0 + H })); | |
| 76 | expect(a.series.uptime[0]).toBe(0.75); | |
| 77 | expect(a.series.uptime[1]).toBeNull(); // in the future | |
| 78 | expect(a.overview.uptime).toBe(0.75); | |
| 79 | expect(aggregate(input({})).overview.uptime).toBeNull(); | |
| 80 | }); | |
| 81 | ||
| 82 | test("room samples show each step's peak; camera fps their mean", () => { | |
| 83 | const a = aggregate(input({ samples: [sample(T0, { watchers: 3, fps: { front: 20 } }), sample(T0 + 60_000, { watchers: 7, fps: { front: 30 } })] })); | |
| 84 | expect(a.series.watchers).toEqual([7, null, null, null]); | |
| 85 | expect(a.series.fps.front).toEqual([25, null, null, null]); | |
| 86 | }); | |
| 87 | ||
| 88 | test("event counts per step, and one version marker per change of commit", () => { | |
| 89 | const events = [ev(T0, "app start", null, "aaa1111"), ev(T0 + 10, "app start", null, "aaa1111"), ev(T0 + 11, "STOP"), ev(T0 + 12, "jam"), ev(T0 + H, "app start", null, "bbb2222")]; | |
| 90 | const a = aggregate(input({ events })); | |
| 91 | expect(a.versions).toEqual([{ at: T0, commit: "aaa1111" }, { at: T0 + H, commit: "bbb2222" }]); | |
| 92 | expect(a.series.restarts).toEqual([2, 1, 0, 0]); | |
| 93 | expect(a.series.stops).toEqual([1, 0, 0, 0]); | |
| 94 | expect(a.overview.jams).toBe(1); | |
| 95 | }); | |
| 96 | }); | |
| 97 | ||
| 98 | describe("parseRange", () => { | |
| 99 | const now = T0 + 24 * H; | |
| 100 | const p = (q: string) => parseRange(new URLSearchParams(q), now); | |
| 101 | test("defaults to the last day at its auto step", () => { | |
| 102 | expect(p("")).toEqual({ from: now - 24 * H, to: now, step: 900 }); | |
| 103 | }); | |
| 104 | test("refuses what is not a range", () => { | |
| 105 | expect(p("from=abc")).toHaveProperty("error"); | |
| 106 | expect(p(`from=${now}&to=${now - 1}`)).toHaveProperty("error"); | |
| 107 | expect(p("step=7")).toHaveProperty("error"); | |
| 108 | expect(p(`from=${now - (MAX_POINTS + 1) * 60_000}&to=${now}&step=60`)).toHaveProperty("error"); | |
| 109 | expect(p(`from=${now - H}&to=${now}&step=300`)).toEqual({ from: now - H, to: now, step: 300 }); | |
| 110 | }); | |
| 111 | }); |
Software/so101/src/data-series.ts added +283
| @@ -0,0 +1,283 @@ | ||
| 1 | /** | |
| 2 | * The Data view's numbers: src/facts.ts's rows over a time range, cut into steps. Plain data in, plain data out. | |
| 3 | * | |
| 4 | * What each number is, so a chart never claims more than was measured: | |
| 5 | * - visitors: distinct people who opened the page (an account id, or the hash of a browser); named = an account. | |
| 6 | * - a session is a turn, placed in the step it began; drivers are distinct people who had one; minutes are the turns' lengths. | |
| 7 | * - snap-backs per 100 releases: both summed over the turns the arm measured, then divided (never an average of ratios). | |
| 8 | * - p95 time to arrive: each turn keeps its own p95 (the arm's src/precision.ts); a step's number is the p95 of its | |
| 9 | * turns' p95s (nearest rank), so one bad turn shows. Waits and turn lengths: nearest-rank p50 and p95 of the turns. | |
| 10 | * - watchers, drivers now, queue: the minute samples; a step shows the most seen. Camera fps: their mean. | |
| 11 | * - uptime: the minutes the arm link was sampled live over the minutes that have passed since the range (or the | |
| 12 | * record) began, in a step and in all. | |
| 13 | * - counts of chat, STOP, jam, block and reconnect events are what happened in the step, nothing smoothed. | |
| 14 | */ | |
| 15 | ||
| 16 | import type { EventRow, SampleRow, TurnRow } from "./facts.ts"; | |
| 17 | ||
| 18 | /** The steps a request may name, in seconds, smallest first. */ | |
| 19 | export const STEPS = [60, 300, 900, 1800, 3600, 3 * 3600, 6 * 3600, 86_400] as const; | |
| 20 | /** The most steps one answer carries. */ | |
| 21 | export const MAX_POINTS = 400; | |
| 22 | const TARGET_POINTS = 96; | |
| 23 | const MINUTE = 60_000; | |
| 24 | ||
| 25 | /** The step that cuts a range into about TARGET_POINTS, never more than MAX_POINTS. */ | |
| 26 | export function autoStep(ms: number): number { | |
| 27 | for (const s of STEPS) if (ms / (s * 1000) <= TARGET_POINTS) return s; | |
| 28 | return STEPS[STEPS.length - 1]!; | |
| 29 | } | |
| 30 | ||
| 31 | /** Nearest-rank percentile; null for no values. */ | |
| 32 | export function percentile(values: readonly number[], p: number): number | null { | |
| 33 | if (values.length === 0) return null; | |
| 34 | const sorted = [...values].sort((a, b) => a - b); | |
| 35 | const rank = Math.max(1, Math.ceil((p / 100) * sorted.length)); | |
| 36 | return sorted[rank - 1]!; | |
| 37 | } | |
| 38 | ||
| 39 | const round = (v: number | null, digits = 1): number | null => (v === null || !Number.isFinite(v) ? null : Number(v.toFixed(digits))); | |
| 40 | const sum = (values: readonly number[]): number => values.reduce((a, b) => a + b, 0); | |
| 41 | const mean = (values: readonly number[]): number | null => (values.length ? sum(values) / values.length : null); | |
| 42 | ||
| 43 | export const TURN_BINS = [10, 30, 60, 120, 300] as const; | |
| 44 | /** Counts per bin: below the first edge, between each pair, and at or past the last. */ | |
| 45 | export function histogram(values: readonly number[], edges: readonly number[]): number[] { | |
| 46 | const out = new Array<number>(edges.length + 1).fill(0); | |
| 47 | for (const v of values) { | |
| 48 | let i = 0; | |
| 49 | while (i < edges.length && v >= edges[i]!) i++; | |
| 50 | out[i]!++; | |
| 51 | } | |
| 52 | return out; | |
| 53 | } | |
| 54 | ||
| 55 | export interface Overview { | |
| 56 | visitors: number; | |
| 57 | sessions: number; | |
| 58 | drivers: number; | |
| 59 | minutes: number; | |
| 60 | snapBacksPer100: number | null; | |
| 61 | arriveP95Ms: number | null; | |
| 62 | jams: number; | |
| 63 | /** 0–1; null with no samples. */ | |
| 64 | uptime: number | null; | |
| 65 | } | |
| 66 | ||
| 67 | export interface Series { | |
| 68 | visitorsAnon: number[]; | |
| 69 | visitorsNamed: number[]; | |
| 70 | sessions: number[]; | |
| 71 | drivers: number[]; | |
| 72 | minutes: number[]; | |
| 73 | releases: number[]; | |
| 74 | snapBacks: number[]; | |
| 75 | snapBacksPer100: Array<number | null>; | |
| 76 | byCause: Record<string, number[]>; | |
| 77 | arriveP95Ms: Array<number | null>; | |
| 78 | watchers: Array<number | null>; | |
| 79 | driving: Array<number | null>; | |
| 80 | queue: Array<number | null>; | |
| 81 | waitP50S: Array<number | null>; | |
| 82 | waitP95S: Array<number | null>; | |
| 83 | turnP50S: Array<number | null>; | |
| 84 | chat: number[]; | |
| 85 | stops: number[]; | |
| 86 | jams: number[]; | |
| 87 | blocks: number[]; | |
| 88 | reconnects: number[]; | |
| 89 | restarts: number[]; | |
| 90 | uptime: Array<number | null>; | |
| 91 | fps: Record<string, Array<number | null>>; | |
| 92 | } | |
| 93 | ||
| 94 | export interface Marker { | |
| 95 | at: number; | |
| 96 | commit: string; | |
| 97 | } | |
| 98 | ||
| 99 | export interface Aggregate { | |
| 100 | from: number; | |
| 101 | to: number; | |
| 102 | step: number; | |
| 103 | /** Each step's start, ms. */ | |
| 104 | t: number[]; | |
| 105 | overview: Overview; | |
| 106 | series: Series; | |
| 107 | /** Who got how far: opened the page, set a name, joined the line, drove, drove under 10 s (each distinct people). */ | |
| 108 | funnel: { visitors: number; named: number; queued: number; drove: number; under10s: number }; | |
| 109 | turnLengths: { edgesS: readonly number[]; counts: number[] }; | |
| 110 | /** How turns ended, by the reason kept. */ | |
| 111 | reasons: Record<string, number>; | |
| 112 | /** Each time the app started on a commit other than the one before. */ | |
| 113 | versions: Marker[]; | |
| 114 | } | |
| 115 | ||
| 116 | export interface Input { | |
| 117 | from: number; | |
| 118 | to: number; | |
| 119 | /** Seconds. */ | |
| 120 | step: number; | |
| 121 | /** Now: no step past it is uptime's fault. */ | |
| 122 | now: number; | |
| 123 | /** The first moment anything was recorded; uptime counts from there. */ | |
| 124 | oldest: number | null; | |
| 125 | turns: readonly TurnRow[]; | |
| 126 | events: readonly EventRow[]; | |
| 127 | samples: readonly SampleRow[]; | |
| 128 | } | |
| 129 | ||
| 130 | export function aggregate(input: Input): Aggregate { | |
| 131 | const { from, to, now } = input; | |
| 132 | const stepMs = input.step * 1000; | |
| 133 | const n = Math.max(1, Math.ceil((to - from) / stepMs)); | |
| 134 | const t = Array.from({ length: n }, (_, i) => from + i * stepMs); | |
| 135 | const at = (ms: number) => Math.min(n - 1, Math.max(0, Math.floor((ms - from) / stepMs))); | |
| 136 | const zeros = () => new Array<number>(n).fill(0); | |
| 137 | const lists = () => Array.from({ length: n }, () => [] as number[]); | |
| 138 | const pct = (l: number[][], p: number, digits = 1) => l.map((v) => round(percentile(v, p), digits)); | |
| 139 | ||
| 140 | // ---- the room's events | |
| 141 | const anon = Array.from({ length: n }, () => new Set<string>()); | |
| 142 | const named = Array.from({ length: n }, () => new Set<string>()); | |
| 143 | const people = { visitors: new Set<string>(), named: new Set<string>(), queued: new Set<string>() }; | |
| 144 | const count = { chat: zeros(), STOP: zeros(), jam: zeros(), block: zeros(), reconnect: zeros(), "app start": zeros() }; | |
| 145 | const versions: Marker[] = []; | |
| 146 | for (const e of input.events) { | |
| 147 | const i = at(e.at); | |
| 148 | if (e.kind === "visit" && e.who) { | |
| 149 | people.visitors.add(e.who); | |
| 150 | (e.who.startsWith("a-") ? named : anon)[i]!.add(e.who); | |
| 151 | } else if (e.kind === "name set" && e.who) people.named.add(e.who); | |
| 152 | else if (e.kind === "joined line" && e.who) people.queued.add(e.who); | |
| 153 | else if (e.kind in count) count[e.kind as keyof typeof count][i]!++; | |
| 154 | if (e.kind === "app start" && e.commit !== versions[versions.length - 1]?.commit) versions.push({ at: e.at, commit: e.commit }); | |
| 155 | } | |
| 156 | for (const s of named) for (const who of s) people.named.add(who); | |
| 157 | ||
| 158 | // ---- turns | |
| 159 | const sessions = zeros(), minutes = zeros(), releases = zeros(), snapBacks = zeros(); | |
| 160 | const driverSets = Array.from({ length: n }, () => new Set<string>()); | |
| 161 | const byCause: Record<string, number[]> = {}; | |
| 162 | const arrive = lists(), waits = lists(), lengths = lists(); | |
| 163 | const drove = new Map<string, number>(); | |
| 164 | const reasons: Record<string, number> = {}; | |
| 165 | const arrives: number[] = []; | |
| 166 | for (const turn of input.turns) { | |
| 167 | const i = at(Date.parse(turn.startedAt)); | |
| 168 | sessions[i]!++; | |
| 169 | driverSets[i]!.add(turn.who); | |
| 170 | minutes[i]! += turn.seconds / 60; | |
| 171 | lengths[i]!.push(turn.seconds); | |
| 172 | waits[i]!.push(turn.waitSeconds); | |
| 173 | drove.set(turn.who, Math.max(drove.get(turn.who) ?? 0, turn.seconds)); | |
| 174 | reasons[turn.ended] = (reasons[turn.ended] ?? 0) + 1; | |
| 175 | if (turn.releases === null) continue; | |
| 176 | releases[i]! += turn.releases; | |
| 177 | for (const [cause, c] of Object.entries(turn.snapBacks ?? {})) { | |
| 178 | snapBacks[i]! += c; | |
| 179 | (byCause[cause] ??= zeros())[i]! += c; | |
| 180 | } | |
| 181 | if (turn.arriveP95Ms !== null) { | |
| 182 | arrive[i]!.push(turn.arriveP95Ms); | |
| 183 | arrives.push(turn.arriveP95Ms); | |
| 184 | } | |
| 185 | } | |
| 186 | ||
| 187 | // ---- the minute samples | |
| 188 | const sampled = { watchers: lists(), drivers: lists(), queue: lists() }; | |
| 189 | const live = zeros(); | |
| 190 | const sampledLink = zeros(); | |
| 191 | const fpsOf: Record<string, number[][]> = {}; | |
| 192 | for (const s of input.samples) { | |
| 193 | const i = at(s.at); | |
| 194 | sampled.watchers[i]!.push(s.watchers); | |
| 195 | sampled.drivers[i]!.push(s.drivers); | |
| 196 | sampled.queue[i]!.push(s.queue); | |
| 197 | sampledLink[i]!++; | |
| 198 | if (s.linkLive) live[i]!++; | |
| 199 | for (const [camera, v] of Object.entries(s.fps)) (fpsOf[camera] ??= lists())[i]!.push(v); | |
| 200 | } | |
| 201 | const peak = (l: number[][]) => l.map((v) => (v.length ? Math.max(...v) : null)); | |
| 202 | const since = Math.max(from, input.oldest ?? from); | |
| 203 | const passed = t.map((start) => Math.max(0, (Math.min(start + stepMs, now, to) - Math.max(start, since)) / MINUTE)); | |
| 204 | const uptime = passed.map((p, i) => (p >= 1 ? round(Math.min(1, live[i]! / Math.floor(p)), 3) : null)); | |
| 205 | const allPassed = Math.floor(sum(passed)); | |
| 206 | ||
| 207 | const allReleases = sum(releases); | |
| 208 | const overview: Overview = { | |
| 209 | visitors: people.visitors.size, | |
| 210 | sessions: input.turns.length, | |
| 211 | drivers: drove.size, | |
| 212 | minutes: round(sum(minutes)) ?? 0, | |
| 213 | snapBacksPer100: allReleases > 0 ? round((100 * sum(snapBacks)) / allReleases) : null, | |
| 214 | arriveP95Ms: round(percentile(arrives, 95), 0), | |
| 215 | jams: sum(count.jam), | |
| 216 | uptime: allPassed >= 1 && sum(sampledLink) > 0 ? round(Math.min(1, sum(live) / allPassed), 3) : null, | |
| 217 | }; | |
| 218 | ||
| 219 | return { | |
| 220 | from, | |
| 221 | to, | |
| 222 | step: input.step, | |
| 223 | t, | |
| 224 | overview, | |
| 225 | series: { | |
| 226 | visitorsAnon: anon.map((s) => s.size), | |
| 227 | visitorsNamed: named.map((s) => s.size), | |
| 228 | sessions, | |
| 229 | drivers: driverSets.map((s) => s.size), | |
| 230 | minutes: minutes.map((m) => round(m) ?? 0), | |
| 231 | releases, | |
| 232 | snapBacks, | |
| 233 | snapBacksPer100: releases.map((r, i) => (r > 0 ? round((100 * snapBacks[i]!) / r) : null)), | |
| 234 | byCause, | |
| 235 | arriveP95Ms: pct(arrive, 95, 0), | |
| 236 | watchers: peak(sampled.watchers), | |
| 237 | driving: peak(sampled.drivers), | |
| 238 | queue: peak(sampled.queue), | |
| 239 | waitP50S: pct(waits, 50), | |
| 240 | waitP95S: pct(waits, 95), | |
| 241 | turnP50S: pct(lengths, 50), | |
| 242 | chat: count.chat, | |
| 243 | stops: count.STOP, | |
| 244 | jams: count.jam, | |
| 245 | blocks: count.block, | |
| 246 | reconnects: count.reconnect, | |
| 247 | restarts: count["app start"], | |
| 248 | uptime, | |
| 249 | fps: Object.fromEntries(Object.entries(fpsOf).map(([camera, l]) => [camera, l.map((v) => round(mean(v)))])), | |
| 250 | }, | |
| 251 | funnel: { | |
| 252 | visitors: people.visitors.size, | |
| 253 | named: people.named.size, | |
| 254 | queued: people.queued.size, | |
| 255 | drove: drove.size, | |
| 256 | under10s: [...drove.values()].filter((s) => s < 10).length, | |
| 257 | }, | |
| 258 | turnLengths: { edgesS: TURN_BINS, counts: histogram(input.turns.map((x) => x.seconds), TURN_BINS) }, | |
| 259 | reasons, | |
| 260 | versions, | |
| 261 | }; | |
| 262 | } | |
| 263 | ||
| 264 | /** A request's `from`, `to` and `step`, checked: ms since the epoch, from before to, a listed step; else why not. */ | |
| 265 | export function parseRange(params: URLSearchParams, now: number): { from: number; to: number; step: number } | { error: string } { | |
| 266 | const read = (name: string) => { | |
| 267 | const v = params.get(name); | |
| 268 | if (v === null || v === "") return undefined; | |
| 269 | return /^\d{1,15}$/.test(v) ? Number(v) : NaN; | |
| 270 | }; | |
| 271 | const to = read("to") ?? now; | |
| 272 | const from = read("from") ?? to - 86_400_000; | |
| 273 | if (Number.isNaN(to) || Number.isNaN(from)) return { error: "from and to are ms since the epoch" }; | |
| 274 | if (from >= to) return { error: "from must be before to" }; | |
| 275 | const span = to - from; | |
| 276 | const named = read("step"); | |
| 277 | const choices = `step is one of ${STEPS.join(", ")} seconds`; | |
| 278 | if (Number.isNaN(named)) return { error: choices }; | |
| 279 | const step = named ?? autoStep(span); | |
| 280 | if (!(STEPS as readonly number[]).includes(step)) return { error: choices }; | |
| 281 | if (span / (step * 1000) > MAX_POINTS) return { error: `at most ${MAX_POINTS} steps: choose a longer step` }; | |
| 282 | return { from, to, step }; | |
| 283 | } |
Software/so101/src/facts.ts +53 −20
| @@ -83,6 +83,40 @@ export interface TurnRow { | ||
| 83 | 83 | arriveP95Ms: number | null; |
| 84 | 84 | } |
| 85 | 85 | |
| 86 | export interface EventRow { | |
| 87 | at: number; | |
| 88 | commit: string; | |
| 89 | kind: EventKind; | |
| 90 | who: string | null; | |
| 91 | } | |
| 92 | ||
| 93 | export interface SampleRow { | |
| 94 | at: number; | |
| 95 | commit: string; | |
| 96 | watchers: number; | |
| 97 | drivers: number; | |
| 98 | queue: number; | |
| 99 | linkLive: boolean; | |
| 100 | fps: Record<string, number>; | |
| 101 | piTemp?: number; | |
| 102 | piUndervolt?: boolean; | |
| 103 | } | |
| 104 | ||
| 105 | const rowToTurn = (r: Record<string, unknown>): TurnRow => ({ | |
| 106 | id: r.id as number, | |
| 107 | at: new Date(r.at as number).toISOString(), | |
| 108 | commit: r.commit_id as string, | |
| 109 | who: r.who as string, | |
| 110 | startedAt: new Date(r.started as number).toISOString(), | |
| 111 | seconds: r.seconds as number, | |
| 112 | ended: r.ended as string, | |
| 113 | waitSeconds: r.wait_seconds as number, | |
| 114 | releases: r.releases as number | null, | |
| 115 | snapBacks: r.snap_backs ? (JSON.parse(r.snap_backs as string) as Partial<Record<Cause, number>>) : null, | |
| 116 | jams: r.jams as number | null, | |
| 117 | arriveP95Ms: r.arrive_p95_ms as number | null, | |
| 118 | }); | |
| 119 | ||
| 86 | 120 | export class Facts { |
| 87 | 121 | readonly commit: string; |
| 88 | 122 | readonly : Db; |
| @@ -190,30 +224,29 @@ export class Facts { | ||
| 190 | 224 | for (const t of ["turns", "events", "samples"]) this..query(`DELETE FROM ${t} WHERE at < ?`).run(before); |
| 191 | 225 | } |
| 192 | 226 | |
| 193 | turns(): TurnRow[] { | |
| 194 | const rows = this..query("SELECT * FROM turns ORDER BY id").all() as Array<Record<string, unknown>>; | |
| 195 | return rows.map((r) => ({ | |
| 196 | id: r.id as number, | |
| 197 | at: new Date(r.at as number).toISOString(), | |
| 198 | commit: r.commit_id as string, | |
| 199 | who: r.who as string, | |
| 200 | startedAt: new Date(r.started as number).toISOString(), | |
| 201 | seconds: r.seconds as number, | |
| 202 | ended: r.ended as string, | |
| 203 | waitSeconds: r.wait_seconds as number, | |
| 204 | releases: r.releases as number | null, | |
| 205 | snapBacks: r.snap_backs ? (JSON.parse(r.snap_backs as string) as Partial<Record<Cause, number>>) : null, | |
| 206 | jams: r.jams as number | null, | |
| 207 | arriveP95Ms: r.arrive_p95_ms as number | null, | |
| 208 | })); | |
| 227 | /** Turns that began in [from, to), or all of them. */ | |
| 228 | turns(from = -Number.MAX_SAFE_INTEGER, to = Number.MAX_SAFE_INTEGER): TurnRow[] { | |
| 229 | const rows = this..query("SELECT * FROM turns WHERE started >= ? AND started < ? ORDER BY id").all(from, to) as Array<Record<string, unknown>>; | |
| 230 | return rows.map(rowToTurn); | |
| 209 | 231 | } |
| 210 | 232 | |
| 211 | events(): Array<{ at: number; commit: string; kind: EventKind; who: string | null }> { | |
| 212 | return (this..query("SELECT at, commit_id, kind, who FROM events ORDER BY at, rowid").all() as Array<{ at: number; commit_id: string; kind: EventKind; who: string | null }>).map((r) => ({ at: r.at, commit: r.commit_id, kind: r.kind, who: r.who })); | |
| 233 | turnById(id: number): TurnRow | null { | |
| 234 | const row = this..query("SELECT * FROM turns WHERE id = ?").get(id) as Record<string, unknown> | null; | |
| 235 | return row ? rowToTurn(row) : null; | |
| 213 | 236 | } |
| 214 | 237 | |
| 215 | samples(): Array<{ at: number; commit: string; watchers: number; drivers: number; queue: number; linkLive: boolean; fps: Record<string, number>; piTemp?: number; piUndervolt?: boolean }> { | |
| 216 | return (this..query("SELECT * FROM samples ORDER BY at, rowid").all() as Array<Record<string, unknown>>).map((r) => ({ | |
| 238 | /** The first moment anything was kept, ms; null for an empty file. */ | |
| 239 | oldest(): number | null { | |
| 240 | const row = this..query("SELECT MIN(at) AS at FROM (SELECT MIN(at) AS at FROM events UNION ALL SELECT MIN(at) FROM samples UNION ALL SELECT MIN(at) FROM turns)").get() as { at: number | null }; | |
| 241 | return row.at; | |
| 242 | } | |
| 243 | ||
| 244 | events(from = -Number.MAX_SAFE_INTEGER, to = Number.MAX_SAFE_INTEGER): EventRow[] { | |
| 245 | return (this..query("SELECT at, commit_id, kind, who FROM events WHERE at >= ? AND at < ? ORDER BY at, rowid").all(from, to) as Array<{ at: number; commit_id: string; kind: EventKind; who: string | null }>).map((r) => ({ at: r.at, commit: r.commit_id, kind: r.kind, who: r.who })); | |
| 246 | } | |
| 247 | ||
| 248 | samples(from = -Number.MAX_SAFE_INTEGER, to = Number.MAX_SAFE_INTEGER): SampleRow[] { | |
| 249 | return (this..query("SELECT * FROM samples WHERE at >= ? AND at < ? ORDER BY at, rowid").all(from, to) as Array<Record<string, unknown>>).map((r) => ({ | |
| 217 | 250 | at: r.at as number, |
| 218 | 251 | commit: r.commit_id as string, |
| 219 | 252 | watchers: r.watchers as number, |
Software/so101/src/relay.ts +3
| @@ -48,6 +48,7 @@ | ||
| 48 | 48 | * GET /camera/<id>/<stream|state|snapshot> one listed camera, one upstream connection shared by all its viewers (src/cameras.ts, src/camera-feed.ts); |
| 49 | 49 | * the "camera offline" picture when its source is down; stream?fps=1-30 holds one viewer to that frame rate |
| 50 | 50 | * WS /camera/<id>/ws the same feed as whole binary frames (JPEG; the offline PNG when its source is down), latest frame wins |
| 51 | * GET /data?from&to&step[&compare=1], ?turn=<id> the facts as series, for the owner and moderators (403 to anyone else) | |
| 51 | 52 | * GET /admin/camera-list the owner's page for the camera list |
| 52 | 53 | * POST /admin/cameras {"name", "source"}, DELETE /admin/cameras/<id> the owner changes the list; viewers see it within seconds |
| 53 | 54 | * GET|PUT|DELETE /leader/calibration the signed-in person's own leader calibration (src/leader-map.js), any time, |
| @@ -78,6 +79,7 @@ import { CameraSocket } from "./camera-socket.ts"; | ||
| 78 | 79 | import { OFFLINE_FRAME } from "./offline-frame.ts"; |
| 79 | 80 | import { ServoBaseline } from "./servo-baseline.ts"; |
| 80 | 81 | import type { PiSync } from "./pi-deploy.ts"; |
| 82 | import { serveData } from "./data-api.ts"; | |
| 81 | 83 | import { handleAdmin, handleLift, type TuneP } from "./admin-routes.ts"; |
| 82 | 84 | import { readPrecision } from "./precision-panel.ts"; |
| 83 | 85 | import { readTuning } from "./tuning-panel.ts"; |
| @@ -772,6 +774,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 772 | 774 | if (lobby && url.pathname === "/admin/lift") return await handleLift(lobby, req, res, readBody); |
| 773 | 775 | if (lobby && (url.pathname === "/admin" || url.pathname === "/admin/health")) return await handleAdmin(lobby, req, res, url, readBody, askPrecision, askServos, videoCards, options.piSync ? () => options.piSync!.versions() : undefined, tuneP); |
| 774 | 776 | if (url.pathname === "/admin/versions") return versionsRead(req, res); |
| 777 | if (url.pathname === "/data") return serveData(facts ?? undefined, lobby ?? undefined, req, res, url); | |
| 775 | 778 | if (url.pathname === "/precision/trace") return await traceRead(req, res, url); |
| 776 | 779 | if (url.pathname === "/precision") return await precisionRead(req, res, url); |
| 777 | 780 | if (url.pathname === LEADER) return await leaderCalibration(req, res); |
new above
offline
The 403 is correct. Please document this door in the README (parameters, who may call it, response keys).