Public: anyone can read this; the crew can change it.
Migrated from github.com/Trevato/SO-ARM100
Grow card: "Camera can't see the plants" hint when the camera's region is mostly blown out or black (split from #584) #597
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
smith/arm:smith/590-grow-card-camera-can-t-see-the-plants-hi → main 2 commits fork 3 threads · 3 unresolved
Software/so101/README.md +1
| @@ -50,6 +50,7 @@ The arm closes itself too: a real arm (ARM_SERVER) that has not answered for mor | ||
| 50 | 50 | The app is a workshop: one Pi, several **stations**, each with its cameras and controls. Today there are two, **the arm** and **the grow box** (spinach under a UV light). `/` is the workshop home, a card per station with its live camera snapshot first and its controls after, and the arm's own page is at `/arm` (its URL is unchanged for anyone who has it). The page and its title say "Workshop"; it shows who is looking ("Signed in as …", or "Watching as a guest · Sign in"), has a skip link, and keeps working while the arm is closed. |
| 51 | 51 | |
| 52 | 52 | - **Any subset present.** The grow box has nothing to enumerate (the light is USB power only), so it is always there. The arm's card is there only while the link to the Pi's `so101 serve` is live; unplugged, closed by the owner or unreachable it says why ("The arm: not connected", "The arm is resting") and offers no controls, and the owner sees **Open now** on it. Plugging the arm back in brings its card back by itself, with no restart. `so101 serve` with no servo board waits for one ("arm not connected; waiting for its servo board") instead of exiting, so the Pi's service stays up for the cameras. |
| 53 | - **"Camera can't see the plants."** On the grow box's card, each still is checked in the browser once it has loaded: the part inside the camera's box (set on Tune, `/admin/camera-list`) is counted for blown-out and black pixels (`visibility` in src/ui/still-frame.js), and when the plants cannot be seen the card says so over the foot of the still, with a link to Tune (src/ui/grow-hint.js). The relay never decodes a JPEG, so the terminal and the JSON doors cannot show this hint; they have no pixels to count. | |
| 53 | 54 | - **The config** is `so101/stations.json` under `FORGE_APP_DATA`: a list of `{id, name, kind: "arm"|"grow", cameras: [camera ids], controls: [{label, href}]}` (src/stations.ts). Without a file it is the arm (camera 1) and the grow box (camera 2, from the camera list): edit it to match the cameras. With only an arm in it, `/` is the arm's page as before. |
| 54 | 55 | - **Three doors.** The page `/admin/stations/edit` (owner; a raw JSON box that needs JavaScript, so it is awkward on a phone: prefer a laptop or the terminal), `GET /stations` (anyone: each station's `present`, why not, and its camera ids and names) and `PUT /admin/stations` (owner; at most 16 KB, 400 on bad JSON or a bad station), and in the terminal `/stations` to list and `/stations set <json list>` (owner) to replace them. |
| 55 | 56 |
Software/so101/src/grow-hint.test.ts added +88
| @@ -0,0 +1,88 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { CANT_SEE } from "./ui/still-frame.js"; | |
| 3 | import { hintHtml, sightOf, watchStills } from "./ui/grow-hint.js"; | |
| 4 | import { renderWorkshop } from "./workshop-page.ts"; | |
| 5 | ||
| 6 | const paint = (w: number, h: number, at: (x: number, y: number) => [number, number, number]) => { | |
| 7 | const d = new Uint8ClampedArray(w * h * 4); | |
| 8 | for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) d.set([...at(x, y), 255], (y * w + x) * 4); | |
| 9 | return d; | |
| 10 | }; | |
| 11 | const blown = paint(20, 20, (x) => (x < 12 ? [40, 120, 40] : [255, 255, 255])); | |
| 12 | const fine = paint(20, 20, () => [60, 130, 50]); | |
| 13 | const region = { x: 0.6, y: 0, w: 0.4, h: 1 }; | |
| 14 | ||
| 15 | /** A page with one grow still whose pixels are `pixels`; `complete` false means its load event has yet to fire. */ | |
| 16 | function page(pixels: Uint8ClampedArray | "taint", regionJson = JSON.stringify(region), complete = true) { | |
| 17 | const slot = { dataset: { region: regionJson }, innerHTML: "" }; | |
| 18 | const listeners: (() => void)[] = []; | |
| 19 | const img = { | |
| 20 | dataset: { sight: "2" }, | |
| 21 | naturalWidth: 20, | |
| 22 | naturalHeight: 20, | |
| 23 | complete, | |
| 24 | addEventListener: (_: string, f: () => void) => void listeners.push(f), | |
| 25 | }; | |
| 26 | const doc = { | |
| 27 | querySelectorAll: (q: string) => (q === "img[data-sight]" ? [img] : []), | |
| 28 | querySelector: (q: string) => (q === '[data-sight-for="2"]' ? slot : null), | |
| 29 | createElement: () => ({ | |
| 30 | width: 0, | |
| 31 | height: 0, | |
| 32 | getContext: () => ({ | |
| 33 | drawImage() {}, | |
| 34 | getImageData() { | |
| 35 | if (pixels === "taint") throw new Error("SecurityError"); | |
| 36 | return { data: pixels }; | |
| 37 | }, | |
| 38 | }), | |
| 39 | }), | |
| 40 | }; | |
| 41 | return { doc, slot, load: () => listeners.forEach((f) => f()) }; | |
| 42 | } | |
| 43 | ||
| 44 | test("an overexposed box is blown, judged inside the region only", () => { | |
| 45 | expect(sightOf(blown, 20, 20, region)).toBe("blown"); | |
| 46 | expect(sightOf(blown, 20, 20, null)).toBe("ok"); | |
| 47 | expect(sightOf(fine, 20, 20, region)).toBe("ok"); | |
| 48 | expect(hintHtml("ok")).toBe(""); | |
| 49 | expect(hintHtml("dark")).toContain('<a href="/admin/camera-list">Tune</a>'); | |
| 50 | }); | |
| 51 | ||
| 52 | test("the page's script fills the hint for an overexposed still, and leaves it empty for a fine one", () => { | |
| 53 | const bad = page(blown); | |
| 54 | watchStills(bad.doc); | |
| 55 | expect(bad.slot.innerHTML).toContain(CANT_SEE); | |
| 56 | expect(bad.slot.innerHTML).toContain("/admin/camera-list"); | |
| 57 | const good = page(fine); | |
| 58 | watchStills(good.doc); | |
| 59 | expect(good.slot.innerHTML).toBe(""); | |
| 60 | }); | |
| 61 | ||
| 62 | test("a still that has not loaded yet is looked at when it does", () => { | |
| 63 | const late = page(blown, undefined, false); | |
| 64 | watchStills(late.doc); | |
| 65 | expect(late.slot.innerHTML).toBe(""); | |
| 66 | late.load(); | |
| 67 | expect(late.slot.innerHTML).toContain(CANT_SEE); | |
| 68 | }); | |
| 69 | ||
| 70 | test("a still the browser will not let us read, or a bad region, leaves the hint empty", () => { | |
| 71 | const tainted = page("taint"); | |
| 72 | watchStills(tainted.doc); | |
| 73 | expect(tainted.slot.innerHTML).toBe(""); | |
| 74 | const bad = page(blown, "{nope"); | |
| 75 | watchStills(bad.doc); | |
| 76 | expect(bad.slot.innerHTML).toBe(""); | |
| 77 | }); | |
| 78 | ||
| 79 | test("the grow card has an empty slot carrying its region, and runs the script; other cards do not", () => { | |
| 80 | const grow = { station: { id: "grow", name: "Grow box", kind: "grow" as const, cameras: [2], controls: [] }, present: true, cameras: [{ id: 2, name: "Side", region }] }; | |
| 81 | const html = renderWorkshop([grow]); | |
| 82 | expect(html).toContain('data-sight="2"'); | |
| 83 | expect(html).toContain('data-sight-for="2" data-region="{"x":0.6'); | |
| 84 | expect(html).toContain("watchStills(document)"); | |
| 85 | expect(html).not.toContain(CANT_SEE); | |
| 86 | const arm = { station: { id: "arm", name: "The arm", kind: "arm" as const, cameras: [1], controls: [] }, present: true, cameras: [{ id: 1, name: "Cam" }] }; | |
| 87 | expect(renderWorkshop([arm])).not.toContain("watchStills"); | |
| 88 | }); |
Software/so101/src/relay.ts +5 −2
| @@ -208,6 +208,9 @@ const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9})(\/try)?)?$/; | ||
| 208 | 208 | /** The cameras page, with the still-frame module under a name that changes with its content (the arm server's /ui/ takes any). */ |
| 209 | 209 | const STILL_FRAME = new URL("./ui/still-frame.js", import.meta.url); |
| 210 | 210 | const STATIONS_PAGE = readFileSync(new URL("./stations-admin.html", import.meta.url), "utf8"); |
| 211 | const GROW_HINT = new URL("./ui/grow-hint.js", import.meta.url); | |
| 212 | /** The grow card's hint module, versioned by its own content and the module it imports. */ | |
| 213 | const SIGHT_VERSION = createHash("sha256").update(readFileSync(GROW_HINT)).update(readFileSync(STILL_FRAME)).digest("hex").slice(0, 12); | |
| 211 | 214 | const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8").replaceAll( |
| 212 | 215 | "/ui/__VERSION__/", |
| 213 | 216 | `/ui/${createHash("sha256").update(readFileSync(STILL_FRAME)).digest("hex").slice(0, 12)}/`, |
| @@ -734,7 +737,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 734 | 737 | station, |
| 735 | 738 | present, |
| 736 | 739 | ...(station.kind === "arm" && rest ? { resting: restingWords(rest) } : {}), |
| 737 | cameras: present ? station.cameras.filter((id) => listed.has(id)).map((id) => ({ id, name: listed.get(id)! })) : [], | |
| 740 | cameras: present ? station.cameras.filter((id) => listed.has(id)).map((id) => ({ id, name: listed.get(id)!, region: cameras.look(id)?.region ?? null })) : [], | |
| 738 | 741 | }; |
| 739 | 742 | }); |
| 740 | 743 | }; |
| @@ -1019,7 +1022,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> { | ||
| 1019 | 1022 | if (req.method === "GET" && /^\/(?:index\.html)?$/.test(url.pathname) && stations.isWorkshop()) { |
| 1020 | 1023 | res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" }); |
| 1021 | 1024 | const who = lobby?.who(req); |
| 1022 | return res.end(renderWorkshop(stationViews(), { signedIn: lobby && !lobby.local ? lobby.signedIn(req) : null, owner: Boolean(who?.owner) })); | |
| 1025 | return res.end(renderWorkshop(stationViews(), { signedIn: lobby && !lobby.local ? lobby.signedIn(req) : null, owner: Boolean(who?.owner) }).replaceAll("/ui/__VERSION__/", `/ui/${SIGHT_VERSION}/`)); | |
| 1023 | 1026 | } |
| 1024 | 1027 | if (req.method === "GET" && PAGE.test(url.pathname) && rest) return restingPage(req, res, rest); |
| 1025 | 1028 | if (req.method === "GET" && PAGE.test(url.pathname)) return await page(req, res, await ask(simulation + path.replace(/^\/arm(?=$|\?)/, "/"), req, undefined, null, false)); |
Software/so101/src/stations.test.ts +12 −1
| @@ -87,7 +87,9 @@ test("with no arm the workshop shows the grow box and serves its camera; pluggin | ||
| 87 | 87 | res.writeHead(200, { "content-type": "image/jpeg" }); |
| 88 | 88 | res.end("jpeg"); |
| 89 | 89 | }); |
| 90 | const sim = await listen((req, res) => void res.writeHead(200, { "content-type": "text/html" }).end("<title>viewer</title>")); | |
| 90 | // the simulation is the app's own server, which serves /ui/ (the relay passes it on whichever arm is live) | |
| 91 | const simServer = await startServer(new FeetechBus(new FakeBus([1, 2, 3])), { host: "127.0.0.1", port: 0, ids: [1, 2, 3], posesFile: join(tmpdir(), `so101-stations-sim-${process.pid}.json`) }); | |
| 92 | const sim = simServer.port; | |
| 91 | 93 | const free = await listen(() => {}); // a port nothing answers on once its server is closed |
| 92 | 94 | servers.pop()!.close(); |
| 93 | 95 | const cameras = new CameraHub({ seed: [{ name: "Grow cam", source: `http://127.0.0.1:${camera}/stream` }] }); |
| @@ -103,6 +105,14 @@ test("with no arm the workshop shows the grow box and serves its camera; pluggin | ||
| 103 | 105 | expect(home).toContain("/camera/1/snapshot"); |
| 104 | 106 | expect(home).toContain("The arm: not connected"); |
| 105 | 107 | expect(home).not.toContain('href="/arm"'); |
| 108 | // the arm is unplugged and the grow card still carries its script, at a versioned path that answers | |
| 109 | const module = /import \{ watchStills \} from "(\/ui\/[0-9a-f]{12}\/grow-hint\.js)"/.exec(home)?.[1]; | |
| 110 | expect(module).toBeTruthy(); | |
| 111 | expect(home).not.toContain("__VERSION__"); | |
| 112 | const served = await fetch(`${base}${module}`); | |
| 113 | expect(served.status).toBe(200); | |
| 114 | expect(await served.text()).toContain("export function watchStills"); | |
| 115 | expect((await fetch(`${base}${module!.replace("grow-hint", "still-frame")}`)).status).toBe(200); | |
| 106 | 116 | const snap = await fetch(`${base}/camera/1/snapshot`); |
| 107 | 117 | expect(snap.status).toBe(200); |
| 108 | 118 | expect(await snap.text()).toBe("jpeg"); |
| @@ -121,6 +131,7 @@ test("with no arm the workshop shows the grow box and serves its camera; pluggin | ||
| 121 | 131 | } finally { |
| 122 | 132 | await Promise.race([relay.close(), new Promise((r) => setTimeout(r, 500))]); |
| 123 | 133 | await Promise.race([plugged.close(), new Promise((r) => setTimeout(r, 500))]); |
| 134 | await Promise.race([simServer.close(), new Promise((r) => setTimeout(r, 500))]); | |
| 124 | 135 | } |
| 125 | 136 | }); |
| 126 | 137 |
Software/so101/src/ui/grow-hint.js added +52
| @@ -0,0 +1,52 @@ | ||
| 1 | /** | |
| 2 | * The grow card's "can't see the plants" hint: the verdict on a still's box (src/ui/still-frame.js), and the words for it. | |
| 3 | * Plain ESM, served at /ui/ for the workshop page and imported by the tests. | |
| 4 | */ | |
| 5 | import { CANT_SEE, cropRect, visibility } from "./still-frame.js"; | |
| 6 | ||
| 7 | /** | |
| 8 | * Whether the plants can be seen in the box of an RGBA still. | |
| 9 | * @param {ArrayLike<number>} data | |
| 10 | * @param {number} width | |
| 11 | * @param {number} height | |
| 12 | * @param {import("./still-frame.js").Rect | null} region fractions of the frame, null for all of it | |
| 13 | * @returns {"ok" | "blown" | "dark"} | |
| 14 | */ | |
| 15 | export function sightOf(data, width, height, region) { | |
| 16 | return visibility(data, width, cropRect(region, width, height)).verdict; | |
| 17 | } | |
| 18 | ||
| 19 | /** The hint for a verdict, as HTML: nothing when the camera sees well, else the words and a link to Tune. */ | |
| 20 | export function hintHtml(verdict) { | |
| 21 | if (!verdict || verdict === "ok") return ""; | |
| 22 | return `<p class="cant-see" role="status">${CANT_SEE} <a href="/admin/camera-list">Tune</a></p>`; | |
This is also rendered server-side from a |
||
| 23 | } | |
| 24 | ||
| 25 | /** | |
| 26 | * Looks at each grow still on the page once it has loaded, and writes the hint into its slot when the box cannot be seen. | |
| 27 | * An image `[data-sight=<camera>]` has its slot `[data-sight-for=<camera>]`, whose `data-region` is the camera's box as JSON. | |
| 28 | * @param {{ querySelectorAll: Function, querySelector: Function, createElement: Function }} doc | |
| 29 | */ | |
| 30 | export function watchStills(doc) { | |
| 31 | for (const img of doc.querySelectorAll("img[data-sight]")) { | |
| 32 | const slot = doc.querySelector(`[data-sight-for="${img.dataset.sight}"]`); | |
| 33 | if (!slot) continue; | |
| 34 | const look = () => { | |
| 35 | try { | |
| 36 | const canvas = doc.createElement("canvas"); | |
| 37 | canvas.width = img.naturalWidth; | |
| 38 | canvas.height = img.naturalHeight; | |
| 39 | if (!canvas.width || !canvas.height) return; | |
| 40 | const g = canvas.getContext("2d"); | |
| 41 | g.drawImage(img, 0, 0); | |
| 42 | const { data } = g.getImageData(0, 0, canvas.width, canvas.height); | |
| 43 | slot.innerHTML = hintHtml(sightOf(data, canvas.width, canvas.height, JSON.parse(slot.dataset.region))); | |
| 44 | } catch { | |
| 45 | // A still the browser will not let us read (a tainted canvas, a failed decode, a bad region) says nothing: | |
| 46 | // a missing hint is the honest answer when we could not look, and a wrong one would send the owner to Tune for nothing. | |
| 47 | } | |
| 48 | }; | |
| 49 | if (img.complete) look(); | |
| 50 | else img.addEventListener("load", look, { once: true }); | |
Minor: the .cant-see overlay covers the bottom of the still. It is only a short bar, but a lazy-loaded second still is checked only when it loads, so the hint can appear after the owner has started looking. Fine for now. |
||
| 51 | } | |
| 52 | } |
Software/so101/src/workshop-page.ts +18 −2
| @@ -3,6 +3,7 @@ | ||
| 3 | 3 | import type { Station } from "./stations.ts"; |
| 4 | 4 | import { renderSignedIn, SIGNED_IN_STYLE, type SignedIn } from "./signed-in.ts"; |
| 5 | 5 | import { esc } from "./story-markdown.ts"; |
| 6 | import type { Rect } from "./ui/still-frame.js"; | |
| 6 | 7 | |
| 7 | 8 | export const WORKSHOP_TITLE = "Workshop"; |
| 8 | 9 | |
| @@ -12,7 +13,7 @@ export interface StationView { | ||
| 12 | 13 | /** Why a present-less arm rests, when it is closed on purpose or cannot be reached (src/resting-page.ts). */ |
| 13 | 14 | resting?: string; |
| 14 | 15 | /** Cameras of the station that are listed. */ |
| 15 | cameras: { id: number; name: string }[]; | |
| 16 | cameras: { id: number; name: string; region?: Rect | null; sight?: "ok" | "blown" | "dark" }[]; | |
| 16 | 17 | } |
| 17 | 18 | |
| 18 | 19 | const STYLE = ` |
| @@ -24,6 +25,10 @@ h1 { margin: .5rem 0 1rem; } h2 { margin: .6rem 0; font-size: 1.15rem; } | ||
| 24 | 25 | .cards { display: grid; gap: 1rem; grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr)); } |
| 25 | 26 | .card { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: .8rem; } |
| 26 | 27 | .card img { width: 100%; aspect-ratio: 4/3; object-fit: cover; border-radius: 6px; background: #000; display: block; } |
| 28 | .still { position: relative; } | |
| 29 | .sight:empty { display: none; } | |
| 30 | .cant-see { position: absolute; left: 0; right: 0; bottom: 0; margin: 0; padding: .4rem .6rem; border-radius: 0 0 6px 6px; background: #000d; color: #fff; font-size: .9rem; } | |
| 31 | .cant-see a { color: #9ecbff; } | |
| 27 | 32 | .card p { color: var(--dim); margin: .4rem 0; } |
| 28 | 33 | .top { max-width: 60rem; margin: 0 auto; padding: .5rem 1rem 0; display: flex; justify-content: space-between; gap: 1rem; font-size: .9rem; color: var(--dim); } |
| 29 | 34 | .top a, .card button { min-height: 44px; display: inline-flex; align-items: center; } |
| @@ -53,11 +58,21 @@ function card({ station, present, resting, cameras }: StationView, looker: Looke | ||
| 53 | 58 | const controls = station.controls.map((c) => `<a href="${esc(c.href)}">${esc(c.label)}</a>`).join(""); |
| 54 | 59 | return `<section class="card" data-station="${esc(station.id)}" data-present="true" data-cameras="none">${head}<p>No camera set up yet.${setup}</p>${controls ? `<div class="controls">${controls}</div>` : ""}</section>`; |
| 55 | 60 | } |
| 56 | const thumbs = cameras.map((c, i) => `<img src="/camera/${c.id}/snapshot" alt="${esc(c.name)}"${i ? ' loading="lazy"' : ""} width="640" height="480">`).join(""); | |
| 61 | const grow = station.kind === "grow"; | |
| 62 | const thumbs = cameras | |
| 63 | .map((c, i) => { | |
| 64 | const img = `<img src="/camera/${c.id}/snapshot" alt="${esc(c.name)}"${i ? ' loading="lazy"' : ""} width="640" height="480"${grow ? ` data-sight="${c.id}"` : ""}>`; | |
| 65 | // The hint sits over the foot of the still, so it moves nothing when the page's script fills it in. | |
| 66 | return grow ? `<div class="still">${img}<div class="sight" data-sight-for="${c.id}" data-region="${esc(JSON.stringify(c.region ?? null))}"></div></div>` : img; | |
| 67 | }) | |
| 68 | .join(""); | |
| 57 | 69 | const controls = station.controls.map((c) => `<a href="${esc(c.href)}">${esc(c.label)}</a>`).join(""); |
| 58 | 70 | return `<section class="card" data-station="${esc(station.id)}" data-present="true">${head}${thumbs}${controls ? `<div class="controls">${controls}</div>` : ""}</section>`; |
| 59 | 71 | } |
| 60 | 72 | |
| 73 | /** The page's one script: the grow stills' check (src/ui/grow-hint.js). */ | |
| 74 | const SIGHT_SCRIPT = `import { watchStills } from "/ui/__VERSION__/grow-hint.js"; watchStills(document);`; | |
| 75 | ||
| 61 | 76 | export function renderWorkshop(views: readonly StationView[], looker: Looker = { signedIn: null, owner: false }): string { |
| 62 | 77 | return `<!doctype html> |
| 63 | 78 | <html lang="en"> |
| @@ -75,6 +90,7 @@ export function renderWorkshop(views: readonly StationView[], looker: Looker = { | ||
| 75 | 90 | <h1>${WORKSHOP_TITLE}</h1> |
| 76 | 91 | <div class="cards">${views.map((v) => card(v, looker)).join("\n")}</div> |
| 77 | 92 | </main> |
| 93 | ${views.some((v) => v.station.kind === "grow" && v.present && v.cameras.length) ? `<script type="module">${SIGHT_SCRIPT}</script>` : ""} | |
| 78 | 94 | </body> |
| 79 | 95 | </html> |
| 80 | 96 | `; |
This tests the unreachable server-side path. The browser script that really shows the hint is untested; test it with fakes.