Public: anyone can read this; the crew can change it.

Migrated from github.com/Trevato/SO-ARM100

workbenchthis machine

Grow card: "Camera can't see the plants" hint when the camera's region is mostly blown out or black (split from #584) #597

Merged opened this

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

smith/arm:smith/590-grow-card-camera-can-t-see-the-plants-hi → main 2 commits fork 3 threads · 3 unresolved

  • Merged. smith/590-grow-card-camera-can-t-see-the-plants-hi was merged into main as 315c1ae.

Review

Sign in to review this change.
6 files changed +176 −5
Software/so101/README.md +1
@@ -50,6 +50,7 @@ The arm closes itself too: a real arm (ARM_SERVER) that has not answered for mor
5050 The app is a workshop: one Pi, several **stations**, each with its cameras and controls. Today there are two, **the arm** and **the grow box** (spinach under a UV light). `/` is the workshop home, a card per station with its live camera snapshot first and its controls after, and the arm's own page is at `/arm` (its URL is unchanged for anyone who has it). The page and its title say "Workshop"; it shows who is looking ("Signed in as …", or "Watching as a guest · Sign in"), has a skip link, and keeps working while the arm is closed.
5151
5252 - **Any subset present.** The grow box has nothing to enumerate (the light is USB power only), so it is always there. The arm's card is there only while the link to the Pi's `so101 serve` is live; unplugged, closed by the owner or unreachable it says why ("The arm: not connected", "The arm is resting") and offers no controls, and the owner sees **Open now** on it. Plugging the arm back in brings its card back by itself, with no restart. `so101 serve` with no servo board waits for one ("arm not connected; waiting for its servo board") instead of exiting, so the Pi's service stays up for the cameras.
53 - **"Camera can't see the plants."** On the grow box's card, each still is checked in the browser once it has loaded: the part inside the camera's box (set on Tune, `/admin/camera-list`) is counted for blown-out and black pixels (`visibility` in src/ui/still-frame.js), and when the plants cannot be seen the card says so over the foot of the still, with a link to Tune (src/ui/grow-hint.js). The relay never decodes a JPEG, so the terminal and the JSON doors cannot show this hint; they have no pixels to count.
5354 - **The config** is `so101/stations.json` under `FORGE_APP_DATA`: a list of `{id, name, kind: "arm"|"grow", cameras: [camera ids], controls: [{label, href}]}` (src/stations.ts). Without a file it is the arm (camera 1) and the grow box (camera 2, from the camera list): edit it to match the cameras. With only an arm in it, `/` is the arm's page as before.
5455 - **Three doors.** The page `/admin/stations/edit` (owner; a raw JSON box that needs JavaScript, so it is awkward on a phone: prefer a laptop or the terminal), `GET /stations` (anyone: each station's `present`, why not, and its camera ids and names) and `PUT /admin/stations` (owner; at most 16 KB, 400 on bad JSON or a bad station), and in the terminal `/stations` to list and `/stations set <json list>` (owner) to replace them.
5556
Software/so101/src/grow-hint.test.ts added +88
@@ -0,0 +1,88 @@
1 import { expect, test } from "bun:test";
2 import { CANT_SEE } from "./ui/still-frame.js";
3 import { hintHtml, sightOf, watchStills } from "./ui/grow-hint.js";
4 import { renderWorkshop } from "./workshop-page.ts";
5
6 const paint = (w: number, h: number, at: (x: number, y: number) => [number, number, number]) => {
7 const d = new Uint8ClampedArray(w * h * 4);
8 for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) d.set([...at(x, y), 255], (y * w + x) * 4);
9 return d;
10 };
11 const blown = paint(20, 20, (x) => (x < 12 ? [40, 120, 40] : [255, 255, 255]));
12 const fine = paint(20, 20, () => [60, 130, 50]);
13 const region = { x: 0.6, y: 0, w: 0.4, h: 1 };
14
15 /** A page with one grow still whose pixels are `pixels`; `complete` false means its load event has yet to fire. */
16 function page(pixels: Uint8ClampedArray | "taint", regionJson = JSON.stringify(region), complete = true) {
17 const slot = { dataset: { region: regionJson }, innerHTML: "" };
18 const listeners: (() => void)[] = [];
19 const img = {
20 dataset: { sight: "2" },
21 naturalWidth: 20,
22 naturalHeight: 20,
23 complete,
24 addEventListener: (_: string, f: () => void) => void listeners.push(f),
25 };
26 const doc = {
27 querySelectorAll: (q: string) => (q === "img[data-sight]" ? [img] : []),
28 querySelector: (q: string) => (q === '[data-sight-for="2"]' ? slot : null),
29 createElement: () => ({
30 width: 0,
31 height: 0,
32 getContext: () => ({
33 drawImage() {},
outdated

This tests the unreachable server-side path. The browser script that really shows the hint is untested; test it with fakes.

34 getImageData() {
35 if (pixels === "taint") throw new Error("SecurityError");
36 return { data: pixels };
37 },
38 }),
39 }),
40 };
41 return { doc, slot, load: () => listeners.forEach((f) => f()) };
42 }
43
44 test("an overexposed box is blown, judged inside the region only", () => {
45 expect(sightOf(blown, 20, 20, region)).toBe("blown");
46 expect(sightOf(blown, 20, 20, null)).toBe("ok");
47 expect(sightOf(fine, 20, 20, region)).toBe("ok");
48 expect(hintHtml("ok")).toBe("");
49 expect(hintHtml("dark")).toContain('<a href="/admin/camera-list">Tune</a>');
50 });
51
52 test("the page's script fills the hint for an overexposed still, and leaves it empty for a fine one", () => {
53 const bad = page(blown);
54 watchStills(bad.doc);
55 expect(bad.slot.innerHTML).toContain(CANT_SEE);
56 expect(bad.slot.innerHTML).toContain("/admin/camera-list");
57 const good = page(fine);
58 watchStills(good.doc);
59 expect(good.slot.innerHTML).toBe("");
60 });
61
62 test("a still that has not loaded yet is looked at when it does", () => {
63 const late = page(blown, undefined, false);
64 watchStills(late.doc);
65 expect(late.slot.innerHTML).toBe("");
66 late.load();
67 expect(late.slot.innerHTML).toContain(CANT_SEE);
68 });
69
70 test("a still the browser will not let us read, or a bad region, leaves the hint empty", () => {
71 const tainted = page("taint");
72 watchStills(tainted.doc);
73 expect(tainted.slot.innerHTML).toBe("");
74 const bad = page(blown, "{nope");
75 watchStills(bad.doc);
76 expect(bad.slot.innerHTML).toBe("");
77 });
78
79 test("the grow card has an empty slot carrying its region, and runs the script; other cards do not", () => {
80 const grow = { station: { id: "grow", name: "Grow box", kind: "grow" as const, cameras: [2], controls: [] }, present: true, cameras: [{ id: 2, name: "Side", region }] };
81 const html = renderWorkshop([grow]);
82 expect(html).toContain('data-sight="2"');
83 expect(html).toContain('data-sight-for="2" data-region="{&quot;x&quot;:0.6');
84 expect(html).toContain("watchStills(document)");
85 expect(html).not.toContain(CANT_SEE);
86 const arm = { station: { id: "arm", name: "The arm", kind: "arm" as const, cameras: [1], controls: [] }, present: true, cameras: [{ id: 1, name: "Cam" }] };
87 expect(renderWorkshop([arm])).not.toContain("watchStills");
88 });
Software/so101/src/relay.ts +5 −2
@@ -208,6 +208,9 @@ const ADMIN_CAMERA = /^\/admin\/cameras(?:\/(\d{1,9})(\/try)?)?$/;
208208 /** The cameras page, with the still-frame module under a name that changes with its content (the arm server's /ui/ takes any). */
209209 const STILL_FRAME = new URL("./ui/still-frame.js", import.meta.url);
210210 const STATIONS_PAGE = readFileSync(new URL("./stations-admin.html", import.meta.url), "utf8");
211 const GROW_HINT = new URL("./ui/grow-hint.js", import.meta.url);
212 /** The grow card's hint module, versioned by its own content and the module it imports. */
213 const SIGHT_VERSION = createHash("sha256").update(readFileSync(GROW_HINT)).update(readFileSync(STILL_FRAME)).digest("hex").slice(0, 12);
211214 const CAMERA_PAGE = readFileSync(new URL("./camera-list.html", import.meta.url), "utf8").replaceAll(
212215 "/ui/__VERSION__/",
213216 `/ui/${createHash("sha256").update(readFileSync(STILL_FRAME)).digest("hex").slice(0, 12)}/`,
@@ -734,7 +737,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
734737 station,
735738 present,
736739 ...(station.kind === "arm" && rest ? { resting: restingWords(rest) } : {}),
737 cameras: present ? station.cameras.filter((id) => listed.has(id)).map((id) => ({ id, name: listed.get(id)! })) : [],
740 cameras: present ? station.cameras.filter((id) => listed.has(id)).map((id) => ({ id, name: listed.get(id)!, region: cameras.look(id)?.region ?? null })) : [],
738741 };
739742 });
740743 };
@@ -1019,7 +1022,7 @@ export async function startRelay(options: RelayOptions): Promise<RunningRelay> {
10191022 if (req.method === "GET" && /^\/(?:index\.html)?$/.test(url.pathname) && stations.isWorkshop()) {
10201023 res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" });
10211024 const who = lobby?.who(req);
1022 return res.end(renderWorkshop(stationViews(), { signedIn: lobby && !lobby.local ? lobby.signedIn(req) : null, owner: Boolean(who?.owner) }));
1025 return res.end(renderWorkshop(stationViews(), { signedIn: lobby && !lobby.local ? lobby.signedIn(req) : null, owner: Boolean(who?.owner) }).replaceAll("/ui/__VERSION__/", `/ui/${SIGHT_VERSION}/`));
10231026 }
10241027 if (req.method === "GET" && PAGE.test(url.pathname) && rest) return restingPage(req, res, rest);
10251028 if (req.method === "GET" && PAGE.test(url.pathname)) return await page(req, res, await ask(simulation + path.replace(/^\/arm(?=$|\?)/, "/"), req, undefined, null, false));
Software/so101/src/stations.test.ts +12 −1
@@ -87,7 +87,9 @@ test("with no arm the workshop shows the grow box and serves its camera; pluggin
8787 res.writeHead(200, { "content-type": "image/jpeg" });
8888 res.end("jpeg");
8989 });
90 const sim = await listen((req, res) => void res.writeHead(200, { "content-type": "text/html" }).end("<title>viewer</title>"));
90 // the simulation is the app's own server, which serves /ui/ (the relay passes it on whichever arm is live)
91 const simServer = await startServer(new FeetechBus(new FakeBus([1, 2, 3])), { host: "127.0.0.1", port: 0, ids: [1, 2, 3], posesFile: join(tmpdir(), `so101-stations-sim-${process.pid}.json`) });
92 const sim = simServer.port;
9193 const free = await listen(() => {}); // a port nothing answers on once its server is closed
9294 servers.pop()!.close();
9395 const cameras = new CameraHub({ seed: [{ name: "Grow cam", source: `http://127.0.0.1:${camera}/stream` }] });
@@ -103,6 +105,14 @@ test("with no arm the workshop shows the grow box and serves its camera; pluggin
103105 expect(home).toContain("/camera/1/snapshot");
104106 expect(home).toContain("The arm: not connected");
105107 expect(home).not.toContain('href="/arm"');
108 // the arm is unplugged and the grow card still carries its script, at a versioned path that answers
109 const module = /import \{ watchStills \} from "(\/ui\/[0-9a-f]{12}\/grow-hint\.js)"/.exec(home)?.[1];
110 expect(module).toBeTruthy();
111 expect(home).not.toContain("__VERSION__");
112 const served = await fetch(`${base}${module}`);
113 expect(served.status).toBe(200);
114 expect(await served.text()).toContain("export function watchStills");
115 expect((await fetch(`${base}${module!.replace("grow-hint", "still-frame")}`)).status).toBe(200);
106116 const snap = await fetch(`${base}/camera/1/snapshot`);
107117 expect(snap.status).toBe(200);
108118 expect(await snap.text()).toBe("jpeg");
@@ -121,6 +131,7 @@ test("with no arm the workshop shows the grow box and serves its camera; pluggin
121131 } finally {
122132 await Promise.race([relay.close(), new Promise((r) => setTimeout(r, 500))]);
123133 await Promise.race([plugged.close(), new Promise((r) => setTimeout(r, 500))]);
134 await Promise.race([simServer.close(), new Promise((r) => setTimeout(r, 500))]);
124135 }
125136 });
126137
Software/so101/src/ui/grow-hint.js added +52
@@ -0,0 +1,52 @@
1 /**
2 * The grow card's "can't see the plants" hint: the verdict on a still's box (src/ui/still-frame.js), and the words for it.
3 * Plain ESM, served at /ui/ for the workshop page and imported by the tests.
4 */
5 import { CANT_SEE, cropRect, visibility } from "./still-frame.js";
6
7 /**
8 * Whether the plants can be seen in the box of an RGBA still.
9 * @param {ArrayLike<number>} data
10 * @param {number} width
11 * @param {number} height
12 * @param {import("./still-frame.js").Rect | null} region fractions of the frame, null for all of it
13 * @returns {"ok" | "blown" | "dark"}
14 */
15 export function sightOf(data, width, height, region) {
16 return visibility(data, width, cropRect(region, width, height)).verdict;
17 }
18
19 /** The hint for a verdict, as HTML: nothing when the camera sees well, else the words and a link to Tune. */
20 export function hintHtml(verdict) {
21 if (!verdict || verdict === "ok") return "";
22 return `<p class="cant-see" role="status">${CANT_SEE} <a href="/admin/camera-list">Tune</a></p>`;

This is also rendered server-side from a sight the relay never sets. Remove that path or make the relay set it; today the tests exercise dead code.

23 }
24
25 /**
26 * Looks at each grow still on the page once it has loaded, and writes the hint into its slot when the box cannot be seen.
27 * An image `[data-sight=<camera>]` has its slot `[data-sight-for=<camera>]`, whose `data-region` is the camera's box as JSON.
28 * @param {{ querySelectorAll: Function, querySelector: Function, createElement: Function }} doc
29 */
30 export function watchStills(doc) {
31 for (const img of doc.querySelectorAll("img[data-sight]")) {
32 const slot = doc.querySelector(`[data-sight-for="${img.dataset.sight}"]`);
33 if (!slot) continue;
34 const look = () => {
35 try {
36 const canvas = doc.createElement("canvas");
37 canvas.width = img.naturalWidth;
38 canvas.height = img.naturalHeight;
39 if (!canvas.width || !canvas.height) return;
40 const g = canvas.getContext("2d");
41 g.drawImage(img, 0, 0);
42 const { data } = g.getImageData(0, 0, canvas.width, canvas.height);
43 slot.innerHTML = hintHtml(sightOf(data, canvas.width, canvas.height, JSON.parse(slot.dataset.region)));
44 } catch {
45 // A still the browser will not let us read (a tainted canvas, a failed decode, a bad region) says nothing:
46 // a missing hint is the honest answer when we could not look, and a wrong one would send the owner to Tune for nothing.
47 }
48 };
49 if (img.complete) look();
50 else img.addEventListener("load", look, { once: true });

Minor: the .cant-see overlay covers the bottom of the still. It is only a short bar, but a lazy-loaded second still is checked only when it loads, so the hint can appear after the owner has started looking. Fine for now.

51 }
52 }
Software/so101/src/workshop-page.ts +18 −2
@@ -3,6 +3,7 @@
33 import type { Station } from "./stations.ts";
44 import { renderSignedIn, SIGNED_IN_STYLE, type SignedIn } from "./signed-in.ts";
55 import { esc } from "./story-markdown.ts";
6 import type { Rect } from "./ui/still-frame.js";
67
78 export const WORKSHOP_TITLE = "Workshop";
89
@@ -12,7 +13,7 @@ export interface StationView {
1213 /** Why a present-less arm rests, when it is closed on purpose or cannot be reached (src/resting-page.ts). */
1314 resting?: string;
1415 /** Cameras of the station that are listed. */
15 cameras: { id: number; name: string }[];
16 cameras: { id: number; name: string; region?: Rect | null; sight?: "ok" | "blown" | "dark" }[];
1617 }
1718
1819 const STYLE = `
@@ -24,6 +25,10 @@ h1 { margin: .5rem 0 1rem; } h2 { margin: .6rem 0; font-size: 1.15rem; }
2425 .cards { display: grid; gap: 1rem; grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr)); }
2526 .card { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: .8rem; }
2627 .card img { width: 100%; aspect-ratio: 4/3; object-fit: cover; border-radius: 6px; background: #000; display: block; }
28 .still { position: relative; }
29 .sight:empty { display: none; }
30 .cant-see { position: absolute; left: 0; right: 0; bottom: 0; margin: 0; padding: .4rem .6rem; border-radius: 0 0 6px 6px; background: #000d; color: #fff; font-size: .9rem; }
31 .cant-see a { color: #9ecbff; }
2732 .card p { color: var(--dim); margin: .4rem 0; }
2833 .top { max-width: 60rem; margin: 0 auto; padding: .5rem 1rem 0; display: flex; justify-content: space-between; gap: 1rem; font-size: .9rem; color: var(--dim); }
2934 .top a, .card button { min-height: 44px; display: inline-flex; align-items: center; }
@@ -53,11 +58,21 @@ function card({ station, present, resting, cameras }: StationView, looker: Looke
5358 const controls = station.controls.map((c) => `<a href="${esc(c.href)}">${esc(c.label)}</a>`).join("");
5459 return `<section class="card" data-station="${esc(station.id)}" data-present="true" data-cameras="none">${head}<p>No camera set up yet.${setup}</p>${controls ? `<div class="controls">${controls}</div>` : ""}</section>`;
5560 }
56 const thumbs = cameras.map((c, i) => `<img src="/camera/${c.id}/snapshot" alt="${esc(c.name)}"${i ? ' loading="lazy"' : ""} width="640" height="480">`).join("");
61 const grow = station.kind === "grow";
62 const thumbs = cameras
63 .map((c, i) => {
64 const img = `<img src="/camera/${c.id}/snapshot" alt="${esc(c.name)}"${i ? ' loading="lazy"' : ""} width="640" height="480"${grow ? ` data-sight="${c.id}"` : ""}>`;
65 // The hint sits over the foot of the still, so it moves nothing when the page's script fills it in.
66 return grow ? `<div class="still">${img}<div class="sight" data-sight-for="${c.id}" data-region="${esc(JSON.stringify(c.region ?? null))}"></div></div>` : img;
67 })
68 .join("");
5769 const controls = station.controls.map((c) => `<a href="${esc(c.href)}">${esc(c.label)}</a>`).join("");
5870 return `<section class="card" data-station="${esc(station.id)}" data-present="true">${head}${thumbs}${controls ? `<div class="controls">${controls}</div>` : ""}</section>`;
5971 }
6072
73 /** The page's one script: the grow stills' check (src/ui/grow-hint.js). */
74 const SIGHT_SCRIPT = `import { watchStills } from "/ui/__VERSION__/grow-hint.js"; watchStills(document);`;
75
6176 export function renderWorkshop(views: readonly StationView[], looker: Looker = { signedIn: null, owner: false }): string {
6277 return `<!doctype html>
6378 <html lang="en">
@@ -75,6 +90,7 @@ export function renderWorkshop(views: readonly StationView[], looker: Looker = {
7590 <h1>${WORKSHOP_TITLE}</h1>
7691 <div class="cards">${views.map((v) => card(v, looker)).join("\n")}</div>
7792 </main>
93 ${views.some((v) => v.station.kind === "grow" && v.present && v.cameras.length) ? `<script type="module">${SIGHT_SCRIPT}</script>` : ""}
7894 </body>
7995 </html>
8096 `;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs