From 5b491a479d140c34348453197bcfff6fbbdbeefc Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Thu, 23 Jul 2026 08:19:54 +0800 Subject: [PATCH] Add picture viewer module --- .github/workflows/coverage.yaml | 9 + .mise/config.coverage.toml | 79 +++++ .mise/config.rust.toml | 6 + .mise/config.toml | 10 + Cargo.lock | 19 + Cargo.toml | 1 + .../rules/doc-summary-ends-with-period.yaml | 5 + services/ws-modules/pic-viewer/Cargo.toml | 47 +++ services/ws-modules/pic-viewer/build.rs | 20 ++ services/ws-modules/pic-viewer/src/lib.rs | 329 ++++++++++++++++++ services/ws-modules/pic-viewer/tests/parse.rs | 136 ++++++++ .../ws-modules/pic-viewer/tests/show_image.rs | 87 +++++ .../ws-modules/pyeye1/pkg/et_ws_pyeye1.js | 6 +- .../ws-modules/pyeye1/pyeye1/eye_detection.py | 37 +- services/ws-wasm-agent/src/lib.rs | 39 ++- 15 files changed, 811 insertions(+), 19 deletions(-) create mode 100644 services/ws-modules/pic-viewer/Cargo.toml create mode 100644 services/ws-modules/pic-viewer/build.rs create mode 100644 services/ws-modules/pic-viewer/src/lib.rs create mode 100644 services/ws-modules/pic-viewer/tests/parse.rs create mode 100644 services/ws-modules/pic-viewer/tests/show_image.rs diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 38c2113..b391120 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -89,6 +89,15 @@ jobs: export CHROMEDRIVER="${CHROMEWEBDRIVER:+$CHROMEWEBDRIVER/chromedriver}" mise run wasm-agent-cov + # The pic-viewer display path only runs in a browser, so it is invisible to the cargo-llvm-cov run. + # Its native parse tests already ride that run; this drives the module's wasm-bindgen display tests in + # the same headless Chrome as the ws-wasm-agent step above and folds their lcov into lcov.info. + - name: Merge pic-viewer coverage into lcov.info + timeout-minutes: 20 + run: | + export CHROMEDRIVER="${CHROMEWEBDRIVER:+$CHROMEWEBDRIVER/chromedriver}" + mise run pic-viewer-cov + - name: Collect Python coverage timeout-minutes: 15 env: diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index e73010a..88ddeb0 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -10,6 +10,16 @@ # The wasm coverage builds need it: minicov's build.rs hardcodes bare `clang` to compile its C profiler runtime # for the guest target, so this wasm-capable clang must be first on PATH (the wasm build tasks prepend it). LLVM # 22 here matches rustc's LLVM, keeping the covmap format aligned with the llvm-tools llc/llvm-cov used downstream. +# +# Known-broken on macOS: mise resolves the [env] _.path prepend below to a PATH position BEHIND /usr/bin, so +# under `mise run` bare `clang` still hits Apple's and minicov's C compile fails with "unable to create target: +# No available targets are compatible with triple wasm32-unknown-unknown". The wasm coverage tasks +# (wasm-agent-cov, pic-viewer-cov) therefore only work locally on Linux until that ordering is fixed; on a Mac, +# run the task body manually in a shell with the conda-clang bin dir genuinely first on PATH. CI (ubuntu) is +# unaffected because the distro clang already has the wasm backend. +# +# Windows is deliberately out of scope for this env (hence the os scoping here): coverage collection runs only +# on the ubuntu CI lane, and there is no intention to make the wasm coverage pipeline work on Windows. [tools] "conda:clang" = { version = "latest", os = ["linux", "macos"] } @@ -279,3 +289,72 @@ rpt="$covdir/report.txt" rg "ws-wasm-agent/src/lib.rs|Filename|TOTAL" "$rpt" || true """ shell = "bash -euo pipefail -c" + +[tasks.pic-viewer-cov] +depends = ["build-wasm-cov-wrapper"] +description = "Coverage for pic-viewer: run its wasm-bindgen display tests headless, emit lcov into lcov.info" +# Coverage for the module's browser-only display path; its native parse tests already ride cargo-llvm-cov. +# show_image's fetch/decode/canvas-draw only runs in a real browser, so this drives the wasm-bindgen browser +# tests (tests/show_image.rs) in headless Chrome, instrumented the same way as wasm-agent-cov -- the +# workspace wrapper's -Cinstrument-coverage + --emit=llvm-ir, wasm-bindgen-test's built-in coverage cfg, and +# et-web/coverage for minicov's profiler runtime -- then converts the .profraw the same way (gut every +# workspace .ll body to `unreachable`, keep the covmap, llc to a fixed x86_64 ELF object, llvm-cov to lcov, +# keep only this module's records). No backend is needed: the display tests are offline. +run = """ +covdir=target/pic-viewer-cov +coreutils mkdir -p "$covdir" +coreutils rm -f "$covdir"/*.profraw "$covdir"/*.o "$covdir"/*.g.ll +# Force a recompile so this run emits a fresh instrumented .ll whose covmap matches the captured .profraw. +rustup target add wasm32-unknown-unknown +cargo clean -p et-ws-pic-viewer --target wasm32-unknown-unknown + +# Resolve the webdriver, honouring a CHROMEDRIVER already in the environment before the mise-pinned one. +driver="${CHROMEDRIVER:-$(mise which chromedriver)}" + +runner="$(mise which wasm-bindgen-test-runner)" +export RUSTC_WORKSPACE_WRAPPER="{{ config_root }}/target/debug/int-wasm-cov-wrapper" +export RUSTFLAGS="--cfg wasm_bindgen_unstable_test_coverage" +export CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER="$runner" +export CHROMEDRIVER="$driver" +export LLVM_PROFILE_FILE="{{ config_root }}/$covdir/pic-viewer-%p.profraw" +cargo test -p et-ws-pic-viewer --features et-web/coverage --target wasm32-unknown-unknown --test show_image + +host="$(rustc +nightly -vV | goawk '/^host:/ { print $2 }')" +llbin="$(rustc +nightly --print sysroot)/lib/rustlib/$host/bin" +gut="$covdir/gut.awk" +coreutils cat > "$gut" <<'AWK' +/^target datalayout/ { next } +/^target triple/ { next } +/^define/ { print; print "start:"; print " unreachable"; print "}"; skip = 1; next } +skip && /^}/ { skip = 0; next } +skip { next } +{ + gsub(/"target-cpu"="[^"]*"/, "") + gsub(/"target-features"="[^"]*"/, "") + if ($0 ~ /^attributes #/ && $0 ~ /\\{[[:space:]]*\\}/) sub(/\\{[[:space:]]*\\}/, "{ nounwind }") + print +} +AWK +"$llbin/llvm-profdata" merge -sparse -o "$covdir/pic-viewer.profdata" "$covdir"/pic-viewer-*.profraw +objs=() +for ll in target/wasm32-unknown-unknown/debug/deps/*.ll; do + name="$(coreutils basename "$ll" .ll)" + goawk -f "$gut" "$ll" > "$covdir/$name.g.ll" + "$llbin/llc" -filetype=obj -mtriple=x86_64-unknown-linux-gnu -o "$covdir/$name.o" "$covdir/$name.g.ll" + objs+=("-object" "$covdir/$name.o") +done +"$llbin/llvm-cov" export --format=lcov --instr-profile "$covdir/pic-viewer.profdata" "${objs[@]}" > "$covdir/all.lcov" +keep="$covdir/keep.awk" +coreutils cat > "$keep" <<'AWK' +{ buf = buf $0 ORS } +/^SF:/ { keep = index($0, want) > 0 } +/^end_of_record$/ { if (keep) printf "%s", buf; buf = ""; keep = 0 } +AWK +coreutils touch lcov.info +goawk -v want="ws-modules/pic-viewer/src/" -f "$keep" "$covdir/all.lcov" >> lcov.info + +rpt="$covdir/report.txt" +"$llbin/llvm-cov" report --instr-profile "$covdir/pic-viewer.profdata" "${objs[@]}" > "$rpt" 2>/dev/null || true +rg "ws-modules/pic-viewer/src/lib.rs|Filename|TOTAL" "$rpt" || true +""" +shell = "bash -euo pipefail -c" diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 062a759..953e85c 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -30,6 +30,12 @@ description = "Build the comm1 workflow WASM module" dir = "services/ws-modules/comm1" run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +[tasks.build-ws-pic-viewer-module] +depends = ["build-wasm-cov-wrapper"] +description = "Build the pic-viewer broadcast picture viewer WASM module" +dir = "services/ws-modules/pic-viewer" +run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" + [tasks.build-ws-except1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the except1 exception-handling demo WASM module" diff --git a/.mise/config.toml b/.mise/config.toml index c820daa..3c6ca6c 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -1276,6 +1276,16 @@ description = "Run headless Chrome tests for the WebSocket WASM client" dir = "services/ws-wasm-agent" run = "env CHROMEDRIVER=\"$(mise which chromedriver)\" wasm-pack test --headless --chrome" +[tasks.test-pic-viewer-firefox] +description = "Run headless Firefox tests for the pic-viewer module (image display onto the canvas)" +dir = "services/ws-modules/pic-viewer" +run = "wasm-pack test --headless --firefox" + +[tasks.test-pic-viewer-chrome] +description = "Run headless Chrome tests for the pic-viewer module (image display onto the canvas)" +dir = "services/ws-modules/pic-viewer" +run = "env CHROMEDRIVER=\"$(mise which chromedriver)\" wasm-pack test --headless --chrome" + [tasks.ws-e2e-chrome] depends = ["test-ws-wasm-agent-chrome", "ws-server"] description = "Run both the ws-server and ws-wasm-agent using Chrome" diff --git a/Cargo.lock b/Cargo.lock index fe5e3ae..9f10506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4557,6 +4557,25 @@ dependencies = [ "web-sys", ] +[[package]] +name = "et-ws-pic-viewer" +version = "0.1.0" +dependencies = [ + "edge-toolkit", + "et-path", + "et-web", + "et-ws-wasm-agent", + "fs-err", + "js-sys", + "serde_json", + "tracing", + "tracing-wasm", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", +] + [[package]] name = "et-ws-pyo3-runner" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index d7b8dfa..d058663 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "services/ws-modules/graphics-info", "services/ws-modules/har1", "services/ws-modules/nfc", + "services/ws-modules/pic-viewer", "services/ws-modules/sensor1", "services/ws-modules/speech-recognition", "services/ws-modules/video1", diff --git a/config/ast-grep/rules/doc-summary-ends-with-period.yaml b/config/ast-grep/rules/doc-summary-ends-with-period.yaml index 17041a0..912c557 100644 --- a/config/ast-grep/rules/doc-summary-ends-with-period.yaml +++ b/config/ast-grep/rules/doc-summary-ends-with-period.yaml @@ -39,6 +39,10 @@ files: - services/websockify/src/lib.rs - services/websockify/tests/relay.rs - services/ws-modules/except1/src/lib.rs + - services/ws-modules/pic-viewer/build.rs + - services/ws-modules/pic-viewer/src/lib.rs + - services/ws-modules/pic-viewer/tests/parse.rs + - services/ws-modules/pic-viewer/tests/show_image.rs - services/ws-pyo3-runner/tests/modules.rs - services/ws-server/src/config.rs - services/ws-server/src/lib.rs @@ -51,6 +55,7 @@ files: - services/ws-wasi-runner/tests/modules.rs - services/ws-wasi-runner/tests/otel_propagation.rs - services/ws-wasi-runner/tests/vector_otlp_relay.rs + - services/ws-wasm-agent/src/lib.rs - services/ws-web-runner/build.rs - services/ws-web-runner/tests/modules.rs - services/ws/tests/config.rs diff --git a/services/ws-modules/pic-viewer/Cargo.toml b/services/ws-modules/pic-viewer/Cargo.toml new file mode 100644 index 0000000..14d83c6 --- /dev/null +++ b/services/ws-modules/pic-viewer/Cargo.toml @@ -0,0 +1,47 @@ +[package] +description = "Picture viewer" +name = "et-ws-pic-viewer" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +crate-type = ["cdylib", "rlib"] +doctest = false +test = false + +[dependencies] +edge-toolkit.workspace = true +et-web.workspace = true +et-ws-wasm-agent.workspace = true +js-sys.workspace = true +serde_json.workspace = true +tracing.workspace = true +tracing-wasm.workspace = true +wasm-bindgen.workspace = true +wasm-bindgen-futures.workspace = true +web-sys = { workspace = true, features = [ + "CanvasRenderingContext2d", + "Document", + "Element", + "HtmlCanvasElement", + "HtmlElement", + "HtmlImageElement", + "Window", + "console", +] } + +[dev-dependencies] +wasm-bindgen-test.workspace = true +# Extra web-sys surface used only by the browser tests; cargo unions it with the [dependencies] features. +# Blob backs the object-URL serving the embedded test image, ImageData the drawn-pixel readback. +web-sys = { workspace = true, features = ["Blob", "ImageData", "Url"] } + +[build-dependencies] +# Stages the favicon test fixture into OUT_DIR (see build.rs). +et-path.workspace = true +fs-err.workspace = true + +[lints] +workspace = true diff --git a/services/ws-modules/pic-viewer/build.rs b/services/ws-modules/pic-viewer/build.rs new file mode 100644 index 0000000..12fc4f4 --- /dev/null +++ b/services/ws-modules/pic-viewer/build.rs @@ -0,0 +1,20 @@ +//! Stage the browser tests' image fixture (the ws-server page favicon) into `OUT_DIR`. +//! +//! The `show_image` browser tests embed a real PNG via `include_bytes!`, and the repo's +//! `no-relative-path-literal` rule (rightly) forbids reaching it through a `../..` literal. Copying it here, +//! anchored on `et_path::find_project_root_from_manifest()`, keeps the checked-in favicon as the single +//! source of truth while giving the macro a stable `OUT_DIR` path to include. + +use std::path::PathBuf; + +#[expect( + clippy::unwrap_used, + reason = "build script: failing the build loudly is exactly right when the fixture can't stage" +)] +fn main() { + let favicon = et_path::find_project_root_from_manifest().join("services/ws-server/static/favicon.png"); + println!("cargo:rerun-if-changed={}", favicon.display()); + + let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); + let _bytes_copied = fs_err::copy(&favicon, out_dir.join("favicon.png")).unwrap(); +} diff --git a/services/ws-modules/pic-viewer/src/lib.rs b/services/ws-modules/pic-viewer/src/lib.rs new file mode 100644 index 0000000..f3a866c --- /dev/null +++ b/services/ws-modules/pic-viewer/src/lib.rs @@ -0,0 +1,329 @@ +//! pic-viewer: broadcast-driven picture viewer for eye captures stored by other agents. +//! +//! `run()` connects the module's own WebSocket client, then polls a queue of capture announcements: every +//! time another agent stores an eye capture, pyeye1 broadcasts a `pyeye1_capture_stored` payload, the server +//! relays it here inside an `et-agent-message` envelope, and this module fetches the announced storage file +//! and draws it onto the page's output canvas. In the demo scenario this module runs on a different device +//! from pyeye1: the capture device announces each stored image and this viewer shows it moments later. +//! +//! The loop stops cleanly after `IDLE_STOP_POLLS` empty polls without a new picture (each displayed picture +//! resets the idle count), after `MAX_RUNTIME_POLLS` overall, or when the page's stop control calls `stop()`. + +#![expect( + clippy::future_not_send, + clippy::single_call_fn, + reason = "browser WASM module: JsFuture is !Send; module-local helpers like wait_for_* are single-use by design" +)] + +use std::cell::{Cell, RefCell}; +use std::collections::VecDeque; +use std::rc::Rc; + +use edge_toolkit::ws::ServerMessage; +use et_ws_wasm_agent::{WsClient, WsClientConfig, set_textarea_value}; +use js_sys::{Promise, Reflect}; +use serde_json::json; +use tracing::info; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; +use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, HtmlImageElement}; + +const POLL_INTERVAL_MS: i32 = 100; +// The idle window doubles as the wait for the first picture: generous enough to walk to the capture device +// and start pyeye1 there, and to ride out the gap between two pyeye1 runs (pyeye1 stops itself after 30 +// seconds, and its captures arrive at least every 5 seconds while it runs with upload consent granted). +// 600 empty 100ms polls = ~60 seconds idle; 3000 loop iterations = ~5 minutes overall. +const IDLE_STOP_POLLS: u32 = 600; +const MAX_RUNTIME_POLLS: u32 = 3_000; +// The `kind` discriminator pyeye1 puts in its capture-stored broadcast payloads. +const CAPTURE_KIND: &str = "pyeye1_capture_stored"; + +/// One decoded capture announcement: who stored the image and the storage path to fetch it from. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct CaptureNotification { + pub from_agent_id: String, + pub filename: String, + pub url: String, +} + +thread_local! { + /// The running viewer's stop flag; `Some` while `run()` is active, shared with `stop()`. + static STOP_FLAG: RefCell>>> = const { RefCell::new(None) }; +} + +#[wasm_bindgen(start)] +pub fn init() { + tracing_wasm::set_as_global_default(); + info!("pic-viewer workflow module initialized"); +} + +/// Return whether a viewer run is currently active (the page's run button toggles run/stop on this). +#[must_use] +#[wasm_bindgen] +pub fn is_running() -> bool { + STOP_FLAG.with(|flag| flag.borrow().is_some()) +} + +/// Request the active viewer loop to stop; it exits at its next poll and cleans up after itself. +#[wasm_bindgen] +pub fn stop() { + STOP_FLAG.with(|flag| { + if let Some(stop_requested) = flag.borrow_mut().take() { + stop_requested.set(true); + } + }); + log("stop requested"); +} + +/// Run the picture-viewer workflow: connect, then display every capture announced by other agents. +#[wasm_bindgen] +pub async fn run() -> Result<(), JsValue> { + if is_running() { + return Ok(()); + } + let stop_requested = Rc::new(Cell::new(false)); + STOP_FLAG.with(|flag| *flag.borrow_mut() = Some(Rc::clone(&stop_requested))); + + let outcome = view_workflow(&stop_requested).await; + + STOP_FLAG.with(|flag| { + let _active = flag.borrow_mut().take(); + }); + hide_canvas(); + outcome +} + +/// Connect the WebSocket client and poll announced captures until an idle/runtime limit or `stop()`. +async fn view_workflow(stop_requested: &Rc>) -> Result<(), JsValue> { + log("entered run()"); + set_module_status("pic-viewer: connecting")?; + + let ws_url = websocket_url()?; + let mut config = WsClientConfig::new(ws_url); + // The viewer needs its own agent identity. Clients on one origin share the retained agent id in + // localStorage, and the server keeps a single session per id -- with a shared id, the page's client and + // this viewer would steal each other's registration on every (re)connect, and broadcast delivery (which + // goes only to the currently registered session) would silently flap away from the viewer. + config.set_use_retained_agent_id(false); + let mut client = WsClient::new(config); + + let pending: Rc>> = Rc::new(RefCell::new(VecDeque::new())); + let on_message_boxed: Box = Box::new({ + let pending = Rc::clone(&pending); + move |value: JsValue| queue_capture_notification(&pending, &value) + }); + let on_message = Closure::wrap(on_message_boxed); + client.set_on_message(on_message.as_ref().clone()); + + client.connect()?; + wait_for_connected(&client).await?; + let agent_id = wait_for_agent_id(&client).await?; + log(&format!("websocket connected with agent_id={agent_id}")); + set_module_status("pic-viewer: waiting for pictures from other agents...")?; + + let mut shown: u32 = 0; + let mut idle_polls: u32 = 0; + let mut total_polls: u32 = 0; + + while !stop_requested.get() { + if total_polls >= MAX_RUNTIME_POLLS { + log("viewer finished automatically after ~5 minutes"); + break; + } + if idle_polls >= IDLE_STOP_POLLS { + log("viewer stopped after ~60 seconds without a new picture"); + break; + } + total_polls = total_polls.saturating_add(1); + + let next = pending.borrow_mut().pop_front(); + let Some(notification) = next else { + idle_polls = idle_polls.saturating_add(1); + sleep_ms(POLL_INTERVAL_MS).await?; + continue; + }; + + if let Err(error) = show_image(¬ification.url).await { + log(&format!("failed to display {}: {error:?}", notification.url)); + continue; + } + shown = shown.saturating_add(1); + idle_polls = 0; + set_module_status(&format!( + "pic-viewer\npictures shown: {shown}\nshowing: {}\nfrom agent: {}", + notification.filename, notification.from_agent_id + ))?; + client.send_client_event( + "pic_viewer", + "displayed", + json!({ + "filename": notification.filename, + "from_agent_id": notification.from_agent_id, + "pictures_shown": shown, + "url": notification.url, + }), + )?; + log(&format!( + "displayed {} from agent {}", + notification.url, notification.from_agent_id + )); + } + + client.disconnect(); + let message = format!("pic-viewer stopped after showing {shown} picture(s)."); + log(&message); + set_module_status(&message)?; + Ok(()) +} + +/// Queue the capture announcement carried by one incoming frame, ignoring all other traffic. +fn queue_capture_notification(pending: &Rc>>, value: &JsValue) { + let Some(data) = value.as_string() else { + return; + }; + if let Some(notification) = parse_capture_notification(&data) { + pending.borrow_mut().push_back(notification); + } +} + +/// Decode one raw WebSocket frame into a capture announcement, or `None` for any other traffic. +/// +/// The socket also carries connect acks, alive responses, and unrelated broadcasts; only a well-formed +/// `et-agent-message` whose payload is a `pyeye1_capture_stored` announcement decodes. The image URL is only +/// accepted as a same-origin `/storage/` path -- announcements arrive from arbitrary peers, and this keeps a +/// buggy or malicious peer from steering the viewer at an external URL. +#[must_use] +pub fn parse_capture_notification(data: &str) -> Option { + let Ok(ServerMessage::AgentMessage { + from_agent_id, message, .. + }) = serde_json::from_str::(data) + else { + return None; + }; + if message.get("kind").and_then(serde_json::Value::as_str) != Some(CAPTURE_KIND) { + return None; + } + let url = message.get("url").and_then(serde_json::Value::as_str)?; + if !url.starts_with("/storage/") { + return None; + } + let filename = message + .get("filename") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .to_string(); + Some(CaptureNotification { + from_agent_id, + filename, + url: url.to_string(), + }) +} + +/// Fetch one announced capture via the browser's own image loader and draw it onto the output canvas. +/// +/// The canvas is zoomed to the page width -- the same presentation as pyeye1's cropped view, so both devices +/// in the demo show comparable output. +pub async fn show_image(url: &str) -> Result<(), JsValue> { + let image = HtmlImageElement::new()?; + image.set_src(url); + // decode() resolves once the image is fetched and ready to draw, and rejects on a failed fetch/decode. + let _decoded = JsFuture::from(image.decode()).await?; + + let canvas = output_canvas()?; + canvas.set_hidden(false); + canvas.set_attribute("style", "width: 100%; height: auto;")?; + canvas.set_width(image.natural_width()); + canvas.set_height(image.natural_height()); + let context = canvas + .get_context("2d")? + .ok_or_else(|| JsValue::from_str("2d canvas context unavailable"))? + .dyn_into::()?; + context.draw_image_with_html_image_element(&image, 0.0, 0.0) +} + +/// Look up the page's shared output canvas (the same element pyeye1 renders its cropped view onto). +fn output_canvas() -> Result { + let element = web_sys::window() + .and_then(|window| window.document()) + .and_then(|document| document.get_element_by_id("video-output-canvas")) + .ok_or_else(|| JsValue::from_str("Missing #video-output-canvas element"))?; + let Ok(canvas) = element.dyn_into::() else { + return Err(JsValue::from_str("#video-output-canvas is not a canvas")); + }; + Ok(canvas) +} + +/// Hide the output canvas again once the viewer stops (best-effort; a missing element is fine). +fn hide_canvas() { + if let Ok(canvas) = output_canvas() { + canvas.set_hidden(true); + } +} + +/// Log one line to the browser console with the module prefix. +fn log(message: &str) { + let line = format!("[pic-viewer] {message}"); + web_sys::console::log_1(&JsValue::from_str(&line)); +} + +/// Replace the page's module-output textarea with the viewer's current status. +fn set_module_status(message: &str) -> Result<(), JsValue> { + set_textarea_value("module-output", message) +} + +/// Wait until the WebSocket client reports the connected state, or time out after ~10 seconds. +async fn wait_for_connected(client: &WsClient) -> Result<(), JsValue> { + for _attempt in 0_u32..100 { + if client.get_state() == "connected" { + return Ok(()); + } + sleep_ms(100).await?; + } + + Err(JsValue::from_str("Timed out waiting for websocket connection")) +} + +/// Wait until the server has acknowledged the connection with an agent id, or time out after ~10 seconds. +async fn wait_for_agent_id(client: &WsClient) -> Result { + for _attempt in 0_u32..100 { + let agent_id = client.get_agent_id(); + if !agent_id.is_empty() { + return Ok(agent_id); + } + sleep_ms(100).await?; + } + + Err(JsValue::from_str("Timed out waiting for assigned agent_id")) +} + +/// Sleep for `duration_ms` via the window's timer, yielding to the browser event loop. +async fn sleep_ms(duration_ms: i32) -> Result<(), JsValue> { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let promise = Promise::new(&mut |resolve, reject| { + let callback = Closure::once_into_js(move || { + let _resolved = resolve.call0(&JsValue::NULL); + }); + + if let Err(error) = + window.set_timeout_with_callback_and_timeout_and_arguments_0(callback.unchecked_ref(), duration_ms) + { + let _rejected = reject.call1(&JsValue::NULL, &error); + } + }); + JsFuture::from(promise).await.map(et_web::ignore) +} + +/// Derive the WebSocket endpoint from the page's own origin (ws:// for http, wss:// for https). +fn websocket_url() -> Result { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let location = Reflect::get(window.as_ref(), &JsValue::from_str("location"))?; + let protocol = Reflect::get(&location, &JsValue::from_str("protocol"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.protocol is unavailable"))?; + let host = Reflect::get(&location, &JsValue::from_str("host"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.host is unavailable"))?; + let ws_protocol = if protocol == "https:" { "wss:" } else { "ws:" }; + Ok(format!("{ws_protocol}//{host}/ws")) +} diff --git a/services/ws-modules/pic-viewer/tests/parse.rs b/services/ws-modules/pic-viewer/tests/parse.rs new file mode 100644 index 0000000..8160dd8 --- /dev/null +++ b/services/ws-modules/pic-viewer/tests/parse.rs @@ -0,0 +1,136 @@ +//! Tests for `parse_capture_notification`, the frame filter feeding the pic-viewer's display queue. +//! +//! The parser is the module's whole trust boundary: every WebSocket frame the viewer's client receives goes +//! through it, and only what it returns ever reaches `show_image`. These run natively (no browser) because +//! the function is pure string-to-struct decoding. + +#![cfg(test)] + +use et_ws_pic_viewer::parse_capture_notification; +use serde_json::json; + +/// The `et-agent-message` envelope from a live end-to-end run, pinning the real wire format. +/// +/// Captured from the server's fan-out of a pyeye1 `pyeye1_capture_stored` broadcast; verbatim except for the +/// added line breaks between JSON tokens, which are insignificant whitespace to the parser. +const LIVE_BROADCAST_FRAME: &str = r#"{"type":"et-agent-message", +"message_id":"019f8c4a-2603-76e1-a8f3-3bd662e809e7", +"from_agent_id":"019f8c4a-25f5-7162-a35a-db901665cefb", +"scope":"broadcast", +"server_received_at":"2026-07-23T00:04:57.475093+00:00", +"message":{"agent_id":"019f8c4a-25f5-7162-a35a-db901665cefb", +"filename":"pyeye1-eye-capture-demo.png", +"kind":"pyeye1_capture_stored", +"url":"/storage/019f8c4a-25f5-7162-a35a-db901665cefb/pyeye1-eye-capture-demo.png"}}"#; + +/// Wrap a payload in a well-formed `et-agent-message` envelope, the shape the server relays broadcasts in. +fn envelope(payload: &serde_json::Value) -> String { + json!({ + "type": "et-agent-message", + "message_id": "test-message-id", + "from_agent_id": "agent-sender", + "scope": "broadcast", + "server_received_at": "2026-07-23T00:00:00+00:00", + "message": payload, + }) + .to_string() +} + +/// A fully valid capture announcement payload for tests to mutate. +fn valid_payload() -> serde_json::Value { + json!({ + "kind": "pyeye1_capture_stored", + "agent_id": "agent-sender", + "filename": "capture.png", + "url": "/storage/agent-sender/capture.png", + }) +} + +#[test] +fn decodes_the_live_broadcast_envelope() { + let notification = parse_capture_notification(LIVE_BROADCAST_FRAME).unwrap(); + assert_eq!(notification.from_agent_id, "019f8c4a-25f5-7162-a35a-db901665cefb"); + assert_eq!(notification.filename, "pyeye1-eye-capture-demo.png"); + assert_eq!( + notification.url, + "/storage/019f8c4a-25f5-7162-a35a-db901665cefb/pyeye1-eye-capture-demo.png" + ); +} + +#[test] +fn decodes_a_direct_scope_announcement_too() { + // A peer could announce via send_agent_message instead of a broadcast; scope is not part of the filter. + let frame = envelope(&valid_payload()).replace(r#""scope":"broadcast""#, r#""scope":"direct""#); + let notification = parse_capture_notification(&frame).unwrap(); + assert_eq!(notification.url, "/storage/agent-sender/capture.png"); +} + +#[test] +fn ignores_non_agent_message_traffic() { + let other_frames = [ + r#"{"type":"et-connect-ack","agent_id":"agent-1","status":"assigned"}"#, + r#"{"type":"et-response","message":"Alive message received"}"#, + r#"{"type":"et-message-status","message_id":"m-1","status":"broadcast","detail":"sent"}"#, + "not json at all", + "", + "42", + ]; + for frame in other_frames { + assert!( + parse_capture_notification(frame).is_none(), + "frame should be ignored: {frame}" + ); + } +} + +#[test] +fn ignores_other_broadcast_kinds() { + let mut wrong_kind = valid_payload(); + wrong_kind["kind"] = json!("some_other_module_event"); + assert!(parse_capture_notification(&envelope(&wrong_kind)).is_none()); + + let mut no_kind = valid_payload(); + let _removed = no_kind.as_object_mut().unwrap().remove("kind"); + assert!(parse_capture_notification(&envelope(&no_kind)).is_none()); +} + +#[test] +fn rejects_missing_or_non_string_urls() { + let mut no_url = valid_payload(); + let _removed = no_url.as_object_mut().unwrap().remove("url"); + assert!(parse_capture_notification(&envelope(&no_url)).is_none()); + + let mut numeric_url = valid_payload(); + numeric_url["url"] = json!(42_i32); + assert!(parse_capture_notification(&envelope(&numeric_url)).is_none()); +} + +#[test] +fn rejects_urls_outside_same_origin_storage() { + // Announcements arrive from arbitrary peers; anything that could steer the viewer off the same-origin + // storage tree must be dropped, including protocol-relative forms. + let hostile_urls = [ + "https://evil.example/steal.png", + "http://evil.example/steal.png", + "//evil.example/steal.png", + "/other/path.png", + "storage/relative.png", + "javascript:alert(1)", + ]; + for url in hostile_urls { + let mut payload = valid_payload(); + payload["url"] = json!(url); + assert!( + parse_capture_notification(&envelope(&payload)).is_none(), + "url should be rejected: {url}" + ); + } +} + +#[test] +fn missing_filename_defaults_to_empty() { + let mut no_filename = valid_payload(); + let _removed = no_filename.as_object_mut().unwrap().remove("filename"); + let notification = parse_capture_notification(&envelope(&no_filename)).unwrap(); + assert_eq!(notification.filename, ""); +} diff --git a/services/ws-modules/pic-viewer/tests/show_image.rs b/services/ws-modules/pic-viewer/tests/show_image.rs new file mode 100644 index 0000000..37fd0cf --- /dev/null +++ b/services/ws-modules/pic-viewer/tests/show_image.rs @@ -0,0 +1,87 @@ +//! Headless-browser tests proving `show_image` really renders an image onto the output canvas. +//! +//! Runs under `wasm-pack test --headless` (the `test-pic-viewer-firefox` / `-chrome` mise tasks) in a real +//! browser, exercising the genuine display path: the browser's own image fetch + decode, then the canvas +//! draw. The test data is the repo's favicon, embedded at compile time and handed to the image loader as a +//! same-origin Blob object-URL, so the drawn pixels can be read back without tainting the canvas. + +#![cfg(test)] +#![cfg(target_arch = "wasm32")] + +use et_ws_pic_viewer::show_image; +use wasm_bindgen::JsCast; +use wasm_bindgen_test::wasm_bindgen_test; + +wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + +/// The ws-server page favicon: a real 64x64 PNG, staged into `OUT_DIR` by build.rs and embedded here. +const FAVICON_PNG: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/favicon.png")); +const FAVICON_WIDTH: u32 = 64; +const FAVICON_HEIGHT: u32 = 64; + +/// Insert the page's shared output canvas (`show_image` looks it up by this id), hidden like the real page. +fn install_output_canvas() -> web_sys::HtmlCanvasElement { + let document = web_sys::window().unwrap().document().unwrap(); + if let Some(stale) = document.get_element_by_id("video-output-canvas") { + stale.remove(); + } + let canvas = document + .create_element("canvas") + .unwrap() + .dyn_into::() + .unwrap(); + canvas.set_id("video-output-canvas"); + canvas.set_hidden(true); + let _appended = document.body().unwrap().append_child(&canvas).unwrap(); + canvas +} + +/// Serve the embedded favicon bytes as a same-origin Blob object-URL the image loader can fetch. +fn favicon_object_url() -> String { + let bytes = js_sys::Uint8Array::from(FAVICON_PNG); + let parts = js_sys::Array::of1(&bytes); + let blob = web_sys::Blob::new_with_u8_array_sequence(&parts).unwrap(); + web_sys::Url::create_object_url_with_blob(&blob).unwrap() +} + +#[wasm_bindgen_test] +async fn shows_the_favicon_on_the_output_canvas() { + let canvas = install_output_canvas(); + let url = favicon_object_url(); + + show_image(&url).await.unwrap(); + web_sys::Url::revoke_object_url(&url).unwrap(); + + // The canvas must be unhidden and sized to the image's natural dimensions... + assert!(!canvas.hidden(), "canvas should be unhidden after a successful display"); + assert_eq!(canvas.width(), FAVICON_WIDTH); + assert_eq!(canvas.height(), FAVICON_HEIGHT); + + // ...and actually carry the drawn pixels: the favicon is an opaque RGB PNG, so every pixel of a real + // draw has full alpha, while an untouched canvas is all-transparent (alpha 0). + let context = canvas + .get_context("2d") + .unwrap() + .unwrap() + .dyn_into::() + .unwrap(); + let center = context.get_image_data(32.0, 32.0, 1.0, 1.0).unwrap(); + let pixel = center.data(); + assert_eq!(pixel[3], 255, "center pixel should be opaque after drawing the favicon"); +} + +#[wasm_bindgen_test] +async fn a_failed_image_load_reports_an_error_and_leaves_the_canvas_hidden() { + let canvas = install_output_canvas(); + + // A revoked object URL can no longer be fetched, so the image decode rejects. + let url = favicon_object_url(); + web_sys::Url::revoke_object_url(&url).unwrap(); + + let outcome = show_image(&url).await; + assert!( + outcome.is_err(), + "a dead URL must surface as an error, not a silent no-op" + ); + assert!(canvas.hidden(), "the canvas must stay hidden when nothing was drawn"); +} diff --git a/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js index 5f98eac..c1cdb34 100644 --- a/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js +++ b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js @@ -200,8 +200,9 @@ function render(resultsJson) { } // Encodes whatever the overlay canvas currently shows (the cropped eye band plus its box/iris/verdict -// overlay) as a PNG and uploads it to the connected agent's storage bucket. Python decides *whether* and -// *when* to call this (gated on the upload-consent checkbox and fired at most once per run); this primitive +// overlay) as a PNG, uploads it to the connected agent's storage bucket, and returns the stored filename. +// Python decides *whether* and *when* to call this (gated on the upload-consent checkbox), and uses the +// returned filename to broadcast the capture announcement the pic-viewer module listens for; this primitive // only performs the one browser-side operation of turning the current frame into stored bytes. async function saveEyeCapture(state) { const canvas = element("video-output-canvas", HTMLCanvasElement); @@ -217,6 +218,7 @@ async function saveEyeCapture(state) { const resp = await fetch(`/storage/${agentId}/${filename}`, { method: "PUT", body: bytes }); if (!resp.ok) throw new Error(`eye capture upload failed: ${resp.status} ${resp.statusText}`); log(`eye capture saved to storage: ${filename} (${bytes.length} bytes)`); + return filename; } // Screening-verdict overlay, top-left. Python sends analysis=null until the first window completes, and each diff --git a/services/ws-modules/pyeye1/pyeye1/eye_detection.py b/services/ws-modules/pyeye1/pyeye1/eye_detection.py index ed258b2..9b5f041 100644 --- a/services/ws-modules/pyeye1/pyeye1/eye_detection.py +++ b/services/ws-modules/pyeye1/pyeye1/eye_detection.py @@ -20,7 +20,7 @@ from statistics import fmean from typing import Any, TypedDict -from et_ws.messages import WsClientEvent +from et_ws.messages import WsBroadcastMessage, WsClientEvent from .gaze_analysis import ( LEFT_IRIS_CENTER, @@ -107,8 +107,8 @@ async def run(platform) -> None: `video_size() -> [w, h]`, `render(json)`, `log(str)`, `set_status(str)`, `should_stop()`, `cleanup()`, `upload_consent() -> bool` (the page's data-upload checkbox). Async members: `connect_ws()`, `start_camera()`, `play_video()`, `sleep(ms)`, `load_landmarker(model_path, bundle_path, wasm_path)`, - `save_eye_capture()` (encodes the current overlay canvas as a PNG and uploads it to the connected - agent's storage bucket), and + `save_eye_capture()` (encodes the current overlay canvas as a PNG, uploads it to the connected agent's + storage bucket, and returns the stored filename), and `infer()`, which returns one FaceLandmarker pass as the JSON string `{"faces": [[x0, y0, x1, y1, ...], ...], "width": W, "height": H}` where each face is the flat list of normalized landmark coordinates. @@ -159,18 +159,21 @@ async def wait_until(platform, predicate, timeout_ms: float, failure: str) -> No async def attempt_eye_capture(platform) -> None: - """Try one `save_eye_capture()`, reporting a failure both locally and server-side without raising. + """Try one `save_eye_capture()`; broadcast a success to other agents, report a failure without raising. Shared by the two independent capture triggers in `sample_loop` (a detection's rising edge, and the - fixed-cadence periodic heartbeat) so a failed upload from either one is handled identically: logged - locally, reported as a server-visible event, and never allowed to abort the sample loop or be misreported - as an inference error. + fixed-cadence periodic heartbeat) so both behave identically: each stored capture is announced to every + other connected agent (the pic-viewer module running on another device listens for these announcements + and displays the file), and a failed upload is logged locally plus reported as a server-visible event, + never allowed to abort the sample loop or be misreported as an inference error. """ try: - await platform.save_eye_capture() + filename = await platform.save_eye_capture() except Exception as exc: platform.log(f"eye capture failed: {exc}") platform.send_event(eye_capture_error_event_json(str(exc))) + return + platform.send_event(capture_broadcast_json(str(platform.agent_id()), str(filename))) async def sample_loop(platform) -> None: @@ -380,6 +383,24 @@ def client_event_json(details: dict[str, object]) -> str: ).model_dump_json() +def capture_broadcast_json(agent_id: str, filename: str) -> str: + """Build the et-broadcast-message JSON announcing a freshly stored eye capture to every other agent. + + The server relays this to each other connected agent inside an `et-agent-message` envelope. The payload's + `kind` is the discriminator the pic-viewer module matches on, and `url` is the same-origin storage path + it fetches and displays. + """ + return WsBroadcastMessage( + type="et-broadcast-message", + message={ + "kind": "pyeye1_capture_stored", + "agent_id": agent_id, + "filename": filename, + "url": f"/storage/{agent_id}/{filename}", + }, + ).model_dump_json() + + def eye_capture_error_event_json(error: str) -> str: """Build the et-client-event JSON envelope for a failed eye-capture upload. diff --git a/services/ws-wasm-agent/src/lib.rs b/services/ws-wasm-agent/src/lib.rs index 8575e2d..8b16a80 100644 --- a/services/ws-wasm-agent/src/lib.rs +++ b/services/ws-wasm-agent/src/lib.rs @@ -71,6 +71,7 @@ pub struct WsClientConfig { alive_interval_ms: u32, max_reconnect_attempts: u32, initial_reconnect_delay_ms: u32, + use_retained_agent_id: bool, } #[wasm_bindgen] @@ -87,6 +88,7 @@ impl WsClientConfig { alive_interval_ms: DEFAULT_ALIVE_INTERVAL_MS, max_reconnect_attempts: 10, initial_reconnect_delay_ms: 1000, + use_retained_agent_id: true, } } @@ -104,6 +106,17 @@ impl WsClientConfig { pub fn set_initial_reconnect_delay(&mut self, delay_ms: u32) { self.initial_reconnect_delay_ms = delay_ms; } + + /// Opt out of the shared retained agent id for an ephemeral, per-client identity. + /// + /// Every client on an origin shares one retained agent id in localStorage, and the server keeps a single + /// session per agent id -- so a page's own client and a module's client using the same id steal each + /// other's registration on every (re)connect. A client that sets this to `false` neither loads nor + /// stores the retained id: the server assigns it a fresh id and it coexists with the page's client. + #[wasm_bindgen(setter)] + pub fn set_use_retained_agent_id(&mut self, use_retained: bool) { + self.use_retained_agent_id = use_retained; + } } // Inner shared state @@ -133,7 +146,11 @@ impl WsClient { #[wasm_bindgen(constructor)] #[must_use] pub fn new(config: WsClientConfig) -> Self { - let agent_id = load_stored_agent_id(); + let agent_id = if config.use_retained_agent_id { + load_stored_agent_id() + } else { + None + }; info!("Creating new WebSocket client with retained agent ID: {:?}", agent_id); let shared = Rc::new(RefCell::new(SharedState { @@ -156,7 +173,7 @@ impl WsClient { } } - /// Connect to the WebSocket server + /// Connect to the WebSocket server. #[wasm_bindgen] #[expect( clippy::too_many_lines, @@ -212,6 +229,7 @@ impl WsClient { let on_message_box: Box = Box::new({ let shared = Rc::clone(&self.shared); let retained_agent_id = Rc::clone(&self.agent_id); + let use_retained_agent_id = self.config.use_retained_agent_id; move |event: MessageEvent| { info!("WebSocket message received"); if let Some(data) = event.data().as_string() { @@ -225,7 +243,9 @@ impl WsClient { agent_id, status ); *retained_agent_id.borrow_mut() = Some(agent_id.clone()); - if let Err(error) = store_agent_id(&agent_id) { + // An ephemeral-identity client must not clobber the page's shared retained + // id with its own throwaway one. + if use_retained_agent_id && let Err(error) = store_agent_id(&agent_id) { warn!("Failed to persist agent ID: {:?}", error); } match status { @@ -319,7 +339,7 @@ impl WsClient { Ok(()) } - /// Disconnect from the WebSocket server + /// Disconnect from the WebSocket server. #[wasm_bindgen] pub fn disconnect(&mut self) { info!("Disconnecting WebSocket client"); @@ -338,7 +358,7 @@ impl WsClient { self.notify_state_change(); } - /// Send an alive message to the server + /// Send an alive message to the server. #[wasm_bindgen] pub fn send_alive(&self) -> Result<(), JsValue> { let state = self.shared.borrow(); @@ -359,7 +379,7 @@ impl WsClient { Ok(()) } - /// Send a custom message to the server + /// Send a custom message to the server. #[wasm_bindgen] pub fn send(&self, message: &str) -> Result<(), JsValue> { let should_queue = { @@ -396,7 +416,7 @@ impl WsClient { } } - /// Get the current connection state + /// Get the current connection state. #[wasm_bindgen] #[must_use] pub fn get_state(&self) -> String { @@ -415,13 +435,13 @@ impl WsClient { self.agent_id.borrow().clone().unwrap_or_default() } - /// Set callback for message events + /// Set callback for message events. #[wasm_bindgen] pub fn set_on_message(&mut self, callback: JsValue) { self.shared.borrow_mut().on_message_callback = Some(callback); } - /// Set callback for state change events + /// Set callback for state change events. #[wasm_bindgen] pub fn set_on_state_change(&mut self, callback: JsValue) { self.shared.borrow_mut().on_state_change_callback = Some(callback); @@ -718,6 +738,7 @@ impl Clone for WsClient { alive_interval_ms: self.config.alive_interval_ms, max_reconnect_attempts: self.config.max_reconnect_attempts, initial_reconnect_delay_ms: self.config.initial_reconnect_delay_ms, + use_retained_agent_id: self.config.use_retained_agent_id, }, agent_id: Rc::clone(&self.agent_id), shared: Rc::clone(&self.shared),