diff --git a/CLAUDE.md b/CLAUDE.md index e45b76ab..e1254cff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1066,6 +1066,24 @@ and drop the `mise exec --` / shell-builtin workarounds. Single Cargo workspace (`Cargo.toml`). Shared dependency versions are declared in `[workspace.dependencies]`. Add new deps there, not in individual crate `[dependencies]`. +## Avoid GPL and LGPL dependencies + +Every license `config/deny.toml`'s `[licenses] allow` list currently accepts is permissive (Apache-2.0, MIT, +BSD, Zlib, ISC, MPL-2.0, etc.) -- none copyleft. Keep it that way: do not add a new direct or transitive +dependency whose license is GPL (any version) or LGPL (any version), and do not add GPL/LGPL to the +`deny.toml` allow list, even when the license is individually OSI-approved (LGPL is). Being OSI-approved is +not sufficient on its own here -- copyleft obligations are a different category of concern from a permissive +license, and accepting one is a project-policy decision, not a mechanical license check. + +Concrete precedent: `viuer` (added for the storage service's tty image-preview feature) pulled in +`ansi_colours` under LGPL-3.0-or-later as an unconditional dependency for its RGB-to-256-color quantization +path. Rather than allow-listing LGPL, the feature was reimplemented as a small hand-rolled ANSI truecolor +renderer in `services/storage/src/tty_image.rs`, built on the `image` crate the workspace already depended on +via `deno_image`/`ws-web-runner` -- truecolor output needs no palette quantization at all, so there was +nothing left for the LGPL dependency to do. If a future dependency's only viable path involves a GPL/LGPL +component, treat that the same way: look for a permissively-licensed alternative or a narrower reimplementation +first, and only bring the question to the user if neither is workable -- don't add the exception yourself. + ## Clippy lints **Never weaken or disable a lint to make code pass -- not the workspace lint config (`[workspace.lints.*]`, diff --git a/Cargo.lock b/Cargo.lock index 45072f4a..4b788b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4330,6 +4330,7 @@ dependencies = [ "edge-toolkit", "fs-err", "futures-util", + "image", "log", "serde", "serde-inline-default", diff --git a/Cargo.toml b/Cargo.toml index ffd011ad..d7b8dfa6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,6 +93,10 @@ futures-util = "0.3" heck = "0.5" hostname = "0.4" humantime-serde = "1" +# Already a workspace dependency via deno_image/ws-web-runner, so this adds no new crate or license to review. +# Explicit format features only (no avif/dds/exr/hdr/ico/pnm/qoi/tga/tiff): et-storage-service only decodes +# the extensions is_image_filename() recognises. +image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg", "png", "webp"] } int-otlp-mock = { path = "libs/otlp-mock", version = "0.1.0" } js-sys = "0.3" kdl = { version = "6", features = ["v1"] } diff --git a/services/storage/Cargo.toml b/services/storage/Cargo.toml index 8ccb4a2b..18ceb414 100644 --- a/services/storage/Cargo.toml +++ b/services/storage/Cargo.toml @@ -19,6 +19,7 @@ actix-web-thiserror.workspace = true edge-toolkit.workspace = true fs-err.workspace = true futures-util.workspace = true +image.workspace = true log.workspace = true serde.workspace = true serde-inline-default.workspace = true diff --git a/services/storage/src/lib.rs b/services/storage/src/lib.rs index 3a0bc860..c7d6627a 100644 --- a/services/storage/src/lib.rs +++ b/services/storage/src/lib.rs @@ -9,6 +9,7 @@ use serde_default::DefaultFromSerde; use thiserror::Error; pub mod routes; +mod tty_image; pub use self::routes::put_file; diff --git a/services/storage/src/routes.rs b/services/storage/src/routes.rs index 5d183b1c..15bbb354 100644 --- a/services/storage/src/routes.rs +++ b/services/storage/src/routes.rs @@ -24,7 +24,7 @@ use std::path::PathBuf; use actix_web::{HttpRequest, HttpResponse, web}; use edge_toolkit::ws_server::AgentRegistry; use futures_util::StreamExt as _; -use tracing::info; +use tracing::{info, warn}; use crate::{StorageConfig, StorageError}; @@ -97,15 +97,56 @@ where let path = agent_dir.join(&filename); info!("Agent {} storing file: {:?}", agent_id, path); - let mut file = tokio::fs::File::create(path).await?; + let mut file = tokio::fs::File::create(&path).await?; + let mut bytes_written: u64 = 0; while let Some(chunk) = payload.next().await { let chunk = chunk?; - let _copied: u64 = tokio::io::copy(&mut chunk.as_ref(), &mut file).await?; + let copied = tokio::io::copy(&mut chunk.as_ref(), &mut file).await?; + bytes_written = bytes_written.saturating_add(copied); + } + + // This handler is the only path a file reaches storage through, so it is where every stored file can be + // watched exactly once with an accurate byte count and zero extra I/O -- a separate filesystem watcher + // would duplicate that work and risk observing a write mid-flight. + if is_image_filename(&filename) { + info!("Agent {} stored image {:?} ({} bytes)", agent_id, path, bytes_written); + show_image_on_tty(&path); } Ok(HttpResponse::Ok().finish()) } +/// Render a thumbnail of the image at `path` directly to stdout, so the operator actually *sees* what was +/// stored rather than just its filename and byte count. +/// +/// This bypasses `tracing` entirely and writes straight to the terminal: the escape sequences (ANSI +/// truecolor half-block art -- see the `tty_image` module) are tty-only presentation, not structured log +/// data, and would otherwise get shipped to the OTLP log exporter as log-record noise. Decode/render +/// failures (a corrupt upload, a non-terminal stdout) only cost tty visibility, not the request, so they're +/// reported via `warn!` rather than via `?`. +#[expect( + clippy::single_call_fn, + reason = "distinct step of put_file; kept separate for readability and testing" +)] +fn show_image_on_tty(path: &std::path::Path) { + if let Err(error) = crate::tty_image::render(path) { + warn!("failed to render stored image {} to tty: {error}", path.display()); + } +} + +/// Return whether `filename`'s extension marks it as an image, for the tty log line in [`put_file`]. +/// +/// Extension-only: the handler streams bytes straight to disk without buffering, so sniffing magic bytes +/// would need a peek-buffer or a post-write read-back, while the filename is already on hand for free. +#[must_use] +pub fn is_image_filename(filename: &std::path::Path) -> bool { + const IMAGE_EXTENSIONS: [&str; 6] = ["png", "jpg", "jpeg", "gif", "webp", "bmp"]; + filename + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| IMAGE_EXTENSIONS.iter().any(|known| known.eq_ignore_ascii_case(ext))) +} + /// Download a file previously written to the named agent's storage bucket. #[cfg(feature = "openapi-spec")] #[utoipa::path( diff --git a/services/storage/src/tty_image.rs b/services/storage/src/tty_image.rs new file mode 100644 index 00000000..ea705fd6 --- /dev/null +++ b/services/storage/src/tty_image.rs @@ -0,0 +1,83 @@ +//! Hand-rolled ANSI truecolor half-block renderer for images on a real terminal. +//! +//! Kept dependency-free beyond the `image` crate (already resolved in the workspace via +//! deno_image/ws-web-runner, so this adds no new crate to review) rather than pulling in a terminal-image +//! crate: `viuer`, the obvious off-the-shelf choice, drags in `ansi_colours` (LGPL-3.0-or-later) as an +//! unconditional dependency for its RGB-to-256-color quantization path -- copyleft, and the first such +//! license this workspace would have had to accept. Emitting 24-bit truecolor SGR codes directly needs no +//! palette quantization at all, so there is nothing for that dependency to do here. +//! +//! Uses the "half-block" technique standard terminal-image tools (viu, chafa, catimg, viuer's own fallback) +//! use: each terminal row encodes two source-pixel rows via the upper-half-block character `\u{2580}`, whose +//! foreground and background colors are set independently, doubling the effective vertical resolution for +//! the same character-cell budget. + +use std::fmt::Write as _; +use std::io::Write as _; +use std::path::Path; + +/// Terminal columns the rendered thumbnail is resized to fit within. +const TARGET_COLUMNS: u32 = 48; + +/// Render a thumbnail of the image at `path` directly to stdout using ANSI truecolor half-block art. +/// +/// Returns the underlying `image` decode error on failure; the caller decides how to report it (this module +/// stays IO-boundary-agnostic rather than picking a logging mechanism itself). +#[expect( + clippy::single_call_fn, + reason = "distinct step of show_image_on_tty; kept separate for readability and testing" +)] +pub fn render(path: &Path) -> image::ImageResult<()> { + let source = image::open(path)?; + if source.width() == 0 || source.height() == 0 { + return Ok(()); + } + + // Each output row consumes two source-pixel rows (the half-block trick above), so the resize target's + // height is twice the column-derived row count; `max(2)` keeps a hairline-thin source image visible as + // at least one row instead of rounding away to nothing. The float math and truncating cast are both + // genuinely needed to turn a source aspect ratio into a terminal row count; TARGET_COLUMNS being a small + // constant keeps the result comfortably within u32 for any real image. + #[expect( + clippy::float_arithmetic, + reason = "aspect-ratio scaling to a row count needs float math" + )] + let aspect = f64::from(source.height()) / f64::from(source.width()); + #[expect( + clippy::as_conversions, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::float_arithmetic, + reason = "converting the aspect-scaled row count (always small and non-negative) back to u32" + )] + let rows = ((aspect * f64::from(TARGET_COLUMNS) / 2.0).round().max(1.0)) as u32; + let sample_height = rows.saturating_mul(2).max(2); + let resized = source + .resize_exact(TARGET_COLUMNS, sample_height, image::imageops::FilterType::Triangle) + .to_rgba8(); + + let mut out = String::new(); + for row in 0..rows { + let top_y = row.saturating_mul(2); + let bottom_y = top_y.saturating_add(1); + for col in 0..TARGET_COLUMNS { + let top = resized.get_pixel(col, top_y); + let bottom = resized.get_pixel(col, bottom_y); + // Writing to a String cannot fail. A named (not bare `_`) binding discards the Result without + // unwrap/expect or a bare `_`/`.ok()` (all denied by this workspace's lint set in different ways). + let _write_result = write!( + out, + "\x1b[38;2;{};{};{}m\x1b[48;2;{};{};{}m\u{2580}", + top[0], top[1], top[2], bottom[0], bottom[1], bottom[2] + ); + } + out.push_str("\x1b[0m\n"); + } + + // One write for the whole thumbnail (not one per escape code) keeps the render from interleaving with + // other stdout lines on a shared terminal. A failed stdout write only costs tty visibility, not + // correctness, so it's discarded here (named binding, not bare `_`/`.ok()`) rather than surfaced as + // this function's own error. + let _write_result = std::io::stdout().write_all(out.as_bytes()); + Ok(()) +} diff --git a/services/storage/tests/image_logging.rs b/services/storage/tests/image_logging.rs new file mode 100644 index 00000000..0accab5d --- /dev/null +++ b/services/storage/tests/image_logging.rs @@ -0,0 +1,29 @@ +//! Tests for `is_image_filename`, the extension check behind the storage service's tty log line for +//! stored images (see `put_file` in `src/routes.rs`). + +#![cfg(test)] + +use std::path::Path; + +use et_storage_service::routes::is_image_filename; + +#[test] +fn recognizes_common_image_extensions_case_insensitively() { + for ext in ["png", "jpg", "jpeg", "gif", "webp", "bmp", "PNG", "Jpg"] { + let filename = format!("capture.{ext}"); + assert!( + is_image_filename(Path::new(&filename)), + "{filename} should be recognized as an image" + ); + } +} + +#[test] +fn rejects_non_image_extensions_and_extensionless_names() { + for name in ["notes.txt", "data.json", "archive.tar.gz", "no-extension", ".hidden"] { + assert!( + !is_image_filename(Path::new(name)), + "{name} should not be recognized as an image" + ); + } +} diff --git a/services/storage/tests/put.rs b/services/storage/tests/put.rs index e4949502..8362e200 100644 --- a/services/storage/tests/put.rs +++ b/services/storage/tests/put.rs @@ -107,6 +107,34 @@ async fn writes_file_for_registered_agent() { assert_eq!(written, body); } +/// An image PUT under `cargo test` has no real terminal on stdout, so `viuer`'s decode-and-render step +/// (triggered by the `.png` extension) is expected to fail internally. The route must still store the file +/// and return 200 regardless -- tty display is a best-effort side effect, never a reason to fail the upload. +#[actix_rt::test] +async fn stores_an_image_and_returns_200_even_though_tty_rendering_cannot_succeed_in_tests() { + let tmp = tempfile::tempdir().unwrap(); + let config = storage_config(&tmp); + let registry = registry_with_agent("agent-1"); + let app = test::init_service( + App::new() + .app_data(web::Data::new(registry)) + .app_data(web::Data::new(config.clone())) + .configure(|cfg| configure::<()>(cfg, &config)), + ) + .await; + + let body = b"\x89PNG\r\n\x1a\nnot a real png, just image-extension-shaped bytes".as_ref(); + let req = test::TestRequest::put() + .uri("/storage/agent-1/capture.png") + .set_payload(body) + .to_request(); + let resp = test::call_service(&app, req).await; + + assert_eq!(resp.status(), StatusCode::OK); + let written = fs_err::read(tmp.path().join("agent-1").join("capture.png")).unwrap(); + assert_eq!(written, body); +} + #[actix_rt::test] async fn surfaces_io_failure_as_500() { // Point the storage root at a *file* (not a directory). The handler's diff --git a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js index a1278b43..3160b8b9 100644 --- a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js +++ b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js @@ -5,6 +5,7 @@ const PYODIDE_BASE_PATH = "/modules/pyodide/"; let pyodide = null; let pyMod = null; +let moduleVersion = null; function loadPyodideScript() { return new Promise((resolve, reject) => { @@ -67,6 +68,7 @@ export default async function init() { pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${wheelName}")`); }; const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); + moduleVersion = pkg.version; const ownWheel = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; await injectWheel(ownWheel); @@ -115,6 +117,10 @@ export async function run() { const el = document.getElementById("module-output"); if (el) el.value = (el.value ? el.value + "\n" : "") + msg; }; + // Diagnostic marker: this is the version fetched from package.json + injected as the wheel filename, so + // seeing the right number here after a version bump proves the module reloaded fresh (no stale-cache reuse + // of the package.json/wheel fetches below, neither of which is cache-busted). + log(`pydata1 version: ${moduleVersion}`); // The Python side runs `Client(base_url=...)` against this origin and // does PUT/GET itself via the generated client + pyodide-http patch. @@ -125,6 +131,7 @@ export async function run() { pyodide.toPy(sleep), pyodide.toPy(log), pyodide.toPy(() => {}), + pyodide.toPy(() => document.getElementById("upload-consent")?.checked ?? false), ); } catch (err) { log(`pydata1 run failed: ${String(err)}`); diff --git a/services/ws-modules/pydata1/pkg/package.json b/services/ws-modules/pydata1/pkg/package.json index e1543682..5fe123dc 100644 --- a/services/ws-modules/pydata1/pkg/package.json +++ b/services/ws-modules/pydata1/pkg/package.json @@ -8,5 +8,5 @@ "main": "et_ws_pydata1.js", "name": "et-ws-pydata1", "type": "module", - "version": "0.1.0" + "version": "0.1.2" } diff --git a/services/ws-modules/pydata1/pydata1/__init__.py b/services/ws-modules/pydata1/pydata1/__init__.py index 8691c650..9c2e3699 100644 --- a/services/ws-modules/pydata1/pydata1/__init__.py +++ b/services/ws-modules/pydata1/pydata1/__init__.py @@ -8,7 +8,7 @@ from et_rest_client.types import File -async def run(agent_id, base_url, sleep_ms, log, set_status) -> None: +async def run(agent_id, base_url, sleep_ms, log, set_status, upload_consent) -> None: """Execute the data1 workflow: store, fetch, verify.""" # httpx ships its own transport stack; in Pyodide that stack has no # network access. `pyodide_http.patch_all()` swaps httpx's transports @@ -48,6 +48,29 @@ async def run(agent_id, base_url, sleep_ms, log, set_status) -> None: set_status(msg) raise RuntimeError("Data mismatch") + # Diagnostic: exercises the exact same "read the page's upload-consent checkbox, then gate a storage + # PUT on it" mechanism pyeye1 uses, but fully within pydata1's headless, camera-free, fully-observable + # workflow -- so the checkbox/upload plumbing itself can be verified without needing a real camera or a + # remote device to drive it. + consent = upload_consent() + log(f"pydata1: upload_consent() returned {consent}") + if consent: + consent_filename = "consent_test.txt" + consent_content = f"consent test at {datetime.now(timezone.utc).isoformat()}" + try: + async with Client(base_url=base_url) as c: + await put_file.asyncio_detailed( + agent_id, + consent_filename, + client=c, + body=File(payload=consent_content.encode("utf-8")), + ) + log(f"pydata1: consent-gated upload succeeded: {consent_filename}") + except Exception as exc: + log(f"pydata1: consent-gated upload failed: {exc}") + else: + log("pydata1: upload consent not granted, skipping consent-gated upload") + await sleep_ms(2000) log("pydata1: workflow complete") set_status("pydata1: workflow complete") diff --git a/services/ws-modules/pydata1/pyproject.toml b/services/ws-modules/pydata1/pyproject.toml index 92c806fb..2bad0a68 100644 --- a/services/ws-modules/pydata1/pyproject.toml +++ b/services/ws-modules/pydata1/pyproject.toml @@ -4,7 +4,7 @@ description = "Python data 1" license = "Apache-2.0 OR MIT" name = "et-ws-pydata1" requires-python = ">=3.10" -version = "0.1.0" +version = "0.1.2" [build-system] build-backend = "uv_build" diff --git a/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js index a4904a6d..5f98eacb 100644 --- a/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js +++ b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js @@ -35,14 +35,17 @@ export default async function init() { const micropip = pyodide.pyimport("micropip"); await micropip.install("pydantic"); - // Install pyeye1's own wheel from pkg/ next to this shim. + // Install pyeye1's own wheel from pkg/ next to this shim. `no-cache` revalidates against the server so a + // rebuilt wheel is picked up immediately -- a stale cached wheel paired with a fresh shim (or vice versa) + // silently breaks the render/analysis payload contract between the two. const installLocalWheel = async (path) => { - const bytes = new Uint8Array(await fetch(new URL(path, import.meta.url)).then((r) => r.arrayBuffer())); + const wheelResp = await fetch(new URL(path, import.meta.url), { cache: "no-cache" }); + const bytes = new Uint8Array(await wheelResp.arrayBuffer()); pyodide.FS.writeFile(`/tmp/${path}`, bytes); pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${path}")`); }; - const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); + const pkg = await fetch(new URL("package.json", import.meta.url), { cache: "no-cache" }).then((r) => r.json()); await installLocalWheel(`${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`); // et-ws is its own ws-module mounted at /modules/et-ws/; delegate its wheel install to its shim. const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); @@ -117,6 +120,8 @@ function platformFor(state) { set_status: setStatus, should_stop: () => runtime !== state, cleanup: () => cleanup(state), + upload_consent: () => document.getElementById("upload-consent")?.checked ?? false, + save_eye_capture: () => saveEyeCapture(state), }; } @@ -177,8 +182,7 @@ function render(resultsJson) { const color = EYE_COLORS[eye.label] ?? "#fffdfa"; ctx.strokeStyle = color; ctx.strokeRect(left, top, Math.max(right - left, 1), Math.max(bottom - top, 1)); - ctx.fillStyle = color; - ctx.fillText(eye.label === "left_eye" ? "L" : "R", left, Math.max(top - 4, 12)); + drawOutlinedText(ctx, eye.label === "left_eye" ? "L" : "R", left, Math.max(top - 4, 12), color, 3 / scale); } ctx.lineWidth = 2 / scale; @@ -195,6 +199,26 @@ function render(resultsJson) { renderAnalysis(ctx, payload.analysis); } +// Encodes whatever the overlay canvas currently shows (the cropped eye band plus its box/iris/verdict +// overlay) as a PNG and uploads it to the connected agent's storage bucket. Python decides *whether* and +// *when* to call this (gated on the upload-consent checkbox and fired at most once per run); this primitive +// only performs the one browser-side operation of turning the current frame into stored bytes. +async function saveEyeCapture(state) { + const canvas = element("video-output-canvas", HTMLCanvasElement); + const blob = await new Promise((resolve, reject) => { + canvas.toBlob( + (result) => (result ? resolve(result) : reject(new Error("canvas.toBlob returned null"))), + "image/png", + ); + }); + const bytes = new Uint8Array(await blob.arrayBuffer()); + const agentId = state.client.get_agent_id(); + const filename = `pyeye1-eye-capture-${Date.now()}.png`; + const resp = await fetch(`/storage/${agentId}/${filename}`, { method: "PUT", body: bytes }); + if (!resp.ok) throw new Error(`eye capture upload failed: ${resp.status} ${resp.statusText}`); + log(`eye capture saved to storage: ${filename} (${bytes.length} bytes)`); +} + // Screening-verdict overlay, top-left. Python sends analysis=null until the first window completes, and each // screening reports status "insufficient_data" until it has enough non-blink samples to rate. function renderAnalysis(ctx, analysis) { @@ -208,11 +232,24 @@ function renderAnalysis(ctx, analysis) { } if (lines.length > 0) lines.push("screening demo -- not a medical diagnosis"); lines.forEach((line, index) => { - ctx.fillStyle = line.includes("DETECTED") ? "#ffb84d" : "#d7e0e8"; - ctx.fillText(line, 8, 20 + index * 20); + const color = line.includes("DETECTED") ? "#1e90ff" : "#d7e0e8"; + drawOutlinedText(ctx, line, 8, 20 + index * 20, color); }); } +// A single fill color can't stay legible against a live video feed: skin tone, lighting, and background all +// vary per frame, and no one color reads well everywhere (the original amber DETECTED text all but vanished +// against warm skin). A dark outline behind the fill guarantees contrast regardless of what's underneath, +// the same trick broadcast captions and video overlays use. +function drawOutlinedText(ctx, text, x, y, fillColor, lineWidth = 3) { + ctx.lineJoin = "round"; + ctx.lineWidth = lineWidth; + ctx.strokeStyle = "rgba(0, 0, 0, 0.85)"; + ctx.strokeText(text, x, y); + ctx.fillStyle = fillColor; + ctx.fillText(text, x, y); +} + function cleanup(state) { if (runtime === state) runtime = null; state?.landmarker?.close?.(); diff --git a/services/ws-modules/pyeye1/pyeye1/eye_detection.py b/services/ws-modules/pyeye1/pyeye1/eye_detection.py index 9583692e..ed258b2d 100644 --- a/services/ws-modules/pyeye1/pyeye1/eye_detection.py +++ b/services/ws-modules/pyeye1/pyeye1/eye_detection.py @@ -55,6 +55,9 @@ ANALYSIS_WINDOW_MS = 2500 MAX_SAMPLES = 600 MAX_RUNTIME_MS = 30_000 +# Independent of the detection-triggered capture below: a fixed-cadence "heartbeat" capture so at least one +# image gets stored periodically even across a long session where the screening indicators never fire. +PERIODIC_CAPTURE_INTERVAL_MS = 5_000 # Setup polling. Python owns all sequencing and timeouts; the JS primitives never loop or wait on their own. POLL_INTERVAL_MS = 100 @@ -101,11 +104,14 @@ async def run(platform) -> None: `platform` supplies the primitives Python cannot implement itself, each a single browser operation with no sequencing, polling, or timeout logic. Sync members: `ws_state()`, `agent_id()`, `send_event(json)`, - `video_size() -> [w, h]`, `render(json)`, `log(str)`, `set_status(str)`, `should_stop()`, `cleanup()`. - Async members: `connect_ws()`, `start_camera()`, `play_video()`, `sleep(ms)`, - `load_landmarker(model_path, bundle_path, wasm_path)`, and `infer()`, which returns one FaceLandmarker - pass as the JSON string `{"faces": [[x0, y0, x1, y1, ...], ...], "width": W, "height": H}` where each - face is the flat list of normalized landmark coordinates. + `video_size() -> [w, h]`, `render(json)`, `log(str)`, `set_status(str)`, `should_stop()`, `cleanup()`, + `upload_consent() -> bool` (the page's data-upload checkbox). Async members: `connect_ws()`, + `start_camera()`, `play_video()`, `sleep(ms)`, `load_landmarker(model_path, bundle_path, wasm_path)`, + `save_eye_capture()` (encodes the current overlay canvas as a PNG and uploads it to the connected + agent's storage bucket), and + `infer()`, which returns one FaceLandmarker pass as the JSON string + `{"faces": [[x0, y0, x1, y1, ...], ...], "width": W, "height": H}` where each face is the flat list of + normalized landmark coordinates. """ platform.set_status(starting_status()) platform.log(model_log_message()) @@ -152,6 +158,21 @@ async def wait_until(platform, predicate, timeout_ms: float, failure: str) -> No waited_ms += POLL_INTERVAL_MS +async def attempt_eye_capture(platform) -> None: + """Try one `save_eye_capture()`, reporting a failure both locally and server-side without raising. + + Shared by the two independent capture triggers in `sample_loop` (a detection's rising edge, and the + fixed-cadence periodic heartbeat) so a failed upload from either one is handled identically: logged + locally, reported as a server-visible event, and never allowed to abort the sample loop or be misreported + as an inference error. + """ + try: + await platform.save_eye_capture() + except Exception as exc: + platform.log(f"eye capture failed: {exc}") + platform.send_event(eye_capture_error_event_json(str(exc))) + + async def sample_loop(platform) -> None: """Sample frames continuously; re-analyze the gaze window (status + event) per `ANALYSIS_INTERVAL_MS`.""" sample_count = 0 @@ -161,6 +182,9 @@ async def sample_loop(platform) -> None: analysis: WindowAnalysis | None = None crop: Box | None = None last_analysis_ms = 0.0 + last_periodic_capture_ms = 0.0 + indicator_was_active = False + captured_for_episode = False while not platform.should_stop(): loop_started = time.monotonic() @@ -183,11 +207,38 @@ async def sample_loop(platform) -> None: while history and (now_s - history[0]["t"]) * 1000.0 > ANALYSIS_WINDOW_MS: history.popleft() - if now_s * 1000.0 - last_analysis_ms >= ANALYSIS_INTERVAL_MS: + is_analysis_tick = now_s * 1000.0 - last_analysis_ms >= ANALYSIS_INTERVAL_MS + if is_analysis_tick: last_analysis_ms = now_s * 1000.0 analysis = analyze_window(list(history)) platform.set_status(status_text(results, analysis)) platform.send_event(client_event_json(event_payload(results, analysis, width, height))) + + # Save one eye capture per detection -- "detection" means a screening indicator (eye + # misalignment or rhythmic oscillation) newly firing, not merely a face/eyes being visible. + # Edge-triggered on the indicator's own rising edge (not-detected -> detected), tracked + # independently of consent: a new episode resets `captured_for_episode` regardless of + # whether consent is granted yet, so if consent arrives partway through an already-active + # episode, that episode still gets its one capture rather than the edge having been silently + # consumed earlier while consent was still off. + indicator_active = analysis["misalignment"]["detected"] or analysis["oscillation"]["detected"] + if indicator_active and not indicator_was_active: + captured_for_episode = False + if indicator_active and not captured_for_episode and platform.upload_consent(): + captured_for_episode = True + await attempt_eye_capture(platform) + indicator_was_active = indicator_active + + # Independent, fixed-cadence capture on top of the detection-triggered one above: fires every + # PERIODIC_CAPTURE_INTERVAL_MS regardless of whether a screening indicator has ever activated, so + # a long quiet session still gets a periodic image, not only ever the first detection. The + # interval tracks wall-clock time unconditionally (like `last_analysis_ms` above) so it stays on + # schedule through stretches with no consent; only the capture attempt itself is gated on consent. + is_periodic_capture_tick = now_s * 1000.0 - last_periodic_capture_ms >= PERIODIC_CAPTURE_INTERVAL_MS + if is_periodic_capture_tick: + last_periodic_capture_ms = now_s * 1000.0 + if platform.upload_consent(): + await attempt_eye_capture(platform) platform.render(results_json(results, analysis, crop)) except Exception as exc: message = f"pyeye1 eye movement screening: inference error\n{exc}" @@ -329,6 +380,21 @@ def client_event_json(details: dict[str, object]) -> str: ).model_dump_json() +def eye_capture_error_event_json(error: str) -> str: + """Build the et-client-event JSON envelope for a failed eye-capture upload. + + `platform.log(...)` alone only reaches the browser's own on-page log, invisible to anyone watching the + server tty; this event puts the failure where it can actually be seen server-side, same as a successful + capture already is (via the storage service's own "stored image" log line). + """ + return WsClientEvent( + type="et-client-event", + capability="pyeye1", + action="eye_capture_failed", + details={"error": error}, + ).model_dump_json() + + def status_text(results: Sequence[FaceEyes], analysis: WindowAnalysis | None) -> str: """Render the browser status text used by the eye movement screening demo.""" eye_count = sum(len(result["eyes"]) for result in results) diff --git a/services/ws-modules/pyeye1/pyproject.toml b/services/ws-modules/pyeye1/pyproject.toml index 9575e0e0..729ee3c1 100644 --- a/services/ws-modules/pyeye1/pyproject.toml +++ b/services/ws-modules/pyeye1/pyproject.toml @@ -4,7 +4,7 @@ description = "Python eye screening (MediaPipe)" license = "Apache-2.0 OR MIT" name = "et-ws-pyeye1" requires-python = ">=3.10" -version = "0.1.0" +version = "0.1.1" [dependency-groups] dev = ["pytest", "pytest-cov"] diff --git a/services/ws-modules/pyeye1/tests/test_run_workflow.py b/services/ws-modules/pyeye1/tests/test_run_workflow.py index 2cf054f2..79ee2b9e 100644 --- a/services/ws-modules/pyeye1/tests/test_run_workflow.py +++ b/services/ws-modules/pyeye1/tests/test_run_workflow.py @@ -22,6 +22,33 @@ ) +def fake_analysis(*, misalignment: bool = False, oscillation: bool = False) -> dict: + """Build a minimal WindowAnalysis-shaped dict with a controlled misalignment/oscillation verdict. + + Used to drive the capture-triggering logic directly (rising edge of a screening indicator) without + needing real multi-second landmark sequences to organically produce a detection through the actual + gaze-analysis math, which is already covered by test_gaze_analysis.py. + """ + return { + "window_ms": 1000.0, + "samples": 20, + "valid_samples": 20, + "misalignment": { + "status": "ok", + "detected": misalignment, + "horizontal_deviation": 0.2 if misalignment else 0.0, + "vertical_deviation": 0.0, + }, + "oscillation": { + "status": "ok", + "detected": oscillation, + "axis": "horizontal" if oscillation else None, + "frequency_hz": 4.0 if oscillation else None, + "amplitude": 0.1 if oscillation else None, + }, + } + + def synthetic_face() -> list[float]: """Build one flat 478*2 landmark list with open eyes and centered irises.""" values = [0.0] * (MESH_LANDMARK_COUNT * 2) @@ -56,6 +83,8 @@ def __init__(self, stop_after: int = 5) -> None: self.landmarker_args: tuple[str, str, str] | None = None self.played = False self.cleaned = False + self.consent = False + self.capture_calls = 0 async def connect_ws(self) -> None: pass @@ -103,6 +132,12 @@ def should_stop(self) -> bool: def cleanup(self) -> None: self.cleaned = True + def upload_consent(self) -> bool: + return self.consent + + async def save_eye_capture(self) -> None: + self.capture_calls += 1 + class NeverConnectingPlatform(FakePlatform): def ws_state(self) -> str: @@ -114,6 +149,19 @@ async def start_camera(self) -> None: raise RuntimeError("Permission denied") +class MidSessionConsentPlatform(FakePlatform): + """Consent flips on only once a few samples have already run, simulating an opt-in mid-session.""" + + def upload_consent(self) -> bool: + return self.infer_calls >= 3 + + +class FailingCapturePlatform(FakePlatform): + async def save_eye_capture(self) -> None: + self.capture_calls += 1 + raise RuntimeError("upload failed") + + class RunWorkflowTests(unittest.IsolatedAsyncioTestCase): async def test_happy_path_drives_the_full_workflow(self) -> None: platform = FakePlatform(stop_after=5) @@ -160,6 +208,134 @@ async def test_camera_failure_propagates_and_still_cleans_up(self) -> None: self.assertTrue(platform.cleaned) self.assertEqual(platform.infer_calls, 0) + async def test_no_capture_while_eyes_are_visible_but_no_indicator_has_fired(self) -> None: + # Regression guard: a face/eyes being visible must never be enough on its own -- only an actual + # screening indicator (misalignment or oscillation) firing counts as a "detection". + platform = FakePlatform(stop_after=5) + platform.consent = True + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 0.0), + patch.object(eye_detection, "analyze_window", return_value=fake_analysis()), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 0) + + async def test_eye_capture_fires_once_on_the_indicators_rising_edge(self) -> None: + platform = FakePlatform(stop_after=5) + platform.consent = True + # not-detected, detected (rising edge -> capture #1), still detected (no repeat), cleared, + # detected again (a new rising edge -> capture #2). + analyses = [ + fake_analysis(), + fake_analysis(misalignment=True), + fake_analysis(misalignment=True), + fake_analysis(), + fake_analysis(oscillation=True), + ] + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 0.0), + patch.object(eye_detection, "analyze_window", side_effect=analyses), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 2) + + async def test_eye_capture_skipped_entirely_without_consent(self) -> None: + platform = FakePlatform(stop_after=5) + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 0.0), + patch.object(eye_detection, "analyze_window", return_value=fake_analysis(misalignment=True)), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 0) + self.assertFalse(any("eye capture" in line for line in platform.logs)) + + async def test_eye_capture_fires_on_first_rising_edge_after_consent_granted_mid_session(self) -> None: + platform = MidSessionConsentPlatform(stop_after=5) + # Indicator is active from the very first sample; consent only turns on at infer_calls >= 3, so the + # capture fires on sample 3 (the first tick where both are true) and not again for samples 4-5, since + # the indicator never drops back down to re-trigger a new rising edge. + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 0.0), + patch.object(eye_detection, "analyze_window", return_value=fake_analysis(misalignment=True)), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 1) + + async def test_eye_capture_failure_is_logged_and_does_not_abort_the_run(self) -> None: + platform = FailingCapturePlatform(stop_after=5) + platform.consent = True + # not-detected, detected (edge #1 -> attempt+fail), not-detected (clears), detected (edge #2 -> + # attempt+fail), not-detected. Two rising edges -> two failed attempts, each independently retried + # rather than the first failure suppressing the second episode's attempt. + analyses = [ + fake_analysis(), + fake_analysis(misalignment=True), + fake_analysis(), + fake_analysis(misalignment=True), + fake_analysis(), + ] + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 0.0), + patch.object(eye_detection, "analyze_window", side_effect=analyses), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 2) + failure_logs = [line for line in platform.logs if "eye capture failed" in line and "upload failed" in line] + self.assertEqual(len(failure_logs), 2) + self.assertEqual(platform.statuses[-1], stopped_status()) + + async def test_periodic_capture_fires_every_sample_when_its_interval_is_forced_to_zero(self) -> None: + # Detection interval is left huge so it can never tick, isolating the periodic mechanism: every + # sample is a periodic-capture tick, and no screening indicator ever fires. + platform = FakePlatform(stop_after=5) + platform.consent = True + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 999_999_999.0), + patch.object(eye_detection, "PERIODIC_CAPTURE_INTERVAL_MS", 0.0), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 5) + + async def test_periodic_capture_is_also_gated_on_consent(self) -> None: + platform = FakePlatform(stop_after=5) + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 999_999_999.0), + patch.object(eye_detection, "PERIODIC_CAPTURE_INTERVAL_MS", 0.0), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 0) + + async def test_periodic_and_detection_triggered_captures_are_additive(self) -> None: + # The indicator fires once (rising edge on the first sample, then stays active) while the periodic + # interval is forced to zero: the detection edge contributes one capture, the periodic heartbeat + # contributes one per sample, and the two mechanisms don't suppress each other. + platform = FakePlatform(stop_after=5) + platform.consent = True + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 0.0), + patch.object(eye_detection, "PERIODIC_CAPTURE_INTERVAL_MS", 0.0), + patch.object(eye_detection, "analyze_window", return_value=fake_analysis(misalignment=True)), + ): + await run(platform) + self.assertEqual(platform.capture_calls, 6) + + async def test_eye_capture_failure_is_also_reported_as_a_server_visible_event(self) -> None: + # platform.log() alone only reaches the browser's own log box; a failure must also be sent as a + # client-event so it's visible server-side without anyone having to watch the browser. + platform = FailingCapturePlatform(stop_after=5) + platform.consent = True + with ( + patch.object(eye_detection, "ANALYSIS_INTERVAL_MS", 0.0), + patch.object(eye_detection, "analyze_window", return_value=fake_analysis(oscillation=True)), + ): + await run(platform) + + failure_events = [json.loads(event) for event in platform.events if "eye_capture_failed" in event] + self.assertEqual(len(failure_events), 1) + self.assertEqual(failure_events[0]["capability"], "pyeye1") + self.assertEqual(failure_events[0]["action"], "eye_capture_failed") + self.assertEqual(failure_events[0]["details"]["error"], "upload failed") + if __name__ == "__main__": unittest.main() diff --git a/services/ws-server/static/app.js b/services/ws-server/static/app.js index f6f06d5f..2833c5e4 100644 --- a/services/ws-server/static/app.js +++ b/services/ws-server/static/app.js @@ -1,6 +1,12 @@ import init, { initTracing, WsClient, WsClientConfig } from "/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"; -console.log("app.js: module loading started"); +// Bump this string on every meaningful app.js edit. index.html loads this file via a plain, non-cache-busted +//