From 9c9222953aa713014e15c5f7f657d56402e146c4 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:02:52 +0200 Subject: [PATCH 01/46] =?UTF-8?q?feat(stage-a-io):=20=E2=9C=A8=20add=20sha?= =?UTF-8?q?red=20Stage-A=20Teensy=20I/O=20library?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PDA1 wire protocol (fragmentation-tolerant parser with CRC resync), v1 ASCII command grammar, typed serial client with idempotent sequence retries and stream-integrity accounting, bounded background I/O worker, .pdq raw-frame writer, JSON run sidecar, calibrated optical log-contrast estimator (clipping/dark-headroom guarded), and a mock controller for hardware-free tests. Wire-compatible with stage-a-controller include/wire_protocol.h. --- Cargo.toml | 1 + stage-a-io/Cargo.toml | 18 ++ stage-a-io/src/client.rs | 329 +++++++++++++++++++++++++++ stage-a-io/src/estimator.rs | 238 ++++++++++++++++++++ stage-a-io/src/lib.rs | 43 ++++ stage-a-io/src/mock.rs | 283 +++++++++++++++++++++++ stage-a-io/src/pdq.rs | 150 ++++++++++++ stage-a-io/src/protocol.rs | 226 +++++++++++++++++++ stage-a-io/src/sidecar.rs | 216 ++++++++++++++++++ stage-a-io/src/transport.rs | 113 ++++++++++ stage-a-io/src/wire.rs | 438 ++++++++++++++++++++++++++++++++++++ stage-a-io/src/worker.rs | 229 +++++++++++++++++++ 12 files changed, 2284 insertions(+) create mode 100644 stage-a-io/Cargo.toml create mode 100644 stage-a-io/src/client.rs create mode 100644 stage-a-io/src/estimator.rs create mode 100644 stage-a-io/src/lib.rs create mode 100644 stage-a-io/src/mock.rs create mode 100644 stage-a-io/src/pdq.rs create mode 100644 stage-a-io/src/protocol.rs create mode 100644 stage-a-io/src/sidecar.rs create mode 100644 stage-a-io/src/transport.rs create mode 100644 stage-a-io/src/wire.rs create mode 100644 stage-a-io/src/worker.rs diff --git a/Cargo.toml b/Cargo.toml index 306f834..98ff631 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "stage-a-io", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/stage-a-io/Cargo.toml b/stage-a-io/Cargo.toml new file mode 100644 index 0000000..7a8d9fa --- /dev/null +++ b/stage-a-io/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "stage-a-io" +description = "Shared Stage-A Teensy I/O: PDA1 wire protocol, serial client, PDQ writer, run sidecars, and the calibrated optical-contrast estimator" +edition.workspace = true +license.workspace = true +version.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +serialport = { version = "4", optional = true } + +[features] +default = ["hardware"] +# Real serial-port transport. Disable for pure-analysis / CI builds. +hardware = ["dep:serialport"] diff --git a/stage-a-io/src/client.rs b/stage-a-io/src/client.rs new file mode 100644 index 0000000..c664be0 --- /dev/null +++ b/stage-a-io/src/client.rs @@ -0,0 +1,329 @@ +//! Typed request/response client over a [`Transport`]. +//! +//! Sends `@ VERB …` commands and demultiplexes the PDA1 frame stream +//! into (a) the matching control reply, (b) async control notices, and +//! (c) data frames (samples / summaries / markers). On a reply timeout the +//! **identical** line (same `seq`) is resent; firmware caches recent replies, +//! so retries are idempotent by construction. + +use std::collections::BTreeMap; +use std::io; +use std::time::{Duration, Instant}; + +use crate::protocol::{Command, ControlMessage, ProtocolError}; +use crate::transport::Transport; +use crate::wire::{Frame, FrameParser, FrameType, ParseEvent}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct StreamIntegrity { + pub skipped_bytes: u64, + pub crc_failures: u64, + pub sequence_gaps: u64, + pub dropped_samples: u64, +} + +impl StreamIntegrity { + /// A run is valid only while the stream shows zero corruption. + pub fn is_clean(&self) -> bool { + self.skipped_bytes == 0 + && self.crc_failures == 0 + && self.sequence_gaps == 0 + && self.dropped_samples == 0 + } +} + +#[derive(Debug)] +pub enum ClientError { + Io(io::Error), + Protocol(ProtocolError), + /// The device replied `-seq ERR …`. + Device { + code: String, + detail: String, + }, + /// No matching reply within the timeout across all retries. + Timeout { + verb: String, + retries: u32, + }, +} + +impl std::fmt::Display for ClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(err) => write!(f, "transport I/O failed: {err}"), + Self::Protocol(err) => write!(f, "protocol violation: {err}"), + Self::Device { code, detail } => { + write!(f, "device rejected command: code={code} detail={detail}") + } + Self::Timeout { verb, retries } => { + write!(f, "no reply to {verb} after {retries} retries") + } + } + } +} + +impl std::error::Error for ClientError {} + +impl From for ClientError { + fn from(err: io::Error) -> Self { + Self::Io(err) + } +} + +impl From for ClientError { + fn from(err: ProtocolError) -> Self { + Self::Protocol(err) + } +} + +/// Non-reply traffic observed while waiting for or between replies. +#[derive(Debug, Clone, PartialEq)] +pub enum DeviceEvent { + Data(Frame), + Async { + name: String, + fields: BTreeMap, + }, +} + +pub struct StageAClient { + transport: T, + parser: FrameParser, + next_sequence: u32, + last_frame_sequence: Option, + integrity: StreamIntegrity, + pending_events: Vec, + reply_timeout: Duration, + max_retries: u32, + read_buf: Vec, +} + +impl StageAClient { + pub fn new(transport: T) -> Self { + Self { + transport, + parser: FrameParser::default(), + next_sequence: 1, + last_frame_sequence: None, + integrity: StreamIntegrity::default(), + pending_events: Vec::new(), + reply_timeout: Duration::from_millis(500), + max_retries: 2, + read_buf: vec![0_u8; 16 * 1024], + } + } + + pub fn with_reply_timeout(mut self, timeout: Duration) -> Self { + self.reply_timeout = timeout; + self + } + + pub fn integrity(&self) -> StreamIntegrity { + self.integrity + } + + /// Sends a command and waits for its `+seq OK` reply, retrying the + /// identical line on timeout. Data/async frames arriving in between are + /// queued for [`StageAClient::poll_events`]. + pub fn request(&mut self, command: &Command) -> Result, ClientError> { + let sequence = self.next_sequence; + self.next_sequence = self.next_sequence.wrapping_add(1); + let line = command.encode(sequence)?; + + for _attempt in 0..=self.max_retries { + self.transport.write_all(&line)?; + let deadline = Instant::now() + self.reply_timeout; + while Instant::now() < deadline { + self.pump()?; + if let Some(reply) = self.take_reply(sequence)? { + return Ok(reply); + } + std::thread::sleep(Duration::from_millis(1)); + } + } + Err(ClientError::Timeout { + verb: command.verb.clone(), + retries: self.max_retries, + }) + } + + /// Drains any pending non-reply device traffic (data frames, async + /// notices) without blocking. + pub fn poll_events(&mut self) -> Result, ClientError> { + self.pump()?; + Ok(std::mem::take(&mut self.pending_events)) + } + + fn pump(&mut self) -> Result<(), ClientError> { + let n = self.transport.read(&mut self.read_buf)?; + if n > 0 { + self.parser.extend(&self.read_buf[..n]); + } + while let Some(event) = self.parser.next_event() { + match event { + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + self.integrity.skipped_bytes += skipped_bytes as u64; + self.integrity.crc_failures += crc_failures as u64; + } + ParseEvent::Frame(frame) => self.accept_frame(frame), + } + } + Ok(()) + } + + fn accept_frame(&mut self, frame: Frame) { + if let Some(last) = self.last_frame_sequence { + let expected = last.wrapping_add(1); + if frame.header.sequence != expected { + self.integrity.sequence_gaps += 1; + } + } + self.last_frame_sequence = Some(frame.header.sequence); + if frame.header.dropped_samples > 0 { + self.integrity.dropped_samples = u64::from(frame.header.dropped_samples); + } + + match frame.header.frame_type { + FrameType::Control => { + // Control payloads are handled by take_reply / async queue; + // keep the raw frame so replies can be matched later. + self.pending_events.push(DeviceEvent::Data(frame)); + } + _ => self.pending_events.push(DeviceEvent::Data(frame)), + } + } + + fn take_reply( + &mut self, + sequence: u32, + ) -> Result>, ClientError> { + let mut result = None; + let mut remaining = Vec::with_capacity(self.pending_events.len()); + for event in std::mem::take(&mut self.pending_events) { + if result.is_some() { + remaining.push(event); + continue; + } + let DeviceEvent::Data(frame) = &event else { + remaining.push(event); + continue; + }; + let Some(text) = frame.control_text() else { + remaining.push(event); + continue; + }; + match ControlMessage::parse(text) { + Ok(ControlMessage::Ok { + sequence: reply_seq, + fields, + }) if reply_seq == sequence => { + result = Some(Ok(fields)); + } + Ok(ControlMessage::Err { + sequence: reply_seq, + code, + detail, + }) if reply_seq == sequence => { + result = Some(Err(ClientError::Device { code, detail })); + } + Ok(ControlMessage::Async { name, fields }) => { + remaining.push(DeviceEvent::Async { name, fields }); + } + // Stale replies to earlier (retried) sequences are dropped; + // malformed control payloads count as corruption. + Ok(_) => {} + Err(_) => { + self.integrity.crc_failures += 0; // parse failure, not CRC + self.integrity.skipped_bytes += frame.payload.len() as u64; + } + } + } + self.pending_events = remaining; + match result { + Some(Ok(fields)) => Ok(Some(fields)), + Some(Err(err)) => Err(err), + None => Ok(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::MockController; + use crate::transport::MockLink; + + #[test] + fn request_reply_round_trip_with_hello() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + let handle = std::thread::spawn(move || controller.serve_n_commands(1)); + let reply = client + .request(&Command::new("HELLO").field("protocol", 1)) + .expect("HELLO replies"); + handle.join().expect("mock thread joins"); + + assert_eq!(reply.get("protocol").map(String::as_str), Some("1")); + assert!(client.integrity().is_clean()); + } + + #[test] + fn timeout_retries_are_idempotent_via_reply_cache() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + controller.drop_first_reply(); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(50)); + + // The controller swallows the first reply; the client must resend the + // identical sequence and accept the cached second reply. The mock + // panics if a retried sequence re-executes the operation. + let handle = std::thread::spawn(move || controller.serve_n_commands(2)); + let reply = client + .request(&Command::new("STATUS")) + .expect("retried STATUS succeeds"); + handle.join().expect("mock thread joins"); + + assert_eq!(reply.get("state").map(String::as_str), Some("SAFE_IDLE")); + assert_eq!(reply.get("executions").map(String::as_str), Some("1")); + } + + #[test] + fn device_error_reply_surfaces_code_and_detail() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + let handle = std::thread::spawn(move || controller.serve_n_commands(1)); + let err = client + .request(&Command::new("CONFIG").field("mode", "A9")) + .expect_err("invalid mode is rejected"); + handle.join().expect("mock thread joins"); + + match err { + ClientError::Device { code, .. } => assert_eq!(code, "BAD_MODE"), + other => panic!("expected device error, got {other:?}"), + } + } + + #[test] + fn overrun_frames_invalidate_integrity() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + controller.emit_summary_with_drops(3); + client.poll_events().expect("poll"); + assert!(!client.integrity().is_clean()); + assert_eq!(client.integrity().dropped_samples, 3); + } +} diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs new file mode 100644 index 0000000..809786a --- /dev/null +++ b/stage-a-io/src/estimator.rs @@ -0,0 +1,238 @@ +//! Calibrated optical log-contrast estimator. +//! +//! `a = ln(I_max / I_min)` is defined by the *measured light*, never by the +//! commanded DAC excursion: the Pockels-cell V→T response is non-linear, so +//! the photodiode ADC trace is the only valid source of `a` +//! (knowledge base: `methodology/camera-calibration.md`, "define `a` from +//! the light, not the drive"). +//! +//! The estimator therefore: +//! - converts ADC codes to volts through a characterised affine calibration, +//! - subtracts the dark level (the detector is DC-coupled; `a` needs true +//! levels including DC), +//! - takes robust percentile extrema rather than raw min/max so single-code +//! noise spikes do not bias the contrast, +//! - refuses to produce a value at all when the window clips (top/bottom of +//! the ADC range) or has no headroom above dark — a wrong `a` is worse +//! than no `a`. + +use serde::{Deserialize, Serialize}; + +/// Affine ADC calibration plus dark level, all in physical units. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AdcCalibration { + /// Volts per ADC code (gain of the whole front end into the ADC). + pub volts_per_code: f64, + /// Voltage at code 0. + pub offset_volts: f64, + /// Dark level (light blocked), in volts after the affine map. + pub dark_volts: f64, + /// Full-scale code (4095 for the Teensy 12-bit ADC). + pub full_scale_code: u16, +} + +impl Default for AdcCalibration { + fn default() -> Self { + Self { + volts_per_code: 3.3 / 4_095.0, + offset_volts: 0.0, + dark_volts: 0.0, + full_scale_code: 4_095, + } + } +} + +impl AdcCalibration { + pub fn code_to_volts(&self, code: u16) -> f64 { + self.offset_volts + f64::from(code) * self.volts_per_code + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ContrastEstimate { + /// Peak-to-peak log-contrast `a = ln(V_max / V_min)` (dark-corrected). + pub a: f64, + pub v_min_volts: f64, + pub v_max_volts: f64, + /// Fraction of samples at or below code 0 + margin. + pub low_clip_fraction: f64, + /// Fraction of samples at or above full scale - margin. + pub high_clip_fraction: f64, + pub sample_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum EstimateError { + /// Fewer samples than the estimator can use robustly. + TooFewSamples { count: usize, minimum: usize }, + /// The window touches the ADC rails — `a` would be silently wrong. + Clipped { + low_fraction_permille: u32, + high_fraction_permille: u32, + }, + /// The dark-corrected minimum is not positive: no optical headroom. + NoHeadroomAboveDark, +} + +impl std::fmt::Display for EstimateError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooFewSamples { count, minimum } => { + write!(f, "only {count} samples (minimum {minimum})") + } + Self::Clipped { + low_fraction_permille, + high_fraction_permille, + } => write!( + f, + "ADC clipping: {low_fraction_permille}‰ low / {high_fraction_permille}‰ high" + ), + Self::NoHeadroomAboveDark => { + f.write_str("dark-corrected minimum is not positive; a is undefined") + } + } + } +} + +impl std::error::Error for EstimateError {} + +pub const MIN_SAMPLES: usize = 64; +/// Codes within this margin of the rails count as clipped. +pub const CLIP_MARGIN_CODES: u16 = 4; +/// Reject the window when more than 1‰ of samples clip. +pub const MAX_CLIP_FRACTION: f64 = 0.001; +/// Robust extrema: 1st / 99th percentile. +const LOW_PERCENTILE: f64 = 0.01; +const HIGH_PERCENTILE: f64 = 0.99; + +/// Estimates the optical log-contrast from one settled, phase-attributed +/// ADC window. The window must span at least a few full modulation cycles; +/// enforcing that is the caller's job (it knows the drive frequency). +pub fn estimate_contrast( + codes: &[u16], + calibration: &AdcCalibration, +) -> Result { + if codes.len() < MIN_SAMPLES { + return Err(EstimateError::TooFewSamples { + count: codes.len(), + minimum: MIN_SAMPLES, + }); + } + + let low_clip_threshold = CLIP_MARGIN_CODES; + let high_clip_threshold = calibration + .full_scale_code + .saturating_sub(CLIP_MARGIN_CODES); + let low_clipped = codes.iter().filter(|&&c| c <= low_clip_threshold).count(); + let high_clipped = codes.iter().filter(|&&c| c >= high_clip_threshold).count(); + let low_clip_fraction = low_clipped as f64 / codes.len() as f64; + let high_clip_fraction = high_clipped as f64 / codes.len() as f64; + if low_clip_fraction > MAX_CLIP_FRACTION || high_clip_fraction > MAX_CLIP_FRACTION { + return Err(EstimateError::Clipped { + low_fraction_permille: (low_clip_fraction * 1_000.0).round() as u32, + high_fraction_permille: (high_clip_fraction * 1_000.0).round() as u32, + }); + } + + let mut sorted = codes.to_vec(); + sorted.sort_unstable(); + let low_code = percentile(&sorted, LOW_PERCENTILE); + let high_code = percentile(&sorted, HIGH_PERCENTILE); + + let v_min = calibration.code_to_volts(low_code) - calibration.dark_volts; + let v_max = calibration.code_to_volts(high_code) - calibration.dark_volts; + if v_min <= 0.0 || v_max <= 0.0 { + return Err(EstimateError::NoHeadroomAboveDark); + } + + Ok(ContrastEstimate { + a: (v_max / v_min).ln(), + v_min_volts: v_min, + v_max_volts: v_max, + low_clip_fraction, + high_clip_fraction, + sample_count: codes.len(), + }) +} + +fn percentile(sorted: &[u16], q: f64) -> u16 { + let index = ((sorted.len() - 1) as f64 * q).round() as usize; + sorted[index.min(sorted.len() - 1)] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sine_codes(center: f64, amplitude: f64, n: usize) -> Vec { + (0..n) + .map(|i| { + let phase = 2.0 * std::f64::consts::PI * (i as f64) * 7.0 / n as f64; + (center + amplitude * phase.sin()) + .round() + .clamp(0.0, 4_095.0) as u16 + }) + .collect() + } + + #[test] + fn recovers_known_contrast_from_synthetic_sine() { + let calibration = AdcCalibration { + dark_volts: 40.0 * (3.3 / 4_095.0), + ..AdcCalibration::default() + }; + // center 2048, amplitude 900 -> dark-corrected V ratio: + let codes = sine_codes(2_048.0, 900.0, 4_096); + let estimate = estimate_contrast(&codes, &calibration).expect("clean window estimates"); + + let expected = ((2_048.0_f64 + 900.0 - 40.0) / (2_048.0 - 900.0 - 40.0)).ln(); + assert!( + (estimate.a - expected).abs() < 0.01, + "a={} expected~{expected}", + estimate.a + ); + assert!(estimate.low_clip_fraction == 0.0 && estimate.high_clip_fraction == 0.0); + } + + #[test] + fn rejects_clipped_windows() { + // Amplitude pushes past full scale -> clipping at the top rail. + let codes = sine_codes(3_500.0, 900.0, 2_048); + let err = estimate_contrast(&codes, &AdcCalibration::default()) + .expect_err("clipped window must be rejected"); + assert!(matches!(err, EstimateError::Clipped { .. })); + } + + #[test] + fn rejects_windows_without_dark_headroom() { + let calibration = AdcCalibration { + dark_volts: 1_300.0 * (3.3 / 4_095.0), + ..AdcCalibration::default() + }; + // Minimum (2048-900=1148) sits below the dark level (1300). + let codes = sine_codes(2_048.0, 900.0, 2_048); + let err = estimate_contrast(&codes, &calibration) + .expect_err("no headroom above dark must be rejected"); + assert_eq!(err, EstimateError::NoHeadroomAboveDark); + } + + #[test] + fn rejects_short_windows() { + let err = estimate_contrast(&[100; 10], &AdcCalibration::default()) + .expect_err("short window rejected"); + assert!(matches!(err, EstimateError::TooFewSamples { .. })); + } + + #[test] + fn single_sample_spikes_do_not_bias_the_contrast() { + let mut codes = sine_codes(2_048.0, 500.0, 4_096); + codes[7] = 4_000; // one hot spike, below the 1 - 99 percentile weight + let clean = estimate_contrast( + &sine_codes(2_048.0, 500.0, 4_096), + &AdcCalibration::default(), + ) + .expect("clean"); + let spiked = estimate_contrast(&codes, &AdcCalibration::default()).expect("spiked"); + assert!((clean.a - spiked.a).abs() < 0.005); + } +} diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs new file mode 100644 index 0000000..67c517f --- /dev/null +++ b/stage-a-io/src/lib.rs @@ -0,0 +1,43 @@ +//! # stage-a-io +//! +//! Shared research-owned I/O library for the Stage-A camera-calibration +//! plugins (`stage-a-monitor`, `stage-a-a1`, `stage-a-a2`, `stage-a-a3`). +//! +//! Scope, per the Stage-A control-software specification: +//! - the v1 ASCII command grammar and PDA1 binary frame format (wire- +//! compatible with `stage-a-controller/include/wire_protocol.h`), +//! - a typed serial client with idempotent sequence retries and stream- +//! integrity accounting (CRC failures, resync skips, sequence gaps, +//! ADC overruns — any of which invalidates a measurement point), +//! - a bounded background I/O worker so plugin `process_frame()` never +//! blocks on serial, +//! - the `.pdq` raw-frame writer and the JSON run sidecar, +//! - the calibrated optical log-contrast estimator (`a` is measured light, +//! never the commanded DAC excursion), +//! - a mock controller for tests and hardware-free development. +//! +//! This crate deliberately contains **no** experiment policy (sweeps, +//! bisection, fits live in the protocol plugins) and **no** augur types — +//! it is plain I/O + numerics, testable without a host. + +pub mod client; +pub mod estimator; +pub mod mock; +pub mod pdq; +pub mod protocol; +pub mod sidecar; +pub mod transport; +pub mod wire; + +pub use client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; +pub use estimator::{estimate_contrast, AdcCalibration, ContrastEstimate, EstimateError}; +pub use pdq::{PdqSummary, PdqWriter}; +pub use protocol::{Command, ControlMessage, ProtocolError}; +pub use sidecar::{DetectorLoad, IntegrityRecord, RunSidecar, TriggerSource}; +#[cfg(feature = "hardware")] +pub use transport::SerialTransport; +pub use transport::{MockLink, MockTransport, Transport}; +pub use wire::{Frame, FrameHeader, FrameParser, FrameType, ParseEvent, SummaryPayload}; +pub use worker::{IoWorker, WorkerOutput, WorkerRequest}; + +pub mod worker; diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs new file mode 100644 index 0000000..fd7dad0 --- /dev/null +++ b/stage-a-io/src/mock.rs @@ -0,0 +1,283 @@ +//! Mock Stage-A controller for tests and hardware-free plugin development. +//! +//! Implements the v1 command surface (`HELLO`, `STATUS`, `CONFIG`, `ARM`, +//! `RUN`, `START`, `STOP`, `PING`, `FAULT_CLEAR`) with the same idempotency +//! contract as the firmware: replies to recent sequences are cached and +//! resent without re-executing the operation. It can also synthesize +//! photodiode sample/summary frames (sinusoidal drive) so the estimator and +//! plugins can be exercised end to end without a Teensy. + +use std::collections::BTreeMap; + +use crate::protocol::ControlMessage; +use crate::transport::Transport; +use crate::wire::{Frame, FrameHeader, FrameType, SummaryPayload, PROTOCOL_VERSION}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MockState { + SafeIdle, + Configured, + Armed, + Running, +} + +impl MockState { + fn name(self) -> &'static str { + match self { + Self::SafeIdle => "SAFE_IDLE", + Self::Configured => "CONFIGURED", + Self::Armed => "ARMED", + Self::Running => "RUNNING", + } + } +} + +pub struct MockController { + transport: T, + state: MockState, + config: BTreeMap, + config_revision: u32, + reply_cache: Vec<(u32, String)>, + executed_sequences: Vec, + /// Commands executed (used to assert idempotency in tests). + executions: u32, + drop_next_reply: bool, + out_sequence: u32, + line_buffer: Vec, + sample_index: u64, + /// Synthetic optical waveform: codes = center + amplitude*sin(phase). + pub synth_center: f64, + pub synth_amplitude: f64, + pub synth_dark_code: f64, +} + +impl MockController { + pub fn new(transport: T) -> Self { + Self { + transport, + state: MockState::SafeIdle, + config: BTreeMap::new(), + config_revision: 0, + reply_cache: Vec::new(), + executed_sequences: Vec::new(), + executions: 0, + drop_next_reply: false, + out_sequence: 0, + line_buffer: Vec::new(), + sample_index: 0, + synth_center: 2_048.0, + synth_amplitude: 900.0, + synth_dark_code: 40.0, + } + } + + /// Swallow the next reply (simulates a lost USB packet) — the client + /// must retry with the identical sequence. + pub fn drop_first_reply(&mut self) { + self.drop_next_reply = true; + } + + pub fn state(&self) -> MockState { + self.state + } + + /// Serves exactly `n` command lines (counting retries), then returns. + pub fn serve_n_commands(&mut self, n: usize) { + let mut served = 0; + let mut buf = [0_u8; 1024]; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while served < n && std::time::Instant::now() < deadline { + let read = self.transport.read(&mut buf).unwrap_or(0); + if read == 0 { + std::thread::sleep(std::time::Duration::from_millis(1)); + continue; + } + self.line_buffer.extend_from_slice(&buf[..read]); + while let Some(pos) = self.line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = self.line_buffer.drain(..=pos).collect(); + if let Ok(text) = std::str::from_utf8(&line) { + self.handle_line(text.trim_end()); + } + served += 1; + if served >= n { + break; + } + } + } + } + + fn handle_line(&mut self, line: &str) { + let Some(rest) = line.strip_prefix('@') else { + return; + }; + let mut parts = rest.split_ascii_whitespace(); + let Some(sequence) = parts.next().and_then(|s| s.parse::().ok()) else { + return; + }; + // Idempotent retry: replay the cached reply without re-executing. + if let Some((_, cached)) = self + .reply_cache + .iter() + .find(|(cached_seq, _)| *cached_seq == sequence) + { + let payload = cached.clone(); + self.send_control(&payload); + return; + } + assert!( + !self.executed_sequences.contains(&sequence), + "sequence {sequence} re-executed — idempotency broken" + ); + + let verb = parts.next().unwrap_or(""); + let fields: BTreeMap = parts + .filter_map(|part| { + let (key, value) = part.split_once('=')?; + Some((key.to_owned(), value.to_owned())) + }) + .collect(); + + self.executions += 1; + self.executed_sequences.push(sequence); + let reply = self.execute(verb, &fields, sequence); + self.reply_cache.push((sequence, reply.clone())); + if self.reply_cache.len() > 8 { + self.reply_cache.remove(0); + } + if self.drop_next_reply { + self.drop_next_reply = false; + return; + } + self.send_control(&reply); + } + + fn execute(&mut self, verb: &str, fields: &BTreeMap, sequence: u32) -> String { + match verb { + "HELLO" => format!( + "+{sequence} OK protocol=1 firmware=0.1.0-mock board=mock dac_bits=12 \ + capabilities=A1,A2,A3" + ), + "STATUS" => format!( + "+{sequence} OK state={} rev={} executions={}", + self.state.name(), + self.config_revision, + self.executions + ), + "PING" => format!("+{sequence} OK state={}", self.state.name()), + "CONFIG" => { + let mode = fields.get("mode").map(String::as_str).unwrap_or(""); + if !matches!(mode, "A1" | "A2" | "A3") { + return format!("-{sequence} ERR code=BAD_MODE detail=mode"); + } + self.config = fields.clone(); + self.config_revision += 1; + self.state = MockState::Configured; + format!("+{sequence} OK rev={}", self.config_revision) + } + "ARM" => { + if self.state != MockState::Configured { + return format!("-{sequence} ERR code=BAD_STATE detail=arm_requires_config"); + } + self.state = MockState::Armed; + format!("+{sequence} OK state=ARMED rev={}", self.config_revision) + } + "RUN" | "START" => { + if !matches!(self.state, MockState::Armed | MockState::Configured) { + return format!("-{sequence} ERR code=BAD_STATE detail=run_requires_arm"); + } + self.state = MockState::Running; + format!("+{sequence} OK state=RUNNING") + } + "STOP" => { + self.state = MockState::SafeIdle; + format!("+{sequence} OK state=SAFE_IDLE") + } + "FAULT_CLEAR" => format!("+{sequence} OK state={}", self.state.name()), + _ => format!("-{sequence} ERR code=BAD_VERB detail={verb}"), + } + } + + fn send_control(&mut self, payload: &str) { + let frame = self.build_frame(FrameType::Control, payload.as_bytes().to_vec(), 0, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } + + fn build_frame( + &mut self, + frame_type: FrameType, + payload: Vec, + sample_rate_hz: u32, + dropped_samples: u32, + ) -> Frame { + self.out_sequence = self.out_sequence.wrapping_add(1); + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type, + flags: 0, + sequence: self.out_sequence, + payload_bytes: 0, + first_sample_index: self.sample_index, + sample_rate_hz, + dropped_samples, + crc32: 0, + }, + payload, + ) + } + + /// Emits one synthetic sinusoidal sample block (`SamplesU16`). + pub fn emit_sine_block(&mut self, samples: usize, rate_hz: u32, freq_hz: f64) { + let mut payload = Vec::with_capacity(samples * 2); + let mut min_code = u16::MAX; + let mut max_code = 0_u16; + let mut sum = 0_u64; + for i in 0..samples { + let t = (self.sample_index + i as u64) as f64 / f64::from(rate_hz); + let value = self.synth_center + + self.synth_amplitude * (2.0 * std::f64::consts::PI * freq_hz * t).sin(); + let code = value.round().clamp(0.0, 4_095.0) as u16; + min_code = min_code.min(code); + max_code = max_code.max(code); + sum += u64::from(code); + payload.extend_from_slice(&code.to_le_bytes()); + } + let frame = self.build_frame(FrameType::SamplesU16, payload, rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + + let summary = SummaryPayload { + min_code, + max_code, + sample_count: samples as u32, + sum_codes: sum, + first_tick_us: 0, + last_tick_us: ((samples as f64 / f64::from(rate_hz)) * 1e6) as u32, + }; + let frame = self.build_frame(FrameType::Summary, summary.encode(), rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + self.sample_index += samples as u64; + } + + /// Emits a summary frame carrying a nonzero overrun counter. + pub fn emit_summary_with_drops(&mut self, dropped: u32) { + let summary = SummaryPayload { + min_code: 0, + max_code: 0, + sample_count: 0, + sum_codes: 0, + first_tick_us: 0, + last_tick_us: 0, + }; + let frame = self.build_frame(FrameType::Summary, summary.encode(), 20_000, dropped); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } +} + +/// Convenience for tests that need a parsed view of a control payload. +pub fn parse_control(text: &str) -> Option { + ControlMessage::parse(text).ok() +} diff --git a/stage-a-io/src/pdq.rs b/stage-a-io/src/pdq.rs new file mode 100644 index 0000000..0e2119f --- /dev/null +++ b/stage-a-io/src/pdq.rs @@ -0,0 +1,150 @@ +//! `.pdq` writer: preserves every valid PDA1 frame verbatim on disk and +//! tracks run validity. +//! +//! Raw ADC waveforms belong in the PDQ file, never in `HostContext` JSON or +//! per-frame plugin output. A CRC error, frame-sequence gap, or nonzero +//! dropped-sample counter invalidates the run — the file is still written +//! (evidence), but the sidecar must record `valid = false`. + +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; + +use crate::client::StreamIntegrity; +use crate::wire::{crc32, Frame}; + +pub struct PdqWriter { + path: PathBuf, + file: BufWriter, + frames_written: u64, + bytes_written: u64, + running_crc_bytes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PdqSummary { + pub path: PathBuf, + pub frames_written: u64, + pub bytes_written: u64, + /// CRC32 over the whole file contents, recorded in the sidecar. + pub file_crc32: u32, + pub integrity: StreamIntegrity, + pub valid: bool, +} + +impl PdqWriter { + pub fn create(path: impl AsRef) -> std::io::Result { + let path = path.as_ref().to_owned(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + Ok(Self { + file: BufWriter::new(File::create(&path)?), + path, + frames_written: 0, + bytes_written: 0, + running_crc_bytes: Vec::new(), + }) + } + + pub fn write_frame(&mut self, frame: &Frame) -> std::io::Result<()> { + let bytes = frame.to_bytes(); + self.file.write_all(&bytes)?; + self.frames_written += 1; + self.bytes_written += bytes.len() as u64; + self.running_crc_bytes.extend_from_slice(&bytes); + Ok(()) + } + + /// Flushes and closes the file, returning the summary for the sidecar. + pub fn finish(mut self, integrity: StreamIntegrity) -> std::io::Result { + self.file.flush()?; + Ok(PdqSummary { + file_crc32: crc32(&self.running_crc_bytes), + path: self.path, + frames_written: self.frames_written, + bytes_written: self.bytes_written, + valid: integrity.is_clean(), + integrity, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::wire::{FrameHeader, FrameType, PROTOCOL_VERSION}; + + fn frame(sequence: u32) -> Frame { + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Control, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 0, + dropped_samples: 0, + crc32: 0, + }, + format!("+{sequence} OK").into_bytes(), + ) + } + + #[test] + fn writes_frames_verbatim_and_reports_validity() { + let dir = std::env::temp_dir().join(format!( + "stage-a-io-pdq-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let path = dir.join("run.pdq"); + + let mut writer = PdqWriter::create(&path).expect("create pdq"); + let first = frame(1); + let second = frame(2); + writer.write_frame(&first).expect("write"); + writer.write_frame(&second).expect("write"); + let summary = writer + .finish(StreamIntegrity::default()) + .expect("finish pdq"); + + assert!(summary.valid); + assert_eq!(summary.frames_written, 2); + let on_disk = std::fs::read(&path).expect("read back"); + let mut expected = first.to_bytes(); + expected.extend_from_slice(&second.to_bytes()); + assert_eq!(on_disk, expected); + assert_eq!(summary.file_crc32, crate::wire::crc32(&expected)); + + std::fs::remove_dir_all(dir).expect("cleanup"); + } + + #[test] + fn integrity_faults_invalidate_the_run_but_keep_the_file() { + let dir = std::env::temp_dir().join(format!( + "stage-a-io-pdq-invalid-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let path = dir.join("run.pdq"); + + let mut writer = PdqWriter::create(&path).expect("create pdq"); + writer.write_frame(&frame(1)).expect("write"); + let summary = writer + .finish(StreamIntegrity { + dropped_samples: 5, + ..StreamIntegrity::default() + }) + .expect("finish pdq"); + + assert!(!summary.valid); + assert!(path.exists(), "evidence file is preserved"); + std::fs::remove_dir_all(dir).expect("cleanup"); + } +} diff --git a/stage-a-io/src/protocol.rs b/stage-a-io/src/protocol.rs new file mode 100644 index 0000000..254130f --- /dev/null +++ b/stage-a-io/src/protocol.rs @@ -0,0 +1,226 @@ +//! ASCII command / control-reply grammar (host → Teensy and CONTROL frame +//! payloads), per the Stage-A serial protocol v1: +//! +//! ```text +//! Host request: @ key=value key=value\n +//! CONTROL reply payload: + OK key=value ... +//! CONTROL reply payload: - ERR code= detail= +//! Async CONTROL payload: ! key=value ... +//! ``` +//! +//! Commands are printable ASCII, max 192 bytes, integer values only. + +use std::collections::BTreeMap; +use std::fmt; + +pub const MAX_COMMAND_BYTES: usize = 192; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Command { + pub verb: String, + /// Ordered key=value fields (insertion order is preserved on the wire; + /// a BTreeMap would silently reorder, so use a Vec of pairs). + pub fields: Vec<(String, String)>, +} + +impl Command { + pub fn new(verb: &str) -> Self { + Self { + verb: verb.to_owned(), + fields: Vec::new(), + } + } + + pub fn field(mut self, key: &str, value: impl fmt::Display) -> Self { + self.fields.push((key.to_owned(), value.to_string())); + self + } + + /// Encodes `@ VERB k=v ...\n`, validating the printable-ASCII and + /// length constraints. + pub fn encode(&self, sequence: u32) -> Result, ProtocolError> { + let mut line = format!("@{sequence} {}", self.verb); + for (key, value) in &self.fields { + line.push(' '); + line.push_str(key); + line.push('='); + line.push_str(value); + } + line.push('\n'); + if line.len() > MAX_COMMAND_BYTES { + return Err(ProtocolError::CommandTooLong(line.len())); + } + if !line + .bytes() + .all(|b| b == b'\n' || (0x20..=0x7E).contains(&b)) + { + return Err(ProtocolError::NonPrintable); + } + Ok(line.into_bytes()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ControlMessage { + /// `+ OK key=value ...` + Ok { + sequence: u32, + fields: BTreeMap, + }, + /// `- ERR code= detail=` + Err { + sequence: u32, + code: String, + detail: String, + }, + /// `! key=value ...` + Async { + name: String, + fields: BTreeMap, + }, +} + +impl ControlMessage { + pub fn parse(text: &str) -> Result { + let text = text.trim_end_matches(['\r', '\n']); + let mut parts = text.split_ascii_whitespace(); + let head = parts.next().ok_or(ProtocolError::EmptyControl)?; + match head.as_bytes().first() { + Some(b'+') => { + let sequence = head[1..] + .parse() + .map_err(|_| ProtocolError::BadSequence(head.to_owned()))?; + let ok = parts.next(); + if ok != Some("OK") { + return Err(ProtocolError::Malformed(text.to_owned())); + } + Ok(Self::Ok { + sequence, + fields: parse_fields(parts), + }) + } + Some(b'-') => { + let sequence = head[1..] + .parse() + .map_err(|_| ProtocolError::BadSequence(head.to_owned()))?; + let err = parts.next(); + if err != Some("ERR") { + return Err(ProtocolError::Malformed(text.to_owned())); + } + let fields = parse_fields(parts); + Ok(Self::Err { + sequence, + code: fields.get("code").cloned().unwrap_or_default(), + detail: fields.get("detail").cloned().unwrap_or_default(), + }) + } + Some(b'!') => Ok(Self::Async { + name: head[1..].to_owned(), + fields: parse_fields(parts), + }), + _ => Err(ProtocolError::Malformed(text.to_owned())), + } + } + + pub fn sequence(&self) -> Option { + match self { + Self::Ok { sequence, .. } | Self::Err { sequence, .. } => Some(*sequence), + Self::Async { .. } => None, + } + } +} + +fn parse_fields<'a>(parts: impl Iterator) -> BTreeMap { + parts + .filter_map(|part| { + let (key, value) = part.split_once('=')?; + Some((key.to_owned(), value.to_owned())) + }) + .collect() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProtocolError { + CommandTooLong(usize), + NonPrintable, + EmptyControl, + BadSequence(String), + Malformed(String), +} + +impl fmt::Display for ProtocolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CommandTooLong(len) => { + write!(f, "command is {len} bytes (max {MAX_COMMAND_BYTES})") + } + Self::NonPrintable => f.write_str("command contains non-printable bytes"), + Self::EmptyControl => f.write_str("empty control payload"), + Self::BadSequence(head) => write!(f, "unparseable sequence in {head:?}"), + Self::Malformed(text) => write!(f, "malformed control payload {text:?}"), + } + } +} + +impl std::error::Error for ProtocolError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encodes_commands_with_ordered_fields() { + let cmd = Command::new("CONFIG") + .field("mode", "A1") + .field("freq_mhz", 12_500) + .field("center_dac", 2_048) + .field("amplitude_dac", 512); + assert_eq!( + String::from_utf8(cmd.encode(3).expect("encodes")).unwrap(), + "@3 CONFIG mode=A1 freq_mhz=12500 center_dac=2048 amplitude_dac=512\n" + ); + } + + #[test] + fn rejects_oversized_and_non_printable_commands() { + let long = Command::new("X").field("k", "y".repeat(200)); + assert!(matches!( + long.encode(1), + Err(ProtocolError::CommandTooLong(_)) + )); + let bad = Command::new("X").field("k", "\u{7f}"); + assert!(matches!(bad.encode(1), Err(ProtocolError::NonPrintable))); + } + + #[test] + fn parses_ok_err_and_async_payloads() { + let ok = ControlMessage::parse("+12 OK state=ARMED rev=4").expect("ok parses"); + match ok { + ControlMessage::Ok { sequence, fields } => { + assert_eq!(sequence, 12); + assert_eq!(fields.get("rev").map(String::as_str), Some("4")); + } + other => panic!("unexpected {other:?}"), + } + + let err = + ControlMessage::parse("-13 ERR code=BOUNDS detail=amplitude_dac").expect("err parses"); + assert_eq!( + err, + ControlMessage::Err { + sequence: 13, + code: "BOUNDS".into(), + detail: "amplitude_dac".into() + } + ); + + let async_msg = ControlMessage::parse("!APPLIED rev=4").expect("async parses"); + match async_msg { + ControlMessage::Async { name, fields } => { + assert_eq!(name, "APPLIED"); + assert_eq!(fields.get("rev").map(String::as_str), Some("4")); + } + other => panic!("unexpected {other:?}"), + } + } +} diff --git a/stage-a-io/src/sidecar.rs b/stage-a-io/src/sidecar.rs new file mode 100644 index 0000000..0b591bc --- /dev/null +++ b/stage-a-io/src/sidecar.rs @@ -0,0 +1,216 @@ +//! Run sidecar (manifest): everything needed to reproduce or audit one +//! Stage-A recording, written as JSON next to the camera RAW / PDQ files. +//! +//! Per the control-software spec, each recording sidecar includes the run +//! ID, plugin/firmware/protocol versions, raw PDQ path and checksum, +//! ADC/front-end calibration, load, configured and measured sample cadence, +//! drop/CRC counters, the ACKed configuration revision, bias set, optical +//! configuration, flux point, measured `a`, and trigger source. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::client::StreamIntegrity; +use crate::estimator::{AdcCalibration, ContrastEstimate}; +use crate::pdq::PdqSummary; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TriggerSource { + /// Teensy waveform phase-0 sync TTL (A1/A3 drive fiducial). + DrivePhase0, + /// Photodiode → comparator 50 % crossing (A2 light fiducial). + Comparator, + /// No hardware trigger wired; software phase recovery in use. + None, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DetectorLoad { + /// 50 Ω — A1/A2 (speed over signal). + FiftyOhm, + /// Characterised high-Z load — A3 only ($f \ll f_c$). + HighZ { nominal_ohms: u64 }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RunSidecar { + pub run_id: String, + pub protocol: String, + pub created_utc: String, + + pub plugin_name: String, + pub plugin_version: String, + pub firmware_version: String, + pub wire_protocol_version: u8, + + /// Path + CRC32 of the raw PDQ photodiode file. + pub pdq_path: PathBuf, + pub pdq_crc32: u32, + pub pdq_frames: u64, + /// Path of the camera RAW recording this run belongs to, if any. + pub camera_raw_path: Option, + + pub adc_calibration: AdcCalibration, + pub detector_load: DetectorLoad, + pub configured_sample_rate_hz: u32, + pub measured_sample_rate_hz: Option, + + pub integrity: IntegrityRecord, + /// Overall validity — false on any drop/CRC/sequence/cadence fault or + /// estimator rejection. An invalid point is re-measured, never patched. + pub valid: bool, + + /// ACKed controller configuration (verbatim key=value fields) and its + /// revision, exactly as the firmware confirmed them. + pub acked_config_revision: Option, + pub acked_config: BTreeMap, + + /// Frozen camera bias set identifier (registry lives in the knowledge + /// base `setup/bias-sets.md`). + pub bias_set: Option, + /// Optical configuration / flux point labels from the run plan. + pub optical_configuration: Option, + pub flux_point: Option, + + /// Measured optical log-contrast for this run/point, when applicable. + pub measured_contrast: Option, + pub trigger_source: TriggerSource, + + /// Free-form notes (operator observations, deviations). + pub notes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct IntegrityRecord { + pub skipped_bytes: u64, + pub crc_failures: u64, + pub sequence_gaps: u64, + pub dropped_samples: u64, +} + +impl From for IntegrityRecord { + fn from(value: StreamIntegrity) -> Self { + Self { + skipped_bytes: value.skipped_bytes, + crc_failures: value.crc_failures, + sequence_gaps: value.sequence_gaps, + dropped_samples: value.dropped_samples, + } + } +} + +impl RunSidecar { + /// Builds a sidecar skeleton from a finished PDQ file. Protocol fields + /// and run metadata are filled by the owning plugin before writing. + pub fn from_pdq(run_id: &str, protocol: &str, pdq: &PdqSummary) -> Self { + Self { + run_id: run_id.to_owned(), + protocol: protocol.to_owned(), + created_utc: now_utc_iso8601(), + plugin_name: String::new(), + plugin_version: String::new(), + firmware_version: String::new(), + wire_protocol_version: crate::wire::PROTOCOL_VERSION, + pdq_path: pdq.path.clone(), + pdq_crc32: pdq.file_crc32, + pdq_frames: pdq.frames_written, + camera_raw_path: None, + adc_calibration: AdcCalibration::default(), + detector_load: DetectorLoad::FiftyOhm, + configured_sample_rate_hz: 0, + measured_sample_rate_hz: None, + integrity: pdq.integrity.into(), + valid: pdq.valid, + acked_config_revision: None, + acked_config: BTreeMap::new(), + bias_set: None, + optical_configuration: None, + flux_point: None, + measured_contrast: None, + trigger_source: TriggerSource::None, + notes: Vec::new(), + } + } + + pub fn write_json(&self, path: impl AsRef) -> std::io::Result<()> { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let json = serde_json::to_vec_pretty(self)?; + std::fs::write(path, json) + } + + pub fn read_json(path: impl AsRef) -> std::io::Result { + let bytes = std::fs::read(path)?; + serde_json::from_slice(&bytes).map_err(std::io::Error::other) + } +} + +fn now_utc_iso8601() -> String { + // Seconds-resolution UTC timestamp without pulling in chrono. + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let days = secs / 86_400; + let (year, month, day) = civil_from_days(days as i64); + let rem = secs % 86_400; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", + rem / 3_600, + (rem % 3_600) / 60, + rem % 60 + ) +} + +/// Howard Hinnant's `civil_from_days` (public domain algorithm). +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = (z - era * 146_097) as u64; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + (if m <= 2 { y + 1 } else { y }, m, d) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sidecar_round_trips_through_json() { + let pdq = PdqSummary { + path: PathBuf::from("/data/A1-20260713-01.pdq"), + frames_written: 128, + bytes_written: 65_536, + file_crc32: 0xDEAD_BEEF, + integrity: StreamIntegrity::default(), + valid: true, + }; + let mut sidecar = RunSidecar::from_pdq("A1-20260713-01", "A1", &pdq); + sidecar.plugin_name = "stage-a-a1".into(); + sidecar.acked_config_revision = Some(4); + sidecar.acked_config.insert("mode".into(), "A1".into()); + sidecar.trigger_source = TriggerSource::DrivePhase0; + + let json = serde_json::to_string(&sidecar).expect("serializes"); + let decoded: RunSidecar = serde_json::from_str(&json).expect("deserializes"); + assert_eq!(decoded, sidecar); + assert!(decoded.created_utc.ends_with('Z')); + } + + #[test] + fn civil_from_days_matches_known_dates() { + assert_eq!(civil_from_days(0), (1970, 1, 1)); + assert_eq!(civil_from_days(20_282), (2025, 7, 13)); + } +} diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs new file mode 100644 index 0000000..27dea22 --- /dev/null +++ b/stage-a-io/src/transport.rs @@ -0,0 +1,113 @@ +//! Byte transports: the real USB serial port and an in-memory mock. +//! +//! Exactly one armed plugin owns the port at a time; opening a busy device +//! is a visible error, never a silent second connection (the OS enforces +//! exclusivity via `serialport`'s exclusive open on POSIX). + +use std::io; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +pub trait Transport: Send { + /// Reads whatever is available into `buf`, blocking up to the + /// transport's timeout. `Ok(0)` means "nothing arrived this poll". + fn read(&mut self, buf: &mut [u8]) -> io::Result; + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()>; +} + +/// Real serial port. Construction fails visibly if the device is busy or +/// absent. +#[cfg(feature = "hardware")] +pub struct SerialTransport { + port: Box, +} + +#[cfg(feature = "hardware")] +impl SerialTransport { + pub fn open(path: &str, baud: u32, poll_timeout: Duration) -> io::Result { + let port = serialport::new(path, baud) + .timeout(poll_timeout) + .open() + .map_err(|err| io::Error::other(format!("opening {path} failed: {err}")))?; + Ok(Self { port }) + } +} + +#[cfg(feature = "hardware")] +impl Transport for SerialTransport { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self.port.read(buf) { + Ok(n) => Ok(n), + Err(err) if err.kind() == io::ErrorKind::TimedOut => Ok(0), + Err(err) => Err(err), + } + } + + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()> { + io::Write::write_all(&mut self.port, bytes) + } +} + +/// Shared in-memory duplex used by tests and the mock controller: the +/// "host" side reads what the "device" side wrote and vice versa. +#[derive(Default)] +struct DuplexState { + to_host: Vec, + to_device: Vec, +} + +#[derive(Clone, Default)] +pub struct MockLink { + state: Arc>, +} + +impl MockLink { + pub fn new() -> Self { + Self::default() + } + + pub fn host_end(&self) -> MockTransport { + MockTransport { + state: Arc::clone(&self.state), + is_host: true, + } + } + + pub fn device_end(&self) -> MockTransport { + MockTransport { + state: Arc::clone(&self.state), + is_host: false, + } + } +} + +pub struct MockTransport { + state: Arc>, + is_host: bool, +} + +impl Transport for MockTransport { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let source = if self.is_host { + &mut state.to_host + } else { + &mut state.to_device + }; + let n = source.len().min(buf.len()); + buf[..n].copy_from_slice(&source[..n]); + source.drain(..n); + Ok(n) + } + + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()> { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let sink = if self.is_host { + &mut state.to_device + } else { + &mut state.to_host + }; + sink.extend_from_slice(bytes); + Ok(()) + } +} diff --git a/stage-a-io/src/wire.rs b/stage-a-io/src/wire.rs new file mode 100644 index 0000000..33e8caa --- /dev/null +++ b/stage-a-io/src/wire.rs @@ -0,0 +1,438 @@ +//! PDA1 binary wire format (Teensy → host). +//! +//! Mirrors `stage-a-controller/include/wire_protocol.h` exactly: a packed +//! 36-byte little-endian header followed by `payload_bytes` of payload, +//! integrity-protected by CRC32 (IEEE, reflected) over the zeroed-CRC header +//! plus payload. The host must tolerate arbitrary USB fragmentation and +//! resynchronise at the next valid magic + CRC. + +/// `"PDA1"` interpreted as a little-endian `u32`. +pub const MAGIC: u32 = 0x3141_4450; +pub const PROTOCOL_VERSION: u8 = 1; +pub const HEADER_BYTES: usize = 36; + +/// Maximum payload the parser will attempt to buffer. Larger claimed sizes +/// are treated as corruption and trigger resynchronisation. +pub const MAX_PAYLOAD_BYTES: usize = 1 << 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameType { + Control, + SamplesU16, + Summary, + Marker, + Unknown(u8), +} + +impl FrameType { + pub fn from_raw(raw: u8) -> Self { + match raw { + 1 => Self::Control, + 2 => Self::SamplesU16, + 3 => Self::Summary, + 4 => Self::Marker, + other => Self::Unknown(other), + } + } + + pub fn to_raw(self) -> u8 { + match self { + Self::Control => 1, + Self::SamplesU16 => 2, + Self::Summary => 3, + Self::Marker => 4, + Self::Unknown(other) => other, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameHeader { + pub version: u8, + pub frame_type: FrameType, + pub flags: u16, + pub sequence: u32, + pub payload_bytes: u32, + pub first_sample_index: u64, + pub sample_rate_hz: u32, + pub dropped_samples: u32, + pub crc32: u32, +} + +impl FrameHeader { + pub fn parse(bytes: &[u8; HEADER_BYTES]) -> Option { + let magic = u32::from_le_bytes(bytes[0..4].try_into().ok()?); + if magic != MAGIC { + return None; + } + Some(Self { + version: bytes[4], + frame_type: FrameType::from_raw(bytes[5]), + flags: u16::from_le_bytes(bytes[6..8].try_into().ok()?), + sequence: u32::from_le_bytes(bytes[8..12].try_into().ok()?), + payload_bytes: u32::from_le_bytes(bytes[12..16].try_into().ok()?), + first_sample_index: u64::from_le_bytes(bytes[16..24].try_into().ok()?), + sample_rate_hz: u32::from_le_bytes(bytes[24..28].try_into().ok()?), + dropped_samples: u32::from_le_bytes(bytes[28..32].try_into().ok()?), + crc32: u32::from_le_bytes(bytes[32..36].try_into().ok()?), + }) + } + + pub fn encode(&self) -> [u8; HEADER_BYTES] { + let mut out = [0_u8; HEADER_BYTES]; + out[0..4].copy_from_slice(&MAGIC.to_le_bytes()); + out[4] = self.version; + out[5] = self.frame_type.to_raw(); + out[6..8].copy_from_slice(&self.flags.to_le_bytes()); + out[8..12].copy_from_slice(&self.sequence.to_le_bytes()); + out[12..16].copy_from_slice(&self.payload_bytes.to_le_bytes()); + out[16..24].copy_from_slice(&self.first_sample_index.to_le_bytes()); + out[24..28].copy_from_slice(&self.sample_rate_hz.to_le_bytes()); + out[28..32].copy_from_slice(&self.dropped_samples.to_le_bytes()); + out[32..36].copy_from_slice(&self.crc32.to_le_bytes()); + out + } +} + +/// One complete, CRC-verified frame. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + pub header: FrameHeader, + pub payload: Vec, +} + +impl Frame { + /// Builds a frame with a freshly computed CRC (mock/firmware side). + pub fn build(mut header: FrameHeader, payload: Vec) -> Self { + header.payload_bytes = payload.len() as u32; + header.crc32 = frame_crc(&header, &payload); + Self { header, payload } + } + + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::with_capacity(HEADER_BYTES + self.payload.len()); + out.extend_from_slice(&self.header.encode()); + out.extend_from_slice(&self.payload); + out + } + + /// Decodes the payload of a `Summary` frame. + pub fn summary(&self) -> Option { + if self.header.frame_type != FrameType::Summary || self.payload.len() != 24 { + return None; + } + let p = &self.payload; + Some(SummaryPayload { + min_code: u16::from_le_bytes(p[0..2].try_into().ok()?), + max_code: u16::from_le_bytes(p[2..4].try_into().ok()?), + sample_count: u32::from_le_bytes(p[4..8].try_into().ok()?), + sum_codes: u64::from_le_bytes(p[8..16].try_into().ok()?), + first_tick_us: u32::from_le_bytes(p[16..20].try_into().ok()?), + last_tick_us: u32::from_le_bytes(p[20..24].try_into().ok()?), + }) + } + + /// Decodes the payload of a `SamplesU16` frame into ADC codes. + pub fn samples(&self) -> Option> { + if self.header.frame_type != FrameType::SamplesU16 || self.payload.len() % 2 != 0 { + return None; + } + Some( + self.payload + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(), + ) + } + + /// The ASCII payload of a `Control` frame. + pub fn control_text(&self) -> Option<&str> { + if self.header.frame_type != FrameType::Control { + return None; + } + std::str::from_utf8(&self.payload).ok() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SummaryPayload { + pub min_code: u16, + pub max_code: u16, + pub sample_count: u32, + pub sum_codes: u64, + pub first_tick_us: u32, + pub last_tick_us: u32, +} + +impl SummaryPayload { + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(24); + out.extend_from_slice(&self.min_code.to_le_bytes()); + out.extend_from_slice(&self.max_code.to_le_bytes()); + out.extend_from_slice(&self.sample_count.to_le_bytes()); + out.extend_from_slice(&self.sum_codes.to_le_bytes()); + out.extend_from_slice(&self.first_tick_us.to_le_bytes()); + out.extend_from_slice(&self.last_tick_us.to_le_bytes()); + out + } + + pub fn mean_code(&self) -> f64 { + if self.sample_count == 0 { + return 0.0; + } + self.sum_codes as f64 / f64::from(self.sample_count) + } +} + +/// CRC32 (IEEE, reflected, init/final 0xFFFF_FFFF) — identical to the +/// firmware's `crc32Update` loop. +pub fn crc32(data: &[u8]) -> u32 { + crc32_update(0xFFFF_FFFF, data) ^ 0xFFFF_FFFF +} + +fn crc32_update(mut crc: u32, data: &[u8]) -> u32 { + for &byte in data { + crc ^= u32::from(byte); + for _ in 0..8 { + let mask = 0_u32.wrapping_sub(crc & 1); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + crc +} + +/// CRC over the zeroed-CRC header plus payload (firmware `frameCrc`). +pub fn frame_crc(header: &FrameHeader, payload: &[u8]) -> u32 { + let mut zeroed = *header; + zeroed.crc32 = 0; + let mut crc = 0xFFFF_FFFF_u32; + crc = crc32_update(crc, &zeroed.encode()); + crc = crc32_update(crc, payload); + crc ^ 0xFFFF_FFFF +} + +/// What the incremental parser reports for each recovered unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ParseEvent { + Frame(Frame), + /// Bytes were skipped or a frame failed its CRC — the stream stays + /// usable, but the run must be flagged invalid. + Corruption { + skipped_bytes: usize, + crc_failures: usize, + }, +} + +/// Incremental PDA1 parser tolerating arbitrary fragmentation. +/// +/// Feed raw serial bytes with [`FrameParser::extend`], then drain complete +/// frames with [`FrameParser::next_event`]. On a bad magic the parser skips +/// forward one byte at a time; on a bad CRC it discards the candidate header +/// and rescans from the next byte, so a corrupted stream re-locks at the +/// next genuine frame boundary. +#[derive(Debug, Default)] +pub struct FrameParser { + buffer: Vec, + skipped_bytes: usize, + crc_failures: usize, +} + +impl FrameParser { + pub fn extend(&mut self, bytes: &[u8]) { + self.buffer.extend_from_slice(bytes); + } + + pub fn next_event(&mut self) -> Option { + loop { + // Scan to the next plausible magic. + let mut offset = 0; + while self.buffer.len() >= offset + 4 + && u32::from_le_bytes(self.buffer[offset..offset + 4].try_into().unwrap()) != MAGIC + { + offset += 1; + } + if offset > 0 { + self.buffer.drain(..offset); + self.skipped_bytes += offset; + } + + if self.buffer.len() < HEADER_BYTES { + return self.take_corruption(); + } + + let header_bytes: [u8; HEADER_BYTES] = self.buffer[..HEADER_BYTES].try_into().unwrap(); + let Some(header) = FrameHeader::parse(&header_bytes) else { + // Magic matched but parse failed (cannot happen today, but + // stay defensive): skip one byte and rescan. + self.buffer.drain(..1); + self.skipped_bytes += 1; + continue; + }; + + let payload_bytes = header.payload_bytes as usize; + if payload_bytes > MAX_PAYLOAD_BYTES { + self.buffer.drain(..1); + self.skipped_bytes += 1; + continue; + } + if self.buffer.len() < HEADER_BYTES + payload_bytes { + // Wait for more bytes; report any corruption noticed so far. + return self.take_corruption(); + } + + let payload = self.buffer[HEADER_BYTES..HEADER_BYTES + payload_bytes].to_vec(); + if frame_crc(&header, &payload) != header.crc32 { + self.crc_failures += 1; + self.buffer.drain(..1); + self.skipped_bytes += 1; + continue; + } + + self.buffer.drain(..HEADER_BYTES + payload_bytes); + if let Some(corruption) = self.take_corruption() { + // Deliver the corruption notice first; the verified frame is + // still buffered as raw bytes, so re-parse it next call. + let frame = Frame { header, payload }; + let mut bytes = frame.to_bytes(); + bytes.extend_from_slice(&self.buffer); + self.buffer = bytes; + return Some(corruption); + } + return Some(ParseEvent::Frame(Frame { header, payload })); + } + } + + fn take_corruption(&mut self) -> Option { + if self.skipped_bytes == 0 && self.crc_failures == 0 { + return None; + } + let event = ParseEvent::Corruption { + skipped_bytes: self.skipped_bytes, + crc_failures: self.crc_failures, + }; + self.skipped_bytes = 0; + self.crc_failures = 0; + Some(event) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn control_frame(sequence: u32, text: &str) -> Frame { + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Control, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 0, + dropped_samples: 0, + crc32: 0, + }, + text.as_bytes().to_vec(), + ) + } + + #[test] + fn round_trips_a_frame_through_arbitrary_fragmentation() { + let frame = control_frame(7, "+7 OK state=SAFE_IDLE"); + let bytes = frame.to_bytes(); + + let mut parser = FrameParser::default(); + for chunk in bytes.chunks(3) { + parser.extend(chunk); + } + assert_eq!(parser.next_event(), Some(ParseEvent::Frame(frame))); + assert_eq!(parser.next_event(), None); + } + + #[test] + fn resynchronises_after_garbage_and_reports_corruption() { + let frame = control_frame(1, "+1 OK"); + let mut bytes = b"garbage!".to_vec(); + bytes.extend_from_slice(&frame.to_bytes()); + + let mut parser = FrameParser::default(); + parser.extend(&bytes); + assert_eq!( + parser.next_event(), + Some(ParseEvent::Corruption { + skipped_bytes: 8, + crc_failures: 0 + }) + ); + assert_eq!(parser.next_event(), Some(ParseEvent::Frame(frame))); + } + + #[test] + fn detects_crc_corruption_and_relocks_on_next_frame() { + let bad = control_frame(1, "+1 OK"); + let good = control_frame(2, "!STATUS state=RUNNING"); + let mut bytes = bad.to_bytes(); + let len = bytes.len(); + bytes[len - 1] ^= 0xFF; // corrupt payload -> CRC mismatch + bytes.extend_from_slice(&good.to_bytes()); + + let mut parser = FrameParser::default(); + parser.extend(&bytes); + let corruption = parser.next_event(); + match corruption { + Some(ParseEvent::Corruption { crc_failures, .. }) => assert!(crc_failures >= 1), + other => panic!("expected corruption, got {other:?}"), + } + assert_eq!(parser.next_event(), Some(ParseEvent::Frame(good))); + } + + #[test] + fn summary_payload_round_trips() { + let summary = SummaryPayload { + min_code: 12, + max_code: 3_900, + sample_count: 256, + sum_codes: 500_000, + first_tick_us: 1_000, + last_tick_us: 13_800, + }; + let frame = Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Summary, + flags: 0, + sequence: 5, + payload_bytes: 0, + first_sample_index: 4_096, + sample_rate_hz: 20_000, + dropped_samples: 0, + crc32: 0, + }, + summary.encode(), + ); + assert_eq!(frame.summary(), Some(summary)); + assert!((summary.mean_code() - 1953.125).abs() < 1e-9); + } + + #[test] + fn samples_frame_decodes_codes() { + let codes = [1_u16, 2, 4_095]; + let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); + let frame = Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::SamplesU16, + flags: 0, + sequence: 9, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 20_000, + dropped_samples: 0, + crc32: 0, + }, + payload, + ); + assert_eq!(frame.samples(), Some(codes.to_vec())); + } +} diff --git a/stage-a-io/src/worker.rs b/stage-a-io/src/worker.rs new file mode 100644 index 0000000..57ffaee --- /dev/null +++ b/stage-a-io/src/worker.rs @@ -0,0 +1,229 @@ +//! Bounded background I/O worker. +//! +//! The owning plugin's `process_frame()` must never block on serial: it only +//! drains this worker's bounded output queue and pushes bounded requests. +//! The worker thread owns the [`StageAClient`] (and thereby the serial +//! port), sends `PING` at 2 Hz while the controller is armed/running, and +//! requests `STOP` on shutdown. Firmware safety does not depend on that +//! STOP arriving — the on-device watchdog falls back to `SAFE_IDLE` — but a +//! clean stop is always attempted. + +use std::collections::BTreeMap; +use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use crate::client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; +use crate::protocol::Command; +use crate::transport::Transport; + +pub const COMMAND_QUEUE_DEPTH: usize = 16; +pub const OUTPUT_QUEUE_DEPTH: usize = 256; +const PING_INTERVAL: Duration = Duration::from_millis(500); +const IDLE_POLL: Duration = Duration::from_millis(5); + +/// Requests the plugin can queue for the worker. +#[derive(Debug, Clone)] +pub enum WorkerRequest { + /// Send a command and report its reply (or error) as a `Reply` output. + Send { tag: u64, command: Command }, + /// Enable/disable the 2 Hz watchdog ping (armed/running phases). + SetPinging(bool), + /// Stop the controller and shut the worker down. + Shutdown { reason: String }, +} + +/// Bounded outputs the plugin drains from `process_frame()`. +#[derive(Debug)] +pub enum WorkerOutput { + Reply { + tag: u64, + result: Result, String>, + }, + Event(DeviceEvent), + Integrity(StreamIntegrity), + /// The worker exited (clean shutdown or transport failure). + Stopped { + reason: String, + }, +} + +pub struct IoWorker { + requests: SyncSender, + outputs: Receiver, + join: Option>, +} + +impl IoWorker { + /// Spawns the worker over an already-open transport. Opening the + /// transport (and failing visibly if the device is busy) is the + /// caller's responsibility, in `LiveCapture` with effects allowed only. + pub fn spawn(client: StageAClient) -> Self { + let (request_tx, request_rx) = std::sync::mpsc::sync_channel(COMMAND_QUEUE_DEPTH); + let (output_tx, output_rx) = std::sync::mpsc::sync_channel(OUTPUT_QUEUE_DEPTH); + let join = std::thread::Builder::new() + .name("stage-a-io".into()) + .spawn(move || run_worker(client, request_rx, output_tx)) + .expect("spawning the stage-a I/O thread must succeed"); + Self { + requests: request_tx, + outputs: output_rx, + join: Some(join), + } + } + + /// Non-blocking enqueue; a full queue is a visible error, not a stall. + pub fn try_send(&self, request: WorkerRequest) -> Result<(), String> { + self.requests.try_send(request).map_err(|err| match err { + TrySendError::Full(_) => "stage-a I/O command queue is full".to_owned(), + TrySendError::Disconnected(_) => "stage-a I/O worker is gone".to_owned(), + }) + } + + /// Drains everything currently queued, without blocking. + pub fn drain_outputs(&self) -> Vec { + let mut out = Vec::new(); + while let Ok(output) = self.outputs.try_recv() { + out.push(output); + } + out + } + + /// Requests a controller STOP and joins the worker. + pub fn shutdown(mut self, reason: &str) { + let _ = self.requests.try_send(WorkerRequest::Shutdown { + reason: reason.to_owned(), + }); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +impl Drop for IoWorker { + fn drop(&mut self) { + let _ = self.requests.try_send(WorkerRequest::Shutdown { + reason: "worker dropped".to_owned(), + }); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +fn run_worker( + mut client: StageAClient, + requests: Receiver, + outputs: SyncSender, +) { + let mut pinging = false; + let mut last_ping = Instant::now(); + let mut last_integrity = client.integrity(); + + let stop_reason = loop { + match requests.recv_timeout(IDLE_POLL) { + Ok(WorkerRequest::Send { tag, command }) => { + let result = client + .request(&command) + .map_err(|err: ClientError| err.to_string()); + if outputs + .try_send(WorkerOutput::Reply { tag, result }) + .is_err() + { + break "output queue closed".to_owned(); + } + } + Ok(WorkerRequest::SetPinging(enabled)) => { + pinging = enabled; + last_ping = Instant::now(); + } + Ok(WorkerRequest::Shutdown { reason }) => break reason, + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break "request queue closed".to_owned(), + } + + match client.poll_events() { + Ok(events) => { + for event in events { + // Bounded best-effort delivery: a full output queue drops + // live telemetry, never blocks the serial loop. Exact + // data is preserved by the PDQ writer downstream of the + // worker owner, which uses Reply-driven flow instead. + let _ = outputs.try_send(WorkerOutput::Event(event)); + } + } + Err(err) => { + let _ = outputs.try_send(WorkerOutput::Reply { + tag: 0, + result: Err(err.to_string()), + }); + break "transport failure".to_owned(); + } + } + + let integrity = client.integrity(); + if integrity != last_integrity { + last_integrity = integrity; + let _ = outputs.try_send(WorkerOutput::Integrity(integrity)); + } + + if pinging && last_ping.elapsed() >= PING_INTERVAL { + last_ping = Instant::now(); + let _ = client.request(&Command::new("PING")); + } + }; + + // Best-effort clean stop; the firmware watchdog is the real guarantee. + let _ = client.request(&Command::new("STOP").field("reason", stop_reason.replace(' ', "_"))); + let _ = outputs.try_send(WorkerOutput::Stopped { + reason: stop_reason, + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::MockController; + use crate::transport::MockLink; + + #[test] + fn worker_round_trips_commands_and_stops_cleanly() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + let worker = IoWorker::spawn(client); + + // HELLO via the worker, served by the mock on this thread. + worker + .try_send(WorkerRequest::Send { + tag: 1, + command: Command::new("HELLO").field("protocol", 1), + }) + .expect("enqueue"); + controller.serve_n_commands(1); + + let deadline = Instant::now() + Duration::from_secs(1); + let mut reply_seen = false; + while Instant::now() < deadline && !reply_seen { + for output in worker.drain_outputs() { + if let WorkerOutput::Reply { tag: 1, result } = output { + let fields = result.expect("HELLO succeeds"); + assert_eq!(fields.get("protocol").map(String::as_str), Some("1")); + reply_seen = true; + } + } + std::thread::sleep(Duration::from_millis(2)); + } + assert!(reply_seen, "HELLO reply must reach the plugin queue"); + + // Shutdown must send STOP to the controller. + let handle = std::thread::spawn(move || { + controller.serve_n_commands(1); + controller + }); + worker.shutdown("test done"); + let controller = handle.join().expect("mock joins"); + assert_eq!(controller.state(), crate::mock::MockState::SafeIdle); + } +} From 3f0d57f98e40ce4173abff2e6f57fb2a647869be Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:08:42 +0200 Subject: [PATCH 02/46] =?UTF-8?q?feat(stage-a-monitor):=20=E2=9C=A8=20add?= =?UTF-8?q?=20commissioning=20monitor=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live decimated photodiode waveform, calibrated clipping-guarded optical contrast, stream-integrity status, and gated manual controller actions (connect/config/start/stop/expert drive). Fails closed on the ABI v5 execution context: serial I/O only in the active live-capture worker; commands are host actions, never persistent settings. --- Cargo.toml | 1 + plugins/stage-a-monitor/Cargo.toml | 15 + plugins/stage-a-monitor/README.md | 39 ++ plugins/stage-a-monitor/plugin.toml | 7 + plugins/stage-a-monitor/src/lib.rs | 808 ++++++++++++++++++++++++++++ stage-a-io/src/transport.rs | 14 + 6 files changed, 884 insertions(+) create mode 100644 plugins/stage-a-monitor/Cargo.toml create mode 100644 plugins/stage-a-monitor/README.md create mode 100644 plugins/stage-a-monitor/plugin.toml create mode 100644 plugins/stage-a-monitor/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 98ff631..ca2a368 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "stage-a-io", + "plugins/stage-a-monitor", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/plugins/stage-a-monitor/Cargo.toml b/plugins/stage-a-monitor/Cargo.toml new file mode 100644 index 0000000..61dfffd --- /dev/null +++ b/plugins/stage-a-monitor/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "augur-plugin-stage-a-monitor" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A commissioning monitor: live photodiode readout, calibrated optical contrast, and gated manual Teensy drive control." + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde_json.workspace = true +stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-monitor/README.md b/plugins/stage-a-monitor/README.md new file mode 100644 index 0000000..8b43896 --- /dev/null +++ b/plugins/stage-a-monitor/README.md @@ -0,0 +1,39 @@ +# Stage-A Monitor + +Commissioning companion for the Stage-A camera-calibration bench: a live +view of the Teensy photodiode DAQ plus **gated** manual controller commands. + +## What it shows + +- **Photodiode waveform** — decimated calibrated trace (volts vs ms) from + the `SamplesU16` stream. +- **Live optical contrast** — `a = ln(V_max/V_min)` from dark-corrected, + clipping-guarded percentile extrema (see `stage-a-io::estimator`). An + invalid window shows *why* (clipped / no headroom / too short) instead of + a silently wrong number. +- **Stream integrity** — CRC failures, resync skips, frame-sequence gaps, + and ADC overruns. Any nonzero counter means the current point is invalid. + +## Controls (host actions on the status table) + +`Connect`, `Disconnect`, `Start acquisition`, `Stop`, and an expert +`Apply drive` modal (integer DAC codes; the optical contrast is always +measured, never assumed from the drive). Commands are actions — not +settings — so a reloaded settings file can never arm hardware. + +## Safety + +The plugin fails closed: the serial port opens only when the host reports +`LiveCapture` with `effects_allowed` (plugin ABI v5 execution context). +Replay and offline analysis can never emit a serial byte, and an existing +connection is shut down the moment effects are revoked. The firmware-side +watchdog independently drops the controller to `SAFE_IDLE` if the host +disappears. + +## Use it for (commissioning checklist) + +1. Wiring / voltage-range check at both detector loads. +2. Dark-level measurement for the estimator calibration. +3. Coherent-crosstalk test (H14): drive on, light blocked — the waveform + view and `a` readout must stay at the noise floor. +4. USB-throughput sanity (watch the integrity counters at full rate). diff --git a/plugins/stage-a-monitor/plugin.toml b/plugins/stage-a-monitor/plugin.toml new file mode 100644 index 0000000..b82d0f5 --- /dev/null +++ b/plugins/stage-a-monitor/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Monitor" +version = "0.2.0" +description = "Live Teensy photodiode readout, calibrated optical contrast, and gated manual drive control for Stage-A commissioning." +domain = "stage-a" +library = "augur_plugin_stage_a_monitor" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs new file mode 100644 index 0000000..f01c1f7 --- /dev/null +++ b/plugins/stage-a-monitor/src/lib.rs @@ -0,0 +1,808 @@ +//! Stage-A commissioning monitor. +//! +//! Live view of the Teensy photodiode DAQ (decimated waveform, calibrated +//! optical log-contrast `a`, clipping/headroom and stream-integrity status) +//! plus **gated** manual controller commands (connect, configure, start, +//! stop) for wiring and crosstalk commissioning. +//! +//! Safety contract (Stage-A control-software spec): +//! - devices open only when `HostContext::execution()` reports +//! `LiveCapture` **and** `effects_allowed` — replay and offline analysis +//! can never touch the serial port, and a stale worker is shut down the +//! moment the context stops permitting effects; +//! - commands are host actions, never persistent settings, so a reloaded +//! settings file cannot re-arm hardware; +//! - `process_frame()` only drains the bounded I/O worker queues. + +use std::collections::BTreeMap; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, + Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, + StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, + TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, + IoWorker, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, +}; + +const WAVEFORM_DATASET_ID: &str = "stage-a-monitor.waveform"; +const STATUS_DATASET_ID: &str = "stage-a-monitor.status"; +const WAVEFORM_VIEW_ID: &str = "stage-a-monitor.waveform.view"; +const STATUS_VIEW_ID: &str = "stage-a-monitor.status.view"; + +const ACTION_CONNECT: &str = "stage-a-monitor.connect"; +const ACTION_DISCONNECT: &str = "stage-a-monitor.disconnect"; +const ACTION_START: &str = "stage-a-monitor.start"; +const ACTION_STOP: &str = "stage-a-monitor.stop"; +const ACTION_APPLY_DRIVE: &str = "stage-a-monitor.apply-drive"; + +/// Retained sample window for the live view + contrast estimate. +const SAMPLE_RING_CAPACITY: usize = 32_768; +/// Points published per waveform refresh (decimated). +const WAVEFORM_POINTS: usize = 1_024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ConnectionState { + Disconnected, + Connected, + Acquiring, +} + +pub struct StageAMonitorPlugin { + enabled: bool, + // -- device -- + worker: Option, + connection: ConnectionState, + firmware: String, + next_tag: u64, + /// Tags of in-flight requests -> human-readable purpose. + in_flight: BTreeMap, + last_error: Option, + integrity: StreamIntegrity, + effects_blocked_reason: Option, + // -- settings -- + port_hint: String, + sample_rate_hz: i64, + block_samples: i64, + calibration: AdcCalibration, + // -- data -- + sample_ring: Vec, + ring_next_sample_index: u64, + sample_rate_seen_hz: u32, + contrast: Option, + contrast_error: Option, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAMonitorPlugin { + fn default() -> Self { + Self { + enabled: false, + worker: None, + connection: ConnectionState::Disconnected, + firmware: String::new(), + next_tag: 1, + in_flight: BTreeMap::new(), + last_error: None, + integrity: StreamIntegrity::default(), + effects_blocked_reason: None, + port_hint: "auto".into(), + sample_rate_hz: 20_000, + block_samples: 256, + calibration: AdcCalibration::default(), + sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), + ring_next_sample_index: 0, + sample_rate_seen_hz: 0, + contrast: None, + contrast_error: None, + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAMonitorPlugin { + fn bump_generation(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn connect(&mut self) { + if self.worker.is_some() { + return; + } + match open_transport(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + } + Err(err) => { + self.last_error = Some(err); + } + } + self.bump_generation(); + } + + fn disconnect(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + worker.shutdown(reason); + } + self.connection = ConnectionState::Disconnected; + self.in_flight.clear(); + self.bump_generation(); + } + + fn start_acquisition(&mut self) { + self.queue_command( + "config", + Command::new("CONFIG") + .field("mode", "A1") + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", self.block_samples) + .field("raw", 1) + .field("summary", 1), + ); + self.queue_command("start", Command::new("START")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(true)); + } + } + + fn stop_acquisition(&mut self) { + self.queue_command("stop", Command::new("STOP").field("reason", "operator")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + + fn apply_drive(&mut self, params: &Value) { + let freq_mhz = params.get("freq_mhz").and_then(Value::as_i64).unwrap_or(0); + let center_dac = params + .get("center_dac") + .and_then(Value::as_i64) + .unwrap_or(2_048); + let amplitude_dac = params + .get("amplitude_dac") + .and_then(Value::as_i64) + .unwrap_or(0); + self.queue_command( + "drive", + Command::new("CONFIG") + .field("mode", "A1") + .field("wave", "SINE") + .field("freq_mhz", freq_mhz) + .field("center_dac", center_dac) + .field("amplitude_dac", amplitude_dac) + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", self.block_samples) + .field("raw", 1) + .field("summary", 1), + ); + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + if outputs.is_empty() { + return; + } + let mut changed = false; + let mut stopped: Option = None; + for output in outputs { + changed = true; + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) => { + self.last_error = Some(format!("{purpose}: {err}")); + } + } + } + WorkerOutput::Event(DeviceEvent::Data(frame)) => { + match frame.header.frame_type { + FrameType::SamplesU16 => { + if let Some(codes) = frame.samples() { + self.sample_rate_seen_hz = frame.header.sample_rate_hz; + self.push_samples(&codes, frame.header.first_sample_index); + } + } + FrameType::Summary | FrameType::Marker | FrameType::Control => {} + FrameType::Unknown(_) => {} + } + } + WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Integrity(integrity) => { + self.integrity = integrity; + } + WorkerOutput::Stopped { reason } => { + stopped = Some(reason); + } + } + } + if let Some(reason) = stopped { + self.worker = None; + self.connection = ConnectionState::Disconnected; + self.last_error = Some(format!("device connection ended: {reason}")); + } + if changed { + self.refresh_contrast(); + self.bump_generation(); + } + } + + fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { + match purpose { + "hello" => { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + self.connection = ConnectionState::Connected; + } + "start" => { + self.connection = ConnectionState::Acquiring; + } + "stop" => { + self.connection = ConnectionState::Connected; + } + _ => {} + } + } + + fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { + self.ring_next_sample_index = first_sample_index + codes.len() as u64; + self.sample_ring.extend_from_slice(codes); + let len = self.sample_ring.len(); + if len > SAMPLE_RING_CAPACITY { + self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); + } + } + + fn refresh_contrast(&mut self) { + if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { + return; + } + match estimate_contrast(&self.sample_ring, &self.calibration) { + Ok(estimate) => { + self.contrast = Some(estimate); + self.contrast_error = None; + } + Err(err) => { + self.contrast = None; + self.contrast_error = Some(err.to_string()); + } + } + } + + fn waveform_dataset(&self) -> Series1dV1 { + let rate = if self.sample_rate_seen_hz > 0 { + f64::from(self.sample_rate_seen_hz) + } else { + self.sample_rate_hz as f64 + }; + let n = self.sample_ring.len(); + let stride = (n / WAVEFORM_POINTS).max(1); + let first_index = self.ring_next_sample_index.saturating_sub(n as u64); + let points: Vec = self + .sample_ring + .iter() + .enumerate() + .step_by(stride) + .map(|(i, &code)| Series1dPoint { + x: (first_index + i as u64) as f64 / rate * 1_000.0, + y: self.calibration.code_to_volts(code), + }) + .collect(); + Series1dV1 { + x_label: "time [ms]".into(), + y_label: "photodiode [V]".into(), + lines: vec![Series1dLine { + name: "photodiode".into(), + points, + }], + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.connection) { + (Some(reason), _) => format!("locked ({reason})"), + (None, ConnectionState::Disconnected) => "disconnected".into(), + (None, ConnectionState::Connected) => "connected".into(), + (None, ConnectionState::Acquiring) => "acquiring".into(), + }; + let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { + (Some(estimate), _) => ( + format!("{:.4}", estimate.a), + format!( + "{:.2}% low / {:.2}% high", + estimate.low_clip_fraction * 100.0, + estimate.high_clip_fraction * 100.0 + ), + ), + (None, Some(err)) => ("invalid".into(), err.clone()), + (None, None) => ("—".into(), "—".into()), + }; + let integrity = if self.integrity.is_clean() { + "clean".to_owned() + } else { + format!( + "crc={} gaps={} skipped={} overruns={}", + self.integrity.crc_failures, + self.integrity.sequence_gaps, + self.integrity.skipped_bytes, + self.integrity.dropped_samples + ) + }; + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("firmware", self.firmware.clone()), + text_column("a", a_text), + text_column("clipping", clip_text), + text_column("integrity", integrity), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("firmware", "Firmware"), + column("a", "a = ln(Vmax/Vmin)"), + column("clipping", "Clipping"), + column("integrity", "Stream integrity"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec<(String, Value)> { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-monitor.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push((request.action_id, request.params)); + } + consumed + } +} + +fn open_transport(port_hint: &str) -> Result, String> { + let path = resolve_port(port_hint)?; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +fn resolve_port(port_hint: &str) -> Result { + if port_hint != "auto" { + return Ok(port_hint.to_owned()); + } + let ports = serial_ports(); + ports + .into_iter() + .next() + .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned()) +} + +fn serial_ports() -> Vec { + serialport_names() + .into_iter() + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() +} + +fn serialport_names() -> Vec { + stage_a_io::transport::available_port_names() +} + +impl Plugin for StageAMonitorPlugin { + fn name(&self) -> &'static str { + "Stage-A Monitor" + } + + fn description(&self) -> &'static str { + "Live Teensy photodiode readout with calibrated optical contrast and gated manual drive control (commissioning)." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect("plugin disabled"); + } + } + + fn reset(&mut self) { + self.sample_ring.clear(); + self.contrast = None; + self.contrast_error = None; + self.bump_generation(); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Fail closed: any pass without live-capture effects tears the + // connection down and refuses commands. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.worker.is_some() { + self.disconnect("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for (action_id, params) in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect("operator"), + ACTION_START => self.start_acquisition(), + ACTION_STOP => self.stop_acquisition(), + ACTION_APPLY_DRIVE => self.apply_drive(¶ms), + _ => {} + } + } + + self.drain_worker(); + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Device".into(), + description: Some( + "Serial DAQ configuration. Connect/start/stop are actions on the status \ + table, never settings — a reloaded settings file can't arm hardware." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Serial port".into(), + tooltip: Some("Teensy USB serial device (auto = first usbmodem)".into()), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "sample_rate_hz".into(), + label: "ADC sample rate".into(), + tooltip: Some("Commanded photodiode sample rate".into()), + kind: SettingKind::I64Slider { + min: 1_000, + max: 100_000, + default: self.sample_rate_hz, + suffix: Some(" Hz".into()), + }, + }, + SettingItem { + key: "dark_millivolts".into(), + label: "Dark level".into(), + tooltip: Some( + "Light-blocked photodiode level; a is computed from dark-corrected \ + voltages" + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 3_300.0, + speed: 1.0, + default: self.calibration.dark_volts * 1_000.0, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "sample_rate_hz" => Some(json!(self.sample_rate_hz)), + "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "sample_rate_hz" => { + self.sample_rate_hz = value + .as_i64() + .ok_or("sample_rate_hz must be an integer")? + .clamp(1_000, 100_000); + Ok(()) + } + "dark_millivolts" => { + let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; + self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + entries.push(StatusEntry::Text(match self.connection { + ConnectionState::Disconnected => "Teensy: disconnected".into(), + ConnectionState::Connected => format!("Teensy: connected ({})", self.firmware), + ConnectionState::Acquiring => format!( + "Teensy: acquiring at {} S/s", + if self.sample_rate_seen_hz > 0 { + self.sample_rate_seen_hz as i64 + } else { + self.sample_rate_hz + } + ), + })); + if let Some(estimate) = &self.contrast { + entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: WAVEFORM_DATASET_ID.into(), + title: "Photodiode waveform".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No photodiode samples yet — connect and start.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Stage-A monitor status".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Monitor idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: WAVEFORM_VIEW_ID.into(), + title: "Photodiode".into(), + dataset_id: WAVEFORM_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Monitor status".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + HostActionDescriptor { + id: ACTION_CONNECT.into(), + title: "Connect".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_DISCONNECT.into(), + title: "Disconnect".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_START.into(), + title: "Start acquisition".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_STOP.into(), + title: "Stop".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_APPLY_DRIVE.into(), + title: "Apply drive (expert)".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: serde_json::to_value(SettingsSchema { + sections: vec![SettingsSection { + label: "Drive".into(), + description: Some( + "Integer DAC drive codes — the optical contrast is measured \ + from the photodiode, never assumed from these values." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "freq_mhz".into(), + label: "Frequency".into(), + tooltip: Some("Drive frequency in millihertz".into()), + kind: SettingKind::I64Drag { + min: 0, + max: 200_000_000, + default: 1_000_000, + }, + }, + SettingItem { + key: "center_dac".into(), + label: "Center DAC code".into(), + tooltip: None, + kind: SettingKind::I64Drag { + min: 0, + max: 4_095, + default: 2_048, + }, + }, + SettingItem { + key: "amplitude_dac".into(), + label: "Amplitude DAC code".into(), + tooltip: None, + kind: SettingKind::I64Drag { + min: 0, + max: 2_047, + default: 0, + }, + }, + ], + }], + }) + .ok(), + }, + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), + _ => 0, + } + } +} + +impl Drop for StageAMonitorPlugin { + fn drop(&mut self) { + self.disconnect("plugin destroyed"); + } +} + +export_plugin!(StageAMonitorPlugin); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn waveform_dataset_decimates_and_calibrates() { + let mut plugin = StageAMonitorPlugin::default(); + plugin.sample_rate_seen_hz = 20_000; + plugin.push_samples(&vec![2_048_u16; 8_192], 0); + let dataset = plugin.waveform_dataset(); + assert_eq!(dataset.lines.len(), 1); + assert!(dataset.lines[0].points.len() <= WAVEFORM_POINTS + 1); + let volts = dataset.lines[0].points[0].y; + assert!((volts - 2_048.0 * 3.3 / 4_095.0).abs() < 1e-9); + } + + #[test] + fn sample_ring_is_bounded() { + let mut plugin = StageAMonitorPlugin::default(); + plugin.push_samples(&vec![1_u16; SAMPLE_RING_CAPACITY], 0); + plugin.push_samples(&vec![2_u16; 4_096], SAMPLE_RING_CAPACITY as u64); + assert_eq!(plugin.sample_ring.len(), SAMPLE_RING_CAPACITY); + assert_eq!(*plugin.sample_ring.last().unwrap(), 2); + } + + #[test] + fn status_dataset_matches_its_schema() { + let plugin = StageAMonitorPlugin::default(); + let dataset = plugin.status_dataset(); + let schema = plugin.status_schema(); + assert_eq!(dataset.columns.len(), schema.columns.len()); + for (data, column) in dataset.columns.iter().zip(&schema.columns) { + assert_eq!(data.column_id, column.id); + assert_eq!(data.len(), 1); + } + } +} diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index 27dea22..fc8482a 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -111,3 +111,17 @@ impl Transport for MockTransport { Ok(()) } } + +/// Names of serial ports visible to the OS (empty without the `hardware` +/// feature). Used by plugins to offer a port picker. +#[cfg(feature = "hardware")] +pub fn available_port_names() -> Vec { + serialport::available_ports() + .map(|ports| ports.into_iter().map(|p| p.port_name).collect()) + .unwrap_or_default() +} + +#[cfg(not(feature = "hardware"))] +pub fn available_port_names() -> Vec { + Vec::new() +} From 8f448c2c53a5fd277fe3177754db81df34dc2349 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:17:09 +0200 Subject: [PATCH 03/46] =?UTF-8?q?feat(stage-a-a1):=20=E2=9C=A8=20add=20A1?= =?UTF-8?q?=20minimum-depth=20Bode=20calibration=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Statistical core: phase folding (hardware EXT_TRIGGER fiducials or software clock-skew recovery via Rayleigh-power frequency scan), Rayleigh detection with Bonferroni-charged trials, median-background phase-locked excess, probit a_min fit with profile CI, hot-pixel mask from an unmodulated reference window. Sweep engine bisects the drive code, grids the bracketed transition, and records measured optical contrast per point. PDQ + sidecar + results export per run; fails closed on the ABI v5 execution context. --- Cargo.toml | 1 + plugins/stage-a-a1/Cargo.toml | 15 + plugins/stage-a-a1/README.md | 66 ++ plugins/stage-a-a1/plugin.toml | 7 + plugins/stage-a-a1/src/analysis.rs | 603 ++++++++++++++ plugins/stage-a-a1/src/lib.rs | 1224 ++++++++++++++++++++++++++++ plugins/stage-a-a1/src/sweep.rs | 363 +++++++++ 7 files changed, 2279 insertions(+) create mode 100644 plugins/stage-a-a1/Cargo.toml create mode 100644 plugins/stage-a-a1/README.md create mode 100644 plugins/stage-a-a1/plugin.toml create mode 100644 plugins/stage-a-a1/src/analysis.rs create mode 100644 plugins/stage-a-a1/src/lib.rs create mode 100644 plugins/stage-a-a1/src/sweep.rs diff --git a/Cargo.toml b/Cargo.toml index ca2a368..e2b3d5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "stage-a-io", "plugins/stage-a-monitor", + "plugins/stage-a-a1", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/plugins/stage-a-a1/Cargo.toml b/plugins/stage-a-a1/Cargo.toml new file mode 100644 index 0000000..fa4841d --- /dev/null +++ b/plugins/stage-a-a1/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "augur-plugin-stage-a-a1" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A A1 event-native Bode calibration: minimum-depth a_min(f) sweep with phase-locked detection." + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde_json.workspace = true +stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md new file mode 100644 index 0000000..48055cf --- /dev/null +++ b/plugins/stage-a-a1/README.md @@ -0,0 +1,66 @@ +# Stage-A A1 — minimum-depth Bode calibration + +Measures `a_min(f)`: the smallest optical log-contrast that still produces +phase-locked camera events, per drive frequency. `|H(f)| = C / a_min(f)`; +the knee of the curve is the pixel bandwidth `f_c(I)`, and the plateau of +`a_min` reads out the contrast quantum `C` (which seeds A3). Protocol +design: knowledge base `methodology/camera-calibration.md` (A1) and +`methodology/stage-a-control-software.md`. + +## How it decides "events just appeared" + +- **Detector — phase, not counts.** Background activity is uniform in + drive phase; modulation events are phase-locked. Each measurement window + is folded and tested with the **Rayleigh test**; background is discounted + automatically instead of subtracting a drifting absolute rate. +- **Cycle fiducial.** With the phase-0 TTL wired into `EXT_TRIGGER`, the + camera-clock edges from `frame.external_triggers()` mark each cycle. + Without the cable, the drive frequency is **refined against the events** + (Rayleigh-power scan over ±ppm around the commanded value — recovers the + Teensy↔camera clock skew); the scan multiplicity is Bonferroni-charged + to the significance threshold. +- **Estimator.** The mean phase-locked events per half-cycle comes from the + positive excess over the median phase-bin occupancy. +- **a_min is a fitted crossing.** The 0→1 step is smeared by shot-noise + first-passage randomness and per-pixel threshold dispersion, so a_min is + the fitted `N = 0.5` crossing of a probit in `ln a`, with a profile + confidence interval. The fitted transition width is a free preview of + the smear (σ_C + FPT). +- **Hot pixels.** An unmodulated reference window at run start builds a + median+5·MAD mask; masked pixels never enter the statistics, and the + mask size is recorded in the sidecar. +- **`a` is measured light.** Every point's contrast comes from the + photodiode ADC through the calibrated, clipping-guarded estimator in + `stage-a-io` — never from the commanded DAC code. Invalid windows + (clipping, CRC/sequence/overrun faults) are re-measured, never patched. + +## Run flow + +`Arm controller` → `Run A1 sweep`: reference window (hot-pixel mask) → +per frequency: bisection on the drive code until the detection boundary is +bracketed → log-spaced grid across the transition → probit fit → +next frequency. Views: `a_min(f)` with CI, live phase histogram, `N(a)` +staircase, run status. Raw PDA1 frames go to +`~/.augur/stage-a-runs/.pdq` with a JSON sidecar and a results +export; final numbers must be recomputed from the camera RAW + PDQ. + +ON and OFF are measured in **separate runs** (settings → Polarity) — the +comparator paths are asymmetric and must never be pooled. + +## Safety + +Fails closed on the ABI v5 execution context exactly like +`stage-a-monitor`: serial I/O only in the active live-capture worker; +Arm/Run/Stop are host actions, never settings; the firmware watchdog +drops to `SAFE_IDLE` independently of host cleanup. + +## Current limitations + +- The Teensy DDS/DAC firmware is still the ADC-only commissioning build — + closed-loop sweeps run against the protocol but the final stimulus + backend is blocked on the hardware freeze (see `stage-a-controller`). +- Marker cycles (periodic full-depth optical anchors) are specced for the + firmware but not yet emitted; the software frequency lock covers the + missing-trigger-cable case meanwhile. +- Measured sample cadence validation and the A5 refractory validity bound + `2fa/C ≪ 1/τ_refr` are recorded, not yet enforced. diff --git a/plugins/stage-a-a1/plugin.toml b/plugins/stage-a-a1/plugin.toml new file mode 100644 index 0000000..62f6089 --- /dev/null +++ b/plugins/stage-a-a1/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A A1 Min-Depth" +version = "0.2.0" +description = "Event-native Bode calibration: a_min(f) via phase-locked detection, drive bisection, and probit fitting." +domain = "stage-a" +library = "augur_plugin_stage_a_a1" +phase = "raw_events" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-a1/src/analysis.rs b/plugins/stage-a-a1/src/analysis.rs new file mode 100644 index 0000000..32cce7c --- /dev/null +++ b/plugins/stage-a-a1/src/analysis.rs @@ -0,0 +1,603 @@ +//! Statistical core of the A1 minimum-depth measurement. +//! +//! ## Why phase, not raw counts +//! +//! Background activity (BA) is uniform in modulation phase; genuine +//! modulation events are phase-locked to the drive. Testing for a +//! phase-locked component (Rayleigh test) therefore discounts uniform +//! background *automatically*, instead of requiring an absolute background +//! rate that drifts with temperature. "Mean events per half-cycle > 1" is +//! kept as the *estimator* (it is the quantity `⌊a·|H|/C⌋` predicts), but +//! the *detector* is the phase test. +//! +//! ## Cycle fiducials without the trigger cable +//! +//! With the phase-0 TTL wired, `frame.external_triggers()` marks each cycle +//! on the camera clock. Without it, the Teensy and camera clocks drift +//! (tens of ppm — folding dies after ~0.1 s at 10 kHz), so the drive +//! frequency is *refined against the events themselves*: scan a small +//! window around the commanded frequency and keep the value maximising the +//! Rayleigh power. The scan multiplicity is charged to the significance +//! test (Bonferroni). +//! +//! ## a_min as a fitted crossing +//! +//! Near threshold the 0→1 step of `⌊a·|H|/C⌋` is smeared by shot-noise +//! first-passage randomness and per-pixel threshold dispersion, so "events +//! just vanish" is not a crisp edge. a_min is defined as the fitted point +//! where the mean phase-locked events per half-cycle crosses 0.5, from a +//! probit-in-ln(a) fit over the transition, with a profile confidence +//! interval. The plateau of a_min(f) reads out the contrast quantum C. + +// --------------------------------------------------------------------------- +// Phase folding +// --------------------------------------------------------------------------- + +/// Folds event timestamps at `frequency_hz` relative to `t0_us`, +/// returning phases in `[0, 1)`. +pub fn fold_phases( + timestamps_us: impl Iterator, + t0_us: u64, + frequency_hz: f64, +) -> Vec { + let period_us = 1.0e6 / frequency_hz; + timestamps_us + .map(|t| { + let dt = t.saturating_sub(t0_us) as f64; + (dt / period_us).fract() + }) + .collect() +} + +/// Folds against explicit cycle-start fiducials (rising trigger edges): +/// each event's phase is its position inside the enclosing cycle. Events +/// before the first or after the last fiducial are dropped (their cycle +/// length is unknown). +pub fn fold_phases_with_fiducials(events: &[u64], cycle_starts_us: &[u64]) -> Vec { + if cycle_starts_us.len() < 2 { + return Vec::new(); + } + let mut phases = Vec::with_capacity(events.len()); + for &t in events { + let idx = match cycle_starts_us.binary_search(&t) { + Ok(i) => i, + Err(0) => continue, + Err(i) => i - 1, + }; + if idx + 1 >= cycle_starts_us.len() { + continue; + } + let start = cycle_starts_us[idx]; + let end = cycle_starts_us[idx + 1]; + if end <= start { + continue; + } + phases.push((t - start) as f64 / (end - start) as f64); + } + phases +} + +// --------------------------------------------------------------------------- +// Rayleigh test +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RayleighResult { + pub n: usize, + /// Resultant length in [0, 1]. + pub r: f64, + /// Z = n·R². + pub z: f64, + /// Approximate p-value under uniformity, `exp(-Z)` with the standard + /// small-sample correction (Zar / Wilkie). + pub p_value: f64, +} + +pub fn rayleigh_test(phases: &[f64]) -> RayleighResult { + let n = phases.len(); + if n == 0 { + return RayleighResult { + n, + r: 0.0, + z: 0.0, + p_value: 1.0, + }; + } + let (mut c, mut s) = (0.0_f64, 0.0_f64); + for &phase in phases { + let angle = 2.0 * std::f64::consts::PI * phase; + c += angle.cos(); + s += angle.sin(); + } + let r = (c * c + s * s).sqrt() / n as f64; + let z = n as f64 * r * r; + let nf = n as f64; + let p = (-z).exp() * (1.0 + (2.0 * z - z * z) / (4.0 * nf) + - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); + RayleighResult { + n, + r, + z, + p_value: p.clamp(0.0, 1.0), + } +} + +// --------------------------------------------------------------------------- +// Frequency refinement (clock-skew recovery without a trigger cable) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FrequencyLock { + pub frequency_hz: f64, + pub rayleigh: RayleighResult, + /// Number of candidate frequencies tested — multiply into the + /// significance threshold (Bonferroni). + pub trials: usize, +} + +/// Scans `±window_ppm` around `nominal_hz` and returns the frequency with +/// the maximum Rayleigh power. The step is chosen so consecutive candidates +/// dephase by ≤ 0.1 cycle over the observation span (finer is wasted). +pub fn refine_frequency( + timestamps_us: &[u64], + nominal_hz: f64, + window_ppm: f64, +) -> Option { + let (&first, &last) = (timestamps_us.first()?, timestamps_us.last()?); + let span_s = (last.saturating_sub(first)) as f64 / 1.0e6; + if span_s <= 0.0 { + return None; + } + let df_step = 0.1 / span_s; + let half_window_hz = nominal_hz * window_ppm * 1e-6; + let steps = ((half_window_hz / df_step).ceil() as i64).clamp(0, 5_000); + let mut best: Option = None; + let trials = (2 * steps + 1) as usize; + for k in -steps..=steps { + let f = nominal_hz + k as f64 * df_step; + if f <= 0.0 { + continue; + } + let phases = fold_phases(timestamps_us.iter().copied(), first, f); + let stat = rayleigh_test(&phases); + if best.as_ref().is_none_or(|b| stat.z > b.rayleigh.z) { + best = Some(FrequencyLock { + frequency_hz: f, + rayleigh: stat, + trials, + }); + } + } + best +} + +// --------------------------------------------------------------------------- +// Phase-locked excess (the events/half-cycle estimator) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseHistogram { + pub bins: Vec, + pub total: usize, +} + +pub fn phase_histogram(phases: &[f64], bin_count: usize) -> PhaseHistogram { + let mut bins = vec![0_u32; bin_count.max(1)]; + for &phase in phases { + let idx = ((phase * bins.len() as f64) as usize).min(bins.len() - 1); + bins[idx] += 1; + } + PhaseHistogram { + bins, + total: phases.len(), + } +} + +/// Estimates the phase-locked event count above the uniform background. +/// +/// The per-bin background is the *median* bin occupancy — robust because +/// the locked cluster occupies a minority of bins. Returns the summed +/// positive excess. Dividing by the number of observed cycles gives the +/// mean phase-locked events per cycle (per polarity: one burst per cycle). +pub fn phase_locked_excess(histogram: &PhaseHistogram) -> f64 { + if histogram.bins.is_empty() { + return 0.0; + } + let mut sorted = histogram.bins.clone(); + sorted.sort_unstable(); + let median = f64::from(sorted[sorted.len() / 2]); + histogram + .bins + .iter() + .map(|&count| (f64::from(count) - median).max(0.0)) + .sum() +} + +// --------------------------------------------------------------------------- +// Detection verdict for one (frequency, amplitude) measurement +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DetectionVerdict { + pub detected: bool, + pub p_value: f64, + /// Bonferroni-corrected significance threshold actually applied. + pub alpha_effective: f64, + /// Mean phase-locked events per cycle (per polarity), background-free. + pub locked_events_per_cycle: f64, +} + +/// Decides whether phase-locked modulation events are present. +/// +/// `alpha` is the per-measurement false-positive budget; `trials` is the +/// look-elsewhere multiplicity (frequency-scan candidates × bisection +/// steps), charged via Bonferroni. +pub fn detect( + rayleigh: RayleighResult, + excess: f64, + observed_cycles: f64, + alpha: f64, + trials: usize, +) -> DetectionVerdict { + let alpha_effective = alpha / trials.max(1) as f64; + DetectionVerdict { + detected: rayleigh.p_value < alpha_effective, + p_value: rayleigh.p_value, + alpha_effective, + locked_events_per_cycle: if observed_cycles > 0.0 { + excess / observed_cycles + } else { + 0.0 + }, + } +} + +// --------------------------------------------------------------------------- +// a_min fit: probit in ln(a) with profile CI +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct MinDepthFit { + /// a at which the mean locked events/half-cycle crosses 0.5. + pub a_min: f64, + /// Profile interval (Δ SSE ≤ SSE_min · (1 + 2/dof)); honest-but-cheap. + pub a_min_low: f64, + pub a_min_high: f64, + /// Transition width in ln(a) — first look at σ_C + FPT smear. + pub sigma_ln_a: f64, + pub points_used: usize, +} + +/// One measured amplitude point for the fit. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DepthPoint { + /// Measured optical log-contrast (photodiode, never the drive code). + pub a: f64, + /// Mean phase-locked events per half-cycle at this contrast. + pub events_per_half_cycle: f64, +} + +fn standard_normal_cdf(z: f64) -> f64 { + // Abramowitz & Stegun 7.1.26 via erf; |error| < 1.5e-7. + let x = z / std::f64::consts::SQRT_2; + let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); + let poly = t + * (0.254_829_592 + + t * (-0.284_496_736 + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); + let erf_abs = 1.0 - poly * (-x * x).exp(); + let erf = if x >= 0.0 { erf_abs } else { -erf_abs }; + 0.5 * (1.0 + erf) +} + +/// Fits `N(a) = Φ((ln a − μ)/σ)` over the transition region and reports +/// `a_min = e^μ` (the N = 0.5 crossing). Points far above the first step +/// (`N > 1.5`) are excluded — there the staircase's higher steps dominate +/// and the single-step model no longer applies. +pub fn fit_min_depth(points: &[DepthPoint]) -> Option { + let usable: Vec = points + .iter() + .copied() + .filter(|p| p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5) + .collect(); + if usable.len() < 3 { + return None; + } + let has_low = usable.iter().any(|p| p.events_per_half_cycle < 0.4); + let has_high = usable.iter().any(|p| p.events_per_half_cycle > 0.6); + if !has_low || !has_high { + return None; + } + + let ln_min = usable.iter().map(|p| p.a.ln()).fold(f64::INFINITY, f64::min); + let ln_max = usable + .iter() + .map(|p| p.a.ln()) + .fold(f64::NEG_INFINITY, f64::max); + + let sse = |mu: f64, sigma: f64| -> f64 { + usable + .iter() + .map(|p| { + let model = standard_normal_cdf((p.a.ln() - mu) / sigma); + let d = p.events_per_half_cycle.min(1.0) - model; + d * d + }) + .sum() + }; + + let mut best = (f64::INFINITY, ln_min, 0.1); + let mu_steps = 200; + for i in 0..=mu_steps { + let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; + for j in 0..40 { + let sigma = 0.005 * 1.2_f64.powi(j); // 0.005 .. ~7 in ln a + let value = sse(mu, sigma); + if value < best.0 { + best = (value, mu, sigma); + } + } + } + let (sse_min, mu_hat, sigma_hat) = best; + let dof = usable.len().saturating_sub(2).max(1) as f64; + let threshold = sse_min * (1.0 + 2.0 / dof) + 1e-12; + + // Profile over mu: the interval where some sigma keeps SSE under the + // threshold. + let mut low = mu_hat; + let mut high = mu_hat; + for i in 0..=mu_steps { + let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; + let feasible = (0..40).any(|j| { + let sigma = 0.005 * 1.2_f64.powi(j); + sse(mu, sigma) <= threshold + }); + if feasible { + low = low.min(mu); + high = high.max(mu); + } + } + + Some(MinDepthFit { + a_min: mu_hat.exp(), + a_min_low: low.exp(), + a_min_high: high.exp(), + sigma_ln_a: sigma_hat, + points_used: usable.len(), + }) +} + +// --------------------------------------------------------------------------- +// Hot-pixel mask (background is heavy-tailed; mask the tail, use the body) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct HotPixelMask { + width: u16, + masked: Vec, +} + +impl HotPixelMask { + /// Builds the mask from per-pixel counts of an *unmodulated* reference + /// window: pixels above `median + 5·MAD` (and above a small absolute + /// floor) are masked. The mask is fixed-pattern and belongs in the run + /// metadata, not just preprocessing. + pub fn from_reference_counts(width: u16, _height: u16, counts: &[u32]) -> Self { + let mut sorted: Vec = counts.to_vec(); + sorted.sort_unstable(); + let median = sorted.get(sorted.len() / 2).copied().unwrap_or(0) as f64; + let mut deviations: Vec = counts + .iter() + .map(|&count| (f64::from(count) - median).abs()) + .collect(); + deviations.sort_by(f64::total_cmp); + let mad = deviations.get(deviations.len() / 2).copied().unwrap_or(0.0); + let threshold = median + 5.0 * mad.max(0.5) + 2.0; + let masked = counts + .iter() + .map(|&count| f64::from(count) > threshold) + .collect(); + Self { width, masked } + } + + pub fn is_masked(&self, x: u16, y: u16) -> bool { + self.masked + .get(y as usize * self.width as usize + x as usize) + .copied() + .unwrap_or(false) + } + + pub fn masked_count(&self) -> usize { + self.masked.iter().filter(|&&m| m).count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Deterministic pseudo-uniform stream (splitmix64 → [0,1)). + struct UniformStream { + state: u64, + } + + impl UniformStream { + fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next(&mut self) -> f64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z = z ^ (z >> 31); + (z >> 11) as f64 / (1_u64 << 53) as f64 + } + + fn take(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.next()).collect() + } + } + + fn uniform_sequence(seed: u64, n: usize) -> Vec { + UniformStream::new(seed).take(n) + } + + /// Synthetic event stream: `per_cycle` phase-locked events per cycle at + /// `locked_phase` (jitter ±0.02) plus `background_rate_hz` uniform noise. + fn synthetic_events( + frequency_hz: f64, + duration_s: f64, + per_cycle: f64, + background_rate_hz: f64, + seed: u64, + ) -> Vec { + let cycles = (frequency_hz * duration_s) as usize; + let period_us = 1.0e6 / frequency_hz; + let mut stream = UniformStream::new(seed); + let mut next = move || stream.next(); + let mut events = Vec::new(); + for cycle in 0..cycles { + let base = cycle as f64 * period_us; + // Bernoulli(per_cycle fractional part) + floor. + let mut count = per_cycle.floor() as usize; + if next() < per_cycle.fract() { + count += 1; + } + for _ in 0..count { + let phase = 0.25 + (next() - 0.5) * 0.04; + events.push((base + phase * period_us) as u64); + } + } + let n_background = (background_rate_hz * duration_s) as usize; + for _ in 0..n_background { + events.push((next() * duration_s * 1.0e6) as u64); + } + events.sort_unstable(); + events + } + + #[test] + fn rayleigh_accepts_uniform_and_rejects_locked_phases() { + let uniform = uniform_sequence(7, 2_000); + let stat = rayleigh_test(&uniform); + assert!(stat.p_value > 0.01, "uniform phases: p={}", stat.p_value); + + let locked: Vec = uniform_sequence(11, 200) + .into_iter() + .map(|u| 0.3 + 0.02 * (u - 0.5)) + .collect(); + let stat = rayleigh_test(&locked); + assert!(stat.p_value < 1e-12, "locked phases: p={}", stat.p_value); + } + + #[test] + fn detection_discounts_uniform_background() { + // 0.8 locked events/cycle at 1 kHz for 0.5 s, drowned in 10x + // background rate: still detected via phase. + let events = synthetic_events(1_000.0, 0.5, 0.8, 8_000.0, 3); + let phases = fold_phases(events.iter().copied(), 0, 1_000.0); + let stat = rayleigh_test(&phases); + assert!(stat.p_value < 1e-6, "p={}", stat.p_value); + + // Background alone must NOT detect. + let noise_only = synthetic_events(1_000.0, 0.5, 0.0, 8_000.0, 5); + let phases = fold_phases(noise_only.iter().copied(), 0, 1_000.0); + let stat = rayleigh_test(&phases); + assert!(stat.p_value > 1e-3, "background-only p={}", stat.p_value); + } + + #[test] + fn phase_locked_excess_recovers_events_per_cycle() { + let frequency = 2_000.0; + let duration = 0.5; + let per_cycle = 0.6; + let events = synthetic_events(frequency, duration, per_cycle, 2_000.0, 9); + let phases = fold_phases(events.iter().copied(), 0, frequency); + let histogram = phase_histogram(&phases, 32); + let cycles = frequency * duration; + let recovered = phase_locked_excess(&histogram) / cycles; + assert!( + (recovered - per_cycle).abs() < 0.12, + "recovered {recovered} vs {per_cycle}" + ); + } + + #[test] + fn frequency_refinement_recovers_clock_skew() { + // Commanded 5 kHz, true (camera-clock) frequency 300 ppm higher — + // the naive fold dephases by 1.5 cycles over the 1 s span and + // collapses, while the refined lock recovers the true frequency. + let true_hz = 5_000.0 * (1.0 + 300e-6); + let events = synthetic_events(true_hz, 1.0, 1.0, 500.0, 13); + let lock = refine_frequency(&events, 5_000.0, 500.0).expect("lock found"); + let recovered_ppm = (lock.frequency_hz / 5_000.0 - 1.0) * 1e6; + // The scan step is 0.1/span = 0.1 Hz = 20 ppm at 5 kHz. + assert!( + (recovered_ppm - 300.0).abs() < 25.0, + "recovered {recovered_ppm} ppm" + ); + let naive = rayleigh_test(&fold_phases(events.iter().copied(), events[0], 5_000.0)); + assert!( + lock.rayleigh.z > naive.z * 5.0, + "lock z={} naive z={}", + lock.rayleigh.z, + naive.z + ); + } + + #[test] + fn fiducial_folding_matches_known_phase() { + let cycle_starts: Vec = (0..100).map(|k| k * 1_000).collect(); + let events: Vec = (0..99).map(|k| k * 1_000 + 250).collect(); + let phases = fold_phases_with_fiducials(&events, &cycle_starts); + assert_eq!(phases.len(), 99); + assert!(phases.iter().all(|p| (p - 0.25).abs() < 1e-9)); + } + + #[test] + fn min_depth_fit_recovers_the_crossing() { + // True a_min = 0.20, smear sigma = 0.15 in ln a. + let mu = 0.2_f64.ln(); + let points: Vec = (0..12) + .map(|i| { + let a = 0.08 * 1.25_f64.powi(i); // 0.08 .. ~0.9 + DepthPoint { + a, + events_per_half_cycle: standard_normal_cdf((a.ln() - mu) / 0.15), + } + }) + .collect(); + let fit = fit_min_depth(&points).expect("fit succeeds"); + assert!( + (fit.a_min - 0.2).abs() < 0.02, + "a_min={} (expected 0.20)", + fit.a_min + ); + assert!(fit.a_min_low <= fit.a_min && fit.a_min <= fit.a_min_high); + assert!((fit.sigma_ln_a - 0.15).abs() < 0.08); + } + + #[test] + fn min_depth_fit_requires_a_bracketed_transition() { + // All points fully above threshold: no crossing to fit. + let points: Vec = (0..6) + .map(|i| DepthPoint { + a: 0.5 + 0.1 * i as f64, + events_per_half_cycle: 1.0, + }) + .collect(); + assert!(fit_min_depth(&points).is_none()); + } + + #[test] + fn hot_pixel_mask_flags_the_tail_only() { + let mut counts = vec![2_u32; 64 * 64]; + counts[5] = 500; // hot + counts[700] = 300; // hot + let mask = HotPixelMask::from_reference_counts(64, 64, &counts); + assert_eq!(mask.masked_count(), 2); + assert!(mask.is_masked(5, 0)); + assert!(!mask.is_masked(6, 0)); + } +} diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs new file mode 100644 index 0000000..c1fd032 --- /dev/null +++ b/plugins/stage-a-a1/src/lib.rs @@ -0,0 +1,1224 @@ +//! Stage-A A1 — event-native Bode calibration, minimum-depth method. +//! +//! Measures `a_min(f)`: the smallest optical log-contrast that still +//! produces phase-locked events, per drive frequency. `|H(f)| = +//! C/a_min(f)`, the knee is `f_c(I)`, and the plateau of `a_min` reads out +//! the contrast quantum `C` (knowledge base: +//! `methodology/camera-calibration.md`, A1 protocol). +//! +//! Division of labour: +//! - `analysis` — phase folding, Rayleigh detection, frequency-skew +//! recovery, phase-locked excess, probit `a_min` fit, hot-pixel mask; +//! - `sweep` — the per-frequency bisection/grid state machine; +//! - this module — device I/O through `stage-a-io` (gated by the ABI v5 +//! execution context), camera-event intake, measurement windows, live +//! views, and the run sidecar. +//! +//! ON and OFF are measured **separately** (never pooled — the paths are +//! asymmetric); select the polarity in the settings and run each sweep. + +mod analysis; +mod sweep; + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, PluginInput, + Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, Command, DeviceEvent, FrameType, IoWorker, PdqWriter, + RunSidecar, StageAClient, StreamIntegrity, TriggerSource, WorkerOutput, WorkerRequest, +}; + +use analysis::{ + detect, fold_phases, fold_phases_with_fiducials, phase_histogram, phase_locked_excess, + rayleigh_test, refine_frequency, HotPixelMask, PhaseHistogram, +}; +use sweep::{Measurement, SweepCommand, SweepEngine, SweepPlan}; + +const AMIN_DATASET_ID: &str = "stage-a-a1.amin"; +const PHASE_DATASET_ID: &str = "stage-a-a1.phase"; +const DEPTH_DATASET_ID: &str = "stage-a-a1.depth"; +const STATUS_DATASET_ID: &str = "stage-a-a1.status"; + +const ACTION_ARM: &str = "stage-a-a1.arm"; +const ACTION_RUN: &str = "stage-a-a1.run"; +const ACTION_STOP: &str = "stage-a-a1.stop"; + +const PHASE_BINS: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RunState { + Idle, + Armed, + Reference, + Sweeping, + Finished, +} + +/// Camera-side accumulation for the current measurement window. +#[derive(Default)] +struct WindowAccumulator { + /// Camera timestamps of polarity-selected, hot-pixel-filtered events. + event_timestamps_us: Vec, + /// Rising phase-0 trigger edges (cycle fiducials) inside the window. + trigger_edges_us: Vec, + /// ADC codes streamed by the Teensy during the window. + adc_codes: Vec, + window_start_us: Option, + latest_camera_ts_us: u64, +} + +impl WindowAccumulator { + fn clear(&mut self) { + self.event_timestamps_us.clear(); + self.trigger_edges_us.clear(); + self.adc_codes.clear(); + self.window_start_us = None; + } + + fn elapsed_us(&self) -> u64 { + self.window_start_us + .map(|start| self.latest_camera_ts_us.saturating_sub(start)) + .unwrap_or(0) + } +} + +pub struct StageAA1Plugin { + enabled: bool, + state: RunState, + // device + worker: Option, + next_tag: u64, + in_flight: BTreeMap, + firmware: String, + integrity: StreamIntegrity, + last_error: Option, + effects_blocked_reason: Option, + // configuration (settings) + port_hint: String, + polarity_on: bool, + freq_start_hz: f64, + freq_stop_hz: f64, + points_per_decade: i64, + cycles_per_measurement: i64, + settle_ms: i64, + alpha: f64, + initial_amplitude_dac: i64, + sample_rate_hz: i64, + calibration: AdcCalibration, + // run + engine: Option, + window: WindowAccumulator, + settle_until_us: Option, + hot_pixels: Option, + reference_counts: Vec, + sensor_size: (u16, u16), + run_id: String, + pdq: Option, + current_phase_histogram: Option, + used_hardware_fiducial: bool, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAA1Plugin { + fn default() -> Self { + Self { + enabled: false, + state: RunState::Idle, + worker: None, + next_tag: 1, + in_flight: BTreeMap::new(), + firmware: String::new(), + integrity: StreamIntegrity::default(), + last_error: None, + effects_blocked_reason: None, + port_hint: "auto".into(), + polarity_on: true, + freq_start_hz: 100.0, + freq_stop_hz: 50_000.0, + points_per_decade: 6, + cycles_per_measurement: 400, + settle_ms: 100, + alpha: 0.001, + initial_amplitude_dac: 512, + sample_rate_hz: 20_000, + calibration: AdcCalibration::default(), + engine: None, + window: WindowAccumulator::default(), + settle_until_us: None, + hot_pixels: None, + reference_counts: Vec::new(), + sensor_size: (0, 0), + run_id: String::new(), + pdq: None, + current_phase_histogram: None, + used_hardware_fiducial: false, + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAA1Plugin { + fn bump(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn frequency_grid(&self) -> Vec { + let start = self.freq_start_hz.max(1.0); + let stop = self.freq_stop_hz.max(start * 1.01); + let per_decade = self.points_per_decade.max(1) as f64; + let decades = (stop / start).log10(); + let n = (decades * per_decade).ceil() as usize + 1; + (0..n) + .map(|i| start * 10f64.powf(i as f64 / per_decade)) + .filter(|&f| f <= stop * 1.0001) + .collect() + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn arm(&mut self) { + if self.worker.is_some() { + return; + } + match open_transport(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + self.state = RunState::Armed; + self.last_error = None; + } + Err(err) => self.last_error = Some(err), + } + self.bump(); + } + + fn start_run(&mut self) { + if self.worker.is_none() { + self.last_error = Some("run: arm the controller first".into()); + return; + } + self.run_id = format!( + "A1-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + ); + let pdq_path = run_data_dir().join(format!("{}.pdq", self.run_id)); + match PdqWriter::create(&pdq_path) { + Ok(writer) => self.pdq = Some(writer), + Err(err) => { + self.last_error = Some(format!("pdq: {err}")); + return; + } + } + let plan = SweepPlan { + frequencies_hz: self.frequency_grid(), + initial_amplitude_dac: self.initial_amplitude_dac.clamp(1, 2_047) as u32, + ..SweepPlan::default() + }; + self.engine = Some(SweepEngine::new(plan)); + self.reference_counts.clear(); + self.hot_pixels = None; + self.window.clear(); + self.settle_until_us = None; + // Reference phase: unmodulated field (amplitude 0) for the + // hot-pixel mask and the background sanity check. + self.send_drive(self.freq_start_hz, 0, "reference"); + self.state = RunState::Reference; + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(true)); + } + self.bump(); + } + + fn stop_run(&mut self, reason: &str) { + self.queue_command("stop", Command::new("STOP").field("reason", reason)); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + self.finish_run(); + self.state = if self.worker.is_some() { + RunState::Armed + } else { + RunState::Idle + }; + self.bump(); + } + + fn disarm(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + worker.shutdown(reason); + } + self.finish_run(); + self.in_flight.clear(); + self.state = RunState::Idle; + self.bump(); + } + + fn finish_run(&mut self) { + if let Some(pdq) = self.pdq.take() { + match pdq.finish(self.integrity) { + Ok(summary) => { + let mut sidecar = RunSidecar::from_pdq(&self.run_id, "A1", &summary); + sidecar.plugin_name = "stage-a-a1".into(); + sidecar.plugin_version = env!("CARGO_PKG_VERSION").into(); + sidecar.firmware_version = self.firmware.clone(); + sidecar.adc_calibration = self.calibration.clone(); + sidecar.configured_sample_rate_hz = self.sample_rate_hz as u32; + sidecar.trigger_source = if self.used_hardware_fiducial { + TriggerSource::DrivePhase0 + } else { + TriggerSource::None + }; + sidecar.valid = summary.valid; + if let Some(mask) = &self.hot_pixels { + sidecar + .notes + .push(format!("hot pixels masked: {}", mask.masked_count())); + } + let sidecar_path = run_data_dir().join(format!("{}.json", self.run_id)); + if let Err(err) = sidecar.write_json(&sidecar_path) { + self.last_error = Some(format!("sidecar: {err}")); + } + if let Some(engine) = &self.engine { + let results_path = + run_data_dir().join(format!("{}.results.json", self.run_id)); + let _ = std::fs::write( + &results_path, + serde_json::to_vec_pretty(&results_json(engine)).unwrap_or_default(), + ); + } + } + Err(err) => self.last_error = Some(format!("pdq finish: {err}")), + } + } + } + + fn send_drive(&mut self, frequency_hz: f64, amplitude_dac: u32, purpose: &str) { + let freq_mhz = (frequency_hz * 1_000.0).round() as i64; + self.queue_command( + purpose, + Command::new("CONFIG") + .field("mode", "A1") + .field("wave", "SINE") + .field("freq_mhz", freq_mhz) + .field("center_dac", 2_048) + .field("amplitude_dac", amplitude_dac) + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", 256) + .field("raw", 1) + .field("summary", 1), + ); + self.queue_command("start", Command::new("START")); + self.window.clear(); + self.settle_until_us = None; // set on the first camera frame seen + self.current_phase_histogram = None; + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + let mut stopped = None; + for output in outputs { + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => { + if purpose == "hello" { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + } + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + WorkerOutput::Event(DeviceEvent::Data(frame)) => { + if let Some(pdq) = &mut self.pdq { + let _ = pdq.write_frame(&frame); + } + if frame.header.frame_type == FrameType::SamplesU16 { + if let Some(codes) = frame.samples() { + self.window.adc_codes.extend_from_slice(&codes); + } + } + } + WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Integrity(integrity) => self.integrity = integrity, + WorkerOutput::Stopped { reason } => stopped = Some(reason), + } + } + if let Some(reason) = stopped { + self.worker = None; + self.last_error = Some(format!("device connection ended: {reason}")); + self.finish_run(); + self.state = RunState::Idle; + self.bump(); + } + } + + fn ingest_camera_frame(&mut self, frame: &PluginFrame<'_>) { + self.sensor_size = (frame.width(), frame.height()); + self.window.latest_camera_ts_us = frame.window_end_us(); + if self.settle_until_us.is_none() { + self.settle_until_us = + Some(frame.window_end_us() + (self.settle_ms.max(0) as u64) * 1_000); + return; + } + let settle_until = self.settle_until_us.unwrap_or(0); + if frame.window_end_us() < settle_until { + return; + } + self.window + .window_start_us + .get_or_insert(frame.window_start_us()); + + if self.state == RunState::Reference { + if self.reference_counts.len() + != frame.width() as usize * frame.height() as usize + { + self.reference_counts = + vec![0; frame.width() as usize * frame.height() as usize]; + } + for event in frame.events() { + let idx = event.y as usize * frame.width() as usize + event.x as usize; + if let Some(slot) = self.reference_counts.get_mut(idx) { + *slot += 1; + } + } + } else { + let mask = self.hot_pixels.as_ref(); + for event in frame.events() { + if event.is_on() != self.polarity_on { + continue; + } + if mask.is_some_and(|m| m.is_masked(event.x, event.y)) { + continue; + } + self.window.event_timestamps_us.push(event.timestamp_us()); + } + } + for trigger in frame.external_triggers() { + if trigger.is_rising() { + self.window.trigger_edges_us.push(trigger.timestamp_us); + } + } + } + + fn window_target_us(&self, frequency_hz: f64) -> u64 { + ((self.cycles_per_measurement.max(10) as f64 / frequency_hz) * 1.0e6) as u64 + } + + fn advance_run(&mut self) { + match self.state { + RunState::Reference => { + // A fixed 0.5 s of unmodulated reference. + if self.window.elapsed_us() < 500_000 { + return; + } + let (width, height) = self.sensor_size; + if width > 0 && !self.reference_counts.is_empty() { + self.hot_pixels = Some(HotPixelMask::from_reference_counts( + width, + height, + &self.reference_counts, + )); + } + self.state = RunState::Sweeping; + let Some(engine) = &self.engine else { + return; + }; + if let SweepCommand::Measure { + frequency_hz, + amplitude_dac, + } = engine.current_command() + { + self.send_drive(frequency_hz, amplitude_dac, "sweep"); + } + self.bump(); + } + RunState::Sweeping => { + let Some(engine) = &self.engine else { + return; + }; + let SweepCommand::Measure { + frequency_hz, + amplitude_dac, + } = engine.current_command() + else { + self.state = RunState::Finished; + self.finish_run(); + self.bump(); + return; + }; + if self.window.elapsed_us() < self.window_target_us(frequency_hz) { + return; + } + let measurement = self.evaluate_window(frequency_hz, amplitude_dac); + let next = { + let engine = self.engine.as_mut().expect("engine exists"); + engine.ingest(measurement) + }; + match next { + SweepCommand::Measure { + frequency_hz, + amplitude_dac, + } => self.send_drive(frequency_hz, amplitude_dac, "sweep"), + SweepCommand::Finished => { + self.queue_command("stop", Command::new("STOP").field("reason", "done")); + self.state = RunState::Finished; + self.finish_run(); + } + } + self.bump(); + } + _ => {} + } + } + + fn evaluate_window(&mut self, frequency_hz: f64, amplitude_dac: u32) -> Measurement { + // Optical contrast from the photodiode trace; any estimator + // rejection or stream fault invalidates the point. + let measured_a = if self.integrity.is_clean() { + estimate_contrast(&self.window.adc_codes, &self.calibration) + .ok() + .map(|estimate| estimate.a) + } else { + None + }; + + let events = &self.window.event_timestamps_us; + let observed_cycles = self.window.elapsed_us() as f64 / 1.0e6 * frequency_hz; + + // Cycle fiducial: hardware phase-0 edges when present, otherwise + // software frequency refinement against the events themselves. + let (phases, trials) = if self.window.trigger_edges_us.len() >= 2 { + self.used_hardware_fiducial = true; + ( + fold_phases_with_fiducials(events, &self.window.trigger_edges_us), + 1, + ) + } else if let Some(lock) = refine_frequency(events, frequency_hz, 100.0) { + ( + fold_phases( + events.iter().copied(), + events.first().copied().unwrap_or(0), + lock.frequency_hz, + ), + lock.trials, + ) + } else { + (Vec::new(), 1) + }; + + let stat = rayleigh_test(&phases); + let histogram = phase_histogram(&phases, PHASE_BINS); + let excess = phase_locked_excess(&histogram); + self.current_phase_histogram = Some(histogram); + let verdict = detect(stat, excess, observed_cycles, self.alpha, trials); + + Measurement { + amplitude_dac, + measured_a, + events_per_half_cycle: verdict.locked_events_per_cycle, + detected: verdict.detected, + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) + || !request.action_id.starts_with("stage-a-a1.") + { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } + + // -- datasets -------------------------------------------------------- + + fn amin_dataset(&self) -> Series1dV1 { + let mut a_min = Vec::new(); + let mut low = Vec::new(); + let mut high = Vec::new(); + if let Some(engine) = &self.engine { + for result in &engine.results { + if let Some(fit) = &result.fit { + a_min.push(Series1dPoint { + x: result.frequency_hz, + y: fit.a_min, + }); + low.push(Series1dPoint { + x: result.frequency_hz, + y: fit.a_min_low, + }); + high.push(Series1dPoint { + x: result.frequency_hz, + y: fit.a_min_high, + }); + } + } + } + Series1dV1 { + x_label: "drive frequency [Hz]".into(), + y_label: "a_min".into(), + lines: vec![ + Series1dLine { + name: "a_min".into(), + points: a_min, + }, + Series1dLine { + name: "CI low".into(), + points: low, + }, + Series1dLine { + name: "CI high".into(), + points: high, + }, + ], + } + } + + fn phase_dataset(&self) -> Series1dV1 { + let points = self + .current_phase_histogram + .as_ref() + .map(|histogram| { + histogram + .bins + .iter() + .enumerate() + .map(|(i, &count)| Series1dPoint { + x: (i as f64 + 0.5) / histogram.bins.len() as f64, + y: f64::from(count), + }) + .collect() + }) + .unwrap_or_default(); + Series1dV1 { + x_label: "drive phase [cycles]".into(), + y_label: "events".into(), + lines: vec![Series1dLine { + name: if self.polarity_on { "ON" } else { "OFF" }.into(), + points, + }], + } + } + + fn depth_dataset(&self) -> Series1dV1 { + let mut points: Vec = self + .engine + .as_ref() + .map(|engine| { + let mut all: Vec = engine + .results + .last() + .map(|result| { + result + .points + .iter() + .map(|p| Series1dPoint { + x: p.a, + y: p.events_per_half_cycle, + }) + .collect() + }) + .unwrap_or_default(); + all.sort_by(|p, q| p.x.total_cmp(&q.x)); + all + }) + .unwrap_or_default(); + points.dedup_by(|p, q| p.x == q.x); + Series1dV1 { + x_label: "measured a".into(), + y_label: "locked events / half-cycle".into(), + lines: vec![Series1dLine { + name: "N(a)".into(), + points, + }], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("progress", "Progress"), + column("fiducial", "Cycle fiducial"), + column("hot_pixels", "Hot pixels"), + column("integrity", "Integrity"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.state) { + (Some(reason), _) => format!("locked ({reason})"), + (None, RunState::Idle) => "idle".into(), + (None, RunState::Armed) => format!("armed ({})", self.firmware), + (None, RunState::Reference) => "reference window (hot-pixel mask)".into(), + (None, RunState::Sweeping) => "sweeping".into(), + (None, RunState::Finished) => "finished".into(), + }; + let progress = self + .engine + .as_ref() + .map(|engine| { + format!( + "{}/{} frequencies", + engine.results.len(), + engine.results.len() + + if engine.is_finished() { 0 } else { 1 } + ) + }) + .unwrap_or_else(|| "—".into()); + let fiducial = if self.used_hardware_fiducial { + "EXT_TRIGGER phase-0".to_owned() + } else { + "software frequency lock".to_owned() + }; + let hot = self + .hot_pixels + .as_ref() + .map(|mask| format!("{} masked", mask.masked_count())) + .unwrap_or_else(|| "—".into()); + let integrity = if self.integrity.is_clean() { + "clean".to_owned() + } else { + format!( + "crc={} gaps={} overruns={}", + self.integrity.crc_failures, + self.integrity.sequence_gaps, + self.integrity.dropped_samples + ) + }; + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("progress", progress), + text_column("fiducial", fiducial), + text_column("hot_pixels", hot), + text_column("integrity", integrity), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } +} + +fn results_json(engine: &SweepEngine) -> Value { + json!({ + "results": engine + .results + .iter() + .map(|result| { + json!({ + "frequency_hz": result.frequency_hz, + "exhausted": result.exhausted, + "measurements": result.measurements, + "fit": result.fit.as_ref().map(|fit| json!({ + "a_min": fit.a_min, + "a_min_low": fit.a_min_low, + "a_min_high": fit.a_min_high, + "sigma_ln_a": fit.sigma_ln_a, + "points_used": fit.points_used, + })), + "points": result + .points + .iter() + .map(|p| json!({"a": p.a, "events_per_half_cycle": p.events_per_half_cycle})) + .collect::>(), + }) + }) + .collect::>(), + }) +} + +fn run_data_dir() -> PathBuf { + let home = std::env::var_os("HOME").map(PathBuf::from).unwrap_or_default(); + home.join(".augur").join("stage-a-runs") +} + +fn open_transport(port_hint: &str) -> Result, String> { + let path = if port_hint == "auto" { + stage_a_io::transport::available_port_names() + .into_iter() + .find(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .ok_or_else(|| "no USB serial device found".to_owned())? + } else { + port_hint.to_owned() + }; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +impl Plugin for StageAA1Plugin { + fn name(&self) -> &'static str { + "Stage-A A1 Min-Depth" + } + + fn description(&self) -> &'static str { + "Event-native Bode calibration: a_min(f) via phase-locked detection, bisection, and probit fitting." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disarm("plugin disabled"); + } + } + + fn reset(&mut self) { + self.window.clear(); + self.current_phase_histogram = None; + self.bump(); + } + + fn input_kind(&self) -> PluginInput { + PluginInput::RawEvents + } + + fn process_frame( + &mut self, + frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = + Some(format!("effects not allowed in {:?}", execution.mode)); + if self.worker.is_some() { + self.disarm("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for action in self.consume_actions(context) { + match action.as_str() { + ACTION_ARM => self.arm(), + ACTION_RUN => self.start_run(), + ACTION_STOP => self.stop_run("operator"), + _ => {} + } + } + + self.drain_worker(); + if matches!(self.state, RunState::Reference | RunState::Sweeping) { + self.ingest_camera_frame(frame); + self.advance_run(); + } + } + + fn settings_schema(&self) -> SettingsSchema { + SettingsSchema { + sections: vec![ + SettingsSection { + label: "Sweep".into(), + description: Some( + "Frequency grid and statistics. ON and OFF are measured in separate \ + runs — never pooled." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "freq_start_hz".into(), + label: "Start frequency".into(), + tooltip: None, + kind: SettingKind::F64Drag { + min: 1.0, + max: 1.0e6, + speed: 10.0, + default: self.freq_start_hz, + }, + }, + SettingItem { + key: "freq_stop_hz".into(), + label: "Stop frequency".into(), + tooltip: None, + kind: SettingKind::F64Drag { + min: 1.0, + max: 1.0e6, + speed: 100.0, + default: self.freq_stop_hz, + }, + }, + SettingItem { + key: "points_per_decade".into(), + label: "Points per decade".into(), + tooltip: None, + kind: SettingKind::I64Slider { + min: 2, + max: 12, + default: self.points_per_decade, + suffix: None, + }, + }, + SettingItem { + key: "cycles_per_measurement".into(), + label: "Cycles per measurement".into(), + tooltip: Some( + "Modulation cycles integrated per amplitude point".into(), + ), + kind: SettingKind::I64Slider { + min: 50, + max: 5_000, + default: self.cycles_per_measurement, + suffix: None, + }, + }, + SettingItem { + key: "polarity_on".into(), + label: "Polarity".into(), + tooltip: Some("Which comparator path this sweep measures".into()), + kind: SettingKind::Enum { + variants: vec!["ON".into(), "OFF".into()], + default: usize::from(!self.polarity_on), + }, + }, + SettingItem { + key: "alpha".into(), + label: "Significance α".into(), + tooltip: Some( + "Per-measurement false-positive budget (Bonferroni-corrected \ + for the frequency scan)" + .into(), + ), + kind: SettingKind::F64Drag { + min: 1e-6, + max: 0.05, + speed: 1e-4, + default: self.alpha, + }, + }, + ], + }, + SettingsSection { + label: "Device".into(), + description: None, + default_open: false, + items: vec![ + SettingItem { + key: "initial_amplitude_dac".into(), + label: "Initial amplitude (DAC)".into(), + tooltip: None, + kind: SettingKind::I64Slider { + min: 1, + max: 2_047, + default: self.initial_amplitude_dac, + suffix: None, + }, + }, + SettingItem { + key: "settle_ms".into(), + label: "Settle time".into(), + tooltip: Some( + "Discarded after each drive change (HVA/Pockels settling + \ + refractory clearing)" + .into(), + ), + kind: SettingKind::I64Slider { + min: 10, + max: 2_000, + default: self.settle_ms, + suffix: Some(" ms".into()), + }, + }, + SettingItem { + key: "dark_millivolts".into(), + label: "Dark level".into(), + tooltip: None, + kind: SettingKind::F64Drag { + min: 0.0, + max: 3_300.0, + speed: 1.0, + default: self.calibration.dark_volts * 1_000.0, + }, + }, + ], + }, + ], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "freq_start_hz" => Some(json!(self.freq_start_hz)), + "freq_stop_hz" => Some(json!(self.freq_stop_hz)), + "points_per_decade" => Some(json!(self.points_per_decade)), + "cycles_per_measurement" => Some(json!(self.cycles_per_measurement)), + "polarity_on" => Some(json!(if self.polarity_on { "ON" } else { "OFF" })), + "alpha" => Some(json!(self.alpha)), + "initial_amplitude_dac" => Some(json!(self.initial_amplitude_dac)), + "settle_ms" => Some(json!(self.settle_ms)), + "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "freq_start_hz" => { + self.freq_start_hz = value.as_f64().ok_or("must be a number")?.max(1.0); + } + "freq_stop_hz" => { + self.freq_stop_hz = value.as_f64().ok_or("must be a number")?.max(1.0); + } + "points_per_decade" => { + self.points_per_decade = value.as_i64().ok_or("must be an integer")?.clamp(2, 12); + } + "cycles_per_measurement" => { + self.cycles_per_measurement = + value.as_i64().ok_or("must be an integer")?.clamp(50, 5_000); + } + "polarity_on" => { + let text = value.as_str().ok_or("must be a string")?; + self.polarity_on = text.eq_ignore_ascii_case("on"); + } + "alpha" => { + self.alpha = value.as_f64().ok_or("must be a number")?.clamp(1e-6, 0.05); + } + "initial_amplitude_dac" => { + self.initial_amplitude_dac = + value.as_i64().ok_or("must be an integer")?.clamp(1, 2_047); + } + "settle_ms" => { + self.settle_ms = value.as_i64().ok_or("must be an integer")?.clamp(10, 2_000); + } + "dark_millivolts" => { + let mv = value.as_f64().ok_or("must be a number")?; + self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); + } + _ => return Err(format!("unknown setting: {key}")), + } + Ok(()) + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + if let Some(engine) = &self.engine { + entries.push(StatusEntry::Text(format!( + "{} frequency points finished", + engine.results.len() + ))); + } + if let Some(err) = &self.last_error { + entries.push(StatusEntry::Text(format!("Error: {err}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: AMIN_DATASET_ID.into(), + title: "a_min(f)".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No fitted frequency points yet.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: PHASE_DATASET_ID.into(), + title: "Phase histogram".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No measurement window yet.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: DEPTH_DATASET_ID.into(), + title: "N(a) at current frequency".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No depth points yet.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "A1 run status".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: format!("{AMIN_DATASET_ID}.view"), + title: "A1 Bode (a_min)".into(), + dataset_id: AMIN_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: format!("{PHASE_DATASET_ID}.view"), + title: "Phase fold".into(), + dataset_id: PHASE_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: format!("{DEPTH_DATASET_ID}.view"), + title: "Depth staircase".into(), + dataset_id: DEPTH_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: format!("{STATUS_DATASET_ID}.view"), + title: "A1 status".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + HostActionDescriptor { + id: ACTION_ARM.into(), + title: "Arm controller".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_RUN.into(), + title: "Run A1 sweep".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_STOP.into(), + title: "Stop".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + AMIN_DATASET_ID => serde_json::to_vec(&self.amin_dataset()).ok(), + PHASE_DATASET_ID => serde_json::to_vec(&self.phase_dataset()).ok(), + DEPTH_DATASET_ID => serde_json::to_vec(&self.depth_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + AMIN_DATASET_ID | PHASE_DATASET_ID | DEPTH_DATASET_ID | STATUS_DATASET_ID => { + self.dataset_generation.max(1) + } + _ => 0, + } + } +} + +impl Drop for StageAA1Plugin { + fn drop(&mut self) { + self.disarm("plugin destroyed"); + } +} + +export_plugin!(StageAA1Plugin); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn frequency_grid_is_log_spaced_and_bounded() { + let mut plugin = StageAA1Plugin::default(); + plugin.freq_start_hz = 100.0; + plugin.freq_stop_hz = 10_000.0; + plugin.points_per_decade = 4; + let grid = plugin.frequency_grid(); + assert!((grid.first().copied().unwrap() - 100.0).abs() < 1e-9); + assert!(grid.last().copied().unwrap() <= 10_000.0 * 1.001); + assert_eq!(grid.len(), 9); + for pair in grid.windows(2) { + let ratio = pair[1] / pair[0]; + assert!((ratio - 10f64.powf(0.25)).abs() < 1e-9); + } + } + + #[test] + fn status_dataset_matches_schema() { + let plugin = StageAA1Plugin::default(); + let dataset = plugin.status_dataset(); + let schema = plugin.status_schema(); + assert_eq!(dataset.columns.len(), schema.columns.len()); + } +} diff --git a/plugins/stage-a-a1/src/sweep.rs b/plugins/stage-a-a1/src/sweep.rs new file mode 100644 index 0000000..1f48cc8 --- /dev/null +++ b/plugins/stage-a-a1/src/sweep.rs @@ -0,0 +1,363 @@ +//! Minimum-depth sweep state machine. +//! +//! For each frequency point: bisect on the integer DAC drive code until the +//! detection boundary is bracketed, then measure a small log-spaced grid +//! across the transition, then fit `a_min` (see `analysis::fit_min_depth`). +//! The engine is pure — device I/O and event analysis happen outside; it +//! only ingests finished measurements and emits the next drive request. +//! Note the asymmetry the whole design hinges on: the *search* variable is +//! the drive code, but every recorded point carries the **measured** +//! optical contrast `a` from the photodiode. + +use crate::analysis::{fit_min_depth, DepthPoint, MinDepthFit}; + +#[derive(Debug, Clone, PartialEq)] +pub struct SweepPlan { + pub frequencies_hz: Vec, + pub initial_amplitude_dac: u32, + pub max_amplitude_dac: u32, + /// Grid points measured across the bracket after bisection. + pub grid_points: usize, + /// Hard cap on measurements per frequency (bisection + grid). + pub max_measurements_per_frequency: usize, +} + +impl Default for SweepPlan { + fn default() -> Self { + Self { + frequencies_hz: Vec::new(), + initial_amplitude_dac: 512, + max_amplitude_dac: 2_047, + grid_points: 6, + max_measurements_per_frequency: 24, + } + } +} + +/// One finished measurement at the currently requested drive. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Measurement { + pub amplitude_dac: u32, + /// Photodiode-measured optical log-contrast. `None` = invalid window + /// (clipped / integrity fault) — the point is discarded and re-measured. + pub measured_a: Option, + pub events_per_half_cycle: f64, + pub detected: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SweepCommand { + /// Configure the drive and measure at these settings. + Measure { frequency_hz: f64, amplitude_dac: u32 }, + /// All frequencies finished. + Finished, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FrequencyResult { + pub frequency_hz: f64, + pub fit: Option, + pub points: Vec, + pub measurements: usize, + /// True when the point budget ran out before the transition was + /// bracketed — a_min is not identifiable from this data. + pub exhausted: bool, +} + +#[derive(Debug, Clone, PartialEq)] +enum Phase { + Bisecting, + Grid { queue: Vec }, +} + +pub struct SweepEngine { + plan: SweepPlan, + frequency_index: usize, + phase: Phase, + current_dac: u32, + measurements_at_frequency: usize, + /// Highest drive code that did NOT detect / lowest that did. + highest_undetected: Option, + lowest_detected: Option, + points: Vec, + invalid_retries: usize, + pub results: Vec, +} + +impl SweepEngine { + pub fn new(plan: SweepPlan) -> Self { + let current_dac = plan.initial_amplitude_dac; + Self { + plan, + frequency_index: 0, + phase: Phase::Bisecting, + current_dac, + measurements_at_frequency: 0, + highest_undetected: None, + lowest_detected: None, + points: Vec::new(), + invalid_retries: 0, + results: Vec::new(), + } + } + + pub fn current_command(&self) -> SweepCommand { + match self.plan.frequencies_hz.get(self.frequency_index) { + Some(&frequency_hz) => SweepCommand::Measure { + frequency_hz, + amplitude_dac: self.current_dac, + }, + None => SweepCommand::Finished, + } + } + + pub fn is_finished(&self) -> bool { + self.frequency_index >= self.plan.frequencies_hz.len() + } + + /// Ingests the finished measurement for the last `Measure` command and + /// advances the state machine. + pub fn ingest(&mut self, measurement: Measurement) -> SweepCommand { + if self.is_finished() { + return SweepCommand::Finished; + } + + let Some(a) = measurement.measured_a else { + // Invalid window: re-measure the same point (bounded retries), + // never silently keep the previous contrast. + self.invalid_retries += 1; + if self.invalid_retries > 3 { + self.finish_frequency(true); + } + return self.current_command(); + }; + self.invalid_retries = 0; + self.measurements_at_frequency += 1; + self.points.push(DepthPoint { + a, + events_per_half_cycle: measurement.events_per_half_cycle, + }); + + if measurement.detected { + self.lowest_detected = Some( + self.lowest_detected + .map_or(measurement.amplitude_dac, |d| d.min(measurement.amplitude_dac)), + ); + } else { + self.highest_undetected = Some( + self.highest_undetected + .map_or(measurement.amplitude_dac, |d| d.max(measurement.amplitude_dac)), + ); + } + + if self.measurements_at_frequency >= self.plan.max_measurements_per_frequency { + self.finish_frequency(!self.bracketed()); + return self.current_command(); + } + + match &mut self.phase { + Phase::Bisecting => { + if self.bracketed() { + let queue = self.grid_queue(); + self.phase = Phase::Grid { queue }; + self.advance_grid(); + } else if measurement.detected { + // Drive down toward the boundary. + let next = ((measurement.amplitude_dac as f64) * 0.65).round() as u32; + if next < 1 { + self.finish_frequency(false); + } else { + self.current_dac = next.max(1); + } + } else { + // Drive up toward the boundary. + let next = ((measurement.amplitude_dac as f64) * 1.5).ceil() as u32; + if next > self.plan.max_amplitude_dac { + // Even full drive shows nothing: unmeasurable point. + self.finish_frequency(true); + } else { + self.current_dac = next; + } + } + } + Phase::Grid { .. } => { + self.advance_grid(); + } + } + self.current_command() + } + + fn bracketed(&self) -> bool { + matches!( + (self.highest_undetected, self.lowest_detected), + (Some(_), Some(_)) + ) + } + + fn grid_queue(&self) -> Vec { + let (Some(low), Some(high)) = (self.highest_undetected, self.lowest_detected) else { + return Vec::new(); + }; + let lo = (low.min(high) as f64 * 0.8).max(1.0); + let hi = (low.max(high) as f64 * 1.25).min(self.plan.max_amplitude_dac as f64); + let n = self.plan.grid_points.max(2); + (0..n) + .map(|i| { + let t = i as f64 / (n - 1) as f64; + (lo * (hi / lo).powf(t)).round() as u32 + }) + .collect() + } + + fn advance_grid(&mut self) { + let next = match &mut self.phase { + Phase::Grid { queue } if !queue.is_empty() => Some(queue.remove(0)), + _ => None, + }; + match next { + Some(dac) => self.current_dac = dac, + None => self.finish_frequency(false), + } + } + + fn finish_frequency(&mut self, exhausted: bool) { + let frequency_hz = self.plan.frequencies_hz[self.frequency_index]; + let fit = if exhausted { + None + } else { + fit_min_depth(&self.points) + }; + self.results.push(FrequencyResult { + frequency_hz, + fit, + points: std::mem::take(&mut self.points), + measurements: self.measurements_at_frequency, + exhausted, + }); + self.frequency_index += 1; + self.phase = Phase::Bisecting; + self.current_dac = self.plan.initial_amplitude_dac; + self.measurements_at_frequency = 0; + self.highest_undetected = None; + self.lowest_detected = None; + self.invalid_retries = 0; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Simulated bench: optical contrast is proportional to the drive code + /// (a = dac / 2000) and the pixel responds with the smeared first step + /// around a_min = 0.2. + fn respond(dac: u32) -> Measurement { + let a = dac as f64 / 2_000.0; + let z = (a.ln() - 0.2_f64.ln()) / 0.12; + let n = 0.5 * (1.0 + erf_approx(z / std::f64::consts::SQRT_2)); + Measurement { + amplitude_dac: dac, + measured_a: Some(a), + events_per_half_cycle: n, + detected: n > 0.15, + } + } + + fn erf_approx(x: f64) -> f64 { + let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); + let poly = t + * (0.254_829_592 + + t * (-0.284_496_736 + + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); + let value = 1.0 - poly * (-x * x).exp(); + if x >= 0.0 { + value + } else { + -value + } + } + + #[test] + fn converges_to_the_synthetic_a_min() { + let mut engine = SweepEngine::new(SweepPlan { + frequencies_hz: vec![1_000.0, 10_000.0], + ..SweepPlan::default() + }); + + let mut guard = 0; + loop { + guard += 1; + assert!(guard < 200, "sweep must terminate"); + match engine.current_command() { + SweepCommand::Finished => break, + SweepCommand::Measure { amplitude_dac, .. } => { + engine.ingest(respond(amplitude_dac)); + } + } + } + + assert_eq!(engine.results.len(), 2); + for result in &engine.results { + let fit = result.fit.as_ref().expect("fit must exist"); + assert!( + (fit.a_min - 0.2).abs() < 0.04, + "f={} a_min={}", + result.frequency_hz, + fit.a_min + ); + assert!(!result.exhausted); + } + } + + #[test] + fn undetectable_frequency_is_reported_exhausted_not_fitted() { + let mut engine = SweepEngine::new(SweepPlan { + frequencies_hz: vec![100_000.0], + ..SweepPlan::default() + }); + let mut guard = 0; + loop { + guard += 1; + assert!(guard < 100); + match engine.current_command() { + SweepCommand::Finished => break, + SweepCommand::Measure { amplitude_dac, .. } => { + engine.ingest(Measurement { + amplitude_dac, + measured_a: Some(amplitude_dac as f64 / 2_000.0), + events_per_half_cycle: 0.0, + detected: false, + }); + } + } + } + assert_eq!(engine.results.len(), 1); + assert!(engine.results[0].exhausted); + assert!(engine.results[0].fit.is_none()); + } + + #[test] + fn invalid_windows_are_retried_then_abandoned() { + let mut engine = SweepEngine::new(SweepPlan { + frequencies_hz: vec![1_000.0], + ..SweepPlan::default() + }); + let mut measures = 0; + loop { + match engine.current_command() { + SweepCommand::Finished => break, + SweepCommand::Measure { amplitude_dac, .. } => { + measures += 1; + assert!(measures < 20); + engine.ingest(Measurement { + amplitude_dac, + measured_a: None, + events_per_half_cycle: 0.0, + detected: false, + }); + } + } + } + assert!(engine.results[0].exhausted); + } +} From 77e859a6ec5e54d7c000dc16611c78667ea56ba7 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:21:31 +0200 Subject: [PATCH 04/46] =?UTF-8?q?docs(stage-a):=20=F0=9F=93=9D=20add=20Sta?= =?UTF-8?q?ge-A=20feature=20brief=20and=20device-ownership=20ADR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature brief for the stage-a-io/monitor/a1 stack, ADR 005 for the device-ownership boundary (plugins own the Teensy, AugurRs stays generic), feature index entry. Note: the legacy plugins (localization/reconstruction/focus-metrics/ evesmlm) on this branch predate the current augur-rs plugin API and do not compile against it — their refresh is in progress on feature/eve-batch-findings; rebasing that work onto plugin ABI v5 only adds the new FfiPreviewFrame.external_triggers field in one test initializer. --- docs/adr/005-stage-a-device-ownership.md | 45 +++++++++++++++ docs/features/README.md | 1 + docs/features/stage-a.md | 72 ++++++++++++++++++++++++ plugins/stage-a-a1/src/analysis.rs | 17 ++++-- plugins/stage-a-a1/src/lib.rs | 14 ++--- plugins/stage-a-a1/src/sweep.rs | 17 ++++-- plugins/stage-a-monitor/src/lib.rs | 22 ++++---- 7 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 docs/adr/005-stage-a-device-ownership.md create mode 100644 docs/features/stage-a.md diff --git a/docs/adr/005-stage-a-device-ownership.md b/docs/adr/005-stage-a-device-ownership.md new file mode 100644 index 0000000..57bcec5 --- /dev/null +++ b/docs/adr/005-stage-a-device-ownership.md @@ -0,0 +1,45 @@ +# ADR 005 — Stage-A device ownership and the `stage-a-io` boundary + +- **Status:** Accepted +- **Date:** 2026-07-13 + +## Context + +The Stage-A camera calibrations (A1–A3) drive a Teensy stimulus/DAQ +controller over USB serial while recording the event camera. Someone has +to own the serial port, the experiment state machines, and the safety +rules. The knowledge-base control-software spec fixes the boundary: +AugurRs stays a generic camera recorder and plugin host and must not gain +laboratory-instrument abstractions. + +## Decision + +1. **Device control lives in removable protocol plugins** (`stage-a-monitor`, + `stage-a-a1`, later `-a2`/`-a3`), one experiment concern per plugin. + Exactly one enabled, armed plugin owns the serial port; opening a busy + device is a visible error. +2. **A shared plain-Rust library `stage-a-io`** (this repo, not a plugin) + owns everything protocol-shaped: PDA1 framing + CRC resync, the ASCII + command grammar with idempotent sequence retries, the bounded I/O + worker, `.pdq` persistence, the run sidecar, and the calibrated optical + contrast estimator. It contains **no experiment policy** (sweeps, + bisection, fits stay in the plugins) and **no augur types** (testable + without a host). +3. **Effects are gated by the host's execution context** (plugin ABI v5): + plugins fail closed unless `LiveCapture && effects_allowed`. Hardware + commands are host actions, never persistent settings. +4. **Wire compatibility is anchored to the firmware header** + (`stage-a-controller/include/wire_protocol.h`); `stage-a-io` mirrors it + with layout tests, and the mock controller implements the same + idempotency contract the firmware promises. + +## Consequences + +- A2/A3 plugins reuse `stage-a-io` unchanged; only their state machines + and views are new code. +- The GUI knows nothing about Teensys; removing the three plugins removes + every trace of lab hardware from the product. +- Protocol changes must land in the firmware header first, then in + `stage-a-io`, keeping a single source of truth for the wire format. +- Plugins depend on `stage-a-io` by path; it is versioned with the + workspace and its API may still move until A2/A3 land. diff --git a/docs/features/README.md b/docs/features/README.md index b39d965..7127a0a 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -4,6 +4,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs +- [Stage-A Calibration Plugins](./stage-a.md) — Teensy-driven Stage-A bench stack: `stage-a-io` shared I/O, commissioning monitor, and the A1 minimum-depth Bode sweep. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Runtime Migration Notes](./plugin-api-v0-2.md) — historical runtime-migration brief, updated with the current interface additions that matter to this repo. - [Plugin Host Views](./plugin-host-views.md) — generic host-rendered datasets, cache generations, and shared view ids. diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md new file mode 100644 index 0000000..7924c6b --- /dev/null +++ b/docs/features/stage-a.md @@ -0,0 +1,72 @@ +# Stage-A calibration plugins (`stage-a-io`, `stage-a-monitor`, `stage-a-a1`) + +> Feature brief — first delivery of the Stage-A camera-calibration stack. +> Design source of truth: knowledge base +> `methodology/stage-a-control-software.md` and +> `methodology/camera-calibration.md` (A1 protocol). + +## Architecture + +```text +AugurRs generic host (camera, RAW, EXT_TRIGGER delivery, execution context — ABI v5) + │ + ├── stage-a-monitor — commissioning: live photodiode view, manual control + └── stage-a-a1 — A1 minimum-depth a_min(f) sweep + │ (exactly one armed plugin owns the device) + ▼ + stage-a-io (this repo, plain lib) ── USB serial ── Teensy stage-a-controller +``` + +AugurRs itself gains no Teensy or serial abstraction — device ownership +lives entirely in these removable plugins (ADR 005). + +## Crates + +| Crate | Role | +|---|---| +| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, mock controller | +| `plugins/stage-a-monitor` | Live decimated waveform, live `a`, integrity status, gated manual CONFIG/START/STOP + expert drive modal | +| `plugins/stage-a-a1` | Phase-locked detection (Rayleigh), hardware/software cycle fiducials, bisection + grid sweep, probit `a_min` fit with CI, hot-pixel mask, PDQ + sidecar + results export | + +## Safety model + +- Serial ports open only when `HostContext::execution()` reports + `LiveCapture` **and** `effects_allowed` (host constructs this fail-closed; + only the active live-capture worker qualifies). Replay can never re-arm + hardware, even from a sidecar that contains a runnable setup. +- All hardware commands are host **actions**; persistent settings never + start hardware after a reload. +- Any CRC error, frame-sequence gap, or ADC overrun invalidates the + measurement point; invalid points are re-measured, never patched, and + the run sidecar records the counters. +- The firmware watchdog (1.5 s) drops the controller to `SAFE_IDLE` + independently of host-side cleanup. + +## Statistics (A1) + +Detection is a phase-uniformity test (background activity is uniform in +drive phase; signal is phase-locked), with the frequency-scan multiplicity +Bonferroni-charged when the software clock-skew lock substitutes for the +missing trigger cable. `a_min` is the fitted `N = 0.5` crossing of a +probit in `ln a` with a profile CI — not a raw bisection endpoint — and +`a` is always the photodiode-measured contrast. Details and rationale: +`plugins/stage-a-a1/README.md`. + +## Verification + +`cargo test` (38 tests): wire fragmentation/CRC-resync/overrun, retry +idempotency against the mock controller, worker round-trip + clean STOP, +estimator recovery/clipping/headroom guards, Rayleigh calibration on +uniform and locked phases, background-immunity, clock-skew recovery +(300 ppm), fiducial folding, probit fit recovery, sweep convergence to a +synthetic `a_min`, exhaustion/invalid-window handling, hot-pixel masking, +dataset/schema consistency. + +## Known gaps + +- Final Teensy DDS/DAC firmware is blocked on the hardware freeze; the + sweep runs against the v1 protocol and the mock meanwhile. +- Marker cycles are protocol-reserved but not yet emitted + (`stage-a-controller/docs/features/a1-marker-cycles.md`). +- `stage-a-a2` / `stage-a-a3` plugins are not yet implemented; A2 + additionally requires the physical trigger cable. diff --git a/plugins/stage-a-a1/src/analysis.rs b/plugins/stage-a-a1/src/analysis.rs index 32cce7c..0c3ce31 100644 --- a/plugins/stage-a-a1/src/analysis.rs +++ b/plugins/stage-a-a1/src/analysis.rs @@ -112,8 +112,9 @@ pub fn rayleigh_test(phases: &[f64]) -> RayleighResult { let r = (c * c + s * s).sqrt() / n as f64; let z = n as f64 * r * r; let nf = n as f64; - let p = (-z).exp() * (1.0 + (2.0 * z - z * z) / (4.0 * nf) - - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); + let p = (-z).exp() + * (1.0 + (2.0 * z - z * z) / (4.0 * nf) + - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); RayleighResult { n, r, @@ -283,7 +284,8 @@ fn standard_normal_cdf(z: f64) -> f64 { let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); let poly = t * (0.254_829_592 - + t * (-0.284_496_736 + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); + + t * (-0.284_496_736 + + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); let erf_abs = 1.0 - poly * (-x * x).exp(); let erf = if x >= 0.0 { erf_abs } else { -erf_abs }; 0.5 * (1.0 + erf) @@ -297,7 +299,9 @@ pub fn fit_min_depth(points: &[DepthPoint]) -> Option { let usable: Vec = points .iter() .copied() - .filter(|p| p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5) + .filter(|p| { + p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5 + }) .collect(); if usable.len() < 3 { return None; @@ -308,7 +312,10 @@ pub fn fit_min_depth(points: &[DepthPoint]) -> Option { return None; } - let ln_min = usable.iter().map(|p| p.a.ln()).fold(f64::INFINITY, f64::min); + let ln_min = usable + .iter() + .map(|p| p.a.ln()) + .fold(f64::INFINITY, f64::min); let ln_max = usable .iter() .map(|p| p.a.ln()) diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs index c1fd032..379a7f8 100644 --- a/plugins/stage-a-a1/src/lib.rs +++ b/plugins/stage-a-a1/src/lib.rs @@ -403,11 +403,8 @@ impl StageAA1Plugin { .get_or_insert(frame.window_start_us()); if self.state == RunState::Reference { - if self.reference_counts.len() - != frame.width() as usize * frame.height() as usize - { - self.reference_counts = - vec![0; frame.width() as usize * frame.height() as usize]; + if self.reference_counts.len() != frame.width() as usize * frame.height() as usize { + self.reference_counts = vec![0; frame.width() as usize * frame.height() as usize]; } for event in frame.events() { let idx = event.y as usize * frame.width() as usize + event.x as usize; @@ -715,8 +712,7 @@ impl StageAA1Plugin { format!( "{}/{} frequencies", engine.results.len(), - engine.results.len() - + if engine.is_finished() { 0 } else { 1 } + engine.results.len() + if engine.is_finished() { 0 } else { 1 } ) }) .unwrap_or_else(|| "—".into()); @@ -786,7 +782,9 @@ fn results_json(engine: &SweepEngine) -> Value { } fn run_data_dir() -> PathBuf { - let home = std::env::var_os("HOME").map(PathBuf::from).unwrap_or_default(); + let home = std::env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_default(); home.join(".augur").join("stage-a-runs") } diff --git a/plugins/stage-a-a1/src/sweep.rs b/plugins/stage-a-a1/src/sweep.rs index 1f48cc8..36f0a25 100644 --- a/plugins/stage-a-a1/src/sweep.rs +++ b/plugins/stage-a-a1/src/sweep.rs @@ -48,7 +48,10 @@ pub struct Measurement { #[derive(Debug, Clone, PartialEq)] pub enum SweepCommand { /// Configure the drive and measure at these settings. - Measure { frequency_hz: f64, amplitude_dac: u32 }, + Measure { + frequency_hz: f64, + amplitude_dac: u32, + }, /// All frequencies finished. Finished, } @@ -139,14 +142,16 @@ impl SweepEngine { }); if measurement.detected { - self.lowest_detected = Some( - self.lowest_detected - .map_or(measurement.amplitude_dac, |d| d.min(measurement.amplitude_dac)), - ); + self.lowest_detected = + Some(self.lowest_detected.map_or(measurement.amplitude_dac, |d| { + d.min(measurement.amplitude_dac) + })); } else { self.highest_undetected = Some( self.highest_undetected - .map_or(measurement.amplitude_dac, |d| d.max(measurement.amplitude_dac)), + .map_or(measurement.amplitude_dac, |d| { + d.max(measurement.amplitude_dac) + }), ); } diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs index f01c1f7..ab159aa 100644 --- a/plugins/stage-a-monitor/src/lib.rs +++ b/plugins/stage-a-monitor/src/lib.rs @@ -26,8 +26,8 @@ use augur_plugin_api::{ }; use serde_json::{json, Value}; use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, - IoWorker, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, + estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, + StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, }; const WAVEFORM_DATASET_ID: &str = "stage-a-monitor.waveform"; @@ -223,18 +223,16 @@ impl StageAMonitorPlugin { } } } - WorkerOutput::Event(DeviceEvent::Data(frame)) => { - match frame.header.frame_type { - FrameType::SamplesU16 => { - if let Some(codes) = frame.samples() { - self.sample_rate_seen_hz = frame.header.sample_rate_hz; - self.push_samples(&codes, frame.header.first_sample_index); - } + WorkerOutput::Event(DeviceEvent::Data(frame)) => match frame.header.frame_type { + FrameType::SamplesU16 => { + if let Some(codes) = frame.samples() { + self.sample_rate_seen_hz = frame.header.sample_rate_hz; + self.push_samples(&codes, frame.header.first_sample_index); } - FrameType::Summary | FrameType::Marker | FrameType::Control => {} - FrameType::Unknown(_) => {} } - } + FrameType::Summary | FrameType::Marker | FrameType::Control => {} + FrameType::Unknown(_) => {} + }, WorkerOutput::Event(DeviceEvent::Async { .. }) => {} WorkerOutput::Integrity(integrity) => { self.integrity = integrity; From d6c6f78cf02540149e87f446dd388ed103475906 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 22:51:11 +0200 Subject: [PATCH 05/46] =?UTF-8?q?feat(evesmlm,plugins):=20=E2=9C=A8=20batc?= =?UTF-8?q?h=20review=20findings=20and=20current=20plugin-API=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the evesmlm suite, localization, and reconstruction plugins plus repo docs to the current augur-rs plugin interface (host-view actions, dataset display metadata/relations, TableSchema extensions, CompactEvent accessors). --- CONTRIBUTING.md | 20 + README.md | 31 +- .../005-investigation-workspace-datasets.md | 43 + docs/architecture.md | 16 +- docs/features/README.md | 6 + docs/features/action-requests-and-refit.md | 93 + .../features/clickable-overlays-source-row.md | 46 + docs/features/evesmlm-temporal-diagnostics.md | 65 + docs/features/evesmlm.md | 23 +- .../investigation-workspace-alignment.md | 79 + docs/features/plugin-host-views.md | 15 +- docs/features/plugin-install-reload.md | 40 + docs/features/reconstruction.md | 17 +- docs/features/tablev1-declarative-metadata.md | 59 + docs/installing-plugins.md | 20 + docs/plugin-api.md | 221 +- plugin-template/README.md | 9 + plugins/evesmlm-candidates/README.md | 23 +- .../evesmlm-candidates/src/eigenfeature.rs | 37 +- plugins/evesmlm-candidates/src/lib.rs | 1284 ++++++++- plugins/evesmlm-candidates/src/types.rs | 44 +- plugins/evesmlm-fitting/README.md | 18 +- plugins/evesmlm-fitting/src/lib.rs | 2534 +++++++++++++++-- plugins/evesmlm-fitting/src/types.rs | 46 + plugins/evesmlm-postproc/README.md | 13 +- plugins/evesmlm-postproc/src/lib.rs | 71 +- plugins/localization/src/lib.rs | 4 +- plugins/reconstruction/README.md | 5 +- plugins/reconstruction/src/lib.rs | 119 +- scripts/install-built-plugins.sh | 19 +- 30 files changed, 4615 insertions(+), 405 deletions(-) create mode 100644 docs/adr/005-investigation-workspace-datasets.md create mode 100644 docs/features/action-requests-and-refit.md create mode 100644 docs/features/clickable-overlays-source-row.md create mode 100644 docs/features/evesmlm-temporal-diagnostics.md create mode 100644 docs/features/investigation-workspace-alignment.md create mode 100644 docs/features/plugin-install-reload.md create mode 100644 docs/features/tablev1-declarative-metadata.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7edc25d..af41406 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -137,6 +137,18 @@ Plugins do not render `egui` directly. Instead, expose: The host owns rendering, export, caching, and window state for declared host views. +When a table dataset should participate in the linked investigation workspace, also populate the additive metadata the host can use: + +- `coordinate_space_2d` +- `coordinate_space_3d` +- `row_id_column` +- `time_column` +- `layer_id` +- `semantic_label` +- `HostDatasetDescriptor.display` + +Prefer structured datasets for selection/linking and use overlays only for supplemental 2D annotations or hit-testing. + ### 7. Write `plugin.toml` Use the runtime format: @@ -160,6 +172,14 @@ cp plugins/my-plugin/plugin.toml ~/.augur/plugins/my-plugin/ cp target/release/libaugur_plugin_my_plugin.dylib ~/.augur/plugins/my-plugin/ ``` +On macOS, either run `./scripts/install-built-plugins.sh --profile release` instead of the manual +copy steps or rewrite the installed dylib id yourself: + +```bash +install_name_tool -id "@loader_path/libaugur_plugin_my_plugin.dylib" \ + ~/.augur/plugins/my-plugin/libaugur_plugin_my_plugin.dylib +``` + Then open `augur-gui`, go to **Plugins**, click **Scan for New Plugins**, and enable the plugin. ## Migrating Older Plugins diff --git a/README.md b/README.md index 40cb651..4d48dd1 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,27 @@ Use this repository for the plugin implementations, template crate, and repo-loc ## Runtime Model - Each plugin ships as a `plugin.toml` manifest plus one platform library (`.dylib`, `.so`, or `.dll`). -- `augur-gui` discovers plugins from `~/.augur/plugins/`, loads the exported `augur_plugin_vtable`, and renders settings, status, and host views through the host. +- `augur-gui` discovers plugins from `~/.augur/plugins/`, loads the exported `augur_plugin_vtable`, and renders settings, status, and linked investigation datasets/views through the host. - Host-owned built-in tools stay in `augur-gui`; they are not runtime plugins in this repository. - Host-owned experiment settings such as pixel scale, sensor geometry, acquisition time, and EventStore budget are published to plugins as `GlobalSettings` on `augur.global_settings`. - Standard shared scientific payloads can also live in companion crates such as `augur-plugin-types`. +## Investigation Workspace Contract + +The host now owns a generic linked workspace across: + +- 2D preview +- 3D inspection +- host-rendered tables + +For plugins, that means: + +- structured datasets are the primary linking mechanism +- stable row ids should be provided when possible +- 2D/3D coordinate metadata should be declared when the plugin has it +- layer/display metadata should describe visibility, color, marker shape, and size +- overlays are supplemental annotations, not the primary integration surface + ## In-Tree Runtime Plugins (work in progress) The plugin crates under `plugins/` are under active development and not yet ready for external use. The template crate and documentation are stable references for writing your own plugins. @@ -37,11 +53,11 @@ The plugin crates under `plugins/` are under active development and not yet read | Plugin | Phase | Notes | |---|---|---| | `localization` | `RawEvents` | Wavelet/Gaussian SMLM localization and standard `LocalizationResults` output | -| `reconstruction` | `DerivedData` | Accumulated localization table plus host-rendered reconstruction windows | +| `reconstruction` | `DerivedData` | Accumulated localization dataset with stable ids, time metadata, density rendering, and 3D inspection | | `focus-metrics` | `DerivedData` | Focus metrics from localization results or FFT preview sharpness | -| `evesmlm-candidates` | `RawEvents` | Event-domain candidate clustering for eveSMLM | -| `evesmlm-fitting` | `DerivedData` | Candidate fitting plus EVE and compatibility localization outputs | -| `evesmlm-postproc` | `DerivedData` | Filtering, drift correction, evaluation, and the later EVE compact view provider | +| `evesmlm-candidates` | `RawEvents` | Event-domain candidate clustering plus accepted/rejected raw-event investigation layers | +| `evesmlm-fitting` | `DerivedData` | Candidate fitting plus shared current-localization datasets, stable ids, and linked 3D inspection | +| `evesmlm-postproc` | `DerivedData` | Filtering, drift correction, evaluation, and the later shared EVE current-localization provider | `plugin-template/` is the starting point for new plugin crates. @@ -62,6 +78,8 @@ cp target/release/libaugur_plugin_localization.dylib ~/.augur/plugins/localizati ``` On Linux, copy the `.so`. On Windows, copy the `.dll`. +On macOS, prefer `./scripts/install-built-plugins.sh --profile release`; it rewrites the copied +plugin dylib id so Plugin Manager reloads do not keep pointing at Cargo's build tree. Then open `augur-gui`, go to **Plugins**, click **Scan for New Plugins**, and enable the plugin. @@ -101,7 +119,8 @@ The current authoring flow is: 2. export the vtable with `export_plugin!` 3. choose `input_kind()` and optional `PluginCapabilities` 4. use `HostContext` for shared payloads, companion crates such as `augur-plugin-types` for reusable payload types, and `CTX_GLOBAL_SETTINGS` for host-owned calibration/settings -5. declare host-rendered outputs with `host_views()` when needed +5. declare host-rendered outputs with `host_views()` when needed and populate stable-id / coordinate / layer metadata when the dataset should participate in linked investigation + - to expose interactive operations, append `HostActionDescriptor`s to `HostViewRegistry.actions` (scope `Dataset`/`Row`/`Cluster`, optional `param_schema`); consume requests from the persistent context key `CTX_INVESTIGATION_ACTION_REQUESTS` 6. build a `cdylib` 7. install `plugin.toml` plus the compiled library into `~/.augur/plugins//` diff --git a/docs/adr/005-investigation-workspace-datasets.md b/docs/adr/005-investigation-workspace-datasets.md new file mode 100644 index 0000000..03754e7 --- /dev/null +++ b/docs/adr/005-investigation-workspace-datasets.md @@ -0,0 +1,43 @@ +# ADR 005: Expose Generic Investigation Datasets From Plugins + +## Status + +Accepted + +## Context + +`augur-gui` now owns a generic linked investigation workspace across 2D preview, 3D inspection, and host-rendered tables. + +That host model depends on richer plugin-side dataset metadata than the older window-centric host-view integration used: + +- stable row ids +- optional 2D and 3D coordinates +- optional time columns +- layer ids and display metadata + +The eveSMLM pipeline also needs stage-local investigation surfaces for tuning, especially at the candidate-finding stage where researchers need to compare accepted and rejected raw events directly. + +## Decision + +Plugins in this repository will align to the investigation workspace through generic structured datasets. + +Rules: + +1. Use table datasets as the primary linking surface for inspectable scientific outputs. +2. Provide `row_id_column` when the plugin can produce stable ids. +3. Provide `coordinate_space_2d`, `coordinate_space_3d`, and `time_column` when the data supports linked 2D/3D inspection. +4. Use `layer_id` plus `HostDatasetDescriptor.display` for visibility and styling defaults. +5. Keep intentionally shared dataset/view ids byte-for-byte identical across providers. +6. Use overlays only for supplemental 2D annotation or hit-testing, not as the primary data contract. +7. When one stage needs multiple logical layers, publish separate datasets/layer ids instead of keying style by plugin name. +8. It is acceptable for multiple rows to share the same stable row id when the intended interaction is "select the whole cluster" rather than "select one raw sample". +9. Stable row keys are dataset-scoped in the current host, so matching row ids across different datasets do not create cross-dataset selection on their own. + +## Consequences + +- the host can keep selection, styling, and filtering generic +- candidate-finding can expose accepted and rejected raw events as separate investigation layers +- candidate centroid overlays can select whole raw-event clusters by reusing `cluster_id` as the stable row key for accepted events +- fitting and post-processing can safely reuse the same current-localization ids without breaking host linking +- fitting can expose rejected fits as a first-class investigation dataset instead of hiding them behind aggregate counters +- plugins carry a little more schema metadata, but avoid plugin-specific host hooks diff --git a/docs/architecture.md b/docs/architecture.md index 2e882ba..c67c61f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,6 +82,7 @@ Plugins declare host-rendered datasets and views through: The host owns: - analysis-panel rendering +- linked 2D/3D investigation state - standalone windows - dataset caching - exports @@ -89,8 +90,19 @@ The host owns: This repository currently uses that mechanism for: -- reconstruction table and density windows -- the shared EVE compact localization panel that can be provided by fitting or post-processing +- reconstruction table, density, and 3D localization inspection +- candidate-stage accepted/rejected raw-event layers for live tuning +- the shared EVE current-localization datasets that can be provided by fitting or post-processing + +For investigation-linked table datasets, the important host-consumed metadata is: + +- stable row ids via `row_id_column` +- 2D and 3D coordinates +- optional time columns +- layer ids and semantic labels +- dataset display metadata for title, default visibility, color, marker shape, and size + +Overlays remain useful for supplemental 2D annotations, but the host now treats structured datasets as the primary linking surface. ## Tradeoffs diff --git a/docs/features/README.md b/docs/features/README.md index b39d965..1b7e22b 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -4,8 +4,14 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs +- [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. +- [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. +- [Investigation Workspace Alignment](./investigation-workspace-alignment.md) — in-tree plugins updated for stable ids, linked 2D/3D/table datasets, and candidate-stage accepted/rejected event inspection. - [Plugin Runtime Migration Notes](./plugin-api-v0-2.md) — historical runtime-migration brief, updated with the current interface additions that matter to this repo. - [Plugin Host Views](./plugin-host-views.md) — generic host-rendered datasets, cache generations, and shared view ids. +- [TableV1 Declarative Metadata](./tablev1-declarative-metadata.md) — plugin-side adoption of row provenance, display formats, and cross-dataset relations for trustworthy table rendering. +- [Clickable 2D Overlays via Marker `source_row`](./clickable-overlays-source-row.md) — plugin-api ABI 4 `source_dataset_id`/`source_row_id` plumbing and failed-fit click-to-select loop. +- [Action Requests And Single-Cluster Refit](./action-requests-and-refit.md) — plugin-declared host actions, eveSMLM refit/commit/discard flow on the `augur.evesmlm.refit_preview` dataset. - [Reconstruction Workflow](./reconstruction.md) — accumulated localization tables rendered and exported by the host. - [eveSMLM Pipeline](./evesmlm.md) — candidate finding, fitting, and post-processing as three chainable plugins. diff --git a/docs/features/action-requests-and-refit.md b/docs/features/action-requests-and-refit.md new file mode 100644 index 0000000..d9be433 --- /dev/null +++ b/docs/features/action-requests-and-refit.md @@ -0,0 +1,93 @@ +# Action Requests And Single-Cluster Refit + +## Summary + +Plugins can declare host-rendered action buttons and consume the requests +the host publishes when the user triggers one. The eveSMLM fitting plugin +is the first concrete consumer: it exposes **Re-fit cluster…**, +**Commit refit**, and **Discard refit preview**. The re-fit action opens a +host-rendered modal driven by the plugin's `param_schema`, runs a +single-cluster fit with the captured parameters, and emits the result as a +separate `augur.evesmlm.refit_preview` dataset so it is visually distinct +from the main pipeline output. + +## Plugin Contract + +Refit is plumbed through the generic host action bus (see +`augur-rs/docs/features/investigation-action-requests.md`). In short: + +- Add `HostActionDescriptor` entries to `HostViewRegistry.actions` in + `host_views()`. Each descriptor declares: + - `id` — stable identifier used to route the request in `process_frame`. + - `title` — button label. + - `scope` — one of `Dataset`, `Row`, `Cluster` with the target + `dataset_id` (and `group_column` for `Cluster`). + - `param_schema: Option` — typically + `serde_json::to_value(my_settings_schema())`. Pass `None` when the + action takes no parameters. +- Read the persistent queue at `CTX_INVESTIGATION_ACTION_REQUESTS` + (`HostActionRequestQueue`). Filter by your cached + `last_consumed_action_request_id` so each request runs exactly once. +- For `Cluster` actions, expect the host to snapshot the selected rows into + `params["__augur_cluster_rows"]`. Plugins can reconstruct the selected + cluster from those rows instead of depending on the next frame to still + contain the same cluster. +- Emit side effects. Publish overlays/datasets for visual preview, or + mutate owned state for commit/discard. + +## Fitting Plugin Implementation + +- Three actions registered in `host_views()`: + - `augur.evesmlm.refit_cluster` — `Cluster` scope on + `augur.evesmlm.candidates.accepted_events` with + `group_column = "cluster_id"`. `param_schema` covers `fit_method`, + `sigma_min_nm`, `sigma_max_nm`, `max_fit_residual`. + - `augur.evesmlm.commit_refit` — `Row` scope on + `augur.evesmlm.refit_preview`, no params. + - `augur.evesmlm.discard_refit` — `Dataset` scope on + `augur.evesmlm.refit_preview`, no params. +- New persistent plugin state: + - `host_results: EveLocalizationResults` / `host_rejected_fits: Vec` — + host-visible history keyed by `cluster_id`, used for persistent tables, + 3D views, and post-commit durability across frames. + - `refit_preview_results: EveLocalizationResults` — preview rows. + - `refit_preview_replaces: Vec>` — parallel vec mapping each + preview row to the current-frame row it replaces on commit (or + `None` to append). + - `last_consumed_action_request_id: u64` — dedupe cursor. +- `process_frame` runs the normal analysis, merges the frame into the + host-visible history, drains the queue, then publishes + `CTX_EVE_LOCALIZATION_RESULTS`. Host tables/3D views therefore keep + committed rows and historical rejected fits visible across frames, while + the reconstruction-facing context publish stays frame-local. +- Preview rows render with a yellow filled-circle marker via + `add_marker_overlay`, distinct from accepted (green cross) and rejected + (red diamond). + +## Scope Resolution Details + +- **Re-fit** reconstructs the selected cluster from the host-supplied + `__augur_cluster_rows` snapshot when available, and only falls back to + the current frame's `EveCandidates` if no snapshot is present. This lets + the action work from persistent/historical selections instead of only the + latest frame. +- **Commit** matches the preview row by `row_id` parsed from the scope + payload, upserts the committed localization into the host-visible + history, drops any matching rejected-fit row for that cluster, and + updates the current frame-local results only if that cluster is still + present in the current frame. +- **Discard** clears the preview list. No other plugin state is touched. + +## Byte-Identical On Discard + +A targeted unit test +(`discard_clears_preview_without_touching_current_results`) clones +`current_results` before discard and asserts byte-identical JSON equality +after. The main pipeline output for the next frame is therefore unchanged +when a request is discarded. + +## References + +- `augur-rs/docs/adr/018-host-action-bus.md` +- `augur-rs/docs/features/investigation-action-requests.md` +- `plugins/evesmlm-fitting/src/lib.rs` diff --git a/docs/features/clickable-overlays-source-row.md b/docs/features/clickable-overlays-source-row.md new file mode 100644 index 0000000..d6ecb1e --- /dev/null +++ b/docs/features/clickable-overlays-source-row.md @@ -0,0 +1,46 @@ +# Clickable 2D Overlays via Marker `source_row` + +## Summary + +The augur-plugin-api ABI (bumped to 4) adds `source_dataset_id` and +`source_row_id` to `FfiMarkerOverlayItem`. Host-side, the viewer uses these +fields — when set — as the authoritative `StableRowKey` on click, instead of +falling back to the `(overlay.dataset_id, marker.stable_id)` pair. This lets a +plugin emit markers on one layer while pointing clicks at rows in a +*different* dataset. + +In-tree plugins now populate `source_row` explicitly: + +- `evesmlm-fitting` — accepted-localization crosses point at + `current_localizations`; rejected-fit diamonds point at `rejected_fits`. +- `evesmlm-postproc` — drift-corrected localization crosses point at + `current_localizations`. +- `evesmlm-candidates` — cluster centroid markers leave `source_row` empty + pending a cluster-addressable dataset (future work). + +## Effect on the EVE Failed-Fit Loop + +Combined with the Stage-2 `rejection_reason` headline and row provenance on +`rejected_fits`, clicking a red diamond in the 2D viewer now: + +1. selects the backing row in the rejected-fit `TableWindow`; +2. shows `rejection_reason` as the summary card heading; +3. auto-seeks the replay transport to the fit's anchor timestamp; +4. keeps the diamond visible while scrubbing inside the fit's declared span. + +No per-frame result cache is involved — the host filters declared rows by +`[span_start_us, span_end_us]` against the current frame window. + +## Code References + +| Path | Role | +| --- | --- | +| `plugins/evesmlm-fitting/src/lib.rs` | Populates `source_row` on accepted crosses and rejected diamonds | +| `plugins/evesmlm-postproc/src/lib.rs` | Populates `source_row` on drift-corrected localization crosses | +| `plugins/evesmlm-candidates/src/lib.rs` | Pending: cluster-addressable dataset for centroid markers | + +## Related + +- [TableV1 Declarative Metadata](./tablev1-declarative-metadata.md) +- [Investigation Workspace Alignment](./investigation-workspace-alignment.md) +- [eveSMLM Pipeline](./evesmlm.md) diff --git a/docs/features/evesmlm-temporal-diagnostics.md b/docs/features/evesmlm-temporal-diagnostics.md new file mode 100644 index 0000000..0efab1a --- /dev/null +++ b/docs/features/evesmlm-temporal-diagnostics.md @@ -0,0 +1,65 @@ +# EVE Temporal Diagnostics + +## Summary + +This feature extends the in-tree eveSMLM pipeline with better live diagnostics for candidate tuning and fit rejection analysis. + +The change adds: + +- temporal candidate clustering over retained event history +- provisional versus complete cluster tracking +- cluster-boundary overlays with clickable centroid markers +- rejected-fit investigation datasets and overlays + +## Candidate Finding + +`EVE Candidate Finding` can now request retained event history from the host and cluster over a configurable temporal lookback window instead of only the current preview frame. + +Tracked clusters keep a stable `cluster_id` while they are visible. A cluster is only published downstream once it has stopped growing for the configured number of stable frames. Until then it remains provisional. + +The candidate overlay now includes: + +- 2-sigma eigenfeature ellipses for DBSCAN and eigenfeature modes +- bounding boxes for frame-based mode +- clickable centroid markers linked to the accepted-events investigation dataset + +Accepted candidate-event rows now intentionally use `cluster_id` as the row-id column so one centroid click can select all raw events that belong to that cluster across the host table and 3D inspection views. + +This is intentionally scoped to the accepted candidate-events dataset. AugurRS still keys selection by `(dataset_id, row_id)`, so matching `cluster_id` values do not create automatic cross-dataset linking into rejected fits or other datasets. + +## Candidate Fitting + +`EVE Candidate Fitting` now records rejected fits with structured rejection reasons instead of only counting them. + +Rejected fits are exposed as a separate host dataset: + +- dataset id: `augur.evesmlm.rejected_fits` +- layer id: `augur.layer.evesmlm.rejected_fits` +- compact/table views for row-wise inspection +- linked 3D view: `augur.evesmlm.rejected_fits.scatter3d` + +Each rejected row carries: + +- stable `row_id` +- source `cluster_id` +- position and timestamp +- sigma values when available +- fit residual +- event count and polarity balance +- rejection reason + +The fitting status output now reports the rejection breakdown across fit failures, sigma-bound rejections, and residual-bound rejections. + +## Investigation Contracts + +This feature keeps the existing host-owned investigation model intact and extends it with two important conventions: + +1. Candidate centroid overlays link into the accepted raw-event dataset by reusing `cluster_id` as the stable row key. +2. Rejected fits are exposed as a first-class structured dataset instead of being implicit in a status count. + +## Verification + +```bash +cargo test -p augur-plugin-evesmlm-candidates +cargo test -p augur-plugin-evesmlm-fitting +``` diff --git a/docs/features/evesmlm.md b/docs/features/evesmlm.md index bdb2813..8ef6cb6 100644 --- a/docs/features/evesmlm.md +++ b/docs/features/evesmlm.md @@ -4,23 +4,38 @@ The eveSMLM pipeline is implemented as three focused plugins so each stage can b ## Stages -1. **EVE Candidate Finding** (`RawEvents`) clusters raw `CdEvent` samples into emitter candidates and publishes `EveCandidates`. -2. **EVE Candidate Fitting** (`DerivedData`) converts each candidate into one or more sub-pixel localization estimates, republishes `EveLocalizationResults` and `LocalizationResults`, and exposes the compact host-view dataset `augur.evesmlm.current_localizations`. -3. **EVE Post-Processing** (`DerivedData`) filters, drift-corrects, and evaluates the fitted localizations, then republishes the same host-view dataset id and view id with the same schema. +1. **EVE Candidate Finding** (`RawEvents`) clusters raw `CdEvent` samples into emitter candidates, can aggregate over retained event history, publishes only stable completed `EveCandidates`, and exposes accepted/rejected raw-event investigation layers plus boundary overlays. +2. **EVE Candidate Fitting** (`DerivedData`) converts each completed candidate into one or more sub-pixel localization estimates, republishes `EveLocalizationResults` and `LocalizationResults`, and exposes both the shared host-view dataset `augur.evesmlm.current_localizations` and the rejected-fit dataset `augur.evesmlm.rejected_fits`. +3. **EVE Post-Processing** (`DerivedData`) filters, drift-corrects, and evaluates the fitted localizations, then republishes the same host-view dataset id and view ids with the same schema and metadata. ## Why Three Plugins - Keeps raw-event grouping separate from numerical fitting, so candidate quality can be inspected directly. +- Lets researchers compare accepted and rejected candidate-stage raw events while tuning clustering thresholds. - Lets researchers compare fitting methods on a fixed candidate set. - Allows post-processing to be toggled or replaced without touching candidate generation. - Preserves compatibility with existing downstream plugins through `LocalizationResults`. ## Host View Resolution +- `EVE Candidate Finding` publishes two investigation datasets for the current analysis window: + - accepted candidate events + - rejected candidate events +- the accepted candidate-events dataset now keys rows by `cluster_id` so centroid overlays can select every event in a cluster at once. +- both candidate datasets now register host tables as well as 3D views, so the investigation workflow has visible table targets for selection and inspection. +- both candidate datasets include timestamps, 2D coordinates, and 3D scatter metadata so the host can color them separately in linked 2D/3D inspection. +- candidate host-view titles stay short (`Accepted Events`, `Rejected Events`) because the host renders + table/window chips in narrow plugin cards; the full dataset ids remain stable. +- candidate table display metadata marks concise `X`, `Y`, `Time`, `Polarity`, and `Cluster` + labels, with accepted events using `Cluster` as the compact-card headline. +- fitting also publishes a rejected-fit investigation dataset and 3D view so fit failures and threshold rejections can be inspected alongside accepted localizations. +- cross-dataset linking is still host-limited: matching `cluster_id` values do not automatically link candidate events to rejected fits because AugurRS selections are scoped by dataset id. - The compact EVE localization panel is declared by both fitting and post-processing. +- the 3D current-localizations view is also declared by both fitting and post-processing - The host resolves duplicate ids in plugin execution order. - When **EVE Post-Processing** is enabled, it becomes the active provider for the panel view. - When post-processing is disabled, the panel falls back automatically to **EVE Candidate Fitting**. +- fitting and post-processing must therefore keep the shared current-localization dataset/view descriptors identical ## Calibration Note @@ -30,7 +45,7 @@ The fitting and post-processing stages now use that host `nm_per_pixel` value au ## Data Flow -`CdEvent` stream -> `EveCandidates` -> `EveLocalizationResults` -> filtered / corrected `EveLocalizationResults` +`CdEvent` stream -> tracked / completed `EveCandidates` -> `EveLocalizationResults` (+ rejected-fit dataset) -> filtered / corrected `EveLocalizationResults` ## Installation diff --git a/docs/features/investigation-workspace-alignment.md b/docs/features/investigation-workspace-alignment.md new file mode 100644 index 0000000..f6d9415 --- /dev/null +++ b/docs/features/investigation-workspace-alignment.md @@ -0,0 +1,79 @@ +# Investigation Workspace Alignment + +## Summary + +This pass aligns the in-tree plugins in `augur-plugins` with the host-owned investigation workspace now implemented in `augur-rs`. + +The goal is not plugin-specific UI. The goal is to expose better generic data contracts so the host can link: + +- 2D preview points +- 3D inspection layers +- host-rendered tables + +## What Changed + +- `evesmlm-candidates` now publishes two generic raw-event investigation datasets: + - accepted candidate events + - rejected candidate events +- those candidate datasets carry: + - stable row ids + - `timestamp_us` + - 2D coordinates + - 3D coordinates using time as the `z` axis + - layer/display metadata for distinct accepted vs rejected styling +- accepted candidate rows can now intentionally share a `cluster_id` row key so one centroid overlay can select every event in that cluster +- candidate datasets must register table views as well as 3D views when the workflow expects row-wise inspection and linked selection +- `evesmlm-fitting` and `evesmlm-postproc` now keep the shared `augur.evesmlm.current_localizations` contract aligned with: + - stable row ids + - `timestamp_us` + - 2D and 3D coordinate metadata + - shared layer/display metadata + - linked marker overlays carrying stable ids +- `evesmlm-fitting` also publishes `augur.evesmlm.rejected_fits` for rejected candidates with timestamps, positions, metrics, and rejection reasons +- matching ids across different datasets still do not link automatically because the host selection model keys rows by dataset id plus stable row id +- `reconstruction` now exposes the accumulated localization dataset as a fuller investigation dataset with: + - stable row ids + - `timestamp_us` + - 3D scatter metadata + - layer/display metadata +- repo-local docs and the template guidance now describe stable ids, dataset/layer metadata, and overlays as supplemental rather than primary integration surfaces + +## Important Contracts + +### Candidate Event Layers + +The candidate-finding stage now surfaces accepted and rejected raw events from the active analysis window as separate host datasets instead of hiding that distinction inside plugin-local logic or centroid-only overlays. + +That makes it possible to tune candidate parameters while seeing: + +- which events survived into clusters +- which events were rejected +- how those two groups distribute over time in the 3D view + +### Shared EVE Current Localizations + +`evesmlm-fitting` and `evesmlm-postproc` intentionally reuse the same dataset id and view ids for current localizations. + +To keep host-side linking trustworthy, those reused descriptors must stay identical across both providers: + +- same schema +- same row-id column +- same coordinate/time metadata +- same layer metadata +- same view descriptors + +The later enabled provider can then replace the dataset payload without breaking selection, styling, or view resolution. + +### Reconstruction + +The reconstruction plugin remains generic. It still publishes one accumulated dataset as the source of truth, but that dataset now participates in the linked investigation model instead of acting only as a density-view backing store. + +## Verification + +```bash +cargo check -p augur-plugin-evesmlm-candidates +cargo test -p augur-plugin-evesmlm-candidates +cargo test -p augur-plugin-evesmlm-fitting +cargo test -p augur-plugin-evesmlm-postproc +cargo test -p augur-plugin-reconstruction +``` diff --git a/docs/features/plugin-host-views.md b/docs/features/plugin-host-views.md index 0670378..12bd335 100644 --- a/docs/features/plugin-host-views.md +++ b/docs/features/plugin-host-views.md @@ -11,19 +11,28 @@ That keeps scientific state in the plugin while letting the host own rendering, ## What Plugins Can Declare - datasets with stable ids and explicit schema metadata +- stable row ids, time columns, and 2D/3D coordinate metadata for linked investigation - analysis-panel views rendered by the host - standalone windows rendered by the host - multiple views backed by the same dataset +- layer/display metadata for host-owned visibility and styling defaults +- supplemental marker overlays for 2D hit-testing when datasets alone are not enough - optional generation counters for cache invalidation ## Current In-Tree Usage -- `Localization Reconstruction` publishes `augur.localization.accumulated` once and lets the host render both: +- `Localization Reconstruction` publishes `augur.localization.accumulated` once and lets the host render: - a `Localization Table` window - a `Reconstruction` density window -- `EVE Candidate Fitting` and `EVE Post-Processing` both publish `augur.evesmlm.current_localizations` with the same schema and the same compact panel view id + - a `Localization Cloud` 3D view +- `EVE Candidate Finding` publishes accepted and rejected raw-event datasets as separate investigation layers +- `EVE Candidate Fitting` and `EVE Post-Processing` both publish `augur.evesmlm.current_localizations` with the same schema and the same view ids -Because the host resolves duplicate ids in plugin execution order, `EVE Post-Processing` becomes the active provider whenever it is enabled; otherwise the compact table falls back to `EVE Candidate Fitting`. +Because the host resolves duplicate ids in plugin execution order, `EVE Post-Processing` becomes the active provider whenever it is enabled; otherwise the shared current-localizations dataset falls back to `EVE Candidate Fitting`. + +`Scatter3dFromTable` descriptors are consumed by AugurRS as main investigation 3D scene layers. +Plugins should still declare them with stable ids and coordinate metadata, but should not rely on +them appearing as separate dock/window chips. ## Why The Split Matters diff --git a/docs/features/plugin-install-reload.md b/docs/features/plugin-install-reload.md new file mode 100644 index 0000000..5399755 --- /dev/null +++ b/docs/features/plugin-install-reload.md @@ -0,0 +1,40 @@ +# Plugin Install And Reload + +## Goal + +Keep locally installed runtime plugins reloadable on macOS even when they are built from an in-flight sibling `augur-rs` checkout. + +## Problem + +Cargo's macOS `cdylib` outputs keep an absolute `LC_ID_DYLIB` that points back into the build tree, for example: + +```text +/path/to/augur-plugins/target/release/deps/libaugur_plugin_localization.dylib +``` + +That identity is harmless when the library stays in `target/`, but it becomes a footgun once the plugin is copied into `~/.augur/plugins//`. The host scans the installed copy, yet dyld can still treat the plugin as the build-tree image identity during later loads or reloads. + +In practice that makes plugin updates look stale: the Plugin Manager can keep reporting an older ABI or older code path even though the copied file in `~/.augur/plugins/` was rebuilt. + +## Repo-Level Fix + +- `scripts/install-built-plugins.sh` still copies each built runtime plugin into the standard `~/.augur/plugins//` layout. +- On macOS, the script now rewrites the copied library's `LC_ID_DYLIB` to `@loader_path/` with `install_name_tool`. +- That keeps the installed artifact self-identified by its installed location instead of Cargo's build-path identity, which makes rescans/reloads behave like the user expects. + +## Authoring Guidance + +- Prefer `./scripts/install-built-plugins.sh --profile release` over manual `cp` steps when installing local plugins on macOS. +- If you do copy a plugin by hand on macOS, rewrite the installed dylib id after copying: + +```bash +install_name_tool -id "@loader_path/libaugur_plugin_my_plugin.dylib" \ + ~/.augur/plugins/my-plugin/libaugur_plugin_my_plugin.dylib +``` + +- After an ABI bump in `augur-plugin-api`, rebuild the plugin and replace the installed runtime library before using **Scan for New Plugins** or **Reload** in `augur-gui`. + +## Verification + +- The installed runtime libraries continue to hash-match the built release artifacts apart from the macOS dylib id rewrite. +- `otool -D ~/.augur/plugins//libaugur_plugin_.dylib` now reports `@loader_path/...` instead of an absolute path into `target/release/deps/`. diff --git a/docs/features/reconstruction.md b/docs/features/reconstruction.md index 3320741..f85ae59 100644 --- a/docs/features/reconstruction.md +++ b/docs/features/reconstruction.md @@ -5,15 +5,26 @@ The reconstruction workflow publishes one accumulated host-view dataset instead ## Components 1. **Localization Reconstruction** (`DerivedData`) reads `LocalizationResults` from `HostContext` and stores a capped nanometer-space accumulation table. -2. **`host_views()`** declares one dataset, `augur.localization.accumulated`, plus two host-rendered window views: +2. **`host_views()`** declares one dataset, `augur.localization.accumulated`, plus host-rendered views for: - `Localization Table` - `Reconstruction` + - `Localization Cloud` 3. **`host_view_dataset()`** serves one columnar `TableV1` snapshot that both windows consume. ## Source Of Truth - the reconstruction plugin owns the only accumulated localization state -- the full table window and density reconstruction window read the same dataset id +- the full table window, density reconstruction window, and 3D scatter inspection all read the same dataset id + +## Investigation Metadata + +The accumulated localization dataset now participates directly in the host investigation workspace through: + +- stable row ids via `id` +- `timestamp_us` as the shared time column +- 2D nanometer coordinates for linked preview/table filtering +- 3D scatter coordinates using `timestamp_us` on the `z` axis +- layer/display metadata for default visibility and styling ## Resource Use @@ -28,7 +39,7 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global ## Data Flow -`LocalizationResults` -> `augur.localization.accumulated` -> host table window / host density window +`LocalizationResults` -> `augur.localization.accumulated` -> host table window / density window / 3D localization cloud ## Installation diff --git a/docs/features/tablev1-declarative-metadata.md b/docs/features/tablev1-declarative-metadata.md new file mode 100644 index 0000000..ae24fe0 --- /dev/null +++ b/docs/features/tablev1-declarative-metadata.md @@ -0,0 +1,59 @@ +# TableV1 Declarative Metadata For Plugins + +## Summary + +Plugins that expose `HostDatasetKind::TableV1` now describe row provenance, cross-dataset +relations, and per-column display formatting declaratively. The host consumes these +descriptors to render timestamps as `mm:ss.uuu`, size columns sensibly, drive summary cards, +auto-seek replay to the anchor timestamp of a selected row, and resolve derived-row selections +back to contributing raw events for 3D emphasis. + +This replaces implicit conventions (where the host guessed from type or column name) with +explicit, serializable metadata carried on `TableSchema` and `HostDatasetDescriptor`. + +## What Plugins Populate + +On `TableSchema`: + +- `provenance: Some(TableRowProvenance { anchor_time_column, span_start_column, span_end_column })` + — typically `anchor_time_column: Some("timestamp_us")`. Spans are used by the host for + span-based visibility and anchor fallback, so `span_start_column` / `span_end_column` should + describe the real contributing interval rather than repeating the anchor timestamp. +- `column_display: Vec` — one entry per column you want formatted: + - timestamp columns → `TableColumnDisplayFormat::TimestampMicros` + - positions, widths, residuals → `FixedPrecision { digits: N }` + - `row_id` columns → `Identifier` with `hidden: true` + - enum-like columns (methods, reasons) → `Category` (promote with `headline: true` in + failure-result schemas to make the reason the summary-card heading) + - Width priority: `High` (~160px) for labels and text; `Medium` (~100px) for numeric; + `Low` (~60px) for compact identifiers. + +On `HostDatasetDescriptor`: + +- `relations: Vec` — + declare joins from this dataset's row to another dataset. Example: candidate-event rows + relate to localizations via `cluster_id`. The host can follow these joins transitively to map a + selected derived row back to raw accepted-event identities. + +All new fields are additive with serde defaults; omitting them keeps the prior behavior. + +## Implemented Datasets + +- `evesmlm-fitting`: `augur.evesmlm.current_localizations`, `augur.evesmlm.rejected_fits` — full + provenance with real `span_start_us` / `span_end_us`, per-column formatting, cluster relations + back to accepted candidate events, and `rejection_reason` marked `headline: true` for rejected + fits. +- `evesmlm-candidates`: accepted/rejected candidate events — provenance on `timestamp_us`, + relation to `current_localizations` via `cluster_id` on accepted events. + +## Descriptor Parity + +`evesmlm-postproc` re-exports the `current_localizations` registry builder from +`evesmlm-fitting`, so the descriptor is structurally identical by construction. A parity test +in `plugins/evesmlm-postproc/src/lib.rs` serializes both registries to JSON and asserts +equality to catch accidental divergence. + +## Related Host Behavior + +See the companion host feature brief: [Investigation Table Trustworthiness](https://github.com/muthmann/augur-rs/blob/main/docs/features/investigation-table-trustworthiness.md) +and [ADR 017](https://github.com/muthmann/augur-rs/blob/main/docs/adr/017-declarative-tablev1-metadata.md). diff --git a/docs/installing-plugins.md b/docs/installing-plugins.md index ba8797b..45a79e9 100644 --- a/docs/installing-plugins.md +++ b/docs/installing-plugins.md @@ -47,6 +47,14 @@ cp target/release/libaugur_plugin_localization.dylib ~/.augur/plugins/localizati Install each plugin into its own directory under `~/.augur/plugins//`. +On macOS, a plain `cp` keeps Cargo's build-path dylib identity in the copied file. Rewrite the +installed copy so reloads do not keep resolving back to the build tree: + +```bash +install_name_tool -id "@loader_path/libaugur_plugin_localization.dylib" \ + ~/.augur/plugins/localization/libaugur_plugin_localization.dylib +``` + ## Install All Built Plugins ```bash @@ -54,6 +62,8 @@ Install each plugin into its own directory under `~/.augur/plugins//`. ``` This copies every plugin that already has a built runtime library in `target/release/`. +On macOS it also rewrites each installed dylib id to `@loader_path/` so Plugin Manager +reloads do not stay pinned to Cargo's original build-path identity. ## Load Or Reload In The GUI @@ -85,6 +95,16 @@ You copied a source directory instead of the built library. Build the plugin and The library was built against an older plugin interface or does not export the runtime vtable. Port it to `augur-plugin-api::Plugin` and export it with `export_plugin!`. +### “plugin ABI mismatch” + +The installed runtime library is stale relative to the host ABI. + +1. Rebuild the plugin against the current sibling `augur-rs` checkout. +2. Replace the installed runtime library in `~/.augur/plugins//`. +3. On macOS, prefer `./scripts/install-built-plugins.sh --profile release` or rewrite the copied dylib id with `install_name_tool -id "@loader_path/" ...`. + +If you overwrote a plugin while `augur-gui` was already running, restart the host once after the ABI bump to clear any previously loaded image from the process. + ### The plugin loads but host-owned settings are missing `GlobalSettings` are published through `augur.global_settings` by newer hosts. If a plugin tolerates `None` there, verify that the installed plugin and the `augur-gui` build come from compatible `augur-rs` / `augur-plugins` revisions. diff --git a/docs/plugin-api.md b/docs/plugin-api.md index f1138d6..c5c884a 100644 --- a/docs/plugin-api.md +++ b/docs/plugin-api.md @@ -1,10 +1,11 @@ # Runtime Plugin API -This repository now follows the runtime-only plugin surface documented in `augur-rs`. +This repository follows the runtime-only plugin surface documented in `augur-rs`. Use the upstream guide as the canonical contract: - [`augur-rs/docs/features/plugin-authoring-guide.md`](https://github.com/muthmann/augur-rs/blob/main/docs/features/plugin-authoring-guide.md) +- [`augur-rs/docs/features/investigation-workspace.md`](https://github.com/muthmann/augur-rs/blob/main/docs/features/investigation-workspace.md) This page summarizes the parts authors working in `augur-plugins` touch most often. @@ -25,37 +26,6 @@ This page summarizes the parts authors working in `augur-plugins` touch most oft - `Series1dV1` - `CTX_GLOBAL_SETTINGS` -## Minimal Plugin - -```rust -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostContext, HostOutput, Plugin, PluginFrame, -}; - -#[derive(Default)] -struct MyPlugin { - enabled: bool, -} - -impl Plugin for MyPlugin { - fn name(&self) -> &'static str { "My Plugin" } - fn enabled(&self) -> bool { self.enabled } - fn set_enabled(&mut self, enabled: bool) { self.enabled = enabled; } - fn reset(&mut self) {} - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - _context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - } -} - -export_plugin!(MyPlugin); -``` - ## Execution Model `input_kind()` and retained history are separate concerns. @@ -94,64 +64,42 @@ context.publish("my.plugin.results", &results)?; let upstream = context.get::("my.plugin.results")?; ``` -Prefer standard shared payloads such as `CTX_LOCALIZATION_RESULTS` when they exist. The standard localization payload now lives in `augur-plugin-types`. If several plugins need the same domain-specific type, put that type in a companion crate instead of copying it into multiple plugin crates. - -Persistent helpers are still available for plugin-owned caches, but shared scientific outputs should normally stay on the per-frame context bus. +Prefer standard shared payloads such as `CTX_LOCALIZATION_RESULTS` when they exist. If several plugins need the same domain-specific type, put that type in a companion crate instead of copying it into multiple plugin crates. ## Host-Owned Global Settings -The host now publishes shared runtime settings on the normal context bus: +The host publishes shared runtime settings on the normal context bus: - key: `CTX_GLOBAL_SETTINGS` - type: `GlobalSettings` -Example: - -```rust -use augur_plugin_api::{GlobalSettings, CTX_GLOBAL_SETTINGS}; - -if let Some(globals) = context.get::(CTX_GLOBAL_SETTINGS)? { - let nm_per_pixel = globals.nm_per_pixel; - let sensor_width = globals.sensor_width; - let sensor_height = globals.sensor_height; - let acq_time_ms = globals.acq_time_ms; - let event_store_budget_bytes = globals.event_store_budget_bytes; - let _ = ( - nm_per_pixel, - sensor_width, - sensor_height, - acq_time_ms, - event_store_budget_bytes, - ); -} -``` - New plugins should prefer `GlobalSettings` over duplicating host-owned defaults such as pixel scale or sensor geometry. -Plugins must tolerate `None` when run against an older host build. - -## Dependencies - -Override `dependencies()` only when the plugin truly requires a specific upstream producer by name: - -```rust -fn dependencies(&self) -> &[&'static str] { - &["EVE Candidate Finding"] -} -``` +## Linked Investigation Datasets -If the plugin can degrade gracefully when an upstream payload is absent, prefer a runtime warning over a hard dependency declaration. +The host now treats structured datasets as the primary integration surface for linked 2D, 3D, and table workflows. Overlays are supplemental. -## Settings And Status +When a table dataset should participate in linked investigation, populate as many of these additive fields as the plugin can support: -Plugins describe settings declaratively through: +- `TableSchema.coordinate_space_2d` +- `TableSchema.coordinate_space_3d` +- `TableSchema.row_id_column` +- `TableSchema.time_column` +- `TableSchema.layer_id` +- `TableSchema.semantic_label` +- `HostDatasetDescriptor.display` + - `layer_title` + - `default_visibility` + - `default_color` + - `default_marker_shape` + - `default_size` -- `settings_schema()` -- `get_setting()` -- `set_setting()` -- optional `status_entries()` +Guidelines: -Common setting kinds include `Bool`, slider/drag values, and `Enum`. The host owns rendering and persistence of the UI state. +- Use stable ids from the scientific data when possible. +- Fall back to deterministic plugin-generated ids when no natural id exists. +- Key reusable shared views by dataset id and keep descriptors byte-for-byte identical across providers that intentionally reuse the same ids. +- Prefer dataset/layer ids for styling and visibility instead of plugin-name-specific logic. ## Host Views @@ -169,6 +117,7 @@ Plugins can declare host-rendered datasets and views through `host_views()` and - `HostViewKind::TableWindow` - `HostViewKind::Density2dFromTable` - `HostViewKind::Scatter2dFromTable` +- `HostViewKind::Scatter3dFromTable` - `HostViewKind::ImageWindow` - `HostViewKind::LineSeriesWindow` @@ -178,10 +127,20 @@ Plugins can declare host-rendered datasets and views through `host_views()` and fn host_views(&self) -> HostViewRegistry { HostViewRegistry { datasets: vec![HostDatasetDescriptor { - id: "example.table".into(), - title: "Example Table".into(), + id: "example.points".into(), + title: "Example Points".into(), kind: HostDatasetKind::TableV1(TableSchema { columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, TableColumn { id: "x".into(), title: "X".into(), @@ -193,48 +152,79 @@ fn host_views(&self) -> HostViewRegistry { value_type: TableValueType::F64, }, ], - coordinate_space_2d: None, + coordinate_space_2d: Some(TableCoordinateSpace2d { + x_column: "x".into(), + y_column: "y".into(), + x_min: 0.0, + x_max: 128.0, + y_min: 0.0, + y_max: 128.0, + }), + coordinate_space_3d: Some(TableCoordinateSpace3d { + x_column: "x".into(), + y_column: "y".into(), + z_column: "timestamp_us".into(), + x_min: 0.0, + x_max: 128.0, + y_min: 0.0, + y_max: 128.0, + z_min: 0.0, + z_max: 5_000.0, + }), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some("example.layer.points".into()), + semantic_label: Some("points".into()), }), empty_message: "No rows yet.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Example points".into()), + default_visibility: Some(true), + default_color: Some([80, 200, 255, 255]), + default_marker_shape: Some(HostMarkerShape::Point), + default_size: Some(3.0), + }), }], views: vec![HostViewDescriptor { - id: "example.table.compact".into(), - title: "Current Rows".into(), - dataset_id: "example.table".into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, + id: "example.points.3d".into(), + title: "Example 3D".into(), + dataset_id: "example.points".into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x".into(), + y_column: "y".into(), + z_column: "timestamp_us".into(), + }, }], } } - -fn host_view_dataset(&self, dataset_id: &str) -> Option> { - if dataset_id != "example.table" { - return None; - } - - let dataset = TableDatasetV1::new(vec![ - TableColumnData { - column_id: "x".into(), - values: TableColumnValues::F64(vec![1.0, 2.0]), - }, - TableColumnData { - column_id: "y".into(), - values: TableColumnValues::F64(vec![3.0, 4.0]), - }, - ]).ok()?; - - serde_json::to_vec(&dataset).ok() -} - -fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - if dataset_id == "example.table" { 1 } else { 0 } -} ``` `host_view_dataset_generation()` is optional but recommended when the host should invalidate a cached snapshot only after the dataset changes. The host owns rendering, exports, caching, and window state. Plugins do not render `egui` directly. +## Marker Overlays + +Use structured datasets for the primary linked-workspace model. Use overlays when the plugin needs extra 2D annotations or hit-testing that supplements the dataset. + +Current overlay helpers: + +- `add_highlight_pixels(...)` +- `add_crosshair_markers(...)` +- `add_marker_overlay(...)` +- `add_warning(...)` + +`add_marker_overlay(...)` supports: + +- point, cross, box, ellipse, diamond, and filled-circle shapes +- per-item color and size +- optional timestamp +- optional stable id +- optional dataset id, layer id, and source label + +That makes it the right choice when a 2D preview annotation should resolve back into the same host selection model. + ## Event History `process_frame()` always receives `event_store: &EventStoreHandle<'_>`. Plugins that need only the current frame can ignore it. History-aware plugins can query: @@ -247,13 +237,14 @@ The host owns rendering, exports, caching, and window state. Plugins do not rend - `collect_events_in_range(start_us, end_us, out)` - `oldest_timestamp_us()` -## Migration From Older Plugin Code +## Migration Notes -When porting older code, replace: +When porting older code: -- `AnalysisPlugin` with `Plugin` -- typed `PluginContext` exchange with `HostContext` -- direct `egui` UI code with declarative settings/status -- special-case host rendering hooks with `host_views()` and `host_view_dataset()` -- duplicated host-owned calibration values with `GlobalSettings` -- compile-time registration with `export_plugin!` plus a built `cdylib` +- replace `AnalysisPlugin` with `Plugin` +- replace typed `PluginContext` exchange with `HostContext` +- replace direct `egui` UI code with declarative settings/status +- replace special-case host rendering hooks with `host_views()` / `host_view_dataset()` +- replace row-index-based linking assumptions with stable row ids where possible +- replace plugin-name-based styling assumptions with dataset/layer metadata +- keep overlays as supplemental annotations, not the primary data contract diff --git a/plugin-template/README.md b/plugin-template/README.md index d525db8..ae32b4c 100644 --- a/plugin-template/README.md +++ b/plugin-template/README.md @@ -22,6 +22,15 @@ If this plugin publishes results to `HostContext` for downstream consumers, desc If this plugin declares datasets or views through `host_views()`, document the dataset ids, view ids, and expected schema here. +For investigation-linked table datasets, also document: + +- which column provides stable row identity +- whether 2D coordinates are exposed +- whether 3D coordinates and time are exposed +- which layer id and display defaults the host should expect + +Prefer structured datasets for linked 2D/3D/table workflows. Use overlays as supplemental annotations rather than the only way to inspect results. + ## Dependencies List any hard upstream plugin dependencies this plugin declares through `dependencies()` (or "None"). diff --git a/plugins/evesmlm-candidates/README.md b/plugins/evesmlm-candidates/README.md index de3a9a1..89c1b47 100644 --- a/plugins/evesmlm-candidates/README.md +++ b/plugins/evesmlm-candidates/README.md @@ -20,26 +20,43 @@ Raw-event candidate discovery for eveSMLM. This plugin groups `CdEvent` samples | Polarity | `Both` | Use positive, negative, or all events | | Epsilon | `3.0` px | Neighborhood radius for DBSCAN | | Min events | `5` | Minimum cluster size | +| Lookback | `66_000` us | Retained-history window used for temporal clustering; set to `0` for single-frame behavior | +| Stable frames | `2` | Number of consecutive no-growth frames before a cluster is published | | Max spatial extent | `5.0` px | Eigenfeature upper bound on the major covariance axis | | Min isotropy | `0.2` | Eigenfeature lower bound on `lambda2 / lambda1` | | Threshold factor | `1.5` | Wavelet threshold multiplier for frame-based mode | | Fit radius | `4` px | Event gathering radius in frame-based mode | | Max candidates | `512` | Safety cap on published candidates | -| Show overlay | `true` | Highlight candidate centroids in the preview | +| Show centroids | `true` | Draw clickable centroid markers linked to accepted candidate events | +| Show boundaries | `true` | Draw 2-sigma ellipses or bounding boxes around visible clusters | +| Show provisional | `true` | Keep still-growing clusters visible in the overlay | ## Execution Phase -`RawEvents` — consumes the raw `CdEvent` stream for the current preview window. +`RawEvents` — consumes the raw `CdEvent` stream for the current preview window and can optionally gather retained events from earlier frames. ## Published Data Publishes `EveCandidates` on the context key `augur.evesmlm.candidates`, containing: -- `clusters: Vec` with raw events, per-pixel histograms, centroid, and bounds +- `clusters: Vec` with stable `cluster_id`, raw events, per-pixel histograms, centroid, bounds, and optional boundary metadata - `frame_window_start_us`, `frame_window_end_us` - `n_events_processed` - `finding_method` +It also exposes two host investigation datasets for the current analysis window: + +- accepted candidate events +- rejected candidate events + +Both datasets carry stable row ids, timestamps, 2D coordinates, and 3D scatter metadata so the host can render accepted and rejected raw events as separate layers during live parameter tuning. + +The plugin now also registers compact and windowed host tables for both datasets, so centroid selection has a visible table target inside AugurRS without requiring plugin-specific UI. + +Accepted candidate-event rows intentionally use the string form of `cluster_id` as the row-id column so one centroid click can select the whole cluster in the accepted-events table and its 3D view. + +That selection is dataset-local: it links the centroid marker to the accepted-events dataset, but it does not cross-select unrelated datasets such as rejected fits because AugurRS stable row keys include the dataset id. + ## Dependencies None. diff --git a/plugins/evesmlm-candidates/src/eigenfeature.rs b/plugins/evesmlm-candidates/src/eigenfeature.rs index e75df0c..60a9e7f 100644 --- a/plugins/evesmlm-candidates/src/eigenfeature.rs +++ b/plugins/evesmlm-candidates/src/eigenfeature.rs @@ -2,6 +2,13 @@ use nalgebra::Matrix2; use crate::EveEvent; +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ClusterEigenInfo { + pub lambda_1: f64, + pub lambda_2: f64, + pub angle_rad: f64, +} + pub fn filter_clusters( events: &[EveEvent], clusters: Vec>, @@ -13,20 +20,20 @@ pub fn filter_clusters( clusters .into_iter() .filter(|indices| { - let Some((lambda_1, lambda_2)) = cluster_eigenvalues(events, indices) else { + let Some(info) = cluster_eigen_info(events, indices) else { return false; }; - let isotropy = if lambda_1 <= 1e-9 { + let isotropy = if info.lambda_1 <= 1e-9 { 1.0 } else { - lambda_2 / lambda_1 + info.lambda_2 / info.lambda_1 }; - lambda_1 <= max_variance && isotropy >= min_isotropy + info.lambda_1 <= max_variance && isotropy >= min_isotropy }) .collect() } -pub fn cluster_eigenvalues(events: &[EveEvent], indices: &[usize]) -> Option<(f64, f64)> { +pub fn cluster_eigen_info(events: &[EveEvent], indices: &[usize]) -> Option { if indices.len() < 2 { return None; } @@ -55,7 +62,21 @@ pub fn cluster_eigenvalues(events: &[EveEvent], indices: &[usize]) -> Option<(f6 covariance /= n.max(1.0); let eigen = covariance.symmetric_eigen(); - let mut eigenvalues = [eigen.eigenvalues[0], eigen.eigenvalues[1]]; - eigenvalues.sort_by(|left, right| right.total_cmp(left)); - Some((eigenvalues[0], eigenvalues[1])) + let major_index = if eigen.eigenvalues[0] >= eigen.eigenvalues[1] { + 0 + } else { + 1 + }; + let minor_index = 1 - major_index; + let major_vector = eigen.eigenvectors.column(major_index); + + Some(ClusterEigenInfo { + lambda_1: eigen.eigenvalues[major_index], + lambda_2: eigen.eigenvalues[minor_index], + angle_rad: major_vector[1].atan2(major_vector[0]), + }) +} + +pub fn cluster_eigenvalues(events: &[EveEvent], indices: &[usize]) -> Option<(f64, f64)> { + cluster_eigen_info(events, indices).map(|info| (info.lambda_1, info.lambda_2)) } diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index 8b3d215..e0bc71e 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -8,16 +8,26 @@ pub mod dbscan; pub mod eigenfeature; pub mod types; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use augur_plugin_api::{ - export_plugin, AnalysisSeverity, EventStoreHandle, FfiCdEvent, FfiPixel, HostContext, - HostOutput, Plugin, PluginFrame, PluginInput, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, + export_plugin, AnalysisSeverity, EventStoreHandle, FfiCdEvent, FfiColorRgba, + FfiMarkerOverlayItem, FfiMarkerShape, FfiPixel, FfiString, HostContext, HostDatasetDescriptor, + HostDatasetDisplayMetadata, HostDatasetKind, HostDatasetRelation, HostMarkerShape, HostOutput, + HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, + PluginCapabilities, PluginFrame, PluginInput, PluginStateKind, SettingItem, SettingKind, + SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, + TableColumnDisplayEntry, TableColumnDisplayFormat, TableColumnDisplayMetadata, TableColumnValues, + TableColumnWidthPriority, TableCoordinateSpace2d, TableCoordinateSpace3d, TableDatasetV1, + TableRowProvenance, TableSchema, TableValueType, }; use serde_json::{json, Value}; -pub use types::{CandidateFindingMethod, EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES}; +use types::TrackedCluster; +pub use types::{ + CandidateFindingMethod, ClusterBoundary, EveCandidates, EveCluster, EveEvent, + CTX_EVE_CANDIDATES, +}; const KERNEL_G1: [f64; 5] = [1.0 / 16.0, 0.25, 3.0 / 8.0, 0.25, 1.0 / 16.0]; const KERNEL_G2: [f64; 9] = [ @@ -31,7 +41,57 @@ const KERNEL_G2: [f64; 9] = [ 0.0, 1.0 / 16.0, ]; -const OVERLAY_COLOR: [u8; 4] = [255, 210, 32, 220]; +const ACCEPTED_EVENTS_COLOR: [u8; 4] = [60, 220, 140, 255]; +const REJECTED_EVENTS_COLOR: [u8; 4] = [255, 110, 110, 235]; +const COMPLETE_BOUNDARY_COLOR: [u8; 4] = [255, 255, 255, 60]; +const PROVISIONAL_BOUNDARY_COLOR: [u8; 4] = [255, 255, 255, 28]; +const COMPLETE_MARKER_COLOR: [u8; 4] = [255, 255, 255, 180]; +const PROVISIONAL_MARKER_COLOR: [u8; 4] = [255, 255, 255, 110]; +const ACCEPTED_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; +const REJECTED_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.rejected_events"; +const ACCEPTED_EVENTS_LAYER_ID: &str = "augur.layer.evesmlm.accepted_events"; +const REJECTED_EVENTS_LAYER_ID: &str = "augur.layer.evesmlm.rejected_events"; +const ACCEPTED_EVENTS_COMPACT_VIEW_ID: &str = "augur.evesmlm.candidates.accepted_events.compact"; +const REJECTED_EVENTS_COMPACT_VIEW_ID: &str = "augur.evesmlm.candidates.rejected_events.compact"; +const ACCEPTED_EVENTS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.accepted_events.table"; +const REJECTED_EVENTS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.rejected_events.table"; +const ACCEPTED_EVENTS_3D_VIEW_ID: &str = "augur.evesmlm.candidates.accepted_events.scatter3d"; +const REJECTED_EVENTS_3D_VIEW_ID: &str = "augur.evesmlm.candidates.rejected_events.scatter3d"; +const CANDIDATE_FINDINGS_DATASET_ID: &str = "augur.evesmlm.candidates.candidate_findings"; +const CANDIDATE_FINDING_PIXELS_DATASET_ID: &str = + "augur.evesmlm.candidates.candidate_finding_pixels"; +const CANDIDATE_FINDINGS_LAYER_ID: &str = "augur.layer.evesmlm.candidate_findings"; +const CANDIDATE_FINDINGS_COMPACT_VIEW_ID: &str = + "augur.evesmlm.candidates.candidate_findings.compact"; +const CANDIDATE_FINDINGS_TABLE_VIEW_ID: &str = + "augur.evesmlm.candidates.candidate_findings.table"; +const CANDIDATE_FINDING_PIXELS_TABLE_VIEW_ID: &str = + "augur.evesmlm.candidates.candidate_finding_pixels.table"; + +#[derive(Debug, Clone)] +struct CandidateEventRow { + event_id: u64, + x_px: f64, + y_px: f64, + timestamp_us: u64, + polarity: bool, + cluster_id: String, +} + +#[derive(Debug, Clone, Default)] +struct CandidateEventDatasets { + accepted: Vec, + rejected: Vec, + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +} + +#[derive(Debug, Clone)] +struct CandidateFinding { + cluster: EveCluster, + method: CandidateFindingMethod, +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum PolarityMode { @@ -99,12 +159,16 @@ pub struct CandidateSettings { pub polarity: PolarityMode, pub epsilon_px: f64, pub min_events: usize, + pub lookback_us: u64, + pub stable_frames: usize, pub max_spatial_extent_px: f64, pub min_isotropy: f64, pub threshold_factor: f64, pub fit_radius_px: usize, pub max_candidates: usize, pub show_overlay: bool, + pub show_boundaries: bool, + pub show_provisional: bool, } impl Default for CandidateSettings { @@ -114,12 +178,16 @@ impl Default for CandidateSettings { polarity: PolarityMode::Both, epsilon_px: 3.0, min_events: 5, + lookback_us: 66_000, + stable_frames: 2, max_spatial_extent_px: 5.0, min_isotropy: 0.2, threshold_factor: 1.5, fit_radius_px: 4, max_candidates: 512, show_overlay: true, + show_boundaries: true, + show_provisional: true, } } } @@ -127,9 +195,19 @@ impl Default for CandidateSettings { pub struct EveSmlmCandidatePlugin { enabled: bool, settings: CandidateSettings, + current_event_datasets: CandidateEventDatasets, last_candidate_count: usize, + last_complete_visible_count: usize, + last_provisional_count: usize, last_event_count: usize, last_status: String, + dataset_generation: u64, + findings: Vec, + findings_generation: u64, + frame_counter: u64, + next_cluster_id: u64, + tracked_clusters: Vec, + event_buffer: Vec, } impl Default for EveSmlmCandidatePlugin { @@ -137,24 +215,106 @@ impl Default for EveSmlmCandidatePlugin { Self { enabled: false, settings: CandidateSettings::default(), + current_event_datasets: CandidateEventDatasets::default(), last_candidate_count: 0, + last_complete_visible_count: 0, + last_provisional_count: 0, last_event_count: 0, last_status: "Enable the plugin to cluster raw eveSMLM events into candidates.".into(), + dataset_generation: 0, + findings: Vec::new(), + findings_generation: 0, + frame_counter: 0, + next_cluster_id: 0, + tracked_clusters: Vec::new(), + event_buffer: Vec::new(), } } } impl EveSmlmCandidatePlugin { + fn reset_tracking_state(&mut self) { + self.frame_counter = 0; + self.next_cluster_id = 0; + self.tracked_clusters.clear(); + self.event_buffer.clear(); + self.findings.clear(); + self.findings_generation = self.findings_generation.wrapping_add(1); + } + + fn append_findings(&mut self, clusters: &[EveCluster]) { + if clusters.is_empty() { + return; + } + + let method = self.settings.finding_method; + self.findings + .extend(clusters.iter().cloned().map(|cluster| CandidateFinding { + cluster, + method, + })); + self.findings_generation = self.findings_generation.wrapping_add(1); + } + + fn collect_analysis_events( + &mut self, + frame: &PluginFrame<'_>, + event_store: &EventStoreHandle<'_>, + ) -> (Vec, u64, u64, bool) { + let mut analysis_events = std::mem::take(&mut self.event_buffer); + let mut analysis_window_start = frame.window_start_us(); + let analysis_window_end = frame.window_end_us(); + let temporal_enabled = self.settings.lookback_us > 0 && event_store.frame_count() > 0; + + analysis_events.clear(); + if temporal_enabled { + let buffered_start = analysis_window_end.saturating_sub(self.settings.lookback_us); + analysis_window_start = buffered_start.max(event_store.oldest_timestamp_us()); + event_store.collect_events_in_range( + analysis_window_start, + analysis_window_end, + &mut analysis_events, + ); + } + + if analysis_events.is_empty() { + analysis_events.extend_from_slice(frame.events()); + analysis_window_start = frame.window_start_us(); + } + + ( + analysis_events, + analysis_window_start, + analysis_window_end, + temporal_enabled, + ) + } + fn analyze_frame( &mut self, frame: &PluginFrame<'_>, raw_events: &[FfiCdEvent], + analysis_window_start_us: u64, + analysis_window_end_us: u64, + temporal_enabled: bool, output: &mut HostOutput<'_>, ) -> EveCandidates { if raw_events.is_empty() { + self.current_event_datasets = CandidateEventDatasets { + sensor_dims: Some((frame.width(), frame.height())), + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, + ..CandidateEventDatasets::default() + }; self.last_candidate_count = 0; + self.last_complete_visible_count = 0; + self.last_provisional_count = 0; self.last_event_count = 0; - self.last_status = "Raw events are unavailable for this preview frame.".into(); + self.last_status = if temporal_enabled { + "No retained raw events are available in the requested temporal lookback.".into() + } else { + "Raw events are unavailable for this preview frame.".into() + }; Self::warning( output, AnalysisSeverity::Info, @@ -171,18 +331,26 @@ impl EveSmlmCandidatePlugin { .collect(); self.last_event_count = filtered_events.len(); if filtered_events.is_empty() { + self.current_event_datasets = CandidateEventDatasets { + sensor_dims: Some((frame.width(), frame.height())), + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, + ..CandidateEventDatasets::default() + }; self.last_candidate_count = 0; + self.last_complete_visible_count = 0; + self.last_provisional_count = 0; self.last_status = "No events passed the configured polarity filter.".into(); return EveCandidates { clusters: Vec::new(), - frame_window_start_us: frame.window_start_us(), - frame_window_end_us: frame.window_end_us(), + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, n_events_processed: 0, finding_method: self.settings.finding_method, }; } - let cluster_indices = match self.settings.finding_method { + let mut cluster_indices = match self.settings.finding_method { CandidateFindingMethod::Dbscan => dbscan::cluster_event_indices( &filtered_events, self.settings.epsilon_px, @@ -207,44 +375,264 @@ impl EveSmlmCandidatePlugin { } }; - let mut clusters = clusters_from_indices(&filtered_events, cluster_indices); - clusters.sort_by_key(|cluster| std::cmp::Reverse(cluster.event_count())); - if clusters.len() > self.settings.max_candidates { - clusters.truncate(self.settings.max_candidates); + cluster_indices.sort_by_key(|indices| std::cmp::Reverse(indices.len())); + if cluster_indices.len() > self.settings.max_candidates { + cluster_indices.truncate(self.settings.max_candidates); } - self.last_candidate_count = clusters.len(); - self.last_status = format!( - "{} candidates from {} events using {}.", - self.last_candidate_count, - self.last_event_count, - self.settings.finding_method.label() + let detected_clusters = clusters_from_indices( + &filtered_events, + cluster_indices.clone(), + self.settings.finding_method, + ); + let (visible_clusters, published_clusters) = + self.update_tracked_clusters(detected_clusters, temporal_enabled); + self.append_findings(&published_clusters); + + self.current_event_datasets = build_candidate_event_datasets( + (frame.width(), frame.height()), + analysis_window_start_us, + analysis_window_end_us, + &filtered_events, + &cluster_indices, + &visible_clusters, ); - if self.settings.show_overlay && !clusters.is_empty() { - let pixels: Vec = clusters - .iter() - .map(|cluster| FfiPixel { - x: cluster.centroid_x.round().max(0.0) as u16, - y: cluster.centroid_y.round().max(0.0) as u16, - }) - .collect(); - output.add_highlight_pixels(&pixels, OVERLAY_COLOR); - } + self.last_candidate_count = published_clusters.len(); + self.last_complete_visible_count = visible_clusters + .iter() + .filter(|cluster| cluster.complete) + .count(); + self.last_provisional_count = visible_clusters + .len() + .saturating_sub(self.last_complete_visible_count); + + let window_span_us = analysis_window_end_us.saturating_sub(analysis_window_start_us); + let boundary_summary = if self.settings.show_boundaries && !visible_clusters.is_empty() { + format!( + " Showing {} for {} visible clusters.", + boundary_label(self.settings.finding_method), + visible_clusters.len() + ) + } else { + String::new() + }; + self.last_status = if temporal_enabled { + format!( + "{} published, {} complete visible, {} provisional from {} events using {} over {} us.{}", + self.last_candidate_count, + self.last_complete_visible_count, + self.last_provisional_count, + self.last_event_count, + self.settings.finding_method.label(), + window_span_us, + boundary_summary + ) + } else { + format!( + "{} published from {} events using {} in the current frame.{}", + self.last_candidate_count, + self.last_event_count, + self.settings.finding_method.label(), + boundary_summary + ) + }; + + self.render_cluster_overlay(frame, &visible_clusters, output); EveCandidates { - clusters, - frame_window_start_us: frame.window_start_us(), - frame_window_end_us: frame.window_end_us(), + clusters: published_clusters, + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, n_events_processed: filtered_events.len(), finding_method: self.settings.finding_method, } } + fn update_tracked_clusters( + &mut self, + mut detected_clusters: Vec, + temporal_enabled: bool, + ) -> (Vec, Vec) { + self.frame_counter = self.frame_counter.wrapping_add(1); + let stable_frames = self.settings.stable_frames.max(1); + let retention_frames = stable_frames.saturating_mul(2).max(1); + let matching_radius = self.settings.epsilon_px.max(0.5); + + let mut candidate_pairs = Vec::new(); + for (detected_index, cluster) in detected_clusters.iter().enumerate() { + for (tracked_index, tracked) in self.tracked_clusters.iter().enumerate() { + let dx = cluster.centroid_x - tracked.centroid_x; + let dy = cluster.centroid_y - tracked.centroid_y; + let distance = (dx * dx + dy * dy).sqrt(); + if distance <= matching_radius { + candidate_pairs.push((distance, detected_index, tracked_index)); + } + } + } + candidate_pairs.sort_by(|left, right| left.0.total_cmp(&right.0)); + + let mut detected_to_tracked = vec![None; detected_clusters.len()]; + let mut tracked_taken = vec![false; self.tracked_clusters.len()]; + for (_, detected_index, tracked_index) in candidate_pairs { + if detected_to_tracked[detected_index].is_none() && !tracked_taken[tracked_index] { + detected_to_tracked[detected_index] = Some(tracked_index); + tracked_taken[tracked_index] = true; + } + } + + for (detected_index, cluster) in detected_clusters.iter_mut().enumerate() { + if let Some(tracked_index) = detected_to_tracked[detected_index] { + let tracked = &mut self.tracked_clusters[tracked_index]; + let current_count = cluster.event_count(); + tracked.centroid_x = cluster.centroid_x; + tracked.centroid_y = cluster.centroid_y; + tracked.last_seen_frame = self.frame_counter; + + if current_count > tracked.event_count { + tracked.event_count = current_count; + tracked.last_grown_frame = self.frame_counter; + tracked.frames_since_growth = 0; + tracked.cluster = cluster.clone(); + if temporal_enabled { + tracked.complete = false; + } + } else { + tracked.frames_since_growth = tracked.frames_since_growth.saturating_add(1); + if current_count == tracked.event_count { + tracked.cluster = cluster.clone(); + } + } + + if !temporal_enabled || tracked.frames_since_growth >= stable_frames { + tracked.complete = true; + } + + tracked.cluster.cluster_id = tracked.id; + tracked.cluster.complete = tracked.complete; + cluster.cluster_id = tracked.id; + cluster.complete = tracked.complete; + } else { + let cluster_id = self.next_cluster_id; + self.next_cluster_id = self.next_cluster_id.wrapping_add(1); + cluster.cluster_id = cluster_id; + cluster.complete = !temporal_enabled; + self.tracked_clusters.push(TrackedCluster { + id: cluster_id, + centroid_x: cluster.centroid_x, + centroid_y: cluster.centroid_y, + event_count: cluster.event_count(), + last_seen_frame: self.frame_counter, + last_grown_frame: self.frame_counter, + frames_since_growth: 0, + complete: cluster.complete, + emitted: false, + cluster: cluster.clone(), + }); + } + } + + for tracked in &mut self.tracked_clusters { + if tracked.last_seen_frame != self.frame_counter { + tracked.frames_since_growth = tracked.frames_since_growth.saturating_add(1); + if temporal_enabled && tracked.frames_since_growth >= stable_frames { + tracked.complete = true; + } + } + if !temporal_enabled { + tracked.complete = true; + } + tracked.cluster.cluster_id = tracked.id; + tracked.cluster.complete = tracked.complete; + } + + let mut published_clusters = Vec::new(); + for tracked in &mut self.tracked_clusters { + if tracked.complete && !tracked.emitted { + tracked.emitted = true; + let mut cluster = tracked.cluster.clone(); + cluster.cluster_id = tracked.id; + cluster.complete = true; + published_clusters.push(cluster); + } + } + + self.tracked_clusters.retain(|tracked| { + self.frame_counter.saturating_sub(tracked.last_seen_frame) as usize <= retention_frames + }); + + (detected_clusters, published_clusters) + } + + fn render_cluster_overlay( + &self, + frame: &PluginFrame<'_>, + visible_clusters: &[EveCluster], + output: &mut HostOutput<'_>, + ) { + let overlay_clusters: Vec<&EveCluster> = visible_clusters + .iter() + .filter(|cluster| cluster.complete || self.settings.show_provisional) + .collect(); + + if self.settings.show_boundaries && !overlay_clusters.is_empty() { + let (complete_pixels, provisional_pixels) = + boundary_pixels(&overlay_clusters, frame.width(), frame.height()); + if !complete_pixels.is_empty() { + output.add_highlight_pixels(&complete_pixels, COMPLETE_BOUNDARY_COLOR); + } + if !provisional_pixels.is_empty() { + output.add_highlight_pixels(&provisional_pixels, PROVISIONAL_BOUNDARY_COLOR); + } + } + + if self.settings.show_overlay && !overlay_clusters.is_empty() { + let stable_ids: Vec = overlay_clusters + .iter() + .map(|cluster| cluster.cluster_id.to_string()) + .collect(); + let markers: Vec = overlay_clusters + .iter() + .zip(stable_ids.iter()) + .map(|(cluster, stable_id)| FfiMarkerOverlayItem { + x: cluster.centroid_x as f32, + y: cluster.centroid_y as f32, + shape: FfiMarkerShape::FilledCircle, + size: 4.0, + color: FfiColorRgba::from_rgba(if cluster.complete { + COMPLETE_MARKER_COLOR + } else { + PROVISIONAL_MARKER_COLOR + }), + timestamp_us: cluster + .events + .last() + .map(|event| event.timestamp) + .unwrap_or(frame.window_end_us()), + has_timestamp: !cluster.events.is_empty(), + stable_id: stable_id.as_str().into(), + source_dataset_id: FfiString::empty(), + source_row_id: FfiString::empty(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(ACCEPTED_EVENTS_DATASET_ID), + Some(ACCEPTED_EVENTS_LAYER_ID), + Some(self.name()), + ); + } + } + pub fn reset(&mut self) { + self.reset_tracking_state(); + self.current_event_datasets = CandidateEventDatasets::default(); self.last_candidate_count = 0; + self.last_complete_visible_count = 0; + self.last_provisional_count = 0; self.last_event_count = 0; self.last_status = "Waiting for the next preview frame.".into(); + self.dataset_generation = self.dataset_generation.wrapping_add(1); } fn parse_usize(value: Value) -> Option { @@ -289,9 +677,20 @@ impl Plugin for EveSmlmCandidatePlugin { frame: &PluginFrame<'_>, output: &mut HostOutput<'_>, context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, + event_store: &EventStoreHandle<'_>, ) { - let candidates = self.analyze_frame(frame, frame.events(), output); + let (analysis_events, analysis_window_start_us, analysis_window_end_us, temporal_enabled) = + self.collect_analysis_events(frame, event_store); + let candidates = self.analyze_frame( + frame, + &analysis_events, + analysis_window_start_us, + analysis_window_end_us, + temporal_enabled, + output, + ); + self.event_buffer = analysis_events; + self.dataset_generation = self.dataset_generation.wrapping_add(1); if let Err(err) = context.publish(CTX_EVE_CANDIDATES, &candidates) { Self::warning( output, @@ -301,6 +700,12 @@ impl Plugin for EveSmlmCandidatePlugin { } } + fn capabilities(&self) -> PluginCapabilities { + PluginCapabilities { + retained_event_history: self.settings.lookback_us > 0, + } + } + fn settings_schema(&self) -> SettingsSchema { SettingsSchema { sections: vec![ @@ -373,10 +778,59 @@ impl Plugin for EveSmlmCandidatePlugin { }, ], }, + SettingsSection { + label: "Temporal aggregation".into(), + description: Some( + "Optionally cluster across retained event history and only publish clusters once they stop growing." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "lookback_us".into(), + label: "Lookback".into(), + tooltip: Some( + "How far back in retained event history to gather events before clustering. Set to 0 for single-frame behavior." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: 500_000, + default: i64::try_from(self.settings.lookback_us).unwrap_or(66_000), + suffix: Some(" us".into()), + }, + }, + SettingItem { + key: "stable_frames".into(), + label: "Stable frames".into(), + tooltip: Some( + "How many consecutive frames without cluster growth are required before a cluster is published to fitting." + .into(), + ), + kind: SettingKind::I64Slider { + min: 1, + max: 8, + default: i64::try_from(self.settings.stable_frames).unwrap_or(2), + suffix: Some(" frames".into()), + }, + }, + SettingItem { + key: "show_provisional".into(), + label: "Show provisional".into(), + tooltip: Some( + "Show still-growing clusters in the preview overlay and boundary layer." + .into(), + ), + kind: SettingKind::Bool { + default: self.settings.show_provisional, + }, + }, + ], + }, SettingsSection { label: "Refinement".into(), description: Some( - "Frame-based mode and eigenfeature filtering use these thresholds to reject broad or anisotropic clusters." + "Frame-based mode, eigenfeature filtering, and preview overlays use these thresholds and display controls." .into(), ), default_open: false, @@ -427,12 +881,26 @@ impl Plugin for EveSmlmCandidatePlugin { }, SettingItem { key: "show_overlay".into(), - label: "Show overlay".into(), - tooltip: Some("Highlight candidate centroids on the preview.".into()), + label: "Show centroids".into(), + tooltip: Some( + "Draw clickable centroid markers that link into the accepted candidate-events dataset." + .into(), + ), kind: SettingKind::Bool { default: self.settings.show_overlay, }, }, + SettingItem { + key: "show_boundaries".into(), + label: "Show boundaries".into(), + tooltip: Some( + "Draw 2-sigma eigenfeature ellipses or bounding boxes around visible clusters." + .into(), + ), + kind: SettingKind::Bool { + default: self.settings.show_boundaries, + }, + }, ], }, ], @@ -445,71 +913,99 @@ impl Plugin for EveSmlmCandidatePlugin { "polarity" => Some(json!(self.settings.polarity.index())), "epsilon_px" => Some(json!(self.settings.epsilon_px)), "min_events" => Some(json!(self.settings.min_events)), + "lookback_us" => Some(json!(self.settings.lookback_us)), + "stable_frames" => Some(json!(self.settings.stable_frames)), "max_spatial_extent_px" => Some(json!(self.settings.max_spatial_extent_px)), "min_isotropy" => Some(json!(self.settings.min_isotropy)), "threshold_factor" => Some(json!(self.settings.threshold_factor)), "fit_radius_px" => Some(json!(self.settings.fit_radius_px)), "max_candidates" => Some(json!(self.settings.max_candidates)), "show_overlay" => Some(json!(self.settings.show_overlay)), + "show_boundaries" => Some(json!(self.settings.show_boundaries)), + "show_provisional" => Some(json!(self.settings.show_provisional)), _ => None, } } fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + let mut reset_tracking = false; match key { "finding_method" => { let Some(value) = Self::parse_usize(value) else { return Err("finding_method must be an integer".into()); }; self.settings.finding_method = CandidateFindingMethod::from_index(value); + reset_tracking = true; } "polarity" => { let Some(value) = Self::parse_usize(value) else { return Err("polarity must be an integer".into()); }; self.settings.polarity = PolarityMode::from_index(value); + reset_tracking = true; } "epsilon_px" => { let Some(value) = value.as_f64() else { return Err("epsilon_px must be numeric".into()); }; self.settings.epsilon_px = value.clamp(1.0, 10.0); + reset_tracking = true; } "min_events" => { let Some(value) = Self::parse_usize(value) else { return Err("min_events must be an integer".into()); }; self.settings.min_events = value.clamp(1, 64); + reset_tracking = true; + } + "lookback_us" => { + let Some(value) = value.as_u64() else { + return Err("lookback_us must be an integer".into()); + }; + self.settings.lookback_us = value.min(500_000); + reset_tracking = true; + } + "stable_frames" => { + let Some(value) = Self::parse_usize(value) else { + return Err("stable_frames must be an integer".into()); + }; + self.settings.stable_frames = value.clamp(1, 8); + reset_tracking = true; } "max_spatial_extent_px" => { let Some(value) = value.as_f64() else { return Err("max_spatial_extent_px must be numeric".into()); }; self.settings.max_spatial_extent_px = value.clamp(1.0, 20.0); + reset_tracking = true; } "min_isotropy" => { let Some(value) = value.as_f64() else { return Err("min_isotropy must be numeric".into()); }; self.settings.min_isotropy = value.clamp(0.0, 1.0); + reset_tracking = true; } "threshold_factor" => { let Some(value) = value.as_f64() else { return Err("threshold_factor must be numeric".into()); }; self.settings.threshold_factor = value.clamp(0.5, 6.0); + reset_tracking = true; } "fit_radius_px" => { let Some(value) = Self::parse_usize(value) else { return Err("fit_radius_px must be an integer".into()); }; self.settings.fit_radius_px = value.clamp(1, 16); + reset_tracking = true; } "max_candidates" => { let Some(value) = Self::parse_usize(value) else { return Err("max_candidates must be an integer".into()); }; self.settings.max_candidates = value.clamp(1, 2048); + reset_tracking = true; } "show_overlay" => { let Some(value) = value.as_bool() else { @@ -517,14 +1013,30 @@ impl Plugin for EveSmlmCandidatePlugin { }; self.settings.show_overlay = value; } + "show_boundaries" => { + let Some(value) = value.as_bool() else { + return Err("show_boundaries must be a boolean".into()); + }; + self.settings.show_boundaries = value; + } + "show_provisional" => { + let Some(value) = value.as_bool() else { + return Err("show_provisional must be a boolean".into()); + }; + self.settings.show_provisional = value; + } _ => return Err(format!("unknown setting: {key}")), } + if reset_tracking { + self.reset_tracking_state(); + } + Ok(()) } fn status_entries(&self) -> Vec { - vec![ + let mut entries = vec![ StatusEntry::Text(self.last_status.clone()), StatusEntry::LabeledValue { label: "Events".into(), @@ -532,16 +1044,508 @@ impl Plugin for EveSmlmCandidatePlugin { color: None, }, StatusEntry::LabeledValue { - label: "Candidates".into(), + label: "Published".into(), value: self.last_candidate_count.to_string(), color: None, }, + StatusEntry::LabeledValue { + label: "Complete".into(), + value: self.last_complete_visible_count.to_string(), + color: None, + }, + StatusEntry::LabeledValue { + label: "Provisional".into(), + value: self.last_provisional_count.to_string(), + color: None, + }, StatusEntry::LabeledValue { label: "Method".into(), value: self.settings.finding_method.label().into(), color: None, }, - ] + ]; + if self.settings.lookback_us > 0 { + entries.push(StatusEntry::LabeledValue { + label: "Lookback".into(), + value: format!("{} us", self.settings.lookback_us), + color: None, + }); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + candidate_event_registry(&self.current_event_datasets) + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + let dataset = match dataset_id { + ACCEPTED_EVENTS_DATASET_ID => { + candidate_events_dataset(&self.current_event_datasets.accepted) + } + REJECTED_EVENTS_DATASET_ID => { + candidate_events_dataset(&self.current_event_datasets.rejected) + } + _ => return None, + }; + serde_json::to_vec(&dataset).ok() + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + ACCEPTED_EVENTS_DATASET_ID | REJECTED_EVENTS_DATASET_ID => self.dataset_generation, + _ => 0, + } + } +} + +fn boundary_label(method: CandidateFindingMethod) -> &'static str { + match method { + CandidateFindingMethod::FrameBased => "bounding boxes", + CandidateFindingMethod::Dbscan | CandidateFindingMethod::Eigenfeature => { + "2-sigma eigenfeature ellipses" + } + } +} + +fn boundary_pixels( + clusters: &[&EveCluster], + width: u16, + height: u16, +) -> (Vec, Vec) { + let mut complete = HashSet::new(); + let mut provisional = HashSet::new(); + + for cluster in clusters { + let target = if cluster.complete { + &mut complete + } else { + &mut provisional + }; + rasterize_cluster_boundary( + cluster.boundary.as_ref(), + width, + height, + !cluster.complete, + target, + ); + } + + let to_pixels = |points: HashSet<(u16, u16)>| { + let mut pixels: Vec<_> = points.into_iter().map(|(x, y)| FfiPixel { x, y }).collect(); + pixels.sort_by_key(|pixel| (pixel.y, pixel.x)); + pixels + }; + + (to_pixels(complete), to_pixels(provisional)) +} + +fn rasterize_cluster_boundary( + boundary: Option<&ClusterBoundary>, + width: u16, + height: u16, + dashed: bool, + out: &mut HashSet<(u16, u16)>, +) { + let Some(boundary) = boundary else { + return; + }; + + match boundary { + ClusterBoundary::BoundingBox { + x_min, + x_max, + y_min, + y_max, + } => { + for (step, x) in (*x_min..=*x_max).enumerate() { + if !dashed || step % 2 == 0 { + push_boundary_pixel(out, width, height, x as f64, f64::from(*y_min)); + push_boundary_pixel(out, width, height, x as f64, f64::from(*y_max)); + } + } + for (step, y) in (*y_min..=*y_max).enumerate() { + if !dashed || step % 2 == 0 { + push_boundary_pixel(out, width, height, f64::from(*x_min), y as f64); + push_boundary_pixel(out, width, height, f64::from(*x_max), y as f64); + } + } + } + ClusterBoundary::Ellipse { + cx, + cy, + semi_major, + semi_minor, + angle_rad, + } => { + let steps = ((semi_major.max(*semi_minor) * 10.0).ceil() as usize).clamp(24, 240); + let cos_angle = angle_rad.cos(); + let sin_angle = angle_rad.sin(); + for step in 0..=steps { + if dashed && step % 2 == 1 { + continue; + } + let theta = std::f64::consts::TAU * step as f64 / steps as f64; + let ellipse_x = semi_major * theta.cos(); + let ellipse_y = semi_minor * theta.sin(); + let rotated_x = ellipse_x * cos_angle - ellipse_y * sin_angle; + let rotated_y = ellipse_x * sin_angle + ellipse_y * cos_angle; + push_boundary_pixel(out, width, height, cx + rotated_x, cy + rotated_y); + } + } + } +} + +fn push_boundary_pixel(out: &mut HashSet<(u16, u16)>, width: u16, height: u16, x: f64, y: f64) { + let x = x.round(); + let y = y.round(); + if x < 0.0 || y < 0.0 { + return; + } + + let x = x as u16; + let y = y as u16; + if x < width && y < height { + out.insert((x, y)); + } +} + +fn candidate_event_registry(datasets: &CandidateEventDatasets) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: ACCEPTED_EVENTS_DATASET_ID.into(), + title: "Accepted EVE events".into(), + kind: HostDatasetKind::TableV1(candidate_events_schema( + datasets, + ACCEPTED_EVENTS_LAYER_ID, + "accepted candidate events", + "cluster_id", + )), + empty_message: "No accepted candidate events in the current analysis window." + .into(), + display: Some(candidate_event_display_metadata( + "Accepted candidate events", + ACCEPTED_EVENTS_COLOR, + )), + relations: vec![HostDatasetRelation { + target_dataset_id: "augur.evesmlm.current_localizations".into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }, + HostDatasetDescriptor { + id: REJECTED_EVENTS_DATASET_ID.into(), + title: "Rejected EVE events".into(), + kind: HostDatasetKind::TableV1(candidate_events_schema( + datasets, + REJECTED_EVENTS_LAYER_ID, + "rejected candidate events", + "event_id", + )), + empty_message: "No rejected candidate events in the current analysis window." + .into(), + display: Some(candidate_event_display_metadata( + "Rejected candidate events", + REJECTED_EVENTS_COLOR, + )), + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: ACCEPTED_EVENTS_COMPACT_VIEW_ID.into(), + title: "Accepted Events".into(), + dataset_id: ACCEPTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: REJECTED_EVENTS_COMPACT_VIEW_ID.into(), + title: "Rejected Events".into(), + dataset_id: REJECTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: ACCEPTED_EVENTS_TABLE_VIEW_ID.into(), + title: "Accepted Events".into(), + dataset_id: ACCEPTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, + HostViewDescriptor { + id: REJECTED_EVENTS_TABLE_VIEW_ID.into(), + title: "Rejected Events".into(), + dataset_id: REJECTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, + HostViewDescriptor { + id: ACCEPTED_EVENTS_3D_VIEW_ID.into(), + title: "Accepted Events 3D".into(), + dataset_id: ACCEPTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + HostViewDescriptor { + id: REJECTED_EVENTS_3D_VIEW_ID.into(), + title: "Rejected Events 3D".into(), + dataset_id: REJECTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), + } +} + +fn candidate_event_display_metadata( + layer_title: &str, + color: [u8; 4], +) -> HostDatasetDisplayMetadata { + HostDatasetDisplayMetadata { + layer_title: Some(layer_title.into()), + default_visibility: Some(true), + default_color: Some(color), + default_marker_shape: Some(HostMarkerShape::Point), + default_size: Some(2.5), + } +} + +fn candidate_events_schema( + datasets: &CandidateEventDatasets, + layer_id: &str, + semantic_label: &str, + row_id_column: &str, +) -> TableSchema { + TableSchema { + columns: vec![ + TableColumn { + id: "event_id".into(), + title: "Event ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "x_px".into(), + title: "X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "y_px".into(), + title: "Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "polarity".into(), + title: "Polarity".into(), + value_type: TableValueType::Bool, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: datasets + .sensor_dims + .map(|(width, height)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min: 0.0, + x_max: f64::from(width), + y_min: 0.0, + y_max: f64::from(height), + }), + coordinate_space_3d: datasets + .sensor_dims + .map(|(width, height)| TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min: 0.0, + x_max: f64::from(width), + y_min: 0.0, + y_max: f64::from(height), + z_min: datasets.frame_window_start_us as f64, + z_max: datasets + .frame_window_end_us + .max(datasets.frame_window_start_us) as f64, + }), + row_id_column: Some(row_id_column.into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(layer_id.into()), + semantic_label: Some(semantic_label.into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("timestamp_us".into()), + span_end_column: Some("timestamp_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "event_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: true, + label: None, + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + width_priority: Some(TableColumnWidthPriority::Medium), + hide_in_compact: false, + label: Some("Time".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: false, + label: Some("X".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: false, + label: Some("Y".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "polarity".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: false, + label: Some("Polarity".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + width_priority: Some(TableColumnWidthPriority::Medium), + hide_in_compact: false, + label: Some("Cluster".into()), + headline: row_id_column == "cluster_id", + }, + }, + ], + } +} + +fn candidate_events_dataset(rows: &[CandidateEventRow]) -> TableDatasetV1 { + TableDatasetV1::new(vec![ + TableColumnData { + column_id: "event_id".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.event_id).collect()), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.timestamp_us).collect()), + }, + TableColumnData { + column_id: "x_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.x_px).collect()), + }, + TableColumnData { + column_id: "y_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.y_px).collect()), + }, + TableColumnData { + column_id: "polarity".into(), + values: TableColumnValues::Bool(rows.iter().map(|row| row.polarity).collect()), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::String( + rows.iter().map(|row| row.cluster_id.clone()).collect(), + ), + }, + ]) + .expect("candidate event columns must stay aligned") +} + +fn candidate_event_row_id(event: &EveEvent, occurrence: u32) -> u64 { + event.timestamp + ^ u64::from(event.x).rotate_left(11) + ^ u64::from(event.y).rotate_left(23) + ^ u64::from(event.polarity as u8).rotate_left(37) + ^ u64::from(occurrence).rotate_left(47) +} + +fn build_candidate_event_datasets( + sensor_dims: (u16, u16), + frame_window_start_us: u64, + frame_window_end_us: u64, + events: &[EveEvent], + cluster_indices: &[Vec], + visible_clusters: &[EveCluster], +) -> CandidateEventDatasets { + let mut cluster_by_event = vec![None; events.len()]; + for (cluster, indices) in visible_clusters.iter().zip(cluster_indices.iter()) { + for &event_index in indices { + if let Some(slot) = cluster_by_event.get_mut(event_index) { + *slot = Some(cluster.cluster_id.to_string()); + } + } + } + + let mut accepted = Vec::new(); + let mut rejected = Vec::new(); + let mut seen_occurrences = HashMap::new(); + for (event_index, event) in events.iter().enumerate() { + let occurrence = seen_occurrences + .entry((event.timestamp, event.x, event.y, event.polarity)) + .or_insert(0u32); + let row = CandidateEventRow { + event_id: candidate_event_row_id(event, *occurrence), + x_px: f64::from(event.x), + y_px: f64::from(event.y), + timestamp_us: event.timestamp, + polarity: event.polarity, + cluster_id: cluster_by_event[event_index].clone().unwrap_or_default(), + }; + *occurrence = occurrence.saturating_add(1); + if cluster_by_event[event_index].is_some() { + accepted.push(row); + } else { + rejected.push(row); + } + } + + CandidateEventDatasets { + accepted, + rejected, + sensor_dims: Some(sensor_dims), + frame_window_start_us, + frame_window_end_us, } } @@ -555,7 +1559,46 @@ fn empty_candidates(frame: &PluginFrame<'_>, method: CandidateFindingMethod) -> } } -fn clusters_from_indices(events: &[EveEvent], cluster_indices: Vec>) -> Vec { +#[allow(clippy::too_many_arguments)] +fn cluster_boundary_for_indices( + events: &[EveEvent], + indices: &[usize], + centroid_x: f64, + centroid_y: f64, + x_min: u16, + x_max: u16, + y_min: u16, + y_max: u16, + method: CandidateFindingMethod, +) -> ClusterBoundary { + if matches!( + method, + CandidateFindingMethod::Dbscan | CandidateFindingMethod::Eigenfeature + ) { + if let Some(info) = eigenfeature::cluster_eigen_info(events, indices) { + return ClusterBoundary::Ellipse { + cx: centroid_x, + cy: centroid_y, + semi_major: (2.0 * info.lambda_1.max(0.0).sqrt()).max(1.0), + semi_minor: (2.0 * info.lambda_2.max(0.0).sqrt()).max(1.0), + angle_rad: info.angle_rad, + }; + } + } + + ClusterBoundary::BoundingBox { + x_min, + x_max, + y_min, + y_max, + } +} + +fn clusters_from_indices( + events: &[EveEvent], + cluster_indices: Vec>, + method: CandidateFindingMethod, +) -> Vec { cluster_indices .into_iter() .filter_map(|indices| { @@ -572,7 +1615,7 @@ fn clusters_from_indices(events: &[EveEvent], cluster_indices: Vec>) let mut y_min = u16::MAX; let mut y_max = 0; - for index in indices { + for &index in &indices { let event = events[index]; cluster_events.push(event); sum_x += f64::from(event.x); @@ -597,15 +1640,24 @@ fn clusters_from_indices(events: &[EveEvent], cluster_indices: Vec>) .collect(); histogram_entries.sort_by_key(|entry| (entry.1, entry.0)); + let centroid_x = sum_x / count; + let centroid_y = sum_y / count; + let boundary = cluster_boundary_for_indices( + events, &indices, centroid_x, centroid_y, x_min, x_max, y_min, y_max, method, + ); + Some(EveCluster { + cluster_id: 0, pixel_histogram: histogram_entries, events: cluster_events, - centroid_x: sum_x / count, - centroid_y: sum_y / count, + centroid_x, + centroid_y, x_min, x_max, y_min, y_max, + complete: false, + boundary: Some(boundary), }) }) .collect() @@ -871,7 +1923,11 @@ mod tests { event(10, 11, true, 4), ]; - let clusters = clusters_from_indices(&events, vec![vec![0, 1, 2, 3]]); + let clusters = clusters_from_indices( + &events, + vec![vec![0, 1, 2, 3]], + CandidateFindingMethod::Dbscan, + ); assert_eq!(clusters.len(), 1); let cluster = &clusters[0]; assert_eq!(cluster.event_count(), 4); @@ -880,6 +1936,7 @@ mod tests { assert_eq!(cluster.pixel_histogram.len(), 3); assert!((cluster.centroid_x - 10.25).abs() < 1e-6); assert!((cluster.centroid_y - 10.25).abs() < 1e-6); + assert!(cluster.boundary.is_some()); } #[test] @@ -892,7 +1949,7 @@ mod tests { }; let image = build_analysis_image(&frame.into_plugin_frame(), &events); - let index = 1usize * 6 + 2usize; + let index = 6usize + 2usize; assert_eq!(image[index], 20.0); } @@ -909,6 +1966,139 @@ mod tests { assert!(maxima.iter().any(|(x, y, _)| (*x, *y) == (3, 3))); } + #[test] + fn candidate_event_datasets_split_accepted_and_rejected_events() { + let events = vec![ + event(10, 10, true, 101), + event(11, 10, true, 102), + event(12, 10, false, 103), + event(30, 20, true, 104), + ]; + let visible_clusters = vec![EveCluster { + cluster_id: 42, + pixel_histogram: vec![(10, 10, 1, 0), (12, 10, 0, 1)], + events: vec![events[0], events[2]], + centroid_x: 11.0, + centroid_y: 10.0, + x_min: 10, + x_max: 12, + y_min: 10, + y_max: 10, + complete: false, + boundary: Some(ClusterBoundary::BoundingBox { + x_min: 10, + x_max: 12, + y_min: 10, + y_max: 10, + }), + }]; + + let datasets = build_candidate_event_datasets( + (32, 24), + 100, + 101, + &events, + &[vec![0, 2]], + &visible_clusters, + ); + assert_eq!(datasets.accepted.len(), 2); + assert_eq!(datasets.rejected.len(), 2); + assert_eq!(datasets.accepted[0].cluster_id, "42"); + assert_eq!(datasets.rejected[0].cluster_id, ""); + } + + #[test] + fn candidate_event_registry_exposes_table_and_3d_views() { + let registry = candidate_event_registry(&CandidateEventDatasets { + accepted: Vec::new(), + rejected: Vec::new(), + sensor_dims: Some((128, 64)), + frame_window_start_us: 10, + frame_window_end_us: 20, + }); + assert_eq!(registry.datasets.len(), 2); + assert_eq!(registry.views.len(), 6); + assert_eq!(registry.views[0].id, ACCEPTED_EVENTS_COMPACT_VIEW_ID); + assert_eq!(registry.views[0].title, "Accepted Events"); + assert!(matches!(registry.views[0].kind, HostViewKind::CompactTable)); + assert_eq!(registry.views[2].id, ACCEPTED_EVENTS_TABLE_VIEW_ID); + assert_eq!(registry.views[2].title, "Accepted Events"); + assert!(matches!(registry.views[2].kind, HostViewKind::TableWindow)); + assert_eq!(registry.views[4].id, ACCEPTED_EVENTS_3D_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.row_id_column.as_deref(), Some("cluster_id")); + let cluster_column = schema.column("cluster_id").expect("cluster id column"); + assert_eq!(cluster_column.value_type, TableValueType::String); + assert_eq!( + schema + .column_display("cluster_id") + .map(|display| display.headline), + Some(true) + ); + assert_eq!( + schema + .coordinate_space_3d + .as_ref() + .map(|space| space.z_column.as_str()), + Some("timestamp_us") + ); + let rejected_schema = match ®istry.datasets[1].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(rejected_schema.row_id_column.as_deref(), Some("event_id")); + } + + #[test] + fn temporal_tracking_waits_for_stable_frames_before_publishing() { + let mut plugin = EveSmlmCandidatePlugin::default(); + plugin.settings.stable_frames = 2; + + let make_cluster = || EveCluster { + cluster_id: 0, + pixel_histogram: vec![(10, 10, 3, 0), (11, 10, 2, 0)], + events: vec![ + event(10, 10, true, 1), + event(10, 10, true, 2), + event(10, 10, true, 3), + event(11, 10, true, 4), + event(11, 10, true, 5), + ], + centroid_x: 10.4, + centroid_y: 10.0, + x_min: 10, + x_max: 11, + y_min: 10, + y_max: 10, + complete: false, + boundary: Some(ClusterBoundary::BoundingBox { + x_min: 10, + x_max: 11, + y_min: 10, + y_max: 10, + }), + }; + + let (visible, published) = plugin.update_tracked_clusters(vec![make_cluster()], true); + assert_eq!(visible.len(), 1); + assert!(!visible[0].complete); + assert!(published.is_empty()); + + let (visible, published) = plugin.update_tracked_clusters(vec![make_cluster()], true); + assert_eq!(visible.len(), 1); + assert!(!visible[0].complete); + assert!(published.is_empty()); + + let (visible, published) = plugin.update_tracked_clusters(vec![make_cluster()], true); + assert_eq!(visible.len(), 1); + assert!(visible[0].complete); + assert_eq!(published.len(), 1); + assert_eq!(published[0].cluster_id, visible[0].cluster_id); + } + struct TestFrame { width: u16, height: u16, diff --git a/plugins/evesmlm-candidates/src/types.rs b/plugins/evesmlm-candidates/src/types.rs index 1f1fbd1..6bf8b1b 100644 --- a/plugins/evesmlm-candidates/src/types.rs +++ b/plugins/evesmlm-candidates/src/types.rs @@ -3,6 +3,10 @@ use serde::{Deserialize, Serialize}; pub const CTX_EVE_CANDIDATES: &str = "augur.evesmlm.candidates"; +fn default_cluster_complete() -> bool { + true +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum CandidateFindingMethod { @@ -33,7 +37,7 @@ pub struct EveEvent { impl From for EveEvent { fn from(value: FfiCdEvent) -> Self { Self { - timestamp: value.timestamp, + timestamp: value.timestamp_us(), x: value.x, y: value.y, polarity: value.polarity != 0, @@ -41,8 +45,28 @@ impl From for EveEvent { } } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ClusterBoundary { + BoundingBox { + x_min: u16, + x_max: u16, + y_min: u16, + y_max: u16, + }, + Ellipse { + cx: f64, + cy: f64, + semi_major: f64, + semi_minor: f64, + angle_rad: f64, + }, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct EveCluster { + #[serde(default)] + pub cluster_id: u64, /// Per-pixel event histogram: (x, y, n_positive, n_negative) pub pixel_histogram: Vec<(u16, u16, u32, u32)>, /// All raw events assigned to this cluster. @@ -55,6 +79,10 @@ pub struct EveCluster { pub x_max: u16, pub y_min: u16, pub y_max: u16, + #[serde(default = "default_cluster_complete")] + pub complete: bool, + #[serde(default)] + pub boundary: Option, } impl EveCluster { @@ -96,3 +124,17 @@ pub struct EveCandidates { pub n_events_processed: usize, pub finding_method: CandidateFindingMethod, } + +#[derive(Debug, Clone)] +pub(crate) struct TrackedCluster { + pub id: u64, + pub centroid_x: f64, + pub centroid_y: f64, + pub event_count: usize, + pub last_seen_frame: u64, + pub last_grown_frame: u64, + pub frames_since_growth: usize, + pub complete: bool, + pub emitted: bool, + pub cluster: EveCluster, +} diff --git a/plugins/evesmlm-fitting/README.md b/plugins/evesmlm-fitting/README.md index 7fdf312..ba5be95 100644 --- a/plugins/evesmlm-fitting/README.md +++ b/plugins/evesmlm-fitting/README.md @@ -23,6 +23,7 @@ Sub-pixel localization for eveSMLM candidate clusters. The plugin consumes `EveC | Sigma max | `200.0` nm | Upper accepted sigma bound for sigma-producing methods | | Max fit residual | `0.5` | Reject fits above this residual | | Show overlay | `true` | Highlight accepted localization positions | +| Show rejected | `false` | Draw rejected fits as linked diamond markers | AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `GlobalSettings`. This plugin uses the host `nm_per_pixel` value automatically for sigma filtering when it is available, while retaining a hidden fallback for older hosts. @@ -34,11 +35,24 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global - `EveLocalizationResults` on `augur.evesmlm.localization_results` - `LocalizationResults` on `augur.localization.results` for compatibility with plugins such as Focus Metrics -- the compact host-view dataset `augur.evesmlm.current_localizations` +- the shared host-view dataset `augur.evesmlm.current_localizations` +- the rejected-fit investigation dataset `augur.evesmlm.rejected_fits` ## Host View -The plugin declares the compact analysis-panel view `augur.evesmlm.current_localizations.compact`. If `EVE Post-Processing` is also enabled, the host resolves that same view id to the later post-processing stage instead. +The plugin declares the shared current-localizations dataset plus: + +- the compact analysis-panel view `augur.evesmlm.current_localizations.compact` +- a linked 3D scatter view over the same dataset +- compact, windowed, and 3D views for rejected fits + +That dataset now carries stable row ids, timestamps, 2D/3D coordinate metadata, and layer/display metadata so the host can keep selection stable across tables, overlays, and 3D inspection. + +Rejected fits are exposed as structured rows with `cluster_id`, timestamps, fit metrics, and a categorical rejection reason so fit failures and threshold rejections can be inspected directly instead of inferred from a counter alone. + +Rejected-fit selection is currently local to the rejected-fits dataset. Matching `cluster_id` values do not create cross-dataset linking back to candidate-event rows because AugurRS stable row keys are scoped by dataset id. + +If `EVE Post-Processing` is also enabled, the host resolves those same dataset/view ids to the later post-processing stage instead. ## Dependencies diff --git a/plugins/evesmlm-fitting/src/lib.rs b/plugins/evesmlm-fitting/src/lib.rs index bd7a370..31b83b3 100644 --- a/plugins/evesmlm-fitting/src/lib.rs +++ b/plugins/evesmlm-fitting/src/lib.rs @@ -3,6 +3,8 @@ //! Consumes `EveCandidates` and localizes each raw-event cluster to //! sub-pixel precision with a configurable fitting backend. +use std::collections::HashMap; + pub mod gaussian; pub mod log_gaussian; pub mod mean_xy; @@ -11,48 +13,140 @@ pub mod radial_symmetry; pub mod types; use augur_plugin_api::{ - export_plugin, AnalysisSeverity, EventStoreHandle, FfiSubpixelMarker, GlobalSettings, + export_plugin, AnalysisSeverity, EventStoreHandle, FfiColorRgba, FfiMarkerOverlayItem, + FfiMarkerShape, GlobalSettings, HostActionDescriptor, HostActionRequestQueue, HostActionScope, HostContext, HostOutput, Plugin, PluginFrame, PluginInput, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, CTX_GLOBAL_SETTINGS, + CTX_INVESTIGATION_ACTION_REQUESTS, HOST_ACTION_CLUSTER_ROWS_PARAM, }; use augur_plugin_api::{ - HostDatasetDescriptor, HostDatasetKind, HostViewDescriptor, HostViewKind, HostViewPlacement, - HostViewRegistry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, + HostDatasetDescriptor, HostDatasetDisplayMetadata, HostDatasetKind, HostDatasetRelation, + HostMarkerShape, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, + TableColumn, TableColumnData, TableColumnDisplayEntry, TableColumnDisplayFormat, + TableColumnDisplayMetadata, TableColumnValues, TableCoordinateSpace2d, TableCoordinateSpace3d, + TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, }; pub use augur_plugin_evesmlm_candidates::{ - CandidateFindingMethod, EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, + EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, }; use augur_plugin_types::{Localization, LocalizationResults, CTX_LOCALIZATION_RESULTS}; use serde_json::{json, Value}; -pub use types::{EveLocalization, EveLocalizationResults, FitMethod, CTX_EVE_LOCALIZATION_RESULTS}; +pub use types::{ + EveLocalization, EveLocalizationResults, FitMethod, RejectedFitRow, RejectionReason, + CTX_EVE_LOCALIZATION_RESULTS, +}; const OVERLAY_COLOR: [u8; 4] = [60, 220, 140, 220]; const CANDIDATE_DEPENDENCY: [&str; 1] = ["EVE Candidate Finding"]; pub const CURRENT_LOCALIZATIONS_DATASET_ID: &str = "augur.evesmlm.current_localizations"; +pub const CURRENT_LOCALIZATIONS_LAYER_ID: &str = "augur.layer.evesmlm.current_localizations"; pub const CURRENT_LOCALIZATIONS_VIEW_ID: &str = "augur.evesmlm.current_localizations.compact"; +pub const CURRENT_LOCALIZATIONS_3D_VIEW_ID: &str = "augur.evesmlm.current_localizations.scatter3d"; +pub const REJECTED_FITS_DATASET_ID: &str = "augur.evesmlm.rejected_fits"; +pub const REJECTED_FITS_LAYER_ID: &str = "augur.layer.evesmlm.rejected_fits"; +pub const REJECTED_FITS_COMPACT_VIEW_ID: &str = "augur.evesmlm.rejected_fits.compact"; +pub const REJECTED_FITS_TABLE_VIEW_ID: &str = "augur.evesmlm.rejected_fits.table"; +pub const REJECTED_FITS_3D_VIEW_ID: &str = "augur.evesmlm.rejected_fits.scatter3d"; + +pub const REFIT_PREVIEW_DATASET_ID: &str = "augur.evesmlm.refit_preview"; +pub const REFIT_PREVIEW_LAYER_ID: &str = "augur.layer.evesmlm.refit_preview"; +pub const REFIT_PREVIEW_VIEW_ID: &str = "augur.evesmlm.refit_preview.compact"; + +pub const ACCEPTED_CANDIDATE_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; + +pub const ACTION_REFIT_CLUSTER: &str = "augur.evesmlm.refit_cluster"; +pub const ACTION_COMMIT_REFIT: &str = "augur.evesmlm.commit_refit"; +pub const ACTION_DISCARD_REFIT: &str = "augur.evesmlm.discard_refit"; pub fn current_localizations_registry() -> HostViewRegistry { + current_localizations_registry_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_registry_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> HostViewRegistry { HostViewRegistry { datasets: vec![HostDatasetDescriptor { id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), title: "Current EVE localizations".into(), - kind: HostDatasetKind::TableV1(current_localizations_schema()), + kind: HostDatasetKind::TableV1(current_localizations_schema_for_results( + results, + sensor_dims, + )), empty_message: "No EVE localizations in the current frame.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Current EVE localizations".into()), + default_visibility: Some(true), + default_color: Some([90, 170, 255, 255]), + default_marker_shape: Some(HostMarkerShape::Cross), + default_size: Some(6.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], }], - views: vec![HostViewDescriptor { - id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), - title: "Current Localizations".into(), - dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }], + views: vec![ + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), + title: "Current Localizations".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_3D_VIEW_ID.into(), + title: "Current Localizations 3D".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), } } pub fn current_localizations_schema() -> TableSchema { + current_localizations_schema_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_schema_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> TableSchema { TableSchema { columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_start_us".into(), + title: "Span Start (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_end_us".into(), + title: "Span End (us)".into(), + value_type: TableValueType::U64, + }, TableColumn { id: "x_px".into(), title: "X (px)".into(), @@ -78,13 +172,172 @@ pub fn current_localizations_schema() -> TableSchema { title: "Events".into(), value_type: TableValueType::U64, }, + TableColumn { + id: "polarity_balance".into(), + title: "Polarity balance".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_residual".into(), + title: "Fit residual".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_method".into(), + title: "Fit method".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: current_localizations_2d_space(results, sensor_dims), + coordinate_space_3d: current_localizations_3d_space(results, sensor_dims), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(CURRENT_LOCALIZATIONS_LAYER_ID.into()), + semantic_label: Some("localizations".into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("span_start_us".into()), + span_end_column: Some("span_end_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "row_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_start_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span start".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_end_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span end".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_residual".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_method".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + ..Default::default() + }, + }, ], - coordinate_space_2d: None, } } pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { TableDatasetV1::new(vec![ + TableColumnData { + column_id: "row_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(localization_row_id) + .collect(), + ), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.cluster_id) + .collect(), + ), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.timestamp_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_start_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_start_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_end_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_end_us) + .collect(), + ), + }, TableColumnData { column_id: "x_px".into(), values: TableColumnValues::F64( @@ -127,187 +380,1543 @@ pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableD .collect(), ), }, + TableColumnData { + column_id: "polarity_balance".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.polarity_balance) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_residual".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.fit_residual) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_method".into(), + values: TableColumnValues::String( + results + .localizations + .iter() + .map(|value| value.fit_method.label().to_owned()) + .collect(), + ), + }, ]) .expect("current localization columns should stay aligned") } -#[derive(Debug, Clone, Copy)] -pub(crate) struct FitEstimate { - pub x: f64, - pub y: f64, - pub sigma_x: f64, - pub sigma_y: f64, - pub residual: f64, +fn current_localizations_2d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results)) + .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min, + x_max, + y_min, + y_max, + }) } -#[derive(Debug, Clone)] -pub struct FittingSettings { - pub fit_method: FitMethod, - pub nm_per_pixel: f64, - pub sigma_min_nm: f64, - pub sigma_max_nm: f64, - pub max_fit_residual: f64, - pub show_overlay: bool, +fn current_localizations_3d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + let (x_min, x_max, y_min, y_max) = sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results))?; + let (z_min, z_max) = localization_time_bounds(results)?; + Some(TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min, + x_max, + y_min, + y_max, + z_min, + z_max, + }) } -impl Default for FittingSettings { - fn default() -> Self { - Self { - fit_method: FitMethod::LogGaussian, - nm_per_pixel: 65.0, - sigma_min_nm: 80.0, - sigma_max_nm: 200.0, - max_fit_residual: 0.5, - show_overlay: true, - } +pub fn refit_preview_registry_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: REFIT_PREVIEW_DATASET_ID.into(), + title: "Refit preview".into(), + kind: HostDatasetKind::TableV1(refit_preview_schema(results, sensor_dims)), + empty_message: "No pending re-fit preview.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Refit preview".into()), + default_visibility: Some(true), + default_color: Some([255, 210, 90, 240]), + default_marker_shape: Some(HostMarkerShape::Circle), + default_size: Some(8.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }], + views: vec![HostViewDescriptor { + id: REFIT_PREVIEW_VIEW_ID.into(), + title: "Refit Preview".into(), + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }], + actions: Vec::new(), } } -pub struct EveSmlmFittingPlugin { - enabled: bool, - settings: FittingSettings, - current_results: EveLocalizationResults, - last_localization_count: usize, - last_rejection_count: usize, - last_status: String, - dataset_generation: u64, +pub fn refit_preview_schema( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> TableSchema { + let mut schema = current_localizations_schema_for_results(results, sensor_dims); + schema.layer_id = Some(REFIT_PREVIEW_LAYER_ID.into()); + schema.semantic_label = Some("refit preview".into()); + schema } -impl Default for EveSmlmFittingPlugin { - fn default() -> Self { - Self { - enabled: false, - settings: FittingSettings::default(), - current_results: EveLocalizationResults::default(), - last_localization_count: 0, - last_rejection_count: 0, - last_status: - "Enable the plugin to fit EVE candidate clusters to sub-pixel localizations.".into(), - dataset_generation: 0, +pub fn refit_preview_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { + current_localizations_dataset(results) +} + +fn refit_action_param_schema() -> SettingsSchema { + SettingsSchema { + sections: vec![SettingsSection { + label: "Refit parameters".into(), + description: Some( + "Re-run the chosen cluster's fit with these parameters and preview the result before committing." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "fit_method".into(), + label: "Method".into(), + tooltip: Some("Fitting backend to use for this cluster.".into()), + kind: SettingKind::Enum { + variants: vec![ + FitMethod::LogGaussian.label().into(), + FitMethod::Gaussian.label().into(), + FitMethod::RadialSymmetry.label().into(), + FitMethod::Phasor.label().into(), + FitMethod::MeanXY.label().into(), + ], + default: FitMethod::LogGaussian.index(), + }, + }, + SettingItem { + key: "sigma_min_nm".into(), + label: "Sigma min".into(), + tooltip: Some("Reject fits with sigma below this bound.".into()), + kind: SettingKind::F64Slider { + min: 10.0, + max: 500.0, + default: FittingSettings::default().sigma_min_nm, + suffix: Some(" nm".into()), + }, + }, + SettingItem { + key: "sigma_max_nm".into(), + label: "Sigma max".into(), + tooltip: Some("Reject fits with sigma above this bound.".into()), + kind: SettingKind::F64Slider { + min: 10.0, + max: 500.0, + default: FittingSettings::default().sigma_max_nm, + suffix: Some(" nm".into()), + }, + }, + SettingItem { + key: "max_fit_residual".into(), + label: "Max residual".into(), + tooltip: Some("Reject fits whose residual exceeds this threshold.".into()), + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: FittingSettings::default().max_fit_residual, + }, + }, + ], + }], + } +} + +fn localization_xy_bounds(results: &EveLocalizationResults) -> Option<(f64, f64, f64, f64)> { + let mut localizations = results.localizations.iter(); + let first = localizations.next()?; + let mut x_min = first.x; + let mut x_max = first.x; + let mut y_min = first.y; + let mut y_max = first.y; + for localization in localizations { + x_min = x_min.min(localization.x); + x_max = x_max.max(localization.x); + y_min = y_min.min(localization.y); + y_max = y_max.max(localization.y); + } + Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) +} + +fn localization_time_bounds(results: &EveLocalizationResults) -> Option<(f64, f64)> { + if let Some(first) = results.localizations.first() { + let mut min_time = first.timestamp_us; + let mut max_time = first.timestamp_us; + for localization in &results.localizations { + min_time = min_time.min(localization.timestamp_us); + max_time = max_time.max(localization.timestamp_us); } + return Some((min_time as f64, max_time.max(min_time) as f64)); + } + + if results.frame_window_end_us >= results.frame_window_start_us { + return Some(( + results.frame_window_start_us as f64, + results.frame_window_end_us as f64, + )); } + + None } -impl EveSmlmFittingPlugin { - fn nm_per_pixel(&self, context: &HostContext<'_>) -> f64 { - context - .get::(CTX_GLOBAL_SETTINGS) - .ok() - .flatten() - .map(|settings| settings.nm_per_pixel) - .unwrap_or(self.settings.nm_per_pixel) +pub fn localization_row_id(localization: &EveLocalization) -> u64 { + localization.cluster_id.rotate_left(3) + ^ localization.timestamp_us + ^ localization.x.to_bits().rotate_left(7) + ^ localization.y.to_bits().rotate_left(19) + ^ localization.sigma_x.to_bits().rotate_left(31) + ^ localization.sigma_y.to_bits().rotate_left(43) + ^ localization.fit_residual.to_bits().rotate_left(53) + ^ (localization.n_events as u64).rotate_left(11) + ^ (localization.fit_method.index() as u64).rotate_left(59) + ^ localization.span_start_us.rotate_left(17) + ^ localization.span_end_us.rotate_left(29) +} + +pub fn rejected_fit_row_id(row: &RejectedFitRow) -> u64 { + row.timestamp_us + ^ row.cluster_id.rotate_left(7) + ^ row.x.to_bits().rotate_left(19) + ^ row.y.to_bits().rotate_left(31) + ^ row.fit_residual.to_bits().rotate_left(43) + ^ (row.rejection_reason as u64).rotate_left(53) + ^ row.span_start_us.rotate_left(17) + ^ row.span_end_us.rotate_left(29) +} + +fn rejected_fits_registry( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: REJECTED_FITS_DATASET_ID.into(), + title: "Rejected EVE fits".into(), + kind: HostDatasetKind::TableV1(rejected_fits_schema( + rows, + sensor_dims, + frame_window_start_us, + frame_window_end_us, + )), + empty_message: "No rejected EVE fits in the current analysis window.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Rejected EVE fits".into()), + default_visibility: Some(false), + default_color: Some([255, 90, 90, 200]), + default_marker_shape: Some(HostMarkerShape::Diamond), + default_size: Some(5.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }], + views: vec![ + HostViewDescriptor { + id: REJECTED_FITS_COMPACT_VIEW_ID.into(), + title: "Rejected Fits".into(), + dataset_id: REJECTED_FITS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: REJECTED_FITS_TABLE_VIEW_ID.into(), + title: "Rejected Fits Table".into(), + dataset_id: REJECTED_FITS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, + HostViewDescriptor { + id: REJECTED_FITS_3D_VIEW_ID.into(), + title: "Rejected Fits 3D".into(), + dataset_id: REJECTED_FITS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), } +} + +fn rejected_fits_schema( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +) -> TableSchema { + TableSchema { + columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_start_us".into(), + title: "Span Start (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_end_us".into(), + title: "Span End (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "x_px".into(), + title: "X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "y_px".into(), + title: "Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_x_px".into(), + title: "Sigma X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_y_px".into(), + title: "Sigma Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_residual".into(), + title: "Fit residual".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "n_events".into(), + title: "Events".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "polarity_balance".into(), + title: "Polarity balance".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "rejection_reason".into(), + title: "Rejection reason".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: rejected_fits_2d_space(rows, sensor_dims), + coordinate_space_3d: rejected_fits_3d_space( + rows, + sensor_dims, + frame_window_start_us, + frame_window_end_us, + ), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(REJECTED_FITS_LAYER_ID.into()), + semantic_label: Some("rejected fits".into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("span_start_us".into()), + span_end_column: Some("span_end_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "row_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_start_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span start".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_end_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span end".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_residual".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "rejection_reason".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + headline: true, + ..Default::default() + }, + }, + ], + } +} + +fn rejected_fits_dataset(rows: &[RejectedFitRow]) -> TableDatasetV1 { + TableDatasetV1::new(vec![ + TableColumnData { + column_id: "row_id".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.row_id).collect()), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.cluster_id).collect()), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.timestamp_us).collect()), + }, + TableColumnData { + column_id: "span_start_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.span_start_us).collect()), + }, + TableColumnData { + column_id: "span_end_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.span_end_us).collect()), + }, + TableColumnData { + column_id: "x_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.x).collect()), + }, + TableColumnData { + column_id: "y_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.y).collect()), + }, + TableColumnData { + column_id: "sigma_x_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.sigma_x).collect()), + }, + TableColumnData { + column_id: "sigma_y_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.sigma_y).collect()), + }, + TableColumnData { + column_id: "fit_residual".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.fit_residual).collect()), + }, + TableColumnData { + column_id: "n_events".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.n_events).collect()), + }, + TableColumnData { + column_id: "polarity_balance".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.polarity_balance).collect()), + }, + TableColumnData { + column_id: "rejection_reason".into(), + values: TableColumnValues::String( + rows.iter() + .map(|row| row.rejection_reason.as_str().to_owned()) + .collect(), + ), + }, + ]) + .expect("rejected-fit columns should stay aligned") +} + +fn rejected_fits_2d_space( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, +) -> Option { + sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| rejected_fit_xy_bounds(rows)) + .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min, + x_max, + y_min, + y_max, + }) +} + +fn rejected_fits_3d_space( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +) -> Option { + let (x_min, x_max, y_min, y_max) = sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| rejected_fit_xy_bounds(rows))?; + let (z_min, z_max) = rejected_fit_time_bounds(rows) + .unwrap_or((frame_window_start_us as f64, frame_window_end_us as f64)); + Some(TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min, + x_max, + y_min, + y_max, + z_min, + z_max, + }) +} + +fn rejected_fit_xy_bounds(rows: &[RejectedFitRow]) -> Option<(f64, f64, f64, f64)> { + let mut rows = rows.iter(); + let first = rows.next()?; + let mut x_min = first.x; + let mut x_max = first.x; + let mut y_min = first.y; + let mut y_max = first.y; + for row in rows { + x_min = x_min.min(row.x); + x_max = x_max.max(row.x); + y_min = y_min.min(row.y); + y_max = y_max.max(row.y); + } + Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) +} + +fn rejected_fit_time_bounds(rows: &[RejectedFitRow]) -> Option<(f64, f64)> { + let mut rows = rows.iter(); + let first = rows.next()?; + let mut min_time = first.timestamp_us; + let mut max_time = first.timestamp_us; + for row in rows { + min_time = min_time.min(row.timestamp_us); + max_time = max_time.max(row.timestamp_us); + } + Some((min_time as f64, max_time.max(min_time) as f64)) +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct FitEstimate { + pub x: f64, + pub y: f64, + pub sigma_x: f64, + pub sigma_y: f64, + pub residual: f64, +} + +#[derive(Debug, Clone)] +pub struct FittingSettings { + pub fit_method: FitMethod, + pub nm_per_pixel: f64, + pub sigma_min_nm: f64, + pub sigma_max_nm: f64, + pub max_fit_residual: f64, + pub show_overlay: bool, + pub show_rejected_overlay: bool, +} + +impl Default for FittingSettings { + fn default() -> Self { + Self { + fit_method: FitMethod::LogGaussian, + nm_per_pixel: 65.0, + sigma_min_nm: 80.0, + sigma_max_nm: 200.0, + max_fit_residual: 0.5, + show_overlay: true, + show_rejected_overlay: false, + } + } +} + +pub struct EveSmlmFittingPlugin { + enabled: bool, + settings: FittingSettings, + current_results: EveLocalizationResults, + current_rejected_fits: Vec, + host_results: EveLocalizationResults, + host_rejected_fits: Vec, + sensor_dims: Option<(u16, u16)>, + last_localization_count: usize, + last_rejection_count: usize, + last_fit_failure_count: usize, + last_sigma_rejection_count: usize, + last_residual_rejection_count: usize, + last_status: String, + dataset_generation: u64, + refit_preview_results: EveLocalizationResults, + /// Parallel to `refit_preview_results.localizations`: for each preview + /// row, the `row_id` of the current localization it should replace on + /// commit (or `None` if commit should append). + refit_preview_replaces: Vec>, + last_consumed_action_request_id: u64, + last_action_notice: Option, +} + +impl Default for EveSmlmFittingPlugin { + fn default() -> Self { + Self { + enabled: false, + settings: FittingSettings::default(), + current_results: EveLocalizationResults::default(), + current_rejected_fits: Vec::new(), + host_results: EveLocalizationResults::default(), + host_rejected_fits: Vec::new(), + sensor_dims: None, + last_localization_count: 0, + last_rejection_count: 0, + last_fit_failure_count: 0, + last_sigma_rejection_count: 0, + last_residual_rejection_count: 0, + last_status: + "Enable the plugin to fit EVE candidate clusters to sub-pixel localizations.".into(), + dataset_generation: 0, + refit_preview_results: EveLocalizationResults::default(), + refit_preview_replaces: Vec::new(), + last_consumed_action_request_id: 0, + last_action_notice: None, + } + } +} + +impl EveSmlmFittingPlugin { + fn nm_per_pixel(&self, context: &HostContext<'_>) -> f64 { + context + .get::(CTX_GLOBAL_SETTINGS) + .ok() + .flatten() + .map(|settings| settings.nm_per_pixel) + .unwrap_or(self.settings.nm_per_pixel) + } + + fn sync_sensor_dims(&mut self, context: &HostContext<'_>, frame: &PluginFrame<'_>) { + self.sensor_dims = context + .get::(CTX_GLOBAL_SETTINGS) + .ok() + .flatten() + .map(|settings| (settings.sensor_width, settings.sensor_height)) + .or(Some((frame.width(), frame.height()))); + } + + fn analyze_candidates( + &mut self, + candidates: Option<&EveCandidates>, + output: &mut HostOutput<'_>, + nm_per_pixel: f64, + ) -> ( + EveLocalizationResults, + LocalizationResults, + Vec, + ) { + let Some(candidates) = candidates else { + self.last_localization_count = 0; + self.last_rejection_count = 0; + self.last_fit_failure_count = 0; + self.last_sigma_rejection_count = 0; + self.last_residual_rejection_count = 0; + self.last_status = "Waiting for EVE Candidate Finding.".into(); + Self::warning( + output, + AnalysisSeverity::Info, + "EVE fitting requires candidate clusters from EVE Candidate Finding.", + ); + return ( + EveLocalizationResults::default(), + LocalizationResults::default(), + Vec::new(), + ); + }; + + let mut localizations = Vec::new(); + let mut rejected_fits = Vec::new(); + let mut fit_failures = 0usize; + let mut sigma_rejections = 0usize; + let mut residual_rejections = 0usize; + for cluster in &candidates.clusters { + let (span_start_us, span_end_us) = cluster_time_span(cluster); + let timestamp_fallback = estimate_timestamp_us( + &cluster.events, + cluster.centroid_x, + cluster.centroid_y, + cluster_extent_radius(cluster), + ); + let Some(fit) = fit_cluster(cluster, self.settings.fit_method) else { + fit_failures += 1; + let mut rejected = RejectedFitRow { + row_id: 0, + cluster_id: cluster.cluster_id, + x: cluster.centroid_x, + y: cluster.centroid_y, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.0, + n_events: cluster.event_count() as u64, + polarity_balance: cluster.polarity_balance(), + rejection_reason: RejectionReason::FitFailed, + timestamp_us: timestamp_fallback, + span_start_us, + span_end_us, + }; + rejected.row_id = rejected_fit_row_id(&rejected); + rejected_fits.push(rejected); + continue; + }; + + if self.settings.fit_method.produces_sigma() { + let sigma_x_nm = fit.sigma_x * nm_per_pixel; + let sigma_y_nm = fit.sigma_y * nm_per_pixel; + if sigma_x_nm < self.settings.sigma_min_nm + || sigma_x_nm > self.settings.sigma_max_nm + || sigma_y_nm < self.settings.sigma_min_nm + || sigma_y_nm > self.settings.sigma_max_nm + { + sigma_rejections += 1; + let timestamp_us = estimate_timestamp_us( + &cluster.events, + fit.x, + fit.y, + fit_radius(cluster, &fit), + ); + let mut rejected = RejectedFitRow { + row_id: 0, + cluster_id: cluster.cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + fit_residual: fit.residual, + n_events: cluster.event_count() as u64, + polarity_balance: cluster.polarity_balance(), + rejection_reason: RejectionReason::SigmaOutOfBounds, + timestamp_us, + span_start_us, + span_end_us, + }; + rejected.row_id = rejected_fit_row_id(&rejected); + rejected_fits.push(rejected); + continue; + } + } + + if fit.residual > self.settings.max_fit_residual { + residual_rejections += 1; + let timestamp_us = + estimate_timestamp_us(&cluster.events, fit.x, fit.y, fit_radius(cluster, &fit)); + let mut rejected = RejectedFitRow { + row_id: 0, + cluster_id: cluster.cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + fit_residual: fit.residual, + n_events: cluster.event_count() as u64, + polarity_balance: cluster.polarity_balance(), + rejection_reason: RejectionReason::ResidualTooHigh, + timestamp_us, + span_start_us, + span_end_us, + }; + rejected.row_id = rejected_fit_row_id(&rejected); + rejected_fits.push(rejected); + continue; + } + + localizations.push(EveLocalization { + cluster_id: cluster.cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + timestamp_us: estimate_timestamp_us( + &cluster.events, + fit.x, + fit.y, + fit_radius(cluster, &fit), + ), + span_start_us, + span_end_us, + n_events: cluster.event_count(), + polarity_balance: cluster.polarity_balance(), + fit_residual: fit.residual, + fit_method: self.settings.fit_method, + }); + } + + self.last_localization_count = localizations.len(); + self.last_fit_failure_count = fit_failures; + self.last_sigma_rejection_count = sigma_rejections; + self.last_residual_rejection_count = residual_rejections; + self.last_rejection_count = fit_failures + sigma_rejections + residual_rejections; + self.last_status = format!( + "{} accepted, {} rejected ({} fit failures, {} sigma bounds, {} residual) with {}.", + self.last_localization_count, + self.last_rejection_count, + self.last_fit_failure_count, + self.last_sigma_rejection_count, + self.last_residual_rejection_count, + self.settings.fit_method.label() + ); + + if self.settings.show_overlay && !localizations.is_empty() { + let stable_ids: Vec = localizations + .iter() + .map(|localization| localization_row_id(localization).to_string()) + .collect(); + let markers: Vec = localizations + .iter() + .zip(stable_ids.iter()) + .map(|(localization, stable_id)| FfiMarkerOverlayItem { + x: localization.x as f32, + y: localization.y as f32, + shape: FfiMarkerShape::Cross, + size: 6.0, + color: FfiColorRgba::from_rgba(OVERLAY_COLOR), + timestamp_us: localization.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(CURRENT_LOCALIZATIONS_DATASET_ID), + Some(CURRENT_LOCALIZATIONS_LAYER_ID), + Some(self.name()), + ); + } + + if self.settings.show_rejected_overlay && !rejected_fits.is_empty() { + let stable_ids: Vec = rejected_fits + .iter() + .map(|row| row.row_id.to_string()) + .collect(); + let markers: Vec = rejected_fits + .iter() + .zip(stable_ids.iter()) + .map(|(row, stable_id)| FfiMarkerOverlayItem { + x: row.x as f32, + y: row.y as f32, + shape: FfiMarkerShape::Diamond, + size: 5.0, + color: FfiColorRgba::from_rgba([255, 90, 90, 180]), + timestamp_us: row.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: REJECTED_FITS_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(REJECTED_FITS_DATASET_ID), + Some(REJECTED_FITS_LAYER_ID), + Some(self.name()), + ); + } + + let eve_results = EveLocalizationResults { + localizations, + frame_window_start_us: candidates.frame_window_start_us, + frame_window_end_us: candidates.frame_window_end_us, + }; + let compatibility_results = to_localization_results(&eve_results); + + (eve_results, compatibility_results, rejected_fits) + } + + pub fn reset(&mut self) { + self.current_results = EveLocalizationResults::default(); + self.current_rejected_fits.clear(); + self.host_results = EveLocalizationResults::default(); + self.host_rejected_fits.clear(); + self.sensor_dims = None; + self.last_localization_count = 0; + self.last_rejection_count = 0; + self.last_fit_failure_count = 0; + self.last_sigma_rejection_count = 0; + self.last_residual_rejection_count = 0; + self.last_status = "Waiting for the next candidate set.".into(); + self.refit_preview_results = EveLocalizationResults::default(); + self.refit_preview_replaces.clear(); + self.last_action_notice = None; + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn parse_usize(value: Value) -> Option { + value.as_u64().and_then(|value| usize::try_from(value).ok()) + } + + fn update_history_bounds(results: &mut EveLocalizationResults) { + let Some(first) = results.localizations.first() else { + results.frame_window_start_us = 0; + results.frame_window_end_us = 0; + return; + }; + let mut start = first.span_start_us; + let mut end = first.span_end_us.max(first.span_start_us); + for localization in &results.localizations[1..] { + start = start.min(localization.span_start_us); + end = end.max(localization.span_end_us.max(localization.span_start_us)); + } + results.frame_window_start_us = start; + results.frame_window_end_us = end; + } + + fn upsert_history_localization(&mut self, localization: EveLocalization) { + self.host_rejected_fits + .retain(|row| row.cluster_id != localization.cluster_id); + if let Some(index) = self + .host_results + .localizations + .iter() + .position(|existing| existing.cluster_id == localization.cluster_id) + { + self.host_results.localizations[index] = localization; + } else { + self.host_results.localizations.push(localization); + } + self.host_results.localizations.sort_by_key(|row| { + ( + row.span_start_us, + row.span_end_us, + row.timestamp_us, + row.cluster_id, + ) + }); + Self::update_history_bounds(&mut self.host_results); + } + + fn upsert_history_rejected_fit(&mut self, row: RejectedFitRow) { + if self + .host_results + .localizations + .iter() + .any(|localization| localization.cluster_id == row.cluster_id) + { + return; + } + if let Some(index) = self + .host_rejected_fits + .iter() + .position(|existing| existing.cluster_id == row.cluster_id) + { + self.host_rejected_fits[index] = row; + } else { + self.host_rejected_fits.push(row); + } + self.host_rejected_fits.sort_by_key(|entry| { + ( + entry.span_start_us, + entry.span_end_us, + entry.timestamp_us, + entry.cluster_id, + ) + }); + } + + fn integrate_frame_history( + &mut self, + localizations: &[EveLocalization], + rejected_fits: &[RejectedFitRow], + ) { + for localization in localizations.iter().cloned() { + self.upsert_history_localization(localization); + } + for row in rejected_fits.iter().cloned() { + self.upsert_history_rejected_fit(row); + } + } + + fn parse_u64_field(value: &Value) -> Option { + value + .as_u64() + .or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok())) + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + } + + fn parse_u16_field(value: &Value) -> Option { + Self::parse_u64_field(value) + .and_then(|value| u16::try_from(value).ok()) + .or_else(|| { + value + .as_f64() + .map(|value| value.round().clamp(0.0, f64::from(u16::MAX)) as u16) + }) + } + + fn parse_bool_field(value: &Value) -> Option { + value + .as_bool() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + } + + fn cluster_from_action_params(params: &Value, expected_cluster_id: u64) -> Option { + let rows = params.get(HOST_ACTION_CLUSTER_ROWS_PARAM)?.as_array()?; + if rows.is_empty() { + return None; + } + + let mut events = Vec::with_capacity(rows.len()); + let mut pixel_histogram: HashMap<(u16, u16), (u32, u32)> = HashMap::new(); + let mut sum_x = 0.0; + let mut sum_y = 0.0; + let mut count: f64 = 0.0; + let mut x_min = u16::MAX; + let mut x_max = 0u16; + let mut y_min = u16::MAX; + let mut y_max = 0u16; + + for row in rows { + let object = row.as_object()?; + let cluster_id = Self::parse_u64_field(object.get("cluster_id")?)?; + if cluster_id != expected_cluster_id { + return None; + } + let x = Self::parse_u16_field(object.get("x_px")?)?; + let y = Self::parse_u16_field(object.get("y_px")?)?; + let timestamp = Self::parse_u64_field(object.get("timestamp_us")?)?; + let polarity = Self::parse_bool_field(object.get("polarity")?)?; + + events.push(EveEvent { + timestamp, + x, + y, + polarity, + }); + + let entry = pixel_histogram.entry((x, y)).or_insert((0, 0)); + if polarity { + entry.0 = entry.0.saturating_add(1); + } else { + entry.1 = entry.1.saturating_add(1); + } + x_min = x_min.min(x); + x_max = x_max.max(x); + y_min = y_min.min(y); + y_max = y_max.max(y); + sum_x += f64::from(x); + sum_y += f64::from(y); + count += 1.0; + } + + if events.is_empty() { + return None; + } + + let mut pixel_histogram: Vec<_> = pixel_histogram + .into_iter() + .map(|((x, y), (positive, negative))| (x, y, positive, negative)) + .collect(); + pixel_histogram.sort_by_key(|(x, y, _, _)| (*y, *x)); + + Some(EveCluster { + cluster_id: expected_cluster_id, + pixel_histogram, + events, + centroid_x: sum_x / count.max(1.0), + centroid_y: sum_y / count.max(1.0), + x_min, + x_max, + y_min, + y_max, + complete: true, + boundary: None, + }) + } + + fn warning(output: &mut HostOutput<'_>, severity: AnalysisSeverity, message: &str) { + output.add_warning("EVE Candidate Fitting", severity, message); + } + + fn handle_action_requests( + &mut self, + context: &mut HostContext<'_>, + output: &mut HostOutput<'_>, + candidates: Option<&EveCandidates>, + nm_per_pixel: f64, + ) { + let queue = match context + .get_persistent::(CTX_INVESTIGATION_ACTION_REQUESTS) + { + Ok(Some(queue)) => queue, + Ok(None) => return, + Err(err) => { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Reading action requests failed: {err}"), + ); + return; + } + }; + + let mut handled_any = false; + for request in &queue.requests { + if request.request_id <= self.last_consumed_action_request_id { + continue; + } + match request.action_id.as_str() { + ACTION_REFIT_CLUSTER => { + self.handle_refit_cluster(request, output, candidates, nm_per_pixel); + handled_any = true; + } + ACTION_COMMIT_REFIT => { + self.handle_commit_refit(request, output); + handled_any = true; + } + ACTION_DISCARD_REFIT => { + self.handle_discard_refit(request, output); + handled_any = true; + } + _ => continue, + } + self.last_consumed_action_request_id = request.request_id; + } + + if handled_any { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + } + + fn handle_refit_cluster( + &mut self, + request: &augur_plugin_api::HostActionRequest, + output: &mut HostOutput<'_>, + candidates: Option<&EveCandidates>, + nm_per_pixel: f64, + ) { + use augur_plugin_api::HostActionScopePayload; + let (dataset_id, group_column, group_value) = match &request.scope_payload { + HostActionScopePayload::Cluster { + dataset_id, + group_column, + group_value, + } => ( + dataset_id.clone(), + group_column.clone(), + group_value.clone(), + ), + _ => { + Self::warning( + output, + AnalysisSeverity::Warning, + "Re-fit action requires a Cluster scope payload.", + ); + return; + } + }; + if dataset_id != ACCEPTED_CANDIDATE_EVENTS_DATASET_ID || group_column != "cluster_id" { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!( + "Ignoring re-fit request for unsupported scope ({dataset_id}/{group_column})." + ), + ); + return; + } + + let cluster_id: u64 = match group_value.parse() { + Ok(value) => value, + Err(_) => { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Re-fit request has non-numeric cluster id: {group_value}"), + ); + return; + } + }; + + let params = &request.params; + let cluster_from_params = Self::cluster_from_action_params(params, cluster_id); + let cluster_from_candidates = candidates.and_then(|candidates| { + candidates + .clusters + .iter() + .find(|cluster| cluster.cluster_id == cluster_id) + .cloned() + }); + let Some(cluster) = cluster_from_params.or(cluster_from_candidates) else { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Re-fit request for cluster {cluster_id} has no usable cluster snapshot."), + ); + return; + }; + let fit_method = params + .get("fit_method") + .and_then(|value| Self::parse_usize(value.clone())) + .map(FitMethod::from_index) + .unwrap_or(self.settings.fit_method); + let sigma_min_nm = params + .get("sigma_min_nm") + .and_then(Value::as_f64) + .unwrap_or(self.settings.sigma_min_nm); + let sigma_max_nm = params + .get("sigma_max_nm") + .and_then(Value::as_f64) + .unwrap_or(self.settings.sigma_max_nm); + let max_fit_residual = params + .get("max_fit_residual") + .and_then(Value::as_f64) + .unwrap_or(self.settings.max_fit_residual); + + let Some(fit) = fit_cluster(&cluster, fit_method) else { + self.last_action_notice = Some(format!("Re-fit failed for cluster {cluster_id}.")); + Self::warning( + output, + AnalysisSeverity::Info, + &format!("Re-fit for cluster {cluster_id} did not converge."), + ); + return; + }; + + if fit_method.produces_sigma() { + let sigma_x_nm = fit.sigma_x * nm_per_pixel; + let sigma_y_nm = fit.sigma_y * nm_per_pixel; + if sigma_x_nm < sigma_min_nm + || sigma_x_nm > sigma_max_nm + || sigma_y_nm < sigma_min_nm + || sigma_y_nm > sigma_max_nm + { + self.last_action_notice = Some(format!( + "Re-fit for cluster {cluster_id} is outside sigma bounds." + )); + Self::warning( + output, + AnalysisSeverity::Info, + &format!("Re-fit for cluster {cluster_id} rejected by sigma bounds."), + ); + return; + } + } + + if fit.residual > max_fit_residual { + self.last_action_notice = Some(format!( + "Re-fit for cluster {cluster_id} exceeds residual threshold." + )); + Self::warning( + output, + AnalysisSeverity::Info, + &format!("Re-fit for cluster {cluster_id} rejected by residual threshold."), + ); + return; + } + + let timestamp_us = + estimate_timestamp_us(&cluster.events, fit.x, fit.y, fit_radius(&cluster, &fit)); + let (span_start_us, span_end_us) = cluster_time_span(&cluster); + let new_localization = EveLocalization { + cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + timestamp_us, + span_start_us, + span_end_us, + n_events: cluster.event_count(), + polarity_balance: cluster.polarity_balance(), + fit_residual: fit.residual, + fit_method, + }; + + let replaces = find_current_localization_for_cluster(&self.host_results, &cluster) + .map(localization_row_id); + + self.refit_preview_results + .localizations + .push(new_localization); + self.refit_preview_replaces.push(replaces); + Self::update_history_bounds(&mut self.refit_preview_results); + + self.last_action_notice = Some(format!( + "Re-fit preview added for cluster {cluster_id} ({}).", + fit_method.label() + )); + } + + fn handle_commit_refit( + &mut self, + request: &augur_plugin_api::HostActionRequest, + output: &mut HostOutput<'_>, + ) { + use augur_plugin_api::HostActionScopePayload; + let (dataset_id, row_id) = match &request.scope_payload { + HostActionScopePayload::Row { dataset_id, row_id } => { + (dataset_id.clone(), row_id.clone()) + } + _ => { + Self::warning( + output, + AnalysisSeverity::Warning, + "Commit action requires a Row scope payload.", + ); + return; + } + }; + if dataset_id != REFIT_PREVIEW_DATASET_ID { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Ignoring commit for unsupported dataset {dataset_id}."), + ); + return; + } + + let target_row_id: u64 = match row_id.parse() { + Ok(value) => value, + Err(_) => { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Commit row_id is not numeric: {row_id}"), + ); + return; + } + }; - fn analyze_candidates( - &mut self, - candidates: Option<&EveCandidates>, - output: &mut HostOutput<'_>, - nm_per_pixel: f64, - ) -> (EveLocalizationResults, LocalizationResults) { - let Some(candidates) = candidates else { - self.last_localization_count = 0; - self.last_rejection_count = 0; - self.last_status = "Waiting for EVE Candidate Finding.".into(); + let index = self + .refit_preview_results + .localizations + .iter() + .position(|localization| localization_row_id(localization) == target_row_id); + let Some(index) = index else { Self::warning( output, AnalysisSeverity::Info, - "EVE fitting requires candidate clusters from EVE Candidate Finding.", - ); - return ( - EveLocalizationResults::default(), - LocalizationResults::default(), + &format!("Commit row {target_row_id} is not in the preview."), ); + return; }; - let mut localizations = Vec::new(); - let mut rejected = 0; - for cluster in &candidates.clusters { - let Some(fit) = fit_cluster(cluster, self.settings.fit_method) else { - rejected += 1; - continue; - }; - - if self.settings.fit_method.produces_sigma() { - let sigma_x_nm = fit.sigma_x * nm_per_pixel; - let sigma_y_nm = fit.sigma_y * nm_per_pixel; - if sigma_x_nm < self.settings.sigma_min_nm - || sigma_x_nm > self.settings.sigma_max_nm - || sigma_y_nm < self.settings.sigma_min_nm - || sigma_y_nm > self.settings.sigma_max_nm - { - rejected += 1; - continue; - } - } - - if fit.residual > self.settings.max_fit_residual { - rejected += 1; - continue; - } + let localization = self.refit_preview_results.localizations.remove(index); + self.refit_preview_replaces.remove(index); + let cluster_id = localization.cluster_id; + self.upsert_history_localization(localization.clone()); + self.host_rejected_fits + .retain(|row| row.cluster_id != cluster_id); - localizations.push(EveLocalization { - x: fit.x, - y: fit.y, - sigma_x: fit.sigma_x, - sigma_y: fit.sigma_y, - timestamp_us: estimate_timestamp_us( - &cluster.events, - fit.x, - fit.y, - fit_radius(cluster, &fit), - ), - n_events: cluster.event_count(), - polarity_balance: cluster.polarity_balance(), - fit_residual: fit.residual, - fit_method: self.settings.fit_method, - }); + if let Some(old_index) = self + .current_results + .localizations + .iter() + .position(|entry| entry.cluster_id == cluster_id) + { + self.current_results.localizations[old_index] = localization; } - self.last_localization_count = localizations.len(); - self.last_rejection_count = rejected; - self.last_status = format!( - "{} localizations accepted, {} rejected with {}.", - self.last_localization_count, - self.last_rejection_count, - self.settings.fit_method.label() - ); + Self::update_history_bounds(&mut self.refit_preview_results); - if self.settings.show_overlay && !localizations.is_empty() { - let markers: Vec = localizations - .iter() - .map(|localization| FfiSubpixelMarker { - x: localization.x as f32, - y: localization.y as f32, - }) - .collect(); - output.add_crosshair_markers(&markers, OVERLAY_COLOR, 5); - } + self.last_action_notice = Some(format!("Committed refit preview row {target_row_id}.")); + } - let eve_results = EveLocalizationResults { - localizations, - frame_window_start_us: candidates.frame_window_start_us, - frame_window_end_us: candidates.frame_window_end_us, + fn handle_discard_refit( + &mut self, + request: &augur_plugin_api::HostActionRequest, + output: &mut HostOutput<'_>, + ) { + use augur_plugin_api::HostActionScopePayload; + let dataset_id = match &request.scope_payload { + HostActionScopePayload::Dataset { dataset_id } => dataset_id.clone(), + _ => { + Self::warning( + output, + AnalysisSeverity::Warning, + "Discard action requires a Dataset scope payload.", + ); + return; + } }; - let compatibility_results = to_localization_results(&eve_results); + if dataset_id != REFIT_PREVIEW_DATASET_ID { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Ignoring discard for unsupported dataset {dataset_id}."), + ); + return; + } - (eve_results, compatibility_results) + let dropped = self.refit_preview_results.localizations.len(); + self.refit_preview_results = EveLocalizationResults::default(); + self.refit_preview_replaces.clear(); + self.last_action_notice = Some(format!("Discarded {dropped} preview row(s).")); } - pub fn reset(&mut self) { - self.current_results = EveLocalizationResults::default(); - self.last_localization_count = 0; - self.last_rejection_count = 0; - self.last_status = "Waiting for the next candidate set.".into(); - self.dataset_generation = self.dataset_generation.wrapping_add(1); + fn emit_refit_preview_overlay(&self, output: &mut HostOutput<'_>) { + let localizations = &self.refit_preview_results.localizations; + let stable_ids: Vec = localizations + .iter() + .map(|localization| localization_row_id(localization).to_string()) + .collect(); + let markers: Vec = localizations + .iter() + .zip(stable_ids.iter()) + .map(|(localization, stable_id)| FfiMarkerOverlayItem { + x: localization.x as f32, + y: localization.y as f32, + shape: FfiMarkerShape::FilledCircle, + size: 8.0, + color: FfiColorRgba::from_rgba([255, 210, 90, 240]), + timestamp_us: localization.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(REFIT_PREVIEW_DATASET_ID), + Some(REFIT_PREVIEW_LAYER_ID), + Some(self.name()), + ); } +} - fn parse_usize(value: Value) -> Option { - value.as_u64().and_then(|value| usize::try_from(value).ok()) +fn find_current_localization_for_cluster<'a>( + results: &'a EveLocalizationResults, + cluster: &EveCluster, +) -> Option<&'a EveLocalization> { + if let Some(localization) = results + .localizations + .iter() + .find(|localization| localization.cluster_id == cluster.cluster_id) + { + return Some(localization); + } + let timestamp_range_us: i64 = 2_000; + let mut best: Option<(f64, &'a EveLocalization)> = None; + for localization in &results.localizations { + let dt = (localization.timestamp_us as i64) + .saturating_sub_unsigned(cluster_anchor_timestamp(cluster)); + if dt.abs() > timestamp_range_us { + continue; + } + let dx = localization.x - cluster.centroid_x; + let dy = localization.y - cluster.centroid_y; + let score = dx * dx + dy * dy + (dt as f64).powi(2) * 1e-6; + if best.map_or(true, |(b, _)| score < b) { + best = Some((score, localization)); + } } + best.map(|(_, localization)| localization) +} - fn warning(output: &mut HostOutput<'_>, severity: AnalysisSeverity, message: &str) { - output.add_warning("EVE Candidate Fitting", severity, message); +fn cluster_anchor_timestamp(cluster: &EveCluster) -> u64 { + if cluster.events.is_empty() { + return 0; } + let sum: u128 = cluster + .events + .iter() + .map(|event| event.timestamp as u128) + .sum(); + (sum / cluster.events.len() as u128) as u64 } impl Plugin for EveSmlmFittingPlugin { @@ -344,11 +1953,12 @@ impl Plugin for EveSmlmFittingPlugin { fn process_frame( &mut self, - _frame: &PluginFrame<'_>, + frame: &PluginFrame<'_>, output: &mut HostOutput<'_>, context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { + self.sync_sensor_dims(context, frame); let nm_per_pixel = self.nm_per_pixel(context); let candidates = match context.get::(CTX_EVE_CANDIDATES) { Ok(value) => value, @@ -362,11 +1972,22 @@ impl Plugin for EveSmlmFittingPlugin { } }; - let (eve_results, compatibility) = + let (eve_results, _compatibility, rejected_fits) = self.analyze_candidates(candidates.as_ref(), output, nm_per_pixel); self.current_results = eve_results.clone(); + self.current_rejected_fits = rejected_fits.clone(); + self.integrate_frame_history(&eve_results.localizations, &rejected_fits); self.dataset_generation = self.dataset_generation.wrapping_add(1); - if let Err(err) = context.publish(CTX_EVE_LOCALIZATION_RESULTS, &eve_results) { + + self.handle_action_requests(context, output, candidates.as_ref(), nm_per_pixel); + + if self.settings.show_overlay && !self.refit_preview_results.localizations.is_empty() { + self.emit_refit_preview_overlay(output); + } + + let published_results = self.current_results.clone(); + let compatibility = to_localization_results(&published_results); + if let Err(err) = context.publish(CTX_EVE_LOCALIZATION_RESULTS, &published_results) { Self::warning( output, AnalysisSeverity::Warning, @@ -462,6 +2083,16 @@ impl Plugin for EveSmlmFittingPlugin { default: self.settings.show_overlay, }, }, + SettingItem { + key: "show_rejected_overlay".into(), + label: "Show rejected".into(), + tooltip: Some( + "Draw rejected fits as linked diamond markers in the preview.".into(), + ), + kind: SettingKind::Bool { + default: self.settings.show_rejected_overlay, + }, + }, ], }], } @@ -474,6 +2105,7 @@ impl Plugin for EveSmlmFittingPlugin { "sigma_max_nm" => Some(json!(self.settings.sigma_max_nm)), "max_fit_residual" => Some(json!(self.settings.max_fit_residual)), "show_overlay" => Some(json!(self.settings.show_overlay)), + "show_rejected_overlay" => Some(json!(self.settings.show_rejected_overlay)), _ => None, } } @@ -520,6 +2152,12 @@ impl Plugin for EveSmlmFittingPlugin { }; self.settings.show_overlay = value; } + "show_rejected_overlay" => { + let Some(value) = value.as_bool() else { + return Err("show_rejected_overlay must be a boolean".into()); + }; + self.settings.show_rejected_overlay = value; + } _ => return Err(format!("unknown setting: {key}")), } @@ -527,7 +2165,7 @@ impl Plugin for EveSmlmFittingPlugin { } fn status_entries(&self) -> Vec { - vec![ + let mut entries = vec![ StatusEntry::Text(self.last_status.clone()), StatusEntry::LabeledValue { label: "Accepted".into(), @@ -544,26 +2182,95 @@ impl Plugin for EveSmlmFittingPlugin { value: self.settings.fit_method.label().into(), color: None, }, - ] + ]; + if self.last_rejection_count > 0 { + entries.push(StatusEntry::LabeledValue { + label: "Fit fail".into(), + value: self.last_fit_failure_count.to_string(), + color: None, + }); + entries.push(StatusEntry::LabeledValue { + label: "Sigma".into(), + value: self.last_sigma_rejection_count.to_string(), + color: None, + }); + entries.push(StatusEntry::LabeledValue { + label: "Residual".into(), + value: self.last_residual_rejection_count.to_string(), + color: None, + }); + } + entries } fn host_views(&self) -> HostViewRegistry { - current_localizations_registry() + let mut registry = + current_localizations_registry_for_results(&self.host_results, self.sensor_dims); + let rejected_registry = rejected_fits_registry( + &self.host_rejected_fits, + self.sensor_dims, + self.host_results.frame_window_start_us, + self.host_results.frame_window_end_us, + ); + registry.datasets.extend(rejected_registry.datasets); + registry.views.extend(rejected_registry.views); + let preview_registry = + refit_preview_registry_for_results(&self.refit_preview_results, self.sensor_dims); + registry.datasets.extend(preview_registry.datasets); + registry.views.extend(preview_registry.views); + + let param_schema = serde_json::to_value(refit_action_param_schema()).ok(); + registry.actions = vec![ + HostActionDescriptor { + id: ACTION_REFIT_CLUSTER.into(), + title: "Re-fit cluster…".into(), + scope: HostActionScope::Cluster { + dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + group_column: "cluster_id".into(), + }, + param_schema, + }, + HostActionDescriptor { + id: ACTION_COMMIT_REFIT.into(), + title: "Commit refit".into(), + scope: HostActionScope::Row { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_DISCARD_REFIT.into(), + title: "Discard refit preview".into(), + scope: HostActionScope::Dataset { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + }, + param_schema: None, + }, + ]; + registry } fn host_view_dataset(&self, dataset_id: &str) -> Option> { - if dataset_id != CURRENT_LOCALIZATIONS_DATASET_ID { - return None; + match dataset_id { + CURRENT_LOCALIZATIONS_DATASET_ID => { + serde_json::to_vec(¤t_localizations_dataset(&self.host_results)).ok() + } + REJECTED_FITS_DATASET_ID => { + serde_json::to_vec(&rejected_fits_dataset(&self.host_rejected_fits)).ok() + } + REFIT_PREVIEW_DATASET_ID => { + serde_json::to_vec(&refit_preview_dataset(&self.refit_preview_results)).ok() + } + _ => None, } - - serde_json::to_vec(¤t_localizations_dataset(&self.current_results)).ok() } fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - if dataset_id == CURRENT_LOCALIZATIONS_DATASET_ID { - self.dataset_generation - } else { - 0 + match dataset_id { + CURRENT_LOCALIZATIONS_DATASET_ID + | REJECTED_FITS_DATASET_ID + | REFIT_PREVIEW_DATASET_ID => self.dataset_generation, + _ => 0, } } } @@ -578,14 +2285,31 @@ fn fit_cluster(cluster: &EveCluster, method: FitMethod) -> Option { } } +fn cluster_extent_radius(cluster: &EveCluster) -> f64 { + let dx = f64::from(cluster.x_max.saturating_sub(cluster.x_min)) + 1.0; + let dy = f64::from(cluster.y_max.saturating_sub(cluster.y_min)) + 1.0; + 0.5 * dx.max(dy).max(1.0) +} + fn fit_radius(cluster: &EveCluster, fit: &FitEstimate) -> f64 { if fit.sigma_x > 0.0 && fit.sigma_y > 0.0 { 2.5 * fit.sigma_x.max(fit.sigma_y).max(1.0) } else { - let dx = f64::from(cluster.x_max.saturating_sub(cluster.x_min)) + 1.0; - let dy = f64::from(cluster.y_max.saturating_sub(cluster.y_min)) + 1.0; - 0.5 * dx.max(dy).max(1.0) + cluster_extent_radius(cluster) + } +} + +fn cluster_time_span(cluster: &EveCluster) -> (u64, u64) { + let Some(first) = cluster.events.first() else { + return (0, 0); + }; + let mut start = first.timestamp; + let mut end = first.timestamp; + for event in &cluster.events[1..] { + start = start.min(event.timestamp); + end = end.max(event.timestamp); } + (start, end.max(start)) } fn estimate_timestamp_us(events: &[EveEvent], x: f64, y: f64, radius: f64) -> u64 { @@ -654,6 +2378,23 @@ mod tests { } } + fn localization(x: f64, y: f64, timestamp_us: u64) -> EveLocalization { + EveLocalization { + cluster_id: timestamp_us, + x, + y, + sigma_x: 0.7, + sigma_y: 0.8, + timestamp_us, + span_start_us: timestamp_us.saturating_sub(1), + span_end_us: timestamp_us.saturating_add(1), + n_events: 7, + polarity_balance: 0.1, + fit_residual: 0.02, + fit_method: FitMethod::LogGaussian, + } + } + fn cluster_from_histogram(entries: &[(u16, u16, u32)]) -> EveCluster { let mut pixel_histogram = Vec::new(); let mut events = Vec::new(); @@ -682,6 +2423,7 @@ mod tests { } EveCluster { + cluster_id: 0, pixel_histogram, events, centroid_x: if total > 0.0 { sum_x / total } else { 0.0 }, @@ -690,6 +2432,8 @@ mod tests { x_max, y_min, y_max, + complete: true, + boundary: None, } } @@ -799,6 +2543,7 @@ mod tests { #[test] fn empty_cluster_returns_none() { let cluster = EveCluster { + cluster_id: 0, pixel_histogram: Vec::new(), events: Vec::new(), centroid_x: 0.0, @@ -807,6 +2552,8 @@ mod tests { x_max: 0, y_min: 0, y_max: 0, + complete: true, + boundary: None, }; assert!(mean_xy::fit(&cluster).is_none()); @@ -819,33 +2566,442 @@ mod tests { let registry = current_localizations_registry(); assert_eq!(registry.datasets.len(), 1); - assert_eq!(registry.views.len(), 1); + assert_eq!(registry.views.len(), 2); assert_eq!(registry.datasets[0].id, CURRENT_LOCALIZATIONS_DATASET_ID); assert_eq!(registry.views[0].id, CURRENT_LOCALIZATIONS_VIEW_ID); + assert_eq!(registry.views[1].id, CURRENT_LOCALIZATIONS_3D_VIEW_ID); + assert!(registry.datasets[0].display.is_some()); } #[test] fn host_view_dataset_is_columnar_and_aligned() { let dataset = current_localizations_dataset(&EveLocalizationResults { - localizations: vec![EveLocalization { - x: 1.5, - y: 2.5, - sigma_x: 0.7, - sigma_y: 0.8, - timestamp_us: 10, - n_events: 7, - polarity_balance: 0.1, - fit_residual: 0.02, - fit_method: FitMethod::LogGaussian, - }], + localizations: vec![localization(1.5, 2.5, 10)], + frame_window_start_us: 0, + frame_window_end_us: 50, + }); + + assert_eq!(dataset.row_count(), 1); + assert_eq!(dataset.columns.len(), 13); + assert_eq!(dataset.columns[0].column_id, "row_id"); + assert_eq!(dataset.columns[1].column_id, "cluster_id"); + assert_eq!(dataset.columns[4].column_id, "span_end_us"); + assert_eq!(dataset.columns[12].column_id, "fit_method"); + } + + #[test] + fn current_localization_schema_exposes_linking_metadata() { + let schema = current_localizations_schema_for_results( + &EveLocalizationResults { + localizations: vec![localization(12.0, 18.0, 15)], + frame_window_start_us: 10, + frame_window_end_us: 20, + }, + Some((128, 64)), + ); + assert_eq!(schema.row_id_column.as_deref(), Some("row_id")); + assert_eq!(schema.time_column.as_deref(), Some("timestamp_us")); + assert_eq!( + schema + .coordinate_space_3d + .as_ref() + .map(|space| space.z_column.as_str()), + Some("timestamp_us") + ); + assert_eq!( + schema.layer_id.as_deref(), + Some(CURRENT_LOCALIZATIONS_LAYER_ID) + ); + let provenance = schema.provenance.as_ref().expect("provenance"); + assert_eq!( + provenance.anchor_time_column.as_deref(), + Some("timestamp_us") + ); + assert_eq!( + provenance.span_start_column.as_deref(), + Some("span_start_us") + ); + assert_eq!(provenance.span_end_column.as_deref(), Some("span_end_us")); + } + + #[test] + fn current_localization_registry_relates_rows_to_accepted_candidate_events() { + let registry = current_localizations_registry_for_results( + &EveLocalizationResults { + localizations: vec![localization(12.0, 18.0, 15)], + frame_window_start_us: 10, + frame_window_end_us: 20, + }, + Some((128, 64)), + ); + let relations = ®istry.datasets[0].relations; + assert_eq!(relations.len(), 1); + assert_eq!( + relations[0].target_dataset_id, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID + ); + assert_eq!(relations[0].via_column, "cluster_id"); + assert_eq!(relations[0].target_column, "cluster_id"); + } + + #[test] + fn current_localization_dataset_uses_repeatable_row_ids() { + let dataset = current_localizations_dataset(&EveLocalizationResults { + localizations: vec![localization(1.0, 2.0, 11), localization(1.0, 2.0, 11)], frame_window_start_us: 0, frame_window_end_us: 50, }); + let ids = match &dataset.column("row_id").expect("row id column").values { + TableColumnValues::U64(values) => values.clone(), + other => panic!("unexpected row id values: {other:?}"), + }; + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[test] + fn rejected_fit_registry_exposes_dataset_table_and_3d_views() { + let registry = rejected_fits_registry( + &[RejectedFitRow { + row_id: 1, + cluster_id: 7, + x: 10.5, + y: 12.5, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.0, + n_events: 5, + polarity_balance: 0.2, + rejection_reason: RejectionReason::FitFailed, + timestamp_us: 15, + span_start_us: 10, + span_end_us: 20, + }], + Some((128, 64)), + 10, + 20, + ); + + assert_eq!(registry.datasets.len(), 1); + assert_eq!(registry.views.len(), 3); + assert_eq!(registry.datasets[0].id, REJECTED_FITS_DATASET_ID); + assert_eq!(registry.views[0].id, REJECTED_FITS_COMPACT_VIEW_ID); + assert!(matches!(registry.views[0].kind, HostViewKind::CompactTable)); + assert_eq!(registry.views[1].id, REJECTED_FITS_TABLE_VIEW_ID); + assert!(matches!(registry.views[1].kind, HostViewKind::TableWindow)); + assert_eq!(registry.views[2].id, REJECTED_FITS_3D_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.row_id_column.as_deref(), Some("row_id")); + assert_eq!(schema.layer_id.as_deref(), Some(REJECTED_FITS_LAYER_ID)); + let provenance = schema.provenance.as_ref().expect("provenance"); + assert_eq!( + provenance.anchor_time_column.as_deref(), + Some("timestamp_us") + ); + assert_eq!( + provenance.span_start_column.as_deref(), + Some("span_start_us") + ); + assert_eq!(provenance.span_end_column.as_deref(), Some("span_end_us")); + assert_eq!(registry.datasets[0].relations.len(), 1); + assert_eq!( + registry.datasets[0].relations[0].target_dataset_id, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID + ); + } + + #[test] + fn rejected_fit_dataset_is_columnar_and_repeatable() { + let row = RejectedFitRow { + row_id: 99, + cluster_id: 5, + x: 4.0, + y: 6.0, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.1, + n_events: 8, + polarity_balance: -0.25, + rejection_reason: RejectionReason::ResidualTooHigh, + timestamp_us: 22, + span_start_us: 20, + span_end_us: 30, + }; + let dataset = rejected_fits_dataset(&[row.clone(), row]); + + assert_eq!(dataset.row_count(), 2); + assert_eq!(dataset.columns.len(), 13); + assert_eq!(dataset.columns[0].column_id, "row_id"); + assert_eq!(dataset.columns[12].column_id, "rejection_reason"); + } + + use std::ffi::c_void; + + use augur_plugin_api::{ + FfiColorRgba as TestFfiColorRgba, FfiMarkerOverlayItem as TestFfiMarkerOverlayItem, + FfiOutputCallbacks, FfiPixel, FfiSlice, FfiString, FfiSubpixelMarker, + }; + + unsafe extern "C" fn noop_pixels( + _ctx: *mut c_void, + _pixels: FfiSlice, + _color: TestFfiColorRgba, + ) { + } + unsafe extern "C" fn noop_crosshairs( + _ctx: *mut c_void, + _markers: FfiSlice, + _color: TestFfiColorRgba, + _arm: u16, + ) { + } + unsafe extern "C" fn noop_marker_overlay( + _ctx: *mut c_void, + _markers: FfiSlice, + _dataset: FfiString, + _layer: FfiString, + _src: FfiString, + ) { + } + unsafe extern "C" fn noop_warning( + _ctx: *mut c_void, + _source: FfiString, + _severity: AnalysisSeverity, + _message: FfiString, + ) { + } + + fn noop_output_callbacks() -> FfiOutputCallbacks { + FfiOutputCallbacks { + ctx: std::ptr::null_mut(), + add_highlight_pixels: noop_pixels, + add_crosshair_markers: noop_crosshairs, + add_marker_overlay: noop_marker_overlay, + add_warning: noop_warning, + } + } + + fn cluster_snapshot_params( + cluster_id: u64, + events: &[(u16, u16, bool, u64)], + fit_method: FitMethod, + ) -> Value { + let rows = events + .iter() + .map(|(x, y, polarity, timestamp_us)| { + json!({ + "cluster_id": cluster_id, + "x_px": x, + "y_px": y, + "polarity": polarity, + "timestamp_us": timestamp_us, + }) + }) + .collect(); + let mut params = serde_json::Map::new(); + params.insert("fit_method".into(), json!(fit_method.index() as u64)); + params.insert(HOST_ACTION_CLUSTER_ROWS_PARAM.into(), Value::Array(rows)); + Value::Object(params) + } + + #[test] + fn refit_preview_registry_uses_distinct_layer_and_dataset_ids() { + let registry = + refit_preview_registry_for_results(&EveLocalizationResults::default(), Some((64, 64))); + assert_eq!(registry.datasets.len(), 1); + assert_eq!(registry.datasets[0].id, REFIT_PREVIEW_DATASET_ID); + assert_eq!(registry.views.len(), 1); + assert_eq!(registry.views[0].id, REFIT_PREVIEW_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.layer_id.as_deref(), Some(REFIT_PREVIEW_LAYER_ID)); + assert_eq!(schema.semantic_label.as_deref(), Some("refit preview")); + } + + #[test] + fn host_views_registers_three_actions_with_expected_scopes() { + let plugin = EveSmlmFittingPlugin::default(); + let registry = plugin.host_views(); + + assert_eq!(registry.actions.len(), 3); + assert_eq!(registry.actions[0].id, ACTION_REFIT_CLUSTER); + assert!(matches!( + registry.actions[0].scope, + HostActionScope::Cluster { ref dataset_id, ref group_column } + if dataset_id == ACCEPTED_CANDIDATE_EVENTS_DATASET_ID + && group_column == "cluster_id" + )); + assert!(registry.actions[0].param_schema.is_some()); + + assert_eq!(registry.actions[1].id, ACTION_COMMIT_REFIT); + assert!(matches!( + registry.actions[1].scope, + HostActionScope::Row { ref dataset_id } if dataset_id == REFIT_PREVIEW_DATASET_ID + )); + assert!(registry.actions[1].param_schema.is_none()); + + assert_eq!(registry.actions[2].id, ACTION_DISCARD_REFIT); + assert!(matches!( + registry.actions[2].scope, + HostActionScope::Dataset { ref dataset_id } if dataset_id == REFIT_PREVIEW_DATASET_ID + )); + } + + #[test] + fn refit_cluster_uses_snapshot_rows_when_current_frame_cluster_is_missing() { + let mut plugin = EveSmlmFittingPlugin::default(); + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_REFIT_CLUSTER.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Cluster { + dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + group_column: "cluster_id".into(), + group_value: "7".into(), + }, + params: cluster_snapshot_params( + 7, + &[(10, 20, true, 100), (12, 20, false, 130)], + FitMethod::MeanXY, + ), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_refit_cluster(&request, &mut output, None, 65.0); + + assert_eq!(plugin.refit_preview_results.localizations.len(), 1); + let preview = &plugin.refit_preview_results.localizations[0]; + assert_eq!(preview.cluster_id, 7); + assert!((preview.x - 11.0).abs() < 1e-6); + assert!((preview.y - 20.0).abs() < 1e-6); + assert_eq!(preview.n_events, 2); + assert_eq!(preview.span_start_us, 100); + assert_eq!(preview.span_end_us, 130); + assert_eq!(plugin.refit_preview_replaces, vec![None]); + } + + #[test] + fn commit_persists_preview_into_host_results_even_without_current_frame_match() { + let mut plugin = EveSmlmFittingPlugin::default(); + let preview = localization(3.5, 4.5, 200); + let preview_row_id = localization_row_id(&preview); + plugin.refit_preview_results.localizations.push(preview); + plugin.refit_preview_replaces.push(None); + plugin.host_rejected_fits.push(RejectedFitRow { + row_id: 99, + cluster_id: 200, + x: 3.0, + y: 4.0, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.2, + n_events: 6, + polarity_balance: 0.1, + rejection_reason: RejectionReason::ResidualTooHigh, + timestamp_us: 180, + span_start_us: 170, + span_end_us: 210, + }); + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_COMMIT_REFIT.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Row { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + row_id: preview_row_id.to_string(), + }, + params: serde_json::json!({}), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_commit_refit(&request, &mut output); + + assert!(plugin.refit_preview_results.localizations.is_empty()); + assert!(plugin.current_results.localizations.is_empty()); + assert_eq!(plugin.host_results.localizations.len(), 1); + assert_eq!(plugin.host_results.localizations[0].cluster_id, 200); + assert_eq!(plugin.host_results.localizations[0].x, 3.5); + assert!(plugin.host_rejected_fits.is_empty()); + + let dataset_bytes = plugin + .host_view_dataset(CURRENT_LOCALIZATIONS_DATASET_ID) + .expect("host dataset bytes"); + let dataset: TableDatasetV1 = + serde_json::from_slice(&dataset_bytes).expect("table dataset should deserialize"); assert_eq!(dataset.row_count(), 1); - assert_eq!(dataset.columns.len(), 5); - assert_eq!(dataset.columns[0].column_id, "x_px"); - assert_eq!(dataset.columns[4].column_id, "n_events"); + } + + #[test] + fn commit_replaces_current_localization_when_cluster_matches_current_frame() { + let mut plugin = EveSmlmFittingPlugin::default(); + let old = localization(1.0, 2.0, 100); + plugin.current_results.localizations.push(old); + + let preview = localization(1.1, 2.1, 100); + let preview_row_id = localization_row_id(&preview); + plugin.refit_preview_results.localizations.push(preview); + plugin.refit_preview_replaces.push(None); + + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_COMMIT_REFIT.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Row { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + row_id: preview_row_id.to_string(), + }, + params: serde_json::json!({}), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_commit_refit(&request, &mut output); + + assert_eq!(plugin.current_results.localizations.len(), 1); + assert_eq!(plugin.current_results.localizations[0].x, 1.1); + assert_eq!(plugin.current_results.localizations[0].y, 2.1); + assert_eq!(plugin.host_results.localizations.len(), 1); + assert_eq!(plugin.host_results.localizations[0].cluster_id, 100); + } + + #[test] + fn discard_clears_preview_without_touching_current_results() { + let mut plugin = EveSmlmFittingPlugin::default(); + plugin + .current_results + .localizations + .push(localization(1.0, 2.0, 100)); + let baseline = plugin.current_results.clone(); + + plugin + .refit_preview_results + .localizations + .push(localization(9.0, 9.0, 900)); + plugin.refit_preview_replaces.push(None); + + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_DISCARD_REFIT.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Dataset { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + }, + params: serde_json::json!({}), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_discard_refit(&request, &mut output); + + assert!(plugin.refit_preview_results.localizations.is_empty()); + assert!(plugin.refit_preview_replaces.is_empty()); + let baseline_bytes = serde_json::to_vec(&baseline).unwrap(); + let after_bytes = serde_json::to_vec(&plugin.current_results).unwrap(); + assert_eq!(baseline_bytes, after_bytes); } } diff --git a/plugins/evesmlm-fitting/src/types.rs b/plugins/evesmlm-fitting/src/types.rs index 3c37295..85684bc 100644 --- a/plugins/evesmlm-fitting/src/types.rs +++ b/plugins/evesmlm-fitting/src/types.rs @@ -2,6 +2,32 @@ use serde::{Deserialize, Serialize}; pub const CTX_EVE_LOCALIZATION_RESULTS: &str = "augur.evesmlm.localization_results"; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RejectionReason { + FitFailed, + SigmaOutOfBounds, + ResidualTooHigh, +} + +impl RejectionReason { + pub fn label(self) -> &'static str { + match self { + Self::FitFailed => "Fit failed", + Self::SigmaOutOfBounds => "Sigma out of bounds", + Self::ResidualTooHigh => "Residual too high", + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::FitFailed => "fit_failed", + Self::SigmaOutOfBounds => "sigma_out_of_bounds", + Self::ResidualTooHigh => "residual_too_high", + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FitMethod { @@ -51,11 +77,14 @@ impl FitMethod { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct EveLocalization { + pub cluster_id: u64, pub x: f64, pub y: f64, pub sigma_x: f64, pub sigma_y: f64, pub timestamp_us: u64, + pub span_start_us: u64, + pub span_end_us: u64, pub n_events: usize, pub polarity_balance: f64, pub fit_residual: f64, @@ -68,3 +97,20 @@ pub struct EveLocalizationResults { pub frame_window_start_us: u64, pub frame_window_end_us: u64, } + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RejectedFitRow { + pub row_id: u64, + pub cluster_id: u64, + pub x: f64, + pub y: f64, + pub sigma_x: f64, + pub sigma_y: f64, + pub fit_residual: f64, + pub n_events: u64, + pub polarity_balance: f64, + pub rejection_reason: RejectionReason, + pub timestamp_us: u64, + pub span_start_us: u64, + pub span_end_us: u64, +} diff --git a/plugins/evesmlm-postproc/README.md b/plugins/evesmlm-postproc/README.md index 932624f..03c5116 100644 --- a/plugins/evesmlm-postproc/README.md +++ b/plugins/evesmlm-postproc/README.md @@ -32,11 +32,20 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global ## Published Data -Publishes filtered and drift-corrected `EveLocalizationResults` on `augur.evesmlm.localization_results`, republishes standard `LocalizationResults` on `augur.localization.results` for downstream compatibility, and serves the compact host-view dataset `augur.evesmlm.current_localizations`. +Publishes filtered and drift-corrected `EveLocalizationResults` on `augur.evesmlm.localization_results`, republishes standard `LocalizationResults` on `augur.localization.results` for downstream compatibility, and serves the shared host-view dataset `augur.evesmlm.current_localizations`. ## Host View -This plugin deliberately reuses the same dataset id and compact panel view id as `EVE Candidate Fitting`. Because post-processing resolves later in the pipeline, it becomes the active provider whenever it is enabled. +This plugin deliberately reuses the same dataset id and view ids as `EVE Candidate Fitting`. + +The shared current-localizations contract includes: + +- stable row ids +- timestamps +- 2D and 3D coordinate metadata +- layer/display metadata + +Because post-processing resolves later in the pipeline, it becomes the active provider whenever it is enabled. ## Dependencies diff --git a/plugins/evesmlm-postproc/src/lib.rs b/plugins/evesmlm-postproc/src/lib.rs index 4d96c65..4413f9d 100644 --- a/plugins/evesmlm-postproc/src/lib.rs +++ b/plugins/evesmlm-postproc/src/lib.rs @@ -10,14 +10,15 @@ pub mod filtering; use std::collections::VecDeque; use augur_plugin_api::{ - export_plugin, AnalysisSeverity, EventStoreHandle, FfiSubpixelMarker, GlobalSettings, - HostContext, HostOutput, HostViewRegistry, Plugin, PluginFrame, PluginInput, SettingItem, - SettingKind, SettingsSchema, SettingsSection, StatusEntry, CTX_GLOBAL_SETTINGS, + export_plugin, AnalysisSeverity, EventStoreHandle, FfiColorRgba, FfiMarkerOverlayItem, + FfiMarkerShape, GlobalSettings, HostContext, HostOutput, HostViewRegistry, Plugin, PluginFrame, + PluginInput, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, + CTX_GLOBAL_SETTINGS, }; pub use augur_plugin_evesmlm_fitting::{ - current_localizations_dataset, current_localizations_registry, to_localization_results, - EveLocalization, EveLocalizationResults, FitMethod, CTX_EVE_LOCALIZATION_RESULTS, - CURRENT_LOCALIZATIONS_DATASET_ID, + current_localizations_dataset, current_localizations_registry_for_results, localization_row_id, + to_localization_results, EveLocalization, EveLocalizationResults, FitMethod, + CTX_EVE_LOCALIZATION_RESULTS, CURRENT_LOCALIZATIONS_DATASET_ID, CURRENT_LOCALIZATIONS_LAYER_ID, }; use augur_plugin_types::CTX_LOCALIZATION_RESULTS; use evaluation::EvaluationState; @@ -63,6 +64,7 @@ pub struct EveSmlmPostProcPlugin { enabled: bool, settings: PostProcSettings, current_results: EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, corrected_history: VecDeque>, evaluation: EvaluationState, last_input_count: usize, @@ -78,6 +80,7 @@ impl Default for EveSmlmPostProcPlugin { enabled: false, settings: PostProcSettings::default(), current_results: EveLocalizationResults::default(), + sensor_dims: None, corrected_history: VecDeque::new(), evaluation: EvaluationState::default(), last_input_count: 0, @@ -91,13 +94,16 @@ impl Default for EveSmlmPostProcPlugin { } impl EveSmlmPostProcPlugin { - fn sync_runtime_settings(&mut self, context: &HostContext<'_>) { + fn sync_runtime_settings(&mut self, context: &HostContext<'_>, frame: &PluginFrame<'_>) { if let Some(settings) = context .get::(CTX_GLOBAL_SETTINGS) .ok() .flatten() { self.settings.nm_per_pixel = settings.nm_per_pixel; + self.sensor_dims = Some((settings.sensor_width, settings.sensor_height)); + } else { + self.sensor_dims = Some((frame.width(), frame.height())); } } @@ -159,15 +165,34 @@ impl EveSmlmPostProcPlugin { self.evaluation.update(&corrected); if self.settings.show_overlay && !corrected.localizations.is_empty() { - let markers: Vec = corrected + let stable_ids: Vec = corrected .localizations .iter() - .map(|localization| FfiSubpixelMarker { + .map(|localization| localization_row_id(localization).to_string()) + .collect(); + let markers: Vec = corrected + .localizations + .iter() + .zip(stable_ids.iter()) + .map(|(localization, stable_id)| FfiMarkerOverlayItem { x: localization.x as f32, y: localization.y as f32, + shape: FfiMarkerShape::Cross, + size: 5.5, + color: FfiColorRgba::from_rgba(OVERLAY_COLOR), + timestamp_us: localization.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), }) .collect(); - output.add_crosshair_markers(&markers, OVERLAY_COLOR, 4); + output.add_marker_overlay( + &markers, + Some(CURRENT_LOCALIZATIONS_DATASET_ID), + Some(CURRENT_LOCALIZATIONS_LAYER_ID), + Some(self.name()), + ); } let mut status = format!( @@ -192,6 +217,7 @@ impl EveSmlmPostProcPlugin { pub fn reset(&mut self) { self.current_results = EveLocalizationResults::default(); + self.sensor_dims = None; self.corrected_history.clear(); self.evaluation.reset(); self.last_input_count = 0; @@ -270,12 +296,12 @@ impl Plugin for EveSmlmPostProcPlugin { fn process_frame( &mut self, - _frame: &PluginFrame<'_>, + frame: &PluginFrame<'_>, output: &mut HostOutput<'_>, context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - self.sync_runtime_settings(context); + self.sync_runtime_settings(context, frame); let input = match context.get::(CTX_EVE_LOCALIZATION_RESULTS) { Ok(value) => value, Err(err) => { @@ -592,7 +618,7 @@ impl Plugin for EveSmlmPostProcPlugin { } fn host_views(&self) -> HostViewRegistry { - current_localizations_registry() + current_localizations_registry_for_results(&self.current_results, self.sensor_dims) } fn host_view_dataset(&self, dataset_id: &str) -> Option> { @@ -618,11 +644,14 @@ mod tests { fn localization(x: f64, y: f64, n_events: usize) -> EveLocalization { EveLocalization { + cluster_id: x.to_bits() ^ y.to_bits(), x, y, sigma_x: 1.2, sigma_y: 1.2, timestamp_us: 0, + span_start_us: 0, + span_end_us: 0, n_events, polarity_balance: 0.0, fit_residual: 0.1, @@ -659,6 +688,22 @@ mod tests { assert!(correction.1.abs() <= 0.1); } + #[test] + fn current_localizations_descriptor_matches_fitting() { + use augur_plugin_evesmlm_fitting::current_localizations_registry_for_results as fitting_registry; + let results = EveLocalizationResults::default(); + let fitting = fitting_registry(&results, None); + let postproc = current_localizations_registry_for_results(&results, None); + let fitting_json = + serde_json::to_value(&fitting).expect("fitting registry should serialize"); + let postproc_json = + serde_json::to_value(&postproc).expect("postproc registry should serialize"); + assert_eq!( + fitting_json, postproc_json, + "postproc must mirror fitting's current_localizations descriptor byte-for-byte", + ); + } + #[test] fn enena_accumulation_collects_expected_nearest_neighbor_distances() { let mut evaluation = EvaluationState::default(); diff --git a/plugins/localization/src/lib.rs b/plugins/localization/src/lib.rs index c542b38..9367630 100644 --- a/plugins/localization/src/lib.rs +++ b/plugins/localization/src/lib.rs @@ -467,7 +467,7 @@ fn build_analysis_image(frame: &PluginFrame<'_>, raw_events: Option<&[FfiCdEvent } let idx = event.y as usize * frame.width() as usize + event.x as usize; let weight = event - .timestamp + .timestamp_us() .saturating_sub(frame.window_start_us()) .max(1) as f64; if event.polarity != 0 { @@ -807,7 +807,7 @@ fn estimate_timestamp_us( continue; } let weight = 1.0 / (1.0 + dist2); - weighted_timestamp += event.timestamp as f64 * weight; + weighted_timestamp += event.timestamp_us() as f64 * weight; weight_sum += weight; } diff --git a/plugins/reconstruction/README.md b/plugins/reconstruction/README.md index d2726b6..b333429 100644 --- a/plugins/reconstruction/README.md +++ b/plugins/reconstruction/README.md @@ -18,12 +18,13 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global ## Host Views -The plugin publishes one dataset, `augur.localization.accumulated`, and two host-rendered window views over that dataset: +The plugin publishes one dataset, `augur.localization.accumulated`, and three host-rendered views over that dataset: - `Localization Table` - `Reconstruction` +- `Localization Cloud` -Both views read the same accumulated source of truth. +The dataset now carries stable row ids, timestamps, nanometer-space 2D coordinates, generic 3D scatter metadata, and layer/display metadata. All views read the same accumulated source of truth. ## Compatibility diff --git a/plugins/reconstruction/src/lib.rs b/plugins/reconstruction/src/lib.rs index de404b4..9d03646 100644 --- a/plugins/reconstruction/src/lib.rs +++ b/plugins/reconstruction/src/lib.rs @@ -12,8 +12,10 @@ use std::collections::VecDeque; const DEFAULT_NM_PER_PIXEL: f64 = 65.0; const DEFAULT_MAX_LOCALIZATIONS: usize = 1_000_000; const ACCUMULATED_DATASET_ID: &str = "augur.localization.accumulated"; +const ACCUMULATED_LAYER_ID: &str = "augur.layer.localization.accumulated"; const LOCALIZATION_TABLE_VIEW_ID: &str = "augur.localization.accumulated.table"; const RECONSTRUCTION_VIEW_ID: &str = "augur.localization.accumulated.density"; +const RECONSTRUCTION_3D_VIEW_ID: &str = "augur.localization.accumulated.scatter3d"; #[derive(Debug, Clone)] struct ReconstructionSettings { @@ -149,6 +151,27 @@ impl ReconstructionPlugin { }) } + fn accumulated_coordinate_space_3d(&self) -> Option { + let (sensor_width, sensor_height) = self.sensor_dims?; + let z_min = self.table.front()?.timestamp_us as f64; + let z_max = self + .table + .back()? + .timestamp_us + .max(self.table.front()?.timestamp_us) as f64; + Some(augur_plugin_api::TableCoordinateSpace3d { + x_column: "x_nm".into(), + y_column: "y_nm".into(), + z_column: "timestamp_us".into(), + x_min: 0.0, + x_max: f64::from(sensor_width) * self.settings.nm_per_pixel, + y_min: 0.0, + y_max: f64::from(sensor_height) * self.settings.nm_per_pixel, + z_min, + z_max, + }) + } + fn accumulated_schema(&self) -> augur_plugin_api::TableSchema { augur_plugin_api::TableSchema { columns: vec![ @@ -199,6 +222,71 @@ impl ReconstructionPlugin { }, ], coordinate_space_2d: self.accumulated_coordinate_space(), + coordinate_space_3d: self.accumulated_coordinate_space_3d(), + row_id_column: Some("id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(ACCUMULATED_LAYER_ID.into()), + semantic_label: Some("localizations".into()), + provenance: Some(augur_plugin_api::TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("timestamp_us".into()), + span_end_column: Some("timestamp_us".into()), + anchor_frame_column: Some("frame".into()), + }), + column_display: vec![ + augur_plugin_api::TableColumnDisplayEntry { + column_id: "id".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "x_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 1, + }), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "y_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 1, + }), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "sigma_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 2, + }), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "uncertainty_xy_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 2, + }), + ..Default::default() + }, + }, + ], } } @@ -269,6 +357,14 @@ impl ReconstructionPlugin { title: "Accumulated localizations".into(), kind: augur_plugin_api::HostDatasetKind::TableV1(self.accumulated_schema()), empty_message: "No accumulated localizations yet.".into(), + display: Some(augur_plugin_api::HostDatasetDisplayMetadata { + layer_title: Some("Accumulated localizations".into()), + default_visibility: Some(true), + default_color: Some([255, 180, 80, 255]), + default_marker_shape: Some(augur_plugin_api::HostMarkerShape::Circle), + default_size: Some(3.5), + }), + relations: Vec::new(), }], views: vec![ augur_plugin_api::HostViewDescriptor { @@ -288,7 +384,19 @@ impl ReconstructionPlugin { y_column: "y_nm".into(), }, }, + augur_plugin_api::HostViewDescriptor { + id: RECONSTRUCTION_3D_VIEW_ID.into(), + title: "Localization Cloud".into(), + dataset_id: ACCUMULATED_DATASET_ID.into(), + placement: augur_plugin_api::HostViewPlacement::Window, + kind: augur_plugin_api::HostViewKind::Scatter3dFromTable { + x_column: "x_nm".into(), + y_column: "y_nm".into(), + z_column: "timestamp_us".into(), + }, + }, ], + actions: Vec::new(), } } } @@ -505,16 +613,23 @@ mod tests { } #[test] - fn host_view_registry_exposes_one_dataset_and_two_window_views() { + fn host_view_registry_exposes_one_dataset_and_investigation_views() { let mut plugin = ReconstructionPlugin::default(); plugin.sensor_dims = Some((1280, 720)); let registry = plugin.host_view_registry(); assert_eq!(registry.datasets.len(), 1); - assert_eq!(registry.views.len(), 2); + assert_eq!(registry.views.len(), 3); assert_eq!(registry.datasets[0].id, ACCUMULATED_DATASET_ID); assert_eq!(registry.views[0].id, LOCALIZATION_TABLE_VIEW_ID); assert_eq!(registry.views[1].id, RECONSTRUCTION_VIEW_ID); + assert_eq!(registry.views[2].id, RECONSTRUCTION_3D_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + augur_plugin_api::HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.row_id_column.as_deref(), Some("id")); + assert_eq!(schema.time_column.as_deref(), Some("timestamp_us")); } } diff --git a/scripts/install-built-plugins.sh b/scripts/install-built-plugins.sh index 80a3cbc..d39a8be 100755 --- a/scripts/install-built-plugins.sh +++ b/scripts/install-built-plugins.sh @@ -96,6 +96,21 @@ find_library_path() { return 1 } +rewrite_macos_install_name() { + local installed_library_path="$1" + + if [[ "$(uname -s)" != "Darwin" ]]; then + return 0 + fi + + if ! command -v install_name_tool >/dev/null 2>&1; then + echo "warning: install_name_tool not found; leaving ${installed_library_path} with Cargo's build-path dylib id" >&2 + return 0 + fi + + install_name_tool -id "@loader_path/$(basename "${installed_library_path}")" "${installed_library_path}" +} + library_extension="$(library_extension)" mkdir -p "${dest_dir}" @@ -128,7 +143,9 @@ for plugin_dir in "${repo_root}"/plugins/*; do install_dir="${dest_dir}/${plugin_id}" mkdir -p "${install_dir}" cp "${manifest_path}" "${install_dir}/plugin.toml" - cp "${library_path}" "${install_dir}/$(basename "${library_path}")" + installed_library_path="${install_dir}/$(basename "${library_path}")" + cp "${library_path}" "${installed_library_path}" + rewrite_macos_install_name "${installed_library_path}" echo "Installed ${plugin_id} -> ${install_dir}" installed=$((installed + 1)) done From 10dabfe8033278e7c6f1c9e387da3605a515ac26 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 22:53:02 +0200 Subject: [PATCH 06/46] =?UTF-8?q?fix(plugins):=20=F0=9F=90=9B=20rebuild=20?= =?UTF-8?q?legacy=20plugins=20against=20plugin=20ABI=20v5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the new FfiPreviewFrame.external_triggers field to the two test initializers in focus-metrics and evesmlm-candidates; all plugins now compile and test against the current augur-rs plugin API (ABI v5). --- plugins/evesmlm-candidates/src/lib.rs | 1 + plugins/focus-metrics/src/lib.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index e0bc71e..c173d45 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -2114,6 +2114,7 @@ mod tests { events: augur_plugin_api::FfiSlice::from_slice( &[] as &[augur_plugin_api::FfiCdEvent] ), + external_triggers: augur_plugin_api::FfiSlice::default(), window_start_us: self.window_start_us, window_end_us: self.window_start_us + 1, })); diff --git a/plugins/focus-metrics/src/lib.rs b/plugins/focus-metrics/src/lib.rs index 47c5d23..49a69ab 100644 --- a/plugins/focus-metrics/src/lib.rs +++ b/plugins/focus-metrics/src/lib.rs @@ -656,6 +656,7 @@ mod tests { height: 16, pixels: FfiSlice::from_slice(&pixels), events: FfiSlice::default(), + external_triggers: FfiSlice::default(), window_start_us: 0, window_end_us: 1_000, }; From 0a4be4d8bc14d0e64748e4ca6949e0583a10c9cc Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 22:53:33 +0200 Subject: [PATCH 07/46] =?UTF-8?q?chore(plugins):=20=F0=9F=A7=B9=20apply=20?= =?UTF-8?q?rustfmt=20across=20the=20workspace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugins/evesmlm-candidates/src/lib.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index c173d45..d51776b 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -17,9 +17,9 @@ use augur_plugin_api::{ HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginCapabilities, PluginFrame, PluginInput, PluginStateKind, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, - TableColumnDisplayEntry, TableColumnDisplayFormat, TableColumnDisplayMetadata, TableColumnValues, - TableColumnWidthPriority, TableCoordinateSpace2d, TableCoordinateSpace3d, TableDatasetV1, - TableRowProvenance, TableSchema, TableValueType, + TableColumnDisplayEntry, TableColumnDisplayFormat, TableColumnDisplayMetadata, + TableColumnValues, TableColumnWidthPriority, TableCoordinateSpace2d, TableCoordinateSpace3d, + TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, }; use serde_json::{json, Value}; @@ -63,8 +63,7 @@ const CANDIDATE_FINDING_PIXELS_DATASET_ID: &str = const CANDIDATE_FINDINGS_LAYER_ID: &str = "augur.layer.evesmlm.candidate_findings"; const CANDIDATE_FINDINGS_COMPACT_VIEW_ID: &str = "augur.evesmlm.candidates.candidate_findings.compact"; -const CANDIDATE_FINDINGS_TABLE_VIEW_ID: &str = - "augur.evesmlm.candidates.candidate_findings.table"; +const CANDIDATE_FINDINGS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.candidate_findings.table"; const CANDIDATE_FINDING_PIXELS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.candidate_finding_pixels.table"; @@ -248,11 +247,12 @@ impl EveSmlmCandidatePlugin { } let method = self.settings.finding_method; - self.findings - .extend(clusters.iter().cloned().map(|cluster| CandidateFinding { - cluster, - method, - })); + self.findings.extend( + clusters + .iter() + .cloned() + .map(|cluster| CandidateFinding { cluster, method }), + ); self.findings_generation = self.findings_generation.wrapping_add(1); } From 9c0f349937daf06837d01d8b2b8ae0f355a43f8b Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 14 Jul 2026 13:43:46 +0200 Subject: [PATCH 08/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20align=20mo?= =?UTF-8?q?ck=20and=20host=20plugins=20with=20firmware=200.2.0=20protocol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mock controller spoke an invented protocol (ARM/RUN/FAULT_CLEAR verbs, capabilities HELLO field, BAD_* error codes, arbitrary CONFIG fields), so tests validated commands the Teensy never accepts. It now mirrors stage-a-controller/src/main.cpp verbatim: verbs, SAFE_IDLE/CONFIGURED/RUNNING state machine, RANGE/STATE/SYNTAX/PROTOCOL/VERB error details, single-entry idempotent reply cache, and unknown-CONFIG-field rejection (the host's feature-detection contract). The reserved v2 waveform fields are only accepted behind an explicit with_waveform_extension() opt-in, which also synthesizes photodiode blocks through a Pockels-like sin² transfer. Host fixes uncovered by the faithful mock: - surface async control notices (watchdog !FAULT) through poll_events even with no request in flight; monitor and A1 now react instead of showing a stale acquiring state - A1 sweep issues STOP before CONFIG between measurement points (CONFIG is illegal while RUNNING) - record the ACKed CONFIG fields in the A1 run sidecar - reject wire frames with an unknown protocol version like the reference parser does - stream the PDQ file CRC incrementally instead of buffering the whole recording in memory - monitor maps the unknown_config_field rejection of drive fields to a clear 'no waveform backend' message --- plugins/stage-a-a1/src/lib.rs | 40 +- plugins/stage-a-monitor/src/lib.rs | 27 +- stage-a-io/src/client.rs | 70 +++- stage-a-io/src/lib.rs | 1 + stage-a-io/src/mock.rs | 606 +++++++++++++++++++++++++---- stage-a-io/src/pdq.rs | 10 +- stage-a-io/src/wire.rs | 28 ++ 7 files changed, 668 insertions(+), 114 deletions(-) diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs index 379a7f8..808be04 100644 --- a/plugins/stage-a-a1/src/lib.rs +++ b/plugins/stage-a-a1/src/lib.rs @@ -122,6 +122,8 @@ pub struct StageAA1Plugin { reference_counts: Vec, sensor_size: (u16, u16), run_id: String, + /// CONFIG reply fields exactly as the controller ACKed them (sidecar). + last_acked_config: BTreeMap, pdq: Option, current_phase_histogram: Option, used_hardware_fiducial: bool, @@ -159,6 +161,7 @@ impl Default for StageAA1Plugin { reference_counts: Vec::new(), sensor_size: (0, 0), run_id: String::new(), + last_acked_config: BTreeMap::new(), pdq: None, current_phase_histogram: None, used_hardware_fiducial: false, @@ -290,6 +293,7 @@ impl StageAA1Plugin { sidecar.firmware_version = self.firmware.clone(); sidecar.adc_calibration = self.calibration.clone(); sidecar.configured_sample_rate_hz = self.sample_rate_hz as u32; + sidecar.acked_config = self.last_acked_config.clone(); sidecar.trigger_source = if self.used_hardware_fiducial { TriggerSource::DrivePhase0 } else { @@ -321,6 +325,10 @@ impl StageAA1Plugin { fn send_drive(&mut self, frequency_hz: f64, amplitude_dac: u32, purpose: &str) { let freq_mhz = (frequency_hz * 1_000.0).round() as i64; + // The firmware only accepts CONFIG from SAFE_IDLE/CONFIGURED, so + // every new drive point must stop the running acquisition first + // (STOP is idempotent and harmless before the first point). + self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); self.queue_command( purpose, Command::new("CONFIG") @@ -346,19 +354,26 @@ impl StageAA1Plugin { }; let outputs = worker.drain_outputs(); let mut stopped = None; + let mut watchdog_fault: Option = None; for output in outputs { match output { WorkerOutput::Reply { tag, result } => { let purpose = self.in_flight.remove(&tag).unwrap_or_default(); match result { - Ok(fields) => { - if purpose == "hello" { + Ok(fields) => match purpose.as_str() { + "hello" => { self.firmware = fields .get("firmware") .cloned() .unwrap_or_else(|| "unknown".into()); } - } + // CONFIG ACKs (drive points) go into the sidecar + // verbatim, per the control-software spec. + "reference" | "sweep" => { + self.last_acked_config = fields; + } + _ => {} + }, Err(err) => self.last_error = Some(format!("{purpose}: {err}")), } } @@ -372,11 +387,28 @@ impl StageAA1Plugin { } } } - WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { + if name == "FAULT" { + watchdog_fault = Some( + fields + .get("code") + .cloned() + .unwrap_or_else(|| "unknown".into()), + ); + } + } WorkerOutput::Integrity(integrity) => self.integrity = integrity, WorkerOutput::Stopped { reason } => stopped = Some(reason), } } + if let Some(code) = watchdog_fault { + // The controller safed itself mid-run; the current point is + // invalid and the run cannot silently continue. + self.last_error = Some(format!("controller fault: {code} — run aborted")); + if matches!(self.state, RunState::Reference | RunState::Sweeping) { + self.stop_run("watchdog_fault"); + } + } if let Some(reason) = stopped { self.worker = None; self.last_error = Some(format!("device connection ended: {reason}")); diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs index ab159aa..6b3e3e9 100644 --- a/plugins/stage-a-monitor/src/lib.rs +++ b/plugins/stage-a-monitor/src/lib.rs @@ -218,6 +218,15 @@ impl StageAMonitorPlugin { let purpose = self.in_flight.remove(&tag).unwrap_or_default(); match result { Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) if err.contains("unknown_config_field") => { + // Feature detection: firmware 0.2.0 has no + // waveform backend and rejects the reserved v2 + // drive fields. + self.last_error = Some(format!( + "{purpose}: firmware has no waveform backend (v1) — drive \ + control needs the mock or the future v2 firmware" + )); + } Err(err) => { self.last_error = Some(format!("{purpose}: {err}")); } @@ -233,7 +242,23 @@ impl StageAMonitorPlugin { FrameType::Summary | FrameType::Marker | FrameType::Control => {} FrameType::Unknown(_) => {} }, - WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { + if name == "FAULT" { + // Firmware watchdog dropped the controller to + // SAFE_IDLE — reflect it instead of showing a stale + // "acquiring" state. + if self.connection == ConnectionState::Acquiring { + self.connection = ConnectionState::Connected; + } + self.last_error = Some(format!( + "controller fault: {} — dropped to SAFE_IDLE", + fields.get("code").map(String::as_str).unwrap_or("unknown") + )); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + } WorkerOutput::Integrity(integrity) => { self.integrity = integrity; } diff --git a/stage-a-io/src/client.rs b/stage-a-io/src/client.rs index c664be0..0b018bb 100644 --- a/stage-a-io/src/client.rs +++ b/stage-a-io/src/client.rs @@ -189,9 +189,19 @@ impl StageAClient { match frame.header.frame_type { FrameType::Control => { - // Control payloads are handled by take_reply / async queue; - // keep the raw frame so replies can be matched later. - self.pending_events.push(DeviceEvent::Data(frame)); + // Classify control payloads immediately so async notices + // (e.g. the watchdog `!FAULT`) surface through poll_events + // even when no request is in flight. Replies stay queued as + // raw frames for take_reply to match by sequence. + match frame.control_text().map(ControlMessage::parse) { + Some(Ok(ControlMessage::Async { name, fields })) => { + self.pending_events + .push(DeviceEvent::Async { name, fields }); + } + Some(Ok(_)) => self.pending_events.push(DeviceEvent::Data(frame)), + // Non-UTF8 or malformed control payload: corruption. + _ => self.integrity.skipped_bytes += frame.payload.len() as u64, + } } _ => self.pending_events.push(DeviceEvent::Data(frame)), } @@ -230,16 +240,10 @@ impl StageAClient { }) if reply_seq == sequence => { result = Some(Err(ClientError::Device { code, detail })); } - Ok(ControlMessage::Async { name, fields }) => { - remaining.push(DeviceEvent::Async { name, fields }); - } - // Stale replies to earlier (retried) sequences are dropped; - // malformed control payloads count as corruption. - Ok(_) => {} - Err(_) => { - self.integrity.crc_failures += 0; // parse failure, not CRC - self.integrity.skipped_bytes += frame.payload.len() as u64; - } + // Stale replies to earlier (retried) sequences are dropped. + // Async / malformed payloads never reach here — accept_frame + // classifies them before queueing. + _ => {} } } self.pending_events = remaining; @@ -285,14 +289,17 @@ mod tests { // The controller swallows the first reply; the client must resend the // identical sequence and accept the cached second reply. The mock // panics if a retried sequence re-executes the operation. - let handle = std::thread::spawn(move || controller.serve_n_commands(2)); + let handle = std::thread::spawn(move || { + controller.serve_n_commands(2); + controller + }); let reply = client .request(&Command::new("STATUS")) .expect("retried STATUS succeeds"); - handle.join().expect("mock thread joins"); + let controller = handle.join().expect("mock thread joins"); assert_eq!(reply.get("state").map(String::as_str), Some("SAFE_IDLE")); - assert_eq!(reply.get("executions").map(String::as_str), Some("1")); + assert_eq!(controller.executions(), 1); } #[test] @@ -304,16 +311,43 @@ mod tests { let handle = std::thread::spawn(move || controller.serve_n_commands(1)); let err = client - .request(&Command::new("CONFIG").field("mode", "A9")) + .request( + &Command::new("CONFIG") + .field("mode", "A9") + .field("rate_hz", 20_000), + ) .expect_err("invalid mode is rejected"); handle.join().expect("mock thread joins"); match err { - ClientError::Device { code, .. } => assert_eq!(code, "BAD_MODE"), + ClientError::Device { code, detail } => { + assert_eq!(code, "RANGE"); + assert_eq!(detail, "invalid_mode"); + } other => panic!("expected device error, got {other:?}"), } } + #[test] + fn watchdog_fault_surfaces_as_async_event_without_a_request_in_flight() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + controller.emit_watchdog_fault(); + let events = client.poll_events().expect("poll"); + match events.as_slice() { + [DeviceEvent::Async { name, fields }] => { + assert_eq!(name, "FAULT"); + assert_eq!(fields.get("code").map(String::as_str), Some("WATCHDOG")); + assert_eq!(fields.get("state").map(String::as_str), Some("SAFE_IDLE")); + } + other => panic!("expected one async FAULT event, got {other:?}"), + } + assert!(client.integrity().is_clean()); + } + #[test] fn overrun_frames_invalidate_integrity() { let link = MockLink::new(); diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs index 67c517f..2a21ff7 100644 --- a/stage-a-io/src/lib.rs +++ b/stage-a-io/src/lib.rs @@ -31,6 +31,7 @@ pub mod wire; pub use client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; pub use estimator::{estimate_contrast, AdcCalibration, ContrastEstimate, EstimateError}; +pub use mock::{MockController, MockState, MockWave}; pub use pdq::{PdqSummary, PdqWriter}; pub use protocol::{Command, ControlMessage, ProtocolError}; pub use sidecar::{DetectorLoad, IntegrityRecord, RunSidecar, TriggerSource}; diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs index fd7dad0..803d919 100644 --- a/stage-a-io/src/mock.rs +++ b/stage-a-io/src/mock.rs @@ -1,23 +1,34 @@ //! Mock Stage-A controller for tests and hardware-free plugin development. //! -//! Implements the v1 command surface (`HELLO`, `STATUS`, `CONFIG`, `ARM`, -//! `RUN`, `START`, `STOP`, `PING`, `FAULT_CLEAR`) with the same idempotency -//! contract as the firmware: replies to recent sequences are cached and -//! resent without re-executing the operation. It can also synthesize -//! photodiode sample/summary frames (sinusoidal drive) so the estimator and -//! plugins can be exercised end to end without a Teensy. - -use std::collections::BTreeMap; +//! Mirrors firmware 0.2.0 (`stage-a-controller/src/main.cpp`) faithfully: +//! the same verbs (`HELLO`, `STATUS`, `CONFIG`, `START`, `STOP`, `PING`), +//! the same state machine (`SAFE_IDLE` → `CONFIGURED` → `RUNNING`), the +//! same error codes/details (`PROTOCOL`, `RANGE`, `STATE`, `SYNTAX`, +//! `VERB`), the same single-entry idempotent reply cache, and rejection of +//! unknown `CONFIG` fields — which is the host's feature-detection +//! mechanism, so it must never be papered over here. +//! +//! [`MockController::with_waveform_extension`] additionally models the +//! *proposed* v2 waveform firmware (`stage-a-controller/docs/features/` +//! `waveform-drive.md`): `wave`/`freq_mhz`/`center_dac`/`amplitude_dac` +//! CONFIG fields, a `capabilities` HELLO entry, and synthetic photodiode +//! blocks derived from the configured drive through a Pockels-like sin² +//! transfer — commanded DAC amplitude maps *non-linearly* to optical +//! contrast, exactly why `a` must be measured, never assumed. use crate::protocol::ControlMessage; use crate::transport::Transport; use crate::wire::{Frame, FrameHeader, FrameType, SummaryPayload, PROTOCOL_VERSION}; +pub const MOCK_MAX_RATE_HZ: u32 = 100_000; +pub const MOCK_MAX_BLOCK_SAMPLES: u32 = 256; +/// Proposed v2 waveform ceiling (matches the drive UI bound: 200 kHz). +pub const MOCK_MAX_FREQ_MHZ: u32 = 200_000_000; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MockState { SafeIdle, Configured, - Armed, Running, } @@ -26,29 +37,87 @@ impl MockState { match self { Self::SafeIdle => "SAFE_IDLE", Self::Configured => "CONFIGURED", - Self::Armed => "ARMED", Self::Running => "RUNNING", } } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MockWave { + Sine, + Square, + Saw, +} + +impl MockWave { + /// Normalised waveform value in [-1, 1] at cycle phase `t` in [0, 1). + fn value(self, t: f64) -> f64 { + match self { + Self::Sine => (2.0 * std::f64::consts::PI * t).sin(), + Self::Square => { + if t < 0.5 { + 1.0 + } else { + -1.0 + } + } + Self::Saw => 2.0 * t - 1.0, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +struct MockConfig { + mode: String, + rate_hz: u32, + block_samples: u32, + raw: bool, + summary: bool, + // v2 waveform extension (None until configured). + wave: Option, + freq_mhz: u32, + center_dac: u32, + amplitude_dac: u32, +} + +impl Default for MockConfig { + fn default() -> Self { + Self { + mode: "A1".into(), + rate_hz: 20_000, + block_samples: 256, + raw: true, + summary: true, + wave: None, + freq_mhz: 0, + center_dac: 2_048, + amplitude_dac: 0, + } + } +} + pub struct MockController { transport: T, state: MockState, - config: BTreeMap, - config_revision: u32, - reply_cache: Vec<(u32, String)>, + config: MockConfig, + /// v2 waveform CONFIG fields accepted (proposed firmware) instead of + /// rejected as `unknown_config_field` (firmware 0.2.0). + waveform_extension: bool, + /// Firmware caches exactly one reply (`cached_request_sequence`). + cached_reply: Option<(u32, String)>, executed_sequences: Vec, - /// Commands executed (used to assert idempotency in tests). executions: u32, drop_next_reply: bool, out_sequence: u32, line_buffer: Vec, sample_index: u64, - /// Synthetic optical waveform: codes = center + amplitude*sin(phase). + /// Synthetic optics for [`MockController::emit_configured_block`]: + /// photodiode code = dark + span * sin²(π/2 · drive/4095). + pub synth_dark_code: f64, + pub synth_span_codes: f64, + // Legacy direct-sine synthesis (emit_sine_block). pub synth_center: f64, pub synth_amplitude: f64, - pub synth_dark_code: f64, } impl MockController { @@ -56,21 +125,28 @@ impl MockController { Self { transport, state: MockState::SafeIdle, - config: BTreeMap::new(), - config_revision: 0, - reply_cache: Vec::new(), + config: MockConfig::default(), + waveform_extension: false, + cached_reply: None, executed_sequences: Vec::new(), executions: 0, drop_next_reply: false, out_sequence: 0, line_buffer: Vec::new(), sample_index: 0, + synth_dark_code: 40.0, + synth_span_codes: 3_800.0, synth_center: 2_048.0, synth_amplitude: 900.0, - synth_dark_code: 40.0, } } + /// Enables the proposed v2 waveform command surface. + pub fn with_waveform_extension(mut self) -> Self { + self.waveform_extension = true; + self + } + /// Swallow the next reply (simulates a lost USB packet) — the client /// must retry with the identical sequence. pub fn drop_first_reply(&mut self) { @@ -81,6 +157,41 @@ impl MockController { self.state } + /// Commands actually executed (idempotent retries excluded). + pub fn executions(&self) -> u32 { + self.executions + } + + /// Handles all complete command lines already received, without + /// blocking — for long-lived in-process mock threads (e.g. a plugin's + /// hardware-free `mock` port). + pub fn poll_commands(&mut self) { + let mut buf = [0_u8; 1024]; + loop { + let read = self.transport.read(&mut buf).unwrap_or(0); + if read == 0 { + break; + } + self.line_buffer.extend_from_slice(&buf[..read]); + } + while let Some(pos) = self.line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = self.line_buffer.drain(..=pos).collect(); + if let Ok(text) = std::str::from_utf8(&line) { + let text = text.trim_end().to_owned(); + self.handle_line(&text); + } + } + } + + /// Wall-clock duration one configured sample block spans — the cadence + /// at which a live mock should call [`Self::emit_configured_block`]. + pub fn block_period(&self) -> std::time::Duration { + let rate = self.config.rate_hz.max(1); + std::time::Duration::from_micros( + u64::from(self.config.block_samples) * 1_000_000 / u64::from(rate), + ) + } + /// Serves exactly `n` command lines (counting retries), then returns. pub fn serve_n_commands(&mut self, n: usize) { let mut served = 0; @@ -108,21 +219,21 @@ impl MockController { fn handle_line(&mut self, line: &str) { let Some(rest) = line.strip_prefix('@') else { + self.send_control("-0 ERR code=SYNTAX detail=expected_sequence_and_verb"); return; }; let mut parts = rest.split_ascii_whitespace(); let Some(sequence) = parts.next().and_then(|s| s.parse::().ok()) else { + self.send_control("-0 ERR code=SYNTAX detail=invalid_sequence"); return; }; // Idempotent retry: replay the cached reply without re-executing. - if let Some((_, cached)) = self - .reply_cache - .iter() - .find(|(cached_seq, _)| *cached_seq == sequence) - { - let payload = cached.clone(); - self.send_control(&payload); - return; + if let Some((cached_seq, cached)) = &self.cached_reply { + if *cached_seq == sequence { + let payload = cached.clone(); + self.send_control(&payload); + return; + } } assert!( !self.executed_sequences.contains(&sequence), @@ -130,7 +241,8 @@ impl MockController { ); let verb = parts.next().unwrap_or(""); - let fields: BTreeMap = parts + // Preserve wire order: firmware validates fields as encountered. + let fields: Vec<(String, String)> = parts .filter_map(|part| { let (key, value) = part.split_once('=')?; Some((key.to_owned(), value.to_owned())) @@ -140,10 +252,7 @@ impl MockController { self.executions += 1; self.executed_sequences.push(sequence); let reply = self.execute(verb, &fields, sequence); - self.reply_cache.push((sequence, reply.clone())); - if self.reply_cache.len() > 8 { - self.reply_cache.remove(0); - } + self.cached_reply = Some((sequence, reply.clone())); if self.drop_next_reply { self.drop_next_reply = false; return; @@ -151,50 +260,147 @@ impl MockController { self.send_control(&reply); } - fn execute(&mut self, verb: &str, fields: &BTreeMap, sequence: u32) -> String { + fn execute(&mut self, verb: &str, fields: &[(String, String)], sequence: u32) -> String { + let field = |key: &str| { + fields + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + }; match verb { - "HELLO" => format!( - "+{sequence} OK protocol=1 firmware=0.1.0-mock board=mock dac_bits=12 \ - capabilities=A1,A2,A3" - ), + "HELLO" => { + if field("protocol") != Some("1") { + return format!("-{sequence} ERR code=PROTOCOL detail=requires_v1"); + } + let capabilities = if self.waveform_extension { + " capabilities=A1,A2,A3,WAVE" + } else { + "" + }; + format!( + "+{sequence} OK protocol=1 firmware=0.2.0-mock board=MOCK adc_bits=12 \ + max_rate_hz={MOCK_MAX_RATE_HZ} dac=AD5628 dac_bus=SPI1 dac_cs=29 \ + dac_channel=1.4 dac_address=3{capabilities}" + ) + } "STATUS" => format!( - "+{sequence} OK state={} rev={} executions={}", + "+{sequence} OK state={} mode={} rate_hz={} block_samples={} raw={} summary={} \ + sample_index={} dropped=0 marker_drops=0 dac=1.4/3 code=0", self.state.name(), - self.config_revision, - self.executions + self.config.mode, + self.config.rate_hz, + self.config.block_samples, + u8::from(self.config.raw), + u8::from(self.config.summary), + self.sample_index, ), - "PING" => format!("+{sequence} OK state={}", self.state.name()), - "CONFIG" => { - let mode = fields.get("mode").map(String::as_str).unwrap_or(""); - if !matches!(mode, "A1" | "A2" | "A3") { - return format!("-{sequence} ERR code=BAD_MODE detail=mode"); - } - self.config = fields.clone(); - self.config_revision += 1; - self.state = MockState::Configured; - format!("+{sequence} OK rev={}", self.config_revision) - } - "ARM" => { + "CONFIG" => self.execute_config(fields, sequence), + "START" => { if self.state != MockState::Configured { - return format!("-{sequence} ERR code=BAD_STATE detail=arm_requires_config"); - } - self.state = MockState::Armed; - format!("+{sequence} OK state=ARMED rev={}", self.config_revision) - } - "RUN" | "START" => { - if !matches!(self.state, MockState::Armed | MockState::Configured) { - return format!("-{sequence} ERR code=BAD_STATE detail=run_requires_arm"); + return format!("-{sequence} ERR code=STATE detail=configure_before_start"); } self.state = MockState::Running; format!("+{sequence} OK state=RUNNING") } + // Firmware ignores extra STOP tokens (e.g. reason=…). "STOP" => { self.state = MockState::SafeIdle; format!("+{sequence} OK state=SAFE_IDLE") } - "FAULT_CLEAR" => format!("+{sequence} OK state={}", self.state.name()), - _ => format!("-{sequence} ERR code=BAD_VERB detail={verb}"), + "PING" => format!("+{sequence} OK watchdog=refreshed"), + _ => format!("-{sequence} ERR code=VERB detail=unsupported_command"), + } + } + + fn execute_config(&mut self, fields: &[(String, String)], sequence: u32) -> String { + if self.state == MockState::Running { + return format!("-{sequence} ERR code=STATE detail=stop_before_config"); + } + let err = |code: &str, detail: &str| format!("-{sequence} ERR code={code} detail={detail}"); + let mut next = self.config.clone(); + let mut saw_mode = false; + let mut saw_rate = false; + for (key, value) in fields { + match key.as_str() { + "mode" => { + saw_mode = true; + if !matches!(value.as_str(), "A1" | "A2" | "A3") { + return err("RANGE", "invalid_mode"); + } + next.mode = value.clone(); + } + "rate_hz" => { + saw_rate = true; + match value.parse::() { + Ok(rate) if (100..=MOCK_MAX_RATE_HZ).contains(&rate) => { + next.rate_hz = rate; + } + _ => return err("RANGE", "invalid_rate_hz"), + } + } + "block_samples" => match value.parse::() { + Ok(block) if (1..=MOCK_MAX_BLOCK_SAMPLES).contains(&block) => { + next.block_samples = block; + } + _ => return err("RANGE", "invalid_block_samples"), + }, + "raw" => match value.as_str() { + "0" => next.raw = false, + "1" => next.raw = true, + _ => return err("RANGE", "invalid_raw_flag"), + }, + "summary" => match value.as_str() { + "0" => next.summary = false, + "1" => next.summary = true, + _ => return err("RANGE", "invalid_summary_flag"), + }, + "wave" if self.waveform_extension => { + next.wave = Some(match value.as_str() { + "SINE" => MockWave::Sine, + "SQUARE" => MockWave::Square, + "SAW" => MockWave::Saw, + _ => return err("RANGE", "invalid_wave"), + }); + } + "freq_mhz" if self.waveform_extension => match value.parse::() { + Ok(freq) if (1..=MOCK_MAX_FREQ_MHZ).contains(&freq) => { + next.freq_mhz = freq; + } + _ => return err("RANGE", "invalid_freq_mhz"), + }, + "center_dac" if self.waveform_extension => match value.parse::() { + Ok(center) if center <= 4_095 => next.center_dac = center, + _ => return err("RANGE", "invalid_center_dac"), + }, + "amplitude_dac" if self.waveform_extension => match value.parse::() { + Ok(amplitude) if amplitude <= 2_047 => next.amplitude_dac = amplitude, + _ => return err("RANGE", "invalid_amplitude_dac"), + }, + // Firmware 0.2.0 rejects unknown fields — the host relies + // on this for feature detection. Never accept silently. + _ => return err("SYNTAX", "unknown_config_field"), + } + } + if !saw_mode || !saw_rate || (!next.raw && !next.summary) { + return err("SYNTAX", "mode_rate_and_output_required"); } + if next.wave.is_some() + && (next.center_dac + next.amplitude_dac > 4_095 + || next.center_dac < next.amplitude_dac) + { + return err("RANGE", "amplitude_exceeds_range"); + } + self.config = next; + self.state = MockState::Configured; + format!( + "+{sequence} OK state=CONFIGURED mode={} rate_hz={} block_samples={} raw={} \ + summary={} backend=mock", + self.config.mode, + self.config.rate_hz, + self.config.block_samples, + u8::from(self.config.raw), + u8::from(self.config.summary), + ) } fn send_control(&mut self, payload: &str) { @@ -203,6 +409,13 @@ impl MockController { let _ = self.transport.write_all(&bytes); } + /// Emits the watchdog fault notice and drops to `SAFE_IDLE`, exactly as + /// the firmware does after 1.5 s without host contact. + pub fn emit_watchdog_fault(&mut self) { + self.state = MockState::SafeIdle; + self.send_control("!FAULT code=WATCHDOG state=SAFE_IDLE"); + } + fn build_frame( &mut self, frame_type: FrameType, @@ -227,38 +440,83 @@ impl MockController { ) } - /// Emits one synthetic sinusoidal sample block (`SamplesU16`). - pub fn emit_sine_block(&mut self, samples: usize, rate_hz: u32, freq_hz: f64) { - let mut payload = Vec::with_capacity(samples * 2); + fn emit_codes_block(&mut self, codes: &[u16], rate_hz: u32, raw: bool, summary: bool) { let mut min_code = u16::MAX; let mut max_code = 0_u16; let mut sum = 0_u64; - for i in 0..samples { - let t = (self.sample_index + i as u64) as f64 / f64::from(rate_hz); - let value = self.synth_center - + self.synth_amplitude * (2.0 * std::f64::consts::PI * freq_hz * t).sin(); - let code = value.round().clamp(0.0, 4_095.0) as u16; + let mut payload = Vec::with_capacity(codes.len() * 2); + for &code in codes { min_code = min_code.min(code); max_code = max_code.max(code); sum += u64::from(code); payload.extend_from_slice(&code.to_le_bytes()); } - let frame = self.build_frame(FrameType::SamplesU16, payload, rate_hz, 0); - let bytes = frame.to_bytes(); - let _ = self.transport.write_all(&bytes); + if raw { + let frame = self.build_frame(FrameType::SamplesU16, payload, rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } + if summary { + let summary_payload = SummaryPayload { + min_code, + max_code, + sample_count: codes.len() as u32, + sum_codes: sum, + first_tick_us: 0, + last_tick_us: ((codes.len() as f64 / f64::from(rate_hz)) * 1e6) as u32, + }; + let frame = self.build_frame(FrameType::Summary, summary_payload.encode(), rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } + self.sample_index += codes.len() as u64; + } - let summary = SummaryPayload { - min_code, - max_code, - sample_count: samples as u32, - sum_codes: sum, - first_tick_us: 0, - last_tick_us: ((samples as f64 / f64::from(rate_hz)) * 1e6) as u32, - }; - let frame = self.build_frame(FrameType::Summary, summary.encode(), rate_hz, 0); - let bytes = frame.to_bytes(); - let _ = self.transport.write_all(&bytes); - self.sample_index += samples as u64; + /// Emits one photodiode block synthesized from the *configured* v2 + /// drive: DAC waveform → Pockels-like sin² intensity transfer → ADC + /// codes. Without a configured `wave` (or with `amplitude_dac = 0`) the + /// output is the flat unmodulated level at `center_dac`. + pub fn emit_configured_block(&mut self) { + if self.state != MockState::Running { + return; + } + let config = self.config.clone(); + let rate = f64::from(config.rate_hz); + let freq_hz = f64::from(config.freq_mhz) / 1_000.0; + let codes: Vec = (0..config.block_samples as u64) + .map(|i| { + let t = (self.sample_index + i) as f64 / rate; + let shape = match (config.wave, config.amplitude_dac) { + (Some(wave), amplitude) if amplitude > 0 && freq_hz > 0.0 => { + wave.value((t * freq_hz).fract()) + } + _ => 0.0, + }; + let drive = f64::from(config.center_dac) + f64::from(config.amplitude_dac) * shape; + let transmission = (std::f64::consts::FRAC_PI_2 * drive / 4_095.0) + .sin() + .powi(2); + (self.synth_dark_code + self.synth_span_codes * transmission) + .round() + .clamp(0.0, 4_095.0) as u16 + }) + .collect(); + self.emit_codes_block(&codes, config.rate_hz, config.raw, config.summary); + } + + /// Emits one synthetic sinusoidal sample block (`SamplesU16` + + /// `Summary`), bypassing the drive model — codes = center + A·sin. + pub fn emit_sine_block(&mut self, samples: usize, rate_hz: u32, freq_hz: f64) { + let codes: Vec = (0..samples as u64) + .map(|i| { + let t = (self.sample_index + i) as f64 / f64::from(rate_hz); + (self.synth_center + + self.synth_amplitude * (2.0 * std::f64::consts::PI * freq_hz * t).sin()) + .round() + .clamp(0.0, 4_095.0) as u16 + }) + .collect(); + self.emit_codes_block(&codes, rate_hz, true, true); } /// Emits a summary frame carrying a nonzero overrun counter. @@ -281,3 +539,179 @@ impl MockController { pub fn parse_control(text: &str) -> Option { ControlMessage::parse(text).ok() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::transport::MockLink; + + fn request(controller: &mut MockController, line: &str) { + let mut bytes = line.as_bytes().to_vec(); + bytes.push(b'\n'); + // Feed the line directly through the device-side buffer path. + controller.line_buffer.extend_from_slice(&bytes); + while let Some(pos) = controller.line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = controller.line_buffer.drain(..=pos).collect(); + let text = std::str::from_utf8(&line).unwrap().trim_end().to_owned(); + controller.handle_line(&text); + } + } + + fn last_control_text(host: &mut crate::transport::MockTransport) -> String { + let mut parser = crate::wire::FrameParser::default(); + let mut buf = [0_u8; 4096]; + let mut last = None; + loop { + let n = crate::transport::Transport::read(host, &mut buf).unwrap(); + if n == 0 { + break; + } + parser.extend(&buf[..n]); + } + while let Some(event) = parser.next_event() { + if let crate::wire::ParseEvent::Frame(frame) = event { + if let Some(text) = frame.control_text() { + last = Some(text.to_owned()); + } + } + } + last.expect("a control frame was emitted") + } + + #[test] + fn matches_firmware_state_machine_and_error_details() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + // START before CONFIG → STATE error, firmware detail string. + request(&mut controller, "@1 START"); + assert!(last_control_text(&mut host).contains("code=STATE detail=configure_before_start")); + + // Valid CONFIG, then START, then CONFIG while running is rejected. + request(&mut controller, "@2 CONFIG mode=A1 rate_hz=20000"); + assert!(last_control_text(&mut host).starts_with("+2 OK state=CONFIGURED")); + request(&mut controller, "@3 START"); + assert_eq!(controller.state(), MockState::Running); + request(&mut controller, "@4 CONFIG mode=A1 rate_hz=20000"); + assert!(last_control_text(&mut host).contains("code=STATE detail=stop_before_config")); + + // STOP always succeeds and ignores extra fields. + request(&mut controller, "@5 STOP reason=test"); + assert_eq!(controller.state(), MockState::SafeIdle); + } + + #[test] + fn firmware_v1_rejects_waveform_fields_as_unknown() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + request( + &mut controller, + "@1 CONFIG mode=A1 wave=SINE freq_mhz=1000000 rate_hz=20000", + ); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=unknown_config_field")); + } + + #[test] + fn hello_requires_protocol_v1_and_advertises_capabilities_only_with_extension() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + request(&mut controller, "@1 HELLO"); + assert!(last_control_text(&mut host).contains("code=PROTOCOL detail=requires_v1")); + request(&mut controller, "@2 HELLO protocol=1"); + assert!(!last_control_text(&mut host).contains("capabilities")); + + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + request(&mut controller, "@1 HELLO protocol=1"); + assert!(last_control_text(&mut host).contains("capabilities=A1,A2,A3,WAVE")); + } + + #[test] + fn waveform_extension_validates_drive_bounds() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + + request( + &mut controller, + "@1 CONFIG mode=A1 rate_hz=20000 wave=TRIANGLE freq_mhz=1000000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=invalid_wave")); + + request( + &mut controller, + "@2 CONFIG mode=A1 rate_hz=20000 wave=SINE freq_mhz=1000000 center_dac=3000 \ + amplitude_dac=2000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=amplitude_exceeds_range")); + + request( + &mut controller, + "@3 CONFIG mode=A1 rate_hz=20000 wave=SAW freq_mhz=1000000 center_dac=2048 \ + amplitude_dac=512", + ); + assert!(last_control_text(&mut host).starts_with("+3 OK state=CONFIGURED")); + } + + #[test] + fn configured_drive_synthesizes_nonlinear_pockels_response() { + let contrast_for_amplitude = |amplitude: u32| -> f64 { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + request( + &mut controller, + &format!( + "@1 CONFIG mode=A1 rate_hz=20000 wave=SINE freq_mhz=100000 center_dac=2048 \ + amplitude_dac={amplitude}" + ), + ); + request(&mut controller, "@2 START"); + let _ = last_control_text(&mut host); + for _ in 0..8 { + controller.emit_configured_block(); + } + + let mut parser = crate::wire::FrameParser::default(); + let mut buf = [0_u8; 65_536]; + loop { + let n = crate::transport::Transport::read(&mut host, &mut buf).unwrap(); + if n == 0 { + break; + } + parser.extend(&buf[..n]); + } + let mut codes = Vec::new(); + while let Some(event) = parser.next_event() { + if let crate::wire::ParseEvent::Frame(frame) = event { + if let Some(samples) = frame.samples() { + codes.extend(samples); + } + } + } + let estimate = crate::estimator::estimate_contrast( + &codes, + &crate::estimator::AdcCalibration { + dark_volts: 40.0 * 3.3 / 4_095.0, + ..Default::default() + }, + ) + .expect("clean synthetic window"); + estimate.a + }; + + let a_small = contrast_for_amplitude(512); + let a_double = contrast_for_amplitude(1_024); + assert!(a_small > 0.0 && a_double > a_small); + // sin² transfer: doubling the DAC amplitude must NOT double a. + assert!( + (a_double / a_small - 2.0).abs() > 0.05, + "a_small={a_small} a_double={a_double} — response looks linear" + ); + } +} diff --git a/stage-a-io/src/pdq.rs b/stage-a-io/src/pdq.rs index 0e2119f..1eb4d1c 100644 --- a/stage-a-io/src/pdq.rs +++ b/stage-a-io/src/pdq.rs @@ -11,14 +11,14 @@ use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use crate::client::StreamIntegrity; -use crate::wire::{crc32, Frame}; +use crate::wire::{Crc32, Frame}; pub struct PdqWriter { path: PathBuf, file: BufWriter, frames_written: u64, bytes_written: u64, - running_crc_bytes: Vec, + running_crc: Crc32, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -43,7 +43,7 @@ impl PdqWriter { path, frames_written: 0, bytes_written: 0, - running_crc_bytes: Vec::new(), + running_crc: Crc32::default(), }) } @@ -52,7 +52,7 @@ impl PdqWriter { self.file.write_all(&bytes)?; self.frames_written += 1; self.bytes_written += bytes.len() as u64; - self.running_crc_bytes.extend_from_slice(&bytes); + self.running_crc.update(&bytes); Ok(()) } @@ -60,7 +60,7 @@ impl PdqWriter { pub fn finish(mut self, integrity: StreamIntegrity) -> std::io::Result { self.file.flush()?; Ok(PdqSummary { - file_crc32: crc32(&self.running_crc_bytes), + file_crc32: self.running_crc.finalize(), path: self.path, frames_written: self.frames_written, bytes_written: self.bytes_written, diff --git a/stage-a-io/src/wire.rs b/stage-a-io/src/wire.rs index 33e8caa..c42a249 100644 --- a/stage-a-io/src/wire.rs +++ b/stage-a-io/src/wire.rs @@ -65,6 +65,11 @@ impl FrameHeader { if magic != MAGIC { return None; } + // Unknown protocol versions are corruption, not future frames: the + // reference host parser resynchronises past them byte by byte. + if bytes[4] != PROTOCOL_VERSION { + return None; + } Some(Self { version: bytes[4], frame_type: FrameType::from_raw(bytes[5]), @@ -190,6 +195,29 @@ pub fn crc32(data: &[u8]) -> u32 { crc32_update(0xFFFF_FFFF, data) ^ 0xFFFF_FFFF } +/// Streaming CRC32 with the same parameters as [`crc32`], for hashing data +/// that is not held in memory at once (e.g. the PDQ file writer). +#[derive(Debug, Clone, Copy)] +pub struct Crc32 { + state: u32, +} + +impl Default for Crc32 { + fn default() -> Self { + Self { state: 0xFFFF_FFFF } + } +} + +impl Crc32 { + pub fn update(&mut self, data: &[u8]) { + self.state = crc32_update(self.state, data); + } + + pub fn finalize(self) -> u32 { + self.state ^ 0xFFFF_FFFF + } +} + fn crc32_update(mut crc: u32, data: &[u8]) -> u32 { for &byte in data { crc ^= u32::from(byte); From 58296eb864e8a70ab93d3c798cc89fcf9579902e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 14 Jul 2026 13:43:56 +0200 Subject: [PATCH 09/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20functi?= =?UTF-8?q?on-generator=20familiarisation=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New plugins/stage-a-funcgen crate: manual Pockels-cell drive control (sine/square/sawtooth, frequency, center/amplitude DAC codes) with the resulting optical amplitude always measured from the photodiode as a = ln(Vmax/Vmin) — the commanded amplitude is a phase-modulation depth and maps non-linearly to light, so it is never reported as an optical level. The default 'mock' port runs the waveform-extended mock controller on an in-process thread and streams a synthetic sin²-transfer photodiode response, so the full control loop works with zero hardware. Against real firmware 0.2.0 the reserved drive fields are feature-detected via the unknown_config_field rejection and reported as 'no waveform backend'; actual output stays blocked on the hardware freeze per stage-a-controller/docs/features/waveform-drive.md. Same fail-closed safety model as stage-a-monitor: LiveCapture + effects gating, drive parameters as settings but application as an explicit action, local DAC-range validation before any command, watchdog fault surfacing. --- Cargo.toml | 1 + docs/features/README.md | 1 + docs/features/stage-a-funcgen.md | 57 ++ docs/features/stage-a.md | 9 +- plugins/stage-a-funcgen/Cargo.toml | 15 + plugins/stage-a-funcgen/README.md | 47 ++ plugins/stage-a-funcgen/plugin.toml | 7 + plugins/stage-a-funcgen/src/lib.rs | 1011 +++++++++++++++++++++++++++ 8 files changed, 1146 insertions(+), 2 deletions(-) create mode 100644 docs/features/stage-a-funcgen.md create mode 100644 plugins/stage-a-funcgen/Cargo.toml create mode 100644 plugins/stage-a-funcgen/README.md create mode 100644 plugins/stage-a-funcgen/plugin.toml create mode 100644 plugins/stage-a-funcgen/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index e2b3d5a..3c96d7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "stage-a-io", "plugins/stage-a-monitor", "plugins/stage-a-a1", + "plugins/stage-a-funcgen", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/docs/features/README.md b/docs/features/README.md index 02e8e28..39e5f48 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -5,6 +5,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs - [Stage-A Calibration Plugins](./stage-a.md) — Teensy-driven Stage-A bench stack: `stage-a-io` shared I/O, commissioning monitor, and the A1 minimum-depth Bode sweep. +- [Stage-A Function Generator](./stage-a-funcgen.md) — familiarisation plugin: manual sine/square/sawtooth drive with photodiode-measured contrast, firmware-faithful mock, and the reserved waveform-drive protocol fields. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-funcgen.md b/docs/features/stage-a-funcgen.md new file mode 100644 index 0000000..3f70170 --- /dev/null +++ b/docs/features/stage-a-funcgen.md @@ -0,0 +1,57 @@ +# Stage-A Function Generator (`stage-a-funcgen`) + +> Feature brief — familiarisation plugin for the Stage-A bench. +> Protocol source of truth: +> `stage-a-controller/docs/features/waveform-drive.md` (reserved v2 fields). + +## Purpose + +Manual Pockels-cell drive control for getting to know the setup: waveform +(sine / square / sawtooth), frequency, and DAC modulation depth, with the +resulting optical amplitude always **measured** from the photodiode as +`a = ln(V_max/V_min)` — the Pockels V→T response is non-linear, so the DAC +excursion never doubles as a light level. + +## Included + +- `plugins/stage-a-funcgen` crate (`augur-plugin-stage-a-funcgen`): connect / + apply / stop actions, live photodiode waveform view, status table with the + measured contrast, clipping, and stream integrity; +- **`mock` port** (default): the waveform-extended mock controller runs on an + in-process thread and streams a synthetic photodiode response through a + Pockels-like sin² transfer — the complete control loop with zero hardware; +- feature detection against real firmware: 0.2.0 rejects the reserved drive + fields with `unknown_config_field`, which the plugin reports as "no + waveform backend" instead of a fault; +- same fail-closed safety model as `stage-a-monitor` (`LiveCapture` + + `effects_allowed` only; drive parameters are settings, applying them is an + explicit action; local bounds check before any command is sent). + +## Firmware-faithful mock (stage-a-io) + +Delivered together with this plugin, `stage-a-io`'s `MockController` now +mirrors firmware 0.2.0 exactly — verbs, state machine (`SAFE_IDLE` → +`CONFIGURED` → `RUNNING`), error codes/details, single-entry idempotent reply +cache, and unknown-CONFIG-field rejection. The previous mock accepted verbs +and fields the device does not speak (`ARM`, `RUN`, `capabilities=`, +`BAD_*`), which let host bugs pass tests: the A1 sweep reconfigured while +RUNNING (now fixed with STOP-before-CONFIG) and watchdog `!FAULT` notices +were invisible outside an in-flight request (now surfaced as async events by +`StageAClient` and handled by all three plugins). + +## Verification + +`cargo test -p stage-a-io -p augur-plugin-stage-a-funcgen +-p augur-plugin-stage-a-monitor -p augur-plugin-stage-a-a1`: mock +state-machine/error fidelity against `main.cpp`, v1 rejection of waveform +fields, drive-bounds validation, nonlinear sin² contrast response, watchdog +fault propagation, and the full mock round trip (connect → apply sine / +square / saw → measured `a` → stop → reconfigure while driving). + +## Known gaps + +- Real firmware cannot emit a waveform yet; the `waveform-drive.md` fields + stay host+mock-only until the hardware freeze resolves the DAC channel and + safe HVA window. +- No PDQ/sidecar recording in this plugin — it is a familiarisation tool; + evidence-grade recording stays with `stage-a-monitor`/`stage-a-a1`. diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md index 7924c6b..98354e8 100644 --- a/docs/features/stage-a.md +++ b/docs/features/stage-a.md @@ -11,6 +11,7 @@ AugurRs generic host (camera, RAW, EXT_TRIGGER delivery, execution context — ABI v5) │ ├── stage-a-monitor — commissioning: live photodiode view, manual control + ├── stage-a-funcgen — familiarisation: manual waveform drive (see stage-a-funcgen.md) └── stage-a-a1 — A1 minimum-depth a_min(f) sweep │ (exactly one armed plugin owns the device) ▼ @@ -24,8 +25,9 @@ lives entirely in these removable plugins (ADR 005). | Crate | Role | |---|---| -| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, mock controller | +| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, firmware-faithful mock controller (0.2.0 surface + opt-in v2 waveform extension) | | `plugins/stage-a-monitor` | Live decimated waveform, live `a`, integrity status, gated manual CONFIG/START/STOP + expert drive modal | +| `plugins/stage-a-funcgen` | Manual waveform drive (sine/square/saw, frequency, DAC depth) with photodiode-measured `a`; in-process mock port for hardware-free familiarisation | | `plugins/stage-a-a1` | Phase-locked detection (Rayleigh), hardware/software cycle fiducials, bisection + grid sweep, probit `a_min` fit with CI, hot-pixel mask, PDQ + sidecar + results export | ## Safety model @@ -65,7 +67,10 @@ dataset/schema consistency. ## Known gaps - Final Teensy DDS/DAC firmware is blocked on the hardware freeze; the - sweep runs against the v1 protocol and the mock meanwhile. + sweep and the function generator run against the reserved waveform-drive + protocol (`stage-a-controller/docs/features/waveform-drive.md`) and the + waveform-extended mock meanwhile — firmware 0.2.0 rejects the drive + fields with `unknown_config_field` (feature detection). - Marker cycles are protocol-reserved but not yet emitted (`stage-a-controller/docs/features/a1-marker-cycles.md`). - `stage-a-a2` / `stage-a-a3` plugins are not yet implemented; A2 diff --git a/plugins/stage-a-funcgen/Cargo.toml b/plugins/stage-a-funcgen/Cargo.toml new file mode 100644 index 0000000..ffb540e --- /dev/null +++ b/plugins/stage-a-funcgen/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "augur-plugin-stage-a-funcgen" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A function generator: manual waveform/frequency/amplitude drive control with photodiode-measured optical contrast." + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde_json.workspace = true +stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-funcgen/README.md b/plugins/stage-a-funcgen/README.md new file mode 100644 index 0000000..fb57f6c --- /dev/null +++ b/plugins/stage-a-funcgen/README.md @@ -0,0 +1,47 @@ +# Stage-A Function Generator + +Manual control of the Stage-A Pockels-cell drive for familiarisation with the +bench: pick a waveform (**sine**, **square**, **sawtooth**), a frequency, and a +DAC modulation depth, hit *Apply drive*, and watch the photodiode respond live. + +## Why the amplitude is "measured", not set + +`amplitude_dac` commands the *phase*-modulation depth of the Pockels cell. The +cell's voltage→transmission response is non-linear (≈ sin²), so the same DAC +excursion produces different optical amplitudes at different working points. +The plugin therefore always reports the **measured** optical log-contrast + +``` +a = ln(V_max / V_min) (dark-corrected photodiode voltages) +``` + +computed by `stage-a-io`'s calibrated, clipping-guarded estimator — never a +value inferred from the commanded DAC codes. + +## Ports + +| Port | Behaviour | +|---|---| +| `mock` (default) | Runs the waveform-extended mock controller in-process: full command round trip, synthetic photodiode stream through a Pockels-like sin² transfer. Zero hardware, zero risk. | +| `auto` / explicit device | Real Teensy over USB serial. Firmware 0.2.0 has **no waveform backend** and rejects the drive fields (`unknown_config_field`); the plugin reports this clearly. Real drive control needs the future v2 DDS firmware (`stage-a-controller/docs/features/waveform-drive.md`), which is blocked on the hardware freeze. | + +## Views and actions + +- **FuncGen photodiode** — live decimated waveform (volts vs. ms). +- **Function generator** status table — state, firmware, waveform-backend + capability, commanded drive, measured `a`, clipping, stream integrity. +- Actions on the status table: *Connect*, *Disconnect*, *Apply drive*, + *Stop drive*. + +## Safety model + +Same contract as `stage-a-monitor`: + +- serial/mock connections open only while the execution context is + `LiveCapture` with effects allowed — replay can never drive hardware; +- waveform/frequency/amplitude are persistent *settings*, but nothing reaches + the controller until the explicit *Apply drive* **action**; +- drives whose `center ± amplitude` leave the 0–4095 DAC range are refused + locally before any command is sent; +- `process_frame()` only drains the bounded I/O worker queues; +- watchdog `!FAULT` notices from the controller are surfaced immediately. diff --git a/plugins/stage-a-funcgen/plugin.toml b/plugins/stage-a-funcgen/plugin.toml new file mode 100644 index 0000000..df29231 --- /dev/null +++ b/plugins/stage-a-funcgen/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Function Generator" +version = "0.2.0" +description = "Manual Pockels-cell drive control (sine/square/sawtooth, frequency, DAC amplitude) with the resulting optical contrast always measured from the photodiode." +domain = "stage-a" +library = "augur_plugin_stage_a_funcgen" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-funcgen/src/lib.rs b/plugins/stage-a-funcgen/src/lib.rs new file mode 100644 index 0000000..694a21c --- /dev/null +++ b/plugins/stage-a-funcgen/src/lib.rs @@ -0,0 +1,1011 @@ +//! Stage-A function generator — familiarisation plugin. +//! +//! Manual control of the Pockels-cell drive: waveform (sine, square, +//! sawtooth), frequency, and the commanded DAC modulation depth +//! (`amplitude_dac`). The commanded amplitude sets the *phase* modulation +//! of the Pockels cell, which maps non-linearly to transmitted intensity — +//! so the optical amplitude shown here is always the photodiode-measured +//! log-contrast `a = ln(V_max/V_min)`, never the DAC excursion. +//! +//! Firmware 0.2.0 has no waveform backend yet: it rejects the reserved v2 +//! drive fields with `unknown_config_field` (the feature-detection +//! contract, `stage-a-controller/docs/features/waveform-drive.md`). Until +//! the DDS firmware lands, select the **`mock`** port: it runs the +//! waveform-extended mock controller in-process and streams a synthetic +//! photodiode response through a Pockels-like sin² transfer — the full +//! control loop with zero hardware and zero risk. +//! +//! Safety contract (same as `stage-a-monitor`): +//! - devices open only when the execution context is `LiveCapture` with +//! `effects_allowed`; anything else tears the connection down; +//! - drive parameters are persistent *settings*, but nothing starts the +//! hardware except an explicit Apply **action**; +//! - `process_frame()` only drains the bounded I/O worker queues. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, + Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, + StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, + TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, + MockController, MockState, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, +}; + +const WAVEFORM_DATASET_ID: &str = "stage-a-funcgen.waveform"; +const STATUS_DATASET_ID: &str = "stage-a-funcgen.status"; +const WAVEFORM_VIEW_ID: &str = "stage-a-funcgen.waveform.view"; +const STATUS_VIEW_ID: &str = "stage-a-funcgen.status.view"; + +const ACTION_CONNECT: &str = "stage-a-funcgen.connect"; +const ACTION_DISCONNECT: &str = "stage-a-funcgen.disconnect"; +const ACTION_APPLY: &str = "stage-a-funcgen.apply"; +const ACTION_STOP: &str = "stage-a-funcgen.stop"; + +/// Retained sample window for the live view + contrast estimate. +const SAMPLE_RING_CAPACITY: usize = 32_768; +/// Points published per waveform refresh (decimated). +const WAVEFORM_POINTS: usize = 1_024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ConnectionState { + Disconnected, + Connected, + Driving, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Wave { + Sine, + Square, + Saw, +} + +impl Wave { + const VARIANTS: [Wave; 3] = [Wave::Sine, Wave::Square, Wave::Saw]; + + fn name(self) -> &'static str { + match self { + Self::Sine => "SINE", + Self::Square => "SQUARE", + Self::Saw => "SAW", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|w| w.name() == name) + } +} + +/// In-process mock controller thread behind the `mock` port. +struct MockService { + stop: Arc, + join: Option>, +} + +impl MockService { + fn spawn() -> (Self, StageAClient) { + let link = stage_a_io::MockLink::new(); + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + let join = std::thread::Builder::new() + .name("stage-a-funcgen-mock".into()) + .spawn(move || { + let mut last_block = Instant::now(); + while !thread_stop.load(Ordering::Relaxed) { + controller.poll_commands(); + if controller.state() == MockState::Running + && last_block.elapsed() >= controller.block_period() + { + last_block = Instant::now(); + controller.emit_configured_block(); + } + std::thread::sleep(Duration::from_millis(1)); + } + }) + .expect("spawning the mock controller thread must succeed"); + ( + Self { + stop, + join: Some(join), + }, + StageAClient::new(link.host_end()), + ) + } +} + +impl Drop for MockService { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +pub struct StageAFuncGenPlugin { + enabled: bool, + // -- device -- + worker: Option, + mock_service: Option, + connection: ConnectionState, + firmware: String, + has_waveform_backend: Option, + next_tag: u64, + in_flight: BTreeMap, + last_error: Option, + integrity: StreamIntegrity, + effects_blocked_reason: Option, + // -- settings (drive parameters; applying them is an explicit action) -- + port_hint: String, + wave: Wave, + frequency_hz: f64, + center_dac: i64, + amplitude_dac: i64, + sample_rate_hz: i64, + calibration: AdcCalibration, + // -- data -- + sample_ring: Vec, + ring_next_sample_index: u64, + sample_rate_seen_hz: u32, + contrast: Option, + contrast_error: Option, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAFuncGenPlugin { + fn default() -> Self { + Self { + enabled: false, + worker: None, + mock_service: None, + connection: ConnectionState::Disconnected, + firmware: String::new(), + has_waveform_backend: None, + next_tag: 1, + in_flight: BTreeMap::new(), + last_error: None, + integrity: StreamIntegrity::default(), + effects_blocked_reason: None, + port_hint: "mock".into(), + wave: Wave::Sine, + frequency_hz: 1_000.0, + center_dac: 2_048, + amplitude_dac: 512, + sample_rate_hz: 20_000, + calibration: AdcCalibration::default(), + sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), + ring_next_sample_index: 0, + sample_rate_seen_hz: 0, + contrast: None, + contrast_error: None, + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAFuncGenPlugin { + fn bump_generation(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn connect(&mut self) { + if self.worker.is_some() { + return; + } + if self.port_hint == "mock" { + let (service, client) = MockService::spawn(); + self.mock_service = Some(service); + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + } else { + match open_serial(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + } + Err(err) => self.last_error = Some(err), + } + } + self.bump_generation(); + } + + fn disconnect(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + // Shut the worker down first: its final STOP still needs the + // mock service (if any) alive to be acknowledged. + worker.shutdown(reason); + } + self.mock_service = None; + self.connection = ConnectionState::Disconnected; + self.firmware.clear(); + self.has_waveform_backend = None; + self.in_flight.clear(); + self.bump_generation(); + } + + /// STOP → CONFIG (drive fields) → START, honouring the firmware state + /// machine (CONFIG is only legal from SAFE_IDLE/CONFIGURED). + fn apply_drive(&mut self) { + let center = self.center_dac.clamp(0, 4_095); + let amplitude = self.amplitude_dac.clamp(0, 2_047); + if center + amplitude > 4_095 || amplitude > center { + self.last_error = Some(format!( + "drive: center {center} ± amplitude {amplitude} exceeds the 0–4095 DAC range" + )); + return; + } + let freq_mhz = ((self.frequency_hz.max(0.001)) * 1_000.0).round() as i64; + self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); + self.queue_command( + "drive", + Command::new("CONFIG") + .field("mode", "A1") + .field("wave", self.wave.name()) + .field("freq_mhz", freq_mhz) + .field("center_dac", center) + .field("amplitude_dac", amplitude) + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", 256) + .field("raw", 1) + .field("summary", 1), + ); + self.queue_command("start", Command::new("START")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(true)); + } + } + + fn stop_drive(&mut self) { + self.queue_command("stop", Command::new("STOP").field("reason", "operator")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + if outputs.is_empty() { + return; + } + let mut changed = false; + let mut stopped: Option = None; + for output in outputs { + changed = true; + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) if err.contains("unknown_config_field") => { + self.has_waveform_backend = Some(false); + self.last_error = Some( + "firmware has no waveform backend (v1) — select the mock port \ + or wait for the v2 DDS firmware" + .into(), + ); + } + Err(err) => { + self.last_error = Some(format!("{purpose}: {err}")); + } + } + } + WorkerOutput::Event(DeviceEvent::Data(frame)) => { + if frame.header.frame_type == FrameType::SamplesU16 { + if let Some(codes) = frame.samples() { + self.sample_rate_seen_hz = frame.header.sample_rate_hz; + self.push_samples(&codes, frame.header.first_sample_index); + } + } + } + WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { + if name == "FAULT" { + if self.connection == ConnectionState::Driving { + self.connection = ConnectionState::Connected; + } + self.last_error = Some(format!( + "controller fault: {} — dropped to SAFE_IDLE", + fields.get("code").map(String::as_str).unwrap_or("unknown") + )); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + } + WorkerOutput::Integrity(integrity) => { + self.integrity = integrity; + } + WorkerOutput::Stopped { reason } => { + stopped = Some(reason); + } + } + } + if let Some(reason) = stopped { + self.worker = None; + self.mock_service = None; + self.connection = ConnectionState::Disconnected; + self.last_error = Some(format!("device connection ended: {reason}")); + } + if changed { + self.refresh_contrast(); + self.bump_generation(); + } + } + + fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { + match purpose { + "hello" => { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + self.has_waveform_backend = Some( + fields + .get("capabilities") + .is_some_and(|caps| caps.split(',').any(|c| c == "WAVE")), + ); + self.connection = ConnectionState::Connected; + } + "drive" => { + self.has_waveform_backend = Some(true); + } + "start" => { + self.connection = ConnectionState::Driving; + } + "stop" => { + if self.connection == ConnectionState::Driving { + self.connection = ConnectionState::Connected; + } + } + _ => {} + } + } + + fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { + self.ring_next_sample_index = first_sample_index + codes.len() as u64; + self.sample_ring.extend_from_slice(codes); + let len = self.sample_ring.len(); + if len > SAMPLE_RING_CAPACITY { + self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); + } + } + + fn refresh_contrast(&mut self) { + if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { + return; + } + match estimate_contrast(&self.sample_ring, &self.calibration) { + Ok(estimate) => { + self.contrast = Some(estimate); + self.contrast_error = None; + } + Err(err) => { + self.contrast = None; + self.contrast_error = Some(err.to_string()); + } + } + } + + fn waveform_dataset(&self) -> Series1dV1 { + let rate = if self.sample_rate_seen_hz > 0 { + f64::from(self.sample_rate_seen_hz) + } else { + self.sample_rate_hz as f64 + }; + let n = self.sample_ring.len(); + let stride = (n / WAVEFORM_POINTS).max(1); + let first_index = self.ring_next_sample_index.saturating_sub(n as u64); + let points: Vec = self + .sample_ring + .iter() + .enumerate() + .step_by(stride) + .map(|(i, &code)| Series1dPoint { + x: (first_index + i as u64) as f64 / rate * 1_000.0, + y: self.calibration.code_to_volts(code), + }) + .collect(); + Series1dV1 { + x_label: "time [ms]".into(), + y_label: "photodiode [V]".into(), + lines: vec![Series1dLine { + name: "photodiode".into(), + points, + }], + } + } + + fn drive_summary(&self) -> String { + format!( + "{} @ {:.3} Hz, {} ± {} DAC", + self.wave.name(), + self.frequency_hz, + self.center_dac, + self.amplitude_dac + ) + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.connection) { + (Some(reason), _) => format!("locked ({reason})"), + (None, ConnectionState::Disconnected) => "disconnected".into(), + (None, ConnectionState::Connected) => "connected".into(), + (None, ConnectionState::Driving) => "driving".into(), + }; + let backend = match self.has_waveform_backend { + Some(true) => "waveform-capable".into(), + Some(false) => "no waveform backend (v1)".into(), + None => "—".into(), + }; + let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { + (Some(estimate), _) => ( + format!("{:.4}", estimate.a), + format!( + "{:.2}% low / {:.2}% high", + estimate.low_clip_fraction * 100.0, + estimate.high_clip_fraction * 100.0 + ), + ), + (None, Some(err)) => ("invalid".into(), err.clone()), + (None, None) => ("—".into(), "—".into()), + }; + let integrity = if self.integrity.is_clean() { + "clean".to_owned() + } else { + format!( + "crc={} gaps={} skipped={} overruns={}", + self.integrity.crc_failures, + self.integrity.sequence_gaps, + self.integrity.skipped_bytes, + self.integrity.dropped_samples + ) + }; + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("firmware", self.firmware.clone()), + text_column("backend", backend), + text_column("drive", self.drive_summary()), + text_column("a", a_text), + text_column("clipping", clip_text), + text_column("integrity", integrity), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("firmware", "Firmware"), + column("backend", "Waveform backend"), + column("drive", "Commanded drive"), + column("a", "Measured a = ln(Vmax/Vmin)"), + column("clipping", "Clipping"), + column("integrity", "Stream integrity"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-funcgen.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } +} + +fn open_serial(port_hint: &str) -> Result, String> { + let path = if port_hint == "auto" { + serial_ports() + .into_iter() + .next() + .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? + } else { + port_hint.to_owned() + }; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +fn serial_ports() -> Vec { + stage_a_io::transport::available_port_names() + .into_iter() + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() +} + +impl Plugin for StageAFuncGenPlugin { + fn name(&self) -> &'static str { + "Stage-A Function Generator" + } + + fn description(&self) -> &'static str { + "Manual Pockels-cell drive (sine/square/sawtooth, frequency, DAC amplitude) with photodiode-measured optical contrast; mock port for hardware-free familiarisation." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect("plugin disabled"); + } + } + + fn reset(&mut self) { + self.sample_ring.clear(); + self.contrast = None; + self.contrast_error = None; + self.bump_generation(); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Fail closed: any pass without live-capture effects tears the + // connection down and refuses commands — even for the mock port, + // so switching the port setting can never bypass the gate. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.worker.is_some() { + self.disconnect("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for action_id in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect("operator"), + ACTION_APPLY => self.apply_drive(), + ACTION_STOP => self.stop_drive(), + _ => {} + } + } + + self.drain_worker(); + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + let wave_variants: Vec = + Wave::VARIANTS.iter().map(|w| w.name().to_owned()).collect(); + let wave_default = Wave::VARIANTS + .iter() + .position(|w| *w == self.wave) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Function generator".into(), + description: Some( + "Drive parameters are settings; nothing reaches the hardware until the \ + Apply action. The optical amplitude is measured from the photodiode — \ + the DAC amplitude is a phase-modulation depth, not a light level." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "mock = in-process simulated controller (no hardware); \ + auto = first Teensy USB serial device" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "wave".into(), + label: "Waveform".into(), + tooltip: Some("SINE, SQUARE, or SAW (sawtooth / Sägezahn)".into()), + kind: SettingKind::Enum { + variants: wave_variants, + default: wave_default, + }, + }, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Drive frequency (sent as integer millihertz)".into()), + kind: SettingKind::F64Drag { + min: 0.001, + max: 200_000.0, + speed: 1.0, + default: self.frequency_hz, + }, + }, + SettingItem { + key: "center_dac".into(), + label: "Center DAC code".into(), + tooltip: Some("Working-point code (0–4095)".into()), + kind: SettingKind::I64Drag { + min: 0, + max: 4_095, + default: self.center_dac, + }, + }, + SettingItem { + key: "amplitude_dac".into(), + label: "Amplitude DAC code".into(), + tooltip: Some( + "Pockels phase-modulation depth (0–2047); the optical contrast \ + this produces is read from the measured a" + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: 2_047, + default: self.amplitude_dac, + }, + }, + SettingItem { + key: "sample_rate_hz".into(), + label: "ADC sample rate".into(), + tooltip: Some("Photodiode sample rate for the feedback stream".into()), + kind: SettingKind::I64Slider { + min: 1_000, + max: 100_000, + default: self.sample_rate_hz, + suffix: Some(" Hz".into()), + }, + }, + SettingItem { + key: "dark_millivolts".into(), + label: "Dark level".into(), + tooltip: Some( + "Light-blocked photodiode level; a is computed from dark-corrected \ + voltages" + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 3_300.0, + speed: 1.0, + default: self.calibration.dark_volts * 1_000.0, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "wave" => Some(json!(self.wave.name())), + "frequency_hz" => Some(json!(self.frequency_hz)), + "center_dac" => Some(json!(self.center_dac)), + "amplitude_dac" => Some(json!(self.amplitude_dac)), + "sample_rate_hz" => Some(json!(self.sample_rate_hz)), + "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "wave" => { + let name = value.as_str().ok_or("wave must be a string")?; + self.wave = Wave::from_name(name) + .ok_or_else(|| format!("unknown waveform: {name} (SINE/SQUARE/SAW)"))?; + Ok(()) + } + "frequency_hz" => { + let hz = value.as_f64().ok_or("frequency_hz must be a number")?; + self.frequency_hz = hz.clamp(0.001, 200_000.0); + Ok(()) + } + "center_dac" => { + self.center_dac = value + .as_i64() + .ok_or("center_dac must be an integer")? + .clamp(0, 4_095); + Ok(()) + } + "amplitude_dac" => { + self.amplitude_dac = value + .as_i64() + .ok_or("amplitude_dac must be an integer")? + .clamp(0, 2_047); + Ok(()) + } + "sample_rate_hz" => { + self.sample_rate_hz = value + .as_i64() + .ok_or("sample_rate_hz must be an integer")? + .clamp(1_000, 100_000); + Ok(()) + } + "dark_millivolts" => { + let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; + self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + entries.push(StatusEntry::Text(match self.connection { + ConnectionState::Disconnected => "FuncGen: disconnected".into(), + ConnectionState::Connected => format!("FuncGen: connected ({})", self.firmware), + ConnectionState::Driving => format!("FuncGen: driving {}", self.drive_summary()), + })); + if let Some(estimate) = &self.contrast { + entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + let dataset_action = |id: &str, title: &str| HostActionDescriptor { + id: id.into(), + title: title.into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }; + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: WAVEFORM_DATASET_ID.into(), + title: "FuncGen photodiode waveform".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No photodiode samples yet — connect and apply a drive.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Function generator status".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Function generator idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: WAVEFORM_VIEW_ID.into(), + title: "FuncGen photodiode".into(), + dataset_id: WAVEFORM_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Function generator".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + dataset_action(ACTION_CONNECT, "Connect"), + dataset_action(ACTION_DISCONNECT, "Disconnect"), + dataset_action(ACTION_APPLY, "Apply drive"), + dataset_action(ACTION_STOP, "Stop drive"), + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), + _ => 0, + } + } +} + +impl Drop for StageAFuncGenPlugin { + fn drop(&mut self) { + self.disconnect("plugin destroyed"); + } +} + +export_plugin!(StageAFuncGenPlugin); + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, Instant}; + + fn drain_until bool>( + plugin: &mut StageAFuncGenPlugin, + timeout: Duration, + mut done: F, + ) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + plugin.drain_worker(); + if done(plugin) { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("condition not reached within {timeout:?}"); + } + + /// Full mock loop: connect → apply sine → measured a appears → stop. + #[test] + fn mock_port_round_trip_measures_optical_contrast() { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.calibration.dark_volts = 40.0 * 3.3 / 4_095.0; + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + assert_eq!(plugin.has_waveform_backend, Some(true)); + assert_eq!(plugin.firmware, "0.2.0-mock"); + + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving && p.contrast.is_some() + }); + let a = plugin.contrast.as_ref().expect("contrast measured").a; + assert!(a > 0.0, "modulated drive must produce positive contrast"); + assert!(plugin.integrity.is_clean()); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + + plugin.stop_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + plugin.disconnect("test done"); + assert_eq!(plugin.connection, ConnectionState::Disconnected); + } + + /// Square and sawtooth are accepted and produce a measurable contrast. + #[test] + fn square_and_saw_waveforms_drive_the_mock() { + for wave in [Wave::Square, Wave::Saw] { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.wave = wave; + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving && p.contrast.is_some() + }); + assert!(plugin.contrast.as_ref().unwrap().a > 0.0); + plugin.disconnect("done"); + } + } + + /// Drives exceeding the DAC range are refused locally, before any + /// command reaches a controller. + #[test] + fn out_of_range_drive_is_rejected_locally() { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.center_dac = 3_000; + plugin.amplitude_dac = 2_000; + plugin.apply_drive(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("exceeds the 0–4095 DAC range"))); + } + + /// Re-applying while driving must STOP first (firmware state machine). + #[test] + fn reapply_while_driving_reconfigures_cleanly() { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving + }); + plugin.frequency_hz = 2_000.0; + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving && p.last_error.is_none() + }); + plugin.disconnect("done"); + } +} From 525cc4d5790f2ce96798a47889619eb104b64198 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 15 Jul 2026 19:56:13 +0200 Subject: [PATCH 10/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20replace=20co?= =?UTF-8?q?mmissioning=20plugins=20with=20minimal=20modulation=20and=20pho?= =?UTF-8?q?todiode=20pair?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - delete stage-a-monitor, stage-a-funcgen, stage-a-a1 (too complex for the current bench stage; retained in git history) - add stage-a-modulation: capped power slider, CONST/SINE/SQUARE with frequency and min threshold, immediate MOD transfer, board-reported DAC code - add stage-a-photodiode: SMA5/pin18/A4 stream readout on the second CDC port with RAW and EXCITATION (I_exc = I_tot - I_pd) modes and rolling chart - extend the stage-a-io mock to firmware 0.3.0 (MOD verb, capabilities) - ADR 006 (two plugins, one port each), rewritten stage-a brief, doc updates --- Cargo.toml | 6 +- docs/adr/006-stage-a-two-plugin-split.md | 48 + docs/features/README.md | 5 +- docs/features/stage-a-funcgen.md | 57 - docs/features/stage-a-modulation.md | 37 + docs/features/stage-a-photodiode.md | 34 + docs/features/stage-a.md | 94 +- plugins/stage-a-a1/README.md | 66 - plugins/stage-a-a1/plugin.toml | 7 - plugins/stage-a-a1/src/analysis.rs | 610 -------- plugins/stage-a-a1/src/lib.rs | 1254 ----------------- plugins/stage-a-a1/src/sweep.rs | 368 ----- plugins/stage-a-funcgen/README.md | 47 - plugins/stage-a-funcgen/plugin.toml | 7 - plugins/stage-a-funcgen/src/lib.rs | 1011 ------------- .../Cargo.toml | 4 +- plugins/stage-a-modulation/README.md | 33 + plugins/stage-a-modulation/plugin.toml | 7 + plugins/stage-a-modulation/src/lib.rs | 844 +++++++++++ plugins/stage-a-monitor/Cargo.toml | 15 - plugins/stage-a-monitor/README.md | 39 - plugins/stage-a-monitor/plugin.toml | 7 - plugins/stage-a-monitor/src/lib.rs | 831 ----------- .../Cargo.toml | 6 +- plugins/stage-a-photodiode/README.md | 25 + plugins/stage-a-photodiode/plugin.toml | 7 + plugins/stage-a-photodiode/src/lib.rs | 774 ++++++++++ stage-a-io/src/lib.rs | 5 +- stage-a-io/src/mock.rs | 172 ++- 29 files changed, 2006 insertions(+), 4414 deletions(-) create mode 100644 docs/adr/006-stage-a-two-plugin-split.md delete mode 100644 docs/features/stage-a-funcgen.md create mode 100644 docs/features/stage-a-modulation.md create mode 100644 docs/features/stage-a-photodiode.md delete mode 100644 plugins/stage-a-a1/README.md delete mode 100644 plugins/stage-a-a1/plugin.toml delete mode 100644 plugins/stage-a-a1/src/analysis.rs delete mode 100644 plugins/stage-a-a1/src/lib.rs delete mode 100644 plugins/stage-a-a1/src/sweep.rs delete mode 100644 plugins/stage-a-funcgen/README.md delete mode 100644 plugins/stage-a-funcgen/plugin.toml delete mode 100644 plugins/stage-a-funcgen/src/lib.rs rename plugins/{stage-a-funcgen => stage-a-modulation}/Cargo.toml (58%) create mode 100644 plugins/stage-a-modulation/README.md create mode 100644 plugins/stage-a-modulation/plugin.toml create mode 100644 plugins/stage-a-modulation/src/lib.rs delete mode 100644 plugins/stage-a-monitor/Cargo.toml delete mode 100644 plugins/stage-a-monitor/README.md delete mode 100644 plugins/stage-a-monitor/plugin.toml delete mode 100644 plugins/stage-a-monitor/src/lib.rs rename plugins/{stage-a-a1 => stage-a-photodiode}/Cargo.toml (51%) create mode 100644 plugins/stage-a-photodiode/README.md create mode 100644 plugins/stage-a-photodiode/plugin.toml create mode 100644 plugins/stage-a-photodiode/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 3c96d7b..dbee199 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,8 @@ [workspace] members = [ "stage-a-io", - "plugins/stage-a-monitor", - "plugins/stage-a-a1", - "plugins/stage-a-funcgen", + "plugins/stage-a-modulation", + "plugins/stage-a-photodiode", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", @@ -29,3 +28,4 @@ egui = "0.27" rustfft = "6" serde = { version = "1", features = ["derive"] } serde_json = "1" +serialport = "4" diff --git a/docs/adr/006-stage-a-two-plugin-split.md b/docs/adr/006-stage-a-two-plugin-split.md new file mode 100644 index 0000000..99971e3 --- /dev/null +++ b/docs/adr/006-stage-a-two-plugin-split.md @@ -0,0 +1,48 @@ +# ADR 006 — Stage-A simplification: two plugins, one serial port each + +- **Status:** Accepted +- **Date:** 2026-07-15 +- **Amends:** ADR 005 (Stage-A device ownership) + +## Context + +The three commissioning plugins (`stage-a-monitor`, `stage-a-funcgen`, `stage-a-a1`, ~3100 lines) +bundled experiment state machines, contrast estimation, and drive control into UIs that were too +complex and opaque for the current bench stage. What the bench actually needs now is: + +1. direct, immediate control of the laser modulation output (capped power slider, + constant/sine/square with frequency), and +2. a plain readout of the photodiode (raw, or inverted to excitation power). + +Both need the same Teensy, but ADR 005 fixes one owner per serial port — and a context-bus +coupling (one plugin republishing data for the other) would make the readout depend on the +control plugin's connection. + +## Decision + +1. **The firmware enumerates two USB CDC ports** (`USB_DUAL_SERIAL`, `stage-a-controller` + ADR 002): port 1 keeps the v1 command protocol; port 2 free-runs a plain-ASCII photodiode + stream. ADR 005's rule is unchanged — one owner per port — there are simply two ports now. +2. **Two minimal plugins replace the three commissioning plugins** (deleted 2026-07-15, retained + in git history): + - `stage-a-modulation` owns the command port (`docs/features/stage-a-modulation.md`); + - `stage-a-photodiode` owns the stream port (`docs/features/stage-a-photodiode.md`). +3. **`stage-a-io` stays** as the protocol library (wire format, client, worker, firmware-faithful + mock — the mock now models firmware 0.3.0's `MOD` verb). The A1/A2/A3 experiment plugins will + build on it again when the bench reaches that stage; the estimator/pdq/sidecar modules remain + for that purpose even though no current plugin uses them. +4. **Immediate transfer replaces the Apply-action pattern** in `stage-a-modulation`: setting + changes are sent to the device as they happen (the operator's explicit request), still behind + the fail-closed execution-context gate. The firmware output is set-and-hold; the explicit + "Output OFF" action is the only stop. + +## Consequences + +- Each plugin is a few hundred transparent lines with a single concern; the photodiode plugin + does not even depend on `stage-a-io`. +- Both plugins work independently — either can connect, disconnect, or crash without affecting + the other. +- Wire-protocol changes still land firmware-first (`stage-a-controller/include/wire_protocol.h` + and command grammar), then in `stage-a-io`'s client/mock. +- The A1 min-depth workflow is gone from the tree until it is rebuilt on the simplified stack; + its last state is tagged by the deletion commit. diff --git a/docs/features/README.md b/docs/features/README.md index 39e5f48..eb5ffe4 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -4,8 +4,9 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs -- [Stage-A Calibration Plugins](./stage-a.md) — Teensy-driven Stage-A bench stack: `stage-a-io` shared I/O, commissioning monitor, and the A1 minimum-depth Bode sweep. -- [Stage-A Function Generator](./stage-a-funcgen.md) — familiarisation plugin: manual sine/square/sawtooth drive with photodiode-measured contrast, firmware-faithful mock, and the reserved waveform-drive protocol fields. +- [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. +- [Stage-A Modulation](./stage-a-modulation.md) — capped power slider + constant/sine/square laser-modulation drive on the command port, applied immediately. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the stream port: raw values or excitation power `I_exc = I_tot − I_pd`. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-funcgen.md b/docs/features/stage-a-funcgen.md deleted file mode 100644 index 3f70170..0000000 --- a/docs/features/stage-a-funcgen.md +++ /dev/null @@ -1,57 +0,0 @@ -# Stage-A Function Generator (`stage-a-funcgen`) - -> Feature brief — familiarisation plugin for the Stage-A bench. -> Protocol source of truth: -> `stage-a-controller/docs/features/waveform-drive.md` (reserved v2 fields). - -## Purpose - -Manual Pockels-cell drive control for getting to know the setup: waveform -(sine / square / sawtooth), frequency, and DAC modulation depth, with the -resulting optical amplitude always **measured** from the photodiode as -`a = ln(V_max/V_min)` — the Pockels V→T response is non-linear, so the DAC -excursion never doubles as a light level. - -## Included - -- `plugins/stage-a-funcgen` crate (`augur-plugin-stage-a-funcgen`): connect / - apply / stop actions, live photodiode waveform view, status table with the - measured contrast, clipping, and stream integrity; -- **`mock` port** (default): the waveform-extended mock controller runs on an - in-process thread and streams a synthetic photodiode response through a - Pockels-like sin² transfer — the complete control loop with zero hardware; -- feature detection against real firmware: 0.2.0 rejects the reserved drive - fields with `unknown_config_field`, which the plugin reports as "no - waveform backend" instead of a fault; -- same fail-closed safety model as `stage-a-monitor` (`LiveCapture` + - `effects_allowed` only; drive parameters are settings, applying them is an - explicit action; local bounds check before any command is sent). - -## Firmware-faithful mock (stage-a-io) - -Delivered together with this plugin, `stage-a-io`'s `MockController` now -mirrors firmware 0.2.0 exactly — verbs, state machine (`SAFE_IDLE` → -`CONFIGURED` → `RUNNING`), error codes/details, single-entry idempotent reply -cache, and unknown-CONFIG-field rejection. The previous mock accepted verbs -and fields the device does not speak (`ARM`, `RUN`, `capabilities=`, -`BAD_*`), which let host bugs pass tests: the A1 sweep reconfigured while -RUNNING (now fixed with STOP-before-CONFIG) and watchdog `!FAULT` notices -were invisible outside an in-flight request (now surfaced as async events by -`StageAClient` and handled by all three plugins). - -## Verification - -`cargo test -p stage-a-io -p augur-plugin-stage-a-funcgen --p augur-plugin-stage-a-monitor -p augur-plugin-stage-a-a1`: mock -state-machine/error fidelity against `main.cpp`, v1 rejection of waveform -fields, drive-bounds validation, nonlinear sin² contrast response, watchdog -fault propagation, and the full mock round trip (connect → apply sine / -square / saw → measured `a` → stop → reconfigure while driving). - -## Known gaps - -- Real firmware cannot emit a waveform yet; the `waveform-drive.md` fields - stay host+mock-only until the hardware freeze resolves the DAC channel and - safe HVA window. -- No PDQ/sidecar recording in this plugin — it is a familiarisation tool; - evidence-grade recording stays with `stage-a-monitor`/`stage-a-a1`. diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md new file mode 100644 index 0000000..c495d9a --- /dev/null +++ b/docs/features/stage-a-modulation.md @@ -0,0 +1,37 @@ +# Stage-A Modulation + +- **Crate:** `plugins/stage-a-modulation` (`augur-plugin-stage-a-modulation`) +- **Firmware:** `stage-a-controller` 0.3.0+ (`MOD` capability), Teensy **command port** +- **Status:** Active (2026-07-15) — replaces `stage-a-funcgen` and the drive half of + `stage-a-monitor` + +## What it is + +The simplest possible laser-modulation control for the Stage-A bench: one power slider in DAC +codes (J23 output, `DAC1.4`), a mode select (`CONST`/`SINE`/`SQUARE`) with frequency +(0.01–2000 Hz) and a min threshold for the periodic modes, and a user-set **max limit** that caps +the slider so a device with a lower tolerated input voltage can never be overdriven from the UI. + +Every accepted setting change is transferred to the Teensy **immediately** as one `MOD` command — +no Apply button, no experiment state machine. The panel shows the modulation and live DAC code the +board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded values. + +## Contract + +- Owns the Teensy **command port** exclusively (one owner per port, ADR 006). The photodiode + stream port belongs to `stage-a-photodiode`. +- Uses `stage-a-io` (`StageAClient`, `IoWorker`, `Command`) for framing, idempotent retries, and + the bounded background I/O thread; `process_frame()` never blocks on serial. +- Fail-closed effects gate: the connection only exists while the host execution context allows + hardware effects. +- Firmware output is **set-and-hold** (`stage-a-controller` ADR 002): disconnecting does not stop + the modulation. The explicit **Output OFF** action sends `MOD wave=OFF`. +- Safety invariants enforced plugin-side: `level ≤ max_level`, `min_level ≤ level`; the firmware + waveform peaks at `level` by construction. +- `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. + +## Verification + +`cargo test -p augur-plugin-stage-a-modulation` — mock round trips: immediate transfer on slider +change, board-code echo, max-cap clamping (including schema regeneration), square drive with min +threshold, Output OFF. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md new file mode 100644 index 0000000..976b9a8 --- /dev/null +++ b/docs/features/stage-a-photodiode.md @@ -0,0 +1,34 @@ +# Stage-A Photodiode + +- **Crate:** `plugins/stage-a-photodiode` (`augur-plugin-stage-a-photodiode`) +- **Firmware:** `stage-a-controller` 0.3.0+ (`PDSTREAM`), Teensy **stream port** (second CDC port) +- **Status:** Active (2026-07-15) — replaces the readout half of `stage-a-monitor` + +## What it is + +A minimal live readout of the photodiode on **board SMA5 → Teensy pin 18 / A4**. The firmware +streams `PD code= n= t_ms=` lines at 50 Hz on its second USB serial port; a +background thread parses them into a bounded ring, and the plugin shows the newest value plus a +rolling chart (1–120 s window). + +Two modes: + +- **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). +- **EXCITATION** — the diode sits behind the PBS in the excitation path and measures the light + removed from the beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set + reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. + +## Contract + +- Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the + plugin is read-only by construction and needs no protocol library — it depends only on + `serialport` and parses one line format. +- Same fail-closed effects gate as the other stage-a plugins for consistent device handling. +- Garbage on the port (e.g. the binary command port picked by mistake) parses to nothing and is + bounded — it can neither grow memory nor produce fake values. +- `mock` port synthesizes a slow sine for hardware-free testing. + +## Verification + +`cargo test -p augur-plugin-stage-a-photodiode` — line parsing (including clamping and rejection), +excitation inversion against the reference, mock reader filling ring/series, ring bound. diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md index 98354e8..ab9167f 100644 --- a/docs/features/stage-a.md +++ b/docs/features/stage-a.md @@ -1,77 +1,31 @@ -# Stage-A calibration plugins (`stage-a-io`, `stage-a-monitor`, `stage-a-a1`) +# Stage-A Bench Stack -> Feature brief — first delivery of the Stage-A camera-calibration stack. -> Design source of truth: knowledge base -> `methodology/stage-a-control-software.md` and -> `methodology/camera-calibration.md` (A1 protocol). +- **Status:** Simplified two-plugin setup (2026-07-15, ADR 006) +- **Firmware:** `stage-a-controller` 0.3.0 (Teensy 4.1 on Hermit V2r1, `USB_DUAL_SERIAL`) -## Architecture +## Current shape -```text -AugurRs generic host (camera, RAW, EXT_TRIGGER delivery, execution context — ABI v5) - │ - ├── stage-a-monitor — commissioning: live photodiode view, manual control - ├── stage-a-funcgen — familiarisation: manual waveform drive (see stage-a-funcgen.md) - └── stage-a-a1 — A1 minimum-depth a_min(f) sweep - │ (exactly one armed plugin owns the device) - ▼ - stage-a-io (this repo, plain lib) ── USB serial ── Teensy stage-a-controller -``` +The Teensy enumerates as **two** USB serial ports, and each is owned by exactly one plugin: -AugurRs itself gains no Teensy or serial abstraction — device ownership -lives entirely in these removable plugins (ADR 005). +| Port | Content | Owner | +|---|---|---| +| command port (first) | v1 ASCII commands + PDA1 binary frames | [`stage-a-modulation`](./stage-a-modulation.md) | +| stream port (second) | free-running `PD code=… n=… t_ms=…` lines, 50 Hz | [`stage-a-photodiode`](./stage-a-photodiode.md) | -## Crates +- **`stage-a-modulation`** — capped power slider + constant/sine/square drive of the laser + modulation input (J23), transferred to the Teensy immediately; shows the board-reported DAC + code. Firmware output is set-and-hold; "Output OFF" is the explicit stop. +- **`stage-a-photodiode`** — live readout of SMA5/pin 18/A4, raw or inverted to excitation power + `I_exc = I_tot − I_pd` against a user-set reference. +- **`stage-a-io`** (shared non-plugin library) — PDA1 wire format, typed client with idempotent + retries, bounded I/O worker, and a firmware-faithful mock (including the 0.3.0 `MOD` verb). + The estimator/pdq/sidecar modules are retained for the future A1–A3 experiment plugins. -| Crate | Role | -|---|---| -| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, firmware-faithful mock controller (0.2.0 surface + opt-in v2 waveform extension) | -| `plugins/stage-a-monitor` | Live decimated waveform, live `a`, integrity status, gated manual CONFIG/START/STOP + expert drive modal | -| `plugins/stage-a-funcgen` | Manual waveform drive (sine/square/saw, frequency, DAC depth) with photodiode-measured `a`; in-process mock port for hardware-free familiarisation | -| `plugins/stage-a-a1` | Phase-locked detection (Rayleigh), hardware/software cycle fiducials, bisection + grid sweep, probit `a_min` fit with CI, hot-pixel mask, PDQ + sidecar + results export | +## History -## Safety model - -- Serial ports open only when `HostContext::execution()` reports - `LiveCapture` **and** `effects_allowed` (host constructs this fail-closed; - only the active live-capture worker qualifies). Replay can never re-arm - hardware, even from a sidecar that contains a runnable setup. -- All hardware commands are host **actions**; persistent settings never - start hardware after a reload. -- Any CRC error, frame-sequence gap, or ADC overrun invalidates the - measurement point; invalid points are re-measured, never patched, and - the run sidecar records the counters. -- The firmware watchdog (1.5 s) drops the controller to `SAFE_IDLE` - independently of host-side cleanup. - -## Statistics (A1) - -Detection is a phase-uniformity test (background activity is uniform in -drive phase; signal is phase-locked), with the frequency-scan multiplicity -Bonferroni-charged when the software clock-skew lock substitutes for the -missing trigger cable. `a_min` is the fitted `N = 0.5` crossing of a -probit in `ln a` with a profile CI — not a raw bisection endpoint — and -`a` is always the photodiode-measured contrast. Details and rationale: -`plugins/stage-a-a1/README.md`. - -## Verification - -`cargo test` (38 tests): wire fragmentation/CRC-resync/overrun, retry -idempotency against the mock controller, worker round-trip + clean STOP, -estimator recovery/clipping/headroom guards, Rayleigh calibration on -uniform and locked phases, background-immunity, clock-skew recovery -(300 ppm), fiducial folding, probit fit recovery, sweep convergence to a -synthetic `a_min`, exhaustion/invalid-window handling, hot-pixel masking, -dataset/schema consistency. - -## Known gaps - -- Final Teensy DDS/DAC firmware is blocked on the hardware freeze; the - sweep and the function generator run against the reserved waveform-drive - protocol (`stage-a-controller/docs/features/waveform-drive.md`) and the - waveform-extended mock meanwhile — firmware 0.2.0 rejects the drive - fields with `unknown_config_field` (feature detection). -- Marker cycles are protocol-reserved but not yet emitted - (`stage-a-controller/docs/features/a1-marker-cycles.md`). -- `stage-a-a2` / `stage-a-a3` plugins are not yet implemented; A2 - additionally requires the physical trigger cable. +The earlier commissioning stack (`stage-a-monitor`, `stage-a-funcgen`, `stage-a-a1` — device +monitor with calibrated contrast, waveform familiarisation, and the A1 minimum-depth Bode sweep) +was removed on 2026-07-15 as too complex for the current bench stage (ADR 006). It remains in git +history; the experiment plugins will be rebuilt on the simplified stack when the bench needs +them. Device-ownership and safety rules: ADR 005 (one owner per port, fail-closed effects gate) +as amended by ADR 006. diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md deleted file mode 100644 index 48055cf..0000000 --- a/plugins/stage-a-a1/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Stage-A A1 — minimum-depth Bode calibration - -Measures `a_min(f)`: the smallest optical log-contrast that still produces -phase-locked camera events, per drive frequency. `|H(f)| = C / a_min(f)`; -the knee of the curve is the pixel bandwidth `f_c(I)`, and the plateau of -`a_min` reads out the contrast quantum `C` (which seeds A3). Protocol -design: knowledge base `methodology/camera-calibration.md` (A1) and -`methodology/stage-a-control-software.md`. - -## How it decides "events just appeared" - -- **Detector — phase, not counts.** Background activity is uniform in - drive phase; modulation events are phase-locked. Each measurement window - is folded and tested with the **Rayleigh test**; background is discounted - automatically instead of subtracting a drifting absolute rate. -- **Cycle fiducial.** With the phase-0 TTL wired into `EXT_TRIGGER`, the - camera-clock edges from `frame.external_triggers()` mark each cycle. - Without the cable, the drive frequency is **refined against the events** - (Rayleigh-power scan over ±ppm around the commanded value — recovers the - Teensy↔camera clock skew); the scan multiplicity is Bonferroni-charged - to the significance threshold. -- **Estimator.** The mean phase-locked events per half-cycle comes from the - positive excess over the median phase-bin occupancy. -- **a_min is a fitted crossing.** The 0→1 step is smeared by shot-noise - first-passage randomness and per-pixel threshold dispersion, so a_min is - the fitted `N = 0.5` crossing of a probit in `ln a`, with a profile - confidence interval. The fitted transition width is a free preview of - the smear (σ_C + FPT). -- **Hot pixels.** An unmodulated reference window at run start builds a - median+5·MAD mask; masked pixels never enter the statistics, and the - mask size is recorded in the sidecar. -- **`a` is measured light.** Every point's contrast comes from the - photodiode ADC through the calibrated, clipping-guarded estimator in - `stage-a-io` — never from the commanded DAC code. Invalid windows - (clipping, CRC/sequence/overrun faults) are re-measured, never patched. - -## Run flow - -`Arm controller` → `Run A1 sweep`: reference window (hot-pixel mask) → -per frequency: bisection on the drive code until the detection boundary is -bracketed → log-spaced grid across the transition → probit fit → -next frequency. Views: `a_min(f)` with CI, live phase histogram, `N(a)` -staircase, run status. Raw PDA1 frames go to -`~/.augur/stage-a-runs/.pdq` with a JSON sidecar and a results -export; final numbers must be recomputed from the camera RAW + PDQ. - -ON and OFF are measured in **separate runs** (settings → Polarity) — the -comparator paths are asymmetric and must never be pooled. - -## Safety - -Fails closed on the ABI v5 execution context exactly like -`stage-a-monitor`: serial I/O only in the active live-capture worker; -Arm/Run/Stop are host actions, never settings; the firmware watchdog -drops to `SAFE_IDLE` independently of host cleanup. - -## Current limitations - -- The Teensy DDS/DAC firmware is still the ADC-only commissioning build — - closed-loop sweeps run against the protocol but the final stimulus - backend is blocked on the hardware freeze (see `stage-a-controller`). -- Marker cycles (periodic full-depth optical anchors) are specced for the - firmware but not yet emitted; the software frequency lock covers the - missing-trigger-cable case meanwhile. -- Measured sample cadence validation and the A5 refractory validity bound - `2fa/C ≪ 1/τ_refr` are recorded, not yet enforced. diff --git a/plugins/stage-a-a1/plugin.toml b/plugins/stage-a-a1/plugin.toml deleted file mode 100644 index 62f6089..0000000 --- a/plugins/stage-a-a1/plugin.toml +++ /dev/null @@ -1,7 +0,0 @@ -name = "Stage-A A1 Min-Depth" -version = "0.2.0" -description = "Event-native Bode calibration: a_min(f) via phase-locked detection, drive bisection, and probit fitting." -domain = "stage-a" -library = "augur_plugin_stage_a_a1" -phase = "raw_events" -min_augur_version = "1.0.0" diff --git a/plugins/stage-a-a1/src/analysis.rs b/plugins/stage-a-a1/src/analysis.rs deleted file mode 100644 index 0c3ce31..0000000 --- a/plugins/stage-a-a1/src/analysis.rs +++ /dev/null @@ -1,610 +0,0 @@ -//! Statistical core of the A1 minimum-depth measurement. -//! -//! ## Why phase, not raw counts -//! -//! Background activity (BA) is uniform in modulation phase; genuine -//! modulation events are phase-locked to the drive. Testing for a -//! phase-locked component (Rayleigh test) therefore discounts uniform -//! background *automatically*, instead of requiring an absolute background -//! rate that drifts with temperature. "Mean events per half-cycle > 1" is -//! kept as the *estimator* (it is the quantity `⌊a·|H|/C⌋` predicts), but -//! the *detector* is the phase test. -//! -//! ## Cycle fiducials without the trigger cable -//! -//! With the phase-0 TTL wired, `frame.external_triggers()` marks each cycle -//! on the camera clock. Without it, the Teensy and camera clocks drift -//! (tens of ppm — folding dies after ~0.1 s at 10 kHz), so the drive -//! frequency is *refined against the events themselves*: scan a small -//! window around the commanded frequency and keep the value maximising the -//! Rayleigh power. The scan multiplicity is charged to the significance -//! test (Bonferroni). -//! -//! ## a_min as a fitted crossing -//! -//! Near threshold the 0→1 step of `⌊a·|H|/C⌋` is smeared by shot-noise -//! first-passage randomness and per-pixel threshold dispersion, so "events -//! just vanish" is not a crisp edge. a_min is defined as the fitted point -//! where the mean phase-locked events per half-cycle crosses 0.5, from a -//! probit-in-ln(a) fit over the transition, with a profile confidence -//! interval. The plateau of a_min(f) reads out the contrast quantum C. - -// --------------------------------------------------------------------------- -// Phase folding -// --------------------------------------------------------------------------- - -/// Folds event timestamps at `frequency_hz` relative to `t0_us`, -/// returning phases in `[0, 1)`. -pub fn fold_phases( - timestamps_us: impl Iterator, - t0_us: u64, - frequency_hz: f64, -) -> Vec { - let period_us = 1.0e6 / frequency_hz; - timestamps_us - .map(|t| { - let dt = t.saturating_sub(t0_us) as f64; - (dt / period_us).fract() - }) - .collect() -} - -/// Folds against explicit cycle-start fiducials (rising trigger edges): -/// each event's phase is its position inside the enclosing cycle. Events -/// before the first or after the last fiducial are dropped (their cycle -/// length is unknown). -pub fn fold_phases_with_fiducials(events: &[u64], cycle_starts_us: &[u64]) -> Vec { - if cycle_starts_us.len() < 2 { - return Vec::new(); - } - let mut phases = Vec::with_capacity(events.len()); - for &t in events { - let idx = match cycle_starts_us.binary_search(&t) { - Ok(i) => i, - Err(0) => continue, - Err(i) => i - 1, - }; - if idx + 1 >= cycle_starts_us.len() { - continue; - } - let start = cycle_starts_us[idx]; - let end = cycle_starts_us[idx + 1]; - if end <= start { - continue; - } - phases.push((t - start) as f64 / (end - start) as f64); - } - phases -} - -// --------------------------------------------------------------------------- -// Rayleigh test -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct RayleighResult { - pub n: usize, - /// Resultant length in [0, 1]. - pub r: f64, - /// Z = n·R². - pub z: f64, - /// Approximate p-value under uniformity, `exp(-Z)` with the standard - /// small-sample correction (Zar / Wilkie). - pub p_value: f64, -} - -pub fn rayleigh_test(phases: &[f64]) -> RayleighResult { - let n = phases.len(); - if n == 0 { - return RayleighResult { - n, - r: 0.0, - z: 0.0, - p_value: 1.0, - }; - } - let (mut c, mut s) = (0.0_f64, 0.0_f64); - for &phase in phases { - let angle = 2.0 * std::f64::consts::PI * phase; - c += angle.cos(); - s += angle.sin(); - } - let r = (c * c + s * s).sqrt() / n as f64; - let z = n as f64 * r * r; - let nf = n as f64; - let p = (-z).exp() - * (1.0 + (2.0 * z - z * z) / (4.0 * nf) - - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); - RayleighResult { - n, - r, - z, - p_value: p.clamp(0.0, 1.0), - } -} - -// --------------------------------------------------------------------------- -// Frequency refinement (clock-skew recovery without a trigger cable) -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct FrequencyLock { - pub frequency_hz: f64, - pub rayleigh: RayleighResult, - /// Number of candidate frequencies tested — multiply into the - /// significance threshold (Bonferroni). - pub trials: usize, -} - -/// Scans `±window_ppm` around `nominal_hz` and returns the frequency with -/// the maximum Rayleigh power. The step is chosen so consecutive candidates -/// dephase by ≤ 0.1 cycle over the observation span (finer is wasted). -pub fn refine_frequency( - timestamps_us: &[u64], - nominal_hz: f64, - window_ppm: f64, -) -> Option { - let (&first, &last) = (timestamps_us.first()?, timestamps_us.last()?); - let span_s = (last.saturating_sub(first)) as f64 / 1.0e6; - if span_s <= 0.0 { - return None; - } - let df_step = 0.1 / span_s; - let half_window_hz = nominal_hz * window_ppm * 1e-6; - let steps = ((half_window_hz / df_step).ceil() as i64).clamp(0, 5_000); - let mut best: Option = None; - let trials = (2 * steps + 1) as usize; - for k in -steps..=steps { - let f = nominal_hz + k as f64 * df_step; - if f <= 0.0 { - continue; - } - let phases = fold_phases(timestamps_us.iter().copied(), first, f); - let stat = rayleigh_test(&phases); - if best.as_ref().is_none_or(|b| stat.z > b.rayleigh.z) { - best = Some(FrequencyLock { - frequency_hz: f, - rayleigh: stat, - trials, - }); - } - } - best -} - -// --------------------------------------------------------------------------- -// Phase-locked excess (the events/half-cycle estimator) -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, PartialEq)] -pub struct PhaseHistogram { - pub bins: Vec, - pub total: usize, -} - -pub fn phase_histogram(phases: &[f64], bin_count: usize) -> PhaseHistogram { - let mut bins = vec![0_u32; bin_count.max(1)]; - for &phase in phases { - let idx = ((phase * bins.len() as f64) as usize).min(bins.len() - 1); - bins[idx] += 1; - } - PhaseHistogram { - bins, - total: phases.len(), - } -} - -/// Estimates the phase-locked event count above the uniform background. -/// -/// The per-bin background is the *median* bin occupancy — robust because -/// the locked cluster occupies a minority of bins. Returns the summed -/// positive excess. Dividing by the number of observed cycles gives the -/// mean phase-locked events per cycle (per polarity: one burst per cycle). -pub fn phase_locked_excess(histogram: &PhaseHistogram) -> f64 { - if histogram.bins.is_empty() { - return 0.0; - } - let mut sorted = histogram.bins.clone(); - sorted.sort_unstable(); - let median = f64::from(sorted[sorted.len() / 2]); - histogram - .bins - .iter() - .map(|&count| (f64::from(count) - median).max(0.0)) - .sum() -} - -// --------------------------------------------------------------------------- -// Detection verdict for one (frequency, amplitude) measurement -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct DetectionVerdict { - pub detected: bool, - pub p_value: f64, - /// Bonferroni-corrected significance threshold actually applied. - pub alpha_effective: f64, - /// Mean phase-locked events per cycle (per polarity), background-free. - pub locked_events_per_cycle: f64, -} - -/// Decides whether phase-locked modulation events are present. -/// -/// `alpha` is the per-measurement false-positive budget; `trials` is the -/// look-elsewhere multiplicity (frequency-scan candidates × bisection -/// steps), charged via Bonferroni. -pub fn detect( - rayleigh: RayleighResult, - excess: f64, - observed_cycles: f64, - alpha: f64, - trials: usize, -) -> DetectionVerdict { - let alpha_effective = alpha / trials.max(1) as f64; - DetectionVerdict { - detected: rayleigh.p_value < alpha_effective, - p_value: rayleigh.p_value, - alpha_effective, - locked_events_per_cycle: if observed_cycles > 0.0 { - excess / observed_cycles - } else { - 0.0 - }, - } -} - -// --------------------------------------------------------------------------- -// a_min fit: probit in ln(a) with profile CI -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct MinDepthFit { - /// a at which the mean locked events/half-cycle crosses 0.5. - pub a_min: f64, - /// Profile interval (Δ SSE ≤ SSE_min · (1 + 2/dof)); honest-but-cheap. - pub a_min_low: f64, - pub a_min_high: f64, - /// Transition width in ln(a) — first look at σ_C + FPT smear. - pub sigma_ln_a: f64, - pub points_used: usize, -} - -/// One measured amplitude point for the fit. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct DepthPoint { - /// Measured optical log-contrast (photodiode, never the drive code). - pub a: f64, - /// Mean phase-locked events per half-cycle at this contrast. - pub events_per_half_cycle: f64, -} - -fn standard_normal_cdf(z: f64) -> f64 { - // Abramowitz & Stegun 7.1.26 via erf; |error| < 1.5e-7. - let x = z / std::f64::consts::SQRT_2; - let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); - let poly = t - * (0.254_829_592 - + t * (-0.284_496_736 - + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); - let erf_abs = 1.0 - poly * (-x * x).exp(); - let erf = if x >= 0.0 { erf_abs } else { -erf_abs }; - 0.5 * (1.0 + erf) -} - -/// Fits `N(a) = Φ((ln a − μ)/σ)` over the transition region and reports -/// `a_min = e^μ` (the N = 0.5 crossing). Points far above the first step -/// (`N > 1.5`) are excluded — there the staircase's higher steps dominate -/// and the single-step model no longer applies. -pub fn fit_min_depth(points: &[DepthPoint]) -> Option { - let usable: Vec = points - .iter() - .copied() - .filter(|p| { - p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5 - }) - .collect(); - if usable.len() < 3 { - return None; - } - let has_low = usable.iter().any(|p| p.events_per_half_cycle < 0.4); - let has_high = usable.iter().any(|p| p.events_per_half_cycle > 0.6); - if !has_low || !has_high { - return None; - } - - let ln_min = usable - .iter() - .map(|p| p.a.ln()) - .fold(f64::INFINITY, f64::min); - let ln_max = usable - .iter() - .map(|p| p.a.ln()) - .fold(f64::NEG_INFINITY, f64::max); - - let sse = |mu: f64, sigma: f64| -> f64 { - usable - .iter() - .map(|p| { - let model = standard_normal_cdf((p.a.ln() - mu) / sigma); - let d = p.events_per_half_cycle.min(1.0) - model; - d * d - }) - .sum() - }; - - let mut best = (f64::INFINITY, ln_min, 0.1); - let mu_steps = 200; - for i in 0..=mu_steps { - let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; - for j in 0..40 { - let sigma = 0.005 * 1.2_f64.powi(j); // 0.005 .. ~7 in ln a - let value = sse(mu, sigma); - if value < best.0 { - best = (value, mu, sigma); - } - } - } - let (sse_min, mu_hat, sigma_hat) = best; - let dof = usable.len().saturating_sub(2).max(1) as f64; - let threshold = sse_min * (1.0 + 2.0 / dof) + 1e-12; - - // Profile over mu: the interval where some sigma keeps SSE under the - // threshold. - let mut low = mu_hat; - let mut high = mu_hat; - for i in 0..=mu_steps { - let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; - let feasible = (0..40).any(|j| { - let sigma = 0.005 * 1.2_f64.powi(j); - sse(mu, sigma) <= threshold - }); - if feasible { - low = low.min(mu); - high = high.max(mu); - } - } - - Some(MinDepthFit { - a_min: mu_hat.exp(), - a_min_low: low.exp(), - a_min_high: high.exp(), - sigma_ln_a: sigma_hat, - points_used: usable.len(), - }) -} - -// --------------------------------------------------------------------------- -// Hot-pixel mask (background is heavy-tailed; mask the tail, use the body) -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone)] -pub struct HotPixelMask { - width: u16, - masked: Vec, -} - -impl HotPixelMask { - /// Builds the mask from per-pixel counts of an *unmodulated* reference - /// window: pixels above `median + 5·MAD` (and above a small absolute - /// floor) are masked. The mask is fixed-pattern and belongs in the run - /// metadata, not just preprocessing. - pub fn from_reference_counts(width: u16, _height: u16, counts: &[u32]) -> Self { - let mut sorted: Vec = counts.to_vec(); - sorted.sort_unstable(); - let median = sorted.get(sorted.len() / 2).copied().unwrap_or(0) as f64; - let mut deviations: Vec = counts - .iter() - .map(|&count| (f64::from(count) - median).abs()) - .collect(); - deviations.sort_by(f64::total_cmp); - let mad = deviations.get(deviations.len() / 2).copied().unwrap_or(0.0); - let threshold = median + 5.0 * mad.max(0.5) + 2.0; - let masked = counts - .iter() - .map(|&count| f64::from(count) > threshold) - .collect(); - Self { width, masked } - } - - pub fn is_masked(&self, x: u16, y: u16) -> bool { - self.masked - .get(y as usize * self.width as usize + x as usize) - .copied() - .unwrap_or(false) - } - - pub fn masked_count(&self) -> usize { - self.masked.iter().filter(|&&m| m).count() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Deterministic pseudo-uniform stream (splitmix64 → [0,1)). - struct UniformStream { - state: u64, - } - - impl UniformStream { - fn new(seed: u64) -> Self { - Self { state: seed } - } - - fn next(&mut self) -> f64 { - self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.state; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z = z ^ (z >> 31); - (z >> 11) as f64 / (1_u64 << 53) as f64 - } - - fn take(&mut self, n: usize) -> Vec { - (0..n).map(|_| self.next()).collect() - } - } - - fn uniform_sequence(seed: u64, n: usize) -> Vec { - UniformStream::new(seed).take(n) - } - - /// Synthetic event stream: `per_cycle` phase-locked events per cycle at - /// `locked_phase` (jitter ±0.02) plus `background_rate_hz` uniform noise. - fn synthetic_events( - frequency_hz: f64, - duration_s: f64, - per_cycle: f64, - background_rate_hz: f64, - seed: u64, - ) -> Vec { - let cycles = (frequency_hz * duration_s) as usize; - let period_us = 1.0e6 / frequency_hz; - let mut stream = UniformStream::new(seed); - let mut next = move || stream.next(); - let mut events = Vec::new(); - for cycle in 0..cycles { - let base = cycle as f64 * period_us; - // Bernoulli(per_cycle fractional part) + floor. - let mut count = per_cycle.floor() as usize; - if next() < per_cycle.fract() { - count += 1; - } - for _ in 0..count { - let phase = 0.25 + (next() - 0.5) * 0.04; - events.push((base + phase * period_us) as u64); - } - } - let n_background = (background_rate_hz * duration_s) as usize; - for _ in 0..n_background { - events.push((next() * duration_s * 1.0e6) as u64); - } - events.sort_unstable(); - events - } - - #[test] - fn rayleigh_accepts_uniform_and_rejects_locked_phases() { - let uniform = uniform_sequence(7, 2_000); - let stat = rayleigh_test(&uniform); - assert!(stat.p_value > 0.01, "uniform phases: p={}", stat.p_value); - - let locked: Vec = uniform_sequence(11, 200) - .into_iter() - .map(|u| 0.3 + 0.02 * (u - 0.5)) - .collect(); - let stat = rayleigh_test(&locked); - assert!(stat.p_value < 1e-12, "locked phases: p={}", stat.p_value); - } - - #[test] - fn detection_discounts_uniform_background() { - // 0.8 locked events/cycle at 1 kHz for 0.5 s, drowned in 10x - // background rate: still detected via phase. - let events = synthetic_events(1_000.0, 0.5, 0.8, 8_000.0, 3); - let phases = fold_phases(events.iter().copied(), 0, 1_000.0); - let stat = rayleigh_test(&phases); - assert!(stat.p_value < 1e-6, "p={}", stat.p_value); - - // Background alone must NOT detect. - let noise_only = synthetic_events(1_000.0, 0.5, 0.0, 8_000.0, 5); - let phases = fold_phases(noise_only.iter().copied(), 0, 1_000.0); - let stat = rayleigh_test(&phases); - assert!(stat.p_value > 1e-3, "background-only p={}", stat.p_value); - } - - #[test] - fn phase_locked_excess_recovers_events_per_cycle() { - let frequency = 2_000.0; - let duration = 0.5; - let per_cycle = 0.6; - let events = synthetic_events(frequency, duration, per_cycle, 2_000.0, 9); - let phases = fold_phases(events.iter().copied(), 0, frequency); - let histogram = phase_histogram(&phases, 32); - let cycles = frequency * duration; - let recovered = phase_locked_excess(&histogram) / cycles; - assert!( - (recovered - per_cycle).abs() < 0.12, - "recovered {recovered} vs {per_cycle}" - ); - } - - #[test] - fn frequency_refinement_recovers_clock_skew() { - // Commanded 5 kHz, true (camera-clock) frequency 300 ppm higher — - // the naive fold dephases by 1.5 cycles over the 1 s span and - // collapses, while the refined lock recovers the true frequency. - let true_hz = 5_000.0 * (1.0 + 300e-6); - let events = synthetic_events(true_hz, 1.0, 1.0, 500.0, 13); - let lock = refine_frequency(&events, 5_000.0, 500.0).expect("lock found"); - let recovered_ppm = (lock.frequency_hz / 5_000.0 - 1.0) * 1e6; - // The scan step is 0.1/span = 0.1 Hz = 20 ppm at 5 kHz. - assert!( - (recovered_ppm - 300.0).abs() < 25.0, - "recovered {recovered_ppm} ppm" - ); - let naive = rayleigh_test(&fold_phases(events.iter().copied(), events[0], 5_000.0)); - assert!( - lock.rayleigh.z > naive.z * 5.0, - "lock z={} naive z={}", - lock.rayleigh.z, - naive.z - ); - } - - #[test] - fn fiducial_folding_matches_known_phase() { - let cycle_starts: Vec = (0..100).map(|k| k * 1_000).collect(); - let events: Vec = (0..99).map(|k| k * 1_000 + 250).collect(); - let phases = fold_phases_with_fiducials(&events, &cycle_starts); - assert_eq!(phases.len(), 99); - assert!(phases.iter().all(|p| (p - 0.25).abs() < 1e-9)); - } - - #[test] - fn min_depth_fit_recovers_the_crossing() { - // True a_min = 0.20, smear sigma = 0.15 in ln a. - let mu = 0.2_f64.ln(); - let points: Vec = (0..12) - .map(|i| { - let a = 0.08 * 1.25_f64.powi(i); // 0.08 .. ~0.9 - DepthPoint { - a, - events_per_half_cycle: standard_normal_cdf((a.ln() - mu) / 0.15), - } - }) - .collect(); - let fit = fit_min_depth(&points).expect("fit succeeds"); - assert!( - (fit.a_min - 0.2).abs() < 0.02, - "a_min={} (expected 0.20)", - fit.a_min - ); - assert!(fit.a_min_low <= fit.a_min && fit.a_min <= fit.a_min_high); - assert!((fit.sigma_ln_a - 0.15).abs() < 0.08); - } - - #[test] - fn min_depth_fit_requires_a_bracketed_transition() { - // All points fully above threshold: no crossing to fit. - let points: Vec = (0..6) - .map(|i| DepthPoint { - a: 0.5 + 0.1 * i as f64, - events_per_half_cycle: 1.0, - }) - .collect(); - assert!(fit_min_depth(&points).is_none()); - } - - #[test] - fn hot_pixel_mask_flags_the_tail_only() { - let mut counts = vec![2_u32; 64 * 64]; - counts[5] = 500; // hot - counts[700] = 300; // hot - let mask = HotPixelMask::from_reference_counts(64, 64, &counts); - assert_eq!(mask.masked_count(), 2); - assert!(mask.is_masked(5, 0)); - assert!(!mask.is_masked(6, 0)); - } -} diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs deleted file mode 100644 index 808be04..0000000 --- a/plugins/stage-a-a1/src/lib.rs +++ /dev/null @@ -1,1254 +0,0 @@ -//! Stage-A A1 — event-native Bode calibration, minimum-depth method. -//! -//! Measures `a_min(f)`: the smallest optical log-contrast that still -//! produces phase-locked events, per drive frequency. `|H(f)| = -//! C/a_min(f)`, the knee is `f_c(I)`, and the plateau of `a_min` reads out -//! the contrast quantum `C` (knowledge base: -//! `methodology/camera-calibration.md`, A1 protocol). -//! -//! Division of labour: -//! - `analysis` — phase folding, Rayleigh detection, frequency-skew -//! recovery, phase-locked excess, probit `a_min` fit, hot-pixel mask; -//! - `sweep` — the per-frequency bisection/grid state machine; -//! - this module — device I/O through `stage-a-io` (gated by the ABI v5 -//! execution context), camera-event intake, measurement windows, live -//! views, and the run sidecar. -//! -//! ON and OFF are measured **separately** (never pooled — the paths are -//! asymmetric); select the polarity in the settings and run each sweep. - -mod analysis; -mod sweep; - -use std::collections::BTreeMap; -use std::path::PathBuf; - -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, PluginInput, - Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, - TableSchema, TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, -}; -use serde_json::{json, Value}; -use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, DeviceEvent, FrameType, IoWorker, PdqWriter, - RunSidecar, StageAClient, StreamIntegrity, TriggerSource, WorkerOutput, WorkerRequest, -}; - -use analysis::{ - detect, fold_phases, fold_phases_with_fiducials, phase_histogram, phase_locked_excess, - rayleigh_test, refine_frequency, HotPixelMask, PhaseHistogram, -}; -use sweep::{Measurement, SweepCommand, SweepEngine, SweepPlan}; - -const AMIN_DATASET_ID: &str = "stage-a-a1.amin"; -const PHASE_DATASET_ID: &str = "stage-a-a1.phase"; -const DEPTH_DATASET_ID: &str = "stage-a-a1.depth"; -const STATUS_DATASET_ID: &str = "stage-a-a1.status"; - -const ACTION_ARM: &str = "stage-a-a1.arm"; -const ACTION_RUN: &str = "stage-a-a1.run"; -const ACTION_STOP: &str = "stage-a-a1.stop"; - -const PHASE_BINS: usize = 32; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RunState { - Idle, - Armed, - Reference, - Sweeping, - Finished, -} - -/// Camera-side accumulation for the current measurement window. -#[derive(Default)] -struct WindowAccumulator { - /// Camera timestamps of polarity-selected, hot-pixel-filtered events. - event_timestamps_us: Vec, - /// Rising phase-0 trigger edges (cycle fiducials) inside the window. - trigger_edges_us: Vec, - /// ADC codes streamed by the Teensy during the window. - adc_codes: Vec, - window_start_us: Option, - latest_camera_ts_us: u64, -} - -impl WindowAccumulator { - fn clear(&mut self) { - self.event_timestamps_us.clear(); - self.trigger_edges_us.clear(); - self.adc_codes.clear(); - self.window_start_us = None; - } - - fn elapsed_us(&self) -> u64 { - self.window_start_us - .map(|start| self.latest_camera_ts_us.saturating_sub(start)) - .unwrap_or(0) - } -} - -pub struct StageAA1Plugin { - enabled: bool, - state: RunState, - // device - worker: Option, - next_tag: u64, - in_flight: BTreeMap, - firmware: String, - integrity: StreamIntegrity, - last_error: Option, - effects_blocked_reason: Option, - // configuration (settings) - port_hint: String, - polarity_on: bool, - freq_start_hz: f64, - freq_stop_hz: f64, - points_per_decade: i64, - cycles_per_measurement: i64, - settle_ms: i64, - alpha: f64, - initial_amplitude_dac: i64, - sample_rate_hz: i64, - calibration: AdcCalibration, - // run - engine: Option, - window: WindowAccumulator, - settle_until_us: Option, - hot_pixels: Option, - reference_counts: Vec, - sensor_size: (u16, u16), - run_id: String, - /// CONFIG reply fields exactly as the controller ACKed them (sidecar). - last_acked_config: BTreeMap, - pdq: Option, - current_phase_histogram: Option, - used_hardware_fiducial: bool, - dataset_generation: u64, - consumed_action_ids: Vec, -} - -impl Default for StageAA1Plugin { - fn default() -> Self { - Self { - enabled: false, - state: RunState::Idle, - worker: None, - next_tag: 1, - in_flight: BTreeMap::new(), - firmware: String::new(), - integrity: StreamIntegrity::default(), - last_error: None, - effects_blocked_reason: None, - port_hint: "auto".into(), - polarity_on: true, - freq_start_hz: 100.0, - freq_stop_hz: 50_000.0, - points_per_decade: 6, - cycles_per_measurement: 400, - settle_ms: 100, - alpha: 0.001, - initial_amplitude_dac: 512, - sample_rate_hz: 20_000, - calibration: AdcCalibration::default(), - engine: None, - window: WindowAccumulator::default(), - settle_until_us: None, - hot_pixels: None, - reference_counts: Vec::new(), - sensor_size: (0, 0), - run_id: String::new(), - last_acked_config: BTreeMap::new(), - pdq: None, - current_phase_histogram: None, - used_hardware_fiducial: false, - dataset_generation: 0, - consumed_action_ids: Vec::new(), - } - } -} - -impl StageAA1Plugin { - fn bump(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn frequency_grid(&self) -> Vec { - let start = self.freq_start_hz.max(1.0); - let stop = self.freq_stop_hz.max(start * 1.01); - let per_decade = self.points_per_decade.max(1) as f64; - let decades = (stop / start).log10(); - let n = (decades * per_decade).ceil() as usize + 1; - (0..n) - .map(|i| start * 10f64.powf(i as f64 / per_decade)) - .filter(|&f| f <= stop * 1.0001) - .collect() - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - - fn arm(&mut self) { - if self.worker.is_some() { - return; - } - match open_transport(&self.port_hint) { - Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - self.state = RunState::Armed; - self.last_error = None; - } - Err(err) => self.last_error = Some(err), - } - self.bump(); - } - - fn start_run(&mut self) { - if self.worker.is_none() { - self.last_error = Some("run: arm the controller first".into()); - return; - } - self.run_id = format!( - "A1-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) - ); - let pdq_path = run_data_dir().join(format!("{}.pdq", self.run_id)); - match PdqWriter::create(&pdq_path) { - Ok(writer) => self.pdq = Some(writer), - Err(err) => { - self.last_error = Some(format!("pdq: {err}")); - return; - } - } - let plan = SweepPlan { - frequencies_hz: self.frequency_grid(), - initial_amplitude_dac: self.initial_amplitude_dac.clamp(1, 2_047) as u32, - ..SweepPlan::default() - }; - self.engine = Some(SweepEngine::new(plan)); - self.reference_counts.clear(); - self.hot_pixels = None; - self.window.clear(); - self.settle_until_us = None; - // Reference phase: unmodulated field (amplitude 0) for the - // hot-pixel mask and the background sanity check. - self.send_drive(self.freq_start_hz, 0, "reference"); - self.state = RunState::Reference; - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(true)); - } - self.bump(); - } - - fn stop_run(&mut self, reason: &str) { - self.queue_command("stop", Command::new("STOP").field("reason", reason)); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - self.finish_run(); - self.state = if self.worker.is_some() { - RunState::Armed - } else { - RunState::Idle - }; - self.bump(); - } - - fn disarm(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - worker.shutdown(reason); - } - self.finish_run(); - self.in_flight.clear(); - self.state = RunState::Idle; - self.bump(); - } - - fn finish_run(&mut self) { - if let Some(pdq) = self.pdq.take() { - match pdq.finish(self.integrity) { - Ok(summary) => { - let mut sidecar = RunSidecar::from_pdq(&self.run_id, "A1", &summary); - sidecar.plugin_name = "stage-a-a1".into(); - sidecar.plugin_version = env!("CARGO_PKG_VERSION").into(); - sidecar.firmware_version = self.firmware.clone(); - sidecar.adc_calibration = self.calibration.clone(); - sidecar.configured_sample_rate_hz = self.sample_rate_hz as u32; - sidecar.acked_config = self.last_acked_config.clone(); - sidecar.trigger_source = if self.used_hardware_fiducial { - TriggerSource::DrivePhase0 - } else { - TriggerSource::None - }; - sidecar.valid = summary.valid; - if let Some(mask) = &self.hot_pixels { - sidecar - .notes - .push(format!("hot pixels masked: {}", mask.masked_count())); - } - let sidecar_path = run_data_dir().join(format!("{}.json", self.run_id)); - if let Err(err) = sidecar.write_json(&sidecar_path) { - self.last_error = Some(format!("sidecar: {err}")); - } - if let Some(engine) = &self.engine { - let results_path = - run_data_dir().join(format!("{}.results.json", self.run_id)); - let _ = std::fs::write( - &results_path, - serde_json::to_vec_pretty(&results_json(engine)).unwrap_or_default(), - ); - } - } - Err(err) => self.last_error = Some(format!("pdq finish: {err}")), - } - } - } - - fn send_drive(&mut self, frequency_hz: f64, amplitude_dac: u32, purpose: &str) { - let freq_mhz = (frequency_hz * 1_000.0).round() as i64; - // The firmware only accepts CONFIG from SAFE_IDLE/CONFIGURED, so - // every new drive point must stop the running acquisition first - // (STOP is idempotent and harmless before the first point). - self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); - self.queue_command( - purpose, - Command::new("CONFIG") - .field("mode", "A1") - .field("wave", "SINE") - .field("freq_mhz", freq_mhz) - .field("center_dac", 2_048) - .field("amplitude_dac", amplitude_dac) - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", 256) - .field("raw", 1) - .field("summary", 1), - ); - self.queue_command("start", Command::new("START")); - self.window.clear(); - self.settle_until_us = None; // set on the first camera frame seen - self.current_phase_histogram = None; - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - let mut stopped = None; - let mut watchdog_fault: Option = None; - for output in outputs { - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => match purpose.as_str() { - "hello" => { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - } - // CONFIG ACKs (drive points) go into the sidecar - // verbatim, per the control-software spec. - "reference" | "sweep" => { - self.last_acked_config = fields; - } - _ => {} - }, - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - WorkerOutput::Event(DeviceEvent::Data(frame)) => { - if let Some(pdq) = &mut self.pdq { - let _ = pdq.write_frame(&frame); - } - if frame.header.frame_type == FrameType::SamplesU16 { - if let Some(codes) = frame.samples() { - self.window.adc_codes.extend_from_slice(&codes); - } - } - } - WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { - if name == "FAULT" { - watchdog_fault = Some( - fields - .get("code") - .cloned() - .unwrap_or_else(|| "unknown".into()), - ); - } - } - WorkerOutput::Integrity(integrity) => self.integrity = integrity, - WorkerOutput::Stopped { reason } => stopped = Some(reason), - } - } - if let Some(code) = watchdog_fault { - // The controller safed itself mid-run; the current point is - // invalid and the run cannot silently continue. - self.last_error = Some(format!("controller fault: {code} — run aborted")); - if matches!(self.state, RunState::Reference | RunState::Sweeping) { - self.stop_run("watchdog_fault"); - } - } - if let Some(reason) = stopped { - self.worker = None; - self.last_error = Some(format!("device connection ended: {reason}")); - self.finish_run(); - self.state = RunState::Idle; - self.bump(); - } - } - - fn ingest_camera_frame(&mut self, frame: &PluginFrame<'_>) { - self.sensor_size = (frame.width(), frame.height()); - self.window.latest_camera_ts_us = frame.window_end_us(); - if self.settle_until_us.is_none() { - self.settle_until_us = - Some(frame.window_end_us() + (self.settle_ms.max(0) as u64) * 1_000); - return; - } - let settle_until = self.settle_until_us.unwrap_or(0); - if frame.window_end_us() < settle_until { - return; - } - self.window - .window_start_us - .get_or_insert(frame.window_start_us()); - - if self.state == RunState::Reference { - if self.reference_counts.len() != frame.width() as usize * frame.height() as usize { - self.reference_counts = vec![0; frame.width() as usize * frame.height() as usize]; - } - for event in frame.events() { - let idx = event.y as usize * frame.width() as usize + event.x as usize; - if let Some(slot) = self.reference_counts.get_mut(idx) { - *slot += 1; - } - } - } else { - let mask = self.hot_pixels.as_ref(); - for event in frame.events() { - if event.is_on() != self.polarity_on { - continue; - } - if mask.is_some_and(|m| m.is_masked(event.x, event.y)) { - continue; - } - self.window.event_timestamps_us.push(event.timestamp_us()); - } - } - for trigger in frame.external_triggers() { - if trigger.is_rising() { - self.window.trigger_edges_us.push(trigger.timestamp_us); - } - } - } - - fn window_target_us(&self, frequency_hz: f64) -> u64 { - ((self.cycles_per_measurement.max(10) as f64 / frequency_hz) * 1.0e6) as u64 - } - - fn advance_run(&mut self) { - match self.state { - RunState::Reference => { - // A fixed 0.5 s of unmodulated reference. - if self.window.elapsed_us() < 500_000 { - return; - } - let (width, height) = self.sensor_size; - if width > 0 && !self.reference_counts.is_empty() { - self.hot_pixels = Some(HotPixelMask::from_reference_counts( - width, - height, - &self.reference_counts, - )); - } - self.state = RunState::Sweeping; - let Some(engine) = &self.engine else { - return; - }; - if let SweepCommand::Measure { - frequency_hz, - amplitude_dac, - } = engine.current_command() - { - self.send_drive(frequency_hz, amplitude_dac, "sweep"); - } - self.bump(); - } - RunState::Sweeping => { - let Some(engine) = &self.engine else { - return; - }; - let SweepCommand::Measure { - frequency_hz, - amplitude_dac, - } = engine.current_command() - else { - self.state = RunState::Finished; - self.finish_run(); - self.bump(); - return; - }; - if self.window.elapsed_us() < self.window_target_us(frequency_hz) { - return; - } - let measurement = self.evaluate_window(frequency_hz, amplitude_dac); - let next = { - let engine = self.engine.as_mut().expect("engine exists"); - engine.ingest(measurement) - }; - match next { - SweepCommand::Measure { - frequency_hz, - amplitude_dac, - } => self.send_drive(frequency_hz, amplitude_dac, "sweep"), - SweepCommand::Finished => { - self.queue_command("stop", Command::new("STOP").field("reason", "done")); - self.state = RunState::Finished; - self.finish_run(); - } - } - self.bump(); - } - _ => {} - } - } - - fn evaluate_window(&mut self, frequency_hz: f64, amplitude_dac: u32) -> Measurement { - // Optical contrast from the photodiode trace; any estimator - // rejection or stream fault invalidates the point. - let measured_a = if self.integrity.is_clean() { - estimate_contrast(&self.window.adc_codes, &self.calibration) - .ok() - .map(|estimate| estimate.a) - } else { - None - }; - - let events = &self.window.event_timestamps_us; - let observed_cycles = self.window.elapsed_us() as f64 / 1.0e6 * frequency_hz; - - // Cycle fiducial: hardware phase-0 edges when present, otherwise - // software frequency refinement against the events themselves. - let (phases, trials) = if self.window.trigger_edges_us.len() >= 2 { - self.used_hardware_fiducial = true; - ( - fold_phases_with_fiducials(events, &self.window.trigger_edges_us), - 1, - ) - } else if let Some(lock) = refine_frequency(events, frequency_hz, 100.0) { - ( - fold_phases( - events.iter().copied(), - events.first().copied().unwrap_or(0), - lock.frequency_hz, - ), - lock.trials, - ) - } else { - (Vec::new(), 1) - }; - - let stat = rayleigh_test(&phases); - let histogram = phase_histogram(&phases, PHASE_BINS); - let excess = phase_locked_excess(&histogram); - self.current_phase_histogram = Some(histogram); - let verdict = detect(stat, excess, observed_cycles, self.alpha, trials); - - Measurement { - amplitude_dac, - measured_a, - events_per_half_cycle: verdict.locked_events_per_cycle, - detected: verdict.detected, - } - } - - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) - || !request.action_id.starts_with("stage-a-a1.") - { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed - } - - // -- datasets -------------------------------------------------------- - - fn amin_dataset(&self) -> Series1dV1 { - let mut a_min = Vec::new(); - let mut low = Vec::new(); - let mut high = Vec::new(); - if let Some(engine) = &self.engine { - for result in &engine.results { - if let Some(fit) = &result.fit { - a_min.push(Series1dPoint { - x: result.frequency_hz, - y: fit.a_min, - }); - low.push(Series1dPoint { - x: result.frequency_hz, - y: fit.a_min_low, - }); - high.push(Series1dPoint { - x: result.frequency_hz, - y: fit.a_min_high, - }); - } - } - } - Series1dV1 { - x_label: "drive frequency [Hz]".into(), - y_label: "a_min".into(), - lines: vec![ - Series1dLine { - name: "a_min".into(), - points: a_min, - }, - Series1dLine { - name: "CI low".into(), - points: low, - }, - Series1dLine { - name: "CI high".into(), - points: high, - }, - ], - } - } - - fn phase_dataset(&self) -> Series1dV1 { - let points = self - .current_phase_histogram - .as_ref() - .map(|histogram| { - histogram - .bins - .iter() - .enumerate() - .map(|(i, &count)| Series1dPoint { - x: (i as f64 + 0.5) / histogram.bins.len() as f64, - y: f64::from(count), - }) - .collect() - }) - .unwrap_or_default(); - Series1dV1 { - x_label: "drive phase [cycles]".into(), - y_label: "events".into(), - lines: vec![Series1dLine { - name: if self.polarity_on { "ON" } else { "OFF" }.into(), - points, - }], - } - } - - fn depth_dataset(&self) -> Series1dV1 { - let mut points: Vec = self - .engine - .as_ref() - .map(|engine| { - let mut all: Vec = engine - .results - .last() - .map(|result| { - result - .points - .iter() - .map(|p| Series1dPoint { - x: p.a, - y: p.events_per_half_cycle, - }) - .collect() - }) - .unwrap_or_default(); - all.sort_by(|p, q| p.x.total_cmp(&q.x)); - all - }) - .unwrap_or_default(); - points.dedup_by(|p, q| p.x == q.x); - Series1dV1 { - x_label: "measured a".into(), - y_label: "locked events / half-cycle".into(), - lines: vec![Series1dLine { - name: "N(a)".into(), - points, - }], - } - } - - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("progress", "Progress"), - column("fiducial", "Cycle fiducial"), - column("hot_pixels", "Hot pixels"), - column("integrity", "Integrity"), - column("error", "Last error"), - ], - ..TableSchema::default() - } - } - - fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.state) { - (Some(reason), _) => format!("locked ({reason})"), - (None, RunState::Idle) => "idle".into(), - (None, RunState::Armed) => format!("armed ({})", self.firmware), - (None, RunState::Reference) => "reference window (hot-pixel mask)".into(), - (None, RunState::Sweeping) => "sweeping".into(), - (None, RunState::Finished) => "finished".into(), - }; - let progress = self - .engine - .as_ref() - .map(|engine| { - format!( - "{}/{} frequencies", - engine.results.len(), - engine.results.len() + if engine.is_finished() { 0 } else { 1 } - ) - }) - .unwrap_or_else(|| "—".into()); - let fiducial = if self.used_hardware_fiducial { - "EXT_TRIGGER phase-0".to_owned() - } else { - "software frequency lock".to_owned() - }; - let hot = self - .hot_pixels - .as_ref() - .map(|mask| format!("{} masked", mask.masked_count())) - .unwrap_or_else(|| "—".into()); - let integrity = if self.integrity.is_clean() { - "clean".to_owned() - } else { - format!( - "crc={} gaps={} overruns={}", - self.integrity.crc_failures, - self.integrity.sequence_gaps, - self.integrity.dropped_samples - ) - }; - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", state), - text_column("progress", progress), - text_column("fiducial", fiducial), - text_column("hot_pixels", hot), - text_column("integrity", integrity), - text_column("error", self.last_error.clone().unwrap_or_default()), - ], - } - } -} - -fn results_json(engine: &SweepEngine) -> Value { - json!({ - "results": engine - .results - .iter() - .map(|result| { - json!({ - "frequency_hz": result.frequency_hz, - "exhausted": result.exhausted, - "measurements": result.measurements, - "fit": result.fit.as_ref().map(|fit| json!({ - "a_min": fit.a_min, - "a_min_low": fit.a_min_low, - "a_min_high": fit.a_min_high, - "sigma_ln_a": fit.sigma_ln_a, - "points_used": fit.points_used, - })), - "points": result - .points - .iter() - .map(|p| json!({"a": p.a, "events_per_half_cycle": p.events_per_half_cycle})) - .collect::>(), - }) - }) - .collect::>(), - }) -} - -fn run_data_dir() -> PathBuf { - let home = std::env::var_os("HOME") - .map(PathBuf::from) - .unwrap_or_default(); - home.join(".augur").join("stage-a-runs") -} - -fn open_transport(port_hint: &str) -> Result, String> { - let path = if port_hint == "auto" { - stage_a_io::transport::available_port_names() - .into_iter() - .find(|name| name.contains("usbmodem") || name.contains("ttyACM")) - .ok_or_else(|| "no USB serial device found".to_owned())? - } else { - port_hint.to_owned() - }; - let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} - -impl Plugin for StageAA1Plugin { - fn name(&self) -> &'static str { - "Stage-A A1 Min-Depth" - } - - fn description(&self) -> &'static str { - "Event-native Bode calibration: a_min(f) via phase-locked detection, bisection, and probit fitting." - } - - fn enabled(&self) -> bool { - self.enabled - } - - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.disarm("plugin disabled"); - } - } - - fn reset(&mut self) { - self.window.clear(); - self.current_phase_histogram = None; - self.bump(); - } - - fn input_kind(&self) -> PluginInput { - PluginInput::RawEvents - } - - fn process_frame( - &mut self, - frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = - Some(format!("effects not allowed in {:?}", execution.mode)); - if self.worker.is_some() { - self.disarm("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for action in self.consume_actions(context) { - match action.as_str() { - ACTION_ARM => self.arm(), - ACTION_RUN => self.start_run(), - ACTION_STOP => self.stop_run("operator"), - _ => {} - } - } - - self.drain_worker(); - if matches!(self.state, RunState::Reference | RunState::Sweeping) { - self.ingest_camera_frame(frame); - self.advance_run(); - } - } - - fn settings_schema(&self) -> SettingsSchema { - SettingsSchema { - sections: vec![ - SettingsSection { - label: "Sweep".into(), - description: Some( - "Frequency grid and statistics. ON and OFF are measured in separate \ - runs — never pooled." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "freq_start_hz".into(), - label: "Start frequency".into(), - tooltip: None, - kind: SettingKind::F64Drag { - min: 1.0, - max: 1.0e6, - speed: 10.0, - default: self.freq_start_hz, - }, - }, - SettingItem { - key: "freq_stop_hz".into(), - label: "Stop frequency".into(), - tooltip: None, - kind: SettingKind::F64Drag { - min: 1.0, - max: 1.0e6, - speed: 100.0, - default: self.freq_stop_hz, - }, - }, - SettingItem { - key: "points_per_decade".into(), - label: "Points per decade".into(), - tooltip: None, - kind: SettingKind::I64Slider { - min: 2, - max: 12, - default: self.points_per_decade, - suffix: None, - }, - }, - SettingItem { - key: "cycles_per_measurement".into(), - label: "Cycles per measurement".into(), - tooltip: Some( - "Modulation cycles integrated per amplitude point".into(), - ), - kind: SettingKind::I64Slider { - min: 50, - max: 5_000, - default: self.cycles_per_measurement, - suffix: None, - }, - }, - SettingItem { - key: "polarity_on".into(), - label: "Polarity".into(), - tooltip: Some("Which comparator path this sweep measures".into()), - kind: SettingKind::Enum { - variants: vec!["ON".into(), "OFF".into()], - default: usize::from(!self.polarity_on), - }, - }, - SettingItem { - key: "alpha".into(), - label: "Significance α".into(), - tooltip: Some( - "Per-measurement false-positive budget (Bonferroni-corrected \ - for the frequency scan)" - .into(), - ), - kind: SettingKind::F64Drag { - min: 1e-6, - max: 0.05, - speed: 1e-4, - default: self.alpha, - }, - }, - ], - }, - SettingsSection { - label: "Device".into(), - description: None, - default_open: false, - items: vec![ - SettingItem { - key: "initial_amplitude_dac".into(), - label: "Initial amplitude (DAC)".into(), - tooltip: None, - kind: SettingKind::I64Slider { - min: 1, - max: 2_047, - default: self.initial_amplitude_dac, - suffix: None, - }, - }, - SettingItem { - key: "settle_ms".into(), - label: "Settle time".into(), - tooltip: Some( - "Discarded after each drive change (HVA/Pockels settling + \ - refractory clearing)" - .into(), - ), - kind: SettingKind::I64Slider { - min: 10, - max: 2_000, - default: self.settle_ms, - suffix: Some(" ms".into()), - }, - }, - SettingItem { - key: "dark_millivolts".into(), - label: "Dark level".into(), - tooltip: None, - kind: SettingKind::F64Drag { - min: 0.0, - max: 3_300.0, - speed: 1.0, - default: self.calibration.dark_volts * 1_000.0, - }, - }, - ], - }, - ], - } - } - - fn get_setting(&self, key: &str) -> Option { - match key { - "freq_start_hz" => Some(json!(self.freq_start_hz)), - "freq_stop_hz" => Some(json!(self.freq_stop_hz)), - "points_per_decade" => Some(json!(self.points_per_decade)), - "cycles_per_measurement" => Some(json!(self.cycles_per_measurement)), - "polarity_on" => Some(json!(if self.polarity_on { "ON" } else { "OFF" })), - "alpha" => Some(json!(self.alpha)), - "initial_amplitude_dac" => Some(json!(self.initial_amplitude_dac)), - "settle_ms" => Some(json!(self.settle_ms)), - "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "freq_start_hz" => { - self.freq_start_hz = value.as_f64().ok_or("must be a number")?.max(1.0); - } - "freq_stop_hz" => { - self.freq_stop_hz = value.as_f64().ok_or("must be a number")?.max(1.0); - } - "points_per_decade" => { - self.points_per_decade = value.as_i64().ok_or("must be an integer")?.clamp(2, 12); - } - "cycles_per_measurement" => { - self.cycles_per_measurement = - value.as_i64().ok_or("must be an integer")?.clamp(50, 5_000); - } - "polarity_on" => { - let text = value.as_str().ok_or("must be a string")?; - self.polarity_on = text.eq_ignore_ascii_case("on"); - } - "alpha" => { - self.alpha = value.as_f64().ok_or("must be a number")?.clamp(1e-6, 0.05); - } - "initial_amplitude_dac" => { - self.initial_amplitude_dac = - value.as_i64().ok_or("must be an integer")?.clamp(1, 2_047); - } - "settle_ms" => { - self.settle_ms = value.as_i64().ok_or("must be an integer")?.clamp(10, 2_000); - } - "dark_millivolts" => { - let mv = value.as_f64().ok_or("must be a number")?; - self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); - } - _ => return Err(format!("unknown setting: {key}")), - } - Ok(()) - } - - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - if let Some(engine) = &self.engine { - entries.push(StatusEntry::Text(format!( - "{} frequency points finished", - engine.results.len() - ))); - } - if let Some(err) = &self.last_error { - entries.push(StatusEntry::Text(format!("Error: {err}"))); - } - entries - } - - fn host_views(&self) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![ - HostDatasetDescriptor { - id: AMIN_DATASET_ID.into(), - title: "a_min(f)".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No fitted frequency points yet.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: PHASE_DATASET_ID.into(), - title: "Phase histogram".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No measurement window yet.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: DEPTH_DATASET_ID.into(), - title: "N(a) at current frequency".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No depth points yet.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "A1 run status".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Idle.".into(), - display: None, - relations: Vec::new(), - }, - ], - views: vec![ - HostViewDescriptor { - id: format!("{AMIN_DATASET_ID}.view"), - title: "A1 Bode (a_min)".into(), - dataset_id: AMIN_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: format!("{PHASE_DATASET_ID}.view"), - title: "Phase fold".into(), - dataset_id: PHASE_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: format!("{DEPTH_DATASET_ID}.view"), - title: "Depth staircase".into(), - dataset_id: DEPTH_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: format!("{STATUS_DATASET_ID}.view"), - title: "A1 status".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - ], - actions: vec![ - HostActionDescriptor { - id: ACTION_ARM.into(), - title: "Arm controller".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_RUN.into(), - title: "Run A1 sweep".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_STOP.into(), - title: "Stop".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - ], - } - } - - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - AMIN_DATASET_ID => serde_json::to_vec(&self.amin_dataset()).ok(), - PHASE_DATASET_ID => serde_json::to_vec(&self.phase_dataset()).ok(), - DEPTH_DATASET_ID => serde_json::to_vec(&self.depth_dataset()).ok(), - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, - } - } - - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - AMIN_DATASET_ID | PHASE_DATASET_ID | DEPTH_DATASET_ID | STATUS_DATASET_ID => { - self.dataset_generation.max(1) - } - _ => 0, - } - } -} - -impl Drop for StageAA1Plugin { - fn drop(&mut self) { - self.disarm("plugin destroyed"); - } -} - -export_plugin!(StageAA1Plugin); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn frequency_grid_is_log_spaced_and_bounded() { - let mut plugin = StageAA1Plugin::default(); - plugin.freq_start_hz = 100.0; - plugin.freq_stop_hz = 10_000.0; - plugin.points_per_decade = 4; - let grid = plugin.frequency_grid(); - assert!((grid.first().copied().unwrap() - 100.0).abs() < 1e-9); - assert!(grid.last().copied().unwrap() <= 10_000.0 * 1.001); - assert_eq!(grid.len(), 9); - for pair in grid.windows(2) { - let ratio = pair[1] / pair[0]; - assert!((ratio - 10f64.powf(0.25)).abs() < 1e-9); - } - } - - #[test] - fn status_dataset_matches_schema() { - let plugin = StageAA1Plugin::default(); - let dataset = plugin.status_dataset(); - let schema = plugin.status_schema(); - assert_eq!(dataset.columns.len(), schema.columns.len()); - } -} diff --git a/plugins/stage-a-a1/src/sweep.rs b/plugins/stage-a-a1/src/sweep.rs deleted file mode 100644 index 36f0a25..0000000 --- a/plugins/stage-a-a1/src/sweep.rs +++ /dev/null @@ -1,368 +0,0 @@ -//! Minimum-depth sweep state machine. -//! -//! For each frequency point: bisect on the integer DAC drive code until the -//! detection boundary is bracketed, then measure a small log-spaced grid -//! across the transition, then fit `a_min` (see `analysis::fit_min_depth`). -//! The engine is pure — device I/O and event analysis happen outside; it -//! only ingests finished measurements and emits the next drive request. -//! Note the asymmetry the whole design hinges on: the *search* variable is -//! the drive code, but every recorded point carries the **measured** -//! optical contrast `a` from the photodiode. - -use crate::analysis::{fit_min_depth, DepthPoint, MinDepthFit}; - -#[derive(Debug, Clone, PartialEq)] -pub struct SweepPlan { - pub frequencies_hz: Vec, - pub initial_amplitude_dac: u32, - pub max_amplitude_dac: u32, - /// Grid points measured across the bracket after bisection. - pub grid_points: usize, - /// Hard cap on measurements per frequency (bisection + grid). - pub max_measurements_per_frequency: usize, -} - -impl Default for SweepPlan { - fn default() -> Self { - Self { - frequencies_hz: Vec::new(), - initial_amplitude_dac: 512, - max_amplitude_dac: 2_047, - grid_points: 6, - max_measurements_per_frequency: 24, - } - } -} - -/// One finished measurement at the currently requested drive. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct Measurement { - pub amplitude_dac: u32, - /// Photodiode-measured optical log-contrast. `None` = invalid window - /// (clipped / integrity fault) — the point is discarded and re-measured. - pub measured_a: Option, - pub events_per_half_cycle: f64, - pub detected: bool, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum SweepCommand { - /// Configure the drive and measure at these settings. - Measure { - frequency_hz: f64, - amplitude_dac: u32, - }, - /// All frequencies finished. - Finished, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct FrequencyResult { - pub frequency_hz: f64, - pub fit: Option, - pub points: Vec, - pub measurements: usize, - /// True when the point budget ran out before the transition was - /// bracketed — a_min is not identifiable from this data. - pub exhausted: bool, -} - -#[derive(Debug, Clone, PartialEq)] -enum Phase { - Bisecting, - Grid { queue: Vec }, -} - -pub struct SweepEngine { - plan: SweepPlan, - frequency_index: usize, - phase: Phase, - current_dac: u32, - measurements_at_frequency: usize, - /// Highest drive code that did NOT detect / lowest that did. - highest_undetected: Option, - lowest_detected: Option, - points: Vec, - invalid_retries: usize, - pub results: Vec, -} - -impl SweepEngine { - pub fn new(plan: SweepPlan) -> Self { - let current_dac = plan.initial_amplitude_dac; - Self { - plan, - frequency_index: 0, - phase: Phase::Bisecting, - current_dac, - measurements_at_frequency: 0, - highest_undetected: None, - lowest_detected: None, - points: Vec::new(), - invalid_retries: 0, - results: Vec::new(), - } - } - - pub fn current_command(&self) -> SweepCommand { - match self.plan.frequencies_hz.get(self.frequency_index) { - Some(&frequency_hz) => SweepCommand::Measure { - frequency_hz, - amplitude_dac: self.current_dac, - }, - None => SweepCommand::Finished, - } - } - - pub fn is_finished(&self) -> bool { - self.frequency_index >= self.plan.frequencies_hz.len() - } - - /// Ingests the finished measurement for the last `Measure` command and - /// advances the state machine. - pub fn ingest(&mut self, measurement: Measurement) -> SweepCommand { - if self.is_finished() { - return SweepCommand::Finished; - } - - let Some(a) = measurement.measured_a else { - // Invalid window: re-measure the same point (bounded retries), - // never silently keep the previous contrast. - self.invalid_retries += 1; - if self.invalid_retries > 3 { - self.finish_frequency(true); - } - return self.current_command(); - }; - self.invalid_retries = 0; - self.measurements_at_frequency += 1; - self.points.push(DepthPoint { - a, - events_per_half_cycle: measurement.events_per_half_cycle, - }); - - if measurement.detected { - self.lowest_detected = - Some(self.lowest_detected.map_or(measurement.amplitude_dac, |d| { - d.min(measurement.amplitude_dac) - })); - } else { - self.highest_undetected = Some( - self.highest_undetected - .map_or(measurement.amplitude_dac, |d| { - d.max(measurement.amplitude_dac) - }), - ); - } - - if self.measurements_at_frequency >= self.plan.max_measurements_per_frequency { - self.finish_frequency(!self.bracketed()); - return self.current_command(); - } - - match &mut self.phase { - Phase::Bisecting => { - if self.bracketed() { - let queue = self.grid_queue(); - self.phase = Phase::Grid { queue }; - self.advance_grid(); - } else if measurement.detected { - // Drive down toward the boundary. - let next = ((measurement.amplitude_dac as f64) * 0.65).round() as u32; - if next < 1 { - self.finish_frequency(false); - } else { - self.current_dac = next.max(1); - } - } else { - // Drive up toward the boundary. - let next = ((measurement.amplitude_dac as f64) * 1.5).ceil() as u32; - if next > self.plan.max_amplitude_dac { - // Even full drive shows nothing: unmeasurable point. - self.finish_frequency(true); - } else { - self.current_dac = next; - } - } - } - Phase::Grid { .. } => { - self.advance_grid(); - } - } - self.current_command() - } - - fn bracketed(&self) -> bool { - matches!( - (self.highest_undetected, self.lowest_detected), - (Some(_), Some(_)) - ) - } - - fn grid_queue(&self) -> Vec { - let (Some(low), Some(high)) = (self.highest_undetected, self.lowest_detected) else { - return Vec::new(); - }; - let lo = (low.min(high) as f64 * 0.8).max(1.0); - let hi = (low.max(high) as f64 * 1.25).min(self.plan.max_amplitude_dac as f64); - let n = self.plan.grid_points.max(2); - (0..n) - .map(|i| { - let t = i as f64 / (n - 1) as f64; - (lo * (hi / lo).powf(t)).round() as u32 - }) - .collect() - } - - fn advance_grid(&mut self) { - let next = match &mut self.phase { - Phase::Grid { queue } if !queue.is_empty() => Some(queue.remove(0)), - _ => None, - }; - match next { - Some(dac) => self.current_dac = dac, - None => self.finish_frequency(false), - } - } - - fn finish_frequency(&mut self, exhausted: bool) { - let frequency_hz = self.plan.frequencies_hz[self.frequency_index]; - let fit = if exhausted { - None - } else { - fit_min_depth(&self.points) - }; - self.results.push(FrequencyResult { - frequency_hz, - fit, - points: std::mem::take(&mut self.points), - measurements: self.measurements_at_frequency, - exhausted, - }); - self.frequency_index += 1; - self.phase = Phase::Bisecting; - self.current_dac = self.plan.initial_amplitude_dac; - self.measurements_at_frequency = 0; - self.highest_undetected = None; - self.lowest_detected = None; - self.invalid_retries = 0; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Simulated bench: optical contrast is proportional to the drive code - /// (a = dac / 2000) and the pixel responds with the smeared first step - /// around a_min = 0.2. - fn respond(dac: u32) -> Measurement { - let a = dac as f64 / 2_000.0; - let z = (a.ln() - 0.2_f64.ln()) / 0.12; - let n = 0.5 * (1.0 + erf_approx(z / std::f64::consts::SQRT_2)); - Measurement { - amplitude_dac: dac, - measured_a: Some(a), - events_per_half_cycle: n, - detected: n > 0.15, - } - } - - fn erf_approx(x: f64) -> f64 { - let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); - let poly = t - * (0.254_829_592 - + t * (-0.284_496_736 - + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); - let value = 1.0 - poly * (-x * x).exp(); - if x >= 0.0 { - value - } else { - -value - } - } - - #[test] - fn converges_to_the_synthetic_a_min() { - let mut engine = SweepEngine::new(SweepPlan { - frequencies_hz: vec![1_000.0, 10_000.0], - ..SweepPlan::default() - }); - - let mut guard = 0; - loop { - guard += 1; - assert!(guard < 200, "sweep must terminate"); - match engine.current_command() { - SweepCommand::Finished => break, - SweepCommand::Measure { amplitude_dac, .. } => { - engine.ingest(respond(amplitude_dac)); - } - } - } - - assert_eq!(engine.results.len(), 2); - for result in &engine.results { - let fit = result.fit.as_ref().expect("fit must exist"); - assert!( - (fit.a_min - 0.2).abs() < 0.04, - "f={} a_min={}", - result.frequency_hz, - fit.a_min - ); - assert!(!result.exhausted); - } - } - - #[test] - fn undetectable_frequency_is_reported_exhausted_not_fitted() { - let mut engine = SweepEngine::new(SweepPlan { - frequencies_hz: vec![100_000.0], - ..SweepPlan::default() - }); - let mut guard = 0; - loop { - guard += 1; - assert!(guard < 100); - match engine.current_command() { - SweepCommand::Finished => break, - SweepCommand::Measure { amplitude_dac, .. } => { - engine.ingest(Measurement { - amplitude_dac, - measured_a: Some(amplitude_dac as f64 / 2_000.0), - events_per_half_cycle: 0.0, - detected: false, - }); - } - } - } - assert_eq!(engine.results.len(), 1); - assert!(engine.results[0].exhausted); - assert!(engine.results[0].fit.is_none()); - } - - #[test] - fn invalid_windows_are_retried_then_abandoned() { - let mut engine = SweepEngine::new(SweepPlan { - frequencies_hz: vec![1_000.0], - ..SweepPlan::default() - }); - let mut measures = 0; - loop { - match engine.current_command() { - SweepCommand::Finished => break, - SweepCommand::Measure { amplitude_dac, .. } => { - measures += 1; - assert!(measures < 20); - engine.ingest(Measurement { - amplitude_dac, - measured_a: None, - events_per_half_cycle: 0.0, - detected: false, - }); - } - } - } - assert!(engine.results[0].exhausted); - } -} diff --git a/plugins/stage-a-funcgen/README.md b/plugins/stage-a-funcgen/README.md deleted file mode 100644 index fb57f6c..0000000 --- a/plugins/stage-a-funcgen/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Stage-A Function Generator - -Manual control of the Stage-A Pockels-cell drive for familiarisation with the -bench: pick a waveform (**sine**, **square**, **sawtooth**), a frequency, and a -DAC modulation depth, hit *Apply drive*, and watch the photodiode respond live. - -## Why the amplitude is "measured", not set - -`amplitude_dac` commands the *phase*-modulation depth of the Pockels cell. The -cell's voltage→transmission response is non-linear (≈ sin²), so the same DAC -excursion produces different optical amplitudes at different working points. -The plugin therefore always reports the **measured** optical log-contrast - -``` -a = ln(V_max / V_min) (dark-corrected photodiode voltages) -``` - -computed by `stage-a-io`'s calibrated, clipping-guarded estimator — never a -value inferred from the commanded DAC codes. - -## Ports - -| Port | Behaviour | -|---|---| -| `mock` (default) | Runs the waveform-extended mock controller in-process: full command round trip, synthetic photodiode stream through a Pockels-like sin² transfer. Zero hardware, zero risk. | -| `auto` / explicit device | Real Teensy over USB serial. Firmware 0.2.0 has **no waveform backend** and rejects the drive fields (`unknown_config_field`); the plugin reports this clearly. Real drive control needs the future v2 DDS firmware (`stage-a-controller/docs/features/waveform-drive.md`), which is blocked on the hardware freeze. | - -## Views and actions - -- **FuncGen photodiode** — live decimated waveform (volts vs. ms). -- **Function generator** status table — state, firmware, waveform-backend - capability, commanded drive, measured `a`, clipping, stream integrity. -- Actions on the status table: *Connect*, *Disconnect*, *Apply drive*, - *Stop drive*. - -## Safety model - -Same contract as `stage-a-monitor`: - -- serial/mock connections open only while the execution context is - `LiveCapture` with effects allowed — replay can never drive hardware; -- waveform/frequency/amplitude are persistent *settings*, but nothing reaches - the controller until the explicit *Apply drive* **action**; -- drives whose `center ± amplitude` leave the 0–4095 DAC range are refused - locally before any command is sent; -- `process_frame()` only drains the bounded I/O worker queues; -- watchdog `!FAULT` notices from the controller are surfaced immediately. diff --git a/plugins/stage-a-funcgen/plugin.toml b/plugins/stage-a-funcgen/plugin.toml deleted file mode 100644 index df29231..0000000 --- a/plugins/stage-a-funcgen/plugin.toml +++ /dev/null @@ -1,7 +0,0 @@ -name = "Stage-A Function Generator" -version = "0.2.0" -description = "Manual Pockels-cell drive control (sine/square/sawtooth, frequency, DAC amplitude) with the resulting optical contrast always measured from the photodiode." -domain = "stage-a" -library = "augur_plugin_stage_a_funcgen" -phase = "frame_only" -min_augur_version = "1.0.0" diff --git a/plugins/stage-a-funcgen/src/lib.rs b/plugins/stage-a-funcgen/src/lib.rs deleted file mode 100644 index 694a21c..0000000 --- a/plugins/stage-a-funcgen/src/lib.rs +++ /dev/null @@ -1,1011 +0,0 @@ -//! Stage-A function generator — familiarisation plugin. -//! -//! Manual control of the Pockels-cell drive: waveform (sine, square, -//! sawtooth), frequency, and the commanded DAC modulation depth -//! (`amplitude_dac`). The commanded amplitude sets the *phase* modulation -//! of the Pockels cell, which maps non-linearly to transmitted intensity — -//! so the optical amplitude shown here is always the photodiode-measured -//! log-contrast `a = ln(V_max/V_min)`, never the DAC excursion. -//! -//! Firmware 0.2.0 has no waveform backend yet: it rejects the reserved v2 -//! drive fields with `unknown_config_field` (the feature-detection -//! contract, `stage-a-controller/docs/features/waveform-drive.md`). Until -//! the DDS firmware lands, select the **`mock`** port: it runs the -//! waveform-extended mock controller in-process and streams a synthetic -//! photodiode response through a Pockels-like sin² transfer — the full -//! control loop with zero hardware and zero risk. -//! -//! Safety contract (same as `stage-a-monitor`): -//! - devices open only when the execution context is `LiveCapture` with -//! `effects_allowed`; anything else tears the connection down; -//! - drive parameters are persistent *settings*, but nothing starts the -//! hardware except an explicit Apply **action**; -//! - `process_frame()` only drains the bounded I/O worker queues. - -use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::thread::JoinHandle; -use std::time::{Duration, Instant}; - -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, - Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, - StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, -}; -use serde_json::{json, Value}; -use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, - MockController, MockState, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, -}; - -const WAVEFORM_DATASET_ID: &str = "stage-a-funcgen.waveform"; -const STATUS_DATASET_ID: &str = "stage-a-funcgen.status"; -const WAVEFORM_VIEW_ID: &str = "stage-a-funcgen.waveform.view"; -const STATUS_VIEW_ID: &str = "stage-a-funcgen.status.view"; - -const ACTION_CONNECT: &str = "stage-a-funcgen.connect"; -const ACTION_DISCONNECT: &str = "stage-a-funcgen.disconnect"; -const ACTION_APPLY: &str = "stage-a-funcgen.apply"; -const ACTION_STOP: &str = "stage-a-funcgen.stop"; - -/// Retained sample window for the live view + contrast estimate. -const SAMPLE_RING_CAPACITY: usize = 32_768; -/// Points published per waveform refresh (decimated). -const WAVEFORM_POINTS: usize = 1_024; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ConnectionState { - Disconnected, - Connected, - Driving, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Wave { - Sine, - Square, - Saw, -} - -impl Wave { - const VARIANTS: [Wave; 3] = [Wave::Sine, Wave::Square, Wave::Saw]; - - fn name(self) -> &'static str { - match self { - Self::Sine => "SINE", - Self::Square => "SQUARE", - Self::Saw => "SAW", - } - } - - fn from_name(name: &str) -> Option { - Self::VARIANTS.into_iter().find(|w| w.name() == name) - } -} - -/// In-process mock controller thread behind the `mock` port. -struct MockService { - stop: Arc, - join: Option>, -} - -impl MockService { - fn spawn() -> (Self, StageAClient) { - let link = stage_a_io::MockLink::new(); - let stop = Arc::new(AtomicBool::new(false)); - let thread_stop = Arc::clone(&stop); - let mut controller = MockController::new(link.device_end()).with_waveform_extension(); - let join = std::thread::Builder::new() - .name("stage-a-funcgen-mock".into()) - .spawn(move || { - let mut last_block = Instant::now(); - while !thread_stop.load(Ordering::Relaxed) { - controller.poll_commands(); - if controller.state() == MockState::Running - && last_block.elapsed() >= controller.block_period() - { - last_block = Instant::now(); - controller.emit_configured_block(); - } - std::thread::sleep(Duration::from_millis(1)); - } - }) - .expect("spawning the mock controller thread must succeed"); - ( - Self { - stop, - join: Some(join), - }, - StageAClient::new(link.host_end()), - ) - } -} - -impl Drop for MockService { - fn drop(&mut self) { - self.stop.store(true, Ordering::Relaxed); - if let Some(join) = self.join.take() { - let _ = join.join(); - } - } -} - -pub struct StageAFuncGenPlugin { - enabled: bool, - // -- device -- - worker: Option, - mock_service: Option, - connection: ConnectionState, - firmware: String, - has_waveform_backend: Option, - next_tag: u64, - in_flight: BTreeMap, - last_error: Option, - integrity: StreamIntegrity, - effects_blocked_reason: Option, - // -- settings (drive parameters; applying them is an explicit action) -- - port_hint: String, - wave: Wave, - frequency_hz: f64, - center_dac: i64, - amplitude_dac: i64, - sample_rate_hz: i64, - calibration: AdcCalibration, - // -- data -- - sample_ring: Vec, - ring_next_sample_index: u64, - sample_rate_seen_hz: u32, - contrast: Option, - contrast_error: Option, - dataset_generation: u64, - consumed_action_ids: Vec, -} - -impl Default for StageAFuncGenPlugin { - fn default() -> Self { - Self { - enabled: false, - worker: None, - mock_service: None, - connection: ConnectionState::Disconnected, - firmware: String::new(), - has_waveform_backend: None, - next_tag: 1, - in_flight: BTreeMap::new(), - last_error: None, - integrity: StreamIntegrity::default(), - effects_blocked_reason: None, - port_hint: "mock".into(), - wave: Wave::Sine, - frequency_hz: 1_000.0, - center_dac: 2_048, - amplitude_dac: 512, - sample_rate_hz: 20_000, - calibration: AdcCalibration::default(), - sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), - ring_next_sample_index: 0, - sample_rate_seen_hz: 0, - contrast: None, - contrast_error: None, - dataset_generation: 0, - consumed_action_ids: Vec::new(), - } - } -} - -impl StageAFuncGenPlugin { - fn bump_generation(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - - fn connect(&mut self) { - if self.worker.is_some() { - return; - } - if self.port_hint == "mock" { - let (service, client) = MockService::spawn(); - self.mock_service = Some(service); - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - } else { - match open_serial(&self.port_hint) { - Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - } - Err(err) => self.last_error = Some(err), - } - } - self.bump_generation(); - } - - fn disconnect(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - // Shut the worker down first: its final STOP still needs the - // mock service (if any) alive to be acknowledged. - worker.shutdown(reason); - } - self.mock_service = None; - self.connection = ConnectionState::Disconnected; - self.firmware.clear(); - self.has_waveform_backend = None; - self.in_flight.clear(); - self.bump_generation(); - } - - /// STOP → CONFIG (drive fields) → START, honouring the firmware state - /// machine (CONFIG is only legal from SAFE_IDLE/CONFIGURED). - fn apply_drive(&mut self) { - let center = self.center_dac.clamp(0, 4_095); - let amplitude = self.amplitude_dac.clamp(0, 2_047); - if center + amplitude > 4_095 || amplitude > center { - self.last_error = Some(format!( - "drive: center {center} ± amplitude {amplitude} exceeds the 0–4095 DAC range" - )); - return; - } - let freq_mhz = ((self.frequency_hz.max(0.001)) * 1_000.0).round() as i64; - self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); - self.queue_command( - "drive", - Command::new("CONFIG") - .field("mode", "A1") - .field("wave", self.wave.name()) - .field("freq_mhz", freq_mhz) - .field("center_dac", center) - .field("amplitude_dac", amplitude) - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", 256) - .field("raw", 1) - .field("summary", 1), - ); - self.queue_command("start", Command::new("START")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(true)); - } - } - - fn stop_drive(&mut self) { - self.queue_command("stop", Command::new("STOP").field("reason", "operator")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - if outputs.is_empty() { - return; - } - let mut changed = false; - let mut stopped: Option = None; - for output in outputs { - changed = true; - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => self.handle_reply(&purpose, &fields), - Err(err) if err.contains("unknown_config_field") => { - self.has_waveform_backend = Some(false); - self.last_error = Some( - "firmware has no waveform backend (v1) — select the mock port \ - or wait for the v2 DDS firmware" - .into(), - ); - } - Err(err) => { - self.last_error = Some(format!("{purpose}: {err}")); - } - } - } - WorkerOutput::Event(DeviceEvent::Data(frame)) => { - if frame.header.frame_type == FrameType::SamplesU16 { - if let Some(codes) = frame.samples() { - self.sample_rate_seen_hz = frame.header.sample_rate_hz; - self.push_samples(&codes, frame.header.first_sample_index); - } - } - } - WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { - if name == "FAULT" { - if self.connection == ConnectionState::Driving { - self.connection = ConnectionState::Connected; - } - self.last_error = Some(format!( - "controller fault: {} — dropped to SAFE_IDLE", - fields.get("code").map(String::as_str).unwrap_or("unknown") - )); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - } - WorkerOutput::Integrity(integrity) => { - self.integrity = integrity; - } - WorkerOutput::Stopped { reason } => { - stopped = Some(reason); - } - } - } - if let Some(reason) = stopped { - self.worker = None; - self.mock_service = None; - self.connection = ConnectionState::Disconnected; - self.last_error = Some(format!("device connection ended: {reason}")); - } - if changed { - self.refresh_contrast(); - self.bump_generation(); - } - } - - fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { - match purpose { - "hello" => { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - self.has_waveform_backend = Some( - fields - .get("capabilities") - .is_some_and(|caps| caps.split(',').any(|c| c == "WAVE")), - ); - self.connection = ConnectionState::Connected; - } - "drive" => { - self.has_waveform_backend = Some(true); - } - "start" => { - self.connection = ConnectionState::Driving; - } - "stop" => { - if self.connection == ConnectionState::Driving { - self.connection = ConnectionState::Connected; - } - } - _ => {} - } - } - - fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { - self.ring_next_sample_index = first_sample_index + codes.len() as u64; - self.sample_ring.extend_from_slice(codes); - let len = self.sample_ring.len(); - if len > SAMPLE_RING_CAPACITY { - self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); - } - } - - fn refresh_contrast(&mut self) { - if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { - return; - } - match estimate_contrast(&self.sample_ring, &self.calibration) { - Ok(estimate) => { - self.contrast = Some(estimate); - self.contrast_error = None; - } - Err(err) => { - self.contrast = None; - self.contrast_error = Some(err.to_string()); - } - } - } - - fn waveform_dataset(&self) -> Series1dV1 { - let rate = if self.sample_rate_seen_hz > 0 { - f64::from(self.sample_rate_seen_hz) - } else { - self.sample_rate_hz as f64 - }; - let n = self.sample_ring.len(); - let stride = (n / WAVEFORM_POINTS).max(1); - let first_index = self.ring_next_sample_index.saturating_sub(n as u64); - let points: Vec = self - .sample_ring - .iter() - .enumerate() - .step_by(stride) - .map(|(i, &code)| Series1dPoint { - x: (first_index + i as u64) as f64 / rate * 1_000.0, - y: self.calibration.code_to_volts(code), - }) - .collect(); - Series1dV1 { - x_label: "time [ms]".into(), - y_label: "photodiode [V]".into(), - lines: vec![Series1dLine { - name: "photodiode".into(), - points, - }], - } - } - - fn drive_summary(&self) -> String { - format!( - "{} @ {:.3} Hz, {} ± {} DAC", - self.wave.name(), - self.frequency_hz, - self.center_dac, - self.amplitude_dac - ) - } - - fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.connection) { - (Some(reason), _) => format!("locked ({reason})"), - (None, ConnectionState::Disconnected) => "disconnected".into(), - (None, ConnectionState::Connected) => "connected".into(), - (None, ConnectionState::Driving) => "driving".into(), - }; - let backend = match self.has_waveform_backend { - Some(true) => "waveform-capable".into(), - Some(false) => "no waveform backend (v1)".into(), - None => "—".into(), - }; - let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { - (Some(estimate), _) => ( - format!("{:.4}", estimate.a), - format!( - "{:.2}% low / {:.2}% high", - estimate.low_clip_fraction * 100.0, - estimate.high_clip_fraction * 100.0 - ), - ), - (None, Some(err)) => ("invalid".into(), err.clone()), - (None, None) => ("—".into(), "—".into()), - }; - let integrity = if self.integrity.is_clean() { - "clean".to_owned() - } else { - format!( - "crc={} gaps={} skipped={} overruns={}", - self.integrity.crc_failures, - self.integrity.sequence_gaps, - self.integrity.skipped_bytes, - self.integrity.dropped_samples - ) - }; - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", state), - text_column("firmware", self.firmware.clone()), - text_column("backend", backend), - text_column("drive", self.drive_summary()), - text_column("a", a_text), - text_column("clipping", clip_text), - text_column("integrity", integrity), - text_column("error", self.last_error.clone().unwrap_or_default()), - ], - } - } - - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("firmware", "Firmware"), - column("backend", "Waveform backend"), - column("drive", "Commanded drive"), - column("a", "Measured a = ln(Vmax/Vmin)"), - column("clipping", "Clipping"), - column("integrity", "Stream integrity"), - column("error", "Last error"), - ], - ..TableSchema::default() - } - } - - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-funcgen.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed - } -} - -fn open_serial(port_hint: &str) -> Result, String> { - let path = if port_hint == "auto" { - serial_ports() - .into_iter() - .next() - .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? - } else { - port_hint.to_owned() - }; - let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} - -fn serial_ports() -> Vec { - stage_a_io::transport::available_port_names() - .into_iter() - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) - .collect() -} - -impl Plugin for StageAFuncGenPlugin { - fn name(&self) -> &'static str { - "Stage-A Function Generator" - } - - fn description(&self) -> &'static str { - "Manual Pockels-cell drive (sine/square/sawtooth, frequency, DAC amplitude) with photodiode-measured optical contrast; mock port for hardware-free familiarisation." - } - - fn enabled(&self) -> bool { - self.enabled - } - - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.disconnect("plugin disabled"); - } - } - - fn reset(&mut self) { - self.sample_ring.clear(); - self.contrast = None; - self.contrast_error = None; - self.bump_generation(); - } - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - // Fail closed: any pass without live-capture effects tears the - // connection down and refuses commands — even for the mock port, - // so switching the port setting can never bypass the gate. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.worker.is_some() { - self.disconnect("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for action_id in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect("operator"), - ACTION_APPLY => self.apply_drive(), - ACTION_STOP => self.stop_drive(), - _ => {} - } - } - - self.drain_worker(); - } - - fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; - port_variants.extend(serial_ports()); - let port_default = port_variants - .iter() - .position(|p| *p == self.port_hint) - .unwrap_or(0); - let wave_variants: Vec = - Wave::VARIANTS.iter().map(|w| w.name().to_owned()).collect(); - let wave_default = Wave::VARIANTS - .iter() - .position(|w| *w == self.wave) - .unwrap_or(0); - SettingsSchema { - sections: vec![SettingsSection { - label: "Function generator".into(), - description: Some( - "Drive parameters are settings; nothing reaches the hardware until the \ - Apply action. The optical amplitude is measured from the photodiode — \ - the DAC amplitude is a phase-modulation depth, not a light level." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "mock = in-process simulated controller (no hardware); \ - auto = first Teensy USB serial device" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, - }, - }, - SettingItem { - key: "wave".into(), - label: "Waveform".into(), - tooltip: Some("SINE, SQUARE, or SAW (sawtooth / Sägezahn)".into()), - kind: SettingKind::Enum { - variants: wave_variants, - default: wave_default, - }, - }, - SettingItem { - key: "frequency_hz".into(), - label: "Frequency".into(), - tooltip: Some("Drive frequency (sent as integer millihertz)".into()), - kind: SettingKind::F64Drag { - min: 0.001, - max: 200_000.0, - speed: 1.0, - default: self.frequency_hz, - }, - }, - SettingItem { - key: "center_dac".into(), - label: "Center DAC code".into(), - tooltip: Some("Working-point code (0–4095)".into()), - kind: SettingKind::I64Drag { - min: 0, - max: 4_095, - default: self.center_dac, - }, - }, - SettingItem { - key: "amplitude_dac".into(), - label: "Amplitude DAC code".into(), - tooltip: Some( - "Pockels phase-modulation depth (0–2047); the optical contrast \ - this produces is read from the measured a" - .into(), - ), - kind: SettingKind::I64Drag { - min: 0, - max: 2_047, - default: self.amplitude_dac, - }, - }, - SettingItem { - key: "sample_rate_hz".into(), - label: "ADC sample rate".into(), - tooltip: Some("Photodiode sample rate for the feedback stream".into()), - kind: SettingKind::I64Slider { - min: 1_000, - max: 100_000, - default: self.sample_rate_hz, - suffix: Some(" Hz".into()), - }, - }, - SettingItem { - key: "dark_millivolts".into(), - label: "Dark level".into(), - tooltip: Some( - "Light-blocked photodiode level; a is computed from dark-corrected \ - voltages" - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 3_300.0, - speed: 1.0, - default: self.calibration.dark_volts * 1_000.0, - }, - }, - ], - }], - } - } - - fn get_setting(&self, key: &str) -> Option { - match key { - "port" => Some(json!(self.port_hint)), - "wave" => Some(json!(self.wave.name())), - "frequency_hz" => Some(json!(self.frequency_hz)), - "center_dac" => Some(json!(self.center_dac)), - "amplitude_dac" => Some(json!(self.amplitude_dac)), - "sample_rate_hz" => Some(json!(self.sample_rate_hz)), - "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); - Ok(()) - } - "wave" => { - let name = value.as_str().ok_or("wave must be a string")?; - self.wave = Wave::from_name(name) - .ok_or_else(|| format!("unknown waveform: {name} (SINE/SQUARE/SAW)"))?; - Ok(()) - } - "frequency_hz" => { - let hz = value.as_f64().ok_or("frequency_hz must be a number")?; - self.frequency_hz = hz.clamp(0.001, 200_000.0); - Ok(()) - } - "center_dac" => { - self.center_dac = value - .as_i64() - .ok_or("center_dac must be an integer")? - .clamp(0, 4_095); - Ok(()) - } - "amplitude_dac" => { - self.amplitude_dac = value - .as_i64() - .ok_or("amplitude_dac must be an integer")? - .clamp(0, 2_047); - Ok(()) - } - "sample_rate_hz" => { - self.sample_rate_hz = value - .as_i64() - .ok_or("sample_rate_hz must be an integer")? - .clamp(1_000, 100_000); - Ok(()) - } - "dark_millivolts" => { - let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; - self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); - Ok(()) - } - _ => Err(format!("unknown setting: {key}")), - } - } - - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - entries.push(StatusEntry::Text(match self.connection { - ConnectionState::Disconnected => "FuncGen: disconnected".into(), - ConnectionState::Connected => format!("FuncGen: connected ({})", self.firmware), - ConnectionState::Driving => format!("FuncGen: driving {}", self.drive_summary()), - })); - if let Some(estimate) = &self.contrast { - entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); - } - entries - } - - fn host_views(&self) -> HostViewRegistry { - let dataset_action = |id: &str, title: &str| HostActionDescriptor { - id: id.into(), - title: title.into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }; - HostViewRegistry { - datasets: vec![ - HostDatasetDescriptor { - id: WAVEFORM_DATASET_ID.into(), - title: "FuncGen photodiode waveform".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No photodiode samples yet — connect and apply a drive.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "Function generator status".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Function generator idle.".into(), - display: None, - relations: Vec::new(), - }, - ], - views: vec![ - HostViewDescriptor { - id: WAVEFORM_VIEW_ID.into(), - title: "FuncGen photodiode".into(), - dataset_id: WAVEFORM_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: STATUS_VIEW_ID.into(), - title: "Function generator".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - ], - actions: vec![ - dataset_action(ACTION_CONNECT, "Connect"), - dataset_action(ACTION_DISCONNECT, "Disconnect"), - dataset_action(ACTION_APPLY, "Apply drive"), - dataset_action(ACTION_STOP, "Stop drive"), - ], - } - } - - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, - } - } - - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), - _ => 0, - } - } -} - -impl Drop for StageAFuncGenPlugin { - fn drop(&mut self) { - self.disconnect("plugin destroyed"); - } -} - -export_plugin!(StageAFuncGenPlugin); - -#[cfg(test)] -mod tests { - use super::*; - use std::time::{Duration, Instant}; - - fn drain_until bool>( - plugin: &mut StageAFuncGenPlugin, - timeout: Duration, - mut done: F, - ) { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - plugin.drain_worker(); - if done(plugin) { - return; - } - std::thread::sleep(Duration::from_millis(2)); - } - panic!("condition not reached within {timeout:?}"); - } - - /// Full mock loop: connect → apply sine → measured a appears → stop. - #[test] - fn mock_port_round_trip_measures_optical_contrast() { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.calibration.dark_volts = 40.0 * 3.3 / 4_095.0; - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - assert_eq!(plugin.has_waveform_backend, Some(true)); - assert_eq!(plugin.firmware, "0.2.0-mock"); - - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving && p.contrast.is_some() - }); - let a = plugin.contrast.as_ref().expect("contrast measured").a; - assert!(a > 0.0, "modulated drive must produce positive contrast"); - assert!(plugin.integrity.is_clean()); - assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); - - plugin.stop_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - plugin.disconnect("test done"); - assert_eq!(plugin.connection, ConnectionState::Disconnected); - } - - /// Square and sawtooth are accepted and produce a measurable contrast. - #[test] - fn square_and_saw_waveforms_drive_the_mock() { - for wave in [Wave::Square, Wave::Saw] { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.wave = wave; - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving && p.contrast.is_some() - }); - assert!(plugin.contrast.as_ref().unwrap().a > 0.0); - plugin.disconnect("done"); - } - } - - /// Drives exceeding the DAC range are refused locally, before any - /// command reaches a controller. - #[test] - fn out_of_range_drive_is_rejected_locally() { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.center_dac = 3_000; - plugin.amplitude_dac = 2_000; - plugin.apply_drive(); - assert!(plugin - .last_error - .as_deref() - .is_some_and(|err| err.contains("exceeds the 0–4095 DAC range"))); - } - - /// Re-applying while driving must STOP first (firmware state machine). - #[test] - fn reapply_while_driving_reconfigures_cleanly() { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving - }); - plugin.frequency_hz = 2_000.0; - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving && p.last_error.is_none() - }); - plugin.disconnect("done"); - } -} diff --git a/plugins/stage-a-funcgen/Cargo.toml b/plugins/stage-a-modulation/Cargo.toml similarity index 58% rename from plugins/stage-a-funcgen/Cargo.toml rename to plugins/stage-a-modulation/Cargo.toml index ffb540e..2b04e93 100644 --- a/plugins/stage-a-funcgen/Cargo.toml +++ b/plugins/stage-a-modulation/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "augur-plugin-stage-a-funcgen" +name = "augur-plugin-stage-a-modulation" version.workspace = true edition.workspace = true license.workspace = true authors.workspace = true -description = "Stage-A function generator: manual waveform/frequency/amplitude drive control with photodiode-measured optical contrast." +description = "Stage-A laser modulation control: one capped power slider plus constant/sine/square drive of the Teensy DAC (J23), applied immediately." [lib] crate-type = ["cdylib", "rlib"] diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md new file mode 100644 index 0000000..9e21a88 --- /dev/null +++ b/plugins/stage-a-modulation/README.md @@ -0,0 +1,33 @@ +# Stage-A Modulation + +Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy **command port** +(the first of the two USB serial ports enumerated by `stage-a-controller` firmware 0.3.0+). + +## What it does + +- **Power slider** in DAC codes (0–4095). Its upper bound is the **max limit** setting — set that + to the highest code the connected device tolerates and the slider physically cannot exceed it. +- **Mode**: `CONST` (hold the level), `SINE`, or `SQUARE` with a **frequency** (0.01–2000 Hz) and + a **min threshold** — the periodic waveforms swing between the threshold and the slider value. +- Every accepted change is sent to the Teensy **immediately** (one `MOD` command); there is no + Apply button. +- The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and + a 2 Hz `STATUS` poll), not just what was commanded. + +## Actions + +- **Connect / Disconnect** — open/close the command port. Connecting never changes the output; + only changes made while connected are transferred. +- **Output OFF** — sends `MOD wave=OFF` (DAC code 0). Needed because the firmware output is + **set-and-hold**: disconnecting, closing the GUI, or a crash leaves the last modulation running + (`stage-a-controller` ADR 002). + +## Ports + +Select the Teensy *command* port (binary protocol), not the photodiode stream port. `mock` runs an +in-process simulated controller for hardware-free testing; `auto` picks the first +usbmodem/ttyACM device. If you picked the wrong physical port, HELLO simply times out — pick the +other one. + +Hardware commands only flow while the host execution context allows effects (live capture); +otherwise the connection is torn down and the panel shows the lock reason. diff --git a/plugins/stage-a-modulation/plugin.toml b/plugins/stage-a-modulation/plugin.toml new file mode 100644 index 0000000..39b9d48 --- /dev/null +++ b/plugins/stage-a-modulation/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Modulation" +version = "0.3.0" +description = "Laser modulation control: capped power slider plus constant/sine/square drive of the Teensy DAC (J23), applied immediately." +domain = "stage-a" +library = "augur_plugin_stage_a_modulation" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs new file mode 100644 index 0000000..e5b163a --- /dev/null +++ b/plugins/stage-a-modulation/src/lib.rs @@ -0,0 +1,844 @@ +//! Stage-A laser modulation control. +//! +//! Drives the laser modulation input (Hermit J23, `DAC1.4`/address 3) through +//! the firmware 0.3.0 `MOD` command. One power slider (DAC code) whose upper +//! bound is a user-set safety cap, a mode select (constant / sine / square) +//! with frequency and a lower threshold for the periodic modes — and every +//! accepted change is transferred to the Teensy immediately, no Apply button. +//! +//! The plugin owns the Teensy **command port** (the first of the two CDC +//! ports the dual-serial firmware enumerates; the photodiode stream port is +//! owned by `stage-a-photodiode`). The firmware output is set-and-hold: +//! disconnecting does NOT switch the modulation off — use the "Output OFF" +//! action (ADR 002 in `stage-a-controller`). +//! +//! Safety contract: +//! - devices open only when the execution context allows hardware effects; +//! anything else tears the connection down (fail closed); +//! - the level slider cannot exceed the max-level cap, and the firmware +//! output can never exceed the slider (square/sine peak at `level`); +//! - `process_frame()` only drains the bounded I/O worker queues. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, SettingItem, + SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, + TableColumnValues, TableDatasetV1, TableSchema, TableValueType, + CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{Command, IoWorker, MockController, StageAClient, WorkerOutput, WorkerRequest}; + +const STATUS_DATASET_ID: &str = "stage-a-modulation.status"; +const STATUS_VIEW_ID: &str = "stage-a-modulation.status.view"; + +const ACTION_CONNECT: &str = "stage-a-modulation.connect"; +const ACTION_DISCONNECT: &str = "stage-a-modulation.disconnect"; +const ACTION_OUTPUT_OFF: &str = "stage-a-modulation.output-off"; + +const MAX_DAC_CODE: i64 = 4_095; +const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(500); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Const, + Sine, + Square, +} + +impl Mode { + const VARIANTS: [Mode; 3] = [Mode::Const, Mode::Sine, Mode::Square]; + + fn name(self) -> &'static str { + match self { + Self::Const => "CONST", + Self::Sine => "SINE", + Self::Square => "SQUARE", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|m| m.name() == name) + } + + fn is_periodic(self) -> bool { + !matches!(self, Self::Const) + } +} + +/// In-process mock controller thread behind the `mock` port. +struct MockService { + stop: Arc, + join: Option>, +} + +impl MockService { + fn spawn() -> (Self, StageAClient) { + let link = stage_a_io::MockLink::new(); + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let mut controller = MockController::new(link.device_end()); + let join = std::thread::Builder::new() + .name("stage-a-modulation-mock".into()) + .spawn(move || { + while !thread_stop.load(Ordering::Relaxed) { + controller.poll_commands(); + std::thread::sleep(Duration::from_millis(1)); + } + }) + .expect("spawning the mock controller thread must succeed"); + ( + Self { + stop, + join: Some(join), + }, + StageAClient::new(link.host_end()), + ) + } +} + +impl Drop for MockService { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +pub struct StageAModulationPlugin { + enabled: bool, + // -- device -- + worker: Option, + mock_service: Option, + connected: bool, + firmware: String, + next_tag: u64, + in_flight: BTreeMap, + last_error: Option, + effects_blocked_reason: Option, + last_status_poll: Instant, + // -- settings (every accepted change is sent immediately) -- + port_hint: String, + max_level: i64, + level: i64, + min_level: i64, + mode: Mode, + frequency_hz: f64, + dirty: bool, + // -- board-reported state (from MOD replies and STATUS polls) -- + board_code: Option, + board_mod: String, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAModulationPlugin { + fn default() -> Self { + Self { + enabled: false, + worker: None, + mock_service: None, + connected: false, + firmware: String::new(), + next_tag: 1, + in_flight: BTreeMap::new(), + last_error: None, + effects_blocked_reason: None, + last_status_poll: Instant::now(), + port_hint: "mock".into(), + max_level: MAX_DAC_CODE, + level: 0, + min_level: 0, + mode: Mode::Const, + frequency_hz: 10.0, + dirty: false, + board_code: None, + board_mod: "—".into(), + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAModulationPlugin { + fn bump_generation(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn connect(&mut self) { + if self.worker.is_some() { + return; + } + if self.port_hint == "mock" { + let (service, client) = MockService::spawn(); + self.mock_service = Some(service); + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + } else { + match open_serial(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + } + Err(err) => { + self.last_error = Some(err); + return; + } + } + } + // Connecting never drives the output: only changes made while + // connected are transferred. + self.dirty = false; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + self.bump_generation(); + } + + fn disconnect(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + worker.shutdown(reason); + } + self.mock_service = None; + self.connected = false; + self.firmware.clear(); + self.in_flight.clear(); + self.board_code = None; + self.board_mod = "—".into(); + self.bump_generation(); + } + + /// One MOD command carrying the complete current drive settings. + fn send_modulation(&mut self) { + self.dirty = false; + let level = self.level.clamp(0, self.max_level); + let mut command = Command::new("MOD") + .field("wave", self.mode.name()) + .field("level", level); + if self.mode.is_periodic() { + let freq_mhz = (self.frequency_hz.clamp(0.01, 2_000.0) * 1_000.0).round() as i64; + command = command + .field("min", self.min_level.clamp(0, level)) + .field("freq_mhz", freq_mhz); + } + self.queue_command("mod", command); + } + + fn output_off(&mut self) { + self.dirty = false; + self.queue_command("mod", Command::new("MOD").field("wave", "OFF")); + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + if outputs.is_empty() { + return; + } + let mut stopped: Option = None; + for output in outputs { + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + WorkerOutput::Event(_) | WorkerOutput::Integrity(_) => {} + WorkerOutput::Stopped { reason } => stopped = Some(reason), + } + } + if let Some(reason) = stopped { + self.worker = None; + self.mock_service = None; + self.connected = false; + self.last_error = Some(format!("device connection ended: {reason}")); + } + self.bump_generation(); + } + + fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { + if purpose == "hello" { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + self.connected = true; + let has_mod = fields + .get("capabilities") + .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); + if !has_mod { + self.last_error = + Some("firmware has no MOD capability — flash stage-a-controller 0.3.0+".into()); + } + } + // MOD replies and STATUS polls both carry code= and mod_* fields. + if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { + self.board_code = Some(code); + } + if let Some(wave) = fields.get("mod_wave") { + let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); + let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); + let freq_mhz = fields + .get("mod_freq_mhz") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0); + self.board_mod = if wave == "SINE" || wave == "SQUARE" { + format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) + } else { + format!("{wave} level={level}") + }; + } + if purpose == "mod" { + self.last_error = None; + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-modulation.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } + + fn commanded_summary(&self) -> String { + if self.mode.is_periodic() { + format!( + "{} {}..{} @ {:.3} Hz", + self.mode.name(), + self.min_level, + self.level, + self.frequency_hz + ) + } else { + format!("{} level={}", self.mode.name(), self.level) + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.connected) { + (Some(reason), _) => format!("locked ({reason})"), + (None, false) => "disconnected".into(), + (None, true) => format!("connected ({})", self.firmware), + }; + let board_code = self + .board_code + .map_or_else(|| "—".into(), |code| code.to_string()); + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("commanded", self.commanded_summary()), + text_column("board_mod", self.board_mod.clone()), + text_column("board_code", board_code), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("commanded", "Commanded drive"), + column("board_mod", "Board modulation"), + column("board_code", "Board DAC code"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } +} + +fn open_serial(port_hint: &str) -> Result, String> { + let path = if port_hint == "auto" { + serial_ports() + .into_iter() + .next() + .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? + } else { + port_hint.to_owned() + }; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +fn serial_ports() -> Vec { + stage_a_io::transport::available_port_names() + .into_iter() + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() +} + +impl Plugin for StageAModulationPlugin { + fn name(&self) -> &'static str { + "Stage-A Modulation" + } + + fn description(&self) -> &'static str { + "Laser modulation control on the Teensy command port: capped power slider, constant/sine/square with frequency, applied immediately; shows the DAC code the board reports." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect("plugin disabled"); + } + } + + fn reset(&mut self) { + self.bump_generation(); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Fail closed: without live-capture effects the connection is torn + // down and no command leaves the plugin. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.worker.is_some() { + self.disconnect("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for action_id in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect("operator"), + ACTION_OUTPUT_OFF => self.output_off(), + _ => {} + } + } + + if self.dirty && self.connected { + self.send_modulation(); + } + if self.connected && self.last_status_poll.elapsed() >= STATUS_POLL_INTERVAL { + self.last_status_poll = Instant::now(); + self.queue_command("status", Command::new("STATUS")); + } + self.drain_worker(); + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + let mode_variants: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let mode_default = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Laser modulation".into(), + description: Some( + "Every change is sent to the Teensy immediately. The output never exceeds \ + the power slider, and the slider never exceeds the max limit. The firmware \ + holds the output when the plugin disconnects — use Output OFF to drive 0." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "Teensy command port (the FIRST of the two usbmodem ports); \ + mock = in-process simulated controller, auto = first device" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "level".into(), + label: "Power (DAC code)".into(), + tooltip: Some( + "Output level in DAC codes; peak value for sine/square. \ + Capped by the max limit below." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.level, + suffix: None, + }, + }, + SettingItem { + key: "max_level".into(), + label: "Max limit (DAC code)".into(), + tooltip: Some( + "Safety cap: the slider cannot go above this. Set it to the \ + highest code the connected device tolerates at J23." + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.max_level, + }, + }, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, + }, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.frequency_hz, + }, + }, + SettingItem { + key: "min_level".into(), + label: "Min threshold (DAC code)".into(), + tooltip: Some( + "Lower bound for sine/square: the waveform swings between this \ + and the power slider. Ignored in CONST mode." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.min_level, + suffix: None, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "level" => Some(json!(self.level)), + "max_level" => Some(json!(self.max_level)), + "mode" => Some(json!(self.mode.name())), + "frequency_hz" => Some(json!(self.frequency_hz)), + "min_level" => Some(json!(self.min_level)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "level" => { + self.level = value + .as_i64() + .ok_or("level must be an integer")? + .clamp(0, self.max_level); + if self.min_level > self.level { + self.min_level = self.level; + } + self.dirty = true; + Ok(()) + } + "max_level" => { + self.max_level = value + .as_i64() + .ok_or("max_level must be an integer")? + .clamp(0, MAX_DAC_CODE); + // Lowering the cap below the current level lowers the output. + if self.level > self.max_level { + self.level = self.max_level; + self.dirty = true; + } + if self.min_level > self.max_level { + self.min_level = self.max_level; + } + Ok(()) + } + "mode" => { + let name = value.as_str().ok_or("mode must be a string")?; + self.mode = Mode::from_name(name) + .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; + self.dirty = true; + Ok(()) + } + "frequency_hz" => { + let hz = value.as_f64().ok_or("frequency_hz must be a number")?; + self.frequency_hz = hz.clamp(0.01, 2_000.0); + if self.mode.is_periodic() { + self.dirty = true; + } + Ok(()) + } + "min_level" => { + self.min_level = value + .as_i64() + .ok_or("min_level must be an integer")? + .clamp(0, self.level); + if self.mode.is_periodic() { + self.dirty = true; + } + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + entries.push(StatusEntry::Text(if self.connected { + format!("Modulation: connected ({})", self.firmware) + } else { + "Modulation: disconnected".into() + })); + if let Some(code) = self.board_code { + entries.push(StatusEntry::Text(format!( + "Board: code={code} ({})", + self.board_mod + ))); + } + if let Some(error) = &self.last_error { + entries.push(StatusEntry::Text(format!("Error: {error}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + let action = |id: &str, title: &str| HostActionDescriptor { + id: id.into(), + title: title.into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }; + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Laser modulation".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Modulation control idle.".into(), + display: None, + relations: Vec::new(), + }], + views: vec![HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Laser modulation".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }], + actions: vec![ + action(ACTION_CONNECT, "Connect"), + action(ACTION_DISCONNECT, "Disconnect"), + action(ACTION_OUTPUT_OFF, "Output OFF"), + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + STATUS_DATASET_ID => self.dataset_generation.max(1), + _ => 0, + } + } +} + +impl Drop for StageAModulationPlugin { + fn drop(&mut self) { + self.disconnect("plugin destroyed"); + } +} + +export_plugin!(StageAModulationPlugin); + +#[cfg(test)] +mod tests { + use super::*; + + fn drain_until bool>( + plugin: &mut StageAModulationPlugin, + timeout: Duration, + mut done: F, + ) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + plugin.drain_worker(); + if done(plugin) { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("condition not reached within {timeout:?}"); + } + + /// Slider change → MOD sent immediately → board echoes the code. + #[test] + fn level_change_transfers_immediately_and_board_code_is_shown() { + let mut plugin = StageAModulationPlugin::default(); + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); + assert_eq!(plugin.firmware, "0.3.0-mock"); + + plugin + .set_setting("level", json!(1234)) + .expect("level accepted"); + assert!(plugin.dirty); + plugin.send_modulation(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.board_code == Some(1234) + }); + assert!(!plugin.dirty); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + plugin.disconnect("test done"); + } + + /// The max cap bounds the slider, and lowering it re-sends a lower level. + #[test] + fn max_level_caps_the_slider() { + let mut plugin = StageAModulationPlugin::default(); + plugin.set_setting("max_level", json!(1000)).unwrap(); + plugin.set_setting("level", json!(4095)).unwrap(); + assert_eq!(plugin.level, 1000, "slider clamps to the cap"); + + plugin.set_setting("max_level", json!(500)).unwrap(); + assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); + assert!(plugin.dirty, "the lowered level must be transferred"); + + let schema = plugin.settings_schema(); + let level_item = schema.sections[0] + .items + .iter() + .find(|item| item.key == "level") + .expect("level setting exists"); + match &level_item.kind { + SettingKind::I64Slider { max, .. } => assert_eq!(*max, 500), + other => panic!("level must stay a slider, got {other:?}"), + } + } + + /// Square drive with min threshold reaches the mock and starts at min. + #[test] + fn square_with_min_threshold_round_trips() { + let mut plugin = StageAModulationPlugin::default(); + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); + + plugin.set_setting("level", json!(2000)).unwrap(); + plugin.set_setting("mode", json!("SQUARE")).unwrap(); + plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); + plugin.set_setting("min_level", json!(500)).unwrap(); + plugin.send_modulation(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.board_code == Some(500) + }); + assert!(plugin.board_mod.contains("SQUARE 500..2000")); + + plugin.output_off(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.board_code == Some(0) + }); + plugin.disconnect("test done"); + } + + /// min_level can never exceed the level. + #[test] + fn min_threshold_is_clamped_to_level() { + let mut plugin = StageAModulationPlugin::default(); + plugin.set_setting("level", json!(1000)).unwrap(); + plugin.set_setting("min_level", json!(3000)).unwrap(); + assert_eq!(plugin.min_level, 1000); + plugin.set_setting("level", json!(200)).unwrap(); + assert_eq!(plugin.min_level, 200, "lowering level drags min down"); + } +} diff --git a/plugins/stage-a-monitor/Cargo.toml b/plugins/stage-a-monitor/Cargo.toml deleted file mode 100644 index 61dfffd..0000000 --- a/plugins/stage-a-monitor/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "augur-plugin-stage-a-monitor" -version.workspace = true -edition.workspace = true -license.workspace = true -authors.workspace = true -description = "Stage-A commissioning monitor: live photodiode readout, calibrated optical contrast, and gated manual Teensy drive control." - -[lib] -crate-type = ["cdylib", "rlib"] - -[dependencies] -augur-plugin-api.workspace = true -serde_json.workspace = true -stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-monitor/README.md b/plugins/stage-a-monitor/README.md deleted file mode 100644 index 8b43896..0000000 --- a/plugins/stage-a-monitor/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Stage-A Monitor - -Commissioning companion for the Stage-A camera-calibration bench: a live -view of the Teensy photodiode DAQ plus **gated** manual controller commands. - -## What it shows - -- **Photodiode waveform** — decimated calibrated trace (volts vs ms) from - the `SamplesU16` stream. -- **Live optical contrast** — `a = ln(V_max/V_min)` from dark-corrected, - clipping-guarded percentile extrema (see `stage-a-io::estimator`). An - invalid window shows *why* (clipped / no headroom / too short) instead of - a silently wrong number. -- **Stream integrity** — CRC failures, resync skips, frame-sequence gaps, - and ADC overruns. Any nonzero counter means the current point is invalid. - -## Controls (host actions on the status table) - -`Connect`, `Disconnect`, `Start acquisition`, `Stop`, and an expert -`Apply drive` modal (integer DAC codes; the optical contrast is always -measured, never assumed from the drive). Commands are actions — not -settings — so a reloaded settings file can never arm hardware. - -## Safety - -The plugin fails closed: the serial port opens only when the host reports -`LiveCapture` with `effects_allowed` (plugin ABI v5 execution context). -Replay and offline analysis can never emit a serial byte, and an existing -connection is shut down the moment effects are revoked. The firmware-side -watchdog independently drops the controller to `SAFE_IDLE` if the host -disappears. - -## Use it for (commissioning checklist) - -1. Wiring / voltage-range check at both detector loads. -2. Dark-level measurement for the estimator calibration. -3. Coherent-crosstalk test (H14): drive on, light blocked — the waveform - view and `a` readout must stay at the noise floor. -4. USB-throughput sanity (watch the integrity counters at full rate). diff --git a/plugins/stage-a-monitor/plugin.toml b/plugins/stage-a-monitor/plugin.toml deleted file mode 100644 index b82d0f5..0000000 --- a/plugins/stage-a-monitor/plugin.toml +++ /dev/null @@ -1,7 +0,0 @@ -name = "Stage-A Monitor" -version = "0.2.0" -description = "Live Teensy photodiode readout, calibrated optical contrast, and gated manual drive control for Stage-A commissioning." -domain = "stage-a" -library = "augur_plugin_stage_a_monitor" -phase = "frame_only" -min_augur_version = "1.0.0" diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs deleted file mode 100644 index 6b3e3e9..0000000 --- a/plugins/stage-a-monitor/src/lib.rs +++ /dev/null @@ -1,831 +0,0 @@ -//! Stage-A commissioning monitor. -//! -//! Live view of the Teensy photodiode DAQ (decimated waveform, calibrated -//! optical log-contrast `a`, clipping/headroom and stream-integrity status) -//! plus **gated** manual controller commands (connect, configure, start, -//! stop) for wiring and crosstalk commissioning. -//! -//! Safety contract (Stage-A control-software spec): -//! - devices open only when `HostContext::execution()` reports -//! `LiveCapture` **and** `effects_allowed` — replay and offline analysis -//! can never touch the serial port, and a stale worker is shut down the -//! moment the context stops permitting effects; -//! - commands are host actions, never persistent settings, so a reloaded -//! settings file cannot re-arm hardware; -//! - `process_frame()` only drains the bounded I/O worker queues. - -use std::collections::BTreeMap; - -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, - Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, - StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, -}; -use serde_json::{json, Value}; -use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, - StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, -}; - -const WAVEFORM_DATASET_ID: &str = "stage-a-monitor.waveform"; -const STATUS_DATASET_ID: &str = "stage-a-monitor.status"; -const WAVEFORM_VIEW_ID: &str = "stage-a-monitor.waveform.view"; -const STATUS_VIEW_ID: &str = "stage-a-monitor.status.view"; - -const ACTION_CONNECT: &str = "stage-a-monitor.connect"; -const ACTION_DISCONNECT: &str = "stage-a-monitor.disconnect"; -const ACTION_START: &str = "stage-a-monitor.start"; -const ACTION_STOP: &str = "stage-a-monitor.stop"; -const ACTION_APPLY_DRIVE: &str = "stage-a-monitor.apply-drive"; - -/// Retained sample window for the live view + contrast estimate. -const SAMPLE_RING_CAPACITY: usize = 32_768; -/// Points published per waveform refresh (decimated). -const WAVEFORM_POINTS: usize = 1_024; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ConnectionState { - Disconnected, - Connected, - Acquiring, -} - -pub struct StageAMonitorPlugin { - enabled: bool, - // -- device -- - worker: Option, - connection: ConnectionState, - firmware: String, - next_tag: u64, - /// Tags of in-flight requests -> human-readable purpose. - in_flight: BTreeMap, - last_error: Option, - integrity: StreamIntegrity, - effects_blocked_reason: Option, - // -- settings -- - port_hint: String, - sample_rate_hz: i64, - block_samples: i64, - calibration: AdcCalibration, - // -- data -- - sample_ring: Vec, - ring_next_sample_index: u64, - sample_rate_seen_hz: u32, - contrast: Option, - contrast_error: Option, - dataset_generation: u64, - consumed_action_ids: Vec, -} - -impl Default for StageAMonitorPlugin { - fn default() -> Self { - Self { - enabled: false, - worker: None, - connection: ConnectionState::Disconnected, - firmware: String::new(), - next_tag: 1, - in_flight: BTreeMap::new(), - last_error: None, - integrity: StreamIntegrity::default(), - effects_blocked_reason: None, - port_hint: "auto".into(), - sample_rate_hz: 20_000, - block_samples: 256, - calibration: AdcCalibration::default(), - sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), - ring_next_sample_index: 0, - sample_rate_seen_hz: 0, - contrast: None, - contrast_error: None, - dataset_generation: 0, - consumed_action_ids: Vec::new(), - } - } -} - -impl StageAMonitorPlugin { - fn bump_generation(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - - fn connect(&mut self) { - if self.worker.is_some() { - return; - } - match open_transport(&self.port_hint) { - Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - } - Err(err) => { - self.last_error = Some(err); - } - } - self.bump_generation(); - } - - fn disconnect(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - worker.shutdown(reason); - } - self.connection = ConnectionState::Disconnected; - self.in_flight.clear(); - self.bump_generation(); - } - - fn start_acquisition(&mut self) { - self.queue_command( - "config", - Command::new("CONFIG") - .field("mode", "A1") - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", self.block_samples) - .field("raw", 1) - .field("summary", 1), - ); - self.queue_command("start", Command::new("START")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(true)); - } - } - - fn stop_acquisition(&mut self) { - self.queue_command("stop", Command::new("STOP").field("reason", "operator")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - - fn apply_drive(&mut self, params: &Value) { - let freq_mhz = params.get("freq_mhz").and_then(Value::as_i64).unwrap_or(0); - let center_dac = params - .get("center_dac") - .and_then(Value::as_i64) - .unwrap_or(2_048); - let amplitude_dac = params - .get("amplitude_dac") - .and_then(Value::as_i64) - .unwrap_or(0); - self.queue_command( - "drive", - Command::new("CONFIG") - .field("mode", "A1") - .field("wave", "SINE") - .field("freq_mhz", freq_mhz) - .field("center_dac", center_dac) - .field("amplitude_dac", amplitude_dac) - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", self.block_samples) - .field("raw", 1) - .field("summary", 1), - ); - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - if outputs.is_empty() { - return; - } - let mut changed = false; - let mut stopped: Option = None; - for output in outputs { - changed = true; - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => self.handle_reply(&purpose, &fields), - Err(err) if err.contains("unknown_config_field") => { - // Feature detection: firmware 0.2.0 has no - // waveform backend and rejects the reserved v2 - // drive fields. - self.last_error = Some(format!( - "{purpose}: firmware has no waveform backend (v1) — drive \ - control needs the mock or the future v2 firmware" - )); - } - Err(err) => { - self.last_error = Some(format!("{purpose}: {err}")); - } - } - } - WorkerOutput::Event(DeviceEvent::Data(frame)) => match frame.header.frame_type { - FrameType::SamplesU16 => { - if let Some(codes) = frame.samples() { - self.sample_rate_seen_hz = frame.header.sample_rate_hz; - self.push_samples(&codes, frame.header.first_sample_index); - } - } - FrameType::Summary | FrameType::Marker | FrameType::Control => {} - FrameType::Unknown(_) => {} - }, - WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { - if name == "FAULT" { - // Firmware watchdog dropped the controller to - // SAFE_IDLE — reflect it instead of showing a stale - // "acquiring" state. - if self.connection == ConnectionState::Acquiring { - self.connection = ConnectionState::Connected; - } - self.last_error = Some(format!( - "controller fault: {} — dropped to SAFE_IDLE", - fields.get("code").map(String::as_str).unwrap_or("unknown") - )); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - } - WorkerOutput::Integrity(integrity) => { - self.integrity = integrity; - } - WorkerOutput::Stopped { reason } => { - stopped = Some(reason); - } - } - } - if let Some(reason) = stopped { - self.worker = None; - self.connection = ConnectionState::Disconnected; - self.last_error = Some(format!("device connection ended: {reason}")); - } - if changed { - self.refresh_contrast(); - self.bump_generation(); - } - } - - fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { - match purpose { - "hello" => { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - self.connection = ConnectionState::Connected; - } - "start" => { - self.connection = ConnectionState::Acquiring; - } - "stop" => { - self.connection = ConnectionState::Connected; - } - _ => {} - } - } - - fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { - self.ring_next_sample_index = first_sample_index + codes.len() as u64; - self.sample_ring.extend_from_slice(codes); - let len = self.sample_ring.len(); - if len > SAMPLE_RING_CAPACITY { - self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); - } - } - - fn refresh_contrast(&mut self) { - if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { - return; - } - match estimate_contrast(&self.sample_ring, &self.calibration) { - Ok(estimate) => { - self.contrast = Some(estimate); - self.contrast_error = None; - } - Err(err) => { - self.contrast = None; - self.contrast_error = Some(err.to_string()); - } - } - } - - fn waveform_dataset(&self) -> Series1dV1 { - let rate = if self.sample_rate_seen_hz > 0 { - f64::from(self.sample_rate_seen_hz) - } else { - self.sample_rate_hz as f64 - }; - let n = self.sample_ring.len(); - let stride = (n / WAVEFORM_POINTS).max(1); - let first_index = self.ring_next_sample_index.saturating_sub(n as u64); - let points: Vec = self - .sample_ring - .iter() - .enumerate() - .step_by(stride) - .map(|(i, &code)| Series1dPoint { - x: (first_index + i as u64) as f64 / rate * 1_000.0, - y: self.calibration.code_to_volts(code), - }) - .collect(); - Series1dV1 { - x_label: "time [ms]".into(), - y_label: "photodiode [V]".into(), - lines: vec![Series1dLine { - name: "photodiode".into(), - points, - }], - } - } - - fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.connection) { - (Some(reason), _) => format!("locked ({reason})"), - (None, ConnectionState::Disconnected) => "disconnected".into(), - (None, ConnectionState::Connected) => "connected".into(), - (None, ConnectionState::Acquiring) => "acquiring".into(), - }; - let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { - (Some(estimate), _) => ( - format!("{:.4}", estimate.a), - format!( - "{:.2}% low / {:.2}% high", - estimate.low_clip_fraction * 100.0, - estimate.high_clip_fraction * 100.0 - ), - ), - (None, Some(err)) => ("invalid".into(), err.clone()), - (None, None) => ("—".into(), "—".into()), - }; - let integrity = if self.integrity.is_clean() { - "clean".to_owned() - } else { - format!( - "crc={} gaps={} skipped={} overruns={}", - self.integrity.crc_failures, - self.integrity.sequence_gaps, - self.integrity.skipped_bytes, - self.integrity.dropped_samples - ) - }; - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", state), - text_column("firmware", self.firmware.clone()), - text_column("a", a_text), - text_column("clipping", clip_text), - text_column("integrity", integrity), - text_column("error", self.last_error.clone().unwrap_or_default()), - ], - } - } - - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("firmware", "Firmware"), - column("a", "a = ln(Vmax/Vmin)"), - column("clipping", "Clipping"), - column("integrity", "Stream integrity"), - column("error", "Last error"), - ], - ..TableSchema::default() - } - } - - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec<(String, Value)> { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-monitor.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push((request.action_id, request.params)); - } - consumed - } -} - -fn open_transport(port_hint: &str) -> Result, String> { - let path = resolve_port(port_hint)?; - let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} - -fn resolve_port(port_hint: &str) -> Result { - if port_hint != "auto" { - return Ok(port_hint.to_owned()); - } - let ports = serial_ports(); - ports - .into_iter() - .next() - .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned()) -} - -fn serial_ports() -> Vec { - serialport_names() - .into_iter() - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) - .collect() -} - -fn serialport_names() -> Vec { - stage_a_io::transport::available_port_names() -} - -impl Plugin for StageAMonitorPlugin { - fn name(&self) -> &'static str { - "Stage-A Monitor" - } - - fn description(&self) -> &'static str { - "Live Teensy photodiode readout with calibrated optical contrast and gated manual drive control (commissioning)." - } - - fn enabled(&self) -> bool { - self.enabled - } - - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.disconnect("plugin disabled"); - } - } - - fn reset(&mut self) { - self.sample_ring.clear(); - self.contrast = None; - self.contrast_error = None; - self.bump_generation(); - } - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - // Fail closed: any pass without live-capture effects tears the - // connection down and refuses commands. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.worker.is_some() { - self.disconnect("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for (action_id, params) in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect("operator"), - ACTION_START => self.start_acquisition(), - ACTION_STOP => self.stop_acquisition(), - ACTION_APPLY_DRIVE => self.apply_drive(¶ms), - _ => {} - } - } - - self.drain_worker(); - } - - fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["auto".to_owned()]; - port_variants.extend(serial_ports()); - let port_default = port_variants - .iter() - .position(|p| *p == self.port_hint) - .unwrap_or(0); - SettingsSchema { - sections: vec![SettingsSection { - label: "Device".into(), - description: Some( - "Serial DAQ configuration. Connect/start/stop are actions on the status \ - table, never settings — a reloaded settings file can't arm hardware." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Serial port".into(), - tooltip: Some("Teensy USB serial device (auto = first usbmodem)".into()), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, - }, - }, - SettingItem { - key: "sample_rate_hz".into(), - label: "ADC sample rate".into(), - tooltip: Some("Commanded photodiode sample rate".into()), - kind: SettingKind::I64Slider { - min: 1_000, - max: 100_000, - default: self.sample_rate_hz, - suffix: Some(" Hz".into()), - }, - }, - SettingItem { - key: "dark_millivolts".into(), - label: "Dark level".into(), - tooltip: Some( - "Light-blocked photodiode level; a is computed from dark-corrected \ - voltages" - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 3_300.0, - speed: 1.0, - default: self.calibration.dark_volts * 1_000.0, - }, - }, - ], - }], - } - } - - fn get_setting(&self, key: &str) -> Option { - match key { - "port" => Some(json!(self.port_hint)), - "sample_rate_hz" => Some(json!(self.sample_rate_hz)), - "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); - Ok(()) - } - "sample_rate_hz" => { - self.sample_rate_hz = value - .as_i64() - .ok_or("sample_rate_hz must be an integer")? - .clamp(1_000, 100_000); - Ok(()) - } - "dark_millivolts" => { - let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; - self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); - Ok(()) - } - _ => Err(format!("unknown setting: {key}")), - } - } - - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - entries.push(StatusEntry::Text(match self.connection { - ConnectionState::Disconnected => "Teensy: disconnected".into(), - ConnectionState::Connected => format!("Teensy: connected ({})", self.firmware), - ConnectionState::Acquiring => format!( - "Teensy: acquiring at {} S/s", - if self.sample_rate_seen_hz > 0 { - self.sample_rate_seen_hz as i64 - } else { - self.sample_rate_hz - } - ), - })); - if let Some(estimate) = &self.contrast { - entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); - } - entries - } - - fn host_views(&self) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![ - HostDatasetDescriptor { - id: WAVEFORM_DATASET_ID.into(), - title: "Photodiode waveform".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No photodiode samples yet — connect and start.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "Stage-A monitor status".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Monitor idle.".into(), - display: None, - relations: Vec::new(), - }, - ], - views: vec![ - HostViewDescriptor { - id: WAVEFORM_VIEW_ID.into(), - title: "Photodiode".into(), - dataset_id: WAVEFORM_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: STATUS_VIEW_ID.into(), - title: "Monitor status".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - ], - actions: vec![ - HostActionDescriptor { - id: ACTION_CONNECT.into(), - title: "Connect".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_DISCONNECT.into(), - title: "Disconnect".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_START.into(), - title: "Start acquisition".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_STOP.into(), - title: "Stop".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_APPLY_DRIVE.into(), - title: "Apply drive (expert)".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: serde_json::to_value(SettingsSchema { - sections: vec![SettingsSection { - label: "Drive".into(), - description: Some( - "Integer DAC drive codes — the optical contrast is measured \ - from the photodiode, never assumed from these values." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "freq_mhz".into(), - label: "Frequency".into(), - tooltip: Some("Drive frequency in millihertz".into()), - kind: SettingKind::I64Drag { - min: 0, - max: 200_000_000, - default: 1_000_000, - }, - }, - SettingItem { - key: "center_dac".into(), - label: "Center DAC code".into(), - tooltip: None, - kind: SettingKind::I64Drag { - min: 0, - max: 4_095, - default: 2_048, - }, - }, - SettingItem { - key: "amplitude_dac".into(), - label: "Amplitude DAC code".into(), - tooltip: None, - kind: SettingKind::I64Drag { - min: 0, - max: 2_047, - default: 0, - }, - }, - ], - }], - }) - .ok(), - }, - ], - } - } - - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, - } - } - - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), - _ => 0, - } - } -} - -impl Drop for StageAMonitorPlugin { - fn drop(&mut self) { - self.disconnect("plugin destroyed"); - } -} - -export_plugin!(StageAMonitorPlugin); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn waveform_dataset_decimates_and_calibrates() { - let mut plugin = StageAMonitorPlugin::default(); - plugin.sample_rate_seen_hz = 20_000; - plugin.push_samples(&vec![2_048_u16; 8_192], 0); - let dataset = plugin.waveform_dataset(); - assert_eq!(dataset.lines.len(), 1); - assert!(dataset.lines[0].points.len() <= WAVEFORM_POINTS + 1); - let volts = dataset.lines[0].points[0].y; - assert!((volts - 2_048.0 * 3.3 / 4_095.0).abs() < 1e-9); - } - - #[test] - fn sample_ring_is_bounded() { - let mut plugin = StageAMonitorPlugin::default(); - plugin.push_samples(&vec![1_u16; SAMPLE_RING_CAPACITY], 0); - plugin.push_samples(&vec![2_u16; 4_096], SAMPLE_RING_CAPACITY as u64); - assert_eq!(plugin.sample_ring.len(), SAMPLE_RING_CAPACITY); - assert_eq!(*plugin.sample_ring.last().unwrap(), 2); - } - - #[test] - fn status_dataset_matches_its_schema() { - let plugin = StageAMonitorPlugin::default(); - let dataset = plugin.status_dataset(); - let schema = plugin.status_schema(); - assert_eq!(dataset.columns.len(), schema.columns.len()); - for (data, column) in dataset.columns.iter().zip(&schema.columns) { - assert_eq!(data.column_id, column.id); - assert_eq!(data.len(), 1); - } - } -} diff --git a/plugins/stage-a-a1/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml similarity index 51% rename from plugins/stage-a-a1/Cargo.toml rename to plugins/stage-a-photodiode/Cargo.toml index fa4841d..b426623 100644 --- a/plugins/stage-a-a1/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "augur-plugin-stage-a-a1" +name = "augur-plugin-stage-a-photodiode" version.workspace = true edition.workspace = true license.workspace = true authors.workspace = true -description = "Stage-A A1 event-native Bode calibration: minimum-depth a_min(f) sweep with phase-locked detection." +description = "Stage-A photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." [lib] crate-type = ["cdylib", "rlib"] @@ -12,4 +12,4 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true serde_json.workspace = true -stage-a-io = { path = "../../stage-a-io" } +serialport.workspace = true diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md new file mode 100644 index 0000000..5f59e82 --- /dev/null +++ b/plugins/stage-a-photodiode/README.md @@ -0,0 +1,25 @@ +# Stage-A Photodiode + +Live readout of the photodiode on **board SMA5 → Teensy pin 18 / analog input A4**, from the +free-running ASCII stream the `stage-a-controller` firmware (0.3.0+) emits on its **second** USB +serial port (`PD code=… n=… t_ms=…` at 50 Hz). The port carries no commands, so this plugin is +read-only by construction; the command port belongs to `stage-a-modulation`. + +## Modes + +- **RAW** — shows the ADC code and its voltage, `V = code · 3.3 / 4095`. +- **EXCITATION** — the photodiode sits in the excitation path behind the PBS and measures the + light *removed* from the beam: `I_pd = I_tot − I_exc`. Given the user-set reference **I_tot** + (in photodiode volts — the reading with the full beam on the diode), the plugin shows + `I_exc = I_tot − I_pd`. + +## Views + +- a live rolling chart (window length settable, 1–120 s) of the value in the selected mode; +- a compact status table with the newest code/value and Connect/Disconnect actions. + +## Ports + +Select the Teensy *stream* port (the second `usbmodem` port). Picking the command port by mistake +is harmless: its binary frames simply parse to nothing (no values appear) — switch to the other +port. `mock` generates a synthetic slow sine for hardware-free testing. diff --git a/plugins/stage-a-photodiode/plugin.toml b/plugins/stage-a-photodiode/plugin.toml new file mode 100644 index 0000000..702a218 --- /dev/null +++ b/plugins/stage-a-photodiode/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Photodiode" +version = "0.3.0" +description = "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." +domain = "stage-a" +library = "augur_plugin_stage_a_photodiode" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs new file mode 100644 index 0000000..8a6f3a7 --- /dev/null +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -0,0 +1,774 @@ +//! Stage-A photodiode readout. +//! +//! Reads the free-running ASCII stream the `stage-a-controller` firmware +//! (0.3.0+, `USB_DUAL_SERIAL`) emits on its **second** USB serial port: +//! one `PD code= n= t_ms=` line every 20 ms from the +//! photodiode on board SMA5 → Teensy pin 18 / A4. The port carries no +//! commands, so opening it is side-effect free; the command port is owned by +//! `stage-a-modulation`. +//! +//! Two display modes: +//! - **RAW**: the ADC code and its voltage (`V = code · 3.3 / 4095`); +//! - **EXCITATION**: the photodiode sits behind the PBS in the excitation +//! path and sees the light removed from the beam, `I_pd = I_tot − I_exc`. +//! Given the user-set reference `I_tot` (in photodiode volts), the plugin +//! shows `I_exc = I_tot − V_pd`. + +use std::collections::VecDeque; +use std::io::Read; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, + Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, + StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, + TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; + +const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; +const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; +const STATUS_DATASET_ID: &str = "stage-a-photodiode.status"; +const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; + +const ACTION_CONNECT: &str = "stage-a-photodiode.connect"; +const ACTION_DISCONNECT: &str = "stage-a-photodiode.disconnect"; + +const ADC_FULL_SCALE_VOLTS: f64 = 3.3; +const ADC_MAX_CODE: f64 = 4_095.0; +/// Ring capacity: > 2.5 minutes at the firmware's 50 lines/s. +const RING_CAPACITY: usize = 8_192; + +fn code_to_volts(code: f64) -> f64 { + code * ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Raw, + Excitation, +} + +impl Mode { + const VARIANTS: [Mode; 2] = [Mode::Raw, Mode::Excitation]; + + fn name(self) -> &'static str { + match self { + Self::Raw => "RAW", + Self::Excitation => "EXCITATION", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|m| m.name() == name) + } +} + +#[derive(Debug, Clone, Copy)] +struct PdSample { + t_ms: u64, + code: f64, +} + +/// Parses one firmware stream line: `PD code= n= t_ms=`. +fn parse_pd_line(line: &str) -> Option { + let rest = line.trim().strip_prefix("PD ")?; + let mut code = None; + let mut t_ms = None; + for token in rest.split_ascii_whitespace() { + let (key, value) = token.split_once('=')?; + match key { + "code" => code = value.parse::().ok(), + "t_ms" => t_ms = value.parse::().ok(), + "n" => {} + _ => return None, + } + } + Some(PdSample { + t_ms: t_ms?, + code: code?.clamp(0.0, ADC_MAX_CODE), + }) +} + +#[derive(Default)] +struct SharedState { + samples: VecDeque, + latest: Option, + error: Option, +} + +impl SharedState { + fn push(&mut self, sample: PdSample) { + self.latest = Some(sample); + self.samples.push_back(sample); + while self.samples.len() > RING_CAPACITY { + self.samples.pop_front(); + } + } +} + +/// Background reader owning the stream port (or the mock generator). +struct Reader { + stop: Arc, + join: Option>, +} + +impl Reader { + fn spawn_serial( + path: String, + shared: Arc>, + generation: Arc, + ) -> Result { + let port = serialport::new(&path, 115_200) + .timeout(Duration::from_millis(50)) + .open() + .map_err(|err| format!("open {path}: {err}"))?; + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let join = std::thread::Builder::new() + .name("stage-a-photodiode".into()) + .spawn(move || read_lines(port, &shared, &generation, &thread_stop)) + .expect("spawning the photodiode reader thread must succeed"); + Ok(Self { + stop, + join: Some(join), + }) + } + + /// Hardware-free source: synthesizes a slow sine around 1 V at 50 Hz. + fn spawn_mock(shared: Arc>, generation: Arc) -> Self { + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let join = std::thread::Builder::new() + .name("stage-a-photodiode-mock".into()) + .spawn(move || { + let start = Instant::now(); + while !thread_stop.load(Ordering::Relaxed) { + let t = start.elapsed().as_secs_f64(); + let volts = 1.0 + 0.5 * (2.0 * std::f64::consts::PI * 0.2 * t).sin(); + let sample = PdSample { + t_ms: (t * 1_000.0) as u64, + code: volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS, + }; + if let Ok(mut state) = shared.lock() { + state.push(sample); + } + generation.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(Duration::from_millis(20)); + } + }) + .expect("spawning the mock photodiode thread must succeed"); + Self { + stop, + join: Some(join), + } + } +} + +impl Drop for Reader { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +fn read_lines( + mut port: Box, + shared: &Mutex, + generation: &AtomicU64, + stop: &AtomicBool, +) { + let mut line_buffer: Vec = Vec::with_capacity(256); + let mut buf = [0_u8; 512]; + while !stop.load(Ordering::Relaxed) { + let read = match port.read(&mut buf) { + Ok(0) => continue, + Ok(read) => read, + Err(err) if err.kind() == std::io::ErrorKind::TimedOut => continue, + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) => { + if let Ok(mut state) = shared.lock() { + state.error = Some(format!("stream read failed: {err}")); + } + generation.fetch_add(1, Ordering::Relaxed); + return; + } + }; + line_buffer.extend_from_slice(&buf[..read]); + // Never let garbage (e.g. the wrong, binary port) grow the buffer. + if line_buffer.len() > 4_096 { + line_buffer.clear(); + } + while let Some(pos) = line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = line_buffer.drain(..=pos).collect(); + let Ok(text) = std::str::from_utf8(&line) else { + continue; + }; + if let Some(sample) = parse_pd_line(text) { + if let Ok(mut state) = shared.lock() { + state.push(sample); + } + generation.fetch_add(1, Ordering::Relaxed); + } + } + } +} + +pub struct StageAPhotodiodePlugin { + enabled: bool, + reader: Option, + shared: Arc>, + generation: Arc, + effects_blocked_reason: Option, + last_error: Option, + // -- settings -- + port_hint: String, + mode: Mode, + reference_volts: f64, + window_s: f64, + consumed_action_ids: Vec, +} + +impl Default for StageAPhotodiodePlugin { + fn default() -> Self { + Self { + enabled: false, + reader: None, + shared: Arc::new(Mutex::new(SharedState::default())), + generation: Arc::new(AtomicU64::new(1)), + effects_blocked_reason: None, + last_error: None, + port_hint: "mock".into(), + mode: Mode::Raw, + reference_volts: 3.3, + window_s: 10.0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAPhotodiodePlugin { + fn connected(&self) -> bool { + self.reader.is_some() + } + + fn connect(&mut self) { + if self.reader.is_some() { + return; + } + if let Ok(mut state) = self.shared.lock() { + *state = SharedState::default(); + } + self.last_error = None; + if self.port_hint == "mock" { + self.reader = Some(Reader::spawn_mock( + Arc::clone(&self.shared), + Arc::clone(&self.generation), + )); + return; + } + let path = if self.port_hint == "auto" { + match serial_ports().into_iter().next() { + Some(path) => path, + None => { + self.last_error = + Some("no USB serial device found (looked for usbmodem/ttyACM)".into()); + return; + } + } + } else { + self.port_hint.clone() + }; + match Reader::spawn_serial(path, Arc::clone(&self.shared), Arc::clone(&self.generation)) { + Ok(reader) => self.reader = Some(reader), + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + + fn disconnect(&mut self) { + self.reader = None; // Drop joins the thread. + self.generation.fetch_add(1, Ordering::Relaxed); + } + + /// Value shown for one sample under the current mode, in volts. + fn display_volts(&self, code: f64) -> f64 { + match self.mode { + Mode::Raw => code_to_volts(code), + Mode::Excitation => self.reference_volts - code_to_volts(code), + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-photodiode.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } + + fn series_dataset(&self) -> Series1dV1 { + let (points, y_label) = match self.shared.lock() { + Ok(state) => { + let latest_ms = state.latest.map_or(0, |s| s.t_ms); + let window_ms = (self.window_s.max(0.5) * 1_000.0) as u64; + let cutoff = latest_ms.saturating_sub(window_ms); + let points: Vec = state + .samples + .iter() + .filter(|s| s.t_ms >= cutoff) + .map(|s| Series1dPoint { + x: (s.t_ms as f64 - latest_ms as f64) / 1_000.0, + y: self.display_volts(s.code), + }) + .collect(); + let label = match self.mode { + Mode::Raw => "photodiode [V]", + Mode::Excitation => "excitation I_tot − I_pd [V]", + }; + (points, label) + } + Err(_) => (Vec::new(), "photodiode [V]"), + }; + Series1dV1 { + x_label: "time before now [s]".into(), + y_label: y_label.into(), + lines: vec![Series1dLine { + name: match self.mode { + Mode::Raw => "photodiode".into(), + Mode::Excitation => "excitation".into(), + }, + points, + }], + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let (latest, stream_error) = match self.shared.lock() { + Ok(state) => (state.latest, state.error.clone()), + Err(_) => (None, None), + }; + let state = match (&self.effects_blocked_reason, self.connected()) { + (Some(reason), _) => format!("locked ({reason})"), + (None, false) => "disconnected".into(), + (None, true) => format!("reading ({})", self.port_hint), + }; + let (code_text, value_text) = match latest { + Some(sample) => ( + format!("{:.1}", sample.code), + format!("{:.4} V", self.display_volts(sample.code)), + ), + None => ("—".into(), "—".into()), + }; + let error = stream_error + .or_else(|| self.last_error.clone()) + .unwrap_or_default(); + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("mode", self.mode.name().to_owned()), + text_column("code", code_text), + text_column("value", value_text), + text_column("error", error), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("mode", "Mode"), + column("code", "ADC code"), + column("value", "Value"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } +} + +fn serial_ports() -> Vec { + serialport::available_ports() + .map(|ports| { + ports + .into_iter() + .map(|p| p.port_name) + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() + }) + .unwrap_or_default() +} + +impl Plugin for StageAPhotodiodePlugin { + fn name(&self) -> &'static str { + "Stage-A Photodiode" + } + + fn description(&self) -> &'static str { + "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot − I_pd with a user-set reference." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect(); + } + } + + fn reset(&mut self) { + if let Ok(mut state) = self.shared.lock() { + state.samples.clear(); + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // The stream port is read-only, but device access still follows the + // same fail-closed gate as every stage-a plugin. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.reader.is_some() { + self.disconnect(); + } + return; + } + self.effects_blocked_reason = None; + + for action_id in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect(), + _ => {} + } + } + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + let mode_variants: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let mode_default = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Photodiode readout".into(), + description: Some( + "Reads the free-running PD stream on the Teensy's SECOND serial port. \ + EXCITATION shows I_exc = I_tot − I_pd: the diode sits behind the PBS and \ + sees the light removed from the excitation beam." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "Teensy stream port (the SECOND usbmodem port); mock = synthetic \ + data, auto = first device" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some( + "RAW: ADC code and volts as measured. EXCITATION: I_tot − I_pd".into(), + ), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, + }, + SettingItem { + key: "reference_volts".into(), + label: "Reference I_tot".into(), + tooltip: Some( + "Total power reference for EXCITATION mode, in photodiode volts: \ + the PD reading with the full beam diverted into the diode" + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: ADC_FULL_SCALE_VOLTS, + speed: 0.01, + default: self.reference_volts, + }, + }, + SettingItem { + key: "window_s".into(), + label: "Chart window".into(), + tooltip: Some("Seconds of history shown in the live chart".into()), + kind: SettingKind::F64Drag { + min: 1.0, + max: 120.0, + speed: 1.0, + default: self.window_s, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "mode" => Some(json!(self.mode.name())), + "reference_volts" => Some(json!(self.reference_volts)), + "window_s" => Some(json!(self.window_s)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "mode" => { + let name = value.as_str().ok_or("mode must be a string")?; + self.mode = Mode::from_name(name) + .ok_or_else(|| format!("unknown mode: {name} (RAW/EXCITATION)"))?; + Ok(()) + } + "reference_volts" => { + let volts = value.as_f64().ok_or("reference_volts must be a number")?; + self.reference_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); + Ok(()) + } + "window_s" => { + let seconds = value.as_f64().ok_or("window_s must be a number")?; + self.window_s = seconds.clamp(1.0, 120.0); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + let (latest, stream_error) = match self.shared.lock() { + Ok(state) => (state.latest, state.error.clone()), + Err(_) => (None, None), + }; + entries.push(StatusEntry::Text(if self.connected() { + format!("Photodiode: reading ({})", self.port_hint) + } else { + "Photodiode: disconnected".into() + })); + if let Some(sample) = latest { + match self.mode { + Mode::Raw => entries.push(StatusEntry::Text(format!( + "PD: code={:.1} ({:.4} V)", + sample.code, + code_to_volts(sample.code) + ))), + Mode::Excitation => entries.push(StatusEntry::Text(format!( + "Excitation: {:.4} V (I_tot={:.3} V, PD={:.4} V)", + self.display_volts(sample.code), + self.reference_volts, + code_to_volts(sample.code) + ))), + } + } + if let Some(error) = stream_error.or_else(|| self.last_error.clone()) { + entries.push(StatusEntry::Text(format!("Error: {error}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + let action = |id: &str, title: &str| HostActionDescriptor { + id: id.into(), + title: title.into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }; + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: SERIES_DATASET_ID.into(), + title: "Photodiode trace".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No photodiode samples yet — connect the stream port.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Photodiode readout".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Photodiode readout idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: SERIES_VIEW_ID.into(), + title: "Photodiode".into(), + dataset_id: SERIES_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Photodiode readout".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + action(ACTION_CONNECT, "Connect"), + action(ACTION_DISCONNECT, "Disconnect"), + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + SERIES_DATASET_ID => serde_json::to_vec(&self.series_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + SERIES_DATASET_ID | STATUS_DATASET_ID => self.generation.load(Ordering::Relaxed).max(1), + _ => 0, + } + } +} + +export_plugin!(StageAPhotodiodePlugin); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_firmware_stream_lines() { + let sample = parse_pd_line("PD code=1042.3 n=16 t_ms=123456\n").expect("valid line"); + assert!((sample.code - 1042.3).abs() < 1e-9); + assert_eq!(sample.t_ms, 123_456); + + assert!(parse_pd_line("garbage").is_none()); + assert!(parse_pd_line("PD code=abc n=16 t_ms=1").is_none()); + assert!(parse_pd_line("PD code=10 n=16").is_none(), "t_ms required"); + // Codes are clamped into the 12-bit range. + let clamped = parse_pd_line("PD code=9999 n=1 t_ms=5").expect("parses"); + assert_eq!(clamped.code, ADC_MAX_CODE); + } + + #[test] + fn excitation_mode_inverts_against_the_reference() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("mode", json!("EXCITATION")).unwrap(); + plugin.set_setting("reference_volts", json!(2.0)).unwrap(); + // I_pd = 0.5 V → I_exc = I_tot − I_pd = 1.5 V. + let code = 0.5 * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS; + assert!((plugin.display_volts(code) - 1.5).abs() < 1e-9); + // RAW mode shows the measured voltage itself. + plugin.set_setting("mode", json!("RAW")).unwrap(); + assert!((plugin.display_volts(code) - 0.5).abs() < 1e-9); + } + + #[test] + fn mock_reader_fills_the_ring_and_series() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.connect(); + let deadline = Instant::now() + Duration::from_secs(2); + loop { + let count = plugin.shared.lock().unwrap().samples.len(); + if count >= 5 { + break; + } + assert!(Instant::now() < deadline, "mock reader produced no data"); + std::thread::sleep(Duration::from_millis(5)); + } + let series = plugin.series_dataset(); + assert!(!series.lines[0].points.is_empty()); + let generation = plugin.generation.load(Ordering::Relaxed); + assert!(generation > 1); + plugin.disconnect(); + } + + #[test] + fn ring_is_bounded() { + let mut state = SharedState::default(); + for i in 0..(RING_CAPACITY + 100) { + state.push(PdSample { + t_ms: i as u64, + code: 1.0, + }); + } + assert_eq!(state.samples.len(), RING_CAPACITY); + assert_eq!(state.latest.unwrap().t_ms, (RING_CAPACITY + 99) as u64); + } +} diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs index 2a21ff7..2a9eabd 100644 --- a/stage-a-io/src/lib.rs +++ b/stage-a-io/src/lib.rs @@ -1,7 +1,8 @@ //! # stage-a-io //! -//! Shared research-owned I/O library for the Stage-A camera-calibration -//! plugins (`stage-a-monitor`, `stage-a-a1`, `stage-a-a2`, `stage-a-a3`). +//! Shared research-owned I/O library for the Stage-A bench plugins +//! (currently `stage-a-modulation`; the future A1–A3 experiment plugins +//! build on it too — see ADR 006). //! //! Scope, per the Stage-A control-software specification: //! - the v1 ASCII command grammar and PDA1 binary frame format (wire- diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs index 803d919..203733c 100644 --- a/stage-a-io/src/mock.rs +++ b/stage-a-io/src/mock.rs @@ -1,12 +1,13 @@ //! Mock Stage-A controller for tests and hardware-free plugin development. //! -//! Mirrors firmware 0.2.0 (`stage-a-controller/src/main.cpp`) faithfully: -//! the same verbs (`HELLO`, `STATUS`, `CONFIG`, `START`, `STOP`, `PING`), -//! the same state machine (`SAFE_IDLE` → `CONFIGURED` → `RUNNING`), the -//! same error codes/details (`PROTOCOL`, `RANGE`, `STATE`, `SYNTAX`, +//! Mirrors firmware 0.3.0 (`stage-a-controller/src/main.cpp`) faithfully: +//! the same verbs (`HELLO`, `STATUS`, `CONFIG`, `START`, `STOP`, `PING`, +//! `MOD`), the same state machine (`SAFE_IDLE` → `CONFIGURED` → `RUNNING`), +//! the same error codes/details (`PROTOCOL`, `RANGE`, `STATE`, `SYNTAX`, //! `VERB`), the same single-entry idempotent reply cache, and rejection of //! unknown `CONFIG` fields — which is the host's feature-detection -//! mechanism, so it must never be papered over here. +//! mechanism, so it must never be papered over here. `MOD` is set-and-hold +//! exactly like the firmware: `STOP` does not touch the modulation state. //! //! [`MockController::with_waveform_extension`] additionally models the //! *proposed* v2 waveform firmware (`stage-a-controller/docs/features/` @@ -24,6 +25,9 @@ pub const MOCK_MAX_RATE_HZ: u32 = 100_000; pub const MOCK_MAX_BLOCK_SAMPLES: u32 = 256; /// Proposed v2 waveform ceiling (matches the drive UI bound: 200 kHz). pub const MOCK_MAX_FREQ_MHZ: u32 = 200_000_000; +/// Firmware 0.3.0 `MOD` frequency window (`board_config.h`). +pub const MOCK_MOD_MIN_FREQ_MHZ: u32 = 10; +pub const MOCK_MOD_MAX_FREQ_MHZ: u32 = 2_000_000; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MockState { @@ -111,6 +115,12 @@ pub struct MockController { out_sequence: u32, line_buffer: Vec, sample_index: u64, + // Firmware 0.3.0 MOD state (set-and-hold, independent of acquisition). + mod_wave: &'static str, + mod_level: u32, + mod_min: u32, + mod_freq_mhz: u32, + mod_code: u32, /// Synthetic optics for [`MockController::emit_configured_block`]: /// photodiode code = dark + span * sin²(π/2 · drive/4095). pub synth_dark_code: f64, @@ -134,6 +144,11 @@ impl MockController { out_sequence: 0, line_buffer: Vec::new(), sample_index: 0, + mod_wave: "OFF", + mod_level: 0, + mod_min: 0, + mod_freq_mhz: 0, + mod_code: 0, synth_dark_code: 40.0, synth_span_codes: 3_800.0, synth_center: 2_048.0, @@ -273,19 +288,20 @@ impl MockController { return format!("-{sequence} ERR code=PROTOCOL detail=requires_v1"); } let capabilities = if self.waveform_extension { - " capabilities=A1,A2,A3,WAVE" + " capabilities=MOD,PDSTREAM,WAVE" } else { - "" + " capabilities=MOD,PDSTREAM" }; format!( - "+{sequence} OK protocol=1 firmware=0.2.0-mock board=MOCK adc_bits=12 \ + "+{sequence} OK protocol=1 firmware=0.3.0-mock board=MOCK adc_bits=12 \ max_rate_hz={MOCK_MAX_RATE_HZ} dac=AD5628 dac_bus=SPI1 dac_cs=29 \ - dac_channel=1.4 dac_address=3{capabilities}" + dac_channel=1.4 dac_address=3 pd_pin=A4{capabilities}" ) } "STATUS" => format!( "+{sequence} OK state={} mode={} rate_hz={} block_samples={} raw={} summary={} \ - sample_index={} dropped=0 marker_drops=0 dac=1.4/3 code=0", + sample_index={} dropped=0 marker_drops=0 dac=1.4/3 code={} mod_wave={} \ + mod_level={} mod_min={} mod_freq_mhz={}", self.state.name(), self.config.mode, self.config.rate_hz, @@ -293,8 +309,14 @@ impl MockController { u8::from(self.config.raw), u8::from(self.config.summary), self.sample_index, + self.mod_code, + self.mod_wave, + self.mod_level, + self.mod_min, + self.mod_freq_mhz, ), "CONFIG" => self.execute_config(fields, sequence), + "MOD" => self.execute_mod(fields, sequence), "START" => { if self.state != MockState::Configured { return format!("-{sequence} ERR code=STATE detail=configure_before_start"); @@ -403,6 +425,86 @@ impl MockController { ) } + /// Firmware 0.3.0 `MOD` handler: same field grammar, validation order, + /// error details, and reply shape as `main.cpp`. + fn execute_mod(&mut self, fields: &[(String, String)], sequence: u32) -> String { + let err = |code: &str, detail: &str| format!("-{sequence} ERR code={code} detail={detail}"); + let mut wave: Option<&'static str> = None; + let mut level = 0_u32; + let mut saw_level = false; + let mut min_level = 0_u32; + let mut freq_mhz = 0_u32; + let mut saw_freq = false; + for (key, value) in fields { + match key.as_str() { + "wave" => { + wave = Some(match value.as_str() { + "OFF" => "OFF", + "CONST" => "CONST", + "SINE" => "SINE", + "SQUARE" => "SQUARE", + _ => return err("RANGE", "invalid_wave"), + }); + } + "level" => match value.parse::() { + Ok(parsed) if parsed <= 4_095 => { + level = parsed; + saw_level = true; + } + _ => return err("RANGE", "invalid_level"), + }, + "min" => match value.parse::() { + Ok(parsed) if parsed <= 4_095 => min_level = parsed, + _ => return err("RANGE", "invalid_min"), + }, + "freq_mhz" => match value.parse::() { + Ok(parsed) => { + freq_mhz = parsed; + saw_freq = true; + } + _ => return err("RANGE", "invalid_freq_mhz"), + }, + _ => return err("SYNTAX", "unknown_mod_field"), + } + } + let Some(wave) = wave else { + return err("SYNTAX", "wave_required"); + }; + let periodic = wave == "SINE" || wave == "SQUARE"; + if wave != "OFF" && !saw_level { + return err("SYNTAX", "level_required"); + } + if periodic && !saw_freq { + return err("SYNTAX", "freq_mhz_required"); + } + if min_level > level { + return err("RANGE", "min_above_level"); + } + if periodic && !(MOCK_MOD_MIN_FREQ_MHZ..=MOCK_MOD_MAX_FREQ_MHZ).contains(&freq_mhz) { + return err("RANGE", "mod_rejected"); + } + if wave == "OFF" { + level = 0; + min_level = 0; + freq_mhz = 0; + } + self.mod_wave = wave; + self.mod_level = level; + self.mod_min = if wave == "CONST" { level } else { min_level }; + self.mod_freq_mhz = if periodic { freq_mhz } else { 0 }; + // Same initial output as the firmware engine: CONST/OFF hold level, + // square starts low, sine starts at the center. + self.mod_code = match wave { + "SQUARE" => self.mod_min, + "SINE" => (self.mod_min + self.mod_level) / 2, + _ => level, + }; + format!( + "+{sequence} OK mod_wave={} mod_level={} mod_min={} mod_freq_mhz={} code={}", + self.mod_wave, self.mod_level, self.mod_min, self.mod_freq_mhz, self.mod_code + ) + } + fn send_control(&mut self, payload: &str) { let frame = self.build_frame(FrameType::Control, payload.as_bytes().to_vec(), 0, 0); let bytes = frame.to_bytes(); @@ -615,20 +717,64 @@ mod tests { } #[test] - fn hello_requires_protocol_v1_and_advertises_capabilities_only_with_extension() { + fn hello_requires_protocol_v1_and_advertises_capabilities() { let link = MockLink::new(); let mut host = link.host_end(); let mut controller = MockController::new(link.device_end()); request(&mut controller, "@1 HELLO"); assert!(last_control_text(&mut host).contains("code=PROTOCOL detail=requires_v1")); request(&mut controller, "@2 HELLO protocol=1"); - assert!(!last_control_text(&mut host).contains("capabilities")); + assert!(last_control_text(&mut host).contains("capabilities=MOD,PDSTREAM")); let link = MockLink::new(); let mut host = link.host_end(); let mut controller = MockController::new(link.device_end()).with_waveform_extension(); request(&mut controller, "@1 HELLO protocol=1"); - assert!(last_control_text(&mut host).contains("capabilities=A1,A2,A3,WAVE")); + assert!(last_control_text(&mut host).contains("capabilities=MOD,PDSTREAM,WAVE")); + } + + #[test] + fn mod_command_validates_and_holds_across_stop() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + // Validation mirrors the firmware error details. + request(&mut controller, "@1 MOD level=1000"); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=wave_required")); + request(&mut controller, "@2 MOD wave=SINE level=1000"); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=freq_mhz_required")); + request( + &mut controller, + "@3 MOD wave=SQUARE level=100 min=200 freq_mhz=1000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=min_above_level")); + request( + &mut controller, + "@4 MOD wave=SINE level=1000 freq_mhz=99000000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=mod_rejected")); + + // CONST applies immediately; STATUS echoes it; STOP does not clear it. + request(&mut controller, "@5 MOD wave=CONST level=1234"); + assert!(last_control_text(&mut host) + .contains("mod_wave=CONST mod_level=1234 mod_min=1234 mod_freq_mhz=0 code=1234")); + request(&mut controller, "@6 STOP"); + request(&mut controller, "@7 STATUS"); + let status = last_control_text(&mut host); + assert!(status.contains("code=1234"), "{status}"); + assert!(status.contains("mod_wave=CONST"), "{status}"); + + // Square starts at the min threshold; OFF drops to zero. + request( + &mut controller, + "@8 MOD wave=SQUARE level=2000 min=500 freq_mhz=10000", + ); + assert!(last_control_text(&mut host) + .contains("mod_wave=SQUARE mod_level=2000 mod_min=500 mod_freq_mhz=10000 code=500")); + request(&mut controller, "@9 MOD wave=OFF"); + assert!(last_control_text(&mut host) + .contains("mod_wave=OFF mod_level=0 mod_min=0 mod_freq_mhz=0 code=0")); } #[test] From 73f4ce18e2fb167b485a6885645ede35c5e45750 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 10:31:24 +0200 Subject: [PATCH 11/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20exchange?= =?UTF-8?q?=20enum=20settings=20as=20indices=20so=20radio=20buttons=20appl?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host settings UI reads enum values with as_u64() and writes the selected variant index; the modulation/photodiode plugins returned and expected variant name strings, so port and mode radio buttons could never be set. Map indices against the schema's variant list in get_setting/set_setting (names still accepted) and pin the contract with round-trip tests. --- plugins/stage-a-modulation/src/lib.rs | 78 ++++++++++++++++++++++++--- plugins/stage-a-photodiode/src/lib.rs | 77 +++++++++++++++++++++++--- 2 files changed, 141 insertions(+), 14 deletions(-) diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index e5b163a..4c5d06f 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -420,6 +420,29 @@ fn serial_ports() -> Vec { .collect() } +/// The exact variant list the settings schema shows for the port enum — the +/// host exchanges enum settings as indices into this list. +fn port_variants() -> Vec { + let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; + variants.extend(serial_ports()); + variants +} + +/// Host enum widgets send the selected index; string names are also accepted +/// (tests, saved configs). +fn enum_choice(value: &Value, variants: &[String]) -> Result { + if let Some(index) = value.as_u64() { + return variants + .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) + .cloned() + .ok_or_else(|| format!("enum index {index} out of range")); + } + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| "expected an enum index or name".to_owned()) +} + impl Plugin for StageAModulationPlugin { fn name(&self) -> &'static str { "Stage-A Modulation" @@ -486,8 +509,7 @@ impl Plugin for StageAModulationPlugin { } fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; - port_variants.extend(serial_ports()); + let port_variants = port_variants(); let port_default = port_variants .iter() .position(|p| *p == self.port_hint) @@ -593,10 +615,24 @@ impl Plugin for StageAModulationPlugin { fn get_setting(&self, key: &str) -> Option { match key { - "port" => Some(json!(self.port_hint)), + // Enum settings are exchanged as indices into the schema's + // variant list (see the host settings UI). + "port" => { + let index = port_variants() + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + Some(json!(index)) + } "level" => Some(json!(self.level)), "max_level" => Some(json!(self.max_level)), - "mode" => Some(json!(self.mode.name())), + "mode" => { + let index = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + Some(json!(index)) + } "frequency_hz" => Some(json!(self.frequency_hz)), "min_level" => Some(json!(self.min_level)), _ => None, @@ -606,7 +642,7 @@ impl Plugin for StageAModulationPlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + self.port_hint = enum_choice(&value, &port_variants())?; Ok(()) } "level" => { @@ -636,8 +672,10 @@ impl Plugin for StageAModulationPlugin { Ok(()) } "mode" => { - let name = value.as_str().ok_or("mode must be a string")?; - self.mode = Mode::from_name(name) + let mode_names: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let name = enum_choice(&value, &mode_names)?; + self.mode = Mode::from_name(&name) .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; self.dirty = true; Ok(()) @@ -831,6 +869,32 @@ mod tests { plugin.disconnect("test done"); } + /// The host settings UI exchanges enum values as indices into the + /// schema's variant list (radio buttons send `json!(index)`). + #[test] + fn enum_settings_round_trip_as_indices() { + let mut plugin = StageAModulationPlugin::default(); + // Mode: index 2 = SQUARE in the schema's variant order. + plugin + .set_setting("mode", json!(2)) + .expect("index accepted"); + assert_eq!(plugin.mode, Mode::Square); + assert_eq!(plugin.get_setting("mode"), Some(json!(2))); + // Port: index 1 = "auto" (variants start with mock, auto). + plugin + .set_setting("port", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.port_hint, "auto"); + assert_eq!(plugin.get_setting("port"), Some(json!(1))); + // Out-of-range indices are visible errors, not silent no-ops. + assert!(plugin.set_setting("mode", json!(99)).is_err()); + // String names keep working (tests, saved configs). + plugin + .set_setting("mode", json!("SINE")) + .expect("name accepted"); + assert_eq!(plugin.mode, Mode::Sine); + } + /// min_level can never exceed the level. #[test] fn min_threshold_is_clamped_to_level() { diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 8a6f3a7..c9d1699 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -431,6 +431,29 @@ fn serial_ports() -> Vec { .unwrap_or_default() } +/// The exact variant list the settings schema shows for the port enum — the +/// host exchanges enum settings as indices into this list. +fn port_variants() -> Vec { + let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; + variants.extend(serial_ports()); + variants +} + +/// Host enum widgets send the selected index; string names are also accepted +/// (tests, saved configs). +fn enum_choice(value: &Value, variants: &[String]) -> Result { + if let Some(index) = value.as_u64() { + return variants + .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) + .cloned() + .ok_or_else(|| format!("enum index {index} out of range")); + } + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| "expected an enum index or name".to_owned()) +} + impl Plugin for StageAPhotodiodePlugin { fn name(&self) -> &'static str { "Stage-A Photodiode" @@ -490,8 +513,7 @@ impl Plugin for StageAPhotodiodePlugin { } fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; - port_variants.extend(serial_ports()); + let port_variants = port_variants(); let port_default = port_variants .iter() .position(|p| *p == self.port_hint) @@ -570,8 +592,22 @@ impl Plugin for StageAPhotodiodePlugin { fn get_setting(&self, key: &str) -> Option { match key { - "port" => Some(json!(self.port_hint)), - "mode" => Some(json!(self.mode.name())), + // Enum settings are exchanged as indices into the schema's + // variant list (see the host settings UI). + "port" => { + let index = port_variants() + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + Some(json!(index)) + } + "mode" => { + let index = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + Some(json!(index)) + } "reference_volts" => Some(json!(self.reference_volts)), "window_s" => Some(json!(self.window_s)), _ => None, @@ -581,12 +617,14 @@ impl Plugin for StageAPhotodiodePlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + self.port_hint = enum_choice(&value, &port_variants())?; Ok(()) } "mode" => { - let name = value.as_str().ok_or("mode must be a string")?; - self.mode = Mode::from_name(name) + let mode_names: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let name = enum_choice(&value, &mode_names)?; + self.mode = Mode::from_name(&name) .ok_or_else(|| format!("unknown mode: {name} (RAW/EXCITATION)"))?; Ok(()) } @@ -759,6 +797,31 @@ mod tests { plugin.disconnect(); } + /// The host settings UI exchanges enum values as indices into the + /// schema's variant list (radio buttons send `json!(index)`). + #[test] + fn enum_settings_round_trip_as_indices() { + let mut plugin = StageAPhotodiodePlugin::default(); + // Mode: index 1 = EXCITATION. + plugin + .set_setting("mode", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.mode, Mode::Excitation); + assert_eq!(plugin.get_setting("mode"), Some(json!(1))); + // Port: index 1 = "auto" (variants start with mock, auto). + plugin + .set_setting("port", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.port_hint, "auto"); + assert_eq!(plugin.get_setting("port"), Some(json!(1))); + assert!(plugin.set_setting("mode", json!(99)).is_err()); + // String names keep working (tests, saved configs). + plugin + .set_setting("mode", json!("RAW")) + .expect("name accepted"); + assert_eq!(plugin.mode, Mode::Raw); + } + #[test] fn ring_is_bounded() { let mut state = SharedState::default(); From bcb2dfcf84d7e35b747f21280f90df58595e909e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 10:43:56 +0200 Subject: [PATCH 12/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20auto-detect?= =?UTF-8?q?=20the=20correct=20Teensy=20port=20in=20both=20plugins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit auto in stage-a-modulation now probes each attached port with HELLO and picks the command port; auto in stage-a-photodiode listens for PD lines and picks the stream port. Filter macOS port lists to the cu.* callout nodes so each device appears once. Verified against the live Teensy (firmware 0.3.0). --- plugins/stage-a-modulation/README.md | 8 +-- plugins/stage-a-modulation/src/lib.rs | 52 +++++++++++++++----- plugins/stage-a-photodiode/README.md | 7 +-- plugins/stage-a-photodiode/src/lib.rs | 70 ++++++++++++++++++++++++--- 4 files changed, 110 insertions(+), 27 deletions(-) diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index 9e21a88..1b3f233 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -24,10 +24,10 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** ## Ports -Select the Teensy *command* port (binary protocol), not the photodiode stream port. `mock` runs an -in-process simulated controller for hardware-free testing; `auto` picks the first -usbmodem/ttyACM device. If you picked the wrong physical port, HELLO simply times out — pick the -other one. +**Use `auto` (default recommendation):** it probes every attached usbmodem/ttyACM device and +connects to the one that answers `HELLO` — that is always the Teensy command port, never the +photodiode stream port. Explicit ports remain selectable; `mock` runs an in-process simulated +controller for hardware-free testing. Hardware commands only flow while the host execution context allows effects (live capture); otherwise the connection is torn down and the panel shows the lock reason. diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 4c5d06f..a31f72a 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -399,24 +399,51 @@ impl StageAModulationPlugin { } fn open_serial(port_hint: &str) -> Result, String> { - let path = if port_hint == "auto" { - serial_ports() - .into_iter() - .next() - .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? - } else { - port_hint.to_owned() - }; + if port_hint == "auto" { + // The dual-serial Teensy enumerates two ports and only the command + // port answers HELLO — probe until one does. + let candidates = serial_ports(); + if candidates.is_empty() { + return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + } + let mut failures = Vec::new(); + for path in &candidates { + match probe_command_port(path) { + // Restore the client's default reply timeout after probing. + Ok(client) => return Ok(client.with_reply_timeout(Duration::from_millis(500))), + Err(err) => failures.push(format!("{path}: {err}")), + } + } + return Err(format!( + "no Teensy command port answered HELLO ({})", + failures.join("; ") + )); + } + open_path(port_hint) +} + +fn open_path(path: &str) -> Result, String> { let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + stage_a_io::SerialTransport::open(path, 115_200, std::time::Duration::from_millis(20)) .map_err(|err| err.to_string())?; Ok(StageAClient::new(transport)) } +/// Opens `path` and sends HELLO with a short timeout: only the Teensy +/// command port replies (the photodiode stream port never answers). +fn probe_command_port(path: &str) -> Result, String> { + let mut client = open_path(path)?.with_reply_timeout(Duration::from_millis(300)); + client + .request(&Command::new("HELLO").field("protocol", 1)) + .map_err(|err| err.to_string())?; + Ok(client) +} + fn serial_ports() -> Vec { stage_a_io::transport::available_port_names() .into_iter() - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + // macOS lists each device twice; use the callout (cu.*) node only. + .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) .collect() } @@ -535,8 +562,9 @@ impl Plugin for StageAModulationPlugin { key: "port".into(), label: "Port".into(), tooltip: Some( - "Teensy command port (the FIRST of the two usbmodem ports); \ - mock = in-process simulated controller, auto = first device" + "auto (recommended) probes the attached usbmodem ports and picks \ + the one that answers HELLO — the Teensy command port; \ + mock = in-process simulated controller" .into(), ), kind: SettingKind::Enum { diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md index 5f59e82..af2e277 100644 --- a/plugins/stage-a-photodiode/README.md +++ b/plugins/stage-a-photodiode/README.md @@ -20,6 +20,7 @@ read-only by construction; the command port belongs to `stage-a-modulation`. ## Ports -Select the Teensy *stream* port (the second `usbmodem` port). Picking the command port by mistake -is harmless: its binary frames simply parse to nothing (no values appear) — switch to the other -port. `mock` generates a synthetic slow sine for hardware-free testing. +**Use `auto` (default recommendation):** it listens briefly on every attached usbmodem/ttyACM +device and connects to the one actually streaming `PD` lines — that is always the Teensy stream +port. Picking the command port manually by mistake is harmless: its binary frames parse to +nothing (no values appear). `mock` generates a synthetic slow sine for hardware-free testing. diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index c9d1699..49fb4c9 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -275,11 +275,10 @@ impl StageAPhotodiodePlugin { return; } let path = if self.port_hint == "auto" { - match serial_ports().into_iter().next() { - Some(path) => path, - None => { - self.last_error = - Some("no USB serial device found (looked for usbmodem/ttyACM)".into()); + match resolve_auto_port() { + Ok(path) => path, + Err(err) => { + self.last_error = Some(err); return; } } @@ -425,12 +424,66 @@ fn serial_ports() -> Vec { ports .into_iter() .map(|p| p.port_name) - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + // macOS lists each device twice; use the callout (cu.*) node only. + .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) .collect() }) .unwrap_or_default() } +/// Finds the Teensy stream port: the dual-serial firmware free-runs `PD` +/// lines on exactly one of the enumerated ports, so listen briefly on each. +fn resolve_auto_port() -> Result { + let candidates = serial_ports(); + if candidates.is_empty() { + return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + } + for path in &candidates { + if probe_pd_stream(path) { + return Ok(path.clone()); + } + } + Err(format!( + "no port streamed PD lines within 500 ms (tried {})", + candidates.join(", ") + )) +} + +/// True when `path` produces a parsable `PD …` line within the probe window. +fn probe_pd_stream(path: &str) -> bool { + let Ok(mut port) = serialport::new(path, 115_200) + .timeout(Duration::from_millis(100)) + .open() + else { + return false; + }; + let deadline = Instant::now() + Duration::from_millis(500); + let mut collected: Vec = Vec::new(); + let mut buf = [0_u8; 512]; + while Instant::now() < deadline { + match port.read(&mut buf) { + Ok(read) if read > 0 => { + collected.extend_from_slice(&buf[..read]); + if String::from_utf8_lossy(&collected) + .lines() + .any(|line| parse_pd_line(line).is_some()) + { + return true; + } + if collected.len() > 8_192 { + collected.drain(..4_096); + } + } + Ok(_) => {} + Err(err) + if err.kind() == std::io::ErrorKind::TimedOut + || err.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return false, + } + } + false +} + /// The exact variant list the settings schema shows for the port enum — the /// host exchanges enum settings as indices into this list. fn port_variants() -> Vec { @@ -539,8 +592,9 @@ impl Plugin for StageAPhotodiodePlugin { key: "port".into(), label: "Port".into(), tooltip: Some( - "Teensy stream port (the SECOND usbmodem port); mock = synthetic \ - data, auto = first device" + "auto (recommended) listens on the attached usbmodem ports and \ + picks the one streaming PD lines — the Teensy stream port; \ + mock = synthetic data" .into(), ), kind: SettingKind::Enum { From cc4b435d24f98f387e32c9341952601444098b0a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 10:55:28 +0200 Subject: [PATCH 13/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20label=20port?= =?UTF-8?q?=20choices=20with=20their=20USB=20product=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port entries now read '/dev/cu.usbmodem… (Teensyduino Dual Serial)' so the Teensy is recognisable among the enumerated devices; the parenthesised label is display-only and stripped when the setting is applied. stage-a-io gains available_ports_with_labels() for this. --- plugins/stage-a-modulation/src/lib.rs | 25 ++++++++++++++---- plugins/stage-a-photodiode/src/lib.rs | 32 ++++++++++++++++++++--- stage-a-io/src/transport.rs | 37 +++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 9 deletions(-) diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index a31f72a..b941a9d 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -448,13 +448,28 @@ fn serial_ports() -> Vec { } /// The exact variant list the settings schema shows for the port enum — the -/// host exchanges enum settings as indices into this list. +/// host exchanges enum settings as indices into this list. Real ports carry +/// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; +/// only the leading path is the value. fn port_variants() -> Vec { let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; - variants.extend(serial_ports()); + for (name, label) in stage_a_io::transport::available_ports_with_labels() { + if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { + continue; + } + variants.push(match label { + Some(label) => format!("{name} ({label})"), + None => name, + }); + } variants } +/// The path part of a port variant; the parenthesised USB label is display-only. +fn variant_path(variant: &str) -> &str { + variant.split_whitespace().next().unwrap_or(variant) +} + /// Host enum widgets send the selected index; string names are also accepted /// (tests, saved configs). fn enum_choice(value: &Value, variants: &[String]) -> Result { @@ -539,7 +554,7 @@ impl Plugin for StageAModulationPlugin { let port_variants = port_variants(); let port_default = port_variants .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); let mode_variants: Vec = Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); @@ -648,7 +663,7 @@ impl Plugin for StageAModulationPlugin { "port" => { let index = port_variants() .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); Some(json!(index)) } @@ -670,7 +685,7 @@ impl Plugin for StageAModulationPlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = enum_choice(&value, &port_variants())?; + self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } "level" => { diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 49fb4c9..d6f2928 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -488,10 +488,34 @@ fn probe_pd_stream(path: &str) -> bool { /// host exchanges enum settings as indices into this list. fn port_variants() -> Vec { let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; - variants.extend(serial_ports()); + for port in serialport::available_ports().unwrap_or_default() { + if !(port.port_name.contains("cu.usbmodem") || port.port_name.contains("ttyACM")) { + continue; + } + let label = match port.port_type { + serialport::SerialPortType::UsbPort(info) => match (info.manufacturer, info.product) { + (Some(manufacturer), Some(product)) if !product.starts_with(&manufacturer) => { + Some(format!("{manufacturer} {product}")) + } + (_, Some(product)) => Some(product), + (Some(manufacturer), None) => Some(manufacturer), + (None, None) => None, + }, + _ => None, + }; + variants.push(match label { + Some(label) => format!("{} ({label})", port.port_name), + None => port.port_name, + }); + } variants } +/// The path part of a port variant; the parenthesised USB label is display-only. +fn variant_path(variant: &str) -> &str { + variant.split_whitespace().next().unwrap_or(variant) +} + /// Host enum widgets send the selected index; string names are also accepted /// (tests, saved configs). fn enum_choice(value: &Value, variants: &[String]) -> Result { @@ -569,7 +593,7 @@ impl Plugin for StageAPhotodiodePlugin { let port_variants = port_variants(); let port_default = port_variants .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); let mode_variants: Vec = Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); @@ -651,7 +675,7 @@ impl Plugin for StageAPhotodiodePlugin { "port" => { let index = port_variants() .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); Some(json!(index)) } @@ -671,7 +695,7 @@ impl Plugin for StageAPhotodiodePlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = enum_choice(&value, &port_variants())?; + self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } "mode" => { diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index fc8482a..f7e677a 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -125,3 +125,40 @@ pub fn available_port_names() -> Vec { pub fn available_port_names() -> Vec { Vec::new() } + +/// Port names plus a human-readable USB label (manufacturer/product) where +/// the OS provides one — e.g. `("/dev/cu.usbmodem…", Some("Teensyduino Dual +/// Serial"))`. Lets port pickers show which entry is the Teensy. +#[cfg(feature = "hardware")] +pub fn available_ports_with_labels() -> Vec<(String, Option)> { + serialport::available_ports() + .map(|ports| { + ports + .into_iter() + .map(|p| { + let label = match p.port_type { + serialport::SerialPortType::UsbPort(info) => { + match (info.manufacturer, info.product) { + (Some(manufacturer), Some(product)) + if !product.starts_with(&manufacturer) => + { + Some(format!("{manufacturer} {product}")) + } + (_, Some(product)) => Some(product), + (Some(manufacturer), None) => Some(manufacturer), + (None, None) => None, + } + } + _ => None, + }; + (p.port_name, label) + }) + .collect() + }) + .unwrap_or_default() +} + +#[cfg(not(feature = "hardware"))] +pub fn available_ports_with_labels() -> Vec<(String, Option)> { + Vec::new() +} From 449f758b4d734f0403fe8c0bc2bb119bcc8777b4 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 11:21:25 +0200 Subject: [PATCH 14/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20make=20dev?= =?UTF-8?q?ice=20control=20settings-driven=20so=20it=20works=20without=20c?= =?UTF-8?q?amera=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host only calls process_frame() while camera frames flow, so the action-button + effects-gate design never connected on a camera-less bench (verified: the serial port stayed free while the GUI ran). Connect is now a checkbox setting handled in set_setting, all serial I/O lives in a plugin-owned device thread (slider drags coalesce into one pending MOD), status comes from shared state, and replay mode still disconnects defensively. Verified end-to-end against the live Teensy: auto-probe, level 800 -> board code 800, level 0 -> 0. --- docs/adr/006-stage-a-two-plugin-split.md | 10 +- docs/features/stage-a-modulation.md | 12 +- docs/features/stage-a-photodiode.md | 3 +- plugins/stage-a-modulation/README.md | 17 +- plugins/stage-a-modulation/src/lib.rs | 646 ++++++++++++----------- plugins/stage-a-photodiode/src/lib.rs | 123 ++--- 6 files changed, 406 insertions(+), 405 deletions(-) diff --git a/docs/adr/006-stage-a-two-plugin-split.md b/docs/adr/006-stage-a-two-plugin-split.md index 99971e3..618e348 100644 --- a/docs/adr/006-stage-a-two-plugin-split.md +++ b/docs/adr/006-stage-a-two-plugin-split.md @@ -31,10 +31,12 @@ control plugin's connection. mock — the mock now models firmware 0.3.0's `MOD` verb). The A1/A2/A3 experiment plugins will build on it again when the bench reaches that stage; the estimator/pdq/sidecar modules remain for that purpose even though no current plugin uses them. -4. **Immediate transfer replaces the Apply-action pattern** in `stage-a-modulation`: setting - changes are sent to the device as they happen (the operator's explicit request), still behind - the fail-closed execution-context gate. The firmware output is set-and-hold; the explicit - "Output OFF" action is the only stop. +4. **Immediate transfer replaces the Apply-action pattern**, and **all device control is + settings-driven** (connect checkbox, slider changes sent as they happen). Host actions and + the per-frame effects gate are unsuitable here: the host only runs `process_frame()` while + camera frames flow, but the bench must work with no camera attached (amended 2026-07-16). + Replay mode still disconnects the modulation plugin defensively. The firmware output is + set-and-hold; the power slider at 0 is the off switch. ## Consequences diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index c495d9a..49279fa 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -20,12 +20,14 @@ board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded val - Owns the Teensy **command port** exclusively (one owner per port, ADR 006). The photodiode stream port belongs to `stage-a-photodiode`. -- Uses `stage-a-io` (`StageAClient`, `IoWorker`, `Command`) for framing, idempotent retries, and - the bounded background I/O thread; `process_frame()` never blocks on serial. -- Fail-closed effects gate: the connection only exists while the host execution context allows - hardware effects. +- Uses `stage-a-io` (`StageAClient`, `Command`) for framing and idempotent retries; slider drags + coalesce into a single pending command the device thread drains. +- **Frame-independent**: connecting is a checkbox setting and all serial I/O lives in a + plugin-owned device thread, because the host only calls `process_frame()` while camera frames + flow — bench control must work with no camera attached. `process_frame()` only disconnects + defensively in replay mode. - Firmware output is **set-and-hold** (`stage-a-controller` ADR 002): disconnecting does not stop - the modulation. The explicit **Output OFF** action sends `MOD wave=OFF`. + the modulation. The power slider at 0 is the off switch. - Safety invariants enforced plugin-side: `level ≤ max_level`, `min_level ≤ level`; the firmware waveform peaks at `level` by construction. - `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 976b9a8..916ce8c 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -23,7 +23,8 @@ Two modes: - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the plugin is read-only by construction and needs no protocol library — it depends only on `serialport` and parses one line format. -- Same fail-closed effects gate as the other stage-a plugins for consistent device handling. +- **Frame-independent**: connecting is a checkbox setting; the reader thread and all views + work with no camera attached (the host only calls `process_frame()` while frames flow). - Garbage on the port (e.g. the binary command port picked by mistake) parses to nothing and is bounded — it can neither grow memory nor produce fake values. - `mock` port synthesizes a slow sine for hardware-free testing. diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index 1b3f233..b349ba6 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -14,13 +14,14 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** - The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and a 2 Hz `STATUS` poll), not just what was commanded. -## Actions +## Connecting -- **Connect / Disconnect** — open/close the command port. Connecting never changes the output; - only changes made while connected are transferred. -- **Output OFF** — sends `MOD wave=OFF` (DAC code 0). Needed because the firmware output is - **set-and-hold**: disconnecting, closing the GUI, or a crash leaves the last modulation running - (`stage-a-controller` ADR 002). +- **Connect** is a checkbox in the plugin settings — it opens/closes the command port and works + **without a running camera** (device I/O lives in a plugin-owned thread, independent of the + host's frame-driven plugin passes). Connecting never changes the output; only changes made + while connected are transferred. +- **Output off = power slider at 0.** The firmware output is **set-and-hold**: disconnecting, + closing the GUI, or a crash leaves the last modulation running (`stage-a-controller` ADR 002). ## Ports @@ -29,5 +30,5 @@ connects to the one that answers `HELLO` — that is always the Teensy command p photodiode stream port. Explicit ports remain selectable; `mock` runs an in-process simulated controller for hardware-free testing. -Hardware commands only flow while the host execution context allows effects (live capture); -otherwise the connection is torn down and the panel shows the lock reason. +Replaying a recording disconnects the plugin defensively; live control itself needs no +capture session. diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index b941a9d..87fc75f 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -6,45 +6,41 @@ //! with frequency and a lower threshold for the periodic modes — and every //! accepted change is transferred to the Teensy immediately, no Apply button. //! -//! The plugin owns the Teensy **command port** (the first of the two CDC -//! ports the dual-serial firmware enumerates; the photodiode stream port is -//! owned by `stage-a-photodiode`). The firmware output is set-and-hold: -//! disconnecting does NOT switch the modulation off — use the "Output OFF" -//! action (ADR 002 in `stage-a-controller`). +//! **Frame-independent by design.** The host only calls `process_frame()` +//! while camera frames flow, so nothing here depends on it: connecting is a +//! checkbox *setting* (settings arrive from the UI thread at any time), a +//! dedicated device thread owns the serial client, and slider changes are +//! coalesced into a pending-command slot that thread drains. The bench works +//! with no camera attached. `process_frame()` only tears the connection down +//! defensively in replay mode. //! -//! Safety contract: -//! - devices open only when the execution context allows hardware effects; -//! anything else tears the connection down (fail closed); -//! - the level slider cannot exceed the max-level cap, and the firmware -//! output can never exceed the slider (square/sine peak at `level`); -//! - `process_frame()` only drains the bounded I/O worker queues. +//! The plugin owns the Teensy **command port**; the photodiode stream port is +//! owned by `stage-a-photodiode`. The firmware output is set-and-hold +//! (`stage-a-controller` ADR 002): disconnecting does NOT switch the +//! modulation off — drag the power slider to 0 to drive 0 V. use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, SettingItem, - SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, - TableColumnValues, TableDatasetV1, TableSchema, TableValueType, - CTX_INVESTIGATION_ACTION_REQUESTS, + export_plugin, EventStoreHandle, ExecutionMode, HostContext, HostDatasetDescriptor, + HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, + HostViewRegistry, Plugin, PluginFrame, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{Command, IoWorker, MockController, StageAClient, WorkerOutput, WorkerRequest}; +use stage_a_io::{Command, MockController, StageAClient, Transport}; const STATUS_DATASET_ID: &str = "stage-a-modulation.status"; const STATUS_VIEW_ID: &str = "stage-a-modulation.status.view"; -const ACTION_CONNECT: &str = "stage-a-modulation.connect"; -const ACTION_DISCONNECT: &str = "stage-a-modulation.disconnect"; -const ACTION_OUTPUT_OFF: &str = "stage-a-modulation.output-off"; - const MAX_DAC_CODE: i64 = 4_095; const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(500); +const DEVICE_LOOP_TICK: Duration = Duration::from_millis(10); #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Mode { @@ -73,6 +69,41 @@ impl Mode { } } +/// State the device thread reports back for the UI (status entries, table). +#[derive(Default)] +struct DeviceState { + connected: bool, + firmware: String, + board_code: Option, + board_mod: String, + last_error: Option, +} + +/// Everything shared between the plugin (UI thread) and the device thread. +struct SharedLink { + state: Mutex, + /// Latest not-yet-sent command; newer settings overwrite older ones so + /// slider drags coalesce instead of queueing. + pending: Mutex>, + stop: AtomicBool, + generation: AtomicU64, +} + +impl SharedLink { + fn new() -> Self { + Self { + state: Mutex::new(DeviceState::default()), + pending: Mutex::new(None), + stop: AtomicBool::new(false), + generation: AtomicU64::new(1), + } + } + + fn bump(&self) { + self.generation.fetch_add(1, Ordering::Relaxed); + } +} + /// In-process mock controller thread behind the `mock` port. struct MockService { stop: Arc, @@ -113,125 +144,200 @@ impl Drop for MockService { } } +/// Handle to the running device thread; dropping it stops the thread. +struct DeviceLink { + shared: Arc, + join: Option>, + _mock: Option, +} + +impl Drop for DeviceLink { + fn drop(&mut self) { + self.shared.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +/// Device thread: HELLO once, then drain the pending command slot and poll +/// STATUS. All serial I/O lives here — the UI thread never blocks. +fn run_device(mut client: StageAClient, shared: Arc) { + match client.request(&Command::new("HELLO").field("protocol", 1)) { + Ok(fields) => { + let mut state = shared.state.lock().expect("device state lock"); + state.connected = true; + state.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + let has_mod = fields + .get("capabilities") + .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); + state.last_error = (!has_mod).then(|| { + "firmware has no MOD capability — flash stage-a-controller 0.3.0+".to_owned() + }); + } + Err(err) => { + let mut state = shared.state.lock().expect("device state lock"); + state.connected = false; + state.last_error = Some(format!("HELLO failed: {err}")); + shared.bump(); + return; + } + } + shared.bump(); + + let mut last_status = Instant::now() - STATUS_POLL_INTERVAL; + while !shared.stop.load(Ordering::Relaxed) { + let pending = shared.pending.lock().expect("pending lock").take(); + if let Some(command) = pending { + let result = client.request(&command); + apply_reply(&shared, "MOD", result); + } else if last_status.elapsed() >= STATUS_POLL_INTERVAL { + last_status = Instant::now(); + let result = client.request(&Command::new("STATUS")); + apply_reply(&shared, "STATUS", result); + } else { + std::thread::sleep(DEVICE_LOOP_TICK); + } + } + + let mut state = shared.state.lock().expect("device state lock"); + state.connected = false; + shared.bump(); +} + +fn apply_reply( + shared: &SharedLink, + purpose: &str, + result: Result, stage_a_io::ClientError>, +) { + let mut state = shared.state.lock().expect("device state lock"); + match result { + Ok(fields) => { + if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { + state.board_code = Some(code); + } + if let Some(wave) = fields.get("mod_wave") { + let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); + let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); + let freq_mhz = fields + .get("mod_freq_mhz") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0); + state.board_mod = if wave == "SINE" || wave == "SQUARE" { + format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) + } else { + format!("{wave} level={level}") + }; + } + if purpose == "MOD" { + state.last_error = None; + } + } + Err(err) => state.last_error = Some(format!("{purpose}: {err}")), + } + drop(state); + shared.bump(); +} + pub struct StageAModulationPlugin { enabled: bool, - // -- device -- - worker: Option, - mock_service: Option, - connected: bool, - firmware: String, - next_tag: u64, - in_flight: BTreeMap, - last_error: Option, - effects_blocked_reason: Option, - last_status_poll: Instant, + link: Option, + shared: Arc, // -- settings (every accepted change is sent immediately) -- + connect_requested: bool, port_hint: String, max_level: i64, level: i64, min_level: i64, mode: Mode, frequency_hz: f64, - dirty: bool, - // -- board-reported state (from MOD replies and STATUS polls) -- - board_code: Option, - board_mod: String, - dataset_generation: u64, - consumed_action_ids: Vec, + last_error: Option, } impl Default for StageAModulationPlugin { fn default() -> Self { Self { enabled: false, - worker: None, - mock_service: None, - connected: false, - firmware: String::new(), - next_tag: 1, - in_flight: BTreeMap::new(), - last_error: None, - effects_blocked_reason: None, - last_status_poll: Instant::now(), - port_hint: "mock".into(), + link: None, + shared: Arc::new(SharedLink::new()), + connect_requested: false, + port_hint: "auto".into(), max_level: MAX_DAC_CODE, level: 0, min_level: 0, mode: Mode::Const, frequency_hz: 10.0, - dirty: false, - board_code: None, - board_mod: "—".into(), - dataset_generation: 0, - consumed_action_ids: Vec::new(), + last_error: None, } } } impl StageAModulationPlugin { - fn bump_generation(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - fn connect(&mut self) { - if self.worker.is_some() { + if self.link.is_some() { return; } + self.last_error = None; + *self.shared.state.lock().expect("device state lock") = DeviceState::default(); + *self.shared.pending.lock().expect("pending lock") = None; + self.shared.stop.store(false, Ordering::Relaxed); + self.shared.bump(); + + let shared = Arc::clone(&self.shared); + let spawn = |name: &str, f: Box| { + std::thread::Builder::new() + .name(name.to_owned()) + .spawn(f) + .expect("spawning the device thread must succeed") + }; if self.port_hint == "mock" { - let (service, client) = MockService::spawn(); - self.mock_service = Some(service); - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; + let (mock, client) = MockService::spawn(); + let join = spawn( + "stage-a-modulation-device", + Box::new(move || run_device(client, shared)), + ); + self.link = Some(DeviceLink { + shared: Arc::clone(&self.shared), + join: Some(join), + _mock: Some(mock), + }); } else { match open_serial(&self.port_hint) { Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; + let join = spawn( + "stage-a-modulation-device", + Box::new(move || run_device(client, shared)), + ); + self.link = Some(DeviceLink { + shared: Arc::clone(&self.shared), + join: Some(join), + _mock: None, + }); } Err(err) => { self.last_error = Some(err); - return; + self.connect_requested = false; } } } - // Connecting never drives the output: only changes made while - // connected are transferred. - self.dirty = false; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - self.bump_generation(); + // Connecting never drives the output (set-and-hold firmware); only + // changes made while connected are transferred. } - fn disconnect(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - worker.shutdown(reason); - } - self.mock_service = None; - self.connected = false; - self.firmware.clear(); - self.in_flight.clear(); - self.board_code = None; - self.board_mod = "—".into(); - self.bump_generation(); + fn disconnect(&mut self) { + self.link = None; // Drop stops and joins the device thread. + self.shared.bump(); } - /// One MOD command carrying the complete current drive settings. + /// Queues one MOD command carrying the complete current drive settings; + /// newer changes overwrite queued ones (drag coalescing). fn send_modulation(&mut self) { - self.dirty = false; + if self.link.is_none() { + return; + } let level = self.level.clamp(0, self.max_level); let mut command = Command::new("MOD") .field("wave", self.mode.name()) @@ -242,103 +348,16 @@ impl StageAModulationPlugin { .field("min", self.min_level.clamp(0, level)) .field("freq_mhz", freq_mhz); } - self.queue_command("mod", command); - } - - fn output_off(&mut self) { - self.dirty = false; - self.queue_command("mod", Command::new("MOD").field("wave", "OFF")); - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - if outputs.is_empty() { - return; - } - let mut stopped: Option = None; - for output in outputs { - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => self.handle_reply(&purpose, &fields), - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - WorkerOutput::Event(_) | WorkerOutput::Integrity(_) => {} - WorkerOutput::Stopped { reason } => stopped = Some(reason), - } - } - if let Some(reason) = stopped { - self.worker = None; - self.mock_service = None; - self.connected = false; - self.last_error = Some(format!("device connection ended: {reason}")); - } - self.bump_generation(); - } - - fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { - if purpose == "hello" { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - self.connected = true; - let has_mod = fields - .get("capabilities") - .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); - if !has_mod { - self.last_error = - Some("firmware has no MOD capability — flash stage-a-controller 0.3.0+".into()); - } - } - // MOD replies and STATUS polls both carry code= and mod_* fields. - if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { - self.board_code = Some(code); - } - if let Some(wave) = fields.get("mod_wave") { - let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); - let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); - let freq_mhz = fields - .get("mod_freq_mhz") - .and_then(|v| v.parse::().ok()) - .unwrap_or(0.0); - self.board_mod = if wave == "SINE" || wave == "SQUARE" { - format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) - } else { - format!("{wave} level={level}") - }; - } - if purpose == "mod" { - self.last_error = None; - } + *self.shared.pending.lock().expect("pending lock") = Some(command); } - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-modulation.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed + #[cfg(test)] + fn device_connected(&self) -> bool { + self.shared + .state + .lock() + .map(|state| state.connected) + .unwrap_or(false) } fn commanded_summary(&self) -> String { @@ -356,25 +375,39 @@ impl StageAModulationPlugin { } fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.connected) { - (Some(reason), _) => format!("locked ({reason})"), - (None, false) => "disconnected".into(), - (None, true) => format!("connected ({})", self.firmware), + let state = self.shared.state.lock().expect("device state lock"); + let connection = if state.connected { + format!("connected ({})", state.firmware) + } else if self.connect_requested { + "connecting…".into() + } else { + "disconnected".into() }; - let board_code = self + let board_code = state .board_code .map_or_else(|| "—".into(), |code| code.to_string()); + let error = state + .last_error + .clone() + .or_else(|| self.last_error.clone()) + .unwrap_or_default(); + let board_mod = if state.board_mod.is_empty() { + "—".to_owned() + } else { + state.board_mod.clone() + }; + drop(state); let text_column = |id: &str, value: String| TableColumnData { column_id: id.to_owned(), values: TableColumnValues::String(vec![value]), }; TableDatasetV1 { columns: vec![ - text_column("state", state), + text_column("state", connection), text_column("commanded", self.commanded_summary()), - text_column("board_mod", self.board_mod.clone()), + text_column("board_mod", board_mod), text_column("board_code", board_code), - text_column("error", self.last_error.clone().unwrap_or_default()), + text_column("error", error), ], } } @@ -452,7 +485,7 @@ fn serial_ports() -> Vec { /// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; /// only the leading path is the value. fn port_variants() -> Vec { - let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; + let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; for (name, label) in stage_a_io::transport::available_ports_with_labels() { if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { continue; @@ -501,13 +534,12 @@ impl Plugin for StageAModulationPlugin { fn set_enabled(&mut self, enabled: bool) { self.enabled = enabled; if !enabled { - self.disconnect("plugin disabled"); + self.connect_requested = false; + self.disconnect(); } } - fn reset(&mut self) { - self.bump_generation(); - } + fn reset(&mut self) {} fn process_frame( &mut self, @@ -516,38 +548,14 @@ impl Plugin for StageAModulationPlugin { context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - // Fail closed: without live-capture effects the connection is torn - // down and no command leaves the plugin. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.worker.is_some() { - self.disconnect("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for action_id in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect("operator"), - ACTION_OUTPUT_OFF => self.output_off(), - _ => {} - } + // Control is settings-driven and works without camera frames. The + // only frame-pass policy: replaying a recording must never keep a + // hardware connection alive. + if context.execution().mode == ExecutionMode::Replay && self.link.is_some() { + self.connect_requested = false; + self.disconnect(); + self.last_error = Some("disconnected: replay mode".into()); } - - if self.dirty && self.connected { - self.send_modulation(); - } - if self.connected && self.last_status_poll.elapsed() >= STATUS_POLL_INTERVAL { - self.last_status_poll = Instant::now(); - self.queue_command("status", Command::new("STATUS")); - } - self.drain_worker(); } fn settings_schema(&self) -> SettingsSchema { @@ -566,9 +574,10 @@ impl Plugin for StageAModulationPlugin { sections: vec![SettingsSection { label: "Laser modulation".into(), description: Some( - "Every change is sent to the Teensy immediately. The output never exceeds \ - the power slider, and the slider never exceeds the max limit. The firmware \ - holds the output when the plugin disconnects — use Output OFF to drive 0." + "Tick Connect, then every change is sent to the Teensy immediately — no \ + camera required. The output never exceeds the power slider, the slider \ + never exceeds the max limit. The firmware holds the output when \ + disconnected; drag the slider to 0 to drive 0 V." .into(), ), default_open: true, @@ -587,12 +596,24 @@ impl Plugin for StageAModulationPlugin { default: port_default, }, }, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the command port. Connecting never changes the \ + output; disconnecting leaves it held (set-and-hold firmware)." + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, + }, SettingItem { key: "level".into(), label: "Power (DAC code)".into(), tooltip: Some( "Output level in DAC codes; peak value for sine/square. \ - Capped by the max limit below." + Capped by the max limit below. 0 = output off." .into(), ), kind: SettingKind::I64Slider { @@ -667,6 +688,7 @@ impl Plugin for StageAModulationPlugin { .unwrap_or(0); Some(json!(index)) } + "connect" => Some(json!(self.connect_requested)), "level" => Some(json!(self.level)), "max_level" => Some(json!(self.max_level)), "mode" => { @@ -688,6 +710,16 @@ impl Plugin for StageAModulationPlugin { self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } + "connect" => { + let requested = value.as_bool().ok_or("connect must be a boolean")?; + self.connect_requested = requested; + if requested { + self.connect(); + } else { + self.disconnect(); + } + Ok(()) + } "level" => { self.level = value .as_i64() @@ -696,7 +728,7 @@ impl Plugin for StageAModulationPlugin { if self.min_level > self.level { self.min_level = self.level; } - self.dirty = true; + self.send_modulation(); Ok(()) } "max_level" => { @@ -707,7 +739,7 @@ impl Plugin for StageAModulationPlugin { // Lowering the cap below the current level lowers the output. if self.level > self.max_level { self.level = self.max_level; - self.dirty = true; + self.send_modulation(); } if self.min_level > self.max_level { self.min_level = self.max_level; @@ -720,14 +752,14 @@ impl Plugin for StageAModulationPlugin { let name = enum_choice(&value, &mode_names)?; self.mode = Mode::from_name(&name) .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; - self.dirty = true; + self.send_modulation(); Ok(()) } "frequency_hz" => { let hz = value.as_f64().ok_or("frequency_hz must be a number")?; self.frequency_hz = hz.clamp(0.01, 2_000.0); if self.mode.is_periodic() { - self.dirty = true; + self.send_modulation(); } Ok(()) } @@ -737,7 +769,7 @@ impl Plugin for StageAModulationPlugin { .ok_or("min_level must be an integer")? .clamp(0, self.level); if self.mode.is_periodic() { - self.dirty = true; + self.send_modulation(); } Ok(()) } @@ -747,35 +779,27 @@ impl Plugin for StageAModulationPlugin { fn status_entries(&self) -> Vec { let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - entries.push(StatusEntry::Text(if self.connected { - format!("Modulation: connected ({})", self.firmware) + let state = self.shared.state.lock().expect("device state lock"); + entries.push(StatusEntry::Text(if state.connected { + format!("Modulation: connected ({})", state.firmware) + } else if self.connect_requested { + "Modulation: connecting…".into() } else { "Modulation: disconnected".into() })); - if let Some(code) = self.board_code { + if let Some(code) = state.board_code { entries.push(StatusEntry::Text(format!( "Board: code={code} ({})", - self.board_mod + state.board_mod ))); } - if let Some(error) = &self.last_error { + if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } entries } fn host_views(&self) -> HostViewRegistry { - let action = |id: &str, title: &str| HostActionDescriptor { - id: id.into(), - title: title.into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }; HostViewRegistry { datasets: vec![HostDatasetDescriptor { id: STATUS_DATASET_ID.into(), @@ -792,11 +816,7 @@ impl Plugin for StageAModulationPlugin { placement: HostViewPlacement::AnalysisPanel, kind: HostViewKind::CompactTable, }], - actions: vec![ - action(ACTION_CONNECT, "Connect"), - action(ACTION_DISCONNECT, "Disconnect"), - action(ACTION_OUTPUT_OFF, "Output OFF"), - ], + actions: Vec::new(), } } @@ -809,7 +829,7 @@ impl Plugin for StageAModulationPlugin { fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { match dataset_id { - STATUS_DATASET_ID => self.dataset_generation.max(1), + STATUS_DATASET_ID => self.shared.generation.load(Ordering::Relaxed).max(1), _ => 0, } } @@ -817,7 +837,7 @@ impl Plugin for StageAModulationPlugin { impl Drop for StageAModulationPlugin { fn drop(&mut self) { - self.disconnect("plugin destroyed"); + self.disconnect(); } } @@ -827,14 +847,13 @@ export_plugin!(StageAModulationPlugin); mod tests { use super::*; - fn drain_until bool>( - plugin: &mut StageAModulationPlugin, + fn wait_until bool>( + plugin: &StageAModulationPlugin, timeout: Duration, - mut done: F, + done: F, ) { let deadline = Instant::now() + timeout; while Instant::now() < deadline { - plugin.drain_worker(); if done(plugin) { return; } @@ -843,25 +862,31 @@ mod tests { panic!("condition not reached within {timeout:?}"); } - /// Slider change → MOD sent immediately → board echoes the code. + fn board_code(plugin: &StageAModulationPlugin) -> Option { + plugin.shared.state.lock().unwrap().board_code + } + + /// Connect checkbox → slider change → MOD sent by the device thread → + /// board echoes the code. No process_frame involved anywhere. #[test] - fn level_change_transfers_immediately_and_board_code_is_shown() { + fn level_change_transfers_without_frames() { let mut plugin = StageAModulationPlugin::default(); - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); - assert_eq!(plugin.firmware, "0.3.0-mock"); - - plugin - .set_setting("level", json!(1234)) - .expect("level accepted"); - assert!(plugin.dirty); - plugin.send_modulation(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.board_code == Some(1234) + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + assert_eq!( + plugin.shared.state.lock().unwrap().firmware, + "0.3.0-mock".to_owned() + ); + + plugin.set_setting("level", json!(1234)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1234) }); - assert!(!plugin.dirty); - assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); - plugin.disconnect("test done"); + assert!(plugin.shared.state.lock().unwrap().last_error.is_none()); + + plugin.set_setting("connect", json!(false)).unwrap(); + assert!(!plugin.device_connected()); } /// The max cap bounds the slider, and lowering it re-sends a lower level. @@ -874,7 +899,6 @@ mod tests { plugin.set_setting("max_level", json!(500)).unwrap(); assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); - assert!(plugin.dirty, "the lowered level must be transferred"); let schema = plugin.settings_schema(); let level_item = schema.sections[0] @@ -888,28 +912,36 @@ mod tests { } } - /// Square drive with min threshold reaches the mock and starts at min. + /// Square drive with min threshold reaches the mock and starts at min; + /// slider to 0 drives the output to 0. #[test] fn square_with_min_threshold_round_trips() { let mut plugin = StageAModulationPlugin::default(); - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); plugin.set_setting("level", json!(2000)).unwrap(); - plugin.set_setting("mode", json!("SQUARE")).unwrap(); plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); plugin.set_setting("min_level", json!(500)).unwrap(); - plugin.send_modulation(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.board_code == Some(500) + plugin.set_setting("mode", json!("SQUARE")).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(500) }); - assert!(plugin.board_mod.contains("SQUARE 500..2000")); - - plugin.output_off(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.board_code == Some(0) + assert!(plugin + .shared + .state + .lock() + .unwrap() + .board_mod + .contains("SQUARE 500..2000")); + + plugin.set_setting("mode", json!("CONST")).unwrap(); + plugin.set_setting("level", json!(0)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(0) }); - plugin.disconnect("test done"); + plugin.set_setting("connect", json!(false)).unwrap(); } /// The host settings UI exchanges enum values as indices into the @@ -923,11 +955,11 @@ mod tests { .expect("index accepted"); assert_eq!(plugin.mode, Mode::Square); assert_eq!(plugin.get_setting("mode"), Some(json!(2))); - // Port: index 1 = "auto" (variants start with mock, auto). + // Port: index 1 = "mock" (variants start with auto, mock). plugin .set_setting("port", json!(1)) .expect("index accepted"); - assert_eq!(plugin.port_hint, "auto"); + assert_eq!(plugin.port_hint, "mock"); assert_eq!(plugin.get_setting("port"), Some(json!(1))); // Out-of-range indices are visible errors, not silent no-ops. assert!(plugin.set_setting("mode", json!(99)).is_err()); diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index d6f2928..7957f17 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -22,12 +22,11 @@ use std::thread::JoinHandle; use std::time::{Duration, Instant}; use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, - Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, - StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, + export_plugin, EventStoreHandle, HostContext, HostDatasetDescriptor, HostDatasetKind, + HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, + PluginFrame, Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, }; use serde_json::{json, Value}; @@ -36,9 +35,6 @@ const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; const STATUS_DATASET_ID: &str = "stage-a-photodiode.status"; const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; -const ACTION_CONNECT: &str = "stage-a-photodiode.connect"; -const ACTION_DISCONNECT: &str = "stage-a-photodiode.disconnect"; - const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; /// Ring capacity: > 2.5 minutes at the firmware's 50 lines/s. @@ -226,14 +222,13 @@ pub struct StageAPhotodiodePlugin { reader: Option, shared: Arc>, generation: Arc, - effects_blocked_reason: Option, last_error: Option, // -- settings -- + connect_requested: bool, port_hint: String, mode: Mode, reference_volts: f64, window_s: f64, - consumed_action_ids: Vec, } impl Default for StageAPhotodiodePlugin { @@ -243,13 +238,12 @@ impl Default for StageAPhotodiodePlugin { reader: None, shared: Arc::new(Mutex::new(SharedState::default())), generation: Arc::new(AtomicU64::new(1)), - effects_blocked_reason: None, last_error: None, - port_hint: "mock".into(), + connect_requested: false, + port_hint: "auto".into(), mode: Mode::Raw, reference_volts: 3.3, window_s: 10.0, - consumed_action_ids: Vec::new(), } } } @@ -287,7 +281,10 @@ impl StageAPhotodiodePlugin { }; match Reader::spawn_serial(path, Arc::clone(&self.shared), Arc::clone(&self.generation)) { Ok(reader) => self.reader = Some(reader), - Err(err) => self.last_error = Some(err), + Err(err) => { + self.last_error = Some(err); + self.connect_requested = false; + } } self.generation.fetch_add(1, Ordering::Relaxed); } @@ -305,29 +302,6 @@ impl StageAPhotodiodePlugin { } } - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-photodiode.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed - } - fn series_dataset(&self) -> Series1dV1 { let (points, y_label) = match self.shared.lock() { Ok(state) => { @@ -369,10 +343,10 @@ impl StageAPhotodiodePlugin { Ok(state) => (state.latest, state.error.clone()), Err(_) => (None, None), }; - let state = match (&self.effects_blocked_reason, self.connected()) { - (Some(reason), _) => format!("locked ({reason})"), - (None, false) => "disconnected".into(), - (None, true) => format!("reading ({})", self.port_hint), + let state = if self.connected() { + format!("reading ({})", self.port_hint) + } else { + "disconnected".into() }; let (code_text, value_text) = match latest { Some(sample) => ( @@ -547,6 +521,7 @@ impl Plugin for StageAPhotodiodePlugin { fn set_enabled(&mut self, enabled: bool) { self.enabled = enabled; if !enabled { + self.connect_requested = false; self.disconnect(); } } @@ -562,31 +537,12 @@ impl Plugin for StageAPhotodiodePlugin { &mut self, _frame: &PluginFrame<'_>, _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, + _context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - // The stream port is read-only, but device access still follows the - // same fail-closed gate as every stage-a plugin. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.reader.is_some() { - self.disconnect(); - } - return; - } - self.effects_blocked_reason = None; - - for action_id in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect(), - _ => {} - } - } + // Reading is settings-driven (connect checkbox) and works without + // camera frames; the stream port carries no commands, so no replay + // teardown is needed either. } fn settings_schema(&self) -> SettingsSchema { @@ -626,6 +582,16 @@ impl Plugin for StageAPhotodiodePlugin { default: port_default, }, }, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the stream port (read-only, no camera required).".into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, + }, SettingItem { key: "mode".into(), label: "Mode".into(), @@ -679,6 +645,7 @@ impl Plugin for StageAPhotodiodePlugin { .unwrap_or(0); Some(json!(index)) } + "connect" => Some(json!(self.connect_requested)), "mode" => { let index = Mode::VARIANTS .iter() @@ -698,6 +665,16 @@ impl Plugin for StageAPhotodiodePlugin { self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } + "connect" => { + let requested = value.as_bool().ok_or("connect must be a boolean")?; + self.connect_requested = requested; + if requested { + self.connect(); + } else { + self.disconnect(); + } + Ok(()) + } "mode" => { let mode_names: Vec = Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); @@ -722,9 +699,6 @@ impl Plugin for StageAPhotodiodePlugin { fn status_entries(&self) -> Vec { let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } let (latest, stream_error) = match self.shared.lock() { Ok(state) => (state.latest, state.error.clone()), Err(_) => (None, None), @@ -756,14 +730,6 @@ impl Plugin for StageAPhotodiodePlugin { } fn host_views(&self) -> HostViewRegistry { - let action = |id: &str, title: &str| HostActionDescriptor { - id: id.into(), - title: title.into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }; HostViewRegistry { datasets: vec![ HostDatasetDescriptor { @@ -799,10 +765,7 @@ impl Plugin for StageAPhotodiodePlugin { kind: HostViewKind::CompactTable, }, ], - actions: vec![ - action(ACTION_CONNECT, "Connect"), - action(ACTION_DISCONNECT, "Disconnect"), - ], + actions: Vec::new(), } } From 5f323c2012c1cb3db0f4abd5adac2c46384fa12d Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 16:29:29 +0200 Subject: [PATCH 15/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20read=20the?= =?UTF-8?q?=20PDA1=20photodiode=20stream=20at=2020=20kSa/s=20with=20envelo?= =?UTF-8?q?pe=20and=20moving=20average?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track firmware 0.4.0 (ADR 003 in stage-a-controller): the stream port now carries PDA1 SamplesU16 frames at pd_stream_rate_hz instead of 50 Hz ASCII lines, raising the plot's data rate 400x. - parse with stage-a-io's FrameParser (new dep, default-features off: wire parser only); auto port probe detects sample frames - bounded raw ring (130 s / 4 M samples) keyed by device sample index; rate changes and index jumps restart the segment so index/rate stays a consistent time base across acquisition handovers - chart decimates the window into <= 1000 min/mean/max envelope buckets; short windows render raw samples; window down to 10 ms - moving-average indicator for the low-voltage regime: fixed sample window (default 4) or one full period of a user-set sync frequency (window = rate / f), making the mean phase-independent under modulation; overlay line + numeric readout in status views - status table gains rate, moving avg, and stream-integrity columns (device drops, CRC failures, resync bytes, segment restarts) - stage-a-io: gate the Duration import behind the hardware feature so default-features = false builds are warning-free Verified: cargo fmt, clippy -D warnings (photodiode + stage-a-io), 9 plugin tests + 29 stage-a-io tests green. --- docs/features/README.md | 2 +- docs/features/stage-a-photodiode.md | 45 +- plugins/stage-a-photodiode/Cargo.toml | 1 + plugins/stage-a-photodiode/plugin.toml | 2 +- plugins/stage-a-photodiode/src/lib.rs | 751 +++++++++++++++++++------ stage-a-io/src/transport.rs | 1 + 6 files changed, 624 insertions(+), 178 deletions(-) diff --git a/docs/features/README.md b/docs/features/README.md index eb5ffe4..16ed650 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -6,7 +6,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. - [Stage-A Modulation](./stage-a-modulation.md) — capped power slider + constant/sine/square laser-modulation drive on the command port, applied immediately. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the stream port: raw values or excitation power `I_exc = I_tot − I_pd`. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 916ce8c..ca688b4 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -1,15 +1,18 @@ # Stage-A Photodiode - **Crate:** `plugins/stage-a-photodiode` (`augur-plugin-stage-a-photodiode`) -- **Firmware:** `stage-a-controller` 0.3.0+ (`PDSTREAM`), Teensy **stream port** (second CDC port) -- **Status:** Active (2026-07-15) — replaces the readout half of `stage-a-monitor` +- **Firmware:** `stage-a-controller` 0.4.0+ (`PDSTREAM_PDA1`), Teensy **stream port** (second CDC port) +- **Status:** Active (2026-07-16) — replaces the readout half of `stage-a-monitor` ## What it is -A minimal live readout of the photodiode on **board SMA5 → Teensy pin 18 / A4**. The firmware -streams `PD code= n= t_ms=` lines at 50 Hz on its second USB serial port; a -background thread parses them into a bounded ring, and the plugin shows the newest value plus a -rolling chart (1–120 s window). +A live readout of the photodiode on **board SMA5 → Teensy pin 18 / A4**. Firmware 0.4.0 streams +PDA1 `SamplesU16` frames free-running at `pd_stream_rate_hz` (20 kSa/s default) on its second USB +serial port; a background thread parses them with `stage-a-io`'s `FrameParser` into a bounded raw +ring (up to 130 s / 4 M samples), and the plugin renders a rolling chart (10 ms – 120 s window) +plus the newest value. During a command-port acquisition the firmware mirrors the acquisition +blocks here — every rate change or sample-index jump restarts the ring as a new segment, so the +`index / rate` time base is always consistent. Two modes: @@ -18,18 +21,34 @@ Two modes: removed from the beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. +## Chart + +- The visible window is decimated into at most 1 000 buckets; when a bucket covers more than one + sample the chart shows the bucket **mean** plus a **min/max envelope**, so narrow modulation + peaks stay visible at any zoom. Windows short enough to fit raw samples render them directly. +- **Moving average** (for the low-voltage regime): a smoothed overlay line plus a numeric readout. + The window is either a fixed sample count (`avg_samples`, default 4; 1 = off) or — the right + tool for modulated signals — **one full period of a user-given frequency** + (`avg_sync_freq_hz`, e.g. the MOD drive frequency): window = `rate / f` samples, which makes + the mean independent of the modulation phase instead of riding the waveform. + ## Contract - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the - plugin is read-only by construction and needs no protocol library — it depends only on - `serialport` and parses one line format. + plugin is read-only by construction. It reuses `stage-a-io` (`default-features = false`) only + for the PDA1 wire parser — no client, worker, or transport. - **Frame-independent**: connecting is a checkbox setting; the reader thread and all views work with no camera attached (the host only calls `process_frame()` while frames flow). -- Garbage on the port (e.g. the binary command port picked by mistake) parses to nothing and is - bounded — it can neither grow memory nor produce fake values. -- `mock` port synthesizes a slow sine for hardware-free testing. +- Garbage on the port resynchronises at the next CRC-clean frame; skipped bytes and CRC failures + are counted and shown in the status table's integrity column together with the firmware's + cumulative drop counter and the segment-restart count. +- `mock` port synthesizes a noisy 5 Hz sine at 20 kSa/s in firmware-sized blocks for + hardware-free testing. ## Verification -`cargo test -p augur-plugin-stage-a-photodiode` — line parsing (including clamping and rejection), -excitation inversion against the reference, mock reader filling ring/series, ring bound. +`cargo test -p augur-plugin-stage-a-photodiode` — frame ingestion incl. segment restarts on index +jumps and rate changes, duration-bounded ring with aligned indexes, moving-average window +derivation from the sync frequency, newest-window average, envelope decimation bounds and +min ≤ mean ≤ max, raw rendering for short windows, excitation inversion, mock reader, settings +round-trips. diff --git a/plugins/stage-a-photodiode/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml index b426623..a838ebb 100644 --- a/plugins/stage-a-photodiode/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -13,3 +13,4 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true serialport.workspace = true +stage-a-io = { path = "../../stage-a-io", default-features = false } diff --git a/plugins/stage-a-photodiode/plugin.toml b/plugins/stage-a-photodiode/plugin.toml index 702a218..21b4bad 100644 --- a/plugins/stage-a-photodiode/plugin.toml +++ b/plugins/stage-a-photodiode/plugin.toml @@ -1,5 +1,5 @@ name = "Stage-A Photodiode" -version = "0.3.0" +version = "0.4.0" description = "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." domain = "stage-a" library = "augur_plugin_stage_a_photodiode" diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 7957f17..a966045 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -1,11 +1,13 @@ //! Stage-A photodiode readout. //! -//! Reads the free-running ASCII stream the `stage-a-controller` firmware -//! (0.3.0+, `USB_DUAL_SERIAL`) emits on its **second** USB serial port: -//! one `PD code= n= t_ms=` line every 20 ms from the -//! photodiode on board SMA5 → Teensy pin 18 / A4. The port carries no +//! Reads the free-running PDA1 binary frame stream the `stage-a-controller` +//! firmware (0.4.0+, `USB_DUAL_SERIAL`) emits on its **second** USB serial +//! port: `SamplesU16` frames at `pd_stream_rate_hz` (20 kSa/s default) from +//! the photodiode on board SMA5 → Teensy pin 18 / A4. The port carries no //! commands, so opening it is side-effect free; the command port is owned by -//! `stage-a-modulation`. +//! `stage-a-modulation`. While a command-port acquisition runs the firmware +//! mirrors its blocks here (flag 0x0001) — every rate change or sample-index +//! jump is treated as a segment restart. //! //! Two display modes: //! - **RAW**: the ADC code and its voltage (`V = code · 3.3 / 4095`); @@ -13,6 +15,11 @@ //! path and sees the light removed from the beam, `I_pd = I_tot − I_exc`. //! Given the user-set reference `I_tot` (in photodiode volts), the plugin //! shows `I_exc = I_tot − V_pd`. +//! +//! The chart decimates the visible window into min/mean/max envelope buckets +//! and overlays a moving average whose window is either a fixed sample count +//! or — for modulated signals — one full period of a user-given frequency, +//! which makes the mean independent of the modulation phase. use std::collections::VecDeque; use std::io::Read; @@ -29,6 +36,7 @@ use augur_plugin_api::{ TableSchema, TableValueType, }; use serde_json::{json, Value}; +use stage_a_io::{FrameParser, ParseEvent}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; @@ -37,8 +45,17 @@ const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; -/// Ring capacity: > 2.5 minutes at the firmware's 50 lines/s. -const RING_CAPACITY: usize = 8_192; +/// Longest raw history kept, in seconds of samples at the active stream rate. +const RING_SECONDS: f64 = 130.0; +/// Absolute sample cap guarding against absurd advertised rates (8 MiB of +/// codes at most). +const RING_MAX_SAMPLES: usize = 4_000_000; +/// Envelope buckets per rendered chart line; keeps the plot payload bounded +/// no matter how many raw samples the window covers. +const MAX_PLOT_BUCKETS: usize = 1_000; +/// The firmware's default stream rate; the mock mirrors it. +const MOCK_RATE_HZ: u32 = 20_000; +const MOCK_BLOCK_SAMPLES: usize = 256; fn code_to_volts(code: f64) -> f64 { code * ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE @@ -65,45 +82,56 @@ impl Mode { } } -#[derive(Debug, Clone, Copy)] -struct PdSample { - t_ms: u64, - code: f64, -} - -/// Parses one firmware stream line: `PD code= n= t_ms=`. -fn parse_pd_line(line: &str) -> Option { - let rest = line.trim().strip_prefix("PD ")?; - let mut code = None; - let mut t_ms = None; - for token in rest.split_ascii_whitespace() { - let (key, value) = token.split_once('=')?; - match key { - "code" => code = value.parse::().ok(), - "t_ms" => t_ms = value.parse::().ok(), - "n" => {} - _ => return None, - } - } - Some(PdSample { - t_ms: t_ms?, - code: code?.clamp(0.0, ADC_MAX_CODE), - }) -} - #[derive(Default)] struct SharedState { - samples: VecDeque, - latest: Option, + /// Sample rate of the current segment (from the frame headers). + rate_hz: u32, + /// Device sample index of `samples.front()` within the current segment. + ring_first_index: u64, + samples: VecDeque, + latest: Option, + /// Cumulative firmware-side drop counter (latest header value). + device_dropped: u32, + crc_failures: u64, + resync_bytes: u64, + /// Segment restarts observed (rate changes, index jumps, reconnects). + segments: u64, error: Option, } impl SharedState { - fn push(&mut self, sample: PdSample) { - self.latest = Some(sample); - self.samples.push_back(sample); - while self.samples.len() > RING_CAPACITY { - self.samples.pop_front(); + fn ring_capacity(rate_hz: u32) -> usize { + ((f64::from(rate_hz.max(1)) * RING_SECONDS) as usize).min(RING_MAX_SAMPLES) + } + + /// Ingests one `SamplesU16` frame. Any discontinuity — rate change, + /// sample-index jump (drops, acquisition handover), reconnect — restarts + /// the ring: within a segment `index / rate` is a consistent time base. + fn ingest(&mut self, first_index: u64, rate_hz: u32, device_dropped: u32, codes: &[u16]) { + if codes.is_empty() { + return; + } + let expected = self.ring_first_index + self.samples.len() as u64; + let continuous = + !self.samples.is_empty() && rate_hz == self.rate_hz && first_index == expected; + if !continuous { + if !self.samples.is_empty() { + self.segments += 1; + } + self.samples.clear(); + self.ring_first_index = first_index; + self.rate_hz = rate_hz; + } + self.samples.extend(codes.iter().copied()); + self.latest = codes.last().copied(); + self.device_dropped = device_dropped; + let excess = self + .samples + .len() + .saturating_sub(Self::ring_capacity(rate_hz)); + if excess > 0 { + self.samples.drain(..excess); + self.ring_first_index += excess as u64; } } } @@ -128,7 +156,7 @@ impl Reader { let thread_stop = Arc::clone(&stop); let join = std::thread::Builder::new() .name("stage-a-photodiode".into()) - .spawn(move || read_lines(port, &shared, &generation, &thread_stop)) + .spawn(move || read_frames(port, &shared, &generation, &thread_stop)) .expect("spawning the photodiode reader thread must succeed"); Ok(Self { stop, @@ -136,7 +164,8 @@ impl Reader { }) } - /// Hardware-free source: synthesizes a slow sine around 1 V at 50 Hz. + /// Hardware-free source: synthesizes a noisy 5 Hz sine around 1 V in + /// firmware-sized blocks at the firmware's default stream rate. fn spawn_mock(shared: Arc>, generation: Arc) -> Self { let stop = Arc::new(AtomicBool::new(false)); let thread_stop = Arc::clone(&stop); @@ -144,18 +173,24 @@ impl Reader { .name("stage-a-photodiode-mock".into()) .spawn(move || { let start = Instant::now(); + let mut next_index: u64 = 0; while !thread_stop.load(Ordering::Relaxed) { - let t = start.elapsed().as_secs_f64(); - let volts = 1.0 + 0.5 * (2.0 * std::f64::consts::PI * 0.2 * t).sin(); - let sample = PdSample { - t_ms: (t * 1_000.0) as u64, - code: volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS, - }; - if let Ok(mut state) = shared.lock() { - state.push(sample); + let target = (start.elapsed().as_secs_f64() * f64::from(MOCK_RATE_HZ)) as u64; + let mut produced = false; + while next_index + MOCK_BLOCK_SAMPLES as u64 <= target { + let codes: Vec = (0..MOCK_BLOCK_SAMPLES) + .map(|i| mock_code(next_index + i as u64)) + .collect(); + if let Ok(mut state) = shared.lock() { + state.ingest(next_index, MOCK_RATE_HZ, 0, &codes); + } + next_index += MOCK_BLOCK_SAMPLES as u64; + produced = true; + } + if produced { + generation.fetch_add(1, Ordering::Relaxed); } - generation.fetch_add(1, Ordering::Relaxed); - std::thread::sleep(Duration::from_millis(20)); + std::thread::sleep(Duration::from_millis(5)); } }) .expect("spawning the mock photodiode thread must succeed"); @@ -166,6 +201,17 @@ impl Reader { } } +/// Deterministic mock sample: 1 V ± 0.5 V sine at 5 Hz plus ~20 mV of hash +/// noise, so the moving-average indicator has something to smooth. +fn mock_code(index: u64) -> u16 { + let t = index as f64 / f64::from(MOCK_RATE_HZ); + let mut hash = index.wrapping_mul(0x9E37_79B9_7F4A_7C15); + hash ^= hash >> 33; + let noise = (hash as f64 / u64::MAX as f64) - 0.5; + let volts = 1.0 + 0.5 * (2.0 * std::f64::consts::PI * 5.0 * t).sin() + 0.04 * noise; + (volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS).clamp(0.0, ADC_MAX_CODE) as u16 +} + impl Drop for Reader { fn drop(&mut self) { self.stop.store(true, Ordering::Relaxed); @@ -175,14 +221,14 @@ impl Drop for Reader { } } -fn read_lines( +fn read_frames( mut port: Box, shared: &Mutex, generation: &AtomicU64, stop: &AtomicBool, ) { - let mut line_buffer: Vec = Vec::with_capacity(256); - let mut buf = [0_u8; 512]; + let mut parser = FrameParser::default(); + let mut buf = [0_u8; 4_096]; while !stop.load(Ordering::Relaxed) { let read = match port.read(&mut buf) { Ok(0) => continue, @@ -197,23 +243,39 @@ fn read_lines( return; } }; - line_buffer.extend_from_slice(&buf[..read]); - // Never let garbage (e.g. the wrong, binary port) grow the buffer. - if line_buffer.len() > 4_096 { - line_buffer.clear(); - } - while let Some(pos) = line_buffer.iter().position(|&b| b == b'\n') { - let line: Vec = line_buffer.drain(..=pos).collect(); - let Ok(text) = std::str::from_utf8(&line) else { - continue; - }; - if let Some(sample) = parse_pd_line(text) { - if let Ok(mut state) = shared.lock() { - state.push(sample); + parser.extend(&buf[..read]); + let mut changed = false; + while let Some(event) = parser.next_event() { + match event { + ParseEvent::Frame(frame) => { + let Some(codes) = frame.samples() else { + continue; // Control/summary frames are not expected here. + }; + if let Ok(mut state) = shared.lock() { + state.ingest( + frame.header.first_sample_index, + frame.header.sample_rate_hz, + frame.header.dropped_samples, + &codes, + ); + } + changed = true; + } + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + if let Ok(mut state) = shared.lock() { + state.resync_bytes += skipped_bytes as u64; + state.crc_failures += crc_failures as u64; + } + changed = true; } - generation.fetch_add(1, Ordering::Relaxed); } } + if changed { + generation.fetch_add(1, Ordering::Relaxed); + } } } @@ -229,6 +291,8 @@ pub struct StageAPhotodiodePlugin { mode: Mode, reference_volts: f64, window_s: f64, + avg_samples: usize, + avg_sync_freq_hz: f64, } impl Default for StageAPhotodiodePlugin { @@ -244,6 +308,8 @@ impl Default for StageAPhotodiodePlugin { mode: Mode::Raw, reference_volts: 3.3, window_s: 10.0, + avg_samples: 4, + avg_sync_freq_hz: 0.0, } } } @@ -302,59 +368,211 @@ impl StageAPhotodiodePlugin { } } + /// Moving-average window in samples: either the fixed sample count or, + /// when a sync frequency is set, one full period of that frequency — + /// which makes the mean independent of the modulation phase. + fn avg_window_samples(&self, rate_hz: u32) -> usize { + if self.avg_sync_freq_hz > 0.0 && rate_hz > 0 { + (f64::from(rate_hz) / self.avg_sync_freq_hz) + .round() + .max(1.0) as usize + } else { + self.avg_samples.max(1) + } + } + + /// Mean of the newest `avg_window_samples` codes (fewer while filling). + fn current_average_code(&self, state: &SharedState) -> Option { + if state.samples.is_empty() { + return None; + } + let window = self + .avg_window_samples(state.rate_hz) + .min(state.samples.len()); + let start = state.samples.len() - window; + let sum: u64 = state.samples.range(start..).map(|&c| u64::from(c)).sum(); + Some(sum as f64 / window as f64) + } + fn series_dataset(&self) -> Series1dV1 { - let (points, y_label) = match self.shared.lock() { - Ok(state) => { - let latest_ms = state.latest.map_or(0, |s| s.t_ms); - let window_ms = (self.window_s.max(0.5) * 1_000.0) as u64; - let cutoff = latest_ms.saturating_sub(window_ms); - let points: Vec = state - .samples - .iter() - .filter(|s| s.t_ms >= cutoff) - .map(|s| Series1dPoint { - x: (s.t_ms as f64 - latest_ms as f64) / 1_000.0, - y: self.display_volts(s.code), - }) - .collect(); - let label = match self.mode { - Mode::Raw => "photodiode [V]", - Mode::Excitation => "excitation I_tot − I_pd [V]", - }; - (points, label) - } - Err(_) => (Vec::new(), "photodiode [V]"), + let y_label = match self.mode { + Mode::Raw => "photodiode [V]", + Mode::Excitation => "excitation I_tot − I_pd [V]", }; - Series1dV1 { + let trace_name = match self.mode { + Mode::Raw => "photodiode", + Mode::Excitation => "excitation", + }; + let empty = |label: &str| Series1dV1 { x_label: "time before now [s]".into(), - y_label: y_label.into(), + y_label: label.into(), lines: vec![Series1dLine { - name: match self.mode { - Mode::Raw => "photodiode".into(), - Mode::Excitation => "excitation".into(), - }, - points, + name: trace_name.into(), + points: Vec::new(), }], + }; + let Ok(state) = self.shared.lock() else { + return empty(y_label); + }; + let total = state.samples.len(); + if total == 0 || state.rate_hz == 0 { + return empty(y_label); + } + let rate = f64::from(state.rate_hz); + + let visible = ((self.window_s.max(0.001) * rate) as usize) + .max(2) + .min(total); + let start = total - visible; + let latest_x_index = state.ring_first_index + total as u64 - 1; + let bucket_len = visible.div_ceil(MAX_PLOT_BUCKETS).max(1); + let decimating = bucket_len > 1; + + let avg_window = self.avg_window_samples(state.rate_hz); + let avg_enabled = avg_window > 1; + // Prime the running sum with up to `avg_window − 1` samples that + // precede the visible slice, so the average is correct from the + // first visible point on. + let prime_start = start.saturating_sub(avg_window - 1); + let mut avg_sum: u64 = 0; + let mut avg_count: usize = 0; + for &code in state.samples.range(prime_start..start) { + avg_sum += u64::from(code); + avg_count += 1; + } + + let mut mean_points = Vec::with_capacity(MAX_PLOT_BUCKETS + 1); + let mut min_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); + let mut max_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); + let mut avg_points = Vec::with_capacity(if avg_enabled { MAX_PLOT_BUCKETS + 1 } else { 0 }); + + let mut bucket_min = u16::MAX; + let mut bucket_max = u16::MIN; + let mut bucket_sum: u64 = 0; + let mut bucket_n: usize = 0; + for (offset, &code) in state.samples.range(start..).enumerate() { + let i = start + offset; + bucket_min = bucket_min.min(code); + bucket_max = bucket_max.max(code); + bucket_sum += u64::from(code); + bucket_n += 1; + if avg_enabled { + avg_sum += u64::from(code); + avg_count += 1; + if avg_count > avg_window { + avg_sum -= u64::from(state.samples[i - avg_window]); + avg_count -= 1; + } + } + if bucket_n == bucket_len || i == total - 1 { + let x = (state.ring_first_index + i as u64) as f64 / rate + - latest_x_index as f64 / rate; + mean_points.push(Series1dPoint { + x, + y: self.display_volts(bucket_sum as f64 / bucket_n as f64), + }); + if decimating { + // EXCITATION inverts the axis, so min/max swap roles. + let (low, high) = ( + self.display_volts(f64::from(bucket_min)), + self.display_volts(f64::from(bucket_max)), + ); + min_points.push(Series1dPoint { + x, + y: low.min(high), + }); + max_points.push(Series1dPoint { + x, + y: low.max(high), + }); + } + if avg_enabled { + avg_points.push(Series1dPoint { + x, + y: self.display_volts(avg_sum as f64 / avg_count as f64), + }); + } + bucket_min = u16::MAX; + bucket_max = u16::MIN; + bucket_sum = 0; + bucket_n = 0; + } + } + + let mut lines = vec![Series1dLine { + name: trace_name.into(), + points: mean_points, + }]; + if decimating { + lines.push(Series1dLine { + name: "min".into(), + points: min_points, + }); + lines.push(Series1dLine { + name: "max".into(), + points: max_points, + }); + } + if avg_enabled { + lines.push(Series1dLine { + name: format!("avg ({avg_window} spl)"), + points: avg_points, + }); + } + Series1dV1 { + x_label: "time before now [s]".into(), + y_label: y_label.into(), + lines, } } fn status_dataset(&self) -> TableDatasetV1 { - let (latest, stream_error) = match self.shared.lock() { - Ok(state) => (state.latest, state.error.clone()), - Err(_) => (None, None), + let (latest, rate_hz, average, integrity, stream_error) = match self.shared.lock() { + Ok(state) => ( + state.latest, + state.rate_hz, + self.current_average_code(&state), + format!( + "drops={} crc={} resync={} segments={}", + state.device_dropped, state.crc_failures, state.resync_bytes, state.segments + ), + state.error.clone(), + ), + Err(_) => (None, 0, None, String::new(), None), }; - let state = if self.connected() { + let state_text = if self.connected() { format!("reading ({})", self.port_hint) } else { "disconnected".into() }; + let rate_text = if rate_hz > 0 { + format!("{rate_hz} Sa/s") + } else { + "—".into() + }; let (code_text, value_text) = match latest { Some(sample) => ( - format!("{:.1}", sample.code), - format!("{:.4} V", self.display_volts(sample.code)), + format!("{sample}"), + format!("{:.4} V", self.display_volts(f64::from(sample))), ), None => ("—".into(), "—".into()), }; + let avg_text = match average { + Some(code) => { + let window = self.avg_window_samples(rate_hz); + format!( + "{:.4} V ({} spl ≈ {:.2} ms)", + self.display_volts(code), + window, + if rate_hz > 0 { + window as f64 * 1_000.0 / f64::from(rate_hz) + } else { + 0.0 + } + ) + } + None => "—".into(), + }; let error = stream_error .or_else(|| self.last_error.clone()) .unwrap_or_default(); @@ -364,10 +582,13 @@ impl StageAPhotodiodePlugin { }; TableDatasetV1 { columns: vec![ - text_column("state", state), + text_column("state", state_text), text_column("mode", self.mode.name().to_owned()), + text_column("rate", rate_text), text_column("code", code_text), text_column("value", value_text), + text_column("avg", avg_text), + text_column("integrity", integrity), text_column("error", error), ], } @@ -383,8 +604,11 @@ impl StageAPhotodiodePlugin { columns: vec![ column("state", "State"), column("mode", "Mode"), + column("rate", "Rate"), column("code", "ADC code"), column("value", "Value"), + column("avg", "Moving avg"), + column("integrity", "Integrity"), column("error", "Last error"), ], ..TableSchema::default() @@ -405,8 +629,9 @@ fn serial_ports() -> Vec { .unwrap_or_default() } -/// Finds the Teensy stream port: the dual-serial firmware free-runs `PD` -/// lines on exactly one of the enumerated ports, so listen briefly on each. +/// Finds the Teensy stream port: the dual-serial firmware free-runs PDA1 +/// `SamplesU16` frames on exactly one of the enumerated ports, so listen +/// briefly on each. fn resolve_auto_port() -> Result { let candidates = serial_ports(); if candidates.is_empty() { @@ -418,12 +643,14 @@ fn resolve_auto_port() -> Result { } } Err(format!( - "no port streamed PD lines within 500 ms (tried {})", + "no port streamed PDA1 sample frames within 500 ms (tried {})", candidates.join(", ") )) } -/// True when `path` produces a parsable `PD …` line within the probe window. +/// True when `path` produces a CRC-clean `SamplesU16` frame within the probe +/// window. The command port emits frames too, but only control replies and +/// acquisition data — unsolicited sample frames identify the stream port. fn probe_pd_stream(path: &str) -> bool { let Ok(mut port) = serialport::new(path, 115_200) .timeout(Duration::from_millis(100)) @@ -432,20 +659,18 @@ fn probe_pd_stream(path: &str) -> bool { return false; }; let deadline = Instant::now() + Duration::from_millis(500); - let mut collected: Vec = Vec::new(); - let mut buf = [0_u8; 512]; + let mut parser = FrameParser::default(); + let mut buf = [0_u8; 4_096]; while Instant::now() < deadline { match port.read(&mut buf) { Ok(read) if read > 0 => { - collected.extend_from_slice(&buf[..read]); - if String::from_utf8_lossy(&collected) - .lines() - .any(|line| parse_pd_line(line).is_some()) - { - return true; - } - if collected.len() > 8_192 { - collected.drain(..4_096); + parser.extend(&buf[..read]); + while let Some(event) = parser.next_event() { + if let ParseEvent::Frame(frame) = event { + if frame.samples().is_some() { + return true; + } + } } } Ok(_) => {} @@ -511,7 +736,7 @@ impl Plugin for StageAPhotodiodePlugin { } fn description(&self) -> &'static str { - "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot − I_pd with a user-set reference." + "Live photodiode readout (SMA5/pin 18/A4) from the Teensy PDA1 stream port at the full stream rate: raw values or excitation power I_exc = I_tot − I_pd with a user-set reference." } fn enabled(&self) -> bool { @@ -561,9 +786,10 @@ impl Plugin for StageAPhotodiodePlugin { sections: vec![SettingsSection { label: "Photodiode readout".into(), description: Some( - "Reads the free-running PD stream on the Teensy's SECOND serial port. \ - EXCITATION shows I_exc = I_tot − I_pd: the diode sits behind the PBS and \ - sees the light removed from the excitation beam." + "Reads the free-running PDA1 sample stream on the Teensy's SECOND serial \ + port (firmware 0.4.0+, 20 kSa/s default). EXCITATION shows \ + I_exc = I_tot − I_pd: the diode sits behind the PBS and sees the light \ + removed from the excitation beam." .into(), ), default_open: true, @@ -573,8 +799,8 @@ impl Plugin for StageAPhotodiodePlugin { label: "Port".into(), tooltip: Some( "auto (recommended) listens on the attached usbmodem ports and \ - picks the one streaming PD lines — the Teensy stream port; \ - mock = synthetic data" + picks the one streaming PDA1 sample frames — the Teensy stream \ + port; mock = synthetic data" .into(), ), kind: SettingKind::Enum { @@ -621,14 +847,48 @@ impl Plugin for StageAPhotodiodePlugin { SettingItem { key: "window_s".into(), label: "Chart window".into(), - tooltip: Some("Seconds of history shown in the live chart".into()), + tooltip: Some( + "Seconds of history shown in the live chart. Short windows \ + (≤ 50 ms) resolve individual modulation cycles at 20 kSa/s." + .into(), + ), kind: SettingKind::F64Drag { - min: 1.0, + min: 0.01, max: 120.0, - speed: 1.0, + speed: 0.05, default: self.window_s, }, }, + SettingItem { + key: "avg_samples".into(), + label: "Average window".into(), + tooltip: Some( + "Moving-average window in samples (1 = off). Ignored while \ + 'Average sync frequency' is set." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 1_000_000, + default: self.avg_samples as i64, + }, + }, + SettingItem { + key: "avg_sync_freq_hz".into(), + label: "Average sync frequency".into(), + tooltip: Some( + "0 = off. When set to the modulation frequency (Hz), the moving \ + average spans exactly one full period (window = rate / f), so the \ + mean level no longer depends on the modulation phase." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 100_000.0, + speed: 1.0, + default: self.avg_sync_freq_hz, + }, + }, ], }], } @@ -655,6 +915,8 @@ impl Plugin for StageAPhotodiodePlugin { } "reference_volts" => Some(json!(self.reference_volts)), "window_s" => Some(json!(self.window_s)), + "avg_samples" => Some(json!(self.avg_samples)), + "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), _ => None, } } @@ -690,7 +952,17 @@ impl Plugin for StageAPhotodiodePlugin { } "window_s" => { let seconds = value.as_f64().ok_or("window_s must be a number")?; - self.window_s = seconds.clamp(1.0, 120.0); + self.window_s = seconds.clamp(0.01, 120.0); + Ok(()) + } + "avg_samples" => { + let samples = value.as_i64().ok_or("avg_samples must be an integer")?; + self.avg_samples = samples.clamp(1, 1_000_000) as usize; + Ok(()) + } + "avg_sync_freq_hz" => { + let freq = value.as_f64().ok_or("avg_sync_freq_hz must be a number")?; + self.avg_sync_freq_hz = freq.clamp(0.0, 100_000.0); Ok(()) } _ => Err(format!("unknown setting: {key}")), @@ -699,30 +971,47 @@ impl Plugin for StageAPhotodiodePlugin { fn status_entries(&self) -> Vec { let mut entries = Vec::new(); - let (latest, stream_error) = match self.shared.lock() { - Ok(state) => (state.latest, state.error.clone()), - Err(_) => (None, None), + let (latest, rate_hz, average, stream_error) = match self.shared.lock() { + Ok(state) => ( + state.latest, + state.rate_hz, + self.current_average_code(&state), + state.error.clone(), + ), + Err(_) => (None, 0, None, None), }; entries.push(StatusEntry::Text(if self.connected() { - format!("Photodiode: reading ({})", self.port_hint) + if rate_hz > 0 { + format!("Photodiode: reading ({}) @ {rate_hz} Sa/s", self.port_hint) + } else { + format!("Photodiode: reading ({})", self.port_hint) + } } else { "Photodiode: disconnected".into() })); if let Some(sample) = latest { match self.mode { Mode::Raw => entries.push(StatusEntry::Text(format!( - "PD: code={:.1} ({:.4} V)", - sample.code, - code_to_volts(sample.code) + "PD: code={sample} ({:.4} V)", + code_to_volts(f64::from(sample)) ))), Mode::Excitation => entries.push(StatusEntry::Text(format!( "Excitation: {:.4} V (I_tot={:.3} V, PD={:.4} V)", - self.display_volts(sample.code), + self.display_volts(f64::from(sample)), self.reference_volts, - code_to_volts(sample.code) + code_to_volts(f64::from(sample)) ))), } } + if let Some(average) = average { + let window = self.avg_window_samples(rate_hz); + if window > 1 { + entries.push(StatusEntry::Text(format!( + "Avg ({window} spl): {:.4} V", + self.display_volts(average) + ))); + } + } if let Some(error) = stream_error.or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } @@ -790,19 +1079,164 @@ export_plugin!(StageAPhotodiodePlugin); #[cfg(test)] mod tests { use super::*; + use stage_a_io::{Frame, FrameHeader, FrameType}; + + fn sample_frame(sequence: u32, first_index: u64, rate_hz: u32, codes: &[u16]) -> Vec { + let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); + Frame::build( + FrameHeader { + version: stage_a_io::wire::PROTOCOL_VERSION, + frame_type: FrameType::SamplesU16, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: first_index, + sample_rate_hz: rate_hz, + dropped_samples: 0, + crc32: 0, + }, + payload, + ) + .to_bytes() + } + + fn ingest_bytes(state: &mut SharedState, bytes: &[u8]) { + let mut parser = FrameParser::default(); + parser.extend(bytes); + while let Some(event) = parser.next_event() { + match event { + ParseEvent::Frame(frame) => { + let codes = frame.samples().expect("sample frame"); + state.ingest( + frame.header.first_sample_index, + frame.header.sample_rate_hz, + frame.header.dropped_samples, + &codes, + ); + } + ParseEvent::Corruption { .. } => panic!("clean test stream"), + } + } + } + + #[test] + fn ingests_contiguous_frames_and_restarts_on_gaps() { + let mut state = SharedState::default(); + ingest_bytes(&mut state, &sample_frame(0, 0, 20_000, &[1, 2, 3, 4])); + ingest_bytes(&mut state, &sample_frame(1, 4, 20_000, &[5, 6])); + assert_eq!(state.samples.len(), 6); + assert_eq!(state.ring_first_index, 0); + assert_eq!(state.segments, 0); + assert_eq!(state.latest, Some(6)); + + // A sample-index jump (dropped block, acquisition handover) restarts + // the segment instead of silently misaligning the time base. + ingest_bytes(&mut state, &sample_frame(2, 100, 20_000, &[7, 8])); + assert_eq!(state.samples.len(), 2); + assert_eq!(state.ring_first_index, 100); + assert_eq!(state.segments, 1); + + // So does a rate change (mirrored acquisition at another rate). + ingest_bytes(&mut state, &sample_frame(3, 102, 50_000, &[9])); + assert_eq!(state.samples.len(), 1); + assert_eq!(state.rate_hz, 50_000); + assert_eq!(state.segments, 2); + } #[test] - fn parses_firmware_stream_lines() { - let sample = parse_pd_line("PD code=1042.3 n=16 t_ms=123456\n").expect("valid line"); - assert!((sample.code - 1042.3).abs() < 1e-9); - assert_eq!(sample.t_ms, 123_456); + fn ring_is_bounded_by_duration() { + let mut state = SharedState::default(); + let rate = 1_000; // capacity = 130_000 samples + let cap = SharedState::ring_capacity(rate); + let block: Vec = (0..1_000).map(|i| (i % 4_096) as u16).collect(); + let mut index = 0_u64; + for _ in 0..(cap / block.len() + 5) { + state.ingest(index, rate, 0, &block); + index += block.len() as u64; + } + assert_eq!(state.samples.len(), cap); + assert_eq!( + state.ring_first_index + state.samples.len() as u64, + index, + "eviction keeps indexes aligned" + ); + assert_eq!(state.segments, 0, "eviction is not a discontinuity"); + } - assert!(parse_pd_line("garbage").is_none()); - assert!(parse_pd_line("PD code=abc n=16 t_ms=1").is_none()); - assert!(parse_pd_line("PD code=10 n=16").is_none(), "t_ms required"); - // Codes are clamped into the 12-bit range. - let clamped = parse_pd_line("PD code=9999 n=1 t_ms=5").expect("parses"); - assert_eq!(clamped.code, ADC_MAX_CODE); + #[test] + fn moving_average_window_follows_the_sync_frequency() { + let mut plugin = StageAPhotodiodePlugin::default(); + assert_eq!(plugin.avg_window_samples(20_000), 4, "sample default"); + plugin + .set_setting("avg_samples", json!(16)) + .expect("valid setting"); + assert_eq!(plugin.avg_window_samples(20_000), 16); + // One full period of a 2 kHz modulation at 20 kSa/s = 10 samples. + plugin + .set_setting("avg_sync_freq_hz", json!(2_000.0)) + .expect("valid setting"); + assert_eq!(plugin.avg_window_samples(20_000), 10); + // Faster than the sample rate clamps to a single sample. + plugin + .set_setting("avg_sync_freq_hz", json!(50_000.0)) + .expect("valid setting"); + assert_eq!(plugin.avg_window_samples(20_000), 1); + } + + #[test] + fn current_average_uses_the_newest_window() { + let plugin = StageAPhotodiodePlugin::default(); // window = 4 samples + let mut state = SharedState::default(); + state.ingest(0, 20_000, 0, &[0, 0, 0, 0, 100, 200, 300, 400]); + let average = plugin.current_average_code(&state).expect("has samples"); + assert!((average - 250.0).abs() < 1e-9); + } + + #[test] + fn series_dataset_decimates_with_envelope_and_average() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("window_s", json!(120.0)).unwrap(); + plugin.set_setting("avg_samples", json!(50)).unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + let codes: Vec = (0..40_000_u32).map(|i| (i % 4_000) as u16).collect(); + state.ingest(0, 20_000, 0, &codes); + } + let series = plugin.series_dataset(); + let names: Vec<&str> = series.lines.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["photodiode", "min", "max", "avg (50 spl)"]); + for line in &series.lines { + assert!( + line.points.len() <= MAX_PLOT_BUCKETS + 1, + "{} has {} points", + line.name, + line.points.len() + ); + assert!(!line.points.is_empty()); + } + // min ≤ mean ≤ max, and x is "seconds before now" ending at 0. + let (mean, min, max) = (&series.lines[0], &series.lines[1], &series.lines[2]); + for ((m, lo), hi) in mean.points.iter().zip(&min.points).zip(&max.points) { + assert!(lo.y <= m.y + 1e-9 && m.y <= hi.y + 1e-9); + } + let last_x = mean.points.last().unwrap().x; + assert!(last_x.abs() < 1e-9, "trace ends at now, got {last_x}"); + } + + #[test] + fn short_windows_render_raw_samples_without_envelope() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("window_s", json!(0.01)).unwrap(); // 200 samples at 20 kSa/s + plugin.set_setting("avg_samples", json!(1)).unwrap(); // average off + { + let mut state = plugin.shared.lock().unwrap(); + let codes: Vec = (0..1_000_u32).map(|i| (i % 4_000) as u16).collect(); + state.ingest(0, 20_000, 0, &codes); + } + let series = plugin.series_dataset(); + let names: Vec<&str> = series.lines.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["photodiode"], "no envelope, no average"); + assert_eq!(series.lines[0].points.len(), 200); } #[test] @@ -820,12 +1254,15 @@ mod tests { #[test] fn mock_reader_fills_the_ring_and_series() { - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = StageAPhotodiodePlugin { + port_hint: "mock".into(), + ..Default::default() + }; plugin.connect(); let deadline = Instant::now() + Duration::from_secs(2); loop { let count = plugin.shared.lock().unwrap().samples.len(); - if count >= 5 { + if count >= MOCK_BLOCK_SAMPLES { break; } assert!(Instant::now() < deadline, "mock reader produced no data"); @@ -833,6 +1270,7 @@ mod tests { } let series = plugin.series_dataset(); assert!(!series.lines[0].points.is_empty()); + assert_eq!(plugin.shared.lock().unwrap().rate_hz, MOCK_RATE_HZ); let generation = plugin.generation.load(Ordering::Relaxed); assert!(generation > 1); plugin.disconnect(); @@ -862,17 +1300,4 @@ mod tests { .expect("name accepted"); assert_eq!(plugin.mode, Mode::Raw); } - - #[test] - fn ring_is_bounded() { - let mut state = SharedState::default(); - for i in 0..(RING_CAPACITY + 100) { - state.push(PdSample { - t_ms: i as u64, - code: 1.0, - }); - } - assert_eq!(state.samples.len(), RING_CAPACITY); - assert_eq!(state.latest.unwrap().t_ms, (RING_CAPACITY + 99) as u64); - } } diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index f7e677a..e9c9377 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -6,6 +6,7 @@ use std::io; use std::sync::{Arc, Mutex}; +#[cfg(feature = "hardware")] use std::time::Duration; pub trait Transport: Send { From 25535811ec79cd7426a800cfa31194c2aa0c781d Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 16:52:19 +0200 Subject: [PATCH 16/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20monito?= =?UTF-8?q?r-cache=20snapshots=20and=20disk=20recording=20to=20the=20photo?= =?UTF-8?q?diode=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two save modes behind one Data settings section: - monitor cache: the raw ring is now cache_s seconds long (default 20 s, 1-130 s) and a Save-cache-snapshot button dumps it as pd_cache_.csv (sample_index, t_s on the device clock, raw code, raw volts) plus a JSON sidecar carrying rate, integrity counters, display mode, and reference so derived quantities stay reproducible - recording: a Record toggle tees every incoming SamplesU16 frame verbatim to pd_rec_.pdq via stage-a-io's PdqWriter from the reader thread; stopping (or disabling the plugin) finalizes the file and writes a sidecar with per-recording integrity deltas and validity; the mock synthesizes identical wire frames so recordings parse the same without hardware - data_dir uses the new Path setting kind; record/save failures surface through status entries like connect errors; status shows a live REC indicator with recorded seconds Verified: cargo fmt, clippy -D warnings, 13 plugin tests green. --- plugins/stage-a-photodiode/src/lib.rs | 780 ++++++++++++++++++++++---- 1 file changed, 676 insertions(+), 104 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index a966045..6f91025 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -22,12 +22,15 @@ //! which makes the mean independent of the modulation phase. use std::collections::VecDeque; -use std::io::Read; +use std::fs::File; +use std::io::{BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; +use augur_plugin_api::PathDialogKind; use augur_plugin_api::{ export_plugin, EventStoreHandle, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, @@ -36,7 +39,7 @@ use augur_plugin_api::{ TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{FrameParser, ParseEvent}; +use stage_a_io::{FrameParser, ParseEvent, PdqWriter, StreamIntegrity}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; @@ -45,8 +48,10 @@ const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; -/// Longest raw history kept, in seconds of samples at the active stream rate. -const RING_SECONDS: f64 = 130.0; +/// Default monitor cache, in seconds of samples at the active stream rate +/// (user-settable 1–130 s). +const DEFAULT_CACHE_SECONDS: f64 = 20.0; +const MAX_CACHE_SECONDS: f64 = 130.0; /// Absolute sample cap guarding against absurd advertised rates (8 MiB of /// codes at most). const RING_MAX_SAMPLES: usize = 4_000_000; @@ -82,7 +87,6 @@ impl Mode { } } -#[derive(Default)] struct SharedState { /// Sample rate of the current segment (from the frame headers). rate_hz: u32, @@ -96,12 +100,31 @@ struct SharedState { resync_bytes: u64, /// Segment restarts observed (rate changes, index jumps, reconnects). segments: u64, + /// Monitor-cache length driving ring eviction (user setting). + cache_seconds: f64, error: Option, } +impl Default for SharedState { + fn default() -> Self { + Self { + rate_hz: 0, + ring_first_index: 0, + samples: VecDeque::new(), + latest: None, + device_dropped: 0, + crc_failures: 0, + resync_bytes: 0, + segments: 0, + cache_seconds: DEFAULT_CACHE_SECONDS, + error: None, + } + } +} + impl SharedState { - fn ring_capacity(rate_hz: u32) -> usize { - ((f64::from(rate_hz.max(1)) * RING_SECONDS) as usize).min(RING_MAX_SAMPLES) + fn ring_capacity(&self, rate_hz: u32) -> usize { + ((f64::from(rate_hz.max(1)) * self.cache_seconds) as usize).clamp(2, RING_MAX_SAMPLES) } /// Ingests one `SamplesU16` frame. Any discontinuity — rate change, @@ -128,7 +151,7 @@ impl SharedState { let excess = self .samples .len() - .saturating_sub(Self::ring_capacity(rate_hz)); + .saturating_sub(self.ring_capacity(rate_hz)); if excess > 0 { self.samples.drain(..excess); self.ring_first_index += excess as u64; @@ -136,6 +159,66 @@ impl SharedState { } } +/// One active disk recording: every clean `SamplesU16` frame is appended +/// verbatim to a `.pdq` file; `stop` writes the JSON sidecar next to it. +struct RecordingSink { + writer: PdqWriter, + pdq_path: PathBuf, + started_slug: String, + samples_written: u64, + write_error: Option, + /// Integrity counters at recording start, so the sidecar reports deltas + /// for exactly the recorded span. + start_crc_failures: u64, + start_resync_bytes: u64, + start_device_dropped: u32, + start_segments: u64, +} + +type SharedRecording = Arc>>; + +fn record_frame(recording: &SharedRecording, frame: &stage_a_io::Frame, samples: usize) { + let Ok(mut slot) = recording.lock() else { + return; + }; + let Some(sink) = slot.as_mut() else { + return; + }; + if sink.write_error.is_some() { + return; + } + match sink.writer.write_frame(frame) { + Ok(()) => sink.samples_written += samples as u64, + Err(err) => sink.write_error = Some(format!("recording write failed: {err}")), + } +} + +/// `YYYYmmdd_HHMMSS` in UTC without a date-time dependency (Howard Hinnant's +/// civil-from-days algorithm). +fn timestamp_slug() -> String { + let seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let days = (seconds / 86_400) as i64; + let (secs_of_day, z) = ((seconds % 86_400) as u32, days + 719_468); + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097) as u64; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = if month <= 2 { year + 1 } else { year }; + format!( + "{year:04}{month:02}{day:02}_{:02}{:02}{:02}", + secs_of_day / 3_600, + (secs_of_day / 60) % 60, + secs_of_day % 60 + ) +} + /// Background reader owning the stream port (or the mock generator). struct Reader { stop: Arc, @@ -147,6 +230,7 @@ impl Reader { path: String, shared: Arc>, generation: Arc, + recording: SharedRecording, ) -> Result { let port = serialport::new(&path, 115_200) .timeout(Duration::from_millis(50)) @@ -156,7 +240,7 @@ impl Reader { let thread_stop = Arc::clone(&stop); let join = std::thread::Builder::new() .name("stage-a-photodiode".into()) - .spawn(move || read_frames(port, &shared, &generation, &thread_stop)) + .spawn(move || read_frames(port, &shared, &generation, &recording, &thread_stop)) .expect("spawning the photodiode reader thread must succeed"); Ok(Self { stop, @@ -166,7 +250,11 @@ impl Reader { /// Hardware-free source: synthesizes a noisy 5 Hz sine around 1 V in /// firmware-sized blocks at the firmware's default stream rate. - fn spawn_mock(shared: Arc>, generation: Arc) -> Self { + fn spawn_mock( + shared: Arc>, + generation: Arc, + recording: SharedRecording, + ) -> Self { let stop = Arc::new(AtomicBool::new(false)); let thread_stop = Arc::clone(&stop); let join = std::thread::Builder::new() @@ -174,6 +262,7 @@ impl Reader { .spawn(move || { let start = Instant::now(); let mut next_index: u64 = 0; + let mut sequence: u32 = 0; while !thread_stop.load(Ordering::Relaxed) { let target = (start.elapsed().as_secs_f64() * f64::from(MOCK_RATE_HZ)) as u64; let mut produced = false; @@ -181,6 +270,14 @@ impl Reader { let codes: Vec = (0..MOCK_BLOCK_SAMPLES) .map(|i| mock_code(next_index + i as u64)) .collect(); + // Recordings capture real wire frames; synthesize the + // identical framing so mock recordings parse the same. + record_frame( + &recording, + &mock_sample_frame(sequence, next_index, &codes), + codes.len(), + ); + sequence = sequence.wrapping_add(1); if let Ok(mut state) = shared.lock() { state.ingest(next_index, MOCK_RATE_HZ, 0, &codes); } @@ -201,6 +298,24 @@ impl Reader { } } +fn mock_sample_frame(sequence: u32, first_index: u64, codes: &[u16]) -> stage_a_io::Frame { + let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); + stage_a_io::Frame::build( + stage_a_io::FrameHeader { + version: stage_a_io::wire::PROTOCOL_VERSION, + frame_type: stage_a_io::FrameType::SamplesU16, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: first_index, + sample_rate_hz: MOCK_RATE_HZ, + dropped_samples: 0, + crc32: 0, + }, + payload, + ) +} + /// Deterministic mock sample: 1 V ± 0.5 V sine at 5 Hz plus ~20 mV of hash /// noise, so the moving-average indicator has something to smooth. fn mock_code(index: u64) -> u16 { @@ -225,6 +340,7 @@ fn read_frames( mut port: Box, shared: &Mutex, generation: &AtomicU64, + recording: &SharedRecording, stop: &AtomicBool, ) { let mut parser = FrameParser::default(); @@ -251,6 +367,7 @@ fn read_frames( let Some(codes) = frame.samples() else { continue; // Control/summary frames are not expected here. }; + record_frame(recording, &frame, codes.len()); if let Ok(mut state) = shared.lock() { state.ingest( frame.header.first_sample_index, @@ -284,7 +401,10 @@ pub struct StageAPhotodiodePlugin { reader: Option, shared: Arc>, generation: Arc, + recording: SharedRecording, last_error: Option, + /// One-line feedback about the most recent save/recording action. + last_save_note: Option, // -- settings -- connect_requested: bool, port_hint: String, @@ -293,6 +413,7 @@ pub struct StageAPhotodiodePlugin { window_s: f64, avg_samples: usize, avg_sync_freq_hz: f64, + data_dir: String, } impl Default for StageAPhotodiodePlugin { @@ -302,7 +423,9 @@ impl Default for StageAPhotodiodePlugin { reader: None, shared: Arc::new(Mutex::new(SharedState::default())), generation: Arc::new(AtomicU64::new(1)), + recording: Arc::new(Mutex::new(None)), last_error: None, + last_save_note: None, connect_requested: false, port_hint: "auto".into(), mode: Mode::Raw, @@ -310,6 +433,7 @@ impl Default for StageAPhotodiodePlugin { window_s: 10.0, avg_samples: 4, avg_sync_freq_hz: 0.0, + data_dir: String::new(), } } } @@ -331,6 +455,7 @@ impl StageAPhotodiodePlugin { self.reader = Some(Reader::spawn_mock( Arc::clone(&self.shared), Arc::clone(&self.generation), + Arc::clone(&self.recording), )); return; } @@ -345,7 +470,12 @@ impl StageAPhotodiodePlugin { } else { self.port_hint.clone() }; - match Reader::spawn_serial(path, Arc::clone(&self.shared), Arc::clone(&self.generation)) { + match Reader::spawn_serial( + path, + Arc::clone(&self.shared), + Arc::clone(&self.generation), + Arc::clone(&self.recording), + ) { Ok(reader) => self.reader = Some(reader), Err(err) => { self.last_error = Some(err); @@ -360,6 +490,183 @@ impl StageAPhotodiodePlugin { self.generation.fetch_add(1, Ordering::Relaxed); } + fn recording_active(&self) -> bool { + self.recording + .lock() + .map(|slot| slot.is_some()) + .unwrap_or(false) + } + + fn resolved_data_dir(&self) -> Result { + if self.data_dir.trim().is_empty() { + return Err("set the data directory first (Data section)".into()); + } + Ok(PathBuf::from(self.data_dir.trim())) + } + + fn start_recording(&mut self) -> Result<(), String> { + if self.recording_active() { + return Ok(()); + } + let dir = self.resolved_data_dir()?; + let slug = timestamp_slug(); + let pdq_path = dir.join(format!("pd_rec_{slug}.pdq")); + let writer = PdqWriter::create(&pdq_path) + .map_err(|err| format!("creating {} failed: {err}", pdq_path.display()))?; + let (crc, resync, dropped, segments) = match self.shared.lock() { + Ok(state) => ( + state.crc_failures, + state.resync_bytes, + state.device_dropped, + state.segments, + ), + Err(_) => (0, 0, 0, 0), + }; + let sink = RecordingSink { + writer, + pdq_path: pdq_path.clone(), + started_slug: slug, + samples_written: 0, + write_error: None, + start_crc_failures: crc, + start_resync_bytes: resync, + start_device_dropped: dropped, + start_segments: segments, + }; + if let Ok(mut slot) = self.recording.lock() { + *slot = Some(sink); + } + self.last_save_note = Some(format!("recording → {}", pdq_path.display())); + Ok(()) + } + + fn stop_recording(&mut self) -> Result<(), String> { + let Some(sink) = self.recording.lock().ok().and_then(|mut slot| slot.take()) else { + return Ok(()); + }; + let (rate_hz, crc, resync, dropped, segments) = match self.shared.lock() { + Ok(state) => ( + state.rate_hz, + state.crc_failures, + state.resync_bytes, + state.device_dropped, + state.segments, + ), + Err(_) => (0, 0, 0, 0, 0), + }; + let integrity = StreamIntegrity { + skipped_bytes: resync.saturating_sub(sink.start_resync_bytes), + crc_failures: crc.saturating_sub(sink.start_crc_failures), + sequence_gaps: segments.saturating_sub(sink.start_segments), + dropped_samples: u64::from(dropped.saturating_sub(sink.start_device_dropped)), + }; + let write_error = sink.write_error.clone(); + let started = sink.started_slug.clone(); + let samples = sink.samples_written; + let summary = sink + .writer + .finish(integrity) + .map_err(|err| format!("finishing recording failed: {err}"))?; + let sidecar = json!({ + "kind": "recording", + "started_utc": started, + "stopped_utc": timestamp_slug(), + "port": self.port_hint, + "sample_rate_hz": rate_hz, + "samples_written": samples, + "pdq_path": summary.path, + "pdq_frames": summary.frames_written, + "pdq_bytes": summary.bytes_written, + "pdq_crc32": summary.file_crc32, + "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, + "display_mode": self.mode.name(), + "reference_volts": self.reference_volts, + "integrity": { + "resync_bytes": summary.integrity.skipped_bytes, + "crc_failures": summary.integrity.crc_failures, + "segment_restarts": summary.integrity.sequence_gaps, + "device_dropped_samples": summary.integrity.dropped_samples, + }, + "valid": summary.valid && write_error.is_none(), + "write_error": write_error, + }); + let sidecar_path = sink.pdq_path.with_extension("json"); + write_json(&sidecar_path, &sidecar)?; + self.last_save_note = Some(format!( + "saved recording {} ({} samples)", + sink.pdq_path.display(), + samples + )); + Ok(()) + } + + /// Dumps the current monitor cache (ring) as CSV + JSON sidecar. Raw + /// codes and raw volts only — mode/reference land in the sidecar so + /// EXCITATION values stay derivable without baking display state into + /// the data. + fn save_cache_snapshot(&mut self) -> Result<(), String> { + let dir = self.resolved_data_dir()?; + let slug = timestamp_slug(); + let csv_path = dir.join(format!("pd_cache_{slug}.csv")); + let state = self + .shared + .lock() + .map_err(|_| "photodiode state lock poisoned".to_owned())?; + if state.samples.is_empty() || state.rate_hz == 0 { + return Err("no samples cached yet".into()); + } + std::fs::create_dir_all(&dir) + .map_err(|err| format!("creating {} failed: {err}", dir.display()))?; + let file = File::create(&csv_path) + .map_err(|err| format!("creating {} failed: {err}", csv_path.display()))?; + let mut writer = BufWriter::new(file); + let rate = f64::from(state.rate_hz); + writeln!(writer, "sample_index,t_s,code,volts") + .map_err(|err| format!("writing CSV failed: {err}"))?; + for (offset, &code) in state.samples.iter().enumerate() { + let index = state.ring_first_index + offset as u64; + writeln!( + writer, + "{index},{:.9},{code},{:.6}", + index as f64 / rate, + code_to_volts(f64::from(code)) + ) + .map_err(|err| format!("writing CSV failed: {err}"))?; + } + writer + .flush() + .map_err(|err| format!("writing CSV failed: {err}"))?; + + let sidecar = json!({ + "kind": "cache_snapshot", + "created_utc": slug, + "port": self.port_hint, + "sample_rate_hz": state.rate_hz, + "samples": state.samples.len(), + "first_sample_index": state.ring_first_index, + "cache_seconds": state.cache_seconds, + "csv_path": csv_path, + "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, + "display_mode": self.mode.name(), + "reference_volts": self.reference_volts, + "time_base": "t_s = sample_index / sample_rate_hz, device clock, segment-relative", + "integrity": { + "resync_bytes": state.resync_bytes, + "crc_failures": state.crc_failures, + "segment_restarts": state.segments, + "device_dropped_samples": state.device_dropped, + }, + }); + let sample_count = state.samples.len(); + drop(state); + write_json(&csv_path.with_extension("json"), &sidecar)?; + self.last_save_note = Some(format!( + "saved cache {} ({sample_count} samples)", + csv_path.display() + )); + Ok(()) + } + /// Value shown for one sample under the current mode, in volts. fn display_volts(&self, code: f64) -> f64 { match self.mode { @@ -616,6 +923,12 @@ impl StageAPhotodiodePlugin { } } +fn write_json(path: &Path, value: &Value) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(value) + .map_err(|err| format!("serializing sidecar failed: {err}"))?; + std::fs::write(path, bytes).map_err(|err| format!("writing {} failed: {err}", path.display())) +} + fn serial_ports() -> Vec { serialport::available_ports() .map(|ports| { @@ -747,6 +1060,11 @@ impl Plugin for StageAPhotodiodePlugin { self.enabled = enabled; if !enabled { self.connect_requested = false; + // Finalize an active recording so the .pdq/.json pair is complete + // even when the plugin is disabled mid-run. + if let Err(err) = self.stop_recording() { + self.last_error = Some(err); + } self.disconnect(); } } @@ -783,114 +1101,184 @@ impl Plugin for StageAPhotodiodePlugin { .position(|m| *m == self.mode) .unwrap_or(0); SettingsSchema { - sections: vec![SettingsSection { - label: "Photodiode readout".into(), - description: Some( - "Reads the free-running PDA1 sample stream on the Teensy's SECOND serial \ + sections: vec![ + SettingsSection { + label: "Photodiode readout".into(), + description: Some( + "Reads the free-running PDA1 sample stream on the Teensy's SECOND serial \ port (firmware 0.4.0+, 20 kSa/s default). EXCITATION shows \ I_exc = I_tot − I_pd: the diode sits behind the PBS and sees the light \ removed from the excitation beam." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "auto (recommended) listens on the attached usbmodem ports and \ + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "auto (recommended) listens on the attached usbmodem ports and \ picks the one streaming PDA1 sample frames — the Teensy stream \ port; mock = synthetic data" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, }, - }, - SettingItem { - key: "connect".into(), - label: "Connect".into(), - tooltip: Some( - "Opens/closes the stream port (read-only, no camera required).".into(), - ), - kind: SettingKind::Bool { - default: self.connect_requested, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the stream port (read-only, no camera required)." + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, }, - }, - SettingItem { - key: "mode".into(), - label: "Mode".into(), - tooltip: Some( - "RAW: ADC code and volts as measured. EXCITATION: I_tot − I_pd".into(), - ), - kind: SettingKind::Enum { - variants: mode_variants, - default: mode_default, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some( + "RAW: ADC code and volts as measured. EXCITATION: I_tot − I_pd" + .into(), + ), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, }, - }, - SettingItem { - key: "reference_volts".into(), - label: "Reference I_tot".into(), - tooltip: Some( - "Total power reference for EXCITATION mode, in photodiode volts: \ + SettingItem { + key: "reference_volts".into(), + label: "Reference I_tot".into(), + tooltip: Some( + "Total power reference for EXCITATION mode, in photodiode volts: \ the PD reading with the full beam diverted into the diode" - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: ADC_FULL_SCALE_VOLTS, - speed: 0.01, - default: self.reference_volts, + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: ADC_FULL_SCALE_VOLTS, + speed: 0.01, + default: self.reference_volts, + }, }, - }, - SettingItem { - key: "window_s".into(), - label: "Chart window".into(), - tooltip: Some( - "Seconds of history shown in the live chart. Short windows \ + SettingItem { + key: "window_s".into(), + label: "Chart window".into(), + tooltip: Some( + "Seconds of history shown in the live chart. Short windows \ (≤ 50 ms) resolve individual modulation cycles at 20 kSa/s." - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.01, - max: 120.0, - speed: 0.05, - default: self.window_s, + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 120.0, + speed: 0.05, + default: self.window_s, + }, }, - }, - SettingItem { - key: "avg_samples".into(), - label: "Average window".into(), - tooltip: Some( - "Moving-average window in samples (1 = off). Ignored while \ + SettingItem { + key: "avg_samples".into(), + label: "Average window".into(), + tooltip: Some( + "Moving-average window in samples (1 = off). Ignored while \ 'Average sync frequency' is set." - .into(), - ), - kind: SettingKind::I64Drag { - min: 1, - max: 1_000_000, - default: self.avg_samples as i64, + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 1_000_000, + default: self.avg_samples as i64, + }, }, - }, - SettingItem { - key: "avg_sync_freq_hz".into(), - label: "Average sync frequency".into(), - tooltip: Some( - "0 = off. When set to the modulation frequency (Hz), the moving \ + SettingItem { + key: "avg_sync_freq_hz".into(), + label: "Average sync frequency".into(), + tooltip: Some( + "0 = off. When set to the modulation frequency (Hz), the moving \ average spans exactly one full period (window = rate / f), so the \ mean level no longer depends on the modulation phase." - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 100_000.0, - speed: 1.0, - default: self.avg_sync_freq_hz, + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 100_000.0, + speed: 1.0, + default: self.avg_sync_freq_hz, + }, }, - }, - ], - }], + ], + }, + SettingsSection { + label: "Data".into(), + description: Some( + "Monitor cache and disk recording. The cache always holds the last \ + N seconds; recording tees every incoming frame to a .pdq file \ + (+ JSON sidecar) so length is disk-bound. CSV/PDQ store raw codes \ + and raw volts on the device clock; mode and reference go into the \ + sidecar." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "data_dir".into(), + label: "Data directory".into(), + tooltip: Some( + "Where recordings and cache snapshots are written.".into(), + ), + kind: SettingKind::Path { + dialog: PathDialogKind::Directory, + default: self.data_dir.clone(), + }, + }, + SettingItem { + key: "cache_s".into(), + label: "Cache length".into(), + tooltip: Some( + "Seconds of raw samples kept in memory for the chart and \ + cache snapshots." + .into(), + ), + kind: SettingKind::F64Drag { + min: 1.0, + max: MAX_CACHE_SECONDS, + speed: 1.0, + default: self + .shared + .lock() + .map(|state| state.cache_seconds) + .unwrap_or(DEFAULT_CACHE_SECONDS), + }, + }, + SettingItem { + key: "record".into(), + label: "Record to disk".into(), + tooltip: Some( + "Start/stop appending every incoming sample frame to \ + pd_rec_.pdq; stopping writes the JSON sidecar." + .into(), + ), + kind: SettingKind::Bool { + default: self.recording_active(), + }, + }, + SettingItem { + key: "save_snapshot".into(), + label: "Save cache snapshot".into(), + tooltip: Some( + "Write the current cache as pd_cache_.csv \ + (+ JSON sidecar)." + .into(), + ), + kind: SettingKind::Button, + }, + ], + }, + ], } } @@ -917,6 +1305,15 @@ impl Plugin for StageAPhotodiodePlugin { "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), + "data_dir" => Some(json!(self.data_dir)), + "cache_s" => Some(json!(self + .shared + .lock() + .map(|state| state.cache_seconds) + .unwrap_or(DEFAULT_CACHE_SECONDS))), + "record" => Some(json!(self.recording_active())), + // Momentary trigger: never reports as pressed. + "save_snapshot" => Some(json!(false)), _ => None, } } @@ -965,6 +1362,45 @@ impl Plugin for StageAPhotodiodePlugin { self.avg_sync_freq_hz = freq.clamp(0.0, 100_000.0); Ok(()) } + "data_dir" => { + self.data_dir = value + .as_str() + .ok_or("data_dir must be a string")? + .to_owned(); + Ok(()) + } + "cache_s" => { + let seconds = value.as_f64().ok_or("cache_s must be a number")?; + if let Ok(mut state) = self.shared.lock() { + state.cache_seconds = seconds.clamp(1.0, MAX_CACHE_SECONDS); + } + Ok(()) + } + "record" => { + let requested = value.as_bool().ok_or("record must be a boolean")?; + // Failures surface through status entries (like `connect`), + // so a missing data directory doesn't read as a broken UI. + let result = if requested { + self.start_recording() + } else { + self.stop_recording() + }; + if let Err(err) = result { + self.last_error = Some(err); + } else { + self.last_error = None; + } + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + "save_snapshot" => { + match self.save_cache_snapshot() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(()) + } _ => Err(format!("unknown setting: {key}")), } } @@ -1012,6 +1448,25 @@ impl Plugin for StageAPhotodiodePlugin { ))); } } + if self.recording_active() { + let (samples, path) = self + .recording + .lock() + .ok() + .and_then(|slot| { + slot.as_ref() + .map(|sink| (sink.samples_written, sink.pdq_path.display().to_string())) + }) + .unwrap_or((0, String::new())); + let seconds = if rate_hz > 0 { + samples as f64 / f64::from(rate_hz) + } else { + 0.0 + }; + entries.push(StatusEntry::Text(format!("● REC {seconds:.1} s → {path}"))); + } else if let Some(note) = &self.last_save_note { + entries.push(StatusEntry::Text(note.clone())); + } if let Some(error) = stream_error.or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } @@ -1146,8 +1601,9 @@ mod tests { #[test] fn ring_is_bounded_by_duration() { let mut state = SharedState::default(); - let rate = 1_000; // capacity = 130_000 samples - let cap = SharedState::ring_capacity(rate); + let rate = 1_000; // capacity = cache_seconds (20 s default) × rate + let cap = state.ring_capacity(rate); + assert_eq!(cap, 20_000, "default cache is 20 s"); let block: Vec = (0..1_000).map(|i| (i % 4_096) as u16).collect(); let mut index = 0_u64; for _ in 0..(cap / block.len() + 5) { @@ -1276,6 +1732,122 @@ mod tests { plugin.disconnect(); } + fn temp_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "stage-a-photodiode-{tag}-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + + #[test] + fn cache_snapshot_writes_csv_and_sidecar() { + let dir = temp_dir("snapshot"); + let mut plugin = StageAPhotodiodePlugin::default(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(10, 20_000, 0, &[100, 200, 300]); + } + plugin.set_setting("save_snapshot", json!(true)).unwrap(); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + + let mut csv_files: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|ext| ext == "csv")) + .collect(); + assert_eq!(csv_files.len(), 1); + let csv_path = csv_files.pop().unwrap(); + let csv = std::fs::read_to_string(&csv_path).unwrap(); + let mut lines = csv.lines(); + assert_eq!(lines.next(), Some("sample_index,t_s,code,volts")); + let first = lines.next().unwrap(); + assert!(first.starts_with("10,0.000500000,100,"), "{first}"); + assert_eq!(csv.lines().count(), 4, "header + 3 samples"); + + let sidecar: Value = + serde_json::from_slice(&std::fs::read(csv_path.with_extension("json")).unwrap()) + .unwrap(); + assert_eq!(sidecar["kind"], "cache_snapshot"); + assert_eq!(sidecar["sample_rate_hz"], 20_000); + assert_eq!(sidecar["samples"], 3); + + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn snapshot_without_data_dir_reports_an_error() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("save_snapshot", json!(true)).unwrap(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("data directory"))); + } + + #[test] + fn recording_tees_frames_to_pdq_and_writes_a_sidecar() { + let dir = temp_dir("recording"); + let mut plugin = StageAPhotodiodePlugin::default(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + plugin.set_setting("record", json!(true)).unwrap(); + assert!(plugin.recording_active()); + assert_eq!(plugin.get_setting("record"), Some(json!(true))); + + // The reader thread path: every parsed frame is teed to the sink. + let frame = mock_sample_frame(0, 0, &[1, 2, 3, 4]); + record_frame(&plugin.recording, &frame, 4); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(0, MOCK_RATE_HZ, 0, &[1, 2, 3, 4]); + } + + plugin.set_setting("record", json!(false)).unwrap(); + assert!(!plugin.recording_active()); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + + let pdq_path: std::path::PathBuf = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .find(|p| p.extension().is_some_and(|ext| ext == "pdq")) + .expect("pdq written"); + assert_eq!(std::fs::read(&pdq_path).unwrap(), frame.to_bytes()); + + let sidecar: Value = + serde_json::from_slice(&std::fs::read(pdq_path.with_extension("json")).unwrap()) + .unwrap(); + assert_eq!(sidecar["kind"], "recording"); + assert_eq!(sidecar["samples_written"], 4); + assert_eq!(sidecar["pdq_frames"], 1); + assert_eq!(sidecar["valid"], true); + + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn cache_length_setting_drives_ring_capacity() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("cache_s", json!(2.0)).unwrap(); + assert_eq!(plugin.get_setting("cache_s"), Some(json!(2.0))); + let mut state = plugin.shared.lock().unwrap(); + assert_eq!(state.ring_capacity(1_000), 2_000); + let block: Vec = vec![1; 1_000]; + for i in 0..5_u64 { + let first = i * 1_000; + state.ingest(first, 1_000, 0, &block); + } + assert_eq!(state.samples.len(), 2_000); + } + /// The host settings UI exchanges enum values as indices into the /// schema's variant list (radio buttons send `json!(index)`). #[test] From 0cd8be933715ec536ba4f44ab90896ae802a825e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 16:59:45 +0200 Subject: [PATCH 17/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20spectr?= =?UTF-8?q?um=20view=20and=20absolute-time=20axis=20to=20the=20photodiode?= =?UTF-8?q?=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PD Spectrum window: Hann-windowed radix-2 FFT (no deps) over the newest power-of-two window of raw samples (256-16384; ≈1.2 Hz resolution at 20 kSa/s), one-sided amplitude in volts with max-hold bin decimation so narrow peaks survive the plot budget. Window placement means the FFT only runs while the window is open. Verified by test: a synthesized 1 kHz 0.4 V tone is recovered at the right frequency and amplitude. - time_axis setting: BEFORE NOW (scrolling, x ends at 0) or SEGMENT TIME (absolute device-clock seconds) — frozen plots and cursor measurements read as positions instead of implied motion. --- plugins/stage-a-photodiode/src/lib.rs | 291 +++++++++++++++++++++++++- 1 file changed, 286 insertions(+), 5 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 6f91025..ffc2d48 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -42,6 +42,8 @@ use serde_json::{json, Value}; use stage_a_io::{FrameParser, ParseEvent, PdqWriter, StreamIntegrity}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; +const SPECTRUM_DATASET_ID: &str = "stage-a-photodiode.spectrum"; +const SPECTRUM_VIEW_ID: &str = "stage-a-photodiode.spectrum.view"; const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; const STATUS_DATASET_ID: &str = "stage-a-photodiode.status"; const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; @@ -58,6 +60,10 @@ const RING_MAX_SAMPLES: usize = 4_000_000; /// Envelope buckets per rendered chart line; keeps the plot payload bounded /// no matter how many raw samples the window covers. const MAX_PLOT_BUCKETS: usize = 1_000; +/// Spectrum FFT window bounds: 16384 samples ≈ 0.8 s at 20 kSa/s +/// (Δf ≈ 1.2 Hz); below 256 samples a spectrum is not meaningful. +const SPECTRUM_MIN_SAMPLES: usize = 256; +const SPECTRUM_MAX_SAMPLES: usize = 16_384; /// The firmware's default stream rate; the mock mirrors it. const MOCK_RATE_HZ: u32 = 20_000; const MOCK_BLOCK_SAMPLES: usize = 256; @@ -87,6 +93,37 @@ impl Mode { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TimeAxis { + /// Scrolling view: x = seconds before the newest sample (ends at 0). + BeforeNow, + /// Fixed view: x = seconds since the segment start on the device clock — + /// a frozen plot reads as absolute positions, not implied motion. + Segment, +} + +impl TimeAxis { + const VARIANTS: [TimeAxis; 2] = [TimeAxis::BeforeNow, TimeAxis::Segment]; + + fn name(self) -> &'static str { + match self { + Self::BeforeNow => "BEFORE NOW", + Self::Segment => "SEGMENT TIME", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|axis| axis.name() == name) + } + + fn label(self) -> &'static str { + match self { + Self::BeforeNow => "time before now [s]", + Self::Segment => "segment time [s]", + } + } +} + struct SharedState { /// Sample rate of the current segment (from the frame headers). rate_hz: u32, @@ -413,6 +450,7 @@ pub struct StageAPhotodiodePlugin { window_s: f64, avg_samples: usize, avg_sync_freq_hz: f64, + time_axis: TimeAxis, data_dir: String, } @@ -433,6 +471,7 @@ impl Default for StageAPhotodiodePlugin { window_s: 10.0, avg_samples: 4, avg_sync_freq_hz: 0.0, + time_axis: TimeAxis::BeforeNow, data_dir: String::new(), } } @@ -710,8 +749,9 @@ impl StageAPhotodiodePlugin { Mode::Raw => "photodiode", Mode::Excitation => "excitation", }; + let x_label = self.time_axis.label(); let empty = |label: &str| Series1dV1 { - x_label: "time before now [s]".into(), + x_label: x_label.into(), y_label: label.into(), lines: vec![Series1dLine { name: trace_name.into(), @@ -772,8 +812,11 @@ impl StageAPhotodiodePlugin { } } if bucket_n == bucket_len || i == total - 1 { - let x = (state.ring_first_index + i as u64) as f64 / rate - - latest_x_index as f64 / rate; + let device_t = (state.ring_first_index + i as u64) as f64 / rate; + let x = match self.time_axis { + TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, + TimeAxis::Segment => device_t, + }; mean_points.push(Series1dPoint { x, y: self.display_volts(bucket_sum as f64 / bucket_n as f64), @@ -827,12 +870,92 @@ impl StageAPhotodiodePlugin { }); } Series1dV1 { - x_label: "time before now [s]".into(), + x_label: x_label.into(), y_label: y_label.into(), lines, } } + /// Amplitude spectrum of the newest power-of-two window of raw samples + /// (Hann-windowed radix-2 FFT). Only computed while the spectrum window + /// is open — it has Window placement, and the host fetches datasets of + /// closed windows never. + fn spectrum_dataset(&self) -> Series1dV1 { + let empty = Series1dV1 { + x_label: "frequency [Hz]".into(), + y_label: "amplitude [V]".into(), + lines: vec![Series1dLine { + name: "spectrum".into(), + points: Vec::new(), + }], + }; + let Ok(state) = self.shared.lock() else { + return empty; + }; + let total = state.samples.len(); + if total < SPECTRUM_MIN_SAMPLES || state.rate_hz == 0 { + return empty; + } + let available = total.min(SPECTRUM_MAX_SAMPLES); + let n = if available.is_power_of_two() { + available + } else { + available.next_power_of_two() >> 1 + }; + let start = total - n; + let mut real: Vec = state + .samples + .range(start..) + .map(|&code| code_to_volts(f64::from(code))) + .collect(); + let rate = f64::from(state.rate_hz); + drop(state); + + let mean = real.iter().sum::() / n as f64; + // Hann window (coherent gain 0.5) on the demeaned signal. + for (i, value) in real.iter_mut().enumerate() { + let w = 0.5 * (1.0 - (2.0 * std::f64::consts::PI * i as f64 / (n as f64 - 1.0)).cos()); + *value = (*value - mean) * w; + } + let mut imag = vec![0.0_f64; n]; + fft_radix2(&mut real, &mut imag); + + // One-sided amplitude: 2·|X|/(N·0.5); decimate bins by max-hold so + // narrow peaks survive the plot budget. + let bins = n / 2; + let bucket = bins.div_ceil(MAX_PLOT_BUCKETS).max(1); + let mut points = Vec::with_capacity(bins.div_ceil(bucket)); + let mut peak = 0.0_f64; + let mut peak_freq = 0.0_f64; + let mut in_bucket = 0_usize; + for k in 1..bins { + let amplitude = 2.0 * (real[k] * real[k] + imag[k] * imag[k]).sqrt() / (n as f64 * 0.5); + let freq = k as f64 * rate / n as f64; + if amplitude > peak { + peak = amplitude; + peak_freq = freq; + } + in_bucket += 1; + if in_bucket == bucket || k == bins - 1 { + points.push(Series1dPoint { + x: peak_freq, + y: peak, + }); + peak = 0.0; + peak_freq = freq; + in_bucket = 0; + } + } + Series1dV1 { + x_label: "frequency [Hz]".into(), + y_label: "amplitude [V]".into(), + lines: vec![Series1dLine { + name: format!("spectrum ({n} spl, Δf {:.2} Hz)", rate / n as f64), + points, + }], + } + } + fn status_dataset(&self) -> TableDatasetV1 { let (latest, rate_hz, average, integrity, stream_error) = match self.shared.lock() { Ok(state) => ( @@ -923,6 +1046,51 @@ impl StageAPhotodiodePlugin { } } +/// In-place iterative radix-2 Cooley–Tukey FFT. Lengths must be powers of +/// two; sized for the spectrum window (≤ 16384), where it runs in well under +/// a millisecond. +fn fft_radix2(real: &mut [f64], imag: &mut [f64]) { + let n = real.len(); + debug_assert!(n.is_power_of_two() && imag.len() == n); + // Bit-reversal permutation. + let mut j = 0_usize; + for i in 1..n { + let mut bit = n >> 1; + while j & bit != 0 { + j ^= bit; + bit >>= 1; + } + j |= bit; + if i < j { + real.swap(i, j); + imag.swap(i, j); + } + } + let mut len = 2_usize; + while len <= n { + let angle = -2.0 * std::f64::consts::PI / len as f64; + let (step_r, step_i) = (angle.cos(), angle.sin()); + for start in (0..n).step_by(len) { + let (mut w_r, mut w_i) = (1.0_f64, 0.0_f64); + for k in start..start + len / 2 { + let (even_r, even_i) = (real[k], imag[k]); + let (odd_r, odd_i) = ( + real[k + len / 2] * w_r - imag[k + len / 2] * w_i, + real[k + len / 2] * w_i + imag[k + len / 2] * w_r, + ); + real[k] = even_r + odd_r; + imag[k] = even_i + odd_i; + real[k + len / 2] = even_r - odd_r; + imag[k + len / 2] = even_i - odd_i; + let next_r = w_r * step_r - w_i * step_i; + w_i = w_r * step_i + w_i * step_r; + w_r = next_r; + } + } + len <<= 1; + } +} + fn write_json(path: &Path, value: &Value) -> Result<(), String> { let bytes = serde_json::to_vec_pretty(value) .map_err(|err| format!("serializing sidecar failed: {err}"))?; @@ -1210,6 +1378,26 @@ impl Plugin for StageAPhotodiodePlugin { default: self.avg_sync_freq_hz, }, }, + SettingItem { + key: "time_axis".into(), + label: "Time axis".into(), + tooltip: Some( + "BEFORE NOW scrolls (x ends at 0); SEGMENT TIME shows absolute \ + seconds on the device clock — better for frozen plots and \ + cursor measurements." + .into(), + ), + kind: SettingKind::Enum { + variants: TimeAxis::VARIANTS + .iter() + .map(|axis| axis.name().to_owned()) + .collect(), + default: TimeAxis::VARIANTS + .iter() + .position(|axis| *axis == self.time_axis) + .unwrap_or(0), + }, + }, ], }, SettingsSection { @@ -1305,6 +1493,13 @@ impl Plugin for StageAPhotodiodePlugin { "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), + "time_axis" => { + let index = TimeAxis::VARIANTS + .iter() + .position(|axis| *axis == self.time_axis) + .unwrap_or(0); + Some(json!(index)) + } "data_dir" => Some(json!(self.data_dir)), "cache_s" => Some(json!(self .shared @@ -1362,6 +1557,16 @@ impl Plugin for StageAPhotodiodePlugin { self.avg_sync_freq_hz = freq.clamp(0.0, 100_000.0); Ok(()) } + "time_axis" => { + let names: Vec = TimeAxis::VARIANTS + .iter() + .map(|axis| axis.name().to_owned()) + .collect(); + let name = enum_choice(&value, &names)?; + self.time_axis = TimeAxis::from_name(&name) + .ok_or_else(|| format!("unknown time axis: {name}"))?; + Ok(()) + } "data_dir" => { self.data_dir = value .as_str() @@ -1484,6 +1689,16 @@ impl Plugin for StageAPhotodiodePlugin { display: None, relations: Vec::new(), }, + HostDatasetDescriptor { + id: SPECTRUM_DATASET_ID.into(), + title: "Photodiode spectrum".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Not enough samples for a spectrum yet — connect the stream \ + port and wait a moment." + .into(), + display: None, + relations: Vec::new(), + }, HostDatasetDescriptor { id: STATUS_DATASET_ID.into(), title: "Photodiode readout".into(), @@ -1501,6 +1716,13 @@ impl Plugin for StageAPhotodiodePlugin { placement: HostViewPlacement::Window, kind: HostViewKind::LineSeriesWindow, }, + HostViewDescriptor { + id: SPECTRUM_VIEW_ID.into(), + title: "PD Spectrum".into(), + dataset_id: SPECTRUM_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, HostViewDescriptor { id: STATUS_VIEW_ID.into(), title: "Photodiode readout".into(), @@ -1516,6 +1738,7 @@ impl Plugin for StageAPhotodiodePlugin { fn host_view_dataset(&self, dataset_id: &str) -> Option> { match dataset_id { SERIES_DATASET_ID => serde_json::to_vec(&self.series_dataset()).ok(), + SPECTRUM_DATASET_ID => serde_json::to_vec(&self.spectrum_dataset()).ok(), STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), _ => None, } @@ -1523,7 +1746,9 @@ impl Plugin for StageAPhotodiodePlugin { fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { match dataset_id { - SERIES_DATASET_ID | STATUS_DATASET_ID => self.generation.load(Ordering::Relaxed).max(1), + SERIES_DATASET_ID | SPECTRUM_DATASET_ID | STATUS_DATASET_ID => { + self.generation.load(Ordering::Relaxed).max(1) + } _ => 0, } } @@ -1732,6 +1957,62 @@ mod tests { plugin.disconnect(); } + #[test] + fn spectrum_finds_a_synthesized_tone() { + let plugin = StageAPhotodiodePlugin::default(); + let rate = 20_000_u32; + // 1 kHz, 0.4 V amplitude around 1 V — well inside the ADC range. + let codes: Vec = (0..16_384_u64) + .map(|i| { + let t = i as f64 / f64::from(rate); + let volts = 1.0 + 0.4 * (2.0 * std::f64::consts::PI * 1_000.0 * t).sin(); + (volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS) as u16 + }) + .collect(); + plugin.shared.lock().unwrap().ingest(0, rate, 0, &codes); + let spectrum = plugin.spectrum_dataset(); + let points = &spectrum.lines[0].points; + assert!(!points.is_empty()); + let peak = points + .iter() + .max_by(|a, b| a.y.partial_cmp(&b.y).unwrap()) + .unwrap(); + assert!( + (peak.x - 1_000.0).abs() < 5.0, + "peak at {} Hz, expected 1 kHz", + peak.x + ); + assert!( + (peak.y - 0.4).abs() < 0.05, + "peak amplitude {} V, expected ≈0.4 V", + peak.y + ); + } + + #[test] + fn segment_time_axis_uses_absolute_device_time() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("avg_samples", json!(1)).unwrap(); + plugin + .set_setting("time_axis", json!("SEGMENT TIME")) + .unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(40_000, 20_000, 0, &[1, 2, 3, 4]); + } + let series = plugin.series_dataset(); + assert_eq!(series.x_label, "segment time [s]"); + let first = series.lines[0].points.first().unwrap(); + // Sample index 40_000 at 20 kSa/s = 2 s into the segment. + assert!((first.x - 2.0).abs() < 1e-6, "got {}", first.x); + // Default mode still ends at zero. + plugin + .set_setting("time_axis", json!("BEFORE NOW")) + .unwrap(); + let series = plugin.series_dataset(); + assert!(series.lines[0].points.last().unwrap().x.abs() < 1e-9); + } + fn temp_dir(tag: &str) -> std::path::PathBuf { let dir = std::env::temp_dir().join(format!( "stage-a-photodiode-{tag}-{}", From 8d7c7eb8d5dd566d3b4c91cea6d2c6a6d982a805 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 17:04:39 +0200 Subject: [PATCH 18/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20a=20TO?= =?UTF-8?q?ML=20protocol=20executor=20to=20the=20modulation=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Timed MOD sequences without cross-plugin control: the executor lives inside the plugin that already owns the command port. - protocol file: loops = N plus [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, min, frequency_hz — fully validated before the run starts (ranges match the firmware grammar) - executor thread walks the steps on an absolute schedule (no drift accumulation) and feeds the same coalescing pending-command slot the device thread drains, so it never touches the serial port itself; the last step holds after completion (set-and-hold), stop is immediate, and disconnecting aborts the run - protocol_path uses the new Path setting kind; run/stop is a settings-driven toggle that works with no camera; progress (loop, step, summary) shows in the status entries Verified: cargo fmt, clippy -D warnings, 8 plugin tests green incl. an end-to-end run against the mock controller. --- plugins/stage-a-modulation/Cargo.toml | 1 + plugins/stage-a-modulation/src/lib.rs | 617 ++++++++++++++++++++++---- 2 files changed, 528 insertions(+), 90 deletions(-) diff --git a/plugins/stage-a-modulation/Cargo.toml b/plugins/stage-a-modulation/Cargo.toml index 2b04e93..983a927 100644 --- a/plugins/stage-a-modulation/Cargo.toml +++ b/plugins/stage-a-modulation/Cargo.toml @@ -13,3 +13,4 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true stage-a-io = { path = "../../stage-a-io" } +toml = "0.8" diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 87fc75f..4c744ce 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -28,9 +28,9 @@ use std::time::{Duration, Instant}; use augur_plugin_api::{ export_plugin, EventStoreHandle, ExecutionMode, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, - HostViewRegistry, Plugin, PluginFrame, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, - TableSchema, TableValueType, + HostViewRegistry, PathDialogKind, Plugin, PluginFrame, SettingItem, SettingKind, + SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, + TableDatasetV1, TableSchema, TableValueType, }; use serde_json::{json, Value}; use stage_a_io::{Command, MockController, StageAClient, Transport}; @@ -242,10 +242,197 @@ fn apply_reply( shared.bump(); } +/// One validated protocol step: the exact MOD command plus how long to hold +/// it before advancing. +#[derive(Debug, Clone, PartialEq)] +struct ProtocolStep { + duration: Duration, + command: Command, + summary: String, +} + +#[derive(Debug, Clone, Default)] +struct ProtocolProgress { + loops: usize, + total_steps: usize, + /// 1-based while running. + loop_index: usize, + step_index: usize, + summary: String, + finished: bool, + stopped: bool, +} + +/// Running protocol executor; dropping it stops the thread. Commands go +/// through the same coalescing pending slot the device thread drains, so the +/// executor never touches the serial port itself. +struct ProtocolRun { + stop: Arc, + join: Option>, + progress: Arc>, +} + +impl Drop for ProtocolRun { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +/// Parses the TOML protocol format: +/// +/// ```toml +/// loops = 2 # optional, default 1 +/// [[steps]] +/// duration_s = 5.0 +/// wave = "SINE" # OFF | CONST | SINE | SQUARE +/// level = 2000 # required unless OFF +/// min = 0 # optional, periodic only +/// frequency_hz = 100.0 # required for SINE/SQUARE (0.01–2000) +/// ``` +fn parse_protocol(text: &str) -> Result<(Vec, usize), String> { + let table: toml::Table = text + .parse() + .map_err(|err| format!("protocol is not valid TOML: {err}"))?; + let loops = match table.get("loops") { + None => 1, + Some(value) => { + let loops = value.as_integer().ok_or("loops must be an integer")?; + if !(1..=10_000).contains(&loops) { + return Err("loops must be between 1 and 10000".into()); + } + loops as usize + } + }; + let raw_steps = table + .get("steps") + .and_then(|value| value.as_array()) + .ok_or("protocol needs at least one [[steps]] entry")?; + if raw_steps.is_empty() { + return Err("protocol needs at least one [[steps]] entry".into()); + } + + let mut steps = Vec::with_capacity(raw_steps.len()); + for (index, raw) in raw_steps.iter().enumerate() { + let step = raw + .as_table() + .ok_or_else(|| format!("step {} must be a table", index + 1))?; + let context = |msg: &str| format!("step {}: {msg}", index + 1); + + let duration_s = step + .get("duration_s") + .and_then(|value| value.as_float().or(value.as_integer().map(|v| v as f64))) + .ok_or_else(|| context("duration_s is required"))?; + if !(0.001..=3_600.0).contains(&duration_s) { + return Err(context("duration_s must be between 0.001 and 3600")); + } + let wave = step + .get("wave") + .and_then(|value| value.as_str()) + .ok_or_else(|| context("wave is required (OFF/CONST/SINE/SQUARE)"))? + .to_uppercase(); + + let (command, summary) = if wave == "OFF" { + ( + Command::new("MOD").field("wave", "OFF"), + format!("OFF for {duration_s} s"), + ) + } else { + let mode = Mode::from_name(&wave) + .ok_or_else(|| context("wave must be OFF, CONST, SINE, or SQUARE"))?; + let level = step + .get("level") + .and_then(|value| value.as_integer()) + .ok_or_else(|| context("level is required"))?; + if !(0..=MAX_DAC_CODE).contains(&level) { + return Err(context("level must be between 0 and 4095")); + } + let mut command = Command::new("MOD") + .field("wave", mode.name()) + .field("level", level); + let summary; + if mode.is_periodic() { + let frequency_hz = step + .get("frequency_hz") + .and_then(|value| value.as_float().or(value.as_integer().map(|v| v as f64))) + .ok_or_else(|| context("frequency_hz is required for SINE/SQUARE"))?; + if !(0.01..=2_000.0).contains(&frequency_hz) { + return Err(context("frequency_hz must be between 0.01 and 2000")); + } + let min = step + .get("min") + .and_then(|value| value.as_integer()) + .unwrap_or(0); + if !(0..=level).contains(&min) { + return Err(context("min must be between 0 and level")); + } + command = command + .field("min", min) + .field("freq_mhz", (frequency_hz * 1_000.0).round() as i64); + summary = format!( + "{} {min}..{level} @ {frequency_hz} Hz for {duration_s} s", + mode.name() + ); + } else { + summary = format!("CONST level={level} for {duration_s} s"); + } + (command, summary) + }; + steps.push(ProtocolStep { + duration: Duration::from_secs_f64(duration_s), + command, + summary, + }); + } + Ok((steps, loops)) +} + +/// Walks the steps on an absolute schedule (no drift accumulation); the last +/// commanded step holds after completion — set-and-hold, like the firmware. +fn run_protocol( + steps: Vec, + loops: usize, + shared: Arc, + stop: Arc, + progress: Arc>, +) { + let mut next_deadline = Instant::now(); + 'run: for loop_index in 1..=loops { + for (step_index, step) in steps.iter().enumerate() { + if stop.load(Ordering::Relaxed) { + break 'run; + } + if let Ok(mut progress) = progress.lock() { + progress.loop_index = loop_index; + progress.step_index = step_index + 1; + progress.summary = step.summary.clone(); + } + *shared.pending.lock().expect("pending lock") = Some(step.command.clone()); + shared.bump(); + next_deadline += step.duration; + while Instant::now() < next_deadline { + if stop.load(Ordering::Relaxed) { + break 'run; + } + let remaining = next_deadline.saturating_duration_since(Instant::now()); + std::thread::sleep(remaining.min(Duration::from_millis(10))); + } + } + } + if let Ok(mut progress) = progress.lock() { + progress.finished = true; + progress.stopped = stop.load(Ordering::Relaxed); + } + shared.bump(); +} + pub struct StageAModulationPlugin { enabled: bool, link: Option, shared: Arc, + protocol: Option, // -- settings (every accepted change is sent immediately) -- connect_requested: bool, port_hint: String, @@ -254,6 +441,7 @@ pub struct StageAModulationPlugin { min_level: i64, mode: Mode, frequency_hz: f64, + protocol_path: String, last_error: Option, } @@ -263,6 +451,7 @@ impl Default for StageAModulationPlugin { enabled: false, link: None, shared: Arc::new(SharedLink::new()), + protocol: None, connect_requested: false, port_hint: "auto".into(), max_level: MAX_DAC_CODE, @@ -270,6 +459,7 @@ impl Default for StageAModulationPlugin { min_level: 0, mode: Mode::Const, frequency_hz: 10.0, + protocol_path: String::new(), last_error: None, } } @@ -328,10 +518,59 @@ impl StageAModulationPlugin { } fn disconnect(&mut self) { + // A protocol without a device to drain its commands is meaningless. + self.protocol = None; self.link = None; // Drop stops and joins the device thread. self.shared.bump(); } + fn protocol_active(&self) -> bool { + self.protocol + .as_ref() + .is_some_and(|run| !run.progress.lock().map(|p| p.finished).unwrap_or(true)) + } + + fn start_protocol(&mut self) -> Result<(), String> { + if self.protocol_active() { + return Ok(()); + } + if self.link.is_none() { + return Err("connect to the controller before running a protocol".into()); + } + if self.protocol_path.trim().is_empty() { + return Err("choose a protocol file first".into()); + } + let text = std::fs::read_to_string(self.protocol_path.trim()) + .map_err(|err| format!("reading {} failed: {err}", self.protocol_path.trim()))?; + let (steps, loops) = parse_protocol(&text)?; + let stop = Arc::new(AtomicBool::new(false)); + let progress = Arc::new(Mutex::new(ProtocolProgress { + loops, + total_steps: steps.len(), + ..ProtocolProgress::default() + })); + let join = std::thread::Builder::new() + .name("stage-a-modulation-protocol".into()) + .spawn({ + let shared = Arc::clone(&self.shared); + let stop = Arc::clone(&stop); + let progress = Arc::clone(&progress); + move || run_protocol(steps, loops, shared, stop, progress) + }) + .expect("spawning the protocol thread must succeed"); + self.protocol = Some(ProtocolRun { + stop, + join: Some(join), + progress, + }); + Ok(()) + } + + fn stop_protocol(&mut self) { + self.protocol = None; // Drop stops and joins; last command holds. + self.shared.bump(); + } + /// Queues one MOD command carrying the complete current drive settings; /// newer changes overwrite queued ones (drag coalescing). fn send_modulation(&mut self) { @@ -571,109 +810,147 @@ impl Plugin for StageAModulationPlugin { .position(|m| *m == self.mode) .unwrap_or(0); SettingsSchema { - sections: vec![SettingsSection { - label: "Laser modulation".into(), - description: Some( - "Tick Connect, then every change is sent to the Teensy immediately — no \ + sections: vec![ + SettingsSection { + label: "Laser modulation".into(), + description: Some( + "Tick Connect, then every change is sent to the Teensy immediately — no \ camera required. The output never exceeds the power slider, the slider \ never exceeds the max limit. The firmware holds the output when \ disconnected; drag the slider to 0 to drive 0 V." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "auto (recommended) probes the attached usbmodem ports and picks \ + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "auto (recommended) probes the attached usbmodem ports and picks \ the one that answers HELLO — the Teensy command port; \ mock = in-process simulated controller" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, }, - }, - SettingItem { - key: "connect".into(), - label: "Connect".into(), - tooltip: Some( - "Opens/closes the command port. Connecting never changes the \ + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the command port. Connecting never changes the \ output; disconnecting leaves it held (set-and-hold firmware)." - .into(), - ), - kind: SettingKind::Bool { - default: self.connect_requested, + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, }, - }, - SettingItem { - key: "level".into(), - label: "Power (DAC code)".into(), - tooltip: Some( - "Output level in DAC codes; peak value for sine/square. \ + SettingItem { + key: "level".into(), + label: "Power (DAC code)".into(), + tooltip: Some( + "Output level in DAC codes; peak value for sine/square. \ Capped by the max limit below. 0 = output off." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.level, - suffix: None, + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.level, + suffix: None, + }, }, - }, - SettingItem { - key: "max_level".into(), - label: "Max limit (DAC code)".into(), - tooltip: Some( - "Safety cap: the slider cannot go above this. Set it to the \ + SettingItem { + key: "max_level".into(), + label: "Max limit (DAC code)".into(), + tooltip: Some( + "Safety cap: the slider cannot go above this. Set it to the \ highest code the connected device tolerates at J23." - .into(), - ), - kind: SettingKind::I64Drag { - min: 0, - max: MAX_DAC_CODE, - default: self.max_level, + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.max_level, + }, }, - }, - SettingItem { - key: "mode".into(), - label: "Mode".into(), - tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), - kind: SettingKind::Enum { - variants: mode_variants, - default: mode_default, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, }, - }, - SettingItem { - key: "frequency_hz".into(), - label: "Frequency".into(), - tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), - kind: SettingKind::F64Drag { - min: 0.01, - max: 2_000.0, - speed: 1.0, - default: self.frequency_hz, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.frequency_hz, + }, }, - }, - SettingItem { - key: "min_level".into(), - label: "Min threshold (DAC code)".into(), - tooltip: Some( - "Lower bound for sine/square: the waveform swings between this \ + SettingItem { + key: "min_level".into(), + label: "Min threshold (DAC code)".into(), + tooltip: Some( + "Lower bound for sine/square: the waveform swings between this \ and the power slider. Ignored in CONST mode." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.min_level, - suffix: None, + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.min_level, + suffix: None, + }, }, - }, - ], - }], + ], + }, + SettingsSection { + label: "Protocol".into(), + description: Some( + "Timed sequence of MOD steps from a TOML file: `loops = N` plus \ + [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, \ + min, frequency_hz. Steps run on an absolute schedule; the last \ + step holds after completion (set-and-hold). Stopping never \ + switches the output off by itself." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "protocol_path".into(), + label: "Protocol file".into(), + tooltip: Some("TOML protocol file (validated on start).".into()), + kind: SettingKind::Path { + dialog: PathDialogKind::OpenFile, + default: self.protocol_path.clone(), + }, + }, + SettingItem { + key: "protocol_run".into(), + label: "Run protocol".into(), + tooltip: Some( + "Start/stop the loaded protocol. Requires an open connection; \ + manual drive controls stay live and override the current step \ + until the next one begins." + .into(), + ), + kind: SettingKind::Bool { + default: self.protocol_active(), + }, + }, + ], + }, + ], } } @@ -700,6 +977,8 @@ impl Plugin for StageAModulationPlugin { } "frequency_hz" => Some(json!(self.frequency_hz)), "min_level" => Some(json!(self.min_level)), + "protocol_path" => Some(json!(self.protocol_path)), + "protocol_run" => Some(json!(self.protocol_active())), _ => None, } } @@ -773,6 +1052,27 @@ impl Plugin for StageAModulationPlugin { } Ok(()) } + "protocol_path" => { + self.protocol_path = value + .as_str() + .ok_or("protocol_path must be a string")? + .to_owned(); + Ok(()) + } + "protocol_run" => { + let requested = value.as_bool().ok_or("protocol_run must be a boolean")?; + // Failures surface through status entries (like `connect`). + if requested { + match self.start_protocol() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + } else { + self.stop_protocol(); + } + self.shared.bump(); + Ok(()) + } _ => Err(format!("unknown setting: {key}")), } } @@ -793,6 +1093,26 @@ impl Plugin for StageAModulationPlugin { state.board_mod ))); } + if let Some(run) = &self.protocol { + if let Ok(progress) = run.progress.lock() { + entries.push(StatusEntry::Text(if progress.finished { + if progress.stopped { + "Protocol: stopped (last step holds)".into() + } else { + "Protocol: finished (last step holds)".into() + } + } else { + format!( + "Protocol: loop {}/{} step {}/{} — {}", + progress.loop_index, + progress.loops, + progress.step_index, + progress.total_steps, + progress.summary + ) + })); + } + } if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } @@ -944,6 +1264,123 @@ mod tests { plugin.set_setting("connect", json!(false)).unwrap(); } + const TEST_PROTOCOL: &str = r#" +loops = 2 + +[[steps]] +duration_s = 0.03 +wave = "SINE" +level = 2000 +min = 100 +frequency_hz = 100.0 + +[[steps]] +duration_s = 0.03 +wave = "CONST" +level = 750 +"#; + + #[test] + fn protocol_parsing_validates_steps() { + let (steps, loops) = parse_protocol(TEST_PROTOCOL).expect("valid protocol"); + assert_eq!(loops, 2); + assert_eq!(steps.len(), 2); + let encoded = |command: &Command, seq: u32| { + String::from_utf8(command.encode(seq).expect("encodes")).expect("utf8") + }; + assert_eq!( + encoded(&steps[0].command, 1), + "@1 MOD wave=SINE level=2000 min=100 freq_mhz=100000\n" + ); + assert_eq!( + encoded(&steps[1].command, 2), + "@2 MOD wave=CONST level=750\n" + ); + assert!((steps[0].duration.as_secs_f64() - 0.03).abs() < 1e-9); + + assert!(parse_protocol("loops = 1").is_err(), "steps required"); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100").is_err(), + "periodic steps need a frequency" + ); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"CONST\"\nlevel = 9999").is_err(), + "level range enforced" + ); + assert!( + parse_protocol( + "[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100\nmin = 200\nfrequency_hz = 10.0" + ) + .is_err(), + "min above level rejected" + ); + let (off, _) = parse_protocol("[[steps]]\nduration_s = 0.5\nwave = \"OFF\"") + .expect("OFF needs no level"); + assert_eq!(encoded(&off[0].command, 1), "@1 MOD wave=OFF\n"); + } + + /// A protocol against the mock walks every step, holds the last one, and + /// reports finished. + #[test] + fn protocol_runs_to_completion_on_the_mock() { + let dir = std::env::temp_dir().join(format!( + "stage-a-modulation-protocol-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("protocol.toml"); + std::fs::write(&path, TEST_PROTOCOL).unwrap(); + + let mut plugin = StageAModulationPlugin::default(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + + plugin + .set_setting("protocol_path", json!(path.display().to_string())) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(true))); + + // 2 loops × 2 steps × 30 ms ≈ 120 ms; wait for the final CONST 750. + wait_until(&plugin, Duration::from_secs(3), |p| { + !p.protocol_active() && board_code(p) == Some(750) + }); + assert!(!plugin.protocol_active()); + assert_eq!(board_code(&plugin), Some(750), "last step holds"); + let progress = plugin + .protocol + .as_ref() + .unwrap() + .progress + .lock() + .unwrap() + .clone(); + assert!(progress.finished && !progress.stopped); + assert_eq!((progress.loop_index, progress.step_index), (2, 2)); + + plugin.set_setting("connect", json!(false)).unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn protocol_requires_a_connection() { + let mut plugin = StageAModulationPlugin::default(); + plugin + .set_setting("protocol_path", json!("/tmp/x.toml")) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("connect"))); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(false))); + } + /// The host settings UI exchanges enum values as indices into the /// schema's variant list (radio buttons send `json!(index)`). #[test] From 38752e3a2ae50a715abc59a32bda6a6777594c1f Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 17:14:03 +0200 Subject: [PATCH 19/46] =?UTF-8?q?perf(stage-a):=20=E2=9A=A1=20decimate=20t?= =?UTF-8?q?he=20photodiode=20chart=20from=20incremental=20summary=20cells?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepares the plugin for firmware 0.5.0's 500 kSa/s DMA stream (ADR 004 in stage-a-controller): a full raw-window rescan per repaint stops being viable around that rate. - ingest maintains 64:1 min/max/sum summary cells aligned to deque offsets; eviction drops whole cells so the alignment (and the device-clock index base) survives, at the cost of up to one cell of ring slack - chart buckets and every moving-average window combine cells plus raw edge samples via range_summary — O(range/64) instead of O(range), verified exact against naive scans across cell boundaries and after eviction - ring cap raised to 16 M samples (32 s at 500 kSa/s, 32 MiB of codes); cache_s keeps ruling the duration at lower rates Verified: cargo fmt, clippy -D warnings, 16 plugin tests green. --- plugins/stage-a-photodiode/src/lib.rs | 294 ++++++++++++++++++++------ 1 file changed, 224 insertions(+), 70 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index ffc2d48..80afa6d 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -54,9 +54,12 @@ const ADC_MAX_CODE: f64 = 4_095.0; /// (user-settable 1–130 s). const DEFAULT_CACHE_SECONDS: f64 = 20.0; const MAX_CACHE_SECONDS: f64 = 130.0; -/// Absolute sample cap guarding against absurd advertised rates (8 MiB of -/// codes at most). -const RING_MAX_SAMPLES: usize = 4_000_000; +/// Absolute sample cap: 16 M samples = 32 s at the firmware's 500 kSa/s +/// stream rate (32 MiB of codes + ~2 MiB of summary cells). +const RING_MAX_SAMPLES: usize = 16_000_000; +/// Raw samples per incremental summary cell (min/max/sum), the unit both +/// chart decimation and the moving average combine instead of raw rescans. +const SUMMARY_CELL: usize = 64; /// Envelope buckets per rendered chart line; keeps the plot payload bounded /// no matter how many raw samples the window covers. const MAX_PLOT_BUCKETS: usize = 1_000; @@ -130,6 +133,11 @@ struct SharedState { /// Device sample index of `samples.front()` within the current segment. ring_first_index: u64, samples: VecDeque, + /// Incremental 64:1 summaries: `cells[i]` covers deque offsets + /// `[i·CELL, (i+1)·CELL)`. Kept aligned by evicting whole cells, so the + /// chart and moving average never rescan the raw window — at 500 kSa/s a + /// full-window rescan per repaint would not be viable. + cells: VecDeque, latest: Option, /// Cumulative firmware-side drop counter (latest header value). device_dropped: u32, @@ -142,12 +150,39 @@ struct SharedState { error: Option, } +/// min/max/sum over exactly [`SUMMARY_CELL`] consecutive raw samples. +#[derive(Clone, Copy)] +struct SummaryCell { + min: u16, + max: u16, + sum: u32, +} + +/// Accumulated min/max/sum/count over an arbitrary sample range. +#[derive(Clone, Copy)] +struct RangeSummary { + min: u16, + max: u16, + sum: u64, + count: usize, +} + +impl RangeSummary { + fn mean(&self) -> f64 { + if self.count == 0 { + return 0.0; + } + self.sum as f64 / self.count as f64 + } +} + impl Default for SharedState { fn default() -> Self { Self { rate_hz: 0, ring_first_index: 0, samples: VecDeque::new(), + cells: VecDeque::new(), latest: None, device_dropped: 0, crc_failures: 0, @@ -179,20 +214,90 @@ impl SharedState { self.segments += 1; } self.samples.clear(); + self.cells.clear(); self.ring_first_index = first_index; self.rate_hz = rate_hz; } self.samples.extend(codes.iter().copied()); self.latest = codes.last().copied(); self.device_dropped = device_dropped; + + // Summarize every newly completed cell. + while (self.cells.len() + 1) * SUMMARY_CELL <= self.samples.len() { + let start = self.cells.len() * SUMMARY_CELL; + let mut cell = SummaryCell { + min: u16::MAX, + max: u16::MIN, + sum: 0, + }; + for &code in self.samples.range(start..start + SUMMARY_CELL) { + cell.min = cell.min.min(code); + cell.max = cell.max.max(code); + cell.sum += u32::from(code); + } + self.cells.push_back(cell); + } + + // Evict whole cells only, keeping the cell/offset alignment intact; + // the ring may exceed its capacity by up to one cell. let excess = self .samples .len() .saturating_sub(self.ring_capacity(rate_hz)); - if excess > 0 { - self.samples.drain(..excess); - self.ring_first_index += excess as u64; + let evict_cells = excess / SUMMARY_CELL; + if evict_cells > 0 { + let evict = evict_cells * SUMMARY_CELL; + self.samples.drain(..evict); + self.cells.drain(..evict_cells); + self.ring_first_index += evict as u64; + } + } + + /// min/max/sum over deque offsets `[start, end)`, combining whole + /// summary cells with raw samples at the edges: O(range/64 + 128) + /// instead of O(range). + fn range_summary(&self, start: usize, end: usize) -> RangeSummary { + let end = end.min(self.samples.len()); + let mut summary = RangeSummary { + min: u16::MAX, + max: u16::MIN, + sum: 0, + count: 0, + }; + if start >= end { + return summary; + } + summary.count = end - start; + let covered = self.cells.len() * SUMMARY_CELL; + let mut i = start; + + // Raw head up to the next cell boundary. + let head_end = (i.div_ceil(SUMMARY_CELL) * SUMMARY_CELL) + .min(end) + .min(covered.max(i)); + if head_end > i { + for &code in self.samples.range(i..head_end) { + summary.min = summary.min.min(code); + summary.max = summary.max.max(code); + summary.sum += u64::from(code); + } + i = head_end; + } + // Whole cells. + while i + SUMMARY_CELL <= end.min(covered) { + let cell = self.cells[i / SUMMARY_CELL]; + summary.min = summary.min.min(cell.min); + summary.max = summary.max.max(cell.max); + summary.sum += u64::from(cell.sum); + i += SUMMARY_CELL; } + // Raw tail (past the last whole cell in range, or past `covered`). + for &code in self.samples.range(i..end) { + summary.min = summary.min.min(code); + summary.max = summary.max.max(code); + summary.sum += u64::from(code); + } + summary } } @@ -736,8 +841,7 @@ impl StageAPhotodiodePlugin { .avg_window_samples(state.rate_hz) .min(state.samples.len()); let start = state.samples.len() - window; - let sum: u64 = state.samples.range(start..).map(|&c| u64::from(c)).sum(); - Some(sum as f64 / window as f64) + Some(state.range_summary(start, state.samples.len()).mean()) } fn series_dataset(&self) -> Series1dV1 { @@ -777,76 +881,55 @@ impl StageAPhotodiodePlugin { let avg_window = self.avg_window_samples(state.rate_hz); let avg_enabled = avg_window > 1; - // Prime the running sum with up to `avg_window − 1` samples that - // precede the visible slice, so the average is correct from the - // first visible point on. - let prime_start = start.saturating_sub(avg_window - 1); - let mut avg_sum: u64 = 0; - let mut avg_count: usize = 0; - for &code in state.samples.range(prime_start..start) { - avg_sum += u64::from(code); - avg_count += 1; - } let mut mean_points = Vec::with_capacity(MAX_PLOT_BUCKETS + 1); let mut min_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); let mut max_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); let mut avg_points = Vec::with_capacity(if avg_enabled { MAX_PLOT_BUCKETS + 1 } else { 0 }); - let mut bucket_min = u16::MAX; - let mut bucket_max = u16::MIN; - let mut bucket_sum: u64 = 0; - let mut bucket_n: usize = 0; - for (offset, &code) in state.samples.range(start..).enumerate() { - let i = start + offset; - bucket_min = bucket_min.min(code); - bucket_max = bucket_max.max(code); - bucket_sum += u64::from(code); - bucket_n += 1; - if avg_enabled { - avg_sum += u64::from(code); - avg_count += 1; - if avg_count > avg_window { - avg_sum -= u64::from(state.samples[i - avg_window]); - avg_count -= 1; - } + // Every bucket (and every moving-average window) is combined from + // the incremental summary cells plus raw edge samples — the cost per + // rebuild is O(buckets · window/64), independent of the raw rate. + let mut bucket_start = start; + while bucket_start < total { + let bucket_end = (bucket_start + bucket_len).min(total); + let last = bucket_end - 1; + let bucket = state.range_summary(bucket_start, bucket_end); + let device_t = (state.ring_first_index + last as u64) as f64 / rate; + let x = match self.time_axis { + TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, + TimeAxis::Segment => device_t, + }; + mean_points.push(Series1dPoint { + x, + y: self.display_volts(bucket.mean()), + }); + if decimating { + // EXCITATION inverts the axis, so min/max swap roles. + let (low, high) = ( + self.display_volts(f64::from(bucket.min)), + self.display_volts(f64::from(bucket.max)), + ); + min_points.push(Series1dPoint { + x, + y: low.min(high), + }); + max_points.push(Series1dPoint { + x, + y: low.max(high), + }); } - if bucket_n == bucket_len || i == total - 1 { - let device_t = (state.ring_first_index + i as u64) as f64 / rate; - let x = match self.time_axis { - TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, - TimeAxis::Segment => device_t, - }; - mean_points.push(Series1dPoint { + if avg_enabled { + // Trailing window ending at this bucket's last sample; may + // reach before the visible slice (fewer while filling). + let window_start = (last + 1).saturating_sub(avg_window); + let window = state.range_summary(window_start, last + 1); + avg_points.push(Series1dPoint { x, - y: self.display_volts(bucket_sum as f64 / bucket_n as f64), + y: self.display_volts(window.mean()), }); - if decimating { - // EXCITATION inverts the axis, so min/max swap roles. - let (low, high) = ( - self.display_volts(f64::from(bucket_min)), - self.display_volts(f64::from(bucket_max)), - ); - min_points.push(Series1dPoint { - x, - y: low.min(high), - }); - max_points.push(Series1dPoint { - x, - y: low.max(high), - }); - } - if avg_enabled { - avg_points.push(Series1dPoint { - x, - y: self.display_volts(avg_sum as f64 / avg_count as f64), - }); - } - bucket_min = u16::MAX; - bucket_max = u16::MIN; - bucket_sum = 0; - bucket_n = 0; } + bucket_start = bucket_end; } let mut lines = vec![Series1dLine { @@ -1835,12 +1918,22 @@ mod tests { state.ingest(index, rate, 0, &block); index += block.len() as u64; } - assert_eq!(state.samples.len(), cap); + // Whole-cell eviction may leave up to one summary cell of slack. + assert!( + state.samples.len() >= cap && state.samples.len() < cap + SUMMARY_CELL, + "len {} vs cap {cap}", + state.samples.len() + ); assert_eq!( state.ring_first_index + state.samples.len() as u64, index, "eviction keeps indexes aligned" ); + assert_eq!( + state.ring_first_index % SUMMARY_CELL as u64, + 0, + "eviction preserves cell alignment" + ); assert_eq!(state.segments, 0, "eviction is not a discontinuity"); } @@ -1957,6 +2050,62 @@ mod tests { plugin.disconnect(); } + /// The summary cells must agree exactly with a naive raw scan for + /// arbitrary ranges, including after whole-cell eviction. + #[test] + fn range_summary_matches_naive_scans() { + let mut state = SharedState { + cache_seconds: 1.0, // capacity 1000 at rate 1000 → forces eviction + ..SharedState::default() + }; + let mut hash: u64 = 0x243F_6A88_85A3_08D3; + let mut next = || { + hash ^= hash << 13; + hash ^= hash >> 7; + hash ^= hash << 17; + (hash % 4_096) as u16 + }; + let mut index = 0_u64; + for _ in 0..7 { + let block: Vec = (0..333).map(|_| next()).collect(); + state.ingest(index, 1_000, 0, &block); + index += block.len() as u64; + } + assert!(state.samples.len() <= 1_000 + SUMMARY_CELL, "evicted"); + assert!(!state.cells.is_empty()); + + let len = state.samples.len(); + for (start, end) in [ + (0, len), + (0, 1), + (1, SUMMARY_CELL), + (SUMMARY_CELL - 1, SUMMARY_CELL + 1), + (7, 500), + (130, 131), + (len - 3, len), + (len / 3, 2 * len / 3), + ] { + let summary = state.range_summary(start, end); + let raw: Vec = state.samples.range(start..end).copied().collect(); + assert_eq!(summary.count, raw.len(), "count for {start}..{end}"); + assert_eq!( + summary.min, + raw.iter().copied().min().unwrap(), + "min for {start}..{end}" + ); + assert_eq!( + summary.max, + raw.iter().copied().max().unwrap(), + "max for {start}..{end}" + ); + assert_eq!( + summary.sum, + raw.iter().map(|&c| u64::from(c)).sum::(), + "sum for {start}..{end}" + ); + } + } + #[test] fn spectrum_finds_a_synthesized_tone() { let plugin = StageAPhotodiodePlugin::default(); @@ -2126,7 +2275,12 @@ mod tests { let first = i * 1_000; state.ingest(first, 1_000, 0, &block); } - assert_eq!(state.samples.len(), 2_000); + // Whole-cell eviction may leave up to one summary cell of slack. + assert!( + state.samples.len() >= 2_000 && state.samples.len() < 2_000 + SUMMARY_CELL, + "len {}", + state.samples.len() + ); } /// The host settings UI exchanges enum values as indices into the From bb83705ae15a5c8e65d6953be0fe42c667eefa72 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 20 Jul 2026 11:32:10 +0200 Subject: [PATCH 20/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20report=20t?= =?UTF-8?q?he=20legacy=20ASCII=20stream=20as=20a=20firmware-flash=20hint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The photodiode plugin dropped the pre-0.4.0 ASCII PD-line path, so a Teensy running old firmware produced a generic 'no PDA1 sample frames' error that gave no hint at the real cause. The auto-probe now classifies each port (PDA1 frames / legacy ASCII / nothing) and, when it sees the 'PD code=…' ASCII stream, tells the user to flash stage-a-controller 0.4.0+ instead — the actual fix, since the plugin and firmware ship in lockstep. Verified: cargo fmt, clippy -D warnings, 16 tests green. --- plugins/stage-a-photodiode/src/lib.rs | 57 +++++++++++++++++++++------ 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 80afa6d..1e0cbff 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -1201,10 +1201,25 @@ fn resolve_auto_port() -> Result { if candidates.is_empty() { return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); } + let mut saw_legacy_ascii = false; for path in &candidates { - if probe_pd_stream(path) { - return Ok(path.clone()); - } + match probe_pd_stream(path) { + ProbeResult::Pda1SampleFrames => return Ok(path.clone()), + ProbeResult::LegacyAsciiStream => saw_legacy_ascii = true, + ProbeResult::Nothing => {} + } + } + if saw_legacy_ascii { + // The pre-0.4.0 firmware emits `PD code=… n=… t_ms=…` ASCII lines + // instead of PDA1 binary frames. This plugin dropped the ASCII path + // (three-repo lockstep), so the fix is a firmware flash, not a plugin + // setting — say so instead of a generic "no frames". + return Err(format!( + "found the legacy ASCII photodiode stream (pre-0.4.0 firmware) — flash \ + stage-a-controller 0.4.0+ so the stream port emits PDA1 binary frames \ + (tried {})", + candidates.join(", ") + )); } Err(format!( "no port streamed PDA1 sample frames within 500 ms (tried {})", @@ -1212,18 +1227,29 @@ fn resolve_auto_port() -> Result { )) } -/// True when `path` produces a CRC-clean `SamplesU16` frame within the probe -/// window. The command port emits frames too, but only control replies and -/// acquisition data — unsolicited sample frames identify the stream port. -fn probe_pd_stream(path: &str) -> bool { +/// What a brief listen on a candidate port revealed. +enum ProbeResult { + /// CRC-clean PDA1 `SamplesU16` frames — the 0.4.0+ stream port. + Pda1SampleFrames, + /// `PD code=… n=… t_ms=…` ASCII lines — the pre-0.4.0 stream port. + LegacyAsciiStream, + /// Nothing parsable (busy/command port, wrong device, or no data). + Nothing, +} + +/// Listens on `path` for up to 500 ms and classifies what it emits. The +/// command port emits frames too, but only control replies and acquisition +/// data — unsolicited sample frames identify the stream port. +fn probe_pd_stream(path: &str) -> ProbeResult { let Ok(mut port) = serialport::new(path, 115_200) .timeout(Duration::from_millis(100)) .open() else { - return false; + return ProbeResult::Nothing; }; let deadline = Instant::now() + Duration::from_millis(500); let mut parser = FrameParser::default(); + let mut ascii_tail: Vec = Vec::with_capacity(256); let mut buf = [0_u8; 4_096]; while Instant::now() < deadline { match port.read(&mut buf) { @@ -1232,19 +1258,28 @@ fn probe_pd_stream(path: &str) -> bool { while let Some(event) = parser.next_event() { if let ParseEvent::Frame(frame) = event { if frame.samples().is_some() { - return true; + return ProbeResult::Pda1SampleFrames; } } } + // Sniff for the legacy ASCII line format in parallel; a valid + // `PD code=` prefix never appears inside PDA1 binary framing. + ascii_tail.extend_from_slice(&buf[..read]); + if String::from_utf8_lossy(&ascii_tail).contains("PD code=") { + return ProbeResult::LegacyAsciiStream; + } + if ascii_tail.len() > 512 { + ascii_tail.drain(..ascii_tail.len() - 256); + } } Ok(_) => {} Err(err) if err.kind() == std::io::ErrorKind::TimedOut || err.kind() == std::io::ErrorKind::Interrupted => {} - Err(_) => return false, + Err(_) => return ProbeResult::Nothing, } } - false + ProbeResult::Nothing } /// The exact variant list the settings schema shows for the port enum — the From 4046b7aac09204c0a1fce4a59b6e268444a0157e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 19:49:11 +0200 Subject: [PATCH 21/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20the=20?= =?UTF-8?q?A1=20orchestration=20plugin=20and=20shared=20plugin=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces `stage-a-a1` as the measurement owner that drives the modulation and photodiode plugins through a leased device contract, plus the `stage-a-plugin-contract` crate that carries the shared settings/telemetry schema between them. Also lands the supporting work these depend on: - optical waveform generation and inversion in the modulation plugin - Pockels transfer calibration (measured V_null / Vpi) with sweep support - photodiode contrast estimation and monitor-cache snapshots - `.pdq` recording format with SHA-256 integrity sidecars Documented in docs/features/stage-a-a1.md, stage-a-a1-automation.md, stage-a-optical-waveform.md, stage-a-pockels-calibration.md and ADRs 007-011. --- Cargo.toml | 3 + README.md | 3 + docs/adr/005-stage-a-device-ownership.md | 14 +- docs/adr/006-stage-a-two-plugin-split.md | 20 +- docs/adr/007-stage-a-owner-orchestration.md | 68 + .../008-stage-a-optical-waveform-inversion.md | 67 + .../009-stage-a-a1-recording-coordinator.md | 107 + docs/adr/010-stage-a-a1-amplitude-sweep.md | 72 + ...11-stage-a-pockels-transfer-calibration.md | 151 + docs/architecture.md | 22 + docs/features/README.md | 8 +- docs/features/stage-a-a1-automation.md | 110 + docs/features/stage-a-a1.md | 204 + docs/features/stage-a-modulation.md | 64 +- docs/features/stage-a-optical-waveform.md | 125 + docs/features/stage-a-photodiode.md | 32 +- docs/features/stage-a-pockels-calibration.md | 214 + docs/features/stage-a.md | 26 +- plugins/stage-a-a1/Cargo.toml | 20 + plugins/stage-a-a1/README.md | 53 + plugins/stage-a-a1/plugin.toml | 9 + plugins/stage-a-a1/src/lib.rs | 14 + plugins/stage-a-a1/src/phase.rs | 382 ++ plugins/stage-a-a1/src/rates.rs | 294 + plugins/stage-a-a1/src/response_curve.rs | 293 + plugins/stage-a-a1/src/runtime.rs | 3460 +++++++++++ plugins/stage-a-a1/src/types.rs | 26 + plugins/stage-a-modulation/Cargo.toml | 1 + plugins/stage-a-modulation/README.md | 85 +- plugins/stage-a-modulation/plugin.toml | 5 +- plugins/stage-a-modulation/src/calibration.rs | 786 +++ plugins/stage-a-modulation/src/lib.rs | 5096 ++++++++++++++--- plugins/stage-a-modulation/src/waveform.rs | 389 ++ plugins/stage-a-photodiode/Cargo.toml | 1 + plugins/stage-a-photodiode/README.md | 30 +- plugins/stage-a-photodiode/plugin.toml | 1 + plugins/stage-a-photodiode/src/lib.rs | 1716 +++++- stage-a-io/src/estimator.rs | 169 +- stage-a-io/src/lib.rs | 18 +- stage-a-io/src/mock.rs | 101 + stage-a-io/src/pdq.rs | 512 +- stage-a-io/src/sha256.rs | 259 + stage-a-io/src/sidecar.rs | 10 + stage-a-io/src/wire.rs | 87 +- stage-a-plugin-contract/Cargo.toml | 17 + stage-a-plugin-contract/README.md | 47 + stage-a-plugin-contract/src/lib.rs | 853 +++ 47 files changed, 15110 insertions(+), 934 deletions(-) create mode 100644 docs/adr/007-stage-a-owner-orchestration.md create mode 100644 docs/adr/008-stage-a-optical-waveform-inversion.md create mode 100644 docs/adr/009-stage-a-a1-recording-coordinator.md create mode 100644 docs/adr/010-stage-a-a1-amplitude-sweep.md create mode 100644 docs/adr/011-stage-a-pockels-transfer-calibration.md create mode 100644 docs/features/stage-a-a1-automation.md create mode 100644 docs/features/stage-a-a1.md create mode 100644 docs/features/stage-a-optical-waveform.md create mode 100644 docs/features/stage-a-pockels-calibration.md create mode 100644 plugins/stage-a-a1/Cargo.toml create mode 100644 plugins/stage-a-a1/README.md create mode 100644 plugins/stage-a-a1/plugin.toml create mode 100644 plugins/stage-a-a1/src/lib.rs create mode 100644 plugins/stage-a-a1/src/phase.rs create mode 100644 plugins/stage-a-a1/src/rates.rs create mode 100644 plugins/stage-a-a1/src/response_curve.rs create mode 100644 plugins/stage-a-a1/src/runtime.rs create mode 100644 plugins/stage-a-a1/src/types.rs create mode 100644 plugins/stage-a-modulation/src/calibration.rs create mode 100644 plugins/stage-a-modulation/src/waveform.rs create mode 100644 stage-a-io/src/sha256.rs create mode 100644 stage-a-plugin-contract/Cargo.toml create mode 100644 stage-a-plugin-contract/README.md create mode 100644 stage-a-plugin-contract/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index dbee199..abb6328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,8 @@ [workspace] members = [ "stage-a-io", + "stage-a-plugin-contract", + "plugins/stage-a-a1", "plugins/stage-a-modulation", "plugins/stage-a-photodiode", "plugins/localization", @@ -29,3 +31,4 @@ rustfft = "6" serde = { version = "1", features = ["derive"] } serde_json = "1" serialport = "4" +stage-a-plugin-contract = { path = "stage-a-plugin-contract" } diff --git a/README.md b/README.md index 4d48dd1..346be75 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ The plugin crates under `plugins/` are under active development and not yet read | `evesmlm-candidates` | `RawEvents` | Event-domain candidate clustering plus accepted/rejected raw-event investigation layers | | `evesmlm-fitting` | `DerivedData` | Candidate fitting plus shared current-localization datasets, stable ids, and linked 3D inspection | | `evesmlm-postproc` | `DerivedData` | Filtering, drift correction, evaluation, and the later shared EVE current-localization provider | +| `stage-a-modulation` | control service | Sole owner of the Stage-A Teensy command port and ACKed modulation state | +| `stage-a-photodiode` | control service | Sole owner of the Stage-A stream port, PDA1 ingestion, and PDQ persistence | +| `stage-a-a1` | `RawEvents` + orchestration | A1 protocol/schedule validation, raw phase quicklooks, analysis core, and a safety-gated commissioning run through the two owner services | `plugin-template/` is the starting point for new plugin crates. diff --git a/docs/adr/005-stage-a-device-ownership.md b/docs/adr/005-stage-a-device-ownership.md index 57bcec5..fe7588b 100644 --- a/docs/adr/005-stage-a-device-ownership.md +++ b/docs/adr/005-stage-a-device-ownership.md @@ -2,6 +2,7 @@ - **Status:** Accepted - **Date:** 2026-07-13 +- **Amended by:** ADR 006 and ADR 007 ## Context @@ -14,10 +15,11 @@ laboratory-instrument abstractions. ## Decision -1. **Device control lives in removable protocol plugins** (`stage-a-monitor`, - `stage-a-a1`, later `-a2`/`-a3`), one experiment concern per plugin. - Exactly one enabled, armed plugin owns the serial port; opening a busy - device is a visible error. +1. **Device control lives in removable protocol plugins.** The permanent + command-port and stream-port owners are now `stage-a-modulation` and + `stage-a-photodiode` (ADR 006/007). Experiment workflows such as A1/A2/A3 + orchestrate those owners through the host service plane and do not open the + ports themselves. 2. **A shared plain-Rust library `stage-a-io`** (this repo, not a plugin) owns everything protocol-shaped: PDA1 framing + CRC resync, the ASCII command grammar with idempotent sequence retries, the bounded I/O @@ -35,8 +37,8 @@ laboratory-instrument abstractions. ## Consequences -- A2/A3 plugins reuse `stage-a-io` unchanged; only their state machines - and views are new code. +- A1/A2/A3 reuse the owner services and serde-only contracts; they may reuse + hardware-free `stage-a-io` parsing/analysis but not its serial transports. - The GUI knows nothing about Teensys; removing the three plugins removes every trace of lab hardware from the product. - Protocol changes must land in the firmware header first, then in diff --git a/docs/adr/006-stage-a-two-plugin-split.md b/docs/adr/006-stage-a-two-plugin-split.md index 618e348..1ba4f93 100644 --- a/docs/adr/006-stage-a-two-plugin-split.md +++ b/docs/adr/006-stage-a-two-plugin-split.md @@ -3,6 +3,7 @@ - **Status:** Accepted - **Date:** 2026-07-15 - **Amends:** ADR 005 (Stage-A device ownership) +- **Amended by:** ADR 007 (persistent owners with host-routed orchestration) ## Context @@ -21,30 +22,31 @@ control plugin's connection. ## Decision 1. **The firmware enumerates two USB CDC ports** (`USB_DUAL_SERIAL`, `stage-a-controller` - ADR 002): port 1 keeps the v1 command protocol; port 2 free-runs a plain-ASCII photodiode + ADR 002): port 1 keeps the v1 command protocol; port 2 free-runs the PDA1 photodiode stream. ADR 005's rule is unchanged — one owner per port — there are simply two ports now. 2. **Two minimal plugins replace the three commissioning plugins** (deleted 2026-07-15, retained in git history): - `stage-a-modulation` owns the command port (`docs/features/stage-a-modulation.md`); - `stage-a-photodiode` owns the stream port (`docs/features/stage-a-photodiode.md`). 3. **`stage-a-io` stays** as the protocol library (wire format, client, worker, firmware-faithful - mock — the mock now models firmware 0.3.0's `MOD` verb). The A1/A2/A3 experiment plugins will - build on it again when the bench reaches that stage; the estimator/pdq/sidecar modules remain - for that purpose even though no current plugin uses them. + mock — the mock now models firmware 0.3.0's `MOD` verb). Owner plugins use its transport/PDQ + pieces. A1/A2/A3 do not open transports; they use the host-routed owner contract (ADR 007) + and may use hardware-free parsing/analysis helpers. 4. **Immediate transfer replaces the Apply-action pattern**, and **all device control is settings-driven** (connect checkbox, slider changes sent as they happen). Host actions and the per-frame effects gate are unsuitable here: the host only runs `process_frame()` while camera frames flow, but the bench must work with no camera attached (amended 2026-07-16). Replay mode still disconnects the modulation plugin defensively. The firmware output is - set-and-hold; the power slider at 0 is the off switch. + set-and-hold. ADR 008's 2026-07-23 amendment separates Manual/Calibrated drive method from + waveform mode and makes `max_level` the universal DAC ceiling. ## Consequences -- Each plugin is a few hundred transparent lines with a single concern; the photodiode plugin - does not even depend on `stage-a-io`. +- Each owner plugin has a single hardware concern; the photodiode plugin uses + `stage-a-io`'s PDA1 parser and PDQ persistence without taking command-port ownership. - Both plugins work independently — either can connect, disconnect, or crash without affecting the other. - Wire-protocol changes still land firmware-first (`stage-a-controller/include/wire_protocol.h` and command grammar), then in `stage-a-io`'s client/mock. -- The A1 min-depth workflow is gone from the tree until it is rebuilt on the simplified stack; - its last state is tagged by the deletion commit. +- The A1 min-depth workflow is rebuilt as an orchestrator on this stable two-owner + stack; it never becomes a third Teensy owner. diff --git a/docs/adr/007-stage-a-owner-orchestration.md b/docs/adr/007-stage-a-owner-orchestration.md new file mode 100644 index 0000000..add704a --- /dev/null +++ b/docs/adr/007-stage-a-owner-orchestration.md @@ -0,0 +1,68 @@ +# ADR 007 — Persistent Stage-A owners with host-routed orchestration + +- **Status:** Superseded in part (2026-07-20) — the persistent two-owner model and + host-routed control plane still hold, but `stage-a-a1` no longer orchestrates the + A1 acquisition. It was reduced to a read-only live-analysis plugin (two + phase-folded quicklooks + the photodiode-measured `a`); leases, recordings, + protocol/schedule freezing, references/epochs, and the minimum-depth logistic fit + were removed. See [Stage-A A1 Analysis](../features/stage-a-a1.md). +- **Date:** 2026-07-20 +- **Amends:** ADR 005 and ADR 006 + +## Context + +The A1 workflow must coordinate laser modulation, high-rate photodiode capture, +and camera recording. The earlier handoff proposed that `stage-a-a1` open both +Teensy ports while armed. That would create a third hardware owner and duplicate +the control/readout logic already maintained by the two manual plugins. + +The existing Augur frame context cannot solve this safely: it is available only +inside `process_frame`, while device control and reference acquisition must also +progress without camera frames. Augur also loads a GUI mirror and a live-worker +instance of each plugin, so an effectful setting copied between both instances +can make them compete for the same port. + +## Decision + +1. `stage-a-modulation` is permanently the sole command-port owner and source of + truth for requested and controller-ACKed modulation state. +2. `stage-a-photodiode` is permanently the sole stream-port owner and source of + truth for PDA1 ingestion, integrity accounting, and PDQ persistence. +3. `stage-a-a1` is an orchestrator and camera-analysis plugin. It never opens a + Teensy port and never creates a `PdqWriter`. +4. Coordination uses Augur's frame-independent, worker-owned plugin service + plane. Requests are atomic semantic operations with stable plugin IDs, + request IDs, leases, run IDs, expected revisions, explicit success/rejection, + and bounded versioned snapshots. The host routes messages but contains no + Stage-A logic. +5. Manual controls and automation share the same owner-side validation. A held + automation lease prevents competing manual mutations; a deliberate manual + override revokes the lease, becomes a visible workflow fault, and commands + output-off where safe. +6. Camera start/finalize uses the allow-listed plugin-to-host recording command + contract. RAW and PDQ receipts are correlated by immutable run ID and actual + finalized paths; the workflow never claims filesystem atomicity. +7. Raw photodiode arrays do not cross JSON. A1 consumes small live summaries and + parses finalized PDQ data for replayable scientific results. + +## Safety and synchronization + +- Only the canonical live-worker instances may effect hardware. GUI mirrors, + replay, and offline instances are fail-closed. +- Duplicate request IDs return the original terminal response without repeating + an effect. +- Lease expiry, replay transition, plugin disable, worker shutdown, or hard fault + revokes control and requests output-off/finalization. +- Current PDA1 frames do not carry a shared modulation/configuration revision. + Ordered ACKs establish operational order, but scientific cross-port identity is + reported as `UNSYNCED` until firmware supplies a common epoch or marker. + +## Consequences + +- A1/A2/A3 can reuse the same owner services without duplicating serial code. +- The manual plugins remain independently useful and testable. +- ADR 005's statement that each experiment plugin owns the serial port no longer + applies to A1/A2/A3; exclusive ownership now belongs to the two device plugins. +- ADR 006's two-port/two-owner split becomes the stable architecture instead of a + temporary commissioning simplification. + diff --git a/docs/adr/008-stage-a-optical-waveform-inversion.md b/docs/adr/008-stage-a-optical-waveform-inversion.md new file mode 100644 index 0000000..fc599a7 --- /dev/null +++ b/docs/adr/008-stage-a-optical-waveform-inversion.md @@ -0,0 +1,67 @@ +# ADR 008 — Optical waveform inversion for the Stage-A modulator + +- **Status:** Accepted +- **Date:** 2026-07-20 +- **Relates to:** ADR 006 (two-plugin split), `stage-a-controller` waveform drive + +## Context + +The Pockels/PBS amplitude modulator has a `sin²` voltage→transmission transfer. +A pure DAC sine (`DAC_SINE`) therefore produces a distorted optical waveform, +and a 50 % bias only approximately linearises the small signal. The A1 +measurement wants a clean optical target — ideally a **log-intensity** sine, +because the event camera responds to changes in `ln I`. + +Producing that target requires driving the DAC with the *inverse* of the `sin²` +lobe, `V(u) = V_null + (2Vπ/π)·arcsin√u`, which is not a sinusoid. The existing +firmware only synthesises a pure sine from a fixed 256-entry table scaled between +`min`/`level`, so it cannot emit the warped shape as-is. The command line is also +capped at 192 bytes, too small to upload a 256-code table inline. + +## Decision + +1. **Own the inversion in the modulation plugin.** `waveform.rs` computes a + 256-entry DAC warp table from an `OpticalTarget` (`LogSine`/`LinearSine`), the + requested depth `a`, and a `LobeInversion { v_null_dac, v_pi_dac }`. It refuses + (never clamps) a drive whose codes leave `0..4095`. +2. **Keep the inversion parameters settable.** `V_null` and `Vπ` are entered in + DAC codes; no measurement rig is required to start. The scientifically clean + **measured LUT** (sweep constant codes, log the photodiode, freeze the table) + is a documented follow-up that drops in behind the same `warp_table` interface. +3. **Send parameters, not the table, over the wire.** The compact + `MOD wave=WARP freq_mhz=… target=… a_milli=… v_null=… v_pi=…` command fits the + 192-byte limit; the firmware rebuilds the identical table with the same formula + (`stimulus_mod::normalisedIntensity` + `dacForU`) and plays it back through a + `warpIsr`. A chunked table-upload command is the future path for the measured + LUT, which cannot be parameterised. +4. **Preserve `DAC_SINE`.** The pure DAC sine (firmware `SINE`) is unchanged and + remains the default for non-optical work. +5. **Separate drive from measurement.** The requested `a` is only a drive target. + The realised optical depth is always the photodiode-measured `a` + (rejected-complement corrected in the `stage-a-io` estimator), never the + commanded value. +6. **Keep drive method orthogonal to waveform mode (2026-07-23 amendment).** + `MANUAL` defines a DAC band from Power + Min threshold; `CALIBRATED` derives + one from `V_null`, `Vπ`, `I_k`, and `a`. All five waveform modes remain + available with both methods. Manual optical modes pass their DAC endpoints + through the forward `sin²` transfer to derive `(I_k, a)`, then reuse the same + inversion path. The separate `max_level` setting is the hard ceiling for + every drive; it is no longer merely the upper bound of the Power slider. +7. **Treat constant hold separately from modulation headroom (2026-07-23 + amendment).** `CONST` maps `I_k` directly through the inverse lobe and ignores + `a`; periodic modes retain the `I_k·exp(a/2) ≤ 1` ceiling. A rejected + calibrated setting is rolled back so displayed settings always describe the + command that can actually be sent. + +## Consequences + +- The plugin, the `stage-a-io` mock, and the firmware share one small parameter + contract and one formula; the inversion math is duplicated in Rust and C++ but + covered by the Rust round-trip tests (`sin²(warp) ≈ target`). +- Method changes only the operating-band source; mode remains a pure waveform + choice. The UI can therefore hide inactive parameters without filtering modes. +- Real optical output on hardware depends on firmware that supports the `WARP` + command; until flashed, the mode is exercisable only against the in-process + mock and the unit tests. +- The measured-LUT upgrade and the eventual `EXT_TRIGGER` camera marker (see the + A1 analysis brief) remain the two open scientific accuracy items. diff --git a/docs/adr/009-stage-a-a1-recording-coordinator.md b/docs/adr/009-stage-a-a1-recording-coordinator.md new file mode 100644 index 0000000..c56c977 --- /dev/null +++ b/docs/adr/009-stage-a-a1-recording-coordinator.md @@ -0,0 +1,107 @@ +# ADR 009 — Stage-A A1 as a focused recording coordinator + +- **Status:** Accepted +- **Date:** 2026-07-23 +- **Relates to:** ADR 005 (device ownership), ADR 006 (two-plugin split), + ADR 007 (owner orchestration — the earlier, broader orchestrator), + [Stage-A A1 Analysis](../features/stage-a-a1.md), + [Stage-A A1 Automation](../features/stage-a-a1-automation.md) + +## Context + +The A1 measurement records, for one illumination `I_k` and frequency `f`, several +runs while sweeping the modulation depth `a`. Each run must persist the camera +**RAW** stream, the photodiode **PDQ** stream, and enough configuration to +reproduce and analyse it offline — named consistently so repeats of an `(I_k, f)` +pair stay grouped. + +The previous A1 plugin (ADR 007, then the live-analysis MVP that superseded it) +was a *read-only* surface: it folded events into quicklooks and offered a manual +response-curve, but **recorded nothing**. Operators had to start/stop the camera +and photodiode recordings separately, with no shared naming and no single place +capturing the modulation settings and measured `a`. Its controls had also drifted +away from the real workflow: a "Capture camera events" toggle that recorded +nothing, an obsolete fallback frequency and phase-bin width, and interim +phase-anchoring knobs (event latency, self-align) that the now-reliable +`EXT_TRIGGER` makes unnecessary. + +## Decision + +1. **A1 becomes a focused recording coordinator.** One *Start recording* button, + a chosen **folder**, a per-`(I_k, f)` **measurement id** (auto-default, + regenerate, or edit), and a **duration** drive a small ordered state machine: + start and acknowledge the host camera recorder; connect and lease the + photodiode; open and acknowledge the PDQ; run for the requested duration; + atomically finalize the PDQ and release its lease; stop and acknowledge the + camera; then write an A1 config sidecar. This order keeps PDQ cleanup inside + the live-effects window and starts the timer only after both streams exist. + It deliberately **re-adds** recording orchestration that the + live-analysis MVP had dropped — in a narrow form: only camera + photodiode + recording, no drive/lease of the modulation device. + +2. **A1 never drives the Teensy.** The optical drive is armed in the modulation + plugin. A1 only *reads* the published `ModulationStateV1` snapshot into the + sidecar. Reintroducing the modulation drive (settle detection, amplitude + sweep) stays on the [automation roadmap](../features/stage-a-a1-automation.md). + +3. **Consistent naming, recorder-owned directories.** Files share an + `_` stem under an `/` subfolder. The camera RAW path is + relative to the **host output root** and the PDQ path relative to the + **photodiode data root** — each recorder confines its own writes, so A1 cannot + force a single absolute directory. The A1 config sidecar is written under + `//` and records the *resolved* paths of both files, so the + set is linked regardless; pointing all roots at the same experiment directory + co-locates everything physically. + +4. **Camera biases stay owned by the host recorder.** The host writes a companion + `.toml` next to the RAW containing the camera config (biases, ROI). A1 + cannot read biases itself; its sidecar cross-references that file and also + passes the key parameters as recording metadata, which the host and photodiode + embed in their own sidecars. + +5. **Two live quicklooks, clearly scoped.** Keep the **rolling half-period + response** `S_p(t)` (live sanity: are events appearing, is ON/OFF timing sane?) + and the **response probability** `q_p` (binary pixel-cycle statistic vs measured + `a`). Drop the phase-bin rate plot. The authoritative `q_p(a, f)` fit is an + **offline** computation over the recordings; the live `q_p` is a quicklook. + + **`q_p` windows: auto by default, pilot-frozen per row.** Because the + `EXT_TRIGGER` fixes the phase, ON and OFF fall in opposite half-cycles, so the + windows are found directly from the current fold — each anchored on its + histogram peak and grown outward until it drops below a floor (default 10 % of + the peak) or the opposite polarity dominates. This replaces the old + manual-pilot *button* and its window-threshold / self-align knobs. + + The window phase depends on the event latency, which is a *phase* shift `τ·f` + (negligible at low `f`, up to a full cycle at high `f`) and drifts with `I_k`, + so windows must be fixed **per `(I_k, f)` row** and held across the `a`-sweep. + A **Record pilot** action therefore freezes the auto-windows for the row and + writes them into the pilot recording's sidecar; **Record background** captures + the floor `q0`. Both are keyed to the measurement id (one id = one row) and are + auto-reloaded by scanning the measurement folder, so returning to a row reuses + its frozen windows. The live `q_p` remains a quicklook — the authoritative fit + still freezes windows offline from the brightest run. + +6. **Lean the trigger surface.** With `EXT_TRIGGER` now reliable, remove the + fallback frequency, the phase-bin width, the event-latency shift, and the + response-curve self-align/threshold knobs. The trigger marker spacing *defines* + `T`; the modulation acknowledged waveform is the only fallback. + +## Consequences + +- A1 now declares `host_commands = ["start_recording", "stop_recording"]` in its + manifest and holds a photodiode lease while recording (the photodiode's manual + recording UI is locked during that window). The first host-command use triggers + a one-time GUI consent prompt. +- A1 writes one file itself (the `.toml` sidecar) via `std::fs` — a small, bounded + write, not a PDQ/serial writer; hardware ownership is unchanged. +- A recording reports success only after complete host finalization and a valid + typed PDQ finalization receipt. The UI keeps one concise phase/result message, + not a rolling internal log. +- The host returns to Preview before delivering its final receipt, which keeps + repeated recordings and automated sweeps live without an extra operator step. +- True single-directory co-location is a **configuration** convention (align the + recorder roots), not something A1 enforces. Enforcing it would require host and + photodiode path changes and is out of scope. +- The contract and ABI are unchanged: every message used already exists + (`HostCommand`, `PhotodiodeCommandV1` lease/begin/finalize). diff --git a/docs/adr/010-stage-a-a1-amplitude-sweep.md b/docs/adr/010-stage-a-a1-amplitude-sweep.md new file mode 100644 index 0000000..ab036fb --- /dev/null +++ b/docs/adr/010-stage-a-a1-amplitude-sweep.md @@ -0,0 +1,72 @@ +# ADR 010 — Stage-A A1 amplitude sweep via leased optical-depth retargeting + +- **Status:** accepted (2026-07-23) +- **Relates to:** ADR 007 (owner orchestration), ADR 009 (recording + coordinator), [Stage-A A1 Automation](../features/stage-a-a1-automation.md) + +## Context + +The A1 workflow records a response curve `q_p(a, f)`: several recordings at +different modulation depths `a` for one `(I_k, f)` row. With the manual +coordinator (ADR 009) the operator had to retarget the drive in the modulation +plugin and press *Start recording* once per amplitude. The automation roadmap +(§1–§4 of the automation brief) calls for a scoped control path: sweep only +`a`, never the rest of the drive. + +Two structural gaps blocked this: + +1. **No semantic "set depth" command.** The modulation service only exposed + `SetWaveform` (raw DAC band) and `PrepareA1`. Sweeping `a` through raw DAC + values would duplicate the optical-inversion math (ADR 008) and the + calibration state (`V_null`, `Vπ`, `u_k`) outside their owner. +2. **Momentary buttons never reached the live worker.** The host runs a UI + mirror and a live worker per plugin; button presses land on the mirror via + `set_setting(key, true)`, while the worker only receives the settings + snapshot built from `get_setting`. Buttons that returned `false` lost every + press (the root cause of the dead record buttons). + +## Decision + +**1. `ModulationCommandV1::SetOpticalDepth { depth_a_milli }`** (contract +addition, additive to V1). Under an automation lease the modulation owner +re-derives its armed drive with the new depth through the same +`drive_command()` builder the operator path uses; everything else (waveform +shape, frequency, `u_k`, calibration, power cap) stays as armed. The owner +rejects the command when no device link is open, when the armed drive cannot +express a depth (manual DAC method, constant mode), or when the derived drive +violates its own safety validation. The command is applied immediately +(`Applied`), not revision-tracked: the sweep's ground truth for "the drive is +really there" is the photodiode-measured `a`, not a firmware ACK. + +**2. The sweep lives in A1** as a small state machine layered *on top of* the +ADR 009 coordinator: `AcquiringLease → (per point) SettingDepth → Settling → +Recording → …release`. Per point it renews the modulation lease, retargets the +depth, waits until the photodiode-measured `a` holds the target tolerance +(±10 %, at least ±0.05) for the configured dwell (30 s cap, then it records +anyway — the sidecar stores the measured `a`), and hands off to the unchanged +recording coordinator (`…_pNN` stem tag, `sweep.requested_a` / `point_index` / +`point_total` in the sidecar). Any rejection, timeout, or failed point aborts +the sweep and releases the lease (`safe_off = false` — the drive holds; safety +remains the owner's lease-expiry job). + +**3. Press counters for momentary buttons.** Every A1 button exports a +monotonic press counter from `get_setting`; `set_setting` interprets `true` as +a local click and a counter advance as one forwarded press edge, adopting the +first-seen value silently (reloads must not replay presses). The plugin-API +`Button` doc now records this idiom, and `SettingKind::Button` gained an +`enabled` flag (serde-default `true`, backward compatible in both directions) +so prerequisite-less presses can be prevented in the UI instead of rejected +after the fact. + +## Consequences + +- A1 now drives exactly one modulation parameter, under a lease, through the + contract — the "A1 owns no hardware" boundary narrows to "A1 may retarget + the armed drive's depth while leased" (the focused re-introduction ADR 007 + anticipated). +- Manual modulation settings stay locked during a sweep (lease lock), and the + operator's own `depth a` re-applies on the next modulation settings sync + after release. +- The press-counter idiom is the sanctioned pattern for momentary controls in + dual-instance plugins; requested-state booleans (`connect`, `record`, + `protocol_run`) remain correct as-is. diff --git a/docs/adr/011-stage-a-pockels-transfer-calibration.md b/docs/adr/011-stage-a-pockels-transfer-calibration.md new file mode 100644 index 0000000..30070f1 --- /dev/null +++ b/docs/adr/011-stage-a-pockels-transfer-calibration.md @@ -0,0 +1,151 @@ +# ADR 011 — Measured Pockels transfer calibration in the modulation plugin + +- **Status:** Accepted +- **Date:** 2026-07-25 +- **Relates to:** ADR 006 (two-plugin split), ADR 008 (optical waveform + inversion), ADR 010 (amplitude sweep / press-counter idiom), + [Stage-A Pockels Transfer Calibration](../features/stage-a-pockels-calibration.md) + +## Context + +ADR 008 made `V_null`/`Vπ` settable and named the **measured LUT** as the +follow-up. In practice they stayed two bare number fields whose tooltip told the +operator to measure them while the software offered no way to do so. Nothing +related a DAC code to an observed photodiode value, so the whole calibrated +drive rested on numbers typed in from a nominal datasheet — exactly what the +knowledge base warns against (`methodology/pockels-waveform-linearisation.md` +§1: "Do not use nominal `Vπ` as the measurement calibration"). + +## Decision + +### 1. The modulation plugin owns the calibration + +It already owns `V_null`/`Vπ` and the DAC. It reads photodiode levels **read +only** from the control-snapshot broadcast (the same bus A1 reads for the +measured `a`), so no lease, no service command, no coordinating plugin, and no +PDQ recording are involved. The alternative — a lease-based cross-plugin +protocol like ADR 010's sweep — would have moved the calibration state away from +the parameters it calibrates for no gain. + +### 2. `PhotodiodeStreamV1.level` — one additive V1 field + +`PhotodiodeLevelV1 { mean_volts, peak_to_peak_volts, sample_count, +end_sample_index, clipped }`, `#[serde(default)]`. + +`PhotodiodeOpticalSummaryV1` could not serve: it reports *contrast* not level, +applies the geometry transform (which needs an anchor this reading must not +depend on), and **refuses** on clipping or missing headroom — precisely at +`V_null`, where the reject-port detector is brightest. The level is deliberately +fail-open where the optical summary is fail-closed, and always **raw** detector +volts, never the plugin's RAW/EXCITATION display transform. + +`end_sample_index` makes settling *provable*: a point is accepted only from a +window that began after its code was commanded plus a settle margin, on the +device sample clock. No shared wall clock, no sleeps, immune to tick jitter. + +### 3. The detector geometry is an input, not an inference + +The initial design assumed a free-signed amplitude would let the fit *identify* +the port. It cannot. Since `sin²` is symmetric about its peak, +`(v, p₀, p₁)` and `(v + Vπ, p₀ + p₁, −p₁)` describe the measured curve +*identically* — the data cannot say which extremum is zero excitation. This is a +fact about the optics, so it is asked (`Detector port`, default `REJECT PORT`, +which `setup/optical-path.md` settles by construction) and the fit selects the +matching representation. Guessing would place `V_null` a quarter wave off and +silently run the drive on the inverted branch. + +### 4. One-dimensional harmonic fit, not a nonlinear solve + +`sin²(x) = (1 − cos 2x)/2` makes the model a constant plus one sinusoid of +period `2Vπ`, which is linear in its quadrature components. For each candidate +`Vπ`, the phase (hence `V_null`) and both amplitudes come from a 3×3 solve, so +only `Vπ` is searched — a log-spaced scan plus a golden-section refine. + +The rejected alternative, seeding the period from the measured extrema, breaks +on the sweeps that matter: at a realistic `Vπ ≈ 860` the DAC range holds ~2.4 +lobes and the global extrema can sit whole periods apart. + +Where several nulls are valid, the **lowest** in-range one wins: least voltage +across the crystal, most headroom, and predictable for the operator. + +### 5. `enabled` is computed from mirrored settings only + +`settings_schema()` is rendered by the **UI mirror**, which by construction +never owns the device link, a lease, a running sweep, or a fit — all of that +lives on the live worker. A first cut gated the calibration buttons on +`calibration_blocker()` and `fit.is_some()`, which disabled them *permanently*: +the mirror can never satisfy either. The buttons now gate on the one +prerequisite the mirror does know (the operator asked to connect), and the +authoritative interlocks stay worker-side, reported through the status entries +the host already takes from the worker. + +**Rule for this repo:** a `SettingKind::Button { enabled }` may only depend on +state that is itself a setting. Anything else is invisible to the instance that +renders it. The same trap bit the press counters — a baseline folded into the +counter made a fresh worker swallow the operator's first press, so the modulation +plugin now uses A1's `PressLatch` (separate `counter` and `seen`) verbatim. + +### 6. The sweep owns the DAC, so `send_modulation` is silent while it runs + +`apply_live_plugin_snapshot` writes **every** settings key to the worker on +every sync, and most of this plugin's drive handlers call `send_modulation()` +unconditionally rather than on change. Each sync therefore re-armed the +operator's waveform on top of the code the sweep had just commanded: the board +spent the sweep playing the armed drive, every point measured the same +waveform-averaged level, and the fit correctly reported `NoModulation` on a +bench where the light was plainly modulating. + +`send_modulation` now returns early while a sweep is in flight, the same shape +as the existing automation-lease guard — a sweep is simply another owner of the +DAC. Settings changed mid-sweep are withheld rather than rejected, and land on +the board when the sweep finishes: the restore prefers the *current* drive and +falls back to the command captured at sweep start. + +### 7. Robust refit, and fit quality warns rather than blocks + +The first cut refused to apply a fit whose residual exceeded 2 % of the detector +span. On the bench that gate fired at 20.8 % on a sweep whose plot looked +correct, and withheld a usable calibration. + +Measuring the failure modes on a realistic small-signal sweep settled it: 5 mV +of noise gives 3.1 %, drift 3.2 %, hysteresis 5.5 % — but a **single stray +point gives 9.9 % while leaving `Vπ` accurate to three codes**. Residual and +correctness are not the same axis, so a residual threshold is the wrong thing to +block on. (A genuinely wrong fit — an amplifier compressing the top of the +range — gives 15.2 % *and* a `Vπ` off by 250 codes, which the plot shows +plainly.) + +Two changes follow. The fit now runs twice, dropping points beyond `6 × median` +absolute residual before refitting — a median cut, because mean and standard +deviation are themselves inflated by the points being sought. And every quality +measure became a warning; the only meaningless case, no full lobe inside the +commandable range, is already refused inside `fit_transfer`, so the separate +coverage gate was dead code and was removed rather than kept. + +Applying still re-validates the resulting drive and rolls back if it cannot be +armed. The sweep restores the pre-sweep drive on every exit path, and refuses to +run while a lease or protocol owns the DAC. + +### 8. `ModulationStateV1.calibration_id` — one additive V1 field + +Set when a measured fit is applied, `None` when the lobe was typed in by hand, +so a consumer's sidecar can cite which inversion produced a run's optical depth. +Previously unrecoverable. + +## Consequences + +- The reported detector level at the null is a **lower bound** on the + total-power anchor `I_tot`, not the anchor: on the reject port the residual + transmitted floor is not separable from it (knowledge base §4.4). The plugin + labels it as such and derives no maximum achievable `a` from it. Freezing a + real anchor still needs a transmitted-port power measurement. +- `V_null`/`Vπ` need neither a dark measurement nor an anchor, because the + fitted offset and amplitude absorb both. That is what keeps this one button + instead of a protocol. +- Ascending and descending passes are both recorded, so the hysteresis figure + the knowledge base's acceptance test 1 asks for comes out of the normal run. +- Still an analytic `sin²` inversion, not a measured LUT. The archived record + stores the points a LUT would need, so ADR 008's follow-up remains open behind + the same `warp_table` interface. +- A static calibration must never be used to correct dynamic roll-off; doing so + would manufacture the Bode curve A1 exists to measure. diff --git a/docs/architecture.md b/docs/architecture.md index c67c61f..dcba238 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,6 +71,28 @@ Key properties: New plugins should prefer this shared host contract over duplicating pixel scale or sensor geometry in plugin-local defaults. +## Frame-Independent Plugin Services + +Hardware workflows use the host-routed control plane rather than the per-frame +JSON context. One canonical live-worker instance owns effects; GUI mirrors, +replay, and offline instances remain fail-closed. Requests target stable manifest +IDs and carry semantic operation names, request IDs, leases, and explicit +responses. Device plugins validate and execute their own operations; the host is +only the router. + +Stage-A uses a serde-only companion contract so `stage-a-a1` can orchestrate +`stage-a-modulation` and `stage-a-photodiode` without linking their implementation +crates or opening their serial ports. See +[`docs/adr/007-stage-a-owner-orchestration.md`](./adr/007-stage-a-owner-orchestration.md). + +Not every cross-plugin dependency needs that machinery. Because the host +broadcasts each plugin's `control_snapshots()` to every plugin's inbox, a plugin +that only needs to *read* another's published state can do so directly — no +lease, no service request, no router round-trip. The Pockels transfer +calibration reads photodiode levels this way while driving only its own DAC: +[`docs/adr/011-stage-a-pockels-transfer-calibration.md`](./adr/011-stage-a-pockels-transfer-calibration.md). +Reserve the leased service path for *commanding* hardware someone else owns. + ## Host Views Plugins declare host-rendered datasets and views through: diff --git a/docs/features/README.md b/docs/features/README.md index 16ed650..2ff4830 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -5,8 +5,12 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. -- [Stage-A Modulation](./stage-a-modulation.md) — capped power slider + constant/sine/square laser-modulation drive on the command port, applied immediately. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`. +- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. +- [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC so the *optical* output is a log- or linear-intensity sine, inverting the Pockels `sin²` transfer from settable `V_null`/`Vπ`. +- [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`Vπ` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`, plus the geometry-corrected optical depth `a`. +- [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. +- [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-a1-automation.md b/docs/features/stage-a-a1-automation.md new file mode 100644 index 0000000..529a10d --- /dev/null +++ b/docs/features/stage-a-a1-automation.md @@ -0,0 +1,110 @@ +# Stage-A A1 Automation — Plan (partially implemented) + +- **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) +- **Status:** **Partially built.** §1 (scoped A1→modulation control path), §2 + (settle detection), §3 (per-point recording) and the single-row core of §4 + (the amplitude loop) now exist as the **Start sweep** button — see + [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md). Still open: scout phase, + randomized point order, multi-`f`/multi-`I_k` rows, `UNIDENTIFIABLE` stop + rule, and the offline `a50` fit (§5–§6). +- **Relates to:** [Stage-A A1 Analysis](./stage-a-a1.md), + [Optical Waveform Drive](./stage-a-optical-waveform.md), + [ADR 007](../adr/007-stage-a-owner-orchestration.md) (the earlier full + orchestrator this deliberately re-adds in a *focused* form), + [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) (the per-recording + RAW + PDQ + sidecar coordinator, now built — see §3). + +> **Update (2026-07-23):** the manual per-recording coordinator in §3 now exists +> (ADR 009): one *Start recording* button records camera RAW + photodiode PDQ + +> an A1 config sidecar per `(I_k, f)` measurement, over an operator-set duration. +> +> **Update (2026-07-23, later):** the single-row amplitude sweep now exists +> (ADR 010): *Start sweep* leases the modulation owner, retargets the armed +> drive per point via `SetOpticalDepth`, waits for the photodiode-measured `a` +> to settle (tolerance + dwell, 30 s cap), and records each point through the +> §3 coordinator with `sweep.requested_a` / `point_index` / `point_total` in +> the sidecar. Remaining below: scout/randomized order, multi-`f`/`I_k` +> iteration, the `UNIDENTIFIABLE` rule, and the `a50` fit. + +## Goal + +Semi-automate the researcher's normal A1 workflow: for one illumination `I_k` +and frequency `f`, sweep the modulation depth `a` and record the response curve +`q̂_p(a,f)`, automatically starting/stopping/saving each recording with proper +naming and full parameters. Later, repeat over `f` and over `I_k`. + +## Already in place (the foundation) + +- Two live plots `r_{p,k}` and `S_p`, plus the response curve `q̂_p(a)`. +- Marker-anchored phase folding from the firmware **phase-0 EXT_TRIGGER**; the + trigger **defines the frequency** (measured marker spacing); event-latency + handling (marker shift + optional self-alignment). +- **Pilot capture** → frozen ON/OFF phase windows; manual "record point" + appends `(measured a, q_on, q_off)` to the curve. +- **ROI + masked pixels** come from the host camera config (`GlobalSettings`); + `N_valid = |ROI| − |masked|`. +- Photodiode-measured **`a`** (rejected-complement geometry) published and + surfaced in A1. +- Optical drive with a **fixed operating point `I_k`** and swept `a` + (`OPTICAL_LOG_SINE`/`OPTICAL_LINEAR_SINE`), power-capped. +- Events sourced exactly from the retained **EventStore** over a sliding window. + +## To build (the automation) + +### 1. A1 → modulation control path (scoped) +Re-introduce a *focused* control path (the contract + modulation plugin still +support it): acquire a modulation lease, set the optical drive +`(target, I_k, a, f, V_null, Vπ)`, start, stop, release. No full workflow zoo — +just set-amplitude / start / stop. Fixed `I_k`, only `a` varies within a curve. + +### 2. Settle detection +Before collecting a point, wait until the photodiode confirms the optical +waveform has stabilised at the new `a` — e.g. the published `measured_a` is +within tolerance of the target and clip-free for a short dwell. Only then start +the counting window. + +### 3. Per-point recording (proper naming + parameters) +For the pilot, background, and every amplitude point, orchestrate: +- host camera **RAW** recording (re-add the host recording commands), +- photodiode **PDQ** recording (`stage-a-photodiode` begin/finalize), +- a **config sidecar** with everything needed to reproduce/replay: `I_k`, `f`, + requested + measured `a`, ON/OFF windows, ROI, masked pixels, `N_valid`, `M`, + latency, biases, run/session ids, timestamps, settle/clip status. +- **Deterministic naming**: `/A1////-...`. + +### 4. Sweep state machine +`SAFE → BACKGROUND(a=0) → PILOT(high, freeze windows) → SCOUT(locate the +transition) → SWEEP(5–7 settled amplitudes spanning ~10–90 %, randomized or +alternating order) → NEXT_FREQUENCY → … → NEXT_ILLUMINATION`. Windows are frozen +from the pilot and **must not** be re-derived from measurement points. + +### 5. Classification and stop rules +- `q̂_p(a,f) = (1/(N_valid·M)) Σ_i Σ_c z_{i,c,p}`, ON/OFF independent + (already implemented for a single point — the sweep just repeats it). +- If the transition cannot reach ~90 % within the safe amplitude range, mark the + frequency **`UNIDENTIFIABLE`** (do not keep increasing `a`). + +### 6. Final fit (per curve) +Fit the background-floor logistic `p(a) = p0 + (1−p0)·logistic((a−a50)/slope)` +to get **`a50`** with a cycle/spatial-tile bootstrap interval; label quality +(VALIDATED / DEGRADED-no-background / UNVALIDATED-no-pilot). (This was the old +`response.rs`; re-add as the sweep's summary output.) + +## Known hard problem (only if per-cycle cross-stream correlation is ever needed) + +The camera trigger and the photodiode stream marker are the *same* firmware +phase-0 on two clocks, consumed **independently** today — nothing pairs cycle +*k* across the two streams, and nothing needs to. If a future metric correlates +per-cycle optical depth with per-cycle camera response, ordinal matching is +fragile (start offset, asymmetric drops, drift). The robust fix is a **cycle +counter** in the PD `MarkerPayload` plus a **distinctive fiducial pattern** +(e.g. a periodic marker cycle) visible in both streams to align on and detect +drops — see the controller's `a1-marker-cycles.md`. + +## Open decisions to confirm at build time + +- Amplitude list: explicit list vs. min/max/count range (randomized order). +- Mask source already resolved: host `GlobalSettings.masked_pixels`. +- Whether RAW+PDQ per point is always on or gated by a "record" toggle + (user already asked for full RAW + PDQ + params per recording). +- Live is a quicklook; the **RAW/PDQ replay is authoritative** for the final fit. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md new file mode 100644 index 0000000..0b72be2 --- /dev/null +++ b/docs/features/stage-a-a1.md @@ -0,0 +1,204 @@ +# Stage-A A1 Analysis + +- **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) +- **Status:** Recording coordinator + live quicklooks + amplitude sweep +- **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), + [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button + press forwarding) +- **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) + +## Purpose + +A1 has two jobs on the Stage-A bench, both deliberately thin: + +1. **Recording coordinator.** One *Start recording* button records the camera + **RAW** stream and the photodiode **PDQ** stream together for a fixed duration, + grouped under a per-`(I_k, f)` measurement id, and writes an A1 **config + sidecar** (`.toml`) linking them with everything needed to reproduce and + analyse the run offline. +2. **Live sanity quicklooks.** The rolling half-period response `S_p(t)` and the + response probability `q_p`, folded on the modulation period `T`. + +A1 owns no hardware and never drives the Teensy. The optical drive is armed in the +modulation plugin; A1 only *reads* its published settings. + +## The recording workflow + +The experiment sweeps the modulation depth `a = ln(I_max/I_min)` at a fixed +illumination `I_k` and frequency `f`, taking several recordings per `(I_k, f)` +pair (a background `a≈0`, a bright pilot, then settled amplitudes). One +**measurement id = one `(I_k, f)` row**; every recording under it lands in the same +folder. A1 makes each recording one button press: + +| Control | Meaning | +|---|---| +| Output folder | where the A1 config sidecar is written (recommended shared experiment root) | +| Measurement id | one per `(I_k, f)` row; auto-generated default, editable, or press **New id** | +| Sweep min a / max a | the `a`-range for this row; the **Start sweep** button records it, and it is stored in every sidecar | +| Sweep points (count) | how many amplitudes Start sweep records, spaced evenly over `[min a, max a]` | +| Sweep settle (s) | dwell the photodiode-measured `a` must hold the target (±10 %, ≥±0.05) before each sweep recording; 30 s cap, then it records anyway | +| Duration (s) | each recording auto-stops and finalizes after this | +| Start recording (sweep point) | start camera RAW → connect and lease photodiode → start PDQ → auto-stop and save both → sidecar | +| Start sweep (record all points) | per point: lease the modulation owner → retarget the calibrated drive to `a_i` → settle → one recording (`…_pNN`) → next point | +| Record pilot | records a bright reference (`…_pilot`) **and** freezes the ON/OFF windows for the row from the live signal | +| Record background | records an unmodulated reference (`…_background`) **and** captures the false-response floor `q0` | +| Stop (abort recording / sweep) | finalize the current recording early; during a sweep also aborts the remaining points | + +The record and sweep buttons stay **disabled until an output folder is +selected**. + +For manual recordings A1 never drives the Teensy: set the drive (high `a` for +the pilot, `a≈0` for the background) in the modulation plugin, then press the +matching button — the recording captures whatever `a` is currently set. + +**The sweep is the one scoped exception.** Start sweep leases the modulation +owner (`SERVICE_STAGE_A_MODULATION_CONTROL_V1`) and, per point, issues +`ModulationCommandV1::SetOpticalDepth` — which only retargets the *depth* of the +drive the operator already armed (waveform, frequency, operating point `I_k`, +and calibration stay untouched; the owner refuses when a manual-DAC or constant +drive is armed). It renews the lease per point, waits for the +photodiode-measured `a` to settle, hands the point to the normal recording +coordinator, and releases the lease at the end or on abort. Sweep points +require `min a > 0` — record `a≈0` with the background button instead. Sidecars +of sweep recordings additionally carry `sweep.requested_a`, `sweep.point_index` +and `sweep.point_total`. After the sweep releases the lease, the drive holds the +last sweep amplitude until the operator's own `depth a` setting is re-applied +(any modulation settings change re-sends it). + +**Naming.** Files share an `_[_role]` stem under an `/` subfolder +(`_pilot` / `_background` tag the reference runs): + +- `/_.raw` — camera RAW, under the **host output root**, with the host's + own `.toml` sidecar (camera biases, ROI) written next to it. +- `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar, under the + **photodiode data root**. +- `/__config.toml` — the A1 sidecar, under the chosen output folder. + +Each recorder confines its writes to its own root, so A1 cannot force one absolute +directory (see ADR 009). Point the host output root and the photodiode data root +at the same experiment directory to co-locate everything; the A1 sidecar records +the *resolved* paths so the set stays linked either way. + +**A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize +timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the +acknowledged snapshot (frequency, center/amplitude DAC, waveform); the +photodiode-measured `a` (`measured_log_contrast`) and clip fractions; ROI + +masked-pixel count + `N_valid`; trigger info (marker-anchored, marker count, +measured period); and the resolved paths of the RAW (+ its camera-config sidecar) +and the PDQ (+ its sidecar). The **pilot** run additionally records the frozen +ON/OFF windows and the **background** run the floor `q0`, so returning to a +measurement (folder + id) auto-reloads them for the `q_p` plot. + +**Mechanism.** A small control-plane state machine in `process_control` starts +the host camera recorder first and waits for its receipt. Only after the host +has completed the Preview → Recording switch does A1 connect and lease the +photodiode and open the PDQ with the same run id. The duration begins when the +PDQ start receipt arrives, so setup time is never deducted from the requested +recording. On completion A1 atomically finalizes the PDQ and releases its lease +while camera effects are still live, then stops the host recorder, waits for its +final receipt, and writes the config sidecar. A recording is successful only +when the host receipt is complete and the photodiode returns a valid finalized +receipt with both PDQ paths. The status panel shows only the current phase and +one concise result or error message; it does not render an internal event log. +A1 declares `host_commands = ["start_recording", "stop_recording"]` in its +manifest. Every role uses this same lifecycle. + +**Host-side note.** The camera RAW leg restarts the host pipeline into +Recording mode and stops it again at finalize. After the file is finalized, the +host restores Preview before returning the receipt, so a sweep or another button +press can start the next recording automatically. + +**File locations** (three roots, point them at the same experiment directory): +`//.raw` (+ host `.toml`), +`//_pd.pdq` + `_pd.json`, and +`//_config.toml`. + +## The two live plots + +Both fold the camera event stream on `T` (from the firmware phase-0 `EXT_TRIGGER` +marker spacing, which *defines* the frequency; the modulation acknowledged waveform +is the only fallback). Enable **Live analysis** to keep them updating. + +Marker hygiene: preview windows overlap, so the same trigger edge arrives on +several consecutive frames — the marker buffer is sorted and deduplicated on +every merge (duplicates used to fail marker validation and blank the plots). +When marker validation still rejects a fold (dropped-trigger jitter), the +quicklook falls back to the free-running fold on `T` instead of going empty. + +1. **Rolling half-period response** + + ```math + S_p(t) = \frac{N_p(t-T/2,\,t]}{N_\text{valid}} + ``` + + events per valid pixel in the trailing half-cycle, ON and OFF. A live indicator: + are events appearing, does the ON/OFF timing look sane, is the response + saturating? It counts *every* event, so a noisy pixel weighs heavily — it is a + quicklook, not the response metric. + +2. **Response probability** `q_p` + + ```math + z_{i,c,p} = \mathbf{1}[\text{pixel } i \text{ fires in } W_p \text{ during cycle } c], + \qquad + \hat q_p(a,f) = \frac{1}{N_\text{valid} M}\sum_i\sum_c z_{i,c,p} + ``` + + the fraction of valid pixel-cycles that fire at least once in the ON/OFF phase + window `W_p` — each pixel-cycle counts **once** (unlike `S_p`). The windows come + from the row's **pilot** when one has been recorded (frozen, held across the + whole row), otherwise from the trigger-anchored fold automatically: since the + `EXT_TRIGGER` fixes the phase, ON and OFF live in opposite half-cycles, so each + window is anchored on its histogram peak and grown outward until events fall + below the **window floor** (default 10 % of the peak) or the opposite polarity + takes over. `Record point` appends one `(measured a, q_on, q_off)` dot. The ROI + and masked pixels come from the augur-rs camera config + (`N_valid = |ROI| − |masked|`). + + **Why the pilot is per row.** The window phase depends on the event latency, + which is a *phase* shift `τ·f` — negligible at low `f`, up to a full cycle at + high `f` — and also drifts with `I_k`. So the windows must be defined **per + `(I_k, f)` row** and held fixed across that row's `a`-sweep (re-deriving them + per amplitude would bias the curve). One pilot per measurement id captures that + exactly. This live `q_p` stays a quicklook; the **authoritative** `q_p(a, f)` + fit (`a50`, background floor) is computed offline from the recordings. + +## Button presses across the UI-mirror / live-worker split + +The host loads two instances of every dynamic plugin: a **UI mirror** (renders +the settings, never touches hardware) and the **live worker** (runs +`process_frame` / `process_control`, owns the recording state machine). A +`SettingKind::Button` click calls `set_setting(key, true)` **on the mirror +only**; the worker receives settings through the host's snapshot, which carries +whatever `get_setting` returns. A1 therefore exports every button as a +**monotonic press counter** (`PressLatch`): the mirror increments it per click, +the snapshot transports it, and the worker treats a counter advance as exactly +one press edge (the first value a freshly loaded worker sees is adopted +silently, so reloads never replay old presses). This is why the record buttons +used to do nothing — the presses died on the mirror. + +Related: A1 overrides `on_discontinuity` to ignore `SettingsChanged` (raised on +*every* settings sync of any plugin), so the response curve, pilot windows and +background floor survive ordinary UI interaction; source changes and seeks +still reset everything. + +## Where the inputs come from + +| Input | Source | +|---|---| +| camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()` | +| phase-0 markers | rising `frame.external_triggers()` — the host **banks trigger edges from dropped preview frames** into the next processed frame (drain-to-newest and the preview throttle drop whole frames; at low modulation frequencies the survivors alone rarely held 2 markers inside the analysis window) | +| modulation period `T` | measured from the `EXT_TRIGGER` marker spacing; else the modulation plugin's acknowledged waveform — which, since the board-echo fallback, includes the **operator-armed UI drive**, not only service-path (leased) targets | +| optical modulation depth `a` | photodiode plugin's optical summary (`measured_log_contrast`) | +| ROI, masked pixels | augur-rs camera config (`CTX_GLOBAL_SETTINGS`) | + +## Tests + +`cargo test -p augur-plugin-stage-a-a1` covers trigger-defined period, marker-anchored +folding, ON/OFF separation of the rolling dataset, auto-window detection and the `q_p` +path, file-safe id generation, UTC timestamp formatting, the config-sidecar builder, +the pilot-window round-trip through the measurement folder, press-latch edge/baseline +semantics, the jittery-marker free-running fallback, sweep-point spacing, the +sweep-point sidecar fields, the ordered camera → PDQ → PDQ finalize → camera +finalize lifecycle (including envelope identity/revision and save location), and +the selective discontinuity reset. diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index 49279fa..e9077f9 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -7,10 +7,32 @@ ## What it is -The simplest possible laser-modulation control for the Stage-A bench: one power slider in DAC -codes (J23 output, `DAC1.4`), a mode select (`CONST`/`SINE`/`SQUARE`) with frequency -(0.01–2000 Hz) and a min threshold for the periodic modes, and a user-set **max limit** that caps -the slider so a device with a lower tolerated input voltage can never be overdriven from the UI. +Laser-modulation control for the Stage-A bench with two orthogonal axes: + +- **Drive method** defines the DAC operating band. `MANUAL` uses Power + Min threshold; + `CALIBRATED` derives it from `V_null`, `Vπ`, `I_k`, and optical depth `a`. +- **Mode** defines the shape that fills the band: `CONST`, `DAC_SINE`, `SQUARE`, + `OPTICAL_LOG_SINE`, or `OPTICAL_LINEAR_SINE`. All five remain available under both methods. + +The always-visible **max limit** is the hard DAC ceiling for every manual and calibrated drive. +The settings schema shows only the selected method's parameter block and refreshes when Method +changes; Manual is the default. + +| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `Vπ` | +|---|---|---| +| `CONST` | hold `power` | hold the DAC code for `I_k` | +| `DAC_SINE` | DAC sine across the band | DAC sine across the band | +| `SQUARE` | DAC square across the band | DAC square across the band | +| `OPTICAL_LOG_SINE` | intensity log-sine across the band | intensity log-sine about `I_k` | +| `OPTICAL_LINEAR_SINE` | intensity linear-sine across the band | intensity linear-sine about `I_k` | + +Manual optical modes reuse the persisted `V_null`/`Vπ` lobe parameters and derive effective +`(I_k, a)` from the manual DAC band through the forward `sin²` transfer. Both optical modes then +use the same inversion path described in [Optical waveform drive](./stage-a-optical-waveform.md). + +`V_null`/`Vπ` are measured, not typed: the Calibration section sweeps settled `CONST` codes +against the photodiode and fits the lobe — see +[Pockels transfer calibration](./stage-a-pockels-calibration.md). Every accepted setting change is transferred to the Teensy **immediately** as one `MOD` command — no Apply button, no experiment state machine. The panel shows the modulation and live DAC code the @@ -27,13 +49,35 @@ board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded val flow — bench control must work with no camera attached. `process_frame()` only disconnects defensively in replay mode. - Firmware output is **set-and-hold** (`stage-a-controller` ADR 002): disconnecting does not stop - the modulation. The power slider at 0 is the off switch. -- Safety invariants enforced plugin-side: `level ≤ max_level`, `min_level ≤ level`; the firmware - waveform peaks at `level` by construction. + the modulation. Manual Power at 0 drives 0 V; automation has an explicit `SafeOff` operation. +- Safety invariants enforced plugin-side: `min_level ≤ level ≤ max_level` for Manual and every + resolved calibrated/optical peak must be `≤ max_level`; invalid drives are refused. +- Status and commanded summaries include Method and the resolved `(lo, hi, hold)` DAC band. - `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. +- The workflow-owner service and `WaveformV1` automation path remain exact-waveform contracts and + do not use the UI Drive method. +- **`SetOpticalDepth`** (ADR 010): under an automation lease the service can retarget the *depth* + `a` of the drive the operator armed — same `drive_command()` builder as the UI path, everything + else untouched. Refused with no device link, a manual-DAC method, or a constant mode; the derived + drive still passes all safety validation. Used by the A1 amplitude sweep. +- **Link watchdog**: the device thread exits after 5 consecutive serial failures (marking the + device disconnected/faulted), and the control tick reaps a finished device thread and + auto-reconnects with a 2 s backoff while `connect` stays requested. Previously a wedged or dead + link silently swallowed every queued command — the UI kept accepting mode changes while the + board held the old waveform. +- **`protocol_run` forwarding**: the UI mirror records the request and the settings snapshot + starts/stops the protocol on the live worker (which owns the device link); only value + *transitions* act, so re-applied snapshots cannot restart a finished protocol. +- **Board-echo `acknowledged` fallback**: the published `ModulationStateV1.acknowledged` now falls + back to a revision-0 target built from the board's `MOD`/`STATUS` echo (`mod_wave`, `mod_level`, + `mod_min`, `mod_freq_mhz`) when no service-path acknowledgement exists. UI-driven drives never + produce a service ACK, so consumers (A1's fallback modulation period) previously saw no waveform + at all for the normal operator workflow. WARP (optical) echoes map to `Periodic` — the fallback's + consumers only need the frequency. ## Verification -`cargo test -p augur-plugin-stage-a-modulation` — mock round trips: immediate transfer on slider -change, board-code echo, max-cap clamping (including schema regeneration), square drive with min -threshold, Output OFF. +`cargo test -p augur-plugin-stage-a-modulation` covers method/mode enum index round-trips, +conditional settings blocks, method-resolved bands, manual optical-band inversion, hard-ceiling +rejection, immediate mock transfer, board-code echo, square drive, and owner-service fail-safe +behavior. diff --git a/docs/features/stage-a-optical-waveform.md b/docs/features/stage-a-optical-waveform.md new file mode 100644 index 0000000..c0ab98b --- /dev/null +++ b/docs/features/stage-a-optical-waveform.md @@ -0,0 +1,125 @@ +# Stage-A Optical Waveform Drive + +- **Crate:** `plugins/stage-a-modulation` (`waveform.rs`) +- **Firmware:** `stage-a-controller` — `MOD wave=WARP` (`stimulus_mod::configureWarp`) +- **Status:** Analytic inversion, fed by a measured `V_null`/`Vπ` + ([Pockels transfer calibration](./stage-a-pockels-calibration.md)); a fully + measured LUT remains a documented follow-up +- **ADR:** [ADR 008](../adr/008-stage-a-optical-waveform-inversion.md) + +## Why + +The Pockels/PBS amplitude modulator has a `sin²` transfer, so a pure DAC sine +does **not** produce a sinusoidal *optical* target. On one monotonic lobe: + +```math +I(V) = I_\text{floor} + (I_\text{ceil}-I_\text{floor})\,\sin^2[\alpha (V - V_\text{null})], +\qquad \alpha = \frac{\pi}{2 V_\pi}. +``` + +To hit a chosen optical target the DAC must be pre-warped by inverting it: + +```math +u(t) = \frac{I_d(t)-I_\text{floor}}{I_\text{ceil}-I_\text{floor}},\qquad +V(u) = V_\text{null} + \frac{2 V_\pi}{\pi}\,\arcsin\!\sqrt{u}. +``` + +## Targets + +- **`OPTICAL_LOG_SINE`** (recommended A1 input): `ln I_d = ln I_g + (a/2)\sin\omega t`. + The event camera responds to changes in `ln I`, so this is the clean input. +- **`OPTICAL_LINEAR_SINE`**: `I_d = I_c(1 + m\sin\omega t)`, `m = \tanh(a/2)`. + +Both operate around an explicit operating point and are refused if their optical +maximum exceeds the lobe ceiling. `DAC_SINE` remains the pure-DAC sine. + +## Inversion parameters (settable — you do not need a rig to start) + +| Setting | Meaning | +|---|---| +| `V_null` | DAC code at the excitation minimum (`sin² = 0`) | +| `Vπ` | DAC-code quarter-wave distance from `V_null` to the excitation maximum | +| `a` | requested optical log-modulation depth `ln(I_max/I_min)` | +| `I_k` | operating illumination as a normalised lobe intensity `u_k ∈ (0,1]` | + +Get `V_null`/`Vπ` from a two-point check (code giving min light, code giving max +light on one lobe) or from nominal `Vπ ÷ driver volts-per-code`. The drive is +refused (never silently clamped) if `V_null + Vπ` overruns `0..4095`. + +### Fixed operating point `I_k`, swept depth `a` + +`I_k` is the geometric-mean point the modulation swings around: +`u(t) = u_k·exp[(a/2) sin ωt]` (log) or `u_k·(1 + m sin ωt)` (linear). **Hold +`I_k` fixed and sweep `a`** for one response curve. The drive is refused +(`Saturates`) when the peak `u_k·exp(a/2) > 1` — lower `I_k` or `a`. + +`CONST` is the exception because it does not modulate: it maps only `I_k` +through the inverse lobe and ignores `a`. For example, `V_null=1630`, +`Vπ=860` gives DAC `2490` at `I_k=1` and DAC `1685` at `I_k=0.01`. +Periodic modes still require the headroom above. Invalid setting changes are +rejected transactionally, so the UI retains the last applied value instead of +showing a target that the board never received. Photodiode RAW/EXCITATION mode +does not participate in this DAC calculation. + +### Drive method and hard ceiling + +Under `CALIBRATED`, `V_null`/`Vπ`/`I_k`/`a` define the operating band directly. +Under `MANUAL`, the Power + Min-threshold DAC endpoints are passed through the +forward `sin²` transfer and converted to the target law's effective `(I_k, a)`; +the same inverse-warp implementation then fills that band. + +Warp codes are absolute lobe codes and cannot be rescaled without distorting the +target. The plugin therefore **refuses** any drive whose peak exceeds the +always-visible `max_level` hard ceiling. Raise the max limit, or lower the +operating band / `I_k` / `a` / `Vπ`, to fit. + +### Modulation reference range + +For the current method the plugin reports the resolved DAC lower endpoint, +upper endpoint, constant hold code, and peak-to-peak swing. + +### Measured parameters (built) and the measured LUT (still future) + +`V_null`/`Vπ` are no longer typed in from a datasheet: the +[Pockels transfer calibration](./stage-a-pockels-calibration.md) sweeps settled +constant DAC codes, reads the photodiode level at each, and fits the lobe those +two parameters describe. The analytic `sin²` inversion above is unchanged — it is +now fed measured parameters. + +The fully measured **LUT** remains open: keep the swept `(code → optical level)` +table for one monotonic lobe and invert it directly instead of the analytic +form, dropping in behind the same `warp_table` interface and superseding +`V_null`/`Vπ` entirely. The calibration record already archives the points such +a table would need. + +## Wire form (firmware line limit) + +The command line is capped at 192 bytes, too small for a 256-code table, so the +plugin computes and validates the warp table locally (for the operator preview +and range guard) but sends the compact **parameters**: + +``` +MOD wave=WARP freq_mhz= target= a_milli= u_k_milli= v_null= v_pi= +``` + +The firmware rebuilds the identical 256-entry DAC table with the same formula +(`stimulus_mod::normalisedIntensity` + `dacForU`) and plays it back at the drive +frequency. A chunked **table upload** command is the natural extension for the +measured LUT. + +## Relationship to the measured `a` + +The requested `a` here is a *drive* target. The realised optical depth is always +the photodiode-measured `a` from the [photodiode plugin](./stage-a-photodiode.md) +(estimator geometry, rejected-complement corrected), never the commanded value. + +## Tests + +`cargo test -p augur-plugin-stage-a-modulation waveform` verifies both targets +stay in the DAC range, that feeding the warp table back through the `sin²` lobe +recovers the intended optical intensity, that the recovered log-contrast matches +the requested `a`, that a manual DAC band round-trips through +`OpticalDrive::from_dac_band`, and that invalid depth/inversion and lobe overruns +are refused. `cargo test -p stage-a-io mod_warp` covers the mock command surface. +The modulation-plugin tests also pin the full-lobe `CONST` values above and +verify that a rejected periodic `I_k` change cannot diverge from the board target. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index ca688b4..122e6ff 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -14,7 +14,19 @@ plus the newest value. During a command-port acquisition the firmware mirrors th blocks here — every rate change or sample-index jump restarts the ring as a new segment, so the `index / rate` time base is always consistent. -Two modes: +## Phase-0 trigger overlay + +The firmware stamps a device-clock **`Marker` frame** (wire type 4) on the stream at every +modulation phase-0, in step with the J24 camera trigger. Because the chart is on the device +(Teensy) sample clock — not the camera clock — this stream marker is the correctly-aligned phase-0 +source (the camera `EXT_TRIGGER` belongs to A1's camera-clock analysis, not here). + +- **Show phase-0 trigger markers** (opt-in) overlays them as one toggleable vertical curve + ("phase-0 trigger") on the chart. +- The **modulation frequency is derived from the marker spacing** (`f = rate / mean marker gap`) and + shown in the status; the mock emits synthetic markers so the overlay works without hardware. + +## Modes - **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). - **EXCITATION** — the diode sits behind the PBS in the excitation path and measures the light @@ -32,6 +44,21 @@ Two modes: (`avg_sync_freq_hz`, e.g. the MOD drive frequency): window = `rate / f` samples, which makes the mean independent of the modulation phase instead of riding the waveform. +## Data (cache snapshot + disk recording) + +- The monitor cache always holds the last *N* seconds (`cache_s`). **Save cache + snapshot** writes it **once** as `pd_cache_.csv` + JSON sidecar. +- **Start recording** / **Stop recording** buttons tee every incoming sample + frame to `pd_rec_.pdq`; stopping writes the JSON sidecar. Both + buttons (and the snapshot) are disabled until a data directory is selected. +- All three are momentary buttons whose presses are forwarded from the UI + mirror to the live worker as monotonic press counters (`PressLatch`, ADR 010) + and act only on a press **edge**. The previous unguarded `save_snapshot` + handler fired on every host settings sync — one unwanted CSV per settings + change of *any* plugin — and the old `record` checkbox synced the mirror's + always-false state to the worker, so it could never stay recording. The + `record` boolean setting remains as a non-schema compatibility alias. + ## Contract - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the @@ -51,4 +78,5 @@ Two modes: jumps and rate changes, duration-bounded ring with aligned indexes, moving-average window derivation from the sync frequency, newest-window average, envelope decimation bounds and min ≤ mean ≤ max, raw rendering for short windows, excitation inversion, mock reader, settings -round-trips. +round-trips, the forwarded snapshot counter saving exactly once, and the record start/stop +buttons. diff --git a/docs/features/stage-a-pockels-calibration.md b/docs/features/stage-a-pockels-calibration.md new file mode 100644 index 0000000..d1a9eca --- /dev/null +++ b/docs/features/stage-a-pockels-calibration.md @@ -0,0 +1,214 @@ +# Stage-A Pockels Transfer Calibration + +- **Crate:** `plugins/stage-a-modulation` (`calibration.rs`) +- **Depends on:** `stage-a-photodiode` publishing `PhotodiodeStreamV1.level` +- **Status:** built +- **ADR:** [ADR 011](../adr/011-stage-a-pockels-transfer-calibration.md) +- **Knowledge base:** `methodology/pockels-waveform-linearisation.md` §4, + `setup/optical-path.md` + +## Why + +`V_null` and `Vπ` drive every calibrated waveform through the optical inversion +([Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md)), but they were +two bare number fields whose tooltip said *"measure it; do not trust nominal +Vπ"* — with no way to measure it. Nothing in the UI connected a DAC code to an +observed photodiode value, so the operator had to hand-sweep `CONST`, watch a +chart in another panel, and do the arithmetic by eye. + +## What it does + +One button. The modulation plugin steps settled `CONST` DAC codes across +`0..max_level` (49 points up, then the same 49 back down, ~20 s), reads the +photodiode level at each, and fits the lobe: + +```math +P(c) = p_0 + p_1 \sin^2\!\left[\frac{\pi (c - V_\text{null})}{2 V_\pi}\right] +``` + +The fit is then reviewed and applied by a second, explicit press. + +## Why the modulation plugin owns it + +It already owns `V_null`/`Vπ` and the DAC. The host broadcasts every plugin's +control snapshot to every plugin's inbox, so it reads photodiode levels **read +only** — no lease, no service command, no coordinating plugin, and no +photodiode recording. The photodiode simply needs to be connected. + +## Three things the physics forces + +**The detector port is an input, not a result.** `sin²` is symmetric about its +peak, so `(v, p_0, p_1)` and `(v + V_\pi, p_0 + p_1, -p_1)` fit the measured +curve *identically* — the data cannot say which extremum is zero excitation. + +The setting asks one observable question: *when the light reaching the sample +gets brighter, does the photodiode reading go up or down?* Stage-A's photodiode +sits on the PBS **reject** port and reads the light the sample does not get, +`I_pd = I_tot − I_exc`, so it falls as the sample brightens — and reads its +**maximum** at `V_null`. That is `REJECT PORT`, the default. `DIRECT` is for a +detector watching the sample beam itself. Declaring it wrong places `V_null` a +quarter wave off and runs the drive on the inverted branch. + +**The shape needs no dark measurement and no anchor.** `p_0` absorbs the dark +level and any DC offset; `p_1` absorbs the front-end gain. `V_null` and `Vπ` +are immune to both, which is why this procedure is one button and not a +protocol. + +**The absolute scale is *not* recoverable here.** On the reject port the +residual transmitted floor cannot be separated from the total-power anchor +`I_tot` (knowledge base §4.4). The detector level at the null is therefore +reported as a **lower bound** on `I_tot`, explicitly not as the anchor, and no +maximum achievable `a` is derived from it. Freezing a real anchor still needs a +transmitted-port power measurement. + +## How the fit works + +Because `sin²(x) = (1 − cos 2x)/2`, the model is a constant plus **one sinusoid +of period `2Vπ`**, and a sinusoid of known period is linear in its quadrature +components. So for each candidate `Vπ` the phase (hence `V_null`) and both +amplitudes come from a 3×3 linear solve, and only `Vπ` is searched: a +log-spaced scan over every period the sweep can resolve, then a golden-section +refine. + +Seeding the period from the measured extrema — the obvious approach — breaks on +exactly the sweeps that matter. At a realistic `Vπ ≈ 860` the DAC range holds +~2.4 lobes, so the global minimum and maximum can sit whole periods apart. + +Several nulls are valid when a sweep spans multiple lobes; the fit reports the +**lowest** one whose `[V_null, V_null + Vπ]` fits inside the max limit — least +voltage across the crystal, most headroom, and a rule the operator can predict. + +## Settling is proven, not timed + +Every published level carries `end_sample_index` and `sample_count` on the +device sample clock. A point is accepted only from a window that *began* at +least `SETTLE_SAMPLES` (2 000 ≈ 100 ms at 20 kSa/s) after its code was +commanded. No shared wall clock, no sleeps, immune to control-tick jitter. + +## The sweep owns the DAC while it runs + +`send_modulation` is silent for the duration. The host re-applies the *whole* +settings snapshot on every sync and most drive handlers push to the board +unconditionally, so without this the operator's armed waveform would be +re-armed on top of every commanded code — the board would play the armed drive +through the sweep, every point would read the same waveform-averaged level, and +the fit would report "the detector level did not change" on a bench where the +light was plainly modulating. Same shape as the automation-lease guard: a sweep +is another owner of the DAC. + +Settings changed mid-sweep are withheld, not rejected, and reach the board when +the sweep ends — the restore prefers the current drive and falls back to the +command captured at sweep start. + +## Interlocks + +The sweep refuses to start, and aborts if any becomes true mid-run, unless: +hardware effects are allowed on this instance, the command port is connected, +**no automation lease is held** (A1 must not be sweeping the drive at the same +time), no protocol is running, and a photodiode level is arriving. + +It always restores the pre-sweep drive — on completion, abort, stop press, +disconnect, or a stalled stream. A calibration sweep leaves the bench as it +found it. + +## Robustness: strays are dropped, the rest is a warning + +The fit runs twice. The first pass finds the period; points whose residual +exceeds **6× the median** absolute residual are then dropped and the fit is +repeated on what is left. The cut is on the median, not the mean or standard +deviation, because those are themselves dragged out by the very points being +looked for. `6 × median` is roughly 4σ for Gaussian noise, so ordinary scatter +survives untouched. + +This matters because of how the numbers actually behave on a bench. Measured on +a realistic small-signal sweep (90 mV span, `Vπ = 860`, 2.4 lobes): + +| Condition | Residual | Fitted `Vπ` | +|---|---|---| +| clean | 0.0 % | 860 | +| 5 mV noise | 3.1 % | 864 | +| 10 mV drift across the sweep | 3.2 % | 861 | +| 10 mV hysteresis | 5.5 % | 861 | +| **one stray point** | **9.9 %** | **863** | +| amplifier compressing the top of the range | 15.2 % | 1110 ✗ | + +A single bad sample inflates the residual fivefold while leaving `Vπ` accurate +to three codes — and it is invisible in the plot. That is why the residual +**warns and never blocks**: blocking on it withholds a good calibration for a +bad reason. A residual that stays high after rejection, with a visibly poor +overlay, is the real signal — and as the last row shows, it comes with a `Vπ` +that is wrong in a way the plot makes obvious. + +The fit is **never** applied automatically, and applying re-validates the +resulting drive: a calibration that cannot be armed is rolled back rather than +stored. Warnings surface as `Check:` lines in the status: + +| Warning | Meaning | +|---|---| +| residual > 5 % of the span | compare fit and points in the plot before trusting `Vπ` | +| points dropped | a couple is ordinary; a large share means the sweep is the problem | +| hysteresis > 5 % | the cell is drifting, or the settle time is too short | +| clipped points | the extremum they sit on is not where the fit thinks it is | + +There is no separate "lobe coverage" gate: `fit_transfer` already refuses a +sweep in which no full lobe fits inside the commandable range, so `Vπ` is always +measured rather than extrapolated by the time a fit exists. + +## The transfer-curve view + +A `LineSeriesWindow` host view, `Pockels transfer curve`: + +- **before any sweep** — the lobe the *configured* `V_null`/`Vπ` claim, on a + normalised `u` axis, with markers at `V_null` and `V_null + Vπ`. This works + with no hardware attached and is the answer to "what are these two numbers". +- **after a fit** — `measured ↑`, `measured ↓`, the fitted curve, and (while + they differ) the configured lobe on the fit's own scale, in detector volts. + +## Provenance + +Applying writes `pockels-.json` into the optional calibration folder +(points, fit, geometry, residual, hysteresis, and the anchor caveat) and sets +`ModulationStateV1.calibration_id`, so a consumer's sidecar can cite which +inversion produced a run's optical depth. Leaving the folder empty applies the +fit without archiving, and says so. + +## Dual-instance note + +The host renders `settings_schema()` from the **UI mirror**, which never owns +the device link, a lease, a sweep, or a fit. A `SettingKind::Button { enabled }` +may therefore only depend on state that is itself a setting — anything else is +invisible to the instance that draws it and disables the button forever. The +calibration buttons gate on "the operator asked to connect"; every real +interlock is enforced on the worker and reported in the status lines, which the +host does take from the worker. + +## Settings + +| Key | Meaning | +|---|---| +| `detector_geometry` | which PBS port the photodiode watches (`REJECT PORT` default) | +| `calibrate` | measure the transfer curve; press again to abort | +| `calibrate_apply` | write the reviewed fit into `V_null`/`Vπ` | +| `calibration_dir` | optional archive folder for the calibration record | + +`V_null`/`Vπ` remain directly editable as the manual override. + +## Verification + +- `calibration.rs` unit tests recover a known lobe from **both** ports, across + a multi-lobe sweep, and with a null at code 0; they check the geometry input + selects between the two equivalent representations, and that flat sweeps, + short sweeps, and out-of-range lobes are refused. +- An end-to-end test runs the sweep against the mock board, synthesizing the + light the reject-port detector *would* report for whatever code the board is + actually holding — ground truth for commanding, settle gating, point + collection, the fit, and the drive restore. + +## Limits + +- Analytic `sin²` inversion, not a measured LUT (the knowledge base's eventual + target); the calibration record stores the points a LUT would need. +- Dark level and the total-power anchor remain separate measurements. +- Static transfer only. A static calibration must never be used to correct + dynamic roll-off — that would manufacture the Bode curve A1 measures + (knowledge base "Gotchas"). diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md index ab9167f..e3bfbe2 100644 --- a/docs/features/stage-a.md +++ b/docs/features/stage-a.md @@ -1,7 +1,7 @@ # Stage-A Bench Stack -- **Status:** Simplified two-plugin setup (2026-07-15, ADR 006) -- **Firmware:** `stage-a-controller` 0.3.0 (Teensy 4.1 on Hermit V2r1, `USB_DUAL_SERIAL`) +- **Status:** Two persistent owners plus orchestrated experiment workflows (ADR 007) +- **Firmware:** `stage-a-controller` 0.4.0+ (Teensy 4.1 on Hermit V2r1, `USB_DUAL_SERIAL`) ## Current shape @@ -10,22 +10,26 @@ The Teensy enumerates as **two** USB serial ports, and each is owned by exactly | Port | Content | Owner | |---|---|---| | command port (first) | v1 ASCII commands + PDA1 binary frames | [`stage-a-modulation`](./stage-a-modulation.md) | -| stream port (second) | free-running `PD code=… n=… t_ms=…` lines, 50 Hz | [`stage-a-photodiode`](./stage-a-photodiode.md) | +| stream port (second) | free-running PDA1 `SamplesU16` frames, 20 kSa/s default | [`stage-a-photodiode`](./stage-a-photodiode.md) | -- **`stage-a-modulation`** — capped power slider + constant/sine/square drive of the laser - modulation input (J23), transferred to the Teensy immediately; shows the board-reported DAC - code. Firmware output is set-and-hold; "Output OFF" is the explicit stop. +- **`stage-a-modulation`** — Manual/Calibrated operating-band selection plus five independent + waveform modes under one hard DAC ceiling, transferred to J23 immediately; shows the resolved + band and board-reported DAC code. Firmware output is set-and-hold; automation uses an explicit + `SafeOff` operation. - **`stage-a-photodiode`** — live readout of SMA5/pin 18/A4, raw or inverted to excitation power `I_exc = I_tot − I_pd` against a user-set reference. - **`stage-a-io`** (shared non-plugin library) — PDA1 wire format, typed client with idempotent retries, bounded I/O worker, and a firmware-faithful mock (including the 0.3.0 `MOD` verb). - The estimator/pdq/sidecar modules are retained for the future A1–A3 experiment plugins. + The photodiode owner uses the parser/PDQ modules; experiment plugins may use + hardware-free readers/analysis but never open the ports. +- **`stage-a-a1`** — orchestrates both owner services and camera recording; it + never opens a Teensy port or writes PDQ directly. Architecture: ADR 007. ## History -The earlier commissioning stack (`stage-a-monitor`, `stage-a-funcgen`, `stage-a-a1` — device +The earlier commissioning stack (`stage-a-monitor`, `stage-a-funcgen`, old `stage-a-a1` — device monitor with calibrated contrast, waveform familiarisation, and the A1 minimum-depth Bode sweep) was removed on 2026-07-15 as too complex for the current bench stage (ADR 006). It remains in git -history; the experiment plugins will be rebuilt on the simplified stack when the bench needs -them. Device-ownership and safety rules: ADR 005 (one owner per port, fail-closed effects gate) -as amended by ADR 006. +history; the new A1 implementation uses different statistics and host-routed +orchestration. Device-ownership and safety rules: ADR 005 as amended by ADR 006 +and ADR 007. diff --git a/plugins/stage-a-a1/Cargo.toml b/plugins/stage-a-a1/Cargo.toml new file mode 100644 index 0000000..4795fbe --- /dev/null +++ b/plugins/stage-a-a1/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "augur-plugin-stage-a-a1" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A A1 workflow orchestrator and pure minimum-depth analysis core" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde.workspace = true +serde_json.workspace = true +stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } +toml = "0.8" + +[lints.rust] +unsafe_code = "forbid" diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md new file mode 100644 index 0000000..50d98a6 --- /dev/null +++ b/plugins/stage-a-a1/README.md @@ -0,0 +1,53 @@ +# Stage-A A1 Analysis + +`stage-a-a1` is the Stage-A **recording coordinator** plus two live sanity quicklooks. One button +records the camera **RAW** stream and the photodiode **PDQ** stream together for a fixed duration, +groups them under a per-`(I_k, f)` measurement id, and writes an A1 config sidecar (`.toml`) linking +the files with the modulation settings, the measured modulation depth `a`, the ROI, and the trigger +info needed to reproduce and analyse the run offline. A second button, **Start sweep**, repeats that +per amplitude: it leases the modulation owner, retargets the armed calibrated drive to each `a` in +`[Sweep min a, Sweep max a]`, waits for the photodiode-measured `a` to settle, and records every +point (`…_pNN`). Outside the leased sweep A1 owns no hardware and never drives the Teensy — arm the +optical drive in the modulation plugin; A1 only reads its published settings. + +## Recording + +- **Output folder** — where the A1 config sidecar is written (recommended shared experiment root). +- **Measurement id** — one per `(I_k, f)` pair; auto-generated default, editable, or press **New id**. +- **Duration (s)** — each recording auto-stops and finalizes after this. +- **Start recording** — starts camera RAW, then connects/leases the photodiode and starts PDQ; + the timer begins after both acknowledge. It auto-finalizes PDQ first, camera second, then writes + the sidecar. **Stop** saves the current recording early (and aborts a running sweep). +- **Start sweep** — records **Sweep points (count)** amplitudes spanning `[Sweep min a, Sweep max a]` + (min > 0): per point it renews the modulation lease, issues `SetOpticalDepth`, waits for the + measured `a` to hold the target for **Sweep settle (s)** (30 s cap, then records anyway), and runs + one normal recording. Sidecars carry `sweep.requested_a` / `point_index` / `point_total`. +- The record/sweep buttons are disabled until an output folder is selected. + +Files share an `_` stem: `/_.raw` (camera, under the host output root), +`/__pd.pdq` + `.json` (photodiode, under its data root), and +`/__config.toml` (A1, under the chosen folder). Point all three roots at the same +experiment directory to co-locate everything. The host also writes its own `.toml` next to the +RAW with the camera biases/ROI; the A1 sidecar cross-references it. + +## Live quicklooks + +- **Rolling half-period response** `S_p(t) = N_p(t−T/2, t] / N_valid` — events per valid pixel in the + trailing half-cycle, ON and OFF. A live "are events appearing, is the ON/OFF timing sane?" check. +- **Response probability** `q_p` — fraction of valid pixel-cycles that fire at least once in the + ON/OFF phase window (each pixel-cycle counts once, unlike `S_p`). The windows come from the row's + **pilot** when one has been recorded (frozen and held across the row), otherwise auto-detected + from the trigger-anchored fold (each grows out from its histogram peak to the window floor, + default 10 % of peak). `Record pilot` / `Record background` (in the Recording section) capture the + frozen windows and the floor `q0` into the measurement folder and are auto-reloaded when you + return to that folder + id. Record one point per amplitude vs the photodiode-measured `a`. The + authoritative `q_p(a, f)` fit is computed **offline** from the recordings; this is a quicklook. + +The period `T` comes from the firmware phase-0 `EXT_TRIGGER` marker spacing (the trigger *defines* +the frequency), falling back to the modulation plugin's acknowledged waveform. The ROI and masked +pixels come from the augur-rs camera config. + +See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the full brief, +[ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, and +[docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the +planned amplitude-sweep automation on top of this. diff --git a/plugins/stage-a-a1/plugin.toml b/plugins/stage-a-a1/plugin.toml new file mode 100644 index 0000000..4515f43 --- /dev/null +++ b/plugins/stage-a-a1/plugin.toml @@ -0,0 +1,9 @@ +id = "stage-a.a1" +name = "Stage-A A1 Analysis" +version = "0.3.0" +description = "Stage-A A1 recording coordinator: one-button synchronized camera .raw + photodiode .pdq recording with a config sidecar, plus live rolling-response and response-probability quicklooks." +domain = "stage-a" +library = "augur_plugin_stage_a_a1" +phase = "raw_events" +min_augur_version = "1.0.0" +host_commands = ["start_recording", "stop_recording"] diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs new file mode 100644 index 0000000..b67952b --- /dev/null +++ b/plugins/stage-a-a1/src/lib.rs @@ -0,0 +1,14 @@ +//! Pure scientific and workflow core for the Stage-A A1 experiment. +//! +//! This crate intentionally contains no serial transport, Teensy client, or +//! PDQ writer. Hardware ownership remains with the Stage-A modulation and +//! photodiode plugins; this code only validates and analyses immutable inputs. + +pub mod phase; +pub mod rates; +pub mod response_curve; +mod runtime; +pub mod types; + +pub use runtime::StageAA1Plugin; +pub use types::{CameraEvent, Polarity}; diff --git a/plugins/stage-a-a1/src/phase.rs b/plugins/stage-a-a1/src/phase.rs new file mode 100644 index 0000000..0db0032 --- /dev/null +++ b/plugins/stage-a-a1/src/phase.rs @@ -0,0 +1,382 @@ +//! EXT_TRIGGER marker validation and camera-clock phase folding. + +use crate::types::{CameraEvent, Polarity}; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct MarkerValidationConfig { + pub expected_frequency_hz: f64, + pub frequency_tolerance_fraction: f64, + pub max_period_jitter_fraction: f64, + /// Expected complete cycles, when the acquisition declared one. + pub expected_cycles: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MarkerValidation { + pub cycle_count: usize, + pub measured_frequency_hz: f64, + pub mean_period_us: f64, + pub max_period_jitter_fraction: f64, + pub first_marker_us: u64, + pub last_marker_us: u64, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum MarkerError { + InvalidConfiguration(&'static str), + TooFewMarkers { + count: usize, + }, + NonIncreasing { + index: usize, + }, + CycleCount { + expected: usize, + actual: usize, + }, + FrequencyOutOfTolerance { + expected_hz: f64, + measured_hz: f64, + tolerance_fraction: f64, + }, + JitterOutOfTolerance { + measured_fraction: f64, + tolerance_fraction: f64, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FoldedEvent { + pub timestamp_us: u64, + pub x: u16, + pub y: u16, + pub polarity: Polarity, + pub cycle_index: usize, + /// Circular phase in `[0, 1)`. + pub phase: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseFold { + pub markers_us: Vec, + pub validation: MarkerValidation, + pub events: Vec, + pub events_outside_complete_cycles: usize, +} + +impl PhaseFold { + pub fn phase_at(&self, timestamp_us: u64) -> f64 { + let period_us = self.validation.mean_period_us; + (timestamp_us.saturating_sub(self.validation.first_marker_us) as f64 / period_us) + .rem_euclid(1.0) + } +} + +pub fn validate_markers( + markers_us: &[u64], + config: MarkerValidationConfig, +) -> Result { + if !config.expected_frequency_hz.is_finite() || config.expected_frequency_hz <= 0.0 { + return Err(MarkerError::InvalidConfiguration( + "expected frequency must be finite and positive", + )); + } + if !config.frequency_tolerance_fraction.is_finite() + || config.frequency_tolerance_fraction < 0.0 + || !config.max_period_jitter_fraction.is_finite() + || config.max_period_jitter_fraction < 0.0 + { + return Err(MarkerError::InvalidConfiguration( + "marker tolerances must be finite and non-negative", + )); + } + if markers_us.len() < 2 { + return Err(MarkerError::TooFewMarkers { + count: markers_us.len(), + }); + } + + let mut periods = Vec::with_capacity(markers_us.len() - 1); + for (index, pair) in markers_us.windows(2).enumerate() { + if pair[1] <= pair[0] { + return Err(MarkerError::NonIncreasing { index: index + 1 }); + } + periods.push((pair[1] - pair[0]) as f64); + } + + let cycle_count = periods.len(); + if let Some(expected) = config.expected_cycles { + if cycle_count != expected { + return Err(MarkerError::CycleCount { + expected, + actual: cycle_count, + }); + } + } + let mean_period_us = periods.iter().sum::() / cycle_count as f64; + let measured_frequency_hz = 1_000_000.0 / mean_period_us; + let frequency_error = ((measured_frequency_hz - config.expected_frequency_hz) + / config.expected_frequency_hz) + .abs(); + if frequency_error > config.frequency_tolerance_fraction { + return Err(MarkerError::FrequencyOutOfTolerance { + expected_hz: config.expected_frequency_hz, + measured_hz: measured_frequency_hz, + tolerance_fraction: config.frequency_tolerance_fraction, + }); + } + + let max_period_jitter_fraction = periods + .iter() + .map(|period| ((period - mean_period_us) / mean_period_us).abs()) + .fold(0.0_f64, f64::max); + if max_period_jitter_fraction > config.max_period_jitter_fraction { + return Err(MarkerError::JitterOutOfTolerance { + measured_fraction: max_period_jitter_fraction, + tolerance_fraction: config.max_period_jitter_fraction, + }); + } + + Ok(MarkerValidation { + cycle_count, + measured_frequency_hz, + mean_period_us, + max_period_jitter_fraction, + first_marker_us: markers_us[0], + last_marker_us: *markers_us.last().expect("at least two markers"), + }) +} + +/// Folds events against a free-running modulation period, with the phase +/// origin placed at the first event. This is the "phase-0 unanchored" path +/// used until a hardware `EXT_TRIGGER` reaches the camera: bins are relative +/// to the first event, not tied to the drive waveform. Only events inside the +/// whole-cycle span are retained so the rate normalisation matches +/// `cycle_count`. Returns `None` when the period is invalid or the window does +/// not cover at least one whole cycle. +pub fn fold_events_free_running(events: &[CameraEvent], period_us: f64) -> Option { + if !period_us.is_finite() || period_us <= 0.0 || events.is_empty() { + return None; + } + let first = events.iter().map(|event| event.timestamp_us).min()?; + let last = events.iter().map(|event| event.timestamp_us).max()?; + let cycle_count = ((last.saturating_sub(first)) as f64 / period_us).floor() as usize; + if cycle_count == 0 { + return None; + } + + let mut folded = Vec::with_capacity(events.len()); + let mut outside = 0; + for event in events { + let cycles = event.timestamp_us.saturating_sub(first) as f64 / period_us; + let cycle_index = cycles.floor() as usize; + if cycle_index >= cycle_count { + outside += 1; + continue; + } + folded.push(FoldedEvent { + timestamp_us: event.timestamp_us, + x: event.x, + y: event.y, + polarity: event.polarity, + cycle_index, + phase: cycles.fract(), + }); + } + + Some(PhaseFold { + markers_us: Vec::new(), + validation: MarkerValidation { + cycle_count, + measured_frequency_hz: 1_000_000.0 / period_us, + mean_period_us: period_us, + max_period_jitter_fraction: 0.0, + first_marker_us: first, + last_marker_us: first + (cycle_count as f64 * period_us).round() as u64, + }, + events: folded, + events_outside_complete_cycles: outside, + }) +} + +pub fn fold_events( + events: &[CameraEvent], + markers_us: &[u64], + config: MarkerValidationConfig, +) -> Result { + let validation = validate_markers(markers_us, config)?; + let mut folded = Vec::with_capacity(events.len()); + let mut outside = 0; + + for event in events { + let cycle_index = match markers_us.binary_search(&event.timestamp_us) { + Ok(index) if index + 1 < markers_us.len() => index, + Ok(_) => { + outside += 1; + continue; + } + Err(0) => { + outside += 1; + continue; + } + Err(index) if index < markers_us.len() => index - 1, + Err(_) => { + outside += 1; + continue; + } + }; + let start = markers_us[cycle_index]; + let end = markers_us[cycle_index + 1]; + let phase = (event.timestamp_us - start) as f64 / (end - start) as f64; + folded.push(FoldedEvent { + timestamp_us: event.timestamp_us, + x: event.x, + y: event.y, + polarity: event.polarity, + cycle_index, + phase, + }); + } + + Ok(PhaseFold { + markers_us: markers_us.to_vec(), + validation, + events: folded, + events_outside_complete_cycles: outside, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> MarkerValidationConfig { + MarkerValidationConfig { + expected_frequency_hz: 1_000.0, + frequency_tolerance_fraction: 0.01, + max_period_jitter_fraction: 0.02, + expected_cycles: Some(3), + } + } + + #[test] + fn validates_and_folds_against_camera_clock_markers() { + let markers = [10_000, 11_000, 12_000, 13_000]; + let events = [ + CameraEvent { + timestamp_us: 10_250, + x: 1, + y: 2, + polarity: Polarity::On, + }, + CameraEvent { + timestamp_us: 11_750, + x: 3, + y: 4, + polarity: Polarity::Off, + }, + CameraEvent { + timestamp_us: 13_000, + x: 0, + y: 0, + polarity: Polarity::On, + }, + ]; + let fold = fold_events(&events, &markers, config()).expect("valid markers"); + assert_eq!(fold.validation.cycle_count, 3); + assert_eq!(fold.events.len(), 2); + assert_eq!(fold.events_outside_complete_cycles, 1); + assert_eq!(fold.events[0].cycle_index, 0); + assert!((fold.events[0].phase - 0.25).abs() < 1e-12); + assert_eq!(fold.events[1].cycle_index, 1); + assert!((fold.events[1].phase - 0.75).abs() < 1e-12); + } + + #[test] + fn rejects_marker_count_frequency_and_jitter_mismatches() { + let mut wrong_count = config(); + wrong_count.expected_cycles = Some(4); + assert!(matches!( + validate_markers(&[0, 1_000, 2_000, 3_000], wrong_count), + Err(MarkerError::CycleCount { .. }) + )); + + assert!(matches!( + validate_markers(&[0, 2_000, 4_000, 6_000], config()), + Err(MarkerError::FrequencyOutOfTolerance { .. }) + )); + + assert!(matches!( + validate_markers(&[0, 1_000, 2_100, 3_000], config()), + Err(MarkerError::JitterOutOfTolerance { .. }) + )); + } + + #[test] + fn free_running_fold_bins_relative_to_first_event() { + // Period 1000 us; three whole cycles from the first event at 500 us. + let events = [ + CameraEvent { + timestamp_us: 500, + x: 0, + y: 0, + polarity: Polarity::On, + }, + CameraEvent { + timestamp_us: 750, + x: 0, + y: 0, + polarity: Polarity::Off, + }, + CameraEvent { + timestamp_us: 1_750, + x: 0, + y: 0, + polarity: Polarity::On, + }, + // Beyond the last whole cycle -> excluded. + CameraEvent { + timestamp_us: 4_000, + x: 0, + y: 0, + polarity: Polarity::On, + }, + ]; + let fold = fold_events_free_running(&events, 1_000.0).expect("one whole cycle"); + assert_eq!(fold.validation.cycle_count, 3); + assert_eq!(fold.events.len(), 3); + assert_eq!(fold.events_outside_complete_cycles, 1); + assert!((fold.events[0].phase - 0.0).abs() < 1e-12); + assert!((fold.events[1].phase - 0.25).abs() < 1e-12); + assert_eq!(fold.events[1].cycle_index, 0); + assert_eq!(fold.events[2].cycle_index, 1); + assert!((fold.events[2].phase - 0.25).abs() < 1e-12); + } + + #[test] + fn free_running_fold_needs_one_whole_cycle() { + let events = [ + CameraEvent { + timestamp_us: 0, + x: 0, + y: 0, + polarity: Polarity::On, + }, + CameraEvent { + timestamp_us: 400, + x: 0, + y: 0, + polarity: Polarity::On, + }, + ]; + assert!(fold_events_free_running(&events, 1_000.0).is_none()); + } + + #[test] + fn rejects_non_monotonic_markers() { + assert_eq!( + validate_markers(&[0, 1_000, 999, 2_000], config()), + Err(MarkerError::NonIncreasing { index: 2 }) + ); + } +} diff --git a/plugins/stage-a-a1/src/rates.rs b/plugins/stage-a-a1/src/rates.rs new file mode 100644 index 0000000..605a910 --- /dev/null +++ b/plugins/stage-a-a1/src/rates.rs @@ -0,0 +1,294 @@ +//! Phase-bin event rates and rolling half-period operator quicklooks. + +use crate::phase::PhaseFold; +use crate::types::Polarity; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RateLayer { + pub count: u64, + /// Events per valid pixel per second. + pub rate_per_pixel_s: f64, + /// Poisson standard error in the same units as `rate_per_pixel_s`. + pub standard_error: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseRateBin { + pub phase_start: f64, + pub phase_end: f64, + pub run: RateLayer, + pub background: Option, + /// Run minus background. Negative values are intentionally preserved. + pub net_rate_per_pixel_s: Option, + /// Independent Poisson uncertainty propagated in quadrature. + pub net_standard_error: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PolarityPhaseRates { + pub polarity: Polarity, + pub valid_pixels: usize, + pub run_cycles: usize, + pub background_cycles: Option, + pub bins: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseRateSet { + pub on: PolarityPhaseRates, + pub off: PolarityPhaseRates, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum RateError { + ZeroValidPixels, + InvalidBinCount, + EmptyCycles, +} + +pub fn phase_bin_rates( + run: &PhaseFold, + background: Option<&PhaseFold>, + valid_pixels: usize, + bin_count: usize, +) -> Result { + if valid_pixels == 0 { + return Err(RateError::ZeroValidPixels); + } + if bin_count == 0 { + return Err(RateError::InvalidBinCount); + } + if run.validation.cycle_count == 0 + || background.is_some_and(|fold| fold.validation.cycle_count == 0) + { + return Err(RateError::EmptyCycles); + } + + Ok(PhaseRateSet { + on: rates_for_polarity(run, background, valid_pixels, bin_count, Polarity::On), + off: rates_for_polarity(run, background, valid_pixels, bin_count, Polarity::Off), + }) +} + +fn rates_for_polarity( + run: &PhaseFold, + background: Option<&PhaseFold>, + valid_pixels: usize, + bin_count: usize, + polarity: Polarity, +) -> PolarityPhaseRates { + let mut run_counts = vec![0_u64; bin_count]; + let mut background_counts = vec![0_u64; bin_count]; + for event in run.events.iter().filter(|event| event.polarity == polarity) { + run_counts[phase_bin(event.phase, bin_count)] += 1; + } + if let Some(background) = background { + for event in background + .events + .iter() + .filter(|event| event.polarity == polarity) + { + background_counts[phase_bin(event.phase, bin_count)] += 1; + } + } + + let run_bin_s = run.validation.mean_period_us / 1_000_000.0 / bin_count as f64; + let run_exposure = valid_pixels as f64 * run.validation.cycle_count as f64 * run_bin_s; + let background_exposure = background.map(|fold| { + valid_pixels as f64 + * fold.validation.cycle_count as f64 + * (fold.validation.mean_period_us / 1_000_000.0 / bin_count as f64) + }); + + let bins = (0..bin_count) + .map(|index| { + let run_layer = poisson_layer(run_counts[index], run_exposure); + let background_layer = background_exposure + .map(|exposure| poisson_layer(background_counts[index], exposure)); + let (net, net_error) = background_layer.map_or((None, None), |background| { + ( + Some(run_layer.rate_per_pixel_s - background.rate_per_pixel_s), + Some( + (run_layer.standard_error.powi(2) + background.standard_error.powi(2)) + .sqrt(), + ), + ) + }); + PhaseRateBin { + phase_start: index as f64 / bin_count as f64, + phase_end: (index + 1) as f64 / bin_count as f64, + run: run_layer, + background: background_layer, + net_rate_per_pixel_s: net, + net_standard_error: net_error, + } + }) + .collect(); + + PolarityPhaseRates { + polarity, + valid_pixels, + run_cycles: run.validation.cycle_count, + background_cycles: background.map(|fold| fold.validation.cycle_count), + bins, + } +} + +fn phase_bin(phase: f64, bin_count: usize) -> usize { + ((phase.rem_euclid(1.0) * bin_count as f64).floor() as usize).min(bin_count - 1) +} + +fn poisson_layer(count: u64, exposure_pixel_s: f64) -> RateLayer { + RateLayer { + count, + rate_per_pixel_s: count as f64 / exposure_pixel_s, + standard_error: (count as f64).sqrt() / exposure_pixel_s, + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RollingResponsePoint { + pub timestamp_us: u64, + /// Events in `(t - T/2, t]` per valid pixel. + pub run_per_pixel: f64, + /// Integral of the periodic phase-resolved background rate, when enabled. + pub background_per_pixel: Option, + pub net_per_pixel: Option, +} + +pub fn rolling_half_period_response( + run: &PhaseFold, + polarity: Polarity, + valid_pixels: usize, + sample_times_us: &[u64], + background_model: Option<&PolarityPhaseRates>, +) -> Result, RateError> { + if valid_pixels == 0 { + return Err(RateError::ZeroValidPixels); + } + let half_period_us = run.validation.mean_period_us / 2.0; + let period_s = run.validation.mean_period_us / 1_000_000.0; + Ok(sample_times_us + .iter() + .map(|×tamp_us| { + let window_start = timestamp_us as f64 - half_period_us; + let count = run + .events + .iter() + .filter(|event| { + event.polarity == polarity + && event.timestamp_us as f64 > window_start + && event.timestamp_us <= timestamp_us + }) + .count(); + let run_per_pixel = count as f64 / valid_pixels as f64; + let background_per_pixel = background_model.map(|model| { + let start_phase = run.phase_at(timestamp_us.saturating_sub(half_period_us as u64)); + integrate_periodic_rates(model, start_phase, 0.5) * period_s + }); + RollingResponsePoint { + timestamp_us, + run_per_pixel, + background_per_pixel, + net_per_pixel: background_per_pixel.map(|bg| run_per_pixel - bg), + } + }) + .collect()) +} + +/// Integrates rates over a circular phase span and returns rate × phase. +fn integrate_periodic_rates(model: &PolarityPhaseRates, start_phase: f64, phase_span: f64) -> f64 { + let mut total = 0.0; + let start = start_phase.rem_euclid(1.0); + let end = start + phase_span; + for bin in &model.bins { + for offset in [0.0, 1.0] { + let bin_start = bin.phase_start + offset; + let bin_end = bin.phase_end + offset; + let overlap = (end.min(bin_end) - start.max(bin_start)).max(0.0); + let layer = bin.background.unwrap_or(bin.run); + total += overlap * layer.rate_per_pixel_s; + } + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::phase::{fold_events, MarkerValidationConfig}; + use crate::types::CameraEvent; + + fn fold(events: &[CameraEvent]) -> PhaseFold { + fold_events( + events, + &[0, 1_000, 2_000], + MarkerValidationConfig { + expected_frequency_hz: 1_000.0, + frequency_tolerance_fraction: 0.0, + max_period_jitter_fraction: 0.0, + expected_cycles: Some(2), + }, + ) + .unwrap() + } + + fn event(timestamp_us: u64, polarity: Polarity) -> CameraEvent { + CameraEvent { + timestamp_us, + x: 0, + y: 0, + polarity, + } + } + + #[test] + fn computes_raw_background_and_negative_net_rates_per_polarity() { + let run = fold(&[ + event(100, Polarity::On), + event(1_100, Polarity::On), + event(600, Polarity::Off), + ]); + let background = fold(&[ + event(100, Polarity::On), + event(200, Polarity::On), + event(1_100, Polarity::On), + event(1_200, Polarity::On), + ]); + let rates = phase_bin_rates(&run, Some(&background), 10, 2).unwrap(); + let on_first = &rates.on.bins[0]; + assert_eq!(on_first.run.count, 2); + assert_eq!(on_first.background.unwrap().count, 4); + assert!(on_first.net_rate_per_pixel_s.unwrap() < 0.0); + assert!(on_first.net_standard_error.unwrap() > 0.0); + assert_eq!(rates.off.bins[1].run.count, 1); + } + + #[test] + fn rolling_quicklook_uses_open_left_closed_right_window_and_background_integral() { + let run = fold(&[ + event(500, Polarity::On), + event(750, Polarity::On), + event(1_000, Polarity::On), + ]); + let background = fold(&[ + event(100, Polarity::On), + event(600, Polarity::On), + event(1_100, Polarity::On), + event(1_600, Polarity::On), + ]); + let background_rates = phase_bin_rates(&run, Some(&background), 1, 2).unwrap(); + let points = rolling_half_period_response( + &run, + Polarity::On, + 1, + &[1_000], + Some(&background_rates.on), + ) + .unwrap(); + // Event at exactly t-T/2 is excluded; 750 and 1000 are included. + assert_eq!(points[0].run_per_pixel, 2.0); + assert!((points[0].background_per_pixel.unwrap() - 1.0).abs() < 1e-12); + assert!((points[0].net_per_pixel.unwrap() - 1.0).abs() < 1e-12); + } +} diff --git a/plugins/stage-a-a1/src/response_curve.rs b/plugins/stage-a-a1/src/response_curve.rs new file mode 100644 index 0000000..b4540a2 --- /dev/null +++ b/plugins/stage-a-a1/src/response_curve.rs @@ -0,0 +1,293 @@ +//! Auto-windowed Bernoulli response probability `q_p(a, f)`. +//! +//! With the firmware phase-0 `EXT_TRIGGER` anchoring the camera phase, ON and OFF +//! events fall in opposite half-cycles, so the ON/OFF phase windows can be found +//! directly from the current fold — no separate bright "pilot" capture is needed. +//! +//! For each polarity we anchor a window on its phase-histogram peak and grow it +//! outward while the histogram stays above a floor (a fraction of the peak) **and** +//! that polarity still dominates the opposite one. The window therefore ends out in +//! the opposite half-cycle, where the polarity's events have died away, and can +//! never bleed into the other polarity's cluster. +//! +//! The response probability is then, per pixel `i` and cycle `c`: +//! +//! ```text +//! z_{i,c,p} = 1 if pixel i fires at least once in W_p during cycle c, else 0 +//! q_p(a,f) = (1 / (N_valid · M)) · Σ_i Σ_c z_{i,c,p} +//! ``` +//! +//! computed independently for ON and OFF, where `M` is the number of complete +//! valid cycles and `N_valid` is the ROI minus masked pixels. +//! +//! This is the **live quicklook** definition. The authoritative `q_p(a, f)` fit +//! freezes the windows once (from the brightest recording) and applies them to all +//! amplitudes offline — auto-windowing per fold is deliberately not amplitude-frozen. + +use std::collections::HashSet; + +use crate::phase::PhaseFold; +use crate::types::Polarity; + +/// Phase-histogram resolution used for window detection. +pub const HIST_BINS: usize = 64; + +/// Circular phase window `[start, end)` in cycle fraction. When `start > end` +/// the window wraps past 1.0. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PhaseWindow { + pub start: f64, + pub end: f64, +} + +impl PhaseWindow { + pub fn contains(&self, phase: f64) -> bool { + let p = phase.rem_euclid(1.0); + if self.start <= self.end { + p >= self.start && p < self.end + } else { + p >= self.start || p < self.end + } + } +} + +/// Region of interest in pixel coordinates; `x1`/`y1` are exclusive. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Roi { + pub x0: u16, + pub y0: u16, + pub x1: u16, + pub y1: u16, +} + +impl Roi { + pub fn contains(&self, x: u16, y: u16) -> bool { + x >= self.x0 && x < self.x1 && y >= self.y0 && y < self.y1 + } + + pub fn area(&self) -> usize { + usize::from(self.x1.saturating_sub(self.x0)) * usize::from(self.y1.saturating_sub(self.y0)) + } +} + +/// One recorded response-curve point. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ResponsePoint { + pub measured_a: f64, + pub q_on: f64, + pub q_off: f64, + pub cycles: usize, + pub valid_pixels: usize, +} + +/// ON/OFF phase histogram of a fold. +pub fn phase_histogram(fold: &PhaseFold, polarity: Polarity) -> Vec { + let mut histogram = vec![0.0; HIST_BINS]; + for event in fold + .events + .iter() + .filter(|event| event.polarity == polarity) + { + let phase = event.phase.rem_euclid(1.0); + let bin = ((phase * HIST_BINS as f64) as usize).min(HIST_BINS - 1); + histogram[bin] += 1.0; + } + histogram +} + +/// Grows a circular window out from `hist`'s peak while the peak-relative floor is +/// met and this polarity keeps dominating `other`. Returns `None` on an empty +/// histogram. +fn grow_window(hist: &[f64], other: &[f64], floor_fraction: f64) -> Option { + let bins = hist.len(); + let peak = hist.iter().copied().fold(0.0_f64, f64::max); + if peak <= 0.0 || bins == 0 { + return None; + } + let floor = peak * floor_fraction; + let peak_bin = hist + .iter() + .enumerate() + .max_by(|a, b| a.1.total_cmp(b.1)) + .map(|(index, _)| index)?; + + // A bin belongs to this window when it clears the floor and this polarity is + // at least as strong as the opposite one there. + let keep = |index: usize| hist[index] >= floor && hist[index] >= other[index]; + + // The peak anchors the window; grow right then left until a bin fails. + let mut right = peak_bin; + for step in 1..bins { + let index = (peak_bin + step) % bins; + if keep(index) { + right = index; + } else { + break; + } + } + let mut left = peak_bin; + for step in 1..bins { + let index = (peak_bin + bins - step) % bins; + if keep(index) { + left = index; + } else { + break; + } + } + + Some(PhaseWindow { + start: left as f64 / bins as f64, + end: ((right + 1) % bins) as f64 / bins as f64, + }) +} + +/// Detects the ON and OFF phase windows directly from a fold's histograms. +pub fn auto_windows(fold: &PhaseFold, floor_fraction: f64) -> Option<(PhaseWindow, PhaseWindow)> { + let on = phase_histogram(fold, Polarity::On); + let off = phase_histogram(fold, Polarity::Off); + let window_on = grow_window(&on, &off, floor_fraction)?; + let window_off = grow_window(&off, &on, floor_fraction)?; + Some((window_on, window_off)) +} + +/// Computes the ON/OFF Bernoulli response probabilities for one fold against the +/// given windows. `masked` holds pixels excluded inside the ROI. Returns `None` +/// when there are no complete cycles or no valid pixels. +pub fn response_probability( + fold: &PhaseFold, + window_on: PhaseWindow, + window_off: PhaseWindow, + roi: Roi, + masked: &HashSet<(u16, u16)>, +) -> Option<(f64, f64, usize, usize)> { + let cycles = fold.validation.cycle_count; + if cycles == 0 { + return None; + } + let masked_in_roi = masked.iter().filter(|(x, y)| roi.contains(*x, *y)).count(); + let valid_pixels = roi.area().saturating_sub(masked_in_roi); + if valid_pixels == 0 { + return None; + } + + let mut on_hits: HashSet<(usize, u16, u16)> = HashSet::new(); + let mut off_hits: HashSet<(usize, u16, u16)> = HashSet::new(); + for event in &fold.events { + if !roi.contains(event.x, event.y) || masked.contains(&(event.x, event.y)) { + continue; + } + match event.polarity { + Polarity::On if window_on.contains(event.phase) => { + on_hits.insert((event.cycle_index, event.x, event.y)); + } + Polarity::Off if window_off.contains(event.phase) => { + off_hits.insert((event.cycle_index, event.x, event.y)); + } + _ => {} + } + } + + let denom = valid_pixels as f64 * cycles as f64; + Some(( + on_hits.len() as f64 / denom, + off_hits.len() as f64 / denom, + cycles, + valid_pixels, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::phase::fold_events_free_running; + use crate::types::CameraEvent; + + fn event(timestamp_us: u64, x: u16, y: u16, polarity: Polarity) -> CameraEvent { + CameraEvent { + timestamp_us, + x, + y, + polarity, + } + } + + /// Builds a fold: every pixel in an N-wide ROI fires ON near phase 0.2 and + /// OFF near phase 0.7, for `cycles` cycles at period 1000 us. + fn respond_fold(cycles: u64, pixels: u16) -> PhaseFold { + let mut events = Vec::new(); + for cycle in 0..cycles { + let base = cycle * 1_000; + for x in 0..pixels { + events.push(event(base + 200, x, 0, Polarity::On)); + events.push(event(base + 700, x, 0, Polarity::Off)); + } + } + // One trailing event so the free-running fold spans `cycles` whole cycles. + events.push(event(cycles * 1_000 + 10, 0, 0, Polarity::On)); + fold_events_free_running(&events, 1_000.0).expect("whole cycles") + } + + fn windows_disjoint(on: &PhaseWindow, off: &PhaseWindow) -> bool { + (0..HIST_BINS).all(|bin| { + let phase = (bin as f64 + 0.5) / HIST_BINS as f64; + !(on.contains(phase) && off.contains(phase)) + }) + } + + #[test] + fn auto_windows_are_separated_and_classify_a_full_response() { + let fold = respond_fold(20, 4); + let (on, off) = auto_windows(&fold, 0.1).expect("windows"); + assert!(windows_disjoint(&on, &off)); + assert_ne!(on, off); + // Free-running fold anchors phase 0 to the first event (an ON), so the ON + // cluster sits at phase 0.0 and the OFF cluster half a cycle later at 0.5. + assert!(on.contains(0.0) && off.contains(0.5)); + assert!(!on.contains(0.5) && !off.contains(0.0)); + + let roi = Roi { + x0: 0, + y0: 0, + x1: 4, + y1: 1, + }; + let (q_on, q_off, _, valid) = + response_probability(&fold, on, off, roi, &HashSet::new()).expect("counts"); + assert_eq!(valid, 4); + assert!(q_on > 0.98 && q_off > 0.98, "q_on={q_on} q_off={q_off}"); + } + + #[test] + fn partial_pixel_response_gives_proportional_probability() { + let roi = Roi { + x0: 0, + y0: 0, + x1: 4, + y1: 1, + }; + let (on, off) = auto_windows(&respond_fold(20, 4), 0.1).expect("windows"); + + // Only 2 of 4 ROI pixels respond every cycle -> q_on ~ 0.5. + let weak = respond_fold(20, 2); + let (q_on_weak, _, _, valid) = + response_probability(&weak, on, off, roi, &HashSet::new()).expect("counts"); + assert_eq!(valid, 4); + assert!((q_on_weak - 0.5).abs() < 0.05, "q_on_weak={q_on_weak}"); + } + + #[test] + fn masked_pixels_are_subtracted_from_valid_count() { + let roi = Roi { + x0: 0, + y0: 0, + x1: 4, + y1: 1, + }; + let mut masked = HashSet::new(); + masked.insert((3_u16, 0_u16)); + let fold = respond_fold(10, 4); + let (on, off) = auto_windows(&fold, 0.1).expect("windows"); + let (_, _, _, valid) = response_probability(&fold, on, off, roi, &masked).expect("counts"); + assert_eq!(valid, 3); + } +} diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs new file mode 100644 index 0000000..b1baedd --- /dev/null +++ b/plugins/stage-a-a1/src/runtime.rs @@ -0,0 +1,3460 @@ +//! Live A1 recording coordinator. +//! +//! A1 has two jobs on the Stage-A bench, both deliberately thin: +//! +//! 1. **Recording coordinator.** One *Start recording* button records, for a fixed +//! duration, the camera **RAW** stream (host recording) and the photodiode **PDQ** +//! stream (leased `stage-a.photodiode` service) together, grouped under a +//! per-`(I_k, f)` measurement **id** and a shared `_` file stem, and +//! writes an A1 **config sidecar** (`.toml`) linking the two files with the +//! modulation settings, the photodiode-measured modulation depth `a`, the ROI, and +//! the trigger info needed to reproduce and analyse the run offline. A1 owns no +//! hardware and, outside the leased sweep below, never drives the Teensy — the +//! optical drive is armed in the modulation plugin; A1 only *reads* its published +//! settings into the sidecar. The **amplitude sweep** (ADR 010) is the one scoped +//! exception: per sweep point it retargets the armed drive's *depth* through the +//! leased modulation service (`SetOpticalDepth`), waits for the photodiode-measured +//! `a` to settle, and records the point through the same coordinator. +//! +//! 2. **Live sanity quicklooks.** Folding the camera event stream on the modulation +//! period `T` (defined by the firmware phase-0 `EXT_TRIGGER`), it renders the +//! **rolling half-period response** `S_p(t)` (a live "are events appearing, is the +//! ON/OFF timing sane?" indicator) and the **response probability** `q_p` curve +//! (frozen-window Bernoulli statistic vs the measured `a`). The authoritative +//! `q_p(a, f)` fit is computed offline from the recordings; the live plot is a +//! quicklook. + +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, FfiCdEvent, GlobalSettings, HostCommand, HostCommandOutcome, + HostCommandReply, HostCommandRequest, HostContext, HostDatasetDescriptor, HostDatasetKind, + HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, + PathDialogKind, Plugin, PluginCapabilities, PluginControlContext, PluginControlInbox, + PluginDiscontinuity, PluginFrame, PluginInput, PluginRuntimeRole, PluginServiceOutcome, + PluginServiceReply, PluginServiceRequest, RoiV1, Series1dLine, Series1dPoint, Series1dV1, + SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, + TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, TableValueType, + CTX_GLOBAL_SETTINGS, +}; +use serde::Serialize; +use serde_json::{json, Value}; +use stage_a_plugin_contract::{ + ClientId, ConnectionStateV1, LeaseId, ModulationCommandV1, ModulationRequestV1, + ModulationStateV1, PdqReceiptV1, PdqStartSpecV1, PhotodiodeCommandV1, PhotodiodeRequestV1, + PhotodiodeResponseV1, PhotodiodeSummaryV1, RequestId, RunId, SemanticRevision, WaveformV1, + CTX_STAGE_A_MODULATION_STATE_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, + SERVICE_STAGE_A_MODULATION_CONTROL_V1, SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, +}; + +use crate::phase::{fold_events, fold_events_free_running, MarkerValidationConfig, PhaseFold}; +use crate::rates::{rolling_half_period_response, RollingResponsePoint}; +use crate::response_curve::{auto_windows, response_probability, PhaseWindow, ResponsePoint, Roi}; +use crate::types::{CameraEvent, Polarity}; + +const MODULATION_PLUGIN_ID: &str = "stage-a.modulation"; +const PHOTODIODE_PLUGIN_ID: &str = "stage-a.photodiode"; +const A1_PLUGIN_ID: &str = "stage-a.a1"; + +const STATUS_DATASET_ID: &str = "stage-a-a1.status"; +const STATUS_VIEW_ID: &str = "stage-a-a1.status.view"; +const ROLLING_DATASET_ID: &str = "stage-a-a1.rolling-response"; +const ROLLING_VIEW_ID: &str = "stage-a-a1.rolling-response.view"; +const RESPONSE_CURVE_DATASET_ID: &str = "stage-a-a1.response-curve"; +const RESPONSE_CURVE_VIEW_ID: &str = "stage-a-a1.response-curve.view"; + +/// Camera events retained for the live fold. At the bench event rates this is a +/// few seconds of history and keeps the fold cost bounded. +const MAX_EVENTS: usize = 4_000_000; +/// Sample points on the rolling half-period trace. +const ROLLING_SAMPLES: u64 = 256; +/// Default `q_p` window floor: grow each ON/OFF window until it falls to this +/// fraction of its histogram peak (or the opposite polarity takes over). +const DEFAULT_WINDOW_FLOOR: f64 = 0.10; +/// Default analysis window (ms) pulled from the retained EventStore each frame. +const DEFAULT_ANALYSIS_WINDOW_MS: i64 = 2_000; +/// Give up waiting for a control-plane reply after this many milliseconds. +const REPLY_TIMEOUT_MS: u64 = 15_000; +/// Upper bound on retained phase-0 markers in the no-EventStore fallback path. +const MAX_MARKERS: usize = 65_536; +/// Give up waiting for the photodiode-measured `a` to reach a sweep target +/// after this long and record anyway (the sidecar stores the measured value). +const SWEEP_SETTLE_TIMEOUT_MS: u64 = 30_000; + +/// Absolute/relative tolerance for "the measured `a` reached the sweep target". +fn sweep_tolerance(target_a: f64) -> f64 { + (target_a * 0.10).max(0.05) +} + +trait RecordingControl { + fn request_service(&mut self, request: &PluginServiceRequest); + fn request_host(&mut self, request: &HostCommandRequest); +} + +impl RecordingControl for PluginControlContext<'_> { + fn request_service(&mut self, request: &PluginServiceRequest) { + let _ = PluginControlContext::request_service(self, request); + } + + fn request_host(&mut self, request: &HostCommandRequest) { + let _ = PluginControlContext::request_host(self, request); + } +} + +/// Forwards momentary button presses across the host's UI-mirror → live-worker +/// settings snapshot. A click arrives as `true` on the clicked instance; the +/// other instance only ever sees the snapshot value from `get_setting`, so the +/// press is transported as a monotonic counter and a counter advance counts as +/// one press edge. The first counter a fresh instance sees is adopted silently +/// so a reloaded worker does not replay old presses. +#[derive(Debug, Default, Clone, Copy)] +struct PressLatch { + counter: u64, + seen: Option, +} + +impl PressLatch { + /// Interprets a settings write to this button; returns true on a press edge. + fn accept(&mut self, value: &Value) -> bool { + if value.as_bool() == Some(true) { + self.counter += 1; + self.seen = Some(self.counter); + return true; + } + let Some(incoming) = value.as_u64() else { + return false; + }; + match self.seen { + None => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + false + } + Some(seen) if incoming > seen => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + true + } + Some(_) => false, + } + } + + fn value(&self) -> Value { + json!(self.counter) + } +} + +/// Where the coordinated recording is in its lifecycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RecPhase { + Idle, + /// Camera start sent; waiting until the host has switched into recording. + StartingCamera, + /// Camera is running; reconnecting the photodiode after the pipeline switch. + ConnectingPhotodiode, + /// AcquireLease sent to the photodiode; waiting for the grant. + AcquiringLease, + /// Camera is running; waiting for the photodiode PDQ start receipt. + StartingPhotodiode, + /// Camera RAW + photodiode PDQ recording are both in flight. + Running, + /// Photodiode finalize sent; camera keeps recording until PDQ is closed. + StoppingPhotodiode, + /// PDQ is closed; waiting for the host camera finalize receipt. + StoppingCamera, +} + +/// What a recording is for within one `(I_k, f)` measurement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RecRole { + /// One amplitude point of the sweep. + Normal, + /// Bright reference that freezes the ON/OFF windows for the whole row. + Pilot, + /// Unmodulated (`a≈0`) reference that gives the false-response floor. + Background, +} + +impl RecRole { + /// Filename-stem suffix, empty for a normal sweep point. + fn suffix(self) -> &'static str { + match self { + RecRole::Normal => "", + RecRole::Pilot => "_pilot", + RecRole::Background => "_background", + } + } + + fn label(self) -> &'static str { + match self { + RecRole::Normal => "point", + RecRole::Pilot => "pilot", + RecRole::Background => "background", + } + } +} + +/// One coordinated `(camera RAW + photodiode PDQ + sidecar)` recording. +struct Recording { + phase: RecPhase, + role: RecRole, + id: String, + stem: String, + folder: String, + duration_s: u64, + start_unix_ms: u64, + last_activity_ms: u64, + lease_id: LeaseId, + stop_requested: bool, + // outstanding request-id correlation + connect_req: u64, + lease_req: u64, + cam_start_req: u64, + cam_stop_req: u64, + pd_begin_req: u64, + pd_finalize_req: u64, + // captured receipts + connect_accepted: bool, + lease_granted: bool, + cam_raw_path: Option, + cam_finalized_path: Option, + /// True only for a complete host finalization receipt, not a partial file. + cam_complete: bool, + /// The host rejected StartRecording — skip the stop and don't wait for a + /// finalize receipt. + cam_rejected: bool, + pd_pdq_path: Option, + pd_sidecar_path: Option, + pd_finalized: bool, + pd_valid: bool, + /// The photodiode rejected BeginRecording — skip the finalize and don't + /// wait for its receipt. + pd_rejected: bool, +} + +/// Where the amplitude sweep is within its per-point cycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SweepPhase { + /// AcquireLease sent to the modulation owner; waiting for the grant. + AcquiringLease, + /// SetOpticalDepth for the current point sent; waiting for Applied. + SettingDepth, + /// Waiting for the photodiode-measured `a` to settle at the target. + Settling, + /// The per-point recording coordinator owns this phase. + Recording, +} + +/// One "record every point of the amplitude range" run: per point the sweep +/// retargets the leased modulation drive, waits for the photodiode-measured +/// `a` to settle, and hands off to the normal recording coordinator. +struct Sweep { + phase: SweepPhase, + /// Requested `a` per point, ascending over `[min_a, max_a]`. + points: Vec, + index: usize, + lease_id: LeaseId, + lease_granted: bool, + lease_req: u64, + depth_req: u64, + depth_applied: bool, + /// Instant the measured `a` first satisfied the tolerance, for the dwell. + settled_since_ms: Option, + /// Give-up deadline for the settle phase. + settle_deadline_ms: u64, + /// Whether the current point's recording actually started (vs. was + /// refused by validation before it began). + point_started: bool, + last_activity_ms: u64, + stop_requested: bool, +} + +impl Sweep { + fn target_a(&self) -> f64 { + self.points.get(self.index).copied().unwrap_or(0.0) + } + + fn total(&self) -> usize { + self.points.len() + } +} + +pub struct StageAA1Plugin { + enabled: bool, + runtime_role: PluginRuntimeRole, + /// While true, camera events are folded into the live quicklooks. This does + /// not record anything — recording is the separate coordinator below. + live: bool, + modulation: Option, + photodiode: Option, + camera_events: Vec, + /// Reusable buffer for exact events pulled from the retained EventStore. + event_scratch: Vec, + /// Sliding analysis window (ms) for the live fold. + analysis_window_ms: i64, + /// Rising `EXT_TRIGGER` timestamps (firmware phase-0 sync). When present these + /// anchor the fold to the drive on the camera clock; empty falls back to the + /// free-running fold on `T`. + camera_markers_us: Vec, + valid_pixels: usize, + frame_width: u16, + frame_height: u16, + // -- host camera ROI/mask, mirrored from CTX_GLOBAL_SETTINGS -- + host_roi: Option, + masked_pixels: HashSet<(u16, u16)>, + // -- response curve (auto-windowed Bernoulli q_p) -- + /// Window floor as a fraction of the ON/OFF histogram peak (see `auto_windows`). + window_floor: f64, + response_points: Vec, + /// ON/OFF windows frozen from the pilot for the current measurement row. When + /// set they override the per-fold auto-windows so the row's `q_p` is + /// consistent; loaded from the pilot's sidecar in the measurement folder. + pilot_windows: Option<(PhaseWindow, PhaseWindow)>, + /// Background floor `(q0_on, q0_off)` from the `a≈0` reference. + background_floor: Option<(f64, f64)>, + // -- recording coordinator -- + output_folder: String, + measurement_id: String, + /// Sweep range `[min_a, max_a]` for this `(I_k, f)` row (automation template). + min_a: f64, + max_a: f64, + duration_s: i64, + recording: Recording, + /// Whether the most recent recording reached its finalize path (vs. being + /// aborted); the sweep uses this to decide between advancing and stopping. + recording_completed_ok: bool, + request_seq: u64, + pd_revision_seq: u64, + /// Role latched by the Start/Pilot/Background buttons, consumed next tick. + pending_role: Option, + /// `(folder, id)` last scanned for pilot/background sidecars, so the folder is + /// re-read only when the measurement changes. + loaded_key: Option<(String, String)>, + /// One-line operator feedback about the most recent recording action. + message: String, + dataset_generation: u64, + // -- amplitude sweep -- + /// Number of sweep points across `[min_a, max_a]`. + sweep_count: i64, + /// Dwell the measured `a` must hold the target tolerance before recording. + settle_s: f64, + /// Latched by the Start sweep button, consumed next control tick. + sweep_pending: bool, + sweep: Option, + // -- momentary-button press forwarding (see PressLatch) -- + press_start: PressLatch, + press_pilot: PressLatch, + press_background: PressLatch, + press_stop: PressLatch, + press_sweep: PressLatch, + press_clear: PressLatch, + press_record_point: PressLatch, + press_clear_curve: PressLatch, +} + +impl Default for StageAA1Plugin { + fn default() -> Self { + Self { + enabled: false, + runtime_role: PluginRuntimeRole::UiMirror, + live: false, + modulation: None, + photodiode: None, + camera_events: Vec::new(), + event_scratch: Vec::new(), + analysis_window_ms: DEFAULT_ANALYSIS_WINDOW_MS, + camera_markers_us: Vec::new(), + valid_pixels: 0, + frame_width: 0, + frame_height: 0, + host_roi: None, + masked_pixels: HashSet::new(), + window_floor: DEFAULT_WINDOW_FLOOR, + response_points: Vec::new(), + pilot_windows: None, + background_floor: None, + output_folder: String::new(), + measurement_id: generate_measurement_id(), + min_a: 0.0, + max_a: 2.0, + duration_s: 10, + recording: Recording::idle(), + recording_completed_ok: false, + request_seq: 0, + pd_revision_seq: 0, + pending_role: None, + loaded_key: None, + message: String::new(), + dataset_generation: 1, + sweep_count: 5, + settle_s: 2.0, + sweep_pending: false, + sweep: None, + press_start: PressLatch::default(), + press_pilot: PressLatch::default(), + press_background: PressLatch::default(), + press_stop: PressLatch::default(), + press_sweep: PressLatch::default(), + press_clear: PressLatch::default(), + press_record_point: PressLatch::default(), + press_clear_curve: PressLatch::default(), + } + } +} + +impl Recording { + fn idle() -> Self { + Self { + phase: RecPhase::Idle, + role: RecRole::Normal, + id: String::new(), + stem: String::new(), + folder: String::new(), + duration_s: 0, + start_unix_ms: 0, + last_activity_ms: 0, + lease_id: LeaseId::new(String::new()), + stop_requested: false, + connect_req: 0, + lease_req: 0, + cam_start_req: 0, + cam_stop_req: 0, + pd_begin_req: 0, + pd_finalize_req: 0, + connect_accepted: false, + lease_granted: false, + cam_raw_path: None, + cam_finalized_path: None, + cam_complete: false, + cam_rejected: false, + pd_pdq_path: None, + pd_sidecar_path: None, + pd_finalized: false, + pd_valid: false, + pd_rejected: false, + } + } + + fn is_active(&self) -> bool { + self.phase != RecPhase::Idle + } + + fn state_label(&self) -> &'static str { + match self.phase { + RecPhase::Idle => "idle", + RecPhase::StartingCamera => "starting camera", + RecPhase::ConnectingPhotodiode => "connecting photodiode", + RecPhase::AcquiringLease => "acquiring lease", + RecPhase::StartingPhotodiode => "starting photodiode", + RecPhase::Running => "recording", + RecPhase::StoppingPhotodiode => "finalizing photodiode", + RecPhase::StoppingCamera => "finalizing camera", + } + } + + /// Seconds remaining in the fixed-duration window, when running. + fn remaining_s(&self, now_ms: u64) -> Option { + if self.phase != RecPhase::Running { + return None; + } + let elapsed_ms = now_ms.saturating_sub(self.start_unix_ms); + let total_ms = self.duration_s.saturating_mul(1_000); + Some(total_ms.saturating_sub(elapsed_ms) / 1_000) + } +} + +impl StageAA1Plugin { + fn bump(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + /// Sets the concise operator-facing recording result. + fn note(&mut self, message: impl Into) { + self.message = message.into(); + self.bump(); + } + + /// The modulation period `T` in microseconds: measured from the phase-0 + /// markers when present (the trigger *defines* the frequency, latency- + /// invariant), otherwise the modulation plugin's acknowledged waveform. + fn period_us(&self) -> Option { + if let Some(period) = self.measured_period_us() { + return Some(period); + } + let hz = self.acknowledged_frequency_hz()?; + (hz > 0.0).then(|| 1_000_000.0 / hz) + } + + /// Modulation period measured from the phase-0 markers (mean spacing). + fn measured_period_us(&self) -> Option { + if self.camera_markers_us.len() < 2 { + return None; + } + let first = *self.camera_markers_us.first()?; + let last = *self.camera_markers_us.last()?; + let spans = (self.camera_markers_us.len() - 1) as f64; + let period = last.saturating_sub(first) as f64 / spans; + (period > 0.0).then_some(period) + } + + /// Frequency (Hz) from the modulation plugin's acknowledged periodic waveform. + fn acknowledged_frequency_hz(&self) -> Option { + let target = self.modulation.as_ref()?.acknowledged.as_ref()?; + match target.waveform.as_ref()? { + WaveformV1::Periodic { + frequency_millihz, .. + } => (*frequency_millihz > 0).then(|| *frequency_millihz as f64 / 1_000.0), + _ => None, + } + } + + fn is_marker_anchored(&self) -> bool { + self.camera_markers_us.len() >= 2 + } + + fn frequency_source(&self) -> &'static str { + if self.measured_period_us().is_some() { + "trigger" + } else { + "modulation" + } + } + + fn current_fold(&self) -> Option { + let period_us = self.period_us()?; + let marker_fold = self.is_marker_anchored().then(|| { + let expected_hz = 1_000_000.0 / period_us; + fold_events( + &self.camera_events, + &self.camera_markers_us, + MarkerValidationConfig { + expected_frequency_hz: expected_hz, + // Live quicklook: accept real-world drift/jitter rather than + // rejecting the whole fold. + frequency_tolerance_fraction: 0.5, + max_period_jitter_fraction: 0.75, + expected_cycles: None, + }, + ) + .ok() + }); + // A marker glitch (dropped trigger, out-of-tolerance jitter) must not + // blank the live plots — fall back to the free-running fold on T. + marker_fold + .flatten() + .or_else(|| fold_events_free_running(&self.camera_events, period_us)) + } + + /// Optical modulation depth `a` published by the photodiode plugin. + fn measured_a(&self) -> Option { + self.photodiode + .as_ref()? + .optical_summary + .as_ref() + .map(|summary| summary.measured_log_contrast) + } + + /// Current ROI from the host camera config, clamped to the frame. + fn roi(&self) -> Option { + if self.frame_width == 0 || self.frame_height == 0 { + return None; + } + let host = self.host_roi.unwrap_or_default(); + let x0 = host.x.min(self.frame_width); + let y0 = host.y.min(self.frame_height); + let x1 = if host.width == 0 { + self.frame_width + } else { + host.x.saturating_add(host.width).min(self.frame_width) + }; + let y1 = if host.height == 0 { + self.frame_height + } else { + host.y.saturating_add(host.height).min(self.frame_height) + }; + (x1 > x0 && y1 > y0).then_some(Roi { x0, y0, x1, y1 }) + } + + /// Number of valid pixels: ROI area minus masked pixels inside it. + fn valid_pixel_count(&self) -> Option { + let roi = self.roi()?; + let masked = self + .masked_pixels + .iter() + .filter(|(x, y)| roi.contains(*x, *y)) + .count(); + Some(roi.area().saturating_sub(masked)) + } + + /// ON/OFF phase windows for `q_p`: the pilot-frozen windows when a pilot has + /// been recorded for this row, otherwise the per-fold auto-windows. + fn current_windows(&self) -> Option<(PhaseWindow, PhaseWindow)> { + if let Some(windows) = self.pilot_windows { + return Some(windows); + } + auto_windows(&self.current_fold()?, self.window_floor) + } + + /// Whether the `q_p` windows are frozen from a pilot (vs live auto-windows). + fn windows_are_frozen(&self) -> bool { + self.pilot_windows.is_some() + } + + /// ON/OFF response probability for the current fold against `current_windows`. + fn current_response(&self) -> Option<(f64, f64, usize, usize)> { + let fold = self.current_fold()?; + let roi = self.roi()?; + let (window_on, window_off) = self.current_windows()?; + response_probability(&fold, window_on, window_off, roi, &self.masked_pixels) + } + + /// Freezes the ON/OFF windows for this row from the current fold (a pilot). + fn freeze_pilot_windows(&mut self) { + match self + .current_fold() + .and_then(|fold| auto_windows(&fold, self.window_floor)) + { + Some(windows) => { + self.pilot_windows = Some(windows); + self.note("Pilot windows frozen from the live signal"); + } + None => { + self.note("No live signal to freeze windows — enable Live analysis first"); + } + } + } + + /// Captures the background floor `(q0_on, q0_off)` from the current fold. + fn capture_background_floor(&mut self) { + match self.current_response() { + Some((q_on, q_off, _, _)) => { + self.background_floor = Some((q_on, q_off)); + self.note(format!("Background floor captured (q0_on={q_on:.3})")); + } + None => { + self.note("No valid background window yet (need events and a valid ROI)"); + } + } + } + + /// Records one response-curve point at the current photodiode-measured `a`. + fn record_response_point(&mut self) -> Result<(), String> { + let measured_a = self + .measured_a() + .ok_or("no photodiode-measured a available (connect the photodiode)")?; + let (q_on, q_off, cycles, valid_pixels) = self + .current_response() + .ok_or("no valid response window yet (need trigger-anchored events and a valid ROI)")?; + self.response_points.push(ResponsePoint { + measured_a, + q_on, + q_off, + cycles, + valid_pixels, + }); + Ok(()) + } + + fn response_curve_dataset(&self) -> Series1dV1 { + let line = |select: fn(&ResponsePoint) -> f64| { + let mut points: Vec = self + .response_points + .iter() + .map(|point| Series1dPoint { + x: point.measured_a, + y: select(point), + }) + .collect(); + points.sort_by(|a, b| a.x.total_cmp(&b.x)); + points + }; + Series1dV1 { + x_label: "Measured modulation depth a = ln(I_max / I_min)".into(), + y_label: "Response probability q_p = fraction of pixel-cycles that fired".into(), + lines: vec![ + Series1dLine { + name: "ON".into(), + points: line(|point| point.q_on), + }, + Series1dLine { + name: "OFF".into(), + points: line(|point| point.q_off), + }, + ], + } + } + + fn rolling_dataset(&self) -> Series1dV1 { + const X: &str = "Camera time since first event (s)"; + const Y: &str = "Events per valid pixel in the trailing half-cycle T/2"; + let empty = || Series1dV1 { + x_label: X.into(), + y_label: Y.into(), + lines: vec![ + Series1dLine { + name: "ON".into(), + points: Vec::new(), + }, + Series1dLine { + name: "OFF".into(), + points: Vec::new(), + }, + ], + }; + let Some(fold) = self.current_fold() else { + return empty(); + }; + let first = fold.validation.first_marker_us; + let last = fold.validation.last_marker_us; + let samples = ROLLING_SAMPLES.min(last.saturating_sub(first).saturating_add(1)); + if samples < 2 { + return empty(); + } + let sample_times: Vec = (0..samples) + .map(|index| first + (last - first) * index / (samples - 1)) + .collect(); + let line = |polarity: Polarity| { + rolling_half_period_response(&fold, polarity, self.valid_pixels, &sample_times, None) + .map(|points| points_for(&points, first)) + .unwrap_or_default() + }; + Series1dV1 { + x_label: X.into(), + y_label: Y.into(), + lines: vec![ + Series1dLine { + name: "ON".into(), + points: line(Polarity::On), + }, + Series1dLine { + name: "OFF".into(), + points: line(Polarity::Off), + }, + ], + } + } + + /// Latest rolling half-period value per polarity, for the status readout. + fn latest_rolling(&self) -> Option<(f64, f64)> { + let fold = self.current_fold()?; + let at = [fold.validation.last_marker_us]; + let value = |polarity| { + rolling_half_period_response(&fold, polarity, self.valid_pixels, &at, None) + .ok() + .and_then(|points| points.first().map(|point| point.run_per_pixel)) + }; + Some((value(Polarity::On)?, value(Polarity::Off)?)) + } + + fn status_dataset(&self) -> TableDatasetV1 { + let now_ms = now_unix_ms(); + let period_us = self.period_us(); + let frequency = period_us.map(|t| 1_000_000.0 / t); + let source = self.frequency_source(); + let (on_now, off_now) = self + .latest_rolling() + .map_or((None, None), |(on, off)| (Some(on), Some(off))); + let cell = |id: &str, value: String| TableColumnData { + column_id: id.into(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + cell("state", self.recording.state_label().into()), + cell( + "measurement_id", + if self.recording.is_active() { + self.recording.id.clone() + } else { + self.measurement_id.clone() + }, + ), + cell( + "remaining", + self.recording + .remaining_s(now_ms) + .map_or_else(|| "—".into(), |s| format!("{s} s")), + ), + cell( + "frequency", + frequency.map_or_else(|| "—".into(), |hz| format!("{hz:.3} Hz ({source})")), + ), + cell( + "a", + self.measured_a() + .map_or_else(|| "—".into(), |a| format!("{a:.3}")), + ), + cell( + "s_on", + on_now.map_or_else(|| "—".into(), |v| format!("{v:.4}")), + ), + cell( + "s_off", + off_now.map_or_else(|| "—".into(), |v| format!("{v:.4}")), + ), + cell("events", self.camera_events.len().to_string()), + cell( + "message", + if self.message.is_empty() { + "—".into() + } else { + self.message.clone() + }, + ), + ], + } + } + + fn update_snapshots(&mut self, inbox: &PluginControlInbox) { + for snapshot in &inbox.snapshots { + match (snapshot.plugin_id.as_str(), snapshot.topic.as_str()) { + (MODULATION_PLUGIN_ID, CTX_STAGE_A_MODULATION_STATE_V1) => { + if let Ok(state) = serde_json::from_value(snapshot.payload.clone()) { + self.modulation = Some(state); + } + } + (PHOTODIODE_PLUGIN_ID, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1) => { + if let Ok(summary) = serde_json::from_value(snapshot.payload.clone()) { + self.photodiode = Some(summary); + } + } + _ => {} + } + } + } + + fn next_request_id(&mut self) -> u64 { + self.request_seq = self.request_seq.wrapping_add(1); + self.request_seq + } + + /// Wraps a photodiode command in the routed service request A1 emits. + fn photodiode_request(&mut self, command: PhotodiodeCommandV1) -> PluginServiceRequest { + let request_id = self.next_request_id(); + let needs_revision = matches!( + command, + PhotodiodeCommandV1::BeginRecording { .. } + | PhotodiodeCommandV1::FinalizeRecording { .. } + | PhotodiodeCommandV1::AbortRecording { .. } + ); + let mut envelope = + PhotodiodeRequestV1::new(RequestId(request_id), ClientId::new(A1_PLUGIN_ID), command); + envelope.lease_id = Some(self.recording.lease_id.clone()); + if !self.recording.stem.is_empty() { + envelope.run_id = Some(RunId::new(self.recording.stem.clone())); + } + if needs_revision { + let observed = self + .photodiode + .as_ref() + .map(|summary| { + summary + .requested_revision + .into_iter() + .chain(summary.acknowledged_revision) + .map(|revision| revision.0) + .max() + .unwrap_or(0) + }) + .unwrap_or(0); + self.pd_revision_seq = self + .pd_revision_seq + .saturating_add(1) + .max(observed.saturating_add(1)); + envelope.requested_revision = Some(SemanticRevision(self.pd_revision_seq)); + } + envelope.target_owner_instance = self + .photodiode + .as_ref() + .map(|summary| summary.owner_instance.clone()); + envelope.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + payload: serde_json::to_value(&envelope).unwrap_or(Value::Null), + } + } + + /// String metadata embedded in both recorders' own sidecars. + fn recording_metadata(&self) -> BTreeMap { + let mut meta = BTreeMap::new(); + meta.insert("a1_measurement_id".into(), self.recording.id.clone()); + meta.insert("a1_stem".into(), self.recording.stem.clone()); + meta.insert("a1_role".into(), self.recording.role.label().into()); + meta.insert( + "a1_duration_s".into(), + self.recording.duration_s.to_string(), + ); + meta.insert("sweep_min_a".into(), format!("{:.6}", self.min_a)); + meta.insert("sweep_max_a".into(), format!("{:.6}", self.max_a)); + if let Some(sweep) = self + .sweep + .as_ref() + .filter(|sweep| sweep.phase == SweepPhase::Recording) + { + meta.insert( + "sweep_requested_a".into(), + format!("{:.6}", sweep.target_a()), + ); + meta.insert("sweep_point_index".into(), (sweep.index + 1).to_string()); + meta.insert("sweep_point_total".into(), sweep.total().to_string()); + } + if let Some(a) = self.measured_a() { + meta.insert("measured_a".into(), format!("{a:.6}")); + } + if let Some(hz) = self.period_us().map(|t| 1_000_000.0 / t) { + meta.insert("modulation_frequency_hz".into(), format!("{hz:.6}")); + } + if let Some(config) = self + .modulation + .as_ref() + .and_then(|s| s.acknowledged.as_ref()) + .and_then(|t| t.a1_configuration.as_ref()) + { + meta.insert("center_dac".into(), config.center_dac.to_string()); + meta.insert("amplitude_dac".into(), config.amplitude_dac.to_string()); + } + if let Some(n) = self.valid_pixel_count() { + meta.insert("n_valid".into(), n.to_string()); + } + meta + } + + /// Kick off a coordinated recording by starting the camera first. Called + /// on the control tick after a record button is pressed. + fn begin_recording(&mut self, context: &mut impl RecordingControl, role: RecRole) { + if self.recording.is_active() { + return; + } + if self.output_folder.trim().is_empty() { + self.note("Set an output folder before recording"); + return; + } + if self.measurement_id.trim().is_empty() { + self.note("Set a measurement id before recording"); + return; + } + let now_ms = now_unix_ms(); + let id = sanitize_stem(self.measurement_id.trim()); + // Sweep points get a stable per-point tag so the row's files sort by + // sweep order as well as by timestamp. + let sweep_tag = self + .sweep + .as_ref() + .filter(|sweep| sweep.phase == SweepPhase::Recording) + .map(|sweep| format!("_p{:02}", sweep.index + 1)) + .unwrap_or_default(); + let stem = format!( + "{id}_{}{}{sweep_tag}", + format_compact_utc(now_ms / 1_000), + role.suffix() + ); + let lease_id = LeaseId::new(format!("a1-{stem}")); + self.recording_completed_ok = false; + let mut recording = Recording::idle(); + recording.role = role; + recording.id = id; + recording.stem = stem; + recording.folder = self.output_folder.trim().to_string(); + recording.duration_s = self.duration_s.max(1) as u64; + // The measurement clock starts only after both recorders acknowledge + // that they are running. + recording.start_unix_ms = 0; + recording.last_activity_ms = now_ms; + recording.lease_id = lease_id; + self.recording = recording; + + // Capture the science reference now (after any folder scan this tick), + // from the live signal at the current drive amplitude. + match role { + RecRole::Pilot => self.freeze_pilot_windows(), + RecRole::Background => self.capture_background_floor(), + RecRole::Normal => {} + } + + self.start_camera(context); + } + + /// Re-reads pilot/background sidecars from the measurement folder when the + /// measurement (folder + id) changes, so the `q_p` plot reuses them. + fn scan_measurement_folder(&mut self) { + let folder = self.output_folder.trim().to_string(); + let id = sanitize_stem(self.measurement_id.trim()); + let key = (folder.clone(), id.clone()); + if self.loaded_key.as_ref() == Some(&key) { + return; + } + self.loaded_key = Some(key); + self.pilot_windows = None; + self.background_floor = None; + if folder.is_empty() || id.is_empty() { + return; + } + let measurement_dir = Path::new(&folder).join(&id); + let measurement_dir = measurement_dir.to_string_lossy(); + if let Some((on, off)) = load_row_windows(&measurement_dir, &id, "_pilot") { + self.pilot_windows = Some((on, off)); + } + self.background_floor = load_row_background(&measurement_dir, &id, "_background"); + } + + /// Start the host camera recorder first. Starting it switches the host from + /// preview into recording and briefly revokes plugin effects, so the PDQ + /// stream must not be opened until the host acknowledges this transition. + fn start_camera(&mut self, context: &mut impl RecordingControl) { + let subdir = self.recording.id.clone(); + let stem = self.recording.stem.clone(); + let metadata = self.recording_metadata(); + + let cam_req = self.next_request_id(); + context.request_host(&HostCommandRequest { + request_id: cam_req, + command: HostCommand::StartRecording { + run_id: stem.clone(), + base_path: format!("{subdir}/{stem}.raw"), + metadata, + }, + }); + self.recording.cam_start_req = cam_req; + self.recording.phase = RecPhase::StartingCamera; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!("Recording {}: starting camera…", self.recording.id)); + } + + fn connect_photodiode(&mut self, context: &mut impl RecordingControl) { + let request = self.photodiode_request(PhotodiodeCommandV1::Connect); + self.recording.connect_req = request.request_id; + context.request_service(&request); + self.recording.phase = RecPhase::ConnectingPhotodiode; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: connecting photodiode…", + self.recording.id + )); + } + + fn acquire_photodiode(&mut self, context: &mut impl RecordingControl) { + let ttl_ms = self + .recording + .duration_s + .saturating_mul(1_000) + .saturating_add(60_000); + let request = self.photodiode_request(PhotodiodeCommandV1::AcquireLease { ttl_ms }); + self.recording.lease_req = request.request_id; + context.request_service(&request); + self.recording.phase = RecPhase::AcquiringLease; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: preparing photodiode…", + self.recording.id + )); + } + + /// Start the PDQ only after the camera recorder is running. + fn start_photodiode(&mut self, context: &mut impl RecordingControl) { + let subdir = self.recording.id.clone(); + let stem = self.recording.stem.clone(); + let spec = PdqStartSpecV1 { + pdq_path: format!("{subdir}/{stem}_pd.pdq"), + sidecar_path: format!("{subdir}/{stem}_pd.json"), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: self.recording_metadata(), + }; + let pd_request = self.photodiode_request(PhotodiodeCommandV1::BeginRecording { + specification: spec, + }); + self.recording.pd_begin_req = pd_request.request_id; + context.request_service(&pd_request); + self.recording.phase = RecPhase::StartingPhotodiode; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: starting photodiode…", + self.recording.id + )); + } + + /// Atomically close the PDQ and release its lease while the camera + /// pipeline is still live. + fn stop_photodiode(&mut self, context: &mut impl RecordingControl) { + if self.recording.lease_granted { + let pd_request = self.photodiode_request(PhotodiodeCommandV1::ReleaseLease { + finalize_recording: true, + reason: "a1 recording complete".into(), + }); + self.recording.pd_finalize_req = pd_request.request_id; + context.request_service(&pd_request); + self.recording.phase = RecPhase::StoppingPhotodiode; + } else { + self.stop_camera(context); + return; + } + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: saving photodiode data…", + self.recording.id + )); + } + + /// Stop the host recorder after the PDQ has been safely finalized. + fn stop_camera(&mut self, context: &mut impl RecordingControl) { + if self.recording.cam_raw_path.is_some() && !self.recording.cam_rejected { + let cam_req = self.next_request_id(); + context.request_host(&HostCommandRequest { + request_id: cam_req, + command: HostCommand::StopRecording, + }); + self.recording.cam_stop_req = cam_req; + self.recording.phase = RecPhase::StoppingCamera; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: saving camera data…", + self.recording.id + )); + } else { + self.finish_recording(context); + } + } + + fn finish_recording(&mut self, context: &mut impl RecordingControl) { + let clean = self.recording.cam_complete + && self.recording.pd_finalized + && self.recording.pd_valid + && self.recording.pd_pdq_path.is_some() + && self.recording.pd_sidecar_path.is_some(); + let sidecar = self.write_sidecar(); + let message = match (sidecar, clean) { + (Ok(path), true) => format!("Saved recording {} → {path}", self.recording.id), + (Ok(path), false) => format!( + "Recording {} was incomplete — metadata saved to {path}", + self.recording.id + ), + (Err(err), _) => format!( + "Recording {} finished, metadata save failed: {err}", + self.recording.id + ), + }; + self.recording_completed_ok = clean; + self.release_and_idle(context, message); + } + + /// Release the photodiode lease (only if we actually hold it) and return to idle. + fn release_and_idle(&mut self, context: &mut impl RecordingControl, message: String) { + if self.recording.lease_granted { + let request = self.photodiode_request(PhotodiodeCommandV1::ReleaseLease { + finalize_recording: true, + reason: "a1 recording complete".into(), + }); + context.request_service(&request); + } + self.recording = Recording::idle(); + self.note(message); + } + + /// Wraps a modulation command in the routed service request A1 emits. + fn modulation_request( + &mut self, + command: ModulationCommandV1, + lease_id: &LeaseId, + ) -> PluginServiceRequest { + let request_id = self.next_request_id(); + let mut envelope = + ModulationRequestV1::new(RequestId(request_id), ClientId::new(A1_PLUGIN_ID), command); + envelope.lease_id = Some(lease_id.clone()); + envelope.target_owner_instance = self + .modulation + .as_ref() + .map(|state| state.owner_instance.clone()); + envelope.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: MODULATION_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + payload: serde_json::to_value(&envelope).unwrap_or(Value::Null), + } + } + + fn modulation_connected(&self) -> bool { + matches!( + self.modulation.as_ref().map(|state| &state.connection), + Some(ConnectionStateV1::Connected { .. }) + ) + } + + /// The requested `a` per sweep point, ascending and inclusive of both ends. + fn sweep_points(&self) -> Vec { + let count = self.sweep_count.clamp(2, 64) as usize; + let span = self.max_a - self.min_a; + (0..count) + .map(|index| self.min_a + span * index as f64 / (count - 1) as f64) + .collect() + } + + /// Worst-case sweep duration, used as the modulation lease TTL. + fn sweep_lease_ttl_ms(&self, remaining_points: usize) -> u64 { + let per_point_ms = (self.duration_s.max(1) as u64) + .saturating_mul(1_000) + .saturating_add(SWEEP_SETTLE_TIMEOUT_MS) + .saturating_add(30_000); + (remaining_points as u64) + .saturating_mul(per_point_ms) + .saturating_add(60_000) + } + + /// Kick off the amplitude sweep: validate, then lease the modulation owner. + fn begin_sweep(&mut self, context: &mut PluginControlContext<'_>) { + if self.recording.is_active() || self.sweep.is_some() { + self.message = "A recording or sweep is already running".into(); + return; + } + if self.output_folder.trim().is_empty() { + self.message = "Set an output folder before sweeping".into(); + return; + } + if self.measurement_id.trim().is_empty() { + self.message = "Set a measurement id before sweeping".into(); + return; + } + if !self.modulation_connected() { + self.message = "Modulation owner is not connected — cannot sweep".into(); + return; + } + if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { + self.message = + "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); + return; + } + if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { + self.message = "Sweep needs max a > min a".into(); + return; + } + let points = self.sweep_points(); + let now_ms = now_unix_ms(); + let lease_id = LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))); + let ttl_ms = self.sweep_lease_ttl_ms(points.len()); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + let lease_req = request.request_id; + let _ = context.request_service(&request); + let total = points.len(); + self.sweep = Some(Sweep { + phase: SweepPhase::AcquiringLease, + points, + index: 0, + lease_id, + lease_granted: false, + lease_req, + depth_req: 0, + depth_applied: false, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: false, + last_activity_ms: now_ms, + stop_requested: false, + }); + self.message = format!("Sweep: acquiring modulation lease for {total} points…"); + } + + /// Release the modulation lease (if held) and clear the sweep. + fn finish_sweep(&mut self, context: &mut PluginControlContext<'_>, message: String) { + if let Some(sweep) = self.sweep.take() { + if sweep.lease_granted { + let request = self.modulation_request( + ModulationCommandV1::ReleaseLease { + safe_off: false, + reason: "a1 sweep finished".into(), + }, + &sweep.lease_id, + ); + let _ = context.request_service(&request); + } + } + self.message = message; + } + + /// Renew the modulation lease and retarget the drive at the current point. + fn send_sweep_depth(&mut self, context: &mut PluginControlContext<'_>) { + let Some(sweep) = self.sweep.as_ref() else { + return; + }; + let lease_id = sweep.lease_id.clone(); + let remaining = sweep.total().saturating_sub(sweep.index); + let target_a = sweep.target_a(); + let index = sweep.index; + let total = sweep.total(); + + let ttl_ms = self.sweep_lease_ttl_ms(remaining); + let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + let _ = context.request_service(&renew); + + let depth = self.modulation_request( + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: (target_a * 1_000.0).round().clamp(0.0, u32::MAX as f64) as u32, + }, + &lease_id, + ); + let depth_req = depth.request_id; + let _ = context.request_service(&depth); + + let now_ms = now_unix_ms(); + if let Some(sweep) = self.sweep.as_mut() { + sweep.phase = SweepPhase::SettingDepth; + sweep.depth_req = depth_req; + sweep.depth_applied = false; + sweep.settled_since_ms = None; + sweep.point_started = false; + sweep.last_activity_ms = now_ms; + } + self.message = format!( + "Sweep point {}/{}: retargeting drive to a = {:.3}…", + index + 1, + total, + target_a + ); + } + + /// Advance the amplitude sweep one control tick. Runs before + /// `drive_recording`, so a point's recording starts on the same tick. + fn drive_sweep(&mut self, context: &mut PluginControlContext<'_>) { + if self.sweep.is_none() { + if std::mem::take(&mut self.sweep_pending) { + self.begin_sweep(context); + } + return; + } + self.sweep_pending = false; + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, depth_applied, last_activity_ms, index, total) = { + let sweep = self.sweep.as_ref().expect("sweep checked above"); + ( + sweep.phase, + sweep.stop_requested, + sweep.lease_granted, + sweep.depth_applied, + sweep.last_activity_ms, + sweep.index, + sweep.total(), + ) + }; + if stop_requested && phase != SweepPhase::Recording { + let message = if self.message.is_empty() { + "Sweep stopped".into() + } else { + self.message.clone() + }; + self.finish_sweep(context, message); + return; + } + match phase { + SweepPhase::AcquiringLease => { + if lease_granted { + self.send_sweep_depth(context); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_sweep( + context, + "Sweep aborted: timed out acquiring the modulation lease".into(), + ); + } + } + SweepPhase::SettingDepth => { + if depth_applied { + let target = self + .sweep + .as_mut() + .map(|sweep| { + sweep.phase = SweepPhase::Settling; + sweep.settled_since_ms = None; + sweep.settle_deadline_ms = now_ms + SWEEP_SETTLE_TIMEOUT_MS; + sweep.target_a() + }) + .unwrap_or_default(); + self.message = format!( + "Sweep point {}/{}: waiting for a to settle at {target:.3}…", + index + 1, + total, + ); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_sweep( + context, + "Sweep aborted: timed out retargeting the modulation drive".into(), + ); + } + } + SweepPhase::Settling => { + let target = self.sweep.as_ref().map(Sweep::target_a).unwrap_or_default(); + let settled = self + .measured_a() + .is_some_and(|measured| (measured - target).abs() <= sweep_tolerance(target)); + let dwell_ms = (self.settle_s.max(0.0) * 1_000.0) as u64; + let mut start_recording = false; + let mut settle_timed_out = false; + if let Some(sweep) = self.sweep.as_mut() { + if settled { + let since = *sweep.settled_since_ms.get_or_insert(now_ms); + if now_ms.saturating_sub(since) >= dwell_ms { + start_recording = true; + } + } else { + sweep.settled_since_ms = None; + } + if !start_recording && now_ms >= sweep.settle_deadline_ms { + // Record anyway: the sidecar stores the *measured* a, + // so an unsettled point is still a usable sample. + start_recording = true; + settle_timed_out = true; + } + if start_recording { + sweep.phase = SweepPhase::Recording; + } + } + if start_recording { + self.pending_role = Some(RecRole::Normal); + if settle_timed_out { + self.message = format!( + "Sweep point {}/{}: a did not settle at {target:.3} — recording anyway", + index + 1, + total, + ); + } + } + } + SweepPhase::Recording => { + if self.pending_role.is_some() || self.recording.is_active() { + if self.recording.is_active() { + if let Some(sweep) = self.sweep.as_mut() { + sweep.point_started = true; + } + if stop_requested { + self.recording.stop_requested = true; + } + } + return; + } + // The recording coordinator is idle again: the point either + // finished, failed, or was refused before starting. + let point_started = self.sweep.as_ref().is_some_and(|sweep| sweep.point_started); + if stop_requested { + let message = self.message.clone(); + self.finish_sweep(context, message); + } else if !point_started || !self.recording_completed_ok { + let message = format!("Sweep aborted: {}", self.message); + self.finish_sweep(context, message); + } else if index + 1 >= total { + self.finish_sweep(context, format!("Sweep complete: {total} points recorded")); + } else { + if let Some(sweep) = self.sweep.as_mut() { + sweep.index += 1; + } + self.send_sweep_depth(context); + } + } + } + } + + /// Routes modulation-service replies belonging to the sweep. Returns true + /// when the reply was consumed. + fn on_sweep_reply(&mut self, reply: &PluginServiceReply) -> bool { + let Some((lease_req, depth_req)) = self + .sweep + .as_ref() + .map(|sweep| (sweep.lease_req, sweep.depth_req)) + else { + return false; + }; + let abort = |this: &mut Self, message: String| { + this.message = message; + if let Some(sweep) = this.sweep.as_mut() { + sweep.stop_requested = true; + } + }; + if reply.request_id == lease_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.sweep.as_mut() { + sweep.lease_granted = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => { + abort( + self, + format!("Sweep aborted: modulation lease rejected: {message}"), + ); + } + } + true + } else if reply.request_id == depth_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.sweep.as_mut() { + sweep.depth_applied = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => { + abort( + self, + format!("Sweep aborted: drive retarget rejected: {message}"), + ); + } + } + true + } else { + false + } + } + + fn on_host_reply(&mut self, reply: &HostCommandReply) { + if reply.request_id == self.recording.cam_start_req { + match &reply.outcome { + HostCommandOutcome::RecordingStarted { + actual_raw_path, .. + } => { + self.recording.cam_raw_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + // Stop the rest of the recording; drive_recording resolves the + // abort from the current phase on the next tick. + self.note(format!("Camera recording rejected ({code}): {message}")); + self.recording.cam_rejected = true; + self.recording.stop_requested = true; + } + _ => {} + } + } else if reply.request_id == self.recording.cam_stop_req { + match &reply.outcome { + HostCommandOutcome::RecordingFinalized { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.cam_complete = true; + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::RecordingPartial { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + self.message = format!("Camera stop failed ({code}): {message}"); + self.recording.cam_rejected = true; + self.recording.last_activity_ms = now_unix_ms(); + } + _ => {} + } + } + } + + fn on_service_reply(&mut self, reply: &PluginServiceReply) { + if self.on_sweep_reply(reply) { + return; + } + let response = match &reply.outcome { + PluginServiceOutcome::Accepted { payload } => { + serde_json::from_value::(payload.clone()).ok() + } + PluginServiceOutcome::Rejected { code, message } => { + if reply.request_id == self.recording.connect_req + || reply.request_id == self.recording.lease_req + || reply.request_id == self.recording.pd_begin_req + { + self.note(format!("Photodiode start failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.stop_requested = true; + } else if reply.request_id == self.recording.pd_finalize_req { + self.note(format!("Photodiode save failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + None + } + }; + let Some(response) = response else { + return; + }; + if reply.request_id == self.recording.connect_req { + self.recording.connect_accepted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.lease_req { + self.recording.lease_granted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.pd_begin_req { + if let Some(PdqReceiptV1::Started(started)) = &response.receipt { + self.recording.pd_pdq_path = Some(started.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(started.sidecar_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + } else if reply.request_id == self.recording.pd_finalize_req { + self.recording.pd_finalized = true; + if let Some(PdqReceiptV1::Finalized(finalized)) = &response.receipt { + self.recording.pd_pdq_path = Some(finalized.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(finalized.sidecar_path.clone()); + self.recording.pd_valid = finalized.valid; + } + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + } + + /// Advance the recording state machine one control tick. + fn drive_recording(&mut self, context: &mut impl RecordingControl) { + let now_ms = now_unix_ms(); + match self.recording.phase { + RecPhase::Idle => { + if let Some(role) = self.pending_role.take() { + self.begin_recording(context, role); + } + } + RecPhase::StartingCamera => { + if self.recording.cam_rejected { + let message = self.message.clone(); + self.release_and_idle(context, message); + } else if self.recording.cam_raw_path.is_some() { + if self.recording.stop_requested { + self.stop_camera(context); + } else { + self.connect_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.cam_rejected = true; + self.release_and_idle(context, "Timed out starting camera recording".into()); + } + } + RecPhase::ConnectingPhotodiode => { + if self.recording.stop_requested && !self.recording.connect_accepted { + self.stop_camera(context); + } else if self.recording.connect_accepted { + if self.recording.stop_requested { + self.stop_camera(context); + } else { + self.acquire_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.note("Timed out connecting the photodiode"); + self.stop_camera(context); + } + } + RecPhase::AcquiringLease => { + if self.recording.pd_rejected { + self.stop_camera(context); + } else if self.recording.lease_granted { + if self.recording.stop_requested { + self.stop_photodiode(context); + } else { + self.start_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.note("Timed out preparing the photodiode"); + self.stop_camera(context); + } + } + RecPhase::StartingPhotodiode => { + if self.recording.pd_pdq_path.is_some() && self.recording.pd_sidecar_path.is_some() + { + self.recording.phase = RecPhase::Running; + self.recording.start_unix_ms = now_ms; + if self.recording.stop_requested { + self.stop_photodiode(context); + } else { + self.note(format!( + "Recording {} for {} s…", + self.recording.id, self.recording.duration_s + )); + } + } else if self.recording.pd_rejected + || now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.stop_photodiode(context); + } + } + RecPhase::Running => { + let elapsed_ms = now_ms.saturating_sub(self.recording.start_unix_ms); + let over = elapsed_ms >= self.recording.duration_s.saturating_mul(1_000); + if over || self.recording.stop_requested { + self.stop_photodiode(context); + } + } + RecPhase::StoppingPhotodiode => { + if self.recording.pd_finalized || self.recording.pd_rejected { + self.stop_camera(context); + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.recording.lease_granted = false; + self.note("Timed out saving photodiode data"); + self.stop_camera(context); + } + } + RecPhase::StoppingCamera => { + if self.recording.cam_finalized_path.is_some() + || self.recording.cam_rejected + || now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + if self.recording.cam_finalized_path.is_none() && !self.recording.cam_rejected { + self.recording.cam_rejected = true; + self.note("Timed out saving camera data"); + } + self.finish_recording(context); + } + } + } + } + + /// Build and write the A1 config sidecar linking the RAW + PDQ files. + fn write_sidecar(&self) -> Result { + let now_ms = now_unix_ms(); + let modulation = self + .modulation + .as_ref() + .and_then(|s| s.acknowledged.as_ref()); + let a1_config = modulation.and_then(|t| t.a1_configuration.as_ref()); + let optical = self + .photodiode + .as_ref() + .and_then(|s| s.optical_summary.as_ref()); + let roi = self.host_roi.unwrap_or_default(); + + let raw_path = self + .recording + .cam_finalized_path + .clone() + .or_else(|| self.recording.cam_raw_path.clone()); + let camera_bias_sidecar = raw_path.as_deref().and_then(sibling_toml); + + let doc = SidecarDoc { + measurement_id: self.recording.id.clone(), + file_stem: self.recording.stem.clone(), + role: self.recording.role.label().into(), + recorded_at_utc: format_iso_utc( + if self.recording.start_unix_ms == 0 { + now_ms + } else { + self.recording.start_unix_ms + } / 1_000, + ), + finalized_at_utc: format_iso_utc(now_ms / 1_000), + duration_s: self.recording.duration_s, + sweep: { + let point = self + .sweep + .as_ref() + .filter(|sweep| sweep.phase == SweepPhase::Recording); + SweepSidecar { + min_a: self.min_a, + max_a: self.max_a, + requested_a: point.map(Sweep::target_a), + point_index: point.map(|sweep| sweep.index + 1), + point_total: point.map(Sweep::total), + } + }, + pilot: (self.recording.role == RecRole::Pilot) + .then_some(self.pilot_windows) + .flatten() + .map(|(on, off)| PilotSidecar { + window_on_start: on.start, + window_on_end: on.end, + window_off_start: off.start, + window_off_end: off.end, + }), + background: (self.recording.role == RecRole::Background) + .then_some(self.background_floor) + .flatten() + .map(|(q_on, q_off)| BackgroundSidecar { q_on, q_off }), + modulation: ModulationSidecar { + frequency_hz: self.period_us().map(|t| 1_000_000.0 / t), + frequency_source: self.frequency_source().into(), + center_dac: a1_config.map(|c| c.center_dac), + amplitude_dac: a1_config.map(|c| c.amplitude_dac), + waveform: modulation + .and_then(|t| t.waveform.as_ref()) + .map(waveform_label), + }, + optical: OpticalSidecar { + measured_a: optical.map(|o| o.measured_log_contrast), + low_clip_fraction: optical.map(|o| o.low_clip_fraction), + high_clip_fraction: optical.map(|o| o.high_clip_fraction), + measured_frequency_hz: optical.and_then(|o| o.measured_frequency_hz), + }, + camera: CameraSidecar { + roi_x: roi.x, + roi_y: roi.y, + roi_width: roi.width, + roi_height: roi.height, + masked_pixels: self.masked_pixels.len(), + n_valid: self.valid_pixel_count(), + }, + trigger: TriggerSidecar { + marker_anchored: self.is_marker_anchored(), + marker_count: self.camera_markers_us.len(), + measured_period_us: self.measured_period_us(), + }, + files: FilesSidecar { + camera_raw: raw_path, + camera_config_sidecar: camera_bias_sidecar, + photodiode_pdq: self.recording.pd_pdq_path.clone(), + photodiode_sidecar: self.recording.pd_sidecar_path.clone(), + }, + }; + + let toml = toml::to_string_pretty(&doc).map_err(|err| err.to_string())?; + let dir = PathBuf::from(&self.recording.folder).join(&self.recording.id); + std::fs::create_dir_all(&dir).map_err(|err| err.to_string())?; + let path = dir.join(format!("{}_config.toml", self.recording.stem)); + std::fs::write(&path, toml).map_err(|err| err.to_string())?; + Ok(path.display().to_string()) + } +} + +// ---- sidecar document ------------------------------------------------------ + +#[derive(Serialize)] +struct SidecarDoc { + measurement_id: String, + file_stem: String, + role: String, + recorded_at_utc: String, + finalized_at_utc: String, + duration_s: u64, + sweep: SweepSidecar, + #[serde(skip_serializing_if = "Option::is_none")] + pilot: Option, + #[serde(skip_serializing_if = "Option::is_none")] + background: Option, + modulation: ModulationSidecar, + optical: OpticalSidecar, + camera: CameraSidecar, + trigger: TriggerSidecar, + files: FilesSidecar, +} + +#[derive(Serialize)] +struct SweepSidecar { + min_a: f64, + max_a: f64, + /// The `a` this sweep point asked the drive for (measured `a` is in + /// `[optical]`); absent on manual recordings. + #[serde(skip_serializing_if = "Option::is_none")] + requested_a: Option, + /// 1-based point position within the sweep; absent on manual recordings. + #[serde(skip_serializing_if = "Option::is_none")] + point_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + point_total: Option, +} + +/// Frozen ON/OFF windows written into a **pilot** recording's sidecar and read +/// back to reuse them across the row. +#[derive(Serialize, serde::Deserialize)] +struct PilotSidecar { + window_on_start: f64, + window_on_end: f64, + window_off_start: f64, + window_off_end: f64, +} + +/// False-response floor written into a **background** recording's sidecar. +#[derive(Serialize, serde::Deserialize)] +struct BackgroundSidecar { + q_on: f64, + q_off: f64, +} + +/// Partial view of a config sidecar for reading the pilot/background sections +/// back; every other section is ignored. +#[derive(serde::Deserialize)] +struct RowSidecar { + #[serde(default)] + pilot: Option, + #[serde(default)] + background: Option, +} + +#[derive(Serialize)] +struct ModulationSidecar { + #[serde(skip_serializing_if = "Option::is_none")] + frequency_hz: Option, + frequency_source: String, + #[serde(skip_serializing_if = "Option::is_none")] + center_dac: Option, + #[serde(skip_serializing_if = "Option::is_none")] + amplitude_dac: Option, + #[serde(skip_serializing_if = "Option::is_none")] + waveform: Option, +} + +#[derive(Serialize)] +struct OpticalSidecar { + #[serde(skip_serializing_if = "Option::is_none")] + measured_a: Option, + #[serde(skip_serializing_if = "Option::is_none")] + low_clip_fraction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + high_clip_fraction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + measured_frequency_hz: Option, +} + +#[derive(Serialize)] +struct CameraSidecar { + roi_x: u16, + roi_y: u16, + roi_width: u16, + roi_height: u16, + masked_pixels: usize, + #[serde(skip_serializing_if = "Option::is_none")] + n_valid: Option, +} + +#[derive(Serialize)] +struct TriggerSidecar { + marker_anchored: bool, + marker_count: usize, + #[serde(skip_serializing_if = "Option::is_none")] + measured_period_us: Option, +} + +#[derive(Serialize)] +struct FilesSidecar { + #[serde(skip_serializing_if = "Option::is_none")] + camera_raw: Option, + #[serde(skip_serializing_if = "Option::is_none")] + camera_config_sidecar: Option, + #[serde(skip_serializing_if = "Option::is_none")] + photodiode_pdq: Option, + #[serde(skip_serializing_if = "Option::is_none")] + photodiode_sidecar: Option, +} + +// ---- free functions -------------------------------------------------------- + +fn ffi_to_camera_event(event: &FfiCdEvent) -> CameraEvent { + CameraEvent { + x: event.x, + y: event.y, + timestamp_us: event.timestamp_us(), + polarity: if event.is_on() { + Polarity::On + } else { + Polarity::Off + }, + } +} + +fn points_for(points: &[RollingResponsePoint], first: u64) -> Vec { + points + .iter() + .map(|point| Series1dPoint { + x: point.timestamp_us.saturating_sub(first) as f64 / 1_000_000.0, + y: point.run_per_pixel, + }) + .collect() +} + +fn waveform_label(waveform: &WaveformV1) -> String { + match waveform { + WaveformV1::Off => "off".into(), + WaveformV1::Constant { level_dac } => format!("constant({level_dac})"), + WaveformV1::Periodic { + waveform, + min_dac, + max_dac, + frequency_millihz, + } => format!( + "periodic({waveform:?}, {min_dac}..{max_dac}, {:.3} Hz)", + *frequency_millihz as f64 / 1_000.0 + ), + } +} + +fn sibling_toml(raw_path: &str) -> Option { + let path = Path::new(raw_path); + let stem = path.file_stem()?.to_string_lossy(); + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + Some(parent.join(format!("{stem}.toml")).display().to_string()) +} + +/// Parses the newest config sidecar in `folder` for measurement `id` whose stem +/// carries `role_tag` (e.g. `_pilot`). Filenames embed a sortable timestamp, so +/// the lexicographically largest matching name is the most recent. +fn load_row_sidecar(folder: &str, id: &str, role_tag: &str) -> Option { + let prefix = format!("{id}_"); + let mut best: Option = None; + for entry in std::fs::read_dir(folder).ok()?.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with(&prefix) + && name.contains(role_tag) + && name.ends_with("_config.toml") + && best.as_ref().is_none_or(|current| name > *current) + { + best = Some(name); + } + } + let text = std::fs::read_to_string(Path::new(folder).join(best?)).ok()?; + toml::from_str::(&text).ok() +} + +fn load_row_windows(folder: &str, id: &str, role_tag: &str) -> Option<(PhaseWindow, PhaseWindow)> { + let pilot = load_row_sidecar(folder, id, role_tag)?.pilot?; + Some(( + PhaseWindow { + start: pilot.window_on_start, + end: pilot.window_on_end, + }, + PhaseWindow { + start: pilot.window_off_start, + end: pilot.window_off_end, + }, + )) +} + +fn load_row_background(folder: &str, id: &str, role_tag: &str) -> Option<(f64, f64)> { + let background = load_row_sidecar(folder, id, role_tag)?.background?; + Some((background.q_on, background.q_off)) +} + +/// Replace anything that is not `[A-Za-z0-9._-]` with `_` so ids are file-safe. +fn sanitize_stem(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + for ch in input.chars() { + if ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-') { + out.push(ch); + } else if !out.ends_with('_') { + out.push('_'); + } + } + let trimmed = out.trim_matches('_').to_string(); + if trimmed.is_empty() { + "A1".into() + } else { + trimmed + } +} + +fn generate_measurement_id() -> String { + let ms = now_unix_ms(); + format!( + "A1-{}-{:04x}", + format_compact_date(ms / 1_000), + (ms & 0xffff) + ) +} + +fn now_unix_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Gregorian date for a count of days since the Unix epoch (Howard Hinnant's +/// civil-from-days algorithm). +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; + let month = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32; + (year + i64::from(month <= 2), month, day) +} + +fn ymd_hms(unix_secs: u64) -> (i64, u32, u32, u64, u64, u64) { + let days = (unix_secs / 86_400) as i64; + let sod = unix_secs % 86_400; + let (y, m, d) = civil_from_days(days); + (y, m, d, sod / 3_600, (sod % 3_600) / 60, sod % 60) +} + +fn format_compact_date(unix_secs: u64) -> String { + let (y, m, d, ..) = ymd_hms(unix_secs); + format!("{y:04}{m:02}{d:02}") +} + +fn format_compact_utc(unix_secs: u64) -> String { + let (y, m, d, hh, mm, ss) = ymd_hms(unix_secs); + format!("{y:04}{m:02}{d:02}-{hh:02}{mm:02}{ss:02}") +} + +fn format_iso_utc(unix_secs: u64) -> String { + let (y, m, d, hh, mm, ss) = ymd_hms(unix_secs); + format!("{y:04}-{m:02}-{d:02}T{hh:02}:{mm:02}:{ss:02}Z") +} + +impl Plugin for StageAA1Plugin { + fn name(&self) -> &'static str { + "Stage-A A1 Analysis" + } + + fn description(&self) -> &'static str { + "Stage-A A1 recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar, plus live rolling-response and response-probability quicklooks." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + } + + fn set_runtime_role(&mut self, role: PluginRuntimeRole) { + self.runtime_role = role; + } + + fn reset(&mut self) { + self.camera_events.clear(); + self.event_scratch.clear(); + self.camera_markers_us.clear(); + self.valid_pixels = 0; + self.response_points.clear(); + self.pilot_windows = None; + self.background_floor = None; + self.loaded_key = None; + self.bump(); + } + + fn on_discontinuity(&mut self, reason: PluginDiscontinuity) { + match reason { + // The host raises SettingsChanged on *every* settings sync of any + // plugin (including our own button presses). The fold window + // rebuilds itself each frame, and the response curve, pilot + // windows, and background floor are operator-owned science state — + // wiping them here made "Record point" appear dead. + PluginDiscontinuity::SettingsChanged => {} + PluginDiscontinuity::Seek + | PluginDiscontinuity::SourceChanged + | PluginDiscontinuity::HistoryEvicted => self.reset(), + } + } + + fn input_kind(&self) -> PluginInput { + PluginInput::RawEvents + } + + fn capabilities(&self) -> PluginCapabilities { + // Request retained event history so the analysis window comes exactly + // from the EventStore rather than best-effort preview frames. + PluginCapabilities { + retained_event_history: true, + } + } + + fn process_frame( + &mut self, + frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + event_store: &EventStoreHandle<'_>, + ) { + self.frame_width = frame.width(); + self.frame_height = frame.height(); + // ROI and masked pixels are owned by the host camera config, not the + // plugin; mirror the latest snapshot each frame. + if let Some(settings) = context + .get::(CTX_GLOBAL_SETTINGS) + .ok() + .flatten() + { + self.host_roi = Some(settings.roi); + self.masked_pixels = settings.masked_pixels.into_iter().collect(); + } + if !self.live { + return; + } + self.valid_pixels = usize::from(frame.width()) * usize::from(frame.height()); + + // Markers (phase-0 sync) only exist on the preview frame, so accumulate + // the rising EXT_TRIGGER edges here regardless of the event source. + // Preview windows overlap, so the same trigger arrives on several + // consecutive frames — dedup after every merge or the duplicate + // timestamps fail marker validation and blank the fold. + self.camera_markers_us.extend( + frame + .external_triggers() + .iter() + .filter(|trigger| trigger.is_rising()) + .map(|trigger| trigger.timestamp_us), + ); + self.camera_markers_us.sort_unstable(); + self.camera_markers_us.dedup(); + if self.camera_markers_us.len() > MAX_MARKERS { + let excess = self.camera_markers_us.len() - MAX_MARKERS; + self.camera_markers_us.drain(..excess); + } + + let window_end = frame.window_end_us(); + let window_us = (self.analysis_window_ms.max(1) as u64).saturating_mul(1_000); + if event_store.frame_count() > 0 { + // Exact path: rebuild the analysis window from the retained event + // history, immune to dropped preview frames. + let window_start = window_end + .saturating_sub(window_us) + .max(event_store.oldest_timestamp_us()); + self.event_scratch.clear(); + event_store.collect_events_in_range(window_start, window_end, &mut self.event_scratch); + self.camera_events.clear(); + self.camera_events + .extend(self.event_scratch.iter().map(ffi_to_camera_event)); + // Keep the marker set on the same window as the events. + self.camera_markers_us + .retain(|&marker| marker >= window_start); + } else if self.camera_events.len() < MAX_EVENTS { + // Fallback (no retained history available): accumulate the + // best-effort preview-frame events. + self.camera_events + .extend(frame.events().iter().map(ffi_to_camera_event)); + } + self.bump(); + } + + fn process_control(&mut self, context: &mut PluginControlContext<'_>) { + let inbox = context.inbox().clone(); + self.update_snapshots(&inbox); + for reply in &inbox.host_replies { + self.on_host_reply(reply); + } + for reply in &inbox.service_replies { + self.on_service_reply(reply); + } + // Reuse the pilot/background captured for this measurement when idle: when + // the folder or id changes, look them up in the folder. + if !self.recording.is_active() { + self.scan_measurement_folder(); + } + // The sweep runs first so a point's recording starts on the same tick. + self.drive_sweep(context); + self.drive_recording(context); + // The fold reflects live snapshots (T, a) even between frames. + self.bump(); + } + + fn settings_schema(&self) -> SettingsSchema { + // The record/sweep buttons stay disabled until the recording has a + // destination, instead of failing with a status message after a click. + let can_record = !self.output_folder.trim().is_empty(); + SettingsSchema { + sections: vec![ + SettingsSection { + label: "Recording".into(), + description: Some( + "Records the camera RAW stream and the photodiode PDQ stream together for \ + a fixed duration and writes an A1 config sidecar (.toml) linking them. \ + Files are grouped under the measurement id and share an _ \ + stem. Arm the optical drive in the modulation plugin first; A1 only reads \ + its settings — it never drives the Teensy. For everything to land in one \ + place, point the host output folder and the photodiode data folder at the \ + same experiment directory as this folder." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "output_folder".into(), + label: "Output folder".into(), + tooltip: Some( + "Directory where the A1 config sidecar is written. Also the \ + recommended shared experiment root for the RAW/PDQ files." + .into(), + ), + kind: SettingKind::Path { + dialog: PathDialogKind::Directory, + default: self.output_folder.clone(), + }, + }, + SettingItem { + key: "measurement_id".into(), + label: "Measurement id (one per I_k, f pair)".into(), + tooltip: Some( + "Groups every repeat of one illumination/frequency pair. Included \ + in every file name. Edit it freely or press New id." + .into(), + ), + kind: SettingKind::Text { + default: self.measurement_id.clone(), + }, + }, + SettingItem { + key: "new_id".into(), + label: "New id".into(), + tooltip: Some("Generate a fresh default measurement id.".into()), + kind: SettingKind::Button { enabled: true }, + }, + SettingItem { + key: "min_a".into(), + label: "Sweep min a".into(), + tooltip: Some( + "Low end of the modulation-depth sweep for this (I_k, f) row. \ + Stored in every sidecar as the automation template; A1 does not \ + drive it — you set the drive in the modulation plugin." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: self.min_a, + }, + }, + SettingItem { + key: "max_a".into(), + label: "Sweep max a".into(), + tooltip: Some( + "High end of the modulation-depth sweep for this (I_k, f) row \ + (also the natural amplitude for the pilot). Stored in every \ + sidecar; A1 does not drive it." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: self.max_a, + }, + }, + SettingItem { + key: "sweep_count".into(), + label: "Sweep points (count)".into(), + tooltip: Some( + "How many amplitudes the Start sweep button records, spaced \ + evenly from Sweep min a to Sweep max a (inclusive)." + .into(), + ), + kind: SettingKind::I64Drag { + min: 2, + max: 64, + default: self.sweep_count, + }, + }, + SettingItem { + key: "settle_s".into(), + label: "Sweep settle (s)".into(), + tooltip: Some( + "After retargeting the drive, the sweep waits until the \ + photodiode-measured a holds the target (±10 %, at least ±0.05) \ + for this long before recording. Gives up after 30 s and records \ + anyway — the sidecar stores the measured a." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 60.0, + speed: 0.1, + default: self.settle_s, + }, + }, + SettingItem { + key: "duration_s".into(), + label: "Duration (s)".into(), + tooltip: Some( + "How long each recording runs before it auto-stops and finalizes." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 3_600, + default: self.duration_s, + }, + }, + SettingItem { + key: "start_recording".into(), + label: "Start recording (sweep point)".into(), + tooltip: Some( + "Acquire the photodiode lease, start the camera RAW + photodiode \ + PDQ recording, auto-stop after the duration, and write the \ + sidecar. Disabled until an output folder is selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "start_sweep".into(), + label: "Start sweep (record all points)".into(), + tooltip: Some( + "Sweeps the modulation depth over [Sweep min a, Sweep max a] in \ + the configured number of points: per point A1 leases the \ + modulation owner, retargets the armed calibrated drive, waits \ + for the photodiode-measured a to settle, and records one sweep \ + point (…_pNN) like the Start recording button. Requires the \ + modulation plugin to have a calibrated periodic/optical drive \ + armed and Sweep min a > 0. Disabled until an output folder is \ + selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_pilot".into(), + label: "Record pilot (freeze ON/OFF windows)".into(), + tooltip: Some( + "Records a bright reference for this row into the same folder \ + (…_pilot) and freezes the ON/OFF windows from the current live \ + signal. Set a high, non-saturating a in the modulation plugin \ + first. The frozen windows are reused for the whole row's q_p. \ + Disabled until an output folder is selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_background".into(), + label: "Record background (a≈0 floor)".into(), + tooltip: Some( + "Records an unmodulated reference (…_background) and captures the \ + false-response floor q0 in the current windows. Set a≈0 in the \ + modulation plugin first. Disabled until an output folder is \ + selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "stop_recording".into(), + label: "Stop (abort recording / sweep)".into(), + tooltip: Some( + "Stop and finalize the current recording before the duration \ + ends; during a sweep this also aborts the remaining points." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + SettingsSection { + label: "Live analysis".into(), + description: Some( + "Live sanity quicklook. Folds the camera event stream on the modulation \ + period T (defined by the firmware phase-0 EXT_TRIGGER) and renders the \ + rolling half-period response S_p(t): events per valid pixel in the \ + trailing T/2, ON and OFF separately. Use it to confirm events are \ + appearing and the ON/OFF timing looks sane before recording. Nothing is \ + recorded here." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "live".into(), + label: "Live analysis".into(), + tooltip: Some( + "Fold incoming events into the live plots. Off freezes the plots \ + at their current values. This does not record anything." + .into(), + ), + kind: SettingKind::Bool { default: self.live }, + }, + SettingItem { + key: "analysis_window_ms".into(), + label: "Analysis window (ms)".into(), + tooltip: Some( + "Trailing window pulled exactly from the retained EventStore. \ + Longer windows cover more cycles; bounded by the host event-store \ + memory budget." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 120_000, + default: self.analysis_window_ms, + }, + }, + SettingItem { + key: "clear".into(), + label: "Clear captured events".into(), + tooltip: Some( + "Empties the fold buffer and resets the live plots.".into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + SettingsSection { + label: "Response probability q_p (live quicklook)".into(), + description: Some( + "Live view of the response-curve metric q_p: the fraction of valid \ + pixel-cycles that fire at least once in the ON/OFF phase window (unlike \ + S_p, each pixel-cycle counts at most once). The ON/OFF windows come from \ + the row's pilot when one has been recorded (frozen, in the Recording \ + section) and otherwise from the trigger-anchored fold automatically — each \ + window grows out from its histogram peak until events drop below the \ + window floor or the opposite polarity takes over. Press Record point at \ + each amplitude to append a q_p(a) dot at the photodiode-measured a. The \ + ROI and masked pixels come from the camera config. The authoritative fit \ + is computed offline from the recordings; this is a quicklook." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "window_floor".into(), + label: "Window floor (fraction of peak)".into(), + tooltip: Some( + "Each ON/OFF window grows out from its histogram peak until events \ + fall below this fraction of the peak (or the opposite polarity \ + takes over). 0.10 = stop at 10 % of the peak." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.02, + max: 0.5, + speed: 0.01, + default: self.window_floor, + }, + }, + SettingItem { + key: "record_point".into(), + label: "Record point (at current a)".into(), + tooltip: Some( + "Computes q_on/q_off for the current buffer against the \ + auto-detected windows and appends a point at the \ + photodiode-measured a." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + SettingItem { + key: "clear_curve".into(), + label: "Clear response curve".into(), + tooltip: Some("Drops the recorded response-curve points.".into()), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + ], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "output_folder" => Some(json!(self.output_folder)), + "measurement_id" => Some(json!(self.measurement_id)), + "min_a" => Some(json!(self.min_a)), + "max_a" => Some(json!(self.max_a)), + "sweep_count" => Some(json!(self.sweep_count)), + "settle_s" => Some(json!(self.settle_s)), + "duration_s" => Some(json!(self.duration_s)), + "live" => Some(json!(self.live)), + "analysis_window_ms" => Some(json!(self.analysis_window_ms)), + "window_floor" => Some(json!(self.window_floor)), + // Button presses are exported as monotonic counters so the host's + // settings snapshot transports them from the UI mirror to the + // live worker (see PressLatch). + "start_recording" => Some(self.press_start.value()), + "record_pilot" => Some(self.press_pilot.value()), + "record_background" => Some(self.press_background.value()), + "stop_recording" => Some(self.press_stop.value()), + "start_sweep" => Some(self.press_sweep.value()), + "clear" => Some(self.press_clear.value()), + "record_point" => Some(self.press_record_point.value()), + "clear_curve" => Some(self.press_clear_curve.value()), + // New id regenerates the measurement id locally; the id itself is + // what synchronizes, so the press must not be forwarded (both + // instances would generate different ids). + "new_id" => Some(json!(false)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "output_folder" => { + self.output_folder = value + .as_str() + .ok_or("output_folder must be a string")? + .to_string(); + } + "measurement_id" => { + self.measurement_id = value + .as_str() + .ok_or("measurement_id must be a string")? + .to_string(); + } + "new_id" if value.as_bool() == Some(true) => { + self.measurement_id = generate_measurement_id(); + } + "min_a" => { + self.min_a = value + .as_f64() + .ok_or("min_a must be a number")? + .clamp(0.0, 10.0); + } + "max_a" => { + self.max_a = value + .as_f64() + .ok_or("max_a must be a number")? + .clamp(0.0, 10.0); + } + "sweep_count" => { + self.sweep_count = value + .as_i64() + .ok_or("sweep_count must be an integer")? + .clamp(2, 64); + } + "settle_s" => { + self.settle_s = value + .as_f64() + .ok_or("settle_s must be a number")? + .clamp(0.0, 60.0); + } + "duration_s" => { + self.duration_s = value + .as_i64() + .ok_or("duration_s must be an integer")? + .clamp(1, 3_600); + } + "start_recording" => { + if self.press_start.accept(&value) { + self.pending_role = Some(RecRole::Normal); + } + } + "record_pilot" => { + if self.press_pilot.accept(&value) { + self.pending_role = Some(RecRole::Pilot); + } + } + "record_background" => { + if self.press_background.accept(&value) { + self.pending_role = Some(RecRole::Background); + } + } + "start_sweep" => { + if self.press_sweep.accept(&value) { + self.sweep_pending = true; + } + } + "stop_recording" => { + if self.press_stop.accept(&value) { + if self.recording.is_active() { + self.recording.stop_requested = true; + } + if let Some(sweep) = self.sweep.as_mut() { + sweep.stop_requested = true; + self.message = "Sweep stop requested".into(); + } + self.sweep_pending = false; + } + } + "live" => { + self.live = value.as_bool().ok_or("live must be a boolean")?; + } + "analysis_window_ms" => { + self.analysis_window_ms = value + .as_i64() + .ok_or("analysis_window_ms must be an integer")? + .clamp(1, 120_000); + } + "clear" => { + if self.press_clear.accept(&value) { + self.camera_events.clear(); + self.event_scratch.clear(); + self.camera_markers_us.clear(); + self.valid_pixels = 0; + } + } + "window_floor" => { + self.window_floor = value + .as_f64() + .ok_or("window_floor must be a number")? + .clamp(0.02, 0.5); + } + "record_point" => { + if self.press_record_point.accept(&value) { + // Report failure via the status message: on the worker the + // press arrives through the settings snapshot, where a + // returned error would be silently dropped. + if let Err(error) = self.record_response_point() { + self.message = format!("Record point failed: {error}"); + } + } + } + "clear_curve" => { + if self.press_clear_curve.accept(&value) { + self.response_points.clear(); + } + } + "new_id" => return Ok(()), + _ => return Err(format!("unknown setting '{key}'")), + } + self.bump(); + Ok(()) + } + + fn status_entries(&self) -> Vec { + let mut entries = vec![StatusEntry::LabeledValue { + label: "Recording".into(), + value: self.recording.state_label().into(), + color: None, + }]; + if self.recording.is_active() { + if let Some(remaining) = self.recording.remaining_s(now_unix_ms()) { + entries.push(StatusEntry::Text(format!( + "{} — {remaining} s remaining", + self.recording.id + ))); + } + } + if let Some(sweep) = &self.sweep { + let phase = match sweep.phase { + SweepPhase::AcquiringLease => "leasing modulation", + SweepPhase::SettingDepth => "retargeting drive", + SweepPhase::Settling => "settling", + SweepPhase::Recording => "recording", + }; + entries.push(StatusEntry::Text(format!( + "Sweep: point {}/{} at a → {:.3} ({phase})", + sweep.index + 1, + sweep.total(), + sweep.target_a() + ))); + } + if !self.message.is_empty() { + entries.push(StatusEntry::Text(self.message.clone())); + } + match self.period_us() { + Some(period_us) => { + let source = self.frequency_source(); + entries.push(StatusEntry::Text(format!( + "T = {:.3} ms ({:.3} Hz, {source})", + period_us / 1_000.0, + 1_000_000.0 / period_us, + ))); + } + None => entries.push(StatusEntry::Text( + "No modulation period (connect modulation or the EXT_TRIGGER)".into(), + )), + } + let anchor = if self.is_marker_anchored() { + format!( + "{} phase-0 markers (trigger-anchored)", + self.camera_markers_us.len() + ) + } else { + "free-running (no EXT_TRIGGER)".into() + }; + entries.push(StatusEntry::Text(format!( + "{} events, {} valid pixels; {anchor}", + self.camera_events.len(), + self.valid_pixels + ))); + entries.push(StatusEntry::Text(match self.measured_a() { + Some(a) => format!("a = {a:.3} (photodiode)"), + None => { + let detail = self + .photodiode + .as_ref() + .map(|summary| connection_label(&summary.connection)) + .unwrap_or("no snapshot"); + format!("a = — (photodiode: {detail})") + } + })); + if let Some((on, off)) = self.latest_rolling() { + entries.push(StatusEntry::Text(format!( + "S_on = {on:.4}, S_off = {off:.4} (events/pixel per T/2)" + ))); + } + let source = if self.windows_are_frozen() { + "pilot-frozen" + } else { + "auto" + }; + let windows = self.current_windows().map_or_else( + || "windows —".into(), + |(on, off)| { + format!( + "windows ({source}) ON [{:.2},{:.2}) OFF [{:.2},{:.2})", + on.start, on.end, off.start, off.end + ) + }, + ); + let valid = self + .valid_pixel_count() + .map_or_else(|| "—".into(), |n| n.to_string()); + entries.push(StatusEntry::Text(format!( + "Response curve: {windows}, N_valid = {valid}, {} point(s)", + self.response_points.len() + ))); + if let Some((q0_on, q0_off)) = self.background_floor { + entries.push(StatusEntry::Text(format!( + "Background floor: q0_on = {q0_on:.3}, q0_off = {q0_off:.3}" + ))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + fn column(id: &str, title: &str) -> TableColumn { + TableColumn { + id: id.into(), + title: title.into(), + value_type: TableValueType::String, + } + } + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "A1 status".into(), + kind: HostDatasetKind::TableV1(TableSchema { + columns: vec![ + column("state", "Recording"), + column("measurement_id", "Measurement id"), + column("remaining", "Remaining"), + column("frequency", "Frequency"), + column("a", "a (photodiode)"), + column("s_on", "S_on"), + column("s_off", "S_off"), + column("events", "Events"), + column("message", "Message"), + ], + ..TableSchema::default() + }), + empty_message: "A1 idle".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: ROLLING_DATASET_ID.into(), + title: "A1 rolling response S_p(t) — live sanity check".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Enable Live analysis; waiting for events and a period".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: RESPONSE_CURVE_DATASET_ID.into(), + title: "A1 response probability q_p(a) — live quicklook".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Capture a pilot, then record points per amplitude".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "A1 status".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: ROLLING_VIEW_ID.into(), + title: "A1 rolling response S_p (ON/OFF)".into(), + dataset_id: ROLLING_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: RESPONSE_CURVE_VIEW_ID.into(), + title: "A1 response probability q_p (ON/OFF)".into(), + dataset_id: RESPONSE_CURVE_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + ], + actions: Vec::new(), + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + ROLLING_DATASET_ID => serde_json::to_vec(&self.rolling_dataset()).ok(), + RESPONSE_CURVE_DATASET_ID => serde_json::to_vec(&self.response_curve_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + matches!( + dataset_id, + STATUS_DATASET_ID | ROLLING_DATASET_ID | RESPONSE_CURVE_DATASET_ID + ) + .then_some(self.dataset_generation) + .unwrap_or(0) + } +} + +fn connection_label(connection: &ConnectionStateV1) -> &'static str { + match connection { + ConnectionStateV1::Connected { .. } => "connected", + ConnectionStateV1::Connecting => "connecting", + ConnectionStateV1::Disconnected => "disconnected", + ConnectionStateV1::Faulted { .. } => "faulted", + } +} + +export_plugin!(StageAA1Plugin); + +#[cfg(test)] +mod tests { + use stage_a_plugin_contract::{ + OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, RequestOutcomeV1, + ResponseCommonV1, Sha256V1, StreamIntegrityV1, CONTRACT_VERSION_V1, + }; + + use super::*; + + #[derive(Default)] + struct ControlSink { + services: Vec, + hosts: Vec, + } + + impl RecordingControl for ControlSink { + fn request_service(&mut self, request: &PluginServiceRequest) { + self.services.push(request.clone()); + } + + fn request_host(&mut self, request: &HostCommandRequest) { + self.hosts.push(request.clone()); + } + } + + fn control_tick( + plugin: &mut StageAA1Plugin, + inbox: PluginControlInbox, + sink: &mut ControlSink, + ) { + for reply in &inbox.host_replies { + plugin.on_host_reply(reply); + } + for reply in &inbox.service_replies { + plugin.on_service_reply(reply); + } + plugin.drive_recording(sink); + } + + fn pd_reply(request_id: u64, receipt: Option) -> PluginServiceReply { + let response = PhotodiodeResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: RequestId(request_id), + owner_instance: OwnerInstanceId::new("pd-test"), + run_id: None, + requested_revision: None, + acknowledged_revision: None, + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(now_unix_ms()), + error: None, + }, + receipt, + }; + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Accepted { + payload: serde_json::to_value(response).expect("response"), + }, + } + } + + fn on(timestamp_us: u64) -> CameraEvent { + CameraEvent { + timestamp_us, + x: 0, + y: 0, + polarity: Polarity::On, + } + } + + /// A plugin whose period comes from marker spacing (no fallback frequency). + fn plugin_with_markers() -> StageAA1Plugin { + StageAA1Plugin { + valid_pixels: 10, + camera_markers_us: vec![0, 1_000, 2_000, 3_000], + ..StageAA1Plugin::default() + } + } + + #[test] + fn period_comes_from_the_trigger_marker_spacing() { + let plugin = plugin_with_markers(); + let period = plugin.period_us().expect("measured period"); + assert!((period - 1_000.0).abs() < 1e-6, "period={period}"); + assert_eq!(plugin.frequency_source(), "trigger"); + } + + #[test] + fn no_markers_and_no_modulation_yields_no_period() { + let plugin = StageAA1Plugin::default(); + assert!(plugin.period_us().is_none()); + assert!(plugin.rolling_dataset().lines[0].points.is_empty()); + } + + #[test] + fn external_triggers_anchor_the_fold() { + let mut plugin = plugin_with_markers(); + for cycle in 0..3 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + assert!(plugin.is_marker_anchored()); + let fold = plugin.current_fold().expect("marker fold"); + assert_eq!(fold.validation.cycle_count, 3); + assert!((fold.events[0].phase - 0.2).abs() < 1e-9); + } + + #[test] + fn rolling_dataset_keeps_on_and_off_separate() { + let mut plugin = plugin_with_markers(); + for cycle in 0..3 { + let base = cycle * 1_000; + plugin.camera_events.push(on(base + 100)); + plugin.camera_events.push(CameraEvent { + polarity: Polarity::Off, + ..on(base + 600) + }); + } + let rolling = plugin.rolling_dataset(); + assert_eq!(rolling.lines.len(), 2); + assert_eq!(rolling.lines[0].name, "ON"); + assert!(rolling.lines[0].points.len() >= 2); + } + + #[test] + fn response_curve_auto_windows_without_a_pilot_and_refuses_without_a() { + let mut plugin = plugin_with_markers(); + plugin.frame_width = 8; + plugin.frame_height = 1; + for cycle in 0..20 { + let base = cycle * 1_000; + for x in 0..4 { + plugin.camera_events.push(CameraEvent { + timestamp_us: base + 200, + x, + y: 0, + polarity: Polarity::On, + }); + plugin.camera_events.push(CameraEvent { + timestamp_us: base + 700, + x, + y: 0, + polarity: Polarity::Off, + }); + } + } + plugin.camera_markers_us = (0..=20).map(|c| c * 1_000).collect(); + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 4, + height: 1, + }); + + // Windows come straight from the fold — no pilot capture needed. + let (q_on, q_off, _, valid) = plugin.current_response().expect("response"); + assert_eq!(valid, 4); + assert!(q_on > 0.9 && q_off > 0.9, "q_on={q_on} q_off={q_off}"); + // Recording a point is still refused without a photodiode-measured a. + assert!(plugin.measured_a().is_none()); + assert!(plugin.record_response_point().is_err()); + } + + #[test] + fn press_latch_distinguishes_clicks_baselines_and_advances() { + let mut latch = PressLatch::default(); + // Direct click on this instance: an edge, and the counter advances. + assert!(latch.accept(&json!(true))); + assert_eq!(latch.value(), json!(1)); + // `false` writes (legacy snapshots) are never edges. + assert!(!latch.accept(&json!(false))); + + // A fresh instance adopts the first forwarded counter silently… + let mut worker = PressLatch::default(); + assert!(!worker.accept(&json!(3))); + // …repeats are not edges… + assert!(!worker.accept(&json!(3))); + // …and only an advance is one press. + assert!(worker.accept(&json!(4))); + assert!(!worker.accept(&json!(4))); + } + + #[test] + fn forwarded_button_counter_latches_the_recording_role() { + let mut plugin = StageAA1Plugin::default(); + // First snapshot after (re)load: adopt the mirror's counter, no press. + plugin + .set_setting("start_recording", json!(2)) + .expect("baseline"); + assert!(plugin.pending_role.is_none()); + // The mirror's counter advances by one click → one press edge. + plugin + .set_setting("start_recording", json!(3)) + .expect("press"); + assert_eq!(plugin.pending_role, Some(RecRole::Normal)); + // Re-applying the same snapshot must not re-press. + plugin.pending_role = None; + plugin + .set_setting("start_recording", json!(3)) + .expect("repeat"); + assert!(plugin.pending_role.is_none()); + } + + #[test] + fn recording_orders_camera_then_pdq_and_saves_inside_the_measurement_folder() { + let folder = std::env::temp_dir().join(format!("a1-lifecycle-{}", now_unix_ms())); + let mut plugin = StageAA1Plugin { + output_folder: folder.display().to_string(), + measurement_id: "A1-row".into(), + duration_s: 1, + pending_role: Some(RecRole::Normal), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!(plugin.recording.phase, RecPhase::StartingCamera); + assert_eq!(sink.hosts.len(), 1); + assert!(sink.services.is_empty(), "PDQ must not start before camera"); + let cam_start_req = sink.hosts[0].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_start_req, + outcome: HostCommandOutcome::RecordingStarted { + actual_raw_path: "/camera/A1-row/run.raw".into(), + started_at: "2026-07-23T00:00:00Z".into(), + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let connect = sink.services.last().expect("connect request"); + let connect_envelope: PhotodiodeRequestV1 = + serde_json::from_value(connect.payload.clone()).expect("connect envelope"); + assert!(matches!( + connect_envelope.command, + PhotodiodeCommandV1::Connect + )); + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(connect.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let acquire = sink.services.last().expect("lease request"); + let acquire_envelope: PhotodiodeRequestV1 = + serde_json::from_value(acquire.payload.clone()).expect("lease envelope"); + assert!(matches!( + acquire_envelope.command, + PhotodiodeCommandV1::AcquireLease { .. } + )); + assert_eq!( + acquire_envelope.run_id.as_ref().map(RunId::as_str), + Some(plugin.recording.stem.as_str()) + ); + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(acquire.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let begin = sink.services.last().expect("begin request"); + let begin_envelope: PhotodiodeRequestV1 = + serde_json::from_value(begin.payload.clone()).expect("begin envelope"); + assert!(matches!( + begin_envelope.command, + PhotodiodeCommandV1::BeginRecording { .. } + )); + assert_eq!(begin_envelope.requested_revision, Some(SemanticRevision(1))); + assert_eq!(plugin.recording.start_unix_ms, 0); + + let run_id = begin_envelope.run_id.expect("run id"); + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply( + begin.request_id, + Some(PdqReceiptV1::Started(PdqStartedReceiptV1 { + run_id: run_id.clone(), + pdq_path: "/pd/A1-row/run_pd.pdq".into(), + sidecar_path: "/pd/A1-row/run_pd.json".into(), + opened_at_unix_ms: now_unix_ms(), + stream_epoch: 1, + first_sample_index: Some(0), + })), + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert_eq!(plugin.recording.phase, RecPhase::Running); + assert!(plugin.recording.start_unix_ms > 0); + + plugin.recording.start_unix_ms = now_unix_ms().saturating_sub(1_000); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let release = sink.services.last().expect("release request"); + let release_envelope: PhotodiodeRequestV1 = + serde_json::from_value(release.payload.clone()).expect("release envelope"); + assert!(matches!( + release_envelope.command, + PhotodiodeCommandV1::ReleaseLease { + finalize_recording: true, + .. + } + )); + assert_eq!(sink.hosts.len(), 1, "camera keeps running until PDQ closes"); + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply( + release.request_id, + Some(PdqReceiptV1::Finalized(PdqFinalizedReceiptV1 { + run_id, + pdq_path: "/pd/A1-row/run_pd.pdq".into(), + sidecar_path: "/pd/A1-row/run_pd.json".into(), + opened_at_unix_ms: now_unix_ms().saturating_sub(1_000), + finalized_at_unix_ms: now_unix_ms(), + file_size_bytes: 64, + sha256: Sha256V1::parse("ab".repeat(32)).expect("sha"), + frames_written: 1, + sample_frames_written: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + segment_count: 1, + integrity: StreamIntegrityV1::default(), + termination: stage_a_plugin_contract::PdqTerminationV1::OperatorStopped, + valid: true, + })), + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert_eq!(plugin.recording.phase, RecPhase::StoppingCamera); + assert_eq!(sink.hosts.len(), 2); + let cam_stop_req = sink.hosts[1].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_stop_req, + outcome: HostCommandOutcome::RecordingFinalized { + actual_raw_path: "/camera/A1-row/run.raw".into(), + size: 128, + sha256: "cd".repeat(32), + duration_us: 1_000_000, + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert_eq!(plugin.recording.phase, RecPhase::Idle); + let measurement_dir = folder.join("A1-row"); + let sidecars: Vec<_> = std::fs::read_dir(&measurement_dir) + .expect("measurement folder") + .flatten() + .map(|entry| entry.path()) + .collect(); + assert_eq!(sidecars.len(), 1); + assert!(sidecars[0] + .file_name() + .is_some_and(|name| name.to_string_lossy().ends_with("_config.toml"))); + assert!(plugin.message.starts_with("Saved recording A1-row")); + + std::fs::remove_dir_all(folder).expect("cleanup"); + } + + #[test] + fn duplicate_or_jittery_markers_still_yield_a_fold() { + // A long trigger dropout leaves a gap far beyond the jitter tolerance: + // marker validation rejects the fold, but the quicklook must fall back + // to the free-running fold instead of blanking. + let mut plugin = StageAA1Plugin { + valid_pixels: 10, + camera_markers_us: vec![0, 1_000, 2_000, 10_000], + ..StageAA1Plugin::default() + }; + for cycle in 0..10 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + let fold = plugin.current_fold().expect("fallback fold"); + assert!(fold.markers_us.is_empty(), "free-running fold expected"); + assert!(!plugin.rolling_dataset().lines[0].points.is_empty()); + } + + #[test] + fn sweep_points_span_the_range_inclusively() { + let plugin = StageAA1Plugin { + min_a: 0.5, + max_a: 2.5, + sweep_count: 5, + ..StageAA1Plugin::default() + }; + let points = plugin.sweep_points(); + assert_eq!(points.len(), 5); + assert!((points[0] - 0.5).abs() < 1e-12); + assert!((points[4] - 2.5).abs() < 1e-12); + assert!((points[2] - 1.5).abs() < 1e-12); + } + + #[test] + fn sweep_point_recordings_carry_the_requested_a_in_the_sidecar() { + let mut plugin = plugin_with_markers(); + plugin.min_a = 0.5; + plugin.max_a = 1.5; + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + points: vec![0.5, 1.0, 1.5], + index: 1, + lease_id: LeaseId::new("a1-sweep-test"), + lease_granted: true, + lease_req: 0, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: true, + last_activity_ms: 0, + stop_requested: false, + }); + plugin.recording.id = "A1-sweeprow".into(); + plugin.recording.stem = "A1-sweeprow_20260723-000000_p02".into(); + plugin.recording.folder = std::env::temp_dir().display().to_string(); + plugin.recording.duration_s = 5; + plugin.recording.start_unix_ms = 1_774_224_000_000; + let path = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&path).expect("read sidecar"); + assert!(text.contains("requested_a = 1.0"), "sidecar: {text}"); + assert!(text.contains("point_index = 2")); + assert!(text.contains("point_total = 3")); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn settings_discontinuities_keep_the_response_curve() { + let mut plugin = StageAA1Plugin::default(); + plugin.response_points.push(ResponsePoint { + measured_a: 1.0, + q_on: 0.5, + q_off: 0.1, + cycles: 10, + valid_pixels: 4, + }); + plugin.on_discontinuity(PluginDiscontinuity::SettingsChanged); + assert_eq!(plugin.response_points.len(), 1); + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + assert!(plugin.response_points.is_empty()); + } + + #[test] + fn measurement_id_generation_is_file_safe_and_prefixed() { + let id = generate_measurement_id(); + assert!(id.starts_with("A1-")); + assert_eq!(sanitize_stem(&id), id); + assert_eq!(sanitize_stem("I_k 3 / f=10Hz"), "I_k_3_f_10Hz"); + } + + #[test] + fn compact_utc_formats_a_known_epoch() { + // 2026-07-23T00:00:00Z = 1_784_764_800 s; +3661 s = 01:01:01. + assert_eq!(format_compact_date(1_784_764_800), "20260723"); + assert_eq!(format_iso_utc(1_784_764_800), "2026-07-23T00:00:00Z"); + assert_eq!(format_compact_utc(1_784_764_800), "20260723-000000"); + assert_eq!( + format_iso_utc(1_784_764_800 + 3_661), + "2026-07-23T01:01:01Z" + ); + } + + #[test] + fn sidecar_serializes_the_expected_sections() { + let mut plugin = plugin_with_markers(); + plugin.frame_width = 4; + plugin.frame_height = 1; + plugin.recording.id = "A1-test".into(); + plugin.recording.stem = "A1-test_20260723-000000".into(); + plugin.recording.folder = std::env::temp_dir().display().to_string(); + plugin.recording.duration_s = 5; + plugin.recording.start_unix_ms = 1_774_224_000_000; + plugin.recording.cam_finalized_path = Some("/data/A1-test/A1-test.raw".into()); + plugin.recording.pd_pdq_path = Some("/pd/A1-test/A1-test_pd.pdq".into()); + let doc = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&doc).expect("read sidecar"); + assert!(text.contains("measurement_id = \"A1-test\"")); + assert!(text.contains("[modulation]")); + assert!(text.contains("[camera]")); + assert!(text.contains("[files]")); + assert!(text.contains("camera_config_sidecar = \"/data/A1-test/A1-test.toml\"")); + let _ = std::fs::remove_file(&doc); + } + + #[test] + fn pilot_windows_round_trip_through_the_folder() { + let dir = std::env::temp_dir().join(format!("a1-pilot-{}", now_unix_ms())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let folder = dir.display().to_string(); + + let mut plugin = plugin_with_markers(); + plugin.output_folder = folder.clone(); + plugin.measurement_id = "A1-row".into(); + plugin.pilot_windows = Some(( + PhaseWindow { + start: 0.10, + end: 0.30, + }, + PhaseWindow { + start: 0.55, + end: 0.80, + }, + )); + // Write a pilot sidecar for the row. + plugin.recording.role = RecRole::Pilot; + plugin.recording.id = "A1-row".into(); + plugin.recording.stem = "A1-row_20260723-000000_pilot".into(); + plugin.recording.folder = folder.clone(); + plugin.write_sidecar().expect("pilot sidecar"); + + // A fresh plugin on the same folder+id auto-loads the frozen windows. + let mut other = StageAA1Plugin { + output_folder: folder.clone(), + measurement_id: "A1-row".into(), + ..StageAA1Plugin::default() + }; + other.scan_measurement_folder(); + let (on, off) = other.pilot_windows.expect("loaded windows"); + assert!((on.start - 0.10).abs() < 1e-9 && (off.end - 0.80).abs() < 1e-9); + assert!(other.windows_are_frozen()); + + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/plugins/stage-a-a1/src/types.rs b/plugins/stage-a-a1/src/types.rs new file mode 100644 index 0000000..91a39cf --- /dev/null +++ b/plugins/stage-a-a1/src/types.rs @@ -0,0 +1,26 @@ +/// Event-camera polarity. A1 always analyses ON and OFF separately. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Polarity { + On, + Off, +} + +impl Polarity { + pub const ALL: [Self; 2] = [Self::On, Self::Off]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::On => "on", + Self::Off => "off", + } + } +} + +/// The event fields needed by the pure A1 analysis. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CameraEvent { + pub timestamp_us: u64, + pub x: u16, + pub y: u16, + pub polarity: Polarity, +} diff --git a/plugins/stage-a-modulation/Cargo.toml b/plugins/stage-a-modulation/Cargo.toml index 983a927..57207f9 100644 --- a/plugins/stage-a-modulation/Cargo.toml +++ b/plugins/stage-a-modulation/Cargo.toml @@ -13,4 +13,5 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true stage-a-io = { path = "../../stage-a-io" } +stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } toml = "0.8" diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index b349ba6..bb29cab 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -5,14 +5,70 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** ## What it does -- **Power slider** in DAC codes (0–4095). Its upper bound is the **max limit** setting — set that - to the highest code the connected device tolerates and the slider physically cannot exceed it. -- **Mode**: `CONST` (hold the level), `SINE`, or `SQUARE` with a **frequency** (0.01–2000 Hz) and - a **min threshold** — the periodic waveforms swing between the threshold and the slider value. +- **Drive method** selects how the DAC operating band is defined: + - `MANUAL`: **Power** is the peak/operating code and **Min threshold** is the lower endpoint. + - `CALIBRATED`: `V_null`, `Vπ`, `I_k`, and optical depth `a` determine the endpoints. + Measure `V_null`/`Vπ` with the built-in transfer sweep — see [Calibration](#calibration--measuring-v_null-and-vπ). +- **Mode** independently selects the waveform that fills that band. All five modes are available + under both methods. +- **Max limit** is always visible and is the hard DAC ceiling for every drive. - Every accepted change is sent to the Teensy **immediately** (one `MOD` command); there is no Apply button. - The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and - a 2 Hz `STATUS` poll), not just what was commanded. + a 2 Hz `STATUS` poll), plus the selected method and resolved DAC band. + +| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `Vπ` | +|---|---|---| +| `CONST` | hold `power` | hold the DAC code for `I_k` | +| `DAC_SINE` | DAC sine across the band | DAC sine across the band | +| `SQUARE` | DAC square across the band | DAC square across the band | +| `OPTICAL_LOG_SINE` | optical log-sine across the band | optical log-sine about `I_k` | +| `OPTICAL_LINEAR_SINE` | optical linear-sine across the band | optical linear-sine about `I_k` | + +Manual optical modes reuse the stored `V_null`/`Vπ` lobe and derive their effective `(I_k, a)` +from the slider band through the forward optical transfer. + +In calibrated `CONST`, `a` is irrelevant: the hold is +`V_null + (2Vπ/π)·asin(sqrt(I_k))`. With `V_null=1630` and `Vπ=860`, this is +2490 at `I_k=1` and 1685 at `I_k=0.01`. Periodic modes still need optical +headroom and reject impossible `I_k`/`a` combinations without changing the +displayed setting or leaving it out of sync with the board. + +## Calibration — measuring `V_null` and `Vπ` + +Do not type these in from a datasheet. Static birefringence, alignment, PBS extinction, driver +gain, temperature, and the actual electrical load all enter the realised map, so measure them: + +1. Connect the command port **and** the photodiode plugin (the sweep reads its published level; + it needs no lease and takes no recording). +2. Set **Detector port**. Stage-A watches the PBS *reject* port, where the detector is + **brightest** at `V_null` — the default. This cannot be inferred from the sweep: a bright and + a dark extremum fit the measured curve equally well, and only the optics say which one is zero + excitation. Getting it wrong puts `V_null` a quarter wave out. +3. Press **Measure transfer curve**. It steps settled `CONST` codes across `0..max limit`, up and + back down (~20 s), and fits the lobe. Your armed drive is restored afterwards, on every exit + path. +4. Read the result in the **Pockels transfer curve** view and the status line, then press + **Apply to V_null / Vπ**. Anything questionable — a high residual, dropped points, + hysteresis, clipping — appears as a `Check:` line but does not block the apply: the plot is + the arbiter, and a single stray sample can inflate the residual fivefold while leaving `Vπ` + accurate to a few codes. Wild points are dropped from the fit automatically. + +The view also works *before* any measurement: it draws the lobe your current `V_null`/`Vπ` claim, +on a normalised axis, with markers at `V_null` and `V_null + Vπ`. + +Two properties worth knowing: + +- `V_null`/`Vπ` need **no** dark measurement and **no** total-power anchor — the fitted offset and + amplitude absorb the dark level and the front-end gain. +- The detector level at the null is reported as a **lower bound** on the total-power anchor + `I_tot`, *not* as the anchor. On the reject port the residual transmitted floor is not separable + from it; freezing a real anchor needs a transmitted-port power measurement. + +Set a **Calibration folder** to archive each applied calibration (points, fit, residual, +hysteresis) and stamp `calibration_id` into the state snapshot, so recordings can cite the +inversion they used. Full detail: [feature brief](../../docs/features/stage-a-pockels-calibration.md), +[ADR 011](../../docs/adr/011-stage-a-pockels-transfer-calibration.md). ## Connecting @@ -20,8 +76,9 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** **without a running camera** (device I/O lives in a plugin-owned thread, independent of the host's frame-driven plugin passes). Connecting never changes the output; only changes made while connected are transferred. -- **Output off = power slider at 0.** The firmware output is **set-and-hold**: disconnecting, - closing the GUI, or a crash leaves the last modulation running (`stage-a-controller` ADR 002). +- The firmware output is **set-and-hold**: disconnecting, closing the GUI, or a crash leaves the + last modulation running (`stage-a-controller` ADR 002). In Manual mode, Power `0` drives `0 V`; + automated workflows use their explicit `SafeOff` command. ## Ports @@ -32,3 +89,17 @@ controller for hardware-free testing. Replaying a recording disconnects the plugin defensively; live control itself needs no capture session. + +## Workflow-owner service + +This plugin is the sole command-port owner for manual operation and automated Stage-A workflows. +The live-worker instance exposes `stage_a.modulation.control.v1` under the stable plugin ID +`stage-a.modulation`; UI-mirror and offline instances never open the port or apply hardware +effects. Automated clients acquire a renewable lease and submit semantic, idempotent commands +(`SetWaveform`, `PrepareA1`, `StartAcquisition`, `StopAcquisition`, `SafeOff`) rather than changing +UI settings or sending raw firmware strings. While leased, manual control settings are locked. + +The bounded `stage_a.modulation_state.v1` snapshot keeps requested and board-acknowledged semantic +revisions separate. Lease expiry, replay/effects revocation, or owner shutdown during an automated +run performs a best-effort controller `STOP` followed by `MOD wave=OFF` before releasing the port. +Automation specifies exact waveforms and therefore does not use the UI Drive method. diff --git a/plugins/stage-a-modulation/plugin.toml b/plugins/stage-a-modulation/plugin.toml index 39b9d48..059aa60 100644 --- a/plugins/stage-a-modulation/plugin.toml +++ b/plugins/stage-a-modulation/plugin.toml @@ -1,6 +1,7 @@ +id = "stage-a.modulation" name = "Stage-A Modulation" -version = "0.3.0" -description = "Laser modulation control: capped power slider plus constant/sine/square drive of the Teensy DAC (J23), applied immediately." +version = "0.4.0" +description = "Laser modulation control: capped power slider plus constant/sine/square/optical drive of the Teensy DAC (J23), applied immediately, with a measured Pockels transfer calibration for V_null/Vπ." domain = "stage-a" library = "augur_plugin_stage_a_modulation" phase = "frame_only" diff --git a/plugins/stage-a-modulation/src/calibration.rs b/plugins/stage-a-modulation/src/calibration.rs new file mode 100644 index 0000000..9bddcfe --- /dev/null +++ b/plugins/stage-a-modulation/src/calibration.rs @@ -0,0 +1,786 @@ +//! Measured Pockels/PBS transfer calibration: fits `V_null` and `Vπ` from a +//! sweep of settled `CONST` DAC codes against the photodiode level. +//! +//! The operator must not have to trust a nominal `Vπ` (knowledge base: +//! `methodology/pockels-waveform-linearisation.md` §4). This module turns a +//! table of `(DAC code, detector volts)` points into the lobe parameters the +//! optical inversion in [`crate::waveform`] needs. +//! +//! # Model +//! +//! ```text +//! P(c) = p0 + p1 · sin²(π (c − V_null) / (2 Vπ)) +//! ``` +//! +//! `p1` is **signed**, because the Stage-A photodiode sits behind the PBS +//! *reject* port and measures the complement `I_pd = I_tot − I_exc`, moving +//! *against* the excitation (knowledge base: `setup/optical-path.md`). +//! +//! The sign cannot be inferred from the sweep. `sin²` is symmetric about its +//! peak, so `(v, p0, p1)` and `(v + Vπ, p0 + p1, −p1)` describe the *same* +//! measured curve exactly; the data alone cannot say which extremum is the +//! excitation null. That is a physical fact about the port, not a fit +//! parameter, so [`fit_transfer`] takes the geometry as an **input** and picks +//! the matching representation. Getting it wrong would place `V_null` a +//! quarter wave off and run the drive on the inverted branch, so it is asked +//! rather than guessed. +//! +//! Two consequences worth stating, because they remove procedure rather than +//! add it: +//! +//! - **The shape is dark- and gain-immune.** `p0` absorbs the dark level and +//! any DC offset, `p1` absorbs the front-end gain. `V_null`/`Vπ` therefore +//! need neither a dark measurement nor the total-power anchor. +//! - **The absolute scale is not recoverable here.** On the reject port the +//! residual transmitted floor cannot be separated from the anchor `I_tot` +//! (knowledge base §4.4), so this module reports the detector extrema and +//! explicitly does *not* derive a maximum achievable `a` from them. +//! +//! # Fit +//! +//! Because `sin²(x) = (1 − cos 2x)/2`, the model is exactly a constant plus +//! **one sinusoid of period `2Vπ`** — and a sinusoid of known period is linear +//! in its quadrature components. So for each candidate `Vπ` the phase (hence +//! `V_null`) and both amplitudes fall out of a 3×3 linear solve, and the +//! search is one-dimensional: scan `Vπ` over every period the sweep can +//! resolve, then refine. See [`solve_harmonic`]. +//! +//! This matters beyond elegance. Seeding the period from the measured extrema +//! — the obvious approach — breaks on exactly the sweeps that matter: with a +//! real `Vπ` near 860 the DAC range holds ~2.4 lobes, so the global minimum +//! and maximum can sit whole periods apart and the seed is meaningless. + +use std::f64::consts::PI; + +use crate::waveform::{LobeInversion, DAC_FULL_SCALE}; + +/// Sweep direction, kept per point so ascending/descending repeatability can +/// be reported (knowledge base §5 acceptance test 1). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Direction { + Ascending, + Descending, +} + +impl Direction { + pub fn label(self) -> &'static str { + match self { + Self::Ascending => "up", + Self::Descending => "down", + } + } +} + +/// One settled `(DAC code, detector level)` measurement. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SweepPoint { + pub code: u16, + pub direction: Direction, + /// Raw detector level in volts, as published by the photodiode owner. + pub volts: f64, + /// Spread over the averaged window; a settle-quality witness. + pub peak_to_peak_volts: f64, + pub clipped: bool, +} + +/// Which port the detector watches. An input to the fit, not an output: the +/// swept curve is identical either way (see the module docs), so this states +/// the bench geometry that resolves the ambiguity. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DetectorGeometry { + /// Detector darkens as excitation rises — the Stage-A PBS reject port, and + /// the default: on this bench the geometry is settled by construction. + RejectedComplement, + /// Detector brightens with excitation (a transmitted-port tap). + Direct, +} + +impl DetectorGeometry { + pub const VARIANTS: [Self; 2] = [Self::RejectedComplement, Self::Direct]; + + /// Named by what the operator can *observe*, not by optics jargon: the + /// question the setting actually asks is which way the photodiode reading + /// moves when the light reaching the sample gets brighter. + pub fn name(self) -> &'static str { + match self { + Self::RejectedComplement => "REJECT PORT (PD falls as light rises)", + Self::Direct => "DIRECT (PD rises with light)", + } + } + + pub fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|kind| kind.name() == name) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TransferFit { + /// DAC code at the excitation minimum. + pub v_null_dac: f64, + /// DAC codes from `v_null` to the excitation maximum (quarter wave). + pub v_pi_dac: f64, + /// Detector volts at the excitation null (`p0`). + pub offset_volts: f64, + /// Signed detector span across one lobe (`p1`); negative on the reject port. + pub span_volts: f64, + pub rms_residual_volts: f64, + /// Residual as a fraction of the detector span — the headline fit quality. + pub quality: f64, + pub geometry: DetectorGeometry, + /// Mean |ascending − descending| at matched codes, as a fraction of the + /// span. `None` when the sweep ran in one direction only. + pub hysteresis: Option, + /// Fraction of one full lobe (`Vπ` codes) the sweep actually covered. + /// Below ~1 the quarter-wave distance is extrapolated, not measured. + pub lobe_coverage: f64, + /// Points discarded as wild before the final fit. A couple is ordinary; a + /// large share means the sweep, not the model, is the problem. + pub rejected_points: usize, + /// Every measured point, rejected ones included, so the plot shows what was + /// actually seen. + pub points: Vec, +} + +impl TransferFit { + pub fn inversion(&self) -> LobeInversion { + LobeInversion { + v_null_dac: self.v_null_dac, + v_pi_dac: self.v_pi_dac, + } + } + + /// Detector extremum at the excitation null. On the reject port this is the + /// detector *maximum* and a **lower bound** on the total-power anchor + /// `I_tot` — not the anchor itself, because the residual transmitted floor + /// is not separable here (knowledge base §4.4). + pub fn detector_volts_at_null(&self) -> f64 { + self.offset_volts + } + + /// Detector extremum at the excitation maximum. + pub fn detector_volts_at_peak(&self) -> f64 { + self.offset_volts + self.span_volts + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FitError { + /// Fewer points than parameters can be resolved from. + TooFewPoints { count: usize, minimum: usize }, + /// The detector never moved: no lobe to fit (light blocked, no drive + /// reaching the cell, or the sweep span sits in a flat region). + NoModulation, + /// A fitted lobe exists but no `[V_null, V_null+Vπ]` fits inside the + /// commandable range, so no monotonic branch is usable. + NoLobeInRange { v_pi_dac: f64 }, +} + +impl std::fmt::Display for FitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooFewPoints { count, minimum } => { + write!(f, "only {count} sweep points (minimum {minimum})") + } + Self::NoModulation => f.write_str( + "the detector level did not change across the sweep; check the light path, \ + the HV amplifier, and that the photodiode is connected", + ), + Self::NoLobeInRange { v_pi_dac } => write!( + f, + "fitted Vπ = {v_pi_dac:.0} DAC codes leaves no full lobe inside the max limit; \ + raise the max limit or re-check the drive gain" + ), + } + } +} + +impl std::error::Error for FitError {} + +/// Smallest usable sweep: four points per fitted parameter. +pub const MIN_POINTS: usize = 16; +/// A detector span below this is treated as noise rather than a lobe. +const MIN_SPAN_VOLTS: f64 = 0.01; + +/// Least-squares solution for one candidate quarter wave `w`. +struct Harmonic { + /// Mean level `A`, and the quadrature amplitudes of `cos`/`sin(πc/w)`. + mean: f64, + amplitude: f64, + phase: f64, + sse: f64, +} + +/// Fits `P = A + B·cos(πc/w) + C·sin(πc/w)` for a fixed `w`. +/// +/// This is the whole trick that makes the search one-dimensional. Because +/// `sin²(x) = (1 − cos 2x)/2`, the lobe model +/// `p0 + p1·sin²(π(c − v)/(2w))` is *exactly* a constant plus one sinusoid of +/// period `2w` — and a sinusoid of known period is **linear** in its +/// quadrature components. So `V_null` (a phase) and both amplitudes drop out +/// of a 3×3 normal-equation solve, and only `Vπ` is ever searched. No seeding +/// from measured extrema, which is what fails once a sweep spans several +/// lobes and the global extrema sit periods apart. +fn solve_harmonic(points: &[SweepPoint], w: f64) -> Harmonic { + let n = points.len() as f64; + let (mut s_c, mut s_s, mut s_cc, mut s_ss, mut s_cs) = (0.0, 0.0, 0.0, 0.0, 0.0); + let (mut s_y, mut s_yc, mut s_ys) = (0.0, 0.0, 0.0); + for point in points { + let theta = PI * f64::from(point.code) / w; + let (sin, cos) = theta.sin_cos(); + s_c += cos; + s_s += sin; + s_cc += cos * cos; + s_ss += sin * sin; + s_cs += cos * sin; + s_y += point.volts; + s_yc += point.volts * cos; + s_ys += point.volts * sin; + } + // Symmetric 3×3 normal equations for (A, B, C), solved by cofactors. + let m = [[n, s_c, s_s], [s_c, s_cc, s_cs], [s_s, s_cs, s_ss]]; + let rhs = [s_y, s_yc, s_ys]; + let cofactor = [ + m[1][1] * m[2][2] - m[1][2] * m[2][1], + m[1][2] * m[2][0] - m[1][0] * m[2][2], + m[1][0] * m[2][1] - m[1][1] * m[2][0], + ]; + let determinant = m[0][0] * cofactor[0] + m[0][1] * cofactor[1] + m[0][2] * cofactor[2]; + if determinant.abs() < 1e-12 { + return Harmonic { + mean: s_y / n, + amplitude: 0.0, + phase: 0.0, + sse: f64::MAX, + }; + } + let solve = |column: usize| { + let mut augmented = m; + for row in 0..3 { + augmented[row][column] = rhs[row]; + } + (augmented[0][0] * (augmented[1][1] * augmented[2][2] - augmented[1][2] * augmented[2][1]) + - augmented[0][1] + * (augmented[1][0] * augmented[2][2] - augmented[1][2] * augmented[2][0]) + + augmented[0][2] + * (augmented[1][0] * augmented[2][1] - augmented[1][1] * augmented[2][0])) + / determinant + }; + let (a, b, c) = (solve(0), solve(1), solve(2)); + let sse = points + .iter() + .map(|point| { + let theta = PI * f64::from(point.code) / w; + let residual = point.volts - (a + b * theta.cos() + c * theta.sin()); + residual * residual + }) + .sum(); + Harmonic { + mean: a, + amplitude: b.hypot(c), + phase: c.atan2(b), + sse, + } +} + +/// Golden-section minimisation of `f` on `[lo, hi]`, used one axis at a time. +fn golden_min(lo: f64, hi: f64, tolerance: f64, f: impl Fn(f64) -> f64) -> f64 { + const INV_PHI: f64 = 0.618_033_988_749_895; + let (mut lo, mut hi) = (lo, hi); + let mut c = hi - (hi - lo) * INV_PHI; + let mut d = lo + (hi - lo) * INV_PHI; + let (mut fc, mut fd) = (f(c), f(d)); + while (hi - lo) > tolerance { + if fc < fd { + hi = d; + d = c; + fd = fc; + c = hi - (hi - lo) * INV_PHI; + fc = f(c); + } else { + lo = c; + c = d; + fc = fd; + d = lo + (hi - lo) * INV_PHI; + fd = f(d); + } + } + 0.5 * (lo + hi) +} + +/// Mean of the points at each distinct code, smoothed over three neighbours, so +/// the seed extrema are not chosen by a single noisy sample. +fn smoothed_profile(points: &[SweepPoint]) -> Vec<(f64, f64)> { + let mut codes: Vec = points.iter().map(|point| point.code).collect(); + codes.sort_unstable(); + codes.dedup(); + let means: Vec<(f64, f64)> = codes + .iter() + .map(|&code| { + let matching: Vec = points + .iter() + .filter(|point| point.code == code) + .map(|point| point.volts) + .collect(); + ( + f64::from(code), + matching.iter().sum::() / matching.len() as f64, + ) + }) + .collect(); + (0..means.len()) + .map(|index| { + let lo = index.saturating_sub(1); + let hi = (index + 2).min(means.len()); + let window = &means[lo..hi]; + ( + means[index].0, + window.iter().map(|(_, v)| v).sum::() / window.len() as f64, + ) + }) + .collect() +} + +/// Shifts `v` by whole lobe periods to the **lowest** null whose lobe +/// `[v, v + w]` fits inside `0..=max_code`. +/// +/// A sweep across several periods finds several equally valid nulls, so the +/// choice needs a rule the operator can predict rather than a nearest-match. +/// The lowest one drives the Pockels cell at the smallest codes — least +/// voltage across the crystal, most headroom under the max limit. +fn select_lobe(v: f64, w: f64, max_code: f64) -> Option { + // Sub-code precision is meaningless on a 12-bit DAC, so a null fitted a + // hair below 0 (or a peak a hair past the ceiling) is snapped into range + // rather than refused — otherwise a lobe nulling exactly at code 0 fails + // on fit noise alone. + const TOLERANCE: f64 = 1.0; + // The model repeats every `2w` in code, and `v + kw` for odd `k` is the + // same branch mirrored, so stepping by `2w` enumerates every null. + let period = 2.0 * w; + let mut candidate = v - period * ((v / period).floor() + 1.0); + while candidate <= max_code + TOLERANCE { + if candidate >= -TOLERANCE && candidate + w <= max_code + TOLERANCE { + return Some(candidate.clamp(0.0, (max_code - w).max(0.0))); + } + candidate += period; + } + None +} + +/// Mean |ascending − descending| at codes visited in both directions, as a +/// fraction of the detector span. +fn hysteresis_fraction(points: &[SweepPoint], span: f64) -> Option { + let mut differences = Vec::new(); + for up in points + .iter() + .filter(|point| point.direction == Direction::Ascending) + { + if let Some(down) = points + .iter() + .find(|point| point.direction == Direction::Descending && point.code == up.code) + { + differences.push((up.volts - down.volts).abs()); + } + } + if differences.is_empty() || span.abs() < f64::EPSILON { + return None; + } + Some(differences.iter().sum::() / differences.len() as f64 / span.abs()) +} + +/// Scans the quarter wave over every period the sweep could resolve, then +/// refines. Returns the best `(Vπ, harmonic)`. +fn fit_period(points: &[SweepPoint], swept_span: f64) -> Option<(f64, Harmonic)> { + // From four samples per lobe (below that the lobe is aliased) out to a + // lobe twice the swept span (a barely-curved arc). Log-spaced, because a + // fixed step wastes resolution at long periods and misses short ones. + let point_spacing = swept_span / points.len().max(2) as f64; + let w_min = (2.0 * point_spacing).max(1.0); + let w_max = (2.0 * swept_span).max(w_min * 1.5); + const SCAN_STEPS: usize = 600; + let log_step = (w_max / w_min).ln() / SCAN_STEPS as f64; + let mut best: Option<(f64, f64)> = None; // (sse, w) + for step in 0..=SCAN_STEPS { + let w = w_min * (log_step * step as f64).exp(); + let sse = solve_harmonic(points, w).sse; + if best.is_none_or(|(previous, _)| sse < previous) { + best = Some((sse, w)); + } + } + let (_, coarse_w) = best?; + // Refine inside one scan cell, where the SSE is unimodal. + let cell = coarse_w * log_step; + let w = golden_min( + (coarse_w - cell).max(w_min * 0.5), + coarse_w + cell, + 1e-3, + |candidate| solve_harmonic(points, candidate).sse, + ); + let harmonic = solve_harmonic(points, w); + Some((w, harmonic)) +} + +/// Points whose residual against `harmonic` is not wildly out of family. +/// +/// The cut is on the **median** absolute residual, not the mean or the +/// standard deviation: those are themselves dragged out by the very points +/// being looked for. `6 × median` is roughly 4σ for Gaussian noise, so ordinary +/// scatter survives untouched and only genuine strays are dropped. +fn without_outliers(points: &[SweepPoint], w: f64, harmonic: &Harmonic) -> Vec { + let residual = |point: &SweepPoint| { + let theta = PI * f64::from(point.code) / w; + point.volts - (harmonic.mean + harmonic.amplitude * (theta - harmonic.phase).cos()) + }; + let mut magnitudes: Vec = points.iter().map(|point| residual(point).abs()).collect(); + magnitudes.sort_by(f64::total_cmp); + let median = magnitudes[magnitudes.len() / 2]; + if median <= 0.0 { + return points.to_vec(); + } + let limit = 6.0 * median; + points + .iter() + .filter(|point| residual(point).abs() <= limit) + .copied() + .collect() +} + +/// Fits the lobe. `max_code` is the highest commandable DAC code (the drive's +/// max limit), which constrains which branch can be used; `geometry` resolves +/// the null/peak ambiguity the data cannot (see the module docs). +pub fn fit_transfer( + points: &[SweepPoint], + max_code: f64, + geometry: DetectorGeometry, +) -> Result { + if points.len() < MIN_POINTS { + return Err(FitError::TooFewPoints { + count: points.len(), + minimum: MIN_POINTS, + }); + } + let profile = smoothed_profile(points); + let min_volts = profile.iter().map(|(_, v)| *v).fold(f64::MAX, f64::min); + let max_volts = profile.iter().map(|(_, v)| *v).fold(f64::MIN, f64::max); + if max_volts - min_volts < MIN_SPAN_VOLTS { + return Err(FitError::NoModulation); + } + + let swept_lo = profile.first().map(|(code, _)| *code).unwrap_or(0.0); + let swept_hi = profile.last().map(|(code, _)| *code).unwrap_or(max_code); + let swept_span = (swept_hi - swept_lo).max(1.0); + + // A single stray point — one window caught mid-settle, one stream hiccup — + // barely moves the fitted period but inflates the RMS residual several + // fold. Fit once, drop the points the fit says are wild, and fit again on + // what is left, so the reported residual describes the curve rather than + // the worst sample. + let (w, harmonic, rejected_points) = { + let first = fit_period(points, swept_span).ok_or(FitError::NoModulation)?; + let kept = without_outliers(points, first.0, &first.1); + if kept.len() < points.len() && kept.len() >= MIN_POINTS { + match fit_period(&kept, swept_span) { + Some((w, harmonic)) => (w, harmonic, points.len() - kept.len()), + None => (first.0, first.1, 0), + } + } else { + (first.0, first.1, 0) + } + }; + // `A + R·cos(θ − φ)` with `θ = πc/w` is the same curve as + // `p0 + p1·sin²(π(c − v)/(2w))` with `|p1| = 2R`. Which of the two signs + // of `p1` applies — and therefore whether the null sits at the phase or a + // quarter wave past it — is the geometry question the data cannot answer. + let radius = harmonic.amplitude; + let (v, p0, p1) = match geometry { + DetectorGeometry::RejectedComplement => ( + harmonic.phase * w / PI, + harmonic.mean + radius, + -2.0 * radius, + ), + DetectorGeometry::Direct => ( + harmonic.phase * w / PI + w, + harmonic.mean - radius, + 2.0 * radius, + ), + }; + if p1.abs() < MIN_SPAN_VOLTS { + return Err(FitError::NoModulation); + } + + let v_null = select_lobe(v, w, max_code).ok_or(FitError::NoLobeInRange { v_pi_dac: w })?; + + // Over the points the fit actually used: dividing the kept residual by the + // full count would flatter the number. + let rms = (harmonic.sse / (points.len() - rejected_points).max(1) as f64).sqrt(); + Ok(TransferFit { + v_null_dac: v_null, + v_pi_dac: w, + offset_volts: p0, + span_volts: p1, + rms_residual_volts: rms, + quality: rms / p1.abs(), + geometry, + hysteresis: hysteresis_fraction(points, p1), + lobe_coverage: swept_span / w, + rejected_points, + // Every measured point is kept for the plot, rejected ones included: + // seeing the strays next to the fit is how the operator judges it. + points: points.to_vec(), + }) +} + +/// Ascending then descending sweep codes over `0..=max_code`. +pub fn sweep_codes( + max_code: u16, + points_per_pass: usize, + both_directions: bool, +) -> Vec<(u16, Direction)> { + let points_per_pass = points_per_pass.max(2); + let max_code = max_code.min(DAC_FULL_SCALE); + let ascending: Vec = (0..points_per_pass) + .map(|index| { + (f64::from(max_code) * index as f64 / (points_per_pass - 1) as f64).round() as u16 + }) + .collect(); + let mut codes: Vec<(u16, Direction)> = ascending + .iter() + .map(|&code| (code, Direction::Ascending)) + .collect(); + if both_directions { + codes.extend( + ascending + .iter() + .rev() + .map(|&code| (code, Direction::Descending)), + ); + } + codes +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Synthesizes a sweep of a known lobe as seen through a given port. + /// `noise` is a deterministic zig-zag, not an RNG, so failures reproduce. + fn synthetic_sweep( + v_null: f64, + v_pi: f64, + offset: f64, + span: f64, + max_code: u16, + noise: f64, + both_directions: bool, + ) -> Vec { + let lobe = LobeInversion { + v_null_dac: v_null, + v_pi_dac: v_pi, + }; + sweep_codes(max_code, 49, both_directions) + .into_iter() + .enumerate() + .map(|(index, (code, direction))| { + let u = lobe.u_for_dac(f64::from(code)); + let wobble = if index % 2 == 0 { noise } else { -noise }; + SweepPoint { + code, + direction, + volts: offset + span * u + wobble, + peak_to_peak_volts: 0.002, + clipped: false, + } + }) + .collect() + } + + #[test] + fn recovers_a_known_lobe_from_the_reject_port() { + // Reject port: detector is brightest (2.4 V) at the excitation null. + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.004, true); + let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement) + .expect("fits the lobe"); + + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null = {}", + fit.v_null_dac + ); + assert!( + (fit.v_pi_dac - 1_600.0).abs() < 10.0, + "Vπ = {}", + fit.v_pi_dac + ); + assert!(fit.span_volts < 0.0, "reject port darkens with excitation"); + assert!((fit.detector_volts_at_null() - 2.4).abs() < 0.02); + assert!(fit.quality < 0.01, "quality = {}", fit.quality); + // Both directions carry the same synthetic curve, so the only + // difference at matched codes is the alternating wobble. + assert!(fit.hysteresis.expect("both directions") < 0.01); + } + + #[test] + fn recovers_the_same_lobe_from_a_direct_detector() { + // Same physical lobe, opposite port: dim at the null, bright at peak. + let points = synthetic_sweep(300.0, 1_600.0, 0.2, 2.2, 4_095, 0.004, true); + let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::Direct).expect("fits the lobe"); + + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null = {}", + fit.v_null_dac + ); + assert!((fit.v_pi_dac - 1_600.0).abs() < 10.0); + assert!(fit.span_volts > 0.0, "direct detector brightens"); + } + + #[test] + fn geometry_selects_between_the_two_equivalent_representations() { + // One curve, two readings. Declaring the wrong port must move V_null by + // exactly a quarter wave — the failure this input exists to prevent. + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, false); + let reject = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + let direct = fit_transfer(&points, 4_095.0, DetectorGeometry::Direct).expect("fits"); + + assert!((reject.v_null_dac - 300.0).abs() < 5.0); + assert!( + ((direct.v_null_dac - reject.v_null_dac).abs() - reject.v_pi_dac).abs() < 10.0, + "direct = {}, reject = {}, Vπ = {}", + direct.v_null_dac, + reject.v_null_dac, + reject.v_pi_dac + ); + // Both describe the measured curve equally well; only the physics + // distinguishes them. + assert!((reject.rms_residual_volts - direct.rms_residual_volts).abs() < 1e-6); + } + + #[test] + fn resolves_a_sweep_spanning_several_lobes() { + // A real Vπ near 860 puts ~2.4 lobes inside the DAC range. Seeding the + // period from the global extrema fails here — they can sit whole + // periods apart — which is why the period is scanned, not seeded. + let points = synthetic_sweep(1_630.0, 860.0, 2.4, -2.2, 4_095, 0.003, true); + let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement) + .expect("fits a multi-lobe sweep"); + assert!((fit.v_pi_dac - 860.0).abs() < 10.0, "Vπ = {}", fit.v_pi_dac); + // Any null is a valid answer as long as it names a real one and the + // lobe it opens fits inside the range. + let offset = (fit.v_null_dac - 1_630.0).rem_euclid(2.0 * 860.0); + assert!( + offset.min(2.0 * 860.0 - offset) < 10.0, + "V_null = {} is not a null of the swept lobe", + fit.v_null_dac + ); + assert!(fit.v_null_dac >= 0.0 && fit.v_null_dac + fit.v_pi_dac <= 4_095.0); + assert!(fit.quality < 0.01, "quality = {}", fit.quality); + assert!(fit.lobe_coverage > 4.0, "coverage = {}", fit.lobe_coverage); + } + + #[test] + fn a_null_at_code_zero_is_not_lost_to_fit_noise() { + // V_null = 0 fits a hair either side of the rail; snapping sub-code + // slack into range is the difference between a usable calibration and + // a refusal. + let points = synthetic_sweep(0.0, 1_200.0, 2.4, -2.2, 4_095, 0.003, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert!(fit.v_null_dac.abs() < 2.0, "V_null = {}", fit.v_null_dac); + } + + #[test] + fn picks_a_lobe_that_fits_inside_the_max_limit() { + // Null at 2600 with Vπ = 1600 would put peak light at 4200, past the + // rail; the previous null one period down (2600 − 3200 < 0) does not + // fit either, so only a lower branch inside the range is acceptable. + let points = synthetic_sweep(1_000.0, 900.0, 2.4, -2.2, 4_095, 0.002, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert!(fit.v_null_dac >= 0.0); + assert!( + fit.v_null_dac + fit.v_pi_dac <= 4_095.0, + "peak light at {} leaves the rail", + fit.v_null_dac + fit.v_pi_dac + ); + } + + #[test] + fn refuses_a_flat_sweep() { + let points: Vec = sweep_codes(4_095, 49, false) + .into_iter() + .map(|(code, direction)| SweepPoint { + code, + direction, + volts: 1.5, + peak_to_peak_volts: 0.001, + clipped: false, + }) + .collect(); + assert_eq!( + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement), + Err(FitError::NoModulation) + ); + } + + #[test] + fn refuses_too_few_points() { + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, false); + assert!(matches!( + fit_transfer(&points[..4], 4_095.0, DetectorGeometry::RejectedComplement), + Err(FitError::TooFewPoints { .. }) + )); + } + + #[test] + fn reports_hysteresis_between_the_two_passes() { + // Descending runs 20 mV below ascending: a real hysteresis signature. + let mut points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, true); + for point in &mut points { + if point.direction == Direction::Descending { + point.volts -= 0.02; + } + } + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + let hysteresis = fit.hysteresis.expect("both directions"); + assert!( + (hysteresis - 0.02 / 2.2).abs() < 1e-3, + "hysteresis = {hysteresis}" + ); + } + + #[test] + fn single_direction_sweep_reports_no_hysteresis() { + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.002, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert_eq!(fit.hysteresis, None); + } + + #[test] + fn lobe_coverage_flags_an_extrapolated_quarter_wave() { + // Sweeping only to code 800 with Vπ = 1600 sees half a lobe. + let points = synthetic_sweep(0.0, 1_600.0, 2.4, -2.2, 800, 0.001, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert!(fit.lobe_coverage < 0.75, "coverage = {}", fit.lobe_coverage); + } + + #[test] + fn sweep_codes_span_the_range_in_both_directions() { + let codes = sweep_codes(4_000, 5, true); + let ascending: Vec = codes + .iter() + .filter(|(_, direction)| *direction == Direction::Ascending) + .map(|(code, _)| *code) + .collect(); + assert_eq!(ascending, [0, 1_000, 2_000, 3_000, 4_000]); + let descending: Vec = codes + .iter() + .filter(|(_, direction)| *direction == Direction::Descending) + .map(|(code, _)| *code) + .collect(); + assert_eq!(descending, [4_000, 3_000, 2_000, 1_000, 0]); + assert_eq!(sweep_codes(4_000, 5, false).len(), 5); + } +} diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 4c744ce..a77c6f2 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -1,10 +1,11 @@ //! Stage-A laser modulation control. //! //! Drives the laser modulation input (Hermit J23, `DAC1.4`/address 3) through -//! the firmware 0.3.0 `MOD` command. One power slider (DAC code) whose upper -//! bound is a user-set safety cap, a mode select (constant / sine / square) -//! with frequency and a lower threshold for the periodic modes — and every -//! accepted change is transferred to the Teensy immediately, no Apply button. +//! the firmware 0.3.0 `MOD` command. Two orthogonal settings define a drive: +//! the method selects a manually entered or optically calibrated DAC band, +//! while the mode selects the waveform that fills that band. A separate max +//! limit is the hard DAC ceiling for every drive. Every accepted change is +//! transferred to the Teensy immediately, with no Apply button. //! //! **Frame-independent by design.** The host only calls `process_frame()` //! while camera frames flow, so nothing here depends on it: connecting is a @@ -19,7 +20,10 @@ //! (`stage-a-controller` ADR 002): disconnecting does NOT switch the //! modulation off — drag the power slider to 0 to drive 0 V. -use std::collections::BTreeMap; +mod calibration; +mod waveform; + +use std::collections::{BTreeMap, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -28,55 +32,248 @@ use std::time::{Duration, Instant}; use augur_plugin_api::{ export_plugin, EventStoreHandle, ExecutionMode, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, - HostViewRegistry, PathDialogKind, Plugin, PluginFrame, SettingItem, SettingKind, - SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, - TableDatasetV1, TableSchema, TableValueType, + HostViewRegistry, PathDialogKind, Plugin, PluginControlContext, PluginControlSnapshot, + PluginFrame, PluginRuntimeRole, PluginServiceOutcome, PluginServiceReply, PluginServiceRequest, + Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{Command, MockController, StageAClient, Transport}; +use stage_a_io::{Command, DeviceEvent, MockController, StageAClient, Transport}; +use stage_a_plugin_contract::{ + A1AcquisitionConfigV1, ClientId, ConnectionStateV1, ControllerStateV1, FreshnessV1, LeaseId, + LeaseSnapshotV1, ModulationCommandV1, ModulationRequestV1, ModulationResponseV1, + ModulationStateV1, ModulationTargetV1, OwnerInstanceId, PhotodiodeLevelV1, PhotodiodeSummaryV1, + RequestOutcomeV1, ResponseCommonV1, RunId, SemanticRevision, ServiceErrorCodeV1, + ServiceErrorV1, SynchronizationV1, UnsyncedReasonV1, WaveformV1, CONTRACT_VERSION_V1, + CTX_STAGE_A_MODULATION_STATE_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, + PLUGIN_ID_STAGE_A_MODULATION, PLUGIN_ID_STAGE_A_PHOTODIODE, + SERVICE_STAGE_A_MODULATION_CONTROL_V1, +}; const STATUS_DATASET_ID: &str = "stage-a-modulation.status"; const STATUS_VIEW_ID: &str = "stage-a-modulation.status.view"; +const CURVE_DATASET_ID: &str = "stage-a-modulation.transfer-curve"; +const CURVE_VIEW_ID: &str = "stage-a-modulation.transfer-curve.view"; + +/// Codes measured per sweep pass. 49 points over the full range put a sample +/// every ~85 codes, ~19 per lobe at a typical Vπ of 860. +const SWEEP_POINTS_PER_PASS: usize = 49; +/// Samples the detector must have taken *after* a code was commanded before its +/// window counts as settled. At the firmware's 20 kSa/s that is 100 ms — enough +/// for the HV amplifier and the cell to arrive, proven from the sample clock +/// rather than assumed from a timer. +const SETTLE_SAMPLES: u64 = 2_000; +/// Give up on a point if no settled level arrives within this long. A stalled +/// photodiode stream must abort the sweep, not hang it. +const POINT_TIMEOUT: Duration = Duration::from_secs(5); +/// Warn (never block) above this residual, as a fraction of the detector span. +/// A clean bench sits near 1 %; a stray point or two reaches ~10 % while `Vπ` +/// stays good, which is why this warns rather than refuses. +const WARN_QUALITY: f64 = 0.05; +/// Warn above this ascending/descending disagreement, as a fraction of the span. +const WARN_HYSTERESIS: f64 = 0.05; const MAX_DAC_CODE: i64 = 4_095; const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(500); const DEVICE_LOOP_TICK: Duration = Duration::from_millis(10); +/// After this many serial requests failing in a row the device thread declares +/// the link dead and exits, so the owner can reap it and reconnect. A wedged +/// link that stays "up" otherwise swallows every queued command while the +/// settings UI keeps responding. +const DEVICE_MAX_CONSECUTIVE_ERRORS: u32 = 5; +/// Minimum spacing between automatic reconnect attempts after the device +/// thread died. +const RECONNECT_BACKOFF_MS: u64 = 2_000; +const REQUEST_CACHE_LIMIT: usize = 256; +const MIN_LEASE_TTL_MS: u64 = 250; +const MAX_LEASE_TTL_MS: u64 = 60_000; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Mode { Const, + /// Pure DAC sine (DAC_SINE): the firmware synthesises a sinusoid directly in + /// DAC codes. The optical output is the non-linear `sin²` of this drive. Sine, Square, + /// OPTICAL_LOG_SINE: the DAC is warped so the *optical* output is a + /// log-intensity sine (the clean A1 target). Requires the lobe inversion. + OpticalLogSine, + /// OPTICAL_LINEAR_SINE: the DAC is warped so the optical output is a + /// linear-intensity sine. + OpticalLinearSine, } impl Mode { - const VARIANTS: [Mode; 3] = [Mode::Const, Mode::Sine, Mode::Square]; + const VARIANTS: [Mode; 5] = [ + Mode::Const, + Mode::Sine, + Mode::Square, + Mode::OpticalLogSine, + Mode::OpticalLinearSine, + ]; fn name(self) -> &'static str { match self { Self::Const => "CONST", - Self::Sine => "SINE", + Self::Sine => "DAC_SINE", Self::Square => "SQUARE", + Self::OpticalLogSine => "OPTICAL_LOG_SINE", + Self::OpticalLinearSine => "OPTICAL_LINEAR_SINE", } } fn from_name(name: &str) -> Option { + // Accept the historical "SINE" alias for the pure DAC sine. + if name == "SINE" { + return Some(Self::Sine); + } Self::VARIANTS.into_iter().find(|m| m.name() == name) } fn is_periodic(self) -> bool { !matches!(self, Self::Const) } + + /// Firmware `wave` token. Optical modes upload a warp table and share the + /// `WARP` playback path. + fn wire_wave(self) -> &'static str { + match self { + Self::Const => "CONST", + Self::Sine => "SINE", + Self::Square => "SQUARE", + Self::OpticalLogSine | Self::OpticalLinearSine => "WARP", + } + } + + fn optical_target(self) -> Option { + match self { + Self::OpticalLogSine => Some(waveform::OpticalTarget::LogSine), + Self::OpticalLinearSine => Some(waveform::OpticalTarget::LinearSine), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DriveMethod { + Manual, + Calibrated, +} + +impl DriveMethod { + const VARIANTS: [Self; 2] = [Self::Manual, Self::Calibrated]; + + fn name(self) -> &'static str { + match self { + Self::Manual => "MANUAL", + Self::Calibrated => "CALIBRATED", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS + .into_iter() + .find(|method| method.name() == name) + } } /// State the device thread reports back for the UI (status entries, table). -#[derive(Default)] struct DeviceState { connected: bool, firmware: String, + capabilities: Vec, board_code: Option, board_mod: String, + /// Structured board-echoed modulation (`mod_wave`/`mod_level`/`mod_min`/ + /// `mod_freq_mhz` reply fields). Lets the published snapshot expose the + /// *operator-armed* drive to consumers (A1 derives its fallback + /// modulation period from it) — UI-driven MOD commands never populate the + /// service-path `acknowledged` target. + board_wave: Option, + board_level: Option, + board_min: Option, + board_freq_millihz: Option, last_error: Option, + controller_state: ControllerStateV1, + requested: Option, + acknowledged: Option, + last_response: Option, + last_device_update_unix_ms: u64, +} + +impl Default for DeviceState { + fn default() -> Self { + Self { + connected: false, + firmware: String::new(), + capabilities: Vec::new(), + board_code: None, + board_mod: String::new(), + board_wave: None, + board_level: None, + board_min: None, + board_freq_millihz: None, + last_error: None, + controller_state: ControllerStateV1::Unknown, + requested: None, + acknowledged: None, + last_response: None, + last_device_update_unix_ms: 0, + } + } +} + +impl DeviceState { + /// Board-echo view of the armed drive as a contract target (revision 0), + /// for the published snapshot when no service-path acknowledgement + /// exists. WARP (optical) drives report as `Periodic` — the consumers of + /// this fallback only need the modulation frequency. + fn board_echo_target(&self) -> Option { + let wave = self.board_wave.as_deref()?; + let level = || u16::try_from(self.board_level.unwrap_or(0)).unwrap_or(0); + let min = || u16::try_from(self.board_min.unwrap_or(0)).unwrap_or(0); + let waveform = match wave { + "OFF" => WaveformV1::Off, + "CONST" => WaveformV1::Constant { level_dac: level() }, + "SINE" | "WARP" => WaveformV1::Periodic { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Sine, + min_dac: min(), + max_dac: level(), + frequency_millihz: self.board_freq_millihz.unwrap_or(0), + }, + "SQUARE" => WaveformV1::Periodic { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Square, + min_dac: min(), + max_dac: level(), + frequency_millihz: self.board_freq_millihz.unwrap_or(0), + }, + _ => return None, + }; + Some(ModulationTargetV1 { + revision: SemanticRevision(0), + waveform: Some(waveform), + a1_configuration: None, + acquisition_running: self.controller_state == ControllerStateV1::Running, + board_dac_code: self.board_code.and_then(|code| u16::try_from(code).ok()), + firmware_configuration_revision: None, + }) + } +} + +#[derive(Clone)] +struct OperationMeta { + request_id: stage_a_plugin_contract::RequestId, + run_id: Option, + requested_revision: SemanticRevision, + target: ModulationTargetV1, + owner_instance: OwnerInstanceId, +} + +struct PendingOperation { + commands: Vec, + purpose: &'static str, + meta: Option, } /// Everything shared between the plugin (UI thread) and the device thread. @@ -84,8 +281,10 @@ struct SharedLink { state: Mutex, /// Latest not-yet-sent command; newer settings overwrite older ones so /// slider drags coalesce instead of queueing. - pending: Mutex>, + pending: Mutex>, + priority: Mutex>, stop: AtomicBool, + fail_closed_on_stop: AtomicBool, generation: AtomicU64, } @@ -94,7 +293,9 @@ impl SharedLink { Self { state: Mutex::new(DeviceState::default()), pending: Mutex::new(None), + priority: Mutex::new(None), stop: AtomicBool::new(false), + fail_closed_on_stop: AtomicBool::new(false), generation: AtomicU64::new(1), } } @@ -115,7 +316,7 @@ impl MockService { let link = stage_a_io::MockLink::new(); let stop = Arc::new(AtomicBool::new(false)); let thread_stop = Arc::clone(&stop); - let mut controller = MockController::new(link.device_end()); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); let join = std::thread::Builder::new() .name("stage-a-modulation-mock".into()) .spawn(move || { @@ -167,10 +368,15 @@ fn run_device(mut client: StageAClient, shared: Arc Ok(fields) => { let mut state = shared.state.lock().expect("device state lock"); state.connected = true; + state.controller_state = ControllerStateV1::SafeIdle; state.firmware = fields .get("firmware") .cloned() .unwrap_or_else(|| "unknown".into()); + state.capabilities = fields + .get("capabilities") + .map(|value| value.split(',').map(str::to_owned).collect()) + .unwrap_or_default(); let has_mod = fields .get("capabilities") .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); @@ -181,6 +387,7 @@ fn run_device(mut client: StageAClient, shared: Arc Err(err) => { let mut state = shared.state.lock().expect("device state lock"); state.connected = false; + state.controller_state = ControllerStateV1::Faulted; state.last_error = Some(format!("HELLO failed: {err}")); shared.bump(); return; @@ -189,18 +396,49 @@ fn run_device(mut client: StageAClient, shared: Arc shared.bump(); let mut last_status = Instant::now() - STATUS_POLL_INTERVAL; + let mut consecutive_errors = 0u32; while !shared.stop.load(Ordering::Relaxed) { - let pending = shared.pending.lock().expect("pending lock").take(); - if let Some(command) = pending { - let result = client.request(&command); - apply_reply(&shared, "MOD", result); + let priority = shared.priority.lock().expect("priority lock").take(); + let pending = priority.or_else(|| shared.pending.lock().expect("pending lock").take()); + if let Some(operation) = pending { + if execute_operation(&mut client, &shared, operation) { + consecutive_errors = 0; + } else { + consecutive_errors += 1; + } } else if last_status.elapsed() >= STATUS_POLL_INTERVAL { last_status = Instant::now(); let result = client.request(&Command::new("STATUS")); - apply_reply(&shared, "STATUS", result); + if result.is_ok() { + consecutive_errors = 0; + } else { + consecutive_errors += 1; + } + apply_status_reply(&shared, "STATUS", result); } else { + if let Ok(events) = client.poll_events() { + apply_device_events(&shared, events); + } std::thread::sleep(DEVICE_LOOP_TICK); } + if consecutive_errors >= DEVICE_MAX_CONSECUTIVE_ERRORS { + // The link is wedged (unplugged cable, stale fd): declare it dead + // so the owner reaps this thread and reconnects, instead of + // silently swallowing every queued command from here on. + let mut state = shared.state.lock().expect("device state lock"); + state.connected = false; + state.controller_state = ControllerStateV1::Faulted; + state.last_error = Some("serial link failed repeatedly — reconnecting".to_owned()); + drop(state); + shared.bump(); + return; + } + } + + if shared.fail_closed_on_stop.load(Ordering::Relaxed) { + let _ = client.request(&Command::new("STOP").field("reason", "owner_shutdown")); + let result = client.request(&Command::new("MOD").field("wave", "OFF")); + apply_status_reply(&shared, "SAFE_OFF", result); } let mut state = shared.state.lock().expect("device state lock"); @@ -208,7 +446,87 @@ fn run_device(mut client: StageAClient, shared: Arc shared.bump(); } -fn apply_reply( +/// Runs one queued operation; returns whether every command succeeded. +fn execute_operation( + client: &mut StageAClient, + shared: &SharedLink, + operation: PendingOperation, +) -> bool { + let mut merged = BTreeMap::new(); + let mut error = None; + for command in &operation.commands { + match client.request(command) { + Ok(fields) => merged.extend(fields), + Err(err) => { + error = Some(err.to_string()); + break; + } + } + if let Ok(events) = client.poll_events() { + apply_device_events(shared, events); + } + } + + let mut state = shared.state.lock().expect("device state lock"); + let succeeded = error.is_none(); + if let Some(message) = error { + state.last_error = Some(format!("{}: {message}", operation.purpose)); + if let Some(meta) = operation.meta { + state.last_response = Some(ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: meta.request_id, + owner_instance: meta.owner_instance, + run_id: meta.run_id, + requested_revision: Some(meta.requested_revision), + acknowledged_revision: state.acknowledged.as_ref().map(|value| value.revision), + outcome: RequestOutcomeV1::Rejected, + completed_at_unix_ms: Some(now_unix_ms()), + error: Some(ServiceErrorV1 { + code: ServiceErrorCodeV1::DeviceRejected, + message, + retryable: false, + }), + }, + controller_state: state.controller_state, + acknowledged_target: state.acknowledged.clone(), + }); + } + } else { + apply_reply_fields(&mut state, &merged); + state.last_error = None; + if let Some(meta) = operation.meta { + let mut acknowledged = meta.target; + acknowledged.board_dac_code = + state.board_code.and_then(|code| u16::try_from(code).ok()); + acknowledged.firmware_configuration_revision = merged + .get("rev") + .and_then(|value| value.parse::().ok()); + state.acknowledged = Some(acknowledged.clone()); + state.last_response = Some(ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: meta.request_id, + owner_instance: meta.owner_instance, + run_id: meta.run_id, + requested_revision: Some(meta.requested_revision), + acknowledged_revision: Some(meta.requested_revision), + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(now_unix_ms()), + error: None, + }, + controller_state: state.controller_state, + acknowledged_target: Some(acknowledged), + }); + } + } + state.last_device_update_unix_ms = now_unix_ms(); + drop(state); + shared.bump(); + succeeded +} + +fn apply_status_reply( shared: &SharedLink, purpose: &str, result: Result, stage_a_io::ClientError>, @@ -216,32 +534,76 @@ fn apply_reply( let mut state = shared.state.lock().expect("device state lock"); match result { Ok(fields) => { - if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { - state.board_code = Some(code); - } - if let Some(wave) = fields.get("mod_wave") { - let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); - let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); - let freq_mhz = fields - .get("mod_freq_mhz") - .and_then(|v| v.parse::().ok()) - .unwrap_or(0.0); - state.board_mod = if wave == "SINE" || wave == "SQUARE" { - format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) - } else { - format!("{wave} level={level}") - }; - } + apply_reply_fields(&mut state, &fields); if purpose == "MOD" { state.last_error = None; } } Err(err) => state.last_error = Some(format!("{purpose}: {err}")), } + state.last_device_update_unix_ms = now_unix_ms(); drop(state); shared.bump(); } +fn apply_reply_fields(state: &mut DeviceState, fields: &BTreeMap) { + if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { + state.board_code = Some(code); + } + if let Some(controller) = fields.get("state") { + state.controller_state = match controller.as_str() { + "SAFE_IDLE" => ControllerStateV1::SafeIdle, + "CONFIGURED" => ControllerStateV1::Configured, + "RUNNING" => ControllerStateV1::Running, + _ => ControllerStateV1::Unknown, + }; + } + if let Some(wave) = fields.get("mod_wave") { + let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); + let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); + let freq_mhz = fields + .get("mod_freq_mhz") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0); + state.board_mod = if wave == "SINE" || wave == "SQUARE" { + format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) + } else { + format!("{wave} level={level}") + }; + state.board_wave = Some(wave.clone()); + state.board_level = fields.get("mod_level").and_then(|v| v.parse().ok()); + state.board_min = fields.get("mod_min").and_then(|v| v.parse().ok()); + state.board_freq_millihz = fields.get("mod_freq_mhz").and_then(|v| v.parse().ok()); + } +} + +fn apply_device_events(shared: &SharedLink, events: Vec) { + let fault = events.into_iter().find_map(|event| match event { + DeviceEvent::Async { name, fields } if name == "FAULT" => Some( + fields + .get("code") + .cloned() + .unwrap_or_else(|| "unknown".into()), + ), + _ => None, + }); + if let Some(code) = fault { + let mut state = shared.state.lock().expect("device state lock"); + state.controller_state = ControllerStateV1::Faulted; + state.last_error = Some(format!("controller fault: {code}")); + state.last_device_update_unix_ms = now_unix_ms(); + drop(state); + shared.bump(); + } +} + +fn now_unix_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0) +} + /// One validated protocol step: the exact MOD command plus how long to hold /// it before advancing. #[derive(Debug, Clone, PartialEq)] @@ -342,6 +704,11 @@ fn parse_protocol(text: &str) -> Result<(Vec, usize), String> { } else { let mode = Mode::from_name(&wave) .ok_or_else(|| context("wave must be OFF, CONST, SINE, or SQUARE"))?; + if mode.optical_target().is_some() { + return Err(context( + "optical warp modes are not available in TOML protocol steps; drive them from the modulation UI", + )); + } let level = step .get("level") .and_then(|value| value.as_integer()) @@ -350,7 +717,7 @@ fn parse_protocol(text: &str) -> Result<(Vec, usize), String> { return Err(context("level must be between 0 and 4095")); } let mut command = Command::new("MOD") - .field("wave", mode.name()) + .field("wave", mode.wire_wave()) .field("level", level); let summary; if mode.is_periodic() { @@ -409,7 +776,11 @@ fn run_protocol( progress.step_index = step_index + 1; progress.summary = step.summary.clone(); } - *shared.pending.lock().expect("pending lock") = Some(step.command.clone()); + *shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![step.command.clone()], + purpose: "PROTOCOL", + meta: None, + }); shared.bump(); next_deadline += step.duration; while Instant::now() < next_deadline { @@ -428,8 +799,92 @@ fn run_protocol( shared.bump(); } +/// A transfer-curve sweep in flight. One point at a time: command a settled +/// `CONST` code, wait for a photodiode window that *starts* after the command, +/// record it, move on. +struct CalibrationSweep { + /// Remaining `(code, direction)` steps, and the points collected so far. + steps: Vec<(u16, calibration::Direction)>, + index: usize, + points: Vec, + /// Detector sample index when the current code was commanded. A level only + /// counts once its window begins after this plus [`SETTLE_SAMPLES`], which + /// needs no shared clock and tolerates any tick jitter. + commanded_at_sample: Option, + /// Wall-clock guard for a stream that stops delivering entirely. + point_started: Instant, + /// Drive to restore when the sweep ends, however it ends. + restore: Option, + /// Max limit in force when the sweep started; the fit's branch constraint. + max_code: u16, +} + +/// Forwards momentary button presses across the host's UI-mirror → live-worker +/// settings snapshot. A click arrives as `true` on the clicked instance; the +/// other instance only ever sees the snapshot value from `get_setting`, so the +/// press is transported as a monotonic counter and a counter advance counts as +/// one press edge. The first counter a fresh instance sees is adopted silently +/// so a reloaded worker does not replay old presses (ADR 010). +/// +/// The baseline is tracked separately from the counter: folding the two +/// together makes a fresh worker mistake the operator's *first* real press for +/// its initial sight of the counter and swallow it. +#[derive(Debug, Default, Clone, Copy)] +struct PressLatch { + counter: u64, + seen: Option, +} + +impl PressLatch { + /// Interprets a settings write to this button; returns true on a press edge. + fn accept(&mut self, value: &Value) -> bool { + if value.as_bool() == Some(true) { + self.counter += 1; + self.seen = Some(self.counter); + return true; + } + let Some(incoming) = value.as_u64() else { + return false; + }; + match self.seen { + None => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + false + } + Some(seen) if incoming > seen => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + true + } + Some(_) => false, + } + } + + fn value(&self) -> Value { + json!(self.counter) + } +} + +impl CalibrationSweep { + fn total(&self) -> usize { + self.steps.len() + } + + fn current(&self) -> Option<(u16, calibration::Direction)> { + self.steps.get(self.index).copied() + } +} + pub struct StageAModulationPlugin { enabled: bool, + runtime_role: PluginRuntimeRole, + effects_allowed: bool, + owner_instance: OwnerInstanceId, + lease: Option, + deferred_release_request: Option, + deferred_release_ack_published: bool, + request_cache: VecDeque<(PluginServiceRequest, PluginServiceReply)>, link: Option, shared: Arc, protocol: Option, @@ -439,16 +894,73 @@ pub struct StageAModulationPlugin { max_level: i64, level: i64, min_level: i64, + method: DriveMethod, mode: Mode, frequency_hz: f64, + // -- optical drive inversion (OPTICAL_* modes) -- + /// Requested optical log-modulation depth `a = ln(I_max / I_min)`. + depth_a: f64, + /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0,1]`. + /// Held fixed while `a` is swept, so one response curve keeps `I_k` constant. + operating_point: f64, + /// DAC code at the excitation minimum of one monotonic Pockels lobe. + v_null_dac: i64, + /// DAC-code quarter-wave distance from `v_null` to the excitation maximum. + v_pi_dac: i64, + // -- measured transfer calibration -- + /// Bench detector geometry. Not inferable from a sweep — see + /// [`calibration::DetectorGeometry`]. + detector_geometry: calibration::DetectorGeometry, + /// Sweep in flight, ticked from `process_control`. + sweep: Option, + /// Last completed fit, awaiting review and an explicit apply. + fit: Option, + /// Set once a fit has been applied to `v_null_dac`/`v_pi_dac`; published on + /// the contract so a consumer's sidecar can cite the inversion in use. + calibration_id: Option, + /// Directory for the archived calibration record; empty means "apply the + /// fit but do not archive it". + calibration_dir: String, + /// Operator-visible outcome of the last sweep or apply. + calibration_status: String, + /// Momentary calibration buttons, forwarded mirror → worker (ADR 010). + press_measure: PressLatch, + press_apply: PressLatch, protocol_path: String, last_error: Option, + /// Last automatic reconnect attempt after the device thread died, for the + /// watchdog backoff in `apply_execution_context`. + last_reconnect_ms: u64, + /// Last `protocol_run` value this instance saw. On the UI mirror this is + /// the operator's request (exported through `get_setting`); everywhere it + /// gates actions to value transitions, because the host re-applies the + /// full settings snapshot on every sync. + protocol_requested: bool, +} + +#[derive(Clone)] +struct ControlLease { + lease_id: LeaseId, + holder: ClientId, + run_id: Option, + expires_at_unix_ms: u64, } impl Default for StageAModulationPlugin { fn default() -> Self { Self { enabled: false, + runtime_role: PluginRuntimeRole::UiMirror, + effects_allowed: false, + owner_instance: OwnerInstanceId::new(format!( + "modulation-{}-{}", + std::process::id(), + now_unix_ms() + )), + lease: None, + deferred_release_request: None, + deferred_release_ack_published: false, + request_cache: VecDeque::new(), link: None, shared: Arc::new(SharedLink::new()), protocol: None, @@ -457,10 +969,25 @@ impl Default for StageAModulationPlugin { max_level: MAX_DAC_CODE, level: 0, min_level: 0, + method: DriveMethod::Manual, mode: Mode::Const, frequency_hz: 10.0, + depth_a: 0.5, + operating_point: 0.5, + v_null_dac: 0, + v_pi_dac: 2_048, + detector_geometry: calibration::DetectorGeometry::RejectedComplement, + sweep: None, + fit: None, + calibration_id: None, + calibration_dir: String::new(), + calibration_status: String::new(), + press_measure: PressLatch::default(), + press_apply: PressLatch::default(), protocol_path: String::new(), last_error: None, + last_reconnect_ms: 0, + protocol_requested: false, } } } @@ -470,10 +997,18 @@ impl StageAModulationPlugin { if self.link.is_some() { return; } + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + self.last_error = Some("connection deferred: hardware effects are not allowed".into()); + return; + } self.last_error = None; *self.shared.state.lock().expect("device state lock") = DeviceState::default(); *self.shared.pending.lock().expect("pending lock") = None; + *self.shared.priority.lock().expect("priority lock") = None; self.shared.stop.store(false, Ordering::Relaxed); + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); self.shared.bump(); let shared = Arc::clone(&self.shared); @@ -571,850 +1106,3901 @@ impl StageAModulationPlugin { self.shared.bump(); } + fn lobe_inversion(&self) -> waveform::LobeInversion { + waveform::LobeInversion { + v_null_dac: self.v_null_dac as f64, + v_pi_dac: self.v_pi_dac as f64, + } + } + + /// Resolves the selected method into the DAC band used by every waveform. + /// The third value is the constant-mode operating code. + fn dac_band(&self) -> Result<(i64, i64, i64), String> { + if self.method == DriveMethod::Manual { + let hi = self.level.clamp(0, self.max_level); + return Ok((self.min_level.clamp(0, hi), hi, hi)); + } + + let u_k = self.operating_point; + let a = self.depth_a; + if !u_k.is_finite() || u_k <= 0.0 || u_k > 1.0 { + return Err("operating point must be in (0, 1]".into()); + } + let inversion = self.lobe_inversion(); + if !inversion.v_pi_dac.is_finite() || inversion.v_pi_dac <= 0.0 { + return Err("Vπ must be finite and positive".into()); + } + + // Constant hold at I_k modulates nothing: no ±a/2 headroom applies, so + // the full (0, 1] range of u_k is expressible (I_k = 1 holds exactly at + // V_null + Vπ). Requiring the modulated band here silently froze the + // drive at the last accepted code whenever u_k·e^{a/2} exceeded 1. + if self.mode == Mode::Const { + let hold = inversion.dac_for_u(u_k).round() as i64; + if hold < 0 { + return Err(format!( + "calibrated hold code {hold} is below 0; re-measure V_null/Vπ" + )); + } + if hold > self.max_level { + return Err(format!( + "calibrated hold {hold} exceeds the max limit {}; raise the max limit or lower I_k / Vπ", + self.max_level + )); + } + return Ok((hold, hold, hold)); + } + + if !a.is_finite() || a <= 0.0 { + return Err("optical depth a must be finite and positive".into()); + } + let u_lo = u_k * (-0.5 * a).exp(); + let u_hi = u_k * (0.5 * a).exp(); + if u_hi > 1.0 { + return Err(format!( + "calibrated optical peak u = {u_hi:.3} exceeds the lobe ceiling; lower a or I_k" + )); + } + + let lo = inversion.dac_for_u(u_lo).round() as i64; + let hi = inversion.dac_for_u(u_hi).round() as i64; + let hold = inversion.dac_for_u(u_k).round() as i64; + if lo < 0 { + return Err(format!( + "calibrated lower DAC code {lo} is below 0; re-measure V_null/Vπ" + )); + } + if hi > self.max_level { + return Err(format!( + "calibrated peak {hi} exceeds the max limit {}; raise the max limit or lower a / I_k / Vπ", + self.max_level + )); + } + Ok((lo, hi, hold)) + } + + fn optical_drive(&self, target: waveform::OpticalTarget) -> waveform::OpticalDrive { + let inversion = self.lobe_inversion(); + match self.method { + DriveMethod::Manual => { + let hi = self.level.clamp(0, self.max_level); + let lo = self.min_level.clamp(0, hi); + waveform::OpticalDrive::from_dac_band(target, inversion, lo as f64, hi as f64) + } + DriveMethod::Calibrated => waveform::OpticalDrive { + target, + depth_a: self.depth_a, + operating_point: self.operating_point, + inversion, + }, + } + } + + /// Builds the single MOD command carrying the complete current drive + /// settings (mode, method, band, frequency). Shared by the operator path + /// (`send_modulation`) and the leased `SetOpticalDepth` service command. + fn drive_command(&self) -> Result { + let (lo, hi, hold) = self.dac_band()?; + let freq_mhz = (self.frequency_hz.clamp(0.01, 2_000.0) * 1_000.0).round() as i64; + Ok(match self.mode { + Mode::Const => Command::new("MOD") + .field("wave", "CONST") + .field("level", hold), + Mode::Sine | Mode::Square => Command::new("MOD") + .field("wave", self.mode.wire_wave()) + .field("level", hi) + .field("min", lo) + .field("freq_mhz", freq_mhz), + Mode::OpticalLogSine | Mode::OpticalLinearSine => { + let target = self + .mode + .optical_target() + .expect("optical modes have a target"); + // Validate the drive locally; the firmware rebuilds the same + // table from compact parameters because a full table does not + // fit on one command line. + self.optical_warp_table(target) + .map_err(|error| format!("optical drive: {error}"))?; + let drive = self.optical_drive(target); + Command::new("MOD") + .field("wave", "WARP") + .field("freq_mhz", freq_mhz) + .field("target", optical_target_token(target)) + .field("a_milli", (drive.depth_a * 1_000.0).round() as i64) + .field( + "u_k_milli", + (drive.operating_point * 1_000.0).round() as i64, + ) + .field("v_null", self.v_null_dac) + .field("v_pi", self.v_pi_dac) + } + }) + } + /// Queues one MOD command carrying the complete current drive settings; /// newer changes overwrite queued ones (drag coalescing). + /// + /// Silent while another owner holds the DAC. Besides an automation lease + /// that now includes a calibration sweep: the host re-applies the *whole* + /// settings snapshot on every sync, and most handlers here call this + /// unconditionally, so without the guard every sync would re-arm the + /// operator's drive on top of the code the sweep just commanded — the + /// sweep would measure the armed waveform instead of its own staircase. fn send_modulation(&mut self) { - if self.link.is_none() { + if self.link.is_none() || self.lease.is_some() || self.sweep.is_some() { return; } - let level = self.level.clamp(0, self.max_level); - let mut command = Command::new("MOD") - .field("wave", self.mode.name()) - .field("level", level); - if self.mode.is_periodic() { - let freq_mhz = (self.frequency_hz.clamp(0.01, 2_000.0) * 1_000.0).round() as i64; - command = command - .field("min", self.min_level.clamp(0, level)) - .field("freq_mhz", freq_mhz); - } - *self.shared.pending.lock().expect("pending lock") = Some(command); + let command = match self.drive_command() { + Ok(command) => command, + Err(error) => { + self.last_error = Some(format!("drive rejected: {error}")); + return; + } + }; + self.last_error = None; + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); } - #[cfg(test)] - fn device_connected(&self) -> bool { - self.shared - .state - .lock() - .map(|state| state.connected) - .unwrap_or(false) + /// Reject a settings update before it can leave the UI showing a drive + /// that was never sent to the board. + fn validate_drive(&mut self) -> Result<(), String> { + match self.drive_command() { + Ok(_) => { + self.last_error = None; + Ok(()) + } + Err(error) => { + self.last_error = Some(format!("drive rejected: {error}")); + Err(error) + } + } } - fn commanded_summary(&self) -> String { - if self.mode.is_periodic() { - format!( - "{} {}..{} @ {:.3} Hz", - self.mode.name(), - self.min_level, - self.level, - self.frequency_hz - ) - } else { - format!("{} level={}", self.mode.name(), self.level) + /// Builds the DAC warp table for the current optical drive settings. The + /// max limit is the hard ceiling; absolute lobe codes cannot be rescaled + /// without distorting the target, so an over-limit drive is refused. + fn optical_warp_table(&self, target: waveform::OpticalTarget) -> Result, String> { + let table = self + .optical_drive(target) + .warp_table() + .map_err(|error| error.to_string())?; + let peak = table.iter().copied().max().unwrap_or(0); + if i64::from(peak) > self.max_level { + return Err(format!( + "optical peak {peak} exceeds the max limit {}; raise the max limit or lower the \ + operating band / a / I_k / Vπ", + self.max_level + )); } + Ok(table) } - fn status_dataset(&self) -> TableDatasetV1 { - let state = self.shared.state.lock().expect("device state lock"); - let connection = if state.connected { - format!("connected ({})", state.firmware) - } else if self.connect_requested { - "connecting…".into() - } else { - "disconnected".into() - }; - let board_code = state - .board_code - .map_or_else(|| "—".into(), |code| code.to_string()); - let error = state - .last_error - .clone() - .or_else(|| self.last_error.clone()) - .unwrap_or_default(); - let board_mod = if state.board_mod.is_empty() { - "—".to_owned() - } else { - state.board_mod.clone() - }; - drop(state); - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", connection), - text_column("commanded", self.commanded_summary()), - text_column("board_mod", board_mod), - text_column("board_code", board_code), - text_column("error", error), - ], - } + // ---- measured transfer calibration ---- + + /// Whether the calibration buttons can be offered, from **mirrored** + /// settings only. + /// + /// `settings_schema()` is rendered by the UI mirror, which never owns the + /// device link, a lease, a sweep, or a fit — those live on the live worker. + /// Gating `enabled` on any of them disables the button permanently. So the + /// affordance uses the one prerequisite the mirror does know (the operator + /// asked to connect) and the authoritative interlocks stay worker-side in + /// [`Self::calibration_blocker`], reported through the status entries the + /// host takes from the worker. + fn calibration_offered(&self) -> bool { + self.connect_requested } - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("commanded", "Commanded drive"), - column("board_mod", "Board modulation"), - column("board_code", "Board DAC code"), - column("error", "Last error"), - ], - ..TableSchema::default() + /// Why a sweep cannot start right now, if it cannot. + fn calibration_blocker(&self) -> Option { + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + return Some("hardware effects are not allowed on this instance".into()); + } + if self.link.is_none() { + return Some("connect the command port first".into()); } + if self.lease.is_some() { + // A1 owns the drive under a lease; two owners stepping the same DAC + // would interleave silently. + return Some("the drive is leased by an automation client".into()); + } + if self.protocol_active() { + return Some("a protocol is running".into()); + } + None } -} -fn open_serial(port_hint: &str) -> Result, String> { - if port_hint == "auto" { - // The dual-serial Teensy enumerates two ports and only the command - // port answers HELLO — probe until one does. - let candidates = serial_ports(); - if candidates.is_empty() { - return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + /// Starts a sweep, remembering the drive to restore afterwards. + fn start_calibration_sweep(&mut self) { + if let Some(blocker) = self.calibration_blocker() { + self.calibration_status = format!("sweep refused: {blocker}"); + return; } - let mut failures = Vec::new(); - for path in &candidates { - match probe_command_port(path) { - // Restore the client's default reply timeout after probing. - Ok(client) => return Ok(client.with_reply_timeout(Duration::from_millis(500))), - Err(err) => failures.push(format!("{path}: {err}")), - } + let max_code = self.max_level.clamp(0, MAX_DAC_CODE) as u16; + if max_code < 2 { + self.calibration_status = "sweep refused: the max limit leaves no range".into(); + return; } - return Err(format!( - "no Teensy command port answered HELLO ({})", - failures.join("; ") - )); + self.sweep = Some(CalibrationSweep { + steps: calibration::sweep_codes(max_code, SWEEP_POINTS_PER_PASS, true), + index: 0, + points: Vec::new(), + commanded_at_sample: None, + point_started: Instant::now(), + // Restoring the drive the operator had armed is part of the + // measurement contract: a sweep must leave the bench as it found it. + restore: self.drive_command().ok(), + max_code, + }); + self.fit = None; + self.calibration_status = "sweep starting…".into(); } - open_path(port_hint) -} -fn open_path(path: &str) -> Result, String> { - let transport = - stage_a_io::SerialTransport::open(path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} + /// Ends the sweep and hands the DAC back to the armed drive. + fn finish_calibration_sweep(&mut self, status: String) { + // Clear the sweep first: it is what silences `send_modulation`. + let restore = self.sweep.take().and_then(|sweep| sweep.restore); + self.calibration_status = status; + if self.link.is_some() { + // Prefer the *current* settings — drive changes made during the + // sweep were withheld from the board, and this is where they land. + // The command captured at the start is the fallback for settings + // that no longer form a valid drive. + if self.drive_command().is_ok() { + self.send_modulation(); + } else if let Some(command) = restore { + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); + } + } + self.shared.bump(); + } -/// Opens `path` and sends HELLO with a short timeout: only the Teensy -/// command port replies (the photodiode stream port never answers). -fn probe_command_port(path: &str) -> Result, String> { - let mut client = open_path(path)?.with_reply_timeout(Duration::from_millis(300)); - client - .request(&Command::new("HELLO").field("protocol", 1)) - .map_err(|err| err.to_string())?; - Ok(client) -} + /// Queues one settled `CONST` code, bypassing the drive builder: a sweep + /// deliberately visits codes the armed drive would refuse. + fn command_sweep_code(&mut self, code: u16) { + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![Command::new("MOD") + .field("wave", "CONST") + .field("level", i64::from(code))], + purpose: "MOD", + meta: None, + }); + } -fn serial_ports() -> Vec { - stage_a_io::transport::available_port_names() - .into_iter() - // macOS lists each device twice; use the callout (cu.*) node only. - .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) - .collect() -} + /// One tick of the sweep. `level` is the newest photodiode reading, if any. + fn drive_calibration(&mut self, level: Option) { + if self.sweep.is_none() { + return; + } + if let Some(blocker) = self.calibration_blocker() { + self.finish_calibration_sweep(format!("sweep aborted: {blocker}")); + return; + } + let Some(level) = level else { + if self + .sweep + .as_ref() + .is_some_and(|sweep| sweep.point_started.elapsed() > POINT_TIMEOUT) + { + self.finish_calibration_sweep( + "sweep aborted: no photodiode level (connect the photodiode plugin)".into(), + ); + } + return; + }; -/// The exact variant list the settings schema shows for the port enum — the -/// host exchanges enum settings as indices into this list. Real ports carry -/// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; -/// only the leading path is the value. -fn port_variants() -> Vec { - let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; - for (name, label) in stage_a_io::transport::available_ports_with_labels() { - if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { - continue; + let Some((code, direction)) = self.sweep.as_ref().and_then(CalibrationSweep::current) + else { + self.complete_calibration_sweep(); + return; + }; + + // Command the point once, then wait for a window that began after it. + let commanded_at = match self.sweep.as_ref().expect("sweep").commanded_at_sample { + Some(sample) => sample, + None => { + self.command_sweep_code(code); + let sweep = self.sweep.as_mut().expect("sweep"); + sweep.commanded_at_sample = Some(level.end_sample_index); + sweep.point_started = Instant::now(); + self.calibration_status = format!( + "sweeping {}/{}…", + self.sweep.as_ref().expect("sweep").index + 1, + self.sweep.as_ref().expect("sweep").total() + ); + return; + } + }; + + let window_start = level.end_sample_index.saturating_sub(level.sample_count); + if window_start < commanded_at + SETTLE_SAMPLES { + if self.sweep.as_ref().expect("sweep").point_started.elapsed() > POINT_TIMEOUT { + self.finish_calibration_sweep( + "sweep aborted: the photodiode stream stalled".into(), + ); + } + return; } - variants.push(match label { - Some(label) => format!("{name} ({label})"), - None => name, + + let sweep = self.sweep.as_mut().expect("sweep"); + sweep.points.push(calibration::SweepPoint { + code, + direction, + volts: level.mean_volts, + peak_to_peak_volts: level.peak_to_peak_volts, + clipped: level.clipped, }); + sweep.index += 1; + sweep.commanded_at_sample = None; + if sweep.index >= sweep.steps.len() { + self.complete_calibration_sweep(); + } } - variants -} -/// The path part of a port variant; the parenthesised USB label is display-only. -fn variant_path(variant: &str) -> &str { - variant.split_whitespace().next().unwrap_or(variant) -} + /// Fits the collected points and leaves the result awaiting an explicit + /// apply — a bad fit silently retargeting the drive is the dangerous case. + fn complete_calibration_sweep(&mut self) { + let Some(sweep) = self.sweep.as_ref() else { + return; + }; + let points = sweep.points.clone(); + let max_code = f64::from(sweep.max_code); + match calibration::fit_transfer(&points, max_code, self.detector_geometry) { + Ok(fit) => { + let status = format!( + "V_null {:.0} Vπ {:.0} span {:.3} V residual {:.1}%{}{} ({:.1} lobes)", + fit.v_null_dac, + fit.v_pi_dac, + fit.span_volts.abs(), + fit.quality * 100.0, + fit.hysteresis + .map(|value| format!(" hysteresis {:.1}%", value * 100.0)) + .unwrap_or_default(), + if fit.rejected_points > 0 { + format!(" {} dropped", fit.rejected_points) + } else { + String::new() + }, + fit.lobe_coverage, + ); + self.fit = Some(fit); + self.finish_calibration_sweep(status); + } + Err(error) => { + self.fit = None; + self.finish_calibration_sweep(format!("fit failed: {error}")); + } + } + } -/// Host enum widgets send the selected index; string names are also accepted -/// (tests, saved configs). -fn enum_choice(value: &Value, variants: &[String]) -> Result { - if let Some(index) = value.as_u64() { - return variants - .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) - .cloned() - .ok_or_else(|| format!("enum index {index} out of range")); + /// Things worth the operator's attention before trusting a fit. Compare + /// them against the transfer-curve plot. + /// + /// Deliberately warnings and not blocks. The only condition that makes a + /// fit meaningless — no full lobe inside the commandable range — is already + /// refused by [`calibration::fit_transfer`] itself, so there is no second + /// fit to reject here. Everything below is a judgement the operator makes + /// against the plot: a single stray sample can push the residual past any + /// threshold while `Vπ` stays accurate to a few codes, so blocking on it + /// would withhold a good calibration for a bad reason. + fn fit_warnings(&self) -> Vec { + let Some(fit) = self.fit.as_ref() else { + return Vec::new(); + }; + let mut warnings = Vec::new(); + if fit.quality > WARN_QUALITY { + warnings.push(format!( + "residual is {:.1}% of the detector span — check the fit against the points \ + in the transfer-curve plot before trusting Vπ", + fit.quality * 100.0 + )); + } + if fit.rejected_points > 0 { + warnings.push(format!( + "{} of {} points were wild and left out of the fit", + fit.rejected_points, + fit.points.len() + )); + } + if let Some(hysteresis) = fit.hysteresis.filter(|value| *value > WARN_HYSTERESIS) { + warnings.push(format!( + "up and down passes differ by {:.1}% of the span — the cell is drifting or \ + the settle time is too short", + hysteresis * 100.0 + )); + } + let clipped = fit.points.iter().filter(|point| point.clipped).count(); + if clipped > 0 { + warnings.push(format!( + "{clipped} points clipped the ADC; the extremum they sit on is not where the \ + fit thinks it is — add attenuation and re-measure" + )); + } + warnings } - value - .as_str() - .map(str::to_owned) - .ok_or_else(|| "expected an enum index or name".to_owned()) -} -impl Plugin for StageAModulationPlugin { - fn name(&self) -> &'static str { - "Stage-A Modulation" + /// Applies the reviewed fit to `V_null`/`Vπ` and archives the record. + fn apply_calibration_fit(&mut self) { + let Some(fit) = self.fit.clone() else { + self.calibration_status = "nothing to apply: measure a transfer curve first".into(); + return; + }; + let previous = (self.v_null_dac, self.v_pi_dac); + self.v_null_dac = fit.v_null_dac.round().clamp(0.0, MAX_DAC_CODE as f64) as i64; + self.v_pi_dac = fit.v_pi_dac.round().clamp(1.0, MAX_DAC_CODE as f64) as i64; + // The applied lobe must still produce a legal drive; a calibration that + // cannot be armed is not an improvement. + if let Err(error) = self.validate_drive() { + self.v_null_dac = previous.0; + self.v_pi_dac = previous.1; + self.calibration_status = format!("not applied: {error}"); + return; + } + let calibration_id = format!("pockels-{}", timestamp_slug()); + let archived = match self.archive_calibration(&calibration_id, &fit) { + Ok(Some(path)) => format!(", archived to {path}"), + Ok(None) => ", not archived (no calibration folder set)".into(), + Err(error) => format!(", archive failed: {error}"), + }; + self.calibration_id = Some(calibration_id); + self.calibration_status = format!( + "applied V_null {} / Vπ {}{archived}", + self.v_null_dac, self.v_pi_dac + ); + self.send_modulation(); + self.shared.bump(); } - fn description(&self) -> &'static str { - "Laser modulation control on the Teensy command port: capped power slider, constant/sine/square with frequency, applied immediately; shows the DAC code the board reports." + /// Writes the calibration record. A calibration is named and never + /// silently overwritten (knowledge base §4.6). + fn archive_calibration( + &self, + calibration_id: &str, + fit: &calibration::TransferFit, + ) -> Result, String> { + if self.calibration_dir.trim().is_empty() { + return Ok(None); + } + let directory = std::path::Path::new(self.calibration_dir.trim()); + std::fs::create_dir_all(directory) + .map_err(|error| format!("creating {}: {error}", directory.display()))?; + let path = directory.join(format!("{calibration_id}.json")); + let record = json!({ + "calibration_id": calibration_id, + "port": self.port_hint, + "max_level": self.max_level, + "detector_geometry": fit.geometry.name(), + "v_null_dac": fit.v_null_dac, + "v_pi_dac": fit.v_pi_dac, + "detector_volts_at_null": fit.detector_volts_at_null(), + "detector_volts_at_peak": fit.detector_volts_at_peak(), + "span_volts": fit.span_volts, + "rms_residual_volts": fit.rms_residual_volts, + "quality": fit.quality, + "hysteresis": fit.hysteresis, + "lobe_coverage": fit.lobe_coverage, + "rejected_points": fit.rejected_points, + "anchor_note": "detector_volts_at_null is a lower bound on the total-power \ + anchor I_tot, not the anchor: on the reject port the residual \ + transmitted floor is not separable from it", + "points": fit + .points + .iter() + .map(|point| json!({ + "code": point.code, + "direction": point.direction.label(), + "volts": point.volts, + "peak_to_peak_volts": point.peak_to_peak_volts, + "clipped": point.clipped, + })) + .collect::>(), + }); + let encoded = serde_json::to_vec_pretty(&record) + .map_err(|error| format!("encoding the calibration record: {error}"))?; + std::fs::write(&path, encoded) + .map_err(|error| format!("writing {}: {error}", path.display()))?; + Ok(Some(path.display().to_string())) } - fn enabled(&self) -> bool { - self.enabled + fn lease_snapshot(&self) -> Option { + self.lease.as_ref().map(|lease| LeaseSnapshotV1 { + lease_id: lease.lease_id.clone(), + holder: lease.holder.clone(), + expires_at_unix_ms: lease.expires_at_unix_ms, + run_id: lease.run_id.clone(), + }) } - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.connect_requested = false; - self.disconnect(); + fn require_lease(&self, request: &ModulationRequestV1) -> Result<(), ServiceErrorV1> { + let lease = self.lease.as_ref().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::LeaseRequired, + "the modulation owner requires an active automation lease", + false, + ) + })?; + if now_unix_ms() > lease.expires_at_unix_ms { + return Err(service_error( + ServiceErrorCodeV1::LeaseExpired, + "the modulation automation lease expired", + false, + )); } + if request.lease_id.as_ref() != Some(&lease.lease_id) || request.requester != lease.holder { + return Err(service_error( + ServiceErrorCodeV1::LeaseMismatch, + "request lease/holder does not match the active lease", + false, + )); + } + if request.run_id != lease.run_id { + return Err(service_error( + ServiceErrorCodeV1::LeaseMismatch, + "request run does not match the leased run", + false, + )); + } + Ok(()) } - fn reset(&mut self) {} - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - // Control is settings-driven and works without camera frames. The - // only frame-pass policy: replaying a recording must never keep a - // hardware connection alive. - if context.execution().mode == ExecutionMode::Replay && self.link.is_some() { - self.connect_requested = false; - self.disconnect(); - self.last_error = Some("disconnected: replay mode".into()); + fn requested_revision( + &self, + request: &ModulationRequestV1, + ) -> Result { + let revision = request.requested_revision.ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "state-changing modulation commands require requested_revision", + false, + ) + })?; + let current = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.requested.as_ref().map(|target| target.revision)); + if current.is_some_and(|current| revision <= current) { + return Err(service_error( + ServiceErrorCodeV1::StaleRequest, + "requested_revision must be newer than the current requested state", + false, + )); } + Ok(revision) } - fn settings_schema(&self) -> SettingsSchema { - let port_variants = port_variants(); - let port_default = port_variants - .iter() - .position(|p| variant_path(p) == self.port_hint) - .unwrap_or(0); - let mode_variants: Vec = - Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); - let mode_default = Mode::VARIANTS - .iter() - .position(|m| *m == self.mode) - .unwrap_or(0); - SettingsSchema { - sections: vec![ - SettingsSection { - label: "Laser modulation".into(), - description: Some( - "Tick Connect, then every change is sent to the Teensy immediately — no \ - camera required. The output never exceeds the power slider, the slider \ - never exceeds the max limit. The firmware holds the output when \ - disconnected; drag the slider to 0 to drive 0 V." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "auto (recommended) probes the attached usbmodem ports and picks \ - the one that answers HELLO — the Teensy command port; \ - mock = in-process simulated controller" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, - }, - }, - SettingItem { - key: "connect".into(), - label: "Connect".into(), - tooltip: Some( - "Opens/closes the command port. Connecting never changes the \ - output; disconnecting leaves it held (set-and-hold firmware)." - .into(), - ), - kind: SettingKind::Bool { - default: self.connect_requested, - }, - }, - SettingItem { - key: "level".into(), - label: "Power (DAC code)".into(), - tooltip: Some( - "Output level in DAC codes; peak value for sine/square. \ - Capped by the max limit below. 0 = output off." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.level, - suffix: None, - }, - }, - SettingItem { - key: "max_level".into(), - label: "Max limit (DAC code)".into(), - tooltip: Some( - "Safety cap: the slider cannot go above this. Set it to the \ - highest code the connected device tolerates at J23." - .into(), - ), - kind: SettingKind::I64Drag { - min: 0, - max: MAX_DAC_CODE, - default: self.max_level, - }, - }, - SettingItem { - key: "mode".into(), - label: "Mode".into(), - tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), - kind: SettingKind::Enum { - variants: mode_variants, - default: mode_default, - }, - }, - SettingItem { - key: "frequency_hz".into(), - label: "Frequency".into(), - tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), - kind: SettingKind::F64Drag { - min: 0.01, - max: 2_000.0, - speed: 1.0, - default: self.frequency_hz, - }, - }, - SettingItem { - key: "min_level".into(), - label: "Min threshold (DAC code)".into(), - tooltip: Some( - "Lower bound for sine/square: the waveform swings between this \ - and the power slider. Ignored in CONST mode." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.min_level, - suffix: None, - }, - }, - ], - }, - SettingsSection { - label: "Protocol".into(), - description: Some( - "Timed sequence of MOD steps from a TOML file: `loops = N` plus \ - [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, \ - min, frequency_hz. Steps run on an absolute schedule; the last \ - step holds after completion (set-and-hold). Stopping never \ - switches the output off by itself." - .into(), - ), - default_open: false, - items: vec![ - SettingItem { - key: "protocol_path".into(), - label: "Protocol file".into(), - tooltip: Some("TOML protocol file (validated on start).".into()), - kind: SettingKind::Path { - dialog: PathDialogKind::OpenFile, - default: self.protocol_path.clone(), - }, - }, - SettingItem { - key: "protocol_run".into(), - label: "Run protocol".into(), - tooltip: Some( - "Start/stop the loaded protocol. Requires an open connection; \ - manual drive controls stay live and override the current step \ - until the next one begins." - .into(), - ), - kind: SettingKind::Bool { - default: self.protocol_active(), - }, - }, - ], - }, - ], + fn base_target(&self, revision: SemanticRevision) -> ModulationTargetV1 { + let state = self.shared.state.lock().expect("device state lock"); + let mut target = state + .requested + .clone() + .or_else(|| state.acknowledged.clone()) + .unwrap_or(ModulationTargetV1 { + revision, + waveform: None, + a1_configuration: None, + acquisition_running: false, + board_dac_code: None, + firmware_configuration_revision: None, + }); + target.revision = revision; + target.board_dac_code = None; + target.firmware_configuration_revision = None; + target + } + + fn queue_service_operation( + &mut self, + request: &ModulationRequestV1, + target: ModulationTargetV1, + commands: Vec, + purpose: &'static str, + priority: bool, + ) -> Result { + if self.link.is_none() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "the Teensy command port is not connected", + true, + )); + } + let revision = target.revision; + let meta = OperationMeta { + request_id: request.request_id, + run_id: request.run_id.clone(), + requested_revision: revision, + target: target.clone(), + owner_instance: self.owner_instance.clone(), + }; + { + let mut state = self.shared.state.lock().expect("device state lock"); + state.requested = Some(target); + } + let operation = PendingOperation { + commands, + purpose, + meta: Some(meta), + }; + if priority { + *self.shared.pending.lock().expect("pending lock") = None; + *self.shared.priority.lock().expect("priority lock") = Some(operation); + } else { + *self.shared.pending.lock().expect("pending lock") = Some(operation); } + self.shared.bump(); + Ok(ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: request.request_id, + owner_instance: self.owner_instance.clone(), + run_id: request.run_id.clone(), + requested_revision: Some(revision), + acknowledged_revision: self + .shared + .state + .lock() + .ok() + .and_then(|state| state.acknowledged.as_ref().map(|value| value.revision)), + outcome: RequestOutcomeV1::InProgress, + completed_at_unix_ms: None, + error: None, + }, + controller_state: self + .shared + .state + .lock() + .map(|state| state.controller_state) + .unwrap_or(ControllerStateV1::Unknown), + acknowledged_target: None, + }) } - fn get_setting(&self, key: &str) -> Option { - match key { - // Enum settings are exchanged as indices into the schema's - // variant list (see the host settings UI). - "port" => { - let index = port_variants() - .iter() - .position(|p| variant_path(p) == self.port_hint) - .unwrap_or(0); - Some(json!(index)) - } - "connect" => Some(json!(self.connect_requested)), - "level" => Some(json!(self.level)), - "max_level" => Some(json!(self.max_level)), - "mode" => { - let index = Mode::VARIANTS - .iter() - .position(|m| *m == self.mode) - .unwrap_or(0); - Some(json!(index)) - } - "frequency_hz" => Some(json!(self.frequency_hz)), - "min_level" => Some(json!(self.min_level)), - "protocol_path" => Some(json!(self.protocol_path)), - "protocol_run" => Some(json!(self.protocol_active())), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "port" => { - self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); - Ok(()) - } - "connect" => { - let requested = value.as_bool().ok_or("connect must be a boolean")?; - self.connect_requested = requested; - if requested { - self.connect(); - } else { - self.disconnect(); - } - Ok(()) - } - "level" => { - self.level = value - .as_i64() - .ok_or("level must be an integer")? - .clamp(0, self.max_level); - if self.min_level > self.level { - self.min_level = self.level; + fn handle_modulation_command( + &mut self, + request: &ModulationRequestV1, + ) -> Result { + match &request.command { + ModulationCommandV1::Connect => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "connection cannot be changed while leased", + false, + )); } - self.send_modulation(); - Ok(()) + self.connect_requested = true; + self.connect(); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "max_level" => { - self.max_level = value - .as_i64() - .ok_or("max_level must be an integer")? - .clamp(0, MAX_DAC_CODE); - // Lowering the cap below the current level lowers the output. - if self.level > self.max_level { - self.level = self.max_level; - self.send_modulation(); + ModulationCommandV1::Disconnect { safe_off, reason } => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "use ReleaseLease while the owner is leased", + false, + )); } - if self.min_level > self.max_level { - self.min_level = self.max_level; + if *safe_off && self.link.is_some() { + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); } - Ok(()) + self.connect_requested = false; + self.disconnect(); + self.last_error = Some(format!("disconnected by service: {reason}")); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "mode" => { - let mode_names: Vec = - Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); - let name = enum_choice(&value, &mode_names)?; - self.mode = Mode::from_name(&name) - .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; - self.send_modulation(); - Ok(()) + ModulationCommandV1::AcquireLease { ttl_ms } => { + let lease_id = request.lease_id.clone().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "AcquireLease requires lease_id", + false, + ) + })?; + if let Some(active) = &self.lease { + if active.lease_id != lease_id || active.holder != request.requester { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "the modulation owner is already leased", + true, + )); + } + } + self.protocol = None; + self.lease = Some(ControlLease { + lease_id, + holder: request.requester.clone(), + run_id: request.run_id.clone(), + expires_at_unix_ms: lease_deadline(*ttl_ms), + }); + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "frequency_hz" => { - let hz = value.as_f64().ok_or("frequency_hz must be a number")?; - self.frequency_hz = hz.clamp(0.01, 2_000.0); - if self.mode.is_periodic() { - self.send_modulation(); + ModulationCommandV1::RenewLease { ttl_ms } => { + self.require_lease(request)?; + if let Some(lease) = &mut self.lease { + lease.expires_at_unix_ms = lease_deadline(*ttl_ms); } - Ok(()) + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "min_level" => { - self.min_level = value - .as_i64() - .ok_or("min_level must be an integer")? - .clamp(0, self.level); - if self.mode.is_periodic() { - self.send_modulation(); + ModulationCommandV1::ReleaseLease { safe_off, reason } => { + self.require_lease(request)?; + if *safe_off { + let revision = request.requested_revision.unwrap_or_else(|| { + let current = self + .shared + .state + .lock() + .ok() + .and_then(|state| { + state.requested.as_ref().map(|value| value.revision.0) + }) + .unwrap_or(0); + SemanticRevision(current.saturating_add(1)) + }); + let mut target = self.base_target(revision); + target.waveform = Some(WaveformV1::Off); + target.acquisition_running = false; + let response = self.queue_service_operation( + request, + target, + vec![ + Command::new("STOP").field("reason", reason.replace(' ', "_")), + Command::new("MOD").field("wave", "OFF"), + ], + "SAFE_OFF", + true, + )?; + self.deferred_release_request = Some(request.request_id); + self.deferred_release_ack_published = false; + return Ok(response); } - Ok(()) + self.lease = None; + self.deferred_release_request = None; + self.shared + .fail_closed_on_stop + .store(false, Ordering::Relaxed); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "protocol_path" => { - self.protocol_path = value - .as_str() - .ok_or("protocol_path must be a string")? - .to_owned(); - Ok(()) + ModulationCommandV1::SafeOff { reason } => { + let revision = request.requested_revision.unwrap_or_else(|| { + let current = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.requested.as_ref().map(|value| value.revision.0)) + .unwrap_or(0); + SemanticRevision(current.saturating_add(1)) + }); + let mut target = self.base_target(revision); + target.waveform = Some(WaveformV1::Off); + target.acquisition_running = false; + self.protocol = None; + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); + self.queue_service_operation( + request, + target, + vec![ + Command::new("STOP").field("reason", reason.replace(' ', "_")), + Command::new("MOD").field("wave", "OFF"), + ], + "SAFE_OFF", + true, + ) } - "protocol_run" => { - let requested = value.as_bool().ok_or("protocol_run must be a boolean")?; - // Failures surface through status entries (like `connect`). - if requested { - match self.start_protocol() { - Ok(()) => self.last_error = None, - Err(err) => self.last_error = Some(err), - } - } else { - self.stop_protocol(); + ModulationCommandV1::SetWaveform { waveform } => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.waveform = Some(waveform.clone()); + self.queue_service_operation( + request, + target, + vec![waveform_command(waveform)], + "SET_WAVEFORM", + false, + ) + } + ModulationCommandV1::SetOpticalDepth { depth_a_milli } => { + self.require_lease(request)?; + if self.link.is_none() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "the modulation owner is not connected to the device", + false, + )); + } + let depth_a = f64::from(*depth_a_milli) / 1_000.0; + if !(0.01..=6.0).contains(&depth_a) { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + format!("optical depth a={depth_a:.3} outside the supported 0.01..=6.0"), + false, + )); } + // Only the calibrated drive expresses an optical depth; the + // manual DAC band and the constant hold do not. + if self.method == DriveMethod::Manual || self.mode == Mode::Const { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "arm a calibrated periodic/optical drive in the modulation plugin \ + before sweeping the optical depth", + false, + )); + } + let previous = self.depth_a; + self.depth_a = depth_a; + let command = match self.drive_command() { + Ok(command) => command, + Err(error) => { + self.depth_a = previous; + return Err(service_error( + ServiceErrorCodeV1::DeviceRejected, + format!("optical depth a={depth_a:.3} rejected: {error}"), + false, + )); + } + }; + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); self.shared.bump(); - Ok(()) + self.immediate_response(request, RequestOutcomeV1::Applied, None) + } + ModulationCommandV1::PrepareA1 { configuration } => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.a1_configuration = Some(configuration.clone()); + target.acquisition_running = false; + self.queue_service_operation( + request, + target, + vec![ + Command::new("STOP").field("reason", "prepare_a1"), + a1_config_command(configuration), + ], + "PREPARE_A1", + false, + ) + } + ModulationCommandV1::StartAcquisition => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.acquisition_running = true; + self.queue_service_operation( + request, + target, + vec![Command::new("START")], + "START", + false, + ) + } + ModulationCommandV1::StopAcquisition { reason } => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.acquisition_running = false; + self.queue_service_operation( + request, + target, + vec![Command::new("STOP").field("reason", reason.replace(' ', "_"))], + "STOP", + false, + ) } - _ => Err(format!("unknown setting: {key}")), } } - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); + fn immediate_response( + &mut self, + request: &ModulationRequestV1, + outcome: RequestOutcomeV1, + error: Option, + ) -> Result { let state = self.shared.state.lock().expect("device state lock"); - entries.push(StatusEntry::Text(if state.connected { - format!("Modulation: connected ({})", state.firmware) + let response = ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: request.request_id, + owner_instance: self.owner_instance.clone(), + run_id: request.run_id.clone(), + requested_revision: request.requested_revision, + acknowledged_revision: state.acknowledged.as_ref().map(|value| value.revision), + outcome, + completed_at_unix_ms: Some(now_unix_ms()), + error, + }, + controller_state: state.controller_state, + acknowledged_target: state.acknowledged.clone(), + }; + drop(state); + self.shared + .state + .lock() + .expect("device state lock") + .last_response = Some(response.clone()); + self.shared.bump(); + Ok(response) + } + + fn control_state(&self) -> ModulationStateV1 { + let state = self.shared.state.lock().expect("device state lock"); + let connection = if state.connected { + ConnectionStateV1::Connected { + port_label: self.port_hint.clone(), + firmware_version: Some(state.firmware.clone()), + } + } else if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { + ConnectionStateV1::Faulted { message: error } } else if self.connect_requested { - "Modulation: connecting…".into() + ConnectionStateV1::Connecting } else { - "Modulation: disconnected".into() - })); - if let Some(code) = state.board_code { - entries.push(StatusEntry::Text(format!( - "Board: code={code} ({})", - state.board_mod - ))); - } - if let Some(run) = &self.protocol { - if let Ok(progress) = run.progress.lock() { - entries.push(StatusEntry::Text(if progress.finished { - if progress.stopped { - "Protocol: stopped (last step holds)".into() - } else { - "Protocol: finished (last step holds)".into() - } - } else { - format!( - "Protocol: loop {}/{} step {}/{} — {}", - progress.loop_index, - progress.loops, - progress.step_index, - progress.total_steps, - progress.summary - ) - })); + ConnectionStateV1::Disconnected + }; + let synchronization = match ( + self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + state.requested.as_ref(), + state.acknowledged.as_ref(), + ) { + (Some(run_id), Some(requested), Some(acknowledged)) + if requested.revision == acknowledged.revision => + { + SynchronizationV1::Synced { + run_id, + acknowledged_revision: acknowledged.revision, + stream_epoch: None, + } } + (None, _, _) => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::NoLease, + detail: None, + }, + _ => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::RequestedRevisionNotAcknowledged, + detail: None, + }, + }; + ModulationStateV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: self.owner_instance.clone(), + service_revision: self.shared.generation.load(Ordering::Relaxed), + connection, + capabilities: state.capabilities.clone(), + lease: self.lease_snapshot(), + controller_state: state.controller_state, + active_run_id: self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + requested: state.requested.clone(), + // Service-path acknowledgements win; otherwise expose the + // board-echoed operator-armed drive (revision 0) so consumers + // like A1 can read the modulation frequency without a lease ever + // having existed. + acknowledged: state + .acknowledged + .clone() + .or_else(|| state.board_echo_target()), + synchronization, + last_response: state.last_response.clone(), + freshness: FreshnessV1 { + observed_at_unix_ms: if state.last_device_update_unix_ms == 0 { + now_unix_ms() + } else { + state.last_device_update_unix_ms + }, + valid_for_ms: 1_500, + }, + calibration_id: self.calibration_id.clone(), } - if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { - entries.push(StatusEntry::Text(format!("Error: {error}"))); - } - entries } - fn host_views(&self) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "Laser modulation".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Modulation control idle.".into(), - display: None, - relations: Vec::new(), - }], - views: vec![HostViewDescriptor { - id: STATUS_VIEW_ID.into(), - title: "Laser modulation".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }], - actions: Vec::new(), + fn expire_lease_if_needed(&mut self) { + let expired = self + .lease + .as_ref() + .is_some_and(|lease| now_unix_ms() > lease.expires_at_unix_ms); + if !expired { + return; } + self.protocol = None; + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); + *self.shared.pending.lock().expect("pending lock") = None; + *self.shared.priority.lock().expect("priority lock") = Some(PendingOperation { + commands: vec![ + Command::new("STOP").field("reason", "lease_expired"), + Command::new("MOD").field("wave", "OFF"), + ], + purpose: "LEASE_EXPIRED_SAFE_OFF", + meta: None, + }); + self.lease = None; + self.last_error = Some("automation lease expired; queued STOP + output off".into()); + self.shared.bump(); } - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, + fn advance_deferred_release(&mut self) { + let Some(request_id) = self.deferred_release_request else { + return; + }; + let terminal_applied = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.last_response.clone()) + .is_some_and(|response| { + response.common.request_id == request_id + && response.common.outcome == RequestOutcomeV1::Applied + }); + if !terminal_applied { + return; + } + if self.deferred_release_ack_published { + self.lease = None; + self.deferred_release_request = None; + self.deferred_release_ack_published = false; + self.shared + .fail_closed_on_stop + .store(false, Ordering::Relaxed); + self.shared.bump(); + } else { + // Preserve the lease for one complete snapshot publication so + // the orchestrator can consume the terminal ACK before the owner + // advertises the release. + self.deferred_release_ack_published = true; } } - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - STATUS_DATASET_ID => self.shared.generation.load(Ordering::Relaxed).max(1), - _ => 0, + fn apply_execution_context(&mut self, execution: &augur_plugin_api::ExecutionContext) { + let allowed = self.runtime_role == PluginRuntimeRole::LiveWorker + && execution.hardware_effects_allowed(); + self.effects_allowed = allowed; + if !allowed { + if self.link.is_some() { + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + } + self.lease = None; + self.deferred_release_request = None; + self.deferred_release_ack_published = false; + return; + } + self.expire_lease_if_needed(); + self.advance_deferred_release(); + // Reap a dead device thread (failed HELLO, wedged serial): a finished + // thread leaves `link` occupied, which both swallows every queued + // command (the settings UI keeps responding while the board holds the + // old waveform) and blocks the auto-reconnect below. + if self + .link + .as_ref() + .and_then(|link| link.join.as_ref()) + .is_some_and(JoinHandle::is_finished) + { + self.link = None; + } + if self.connect_requested && self.link.is_none() { + let now_ms = now_unix_ms(); + if now_ms.saturating_sub(self.last_reconnect_ms) >= RECONNECT_BACKOFF_MS { + self.last_reconnect_ms = now_ms; + self.connect(); + } } } -} -impl Drop for StageAModulationPlugin { - fn drop(&mut self) { - self.disconnect(); + #[cfg(test)] + fn device_connected(&self) -> bool { + self.shared + .state + .lock() + .map(|state| state.connected) + .unwrap_or(false) } -} -export_plugin!(StageAModulationPlugin); + fn commanded_summary(&self) -> String { + match self.dac_band() { + Ok((lo, hi, hold)) if self.mode == Mode::Const => format!( + "{} {} hold={} (band {}..{})", + self.method.name(), + self.mode.name(), + hold, + lo, + hi + ), + Ok((lo, hi, _)) => format!( + "{} {} {}..{} @ {:.3} Hz", + self.method.name(), + self.mode.name(), + lo, + hi, + self.frequency_hz + ), + Err(error) => format!( + "{} {} invalid: {error}", + self.method.name(), + self.mode.name() + ), + } + } -#[cfg(test)] -mod tests { - use super::*; + /// The transfer curve the operator reasons about. Before any sweep it shows + /// the lobe the *configured* `V_null`/`Vπ` claim, on a normalised axis, so + /// the two numbers are legible with no hardware attached; after a fit it + /// shows what was actually measured, in detector volts. + fn curve_dataset(&self) -> Series1dV1 { + let max_code = self.max_level.clamp(1, MAX_DAC_CODE) as f64; + let sample_curve = |scale: f64, offset: f64, inversion: waveform::LobeInversion| { + (0..=256) + .map(|step| { + let code = max_code * f64::from(step) / 256.0; + Series1dPoint { + x: code, + y: offset + scale * inversion.u_for_dac(code), + } + }) + .collect::>() + }; + // Two-point verticals mark the lobe endpoints on whatever y-range the + // rest of the plot spans. + let marker = |name: &str, code: f64, lo: f64, hi: f64| Series1dLine { + name: name.to_owned(), + points: vec![ + Series1dPoint { x: code, y: lo }, + Series1dPoint { x: code, y: hi }, + ], + }; - fn wait_until bool>( - plugin: &StageAModulationPlugin, - timeout: Duration, - done: F, - ) { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if done(plugin) { - return; - } - std::thread::sleep(Duration::from_millis(2)); + let Some(fit) = self.fit.as_ref() else { + let inversion = self.lobe_inversion(); + let mut lines = vec![Series1dLine { + name: "configured lobe".into(), + points: sample_curve(1.0, 0.0, inversion), + }]; + lines.push(marker("V_null", inversion.v_null_dac, 0.0, 1.0)); + lines.push(marker( + "V_null + Vπ", + inversion.v_null_dac + inversion.v_pi_dac, + 0.0, + 1.0, + )); + return Series1dV1 { + x_label: "DAC code".into(), + y_label: "normalised transmission u (not yet measured)".into(), + lines, + }; + }; + + let point_line = |direction: calibration::Direction| Series1dLine { + name: format!("measured {}", direction.label()), + points: fit + .points + .iter() + .filter(|point| point.direction == direction) + .map(|point| Series1dPoint { + x: f64::from(point.code), + y: point.volts, + }) + .collect(), + }; + let (lo, hi) = fit + .points + .iter() + .fold((f64::MAX, f64::MIN), |(lo, hi), point| { + (lo.min(point.volts), hi.max(point.volts)) + }); + let mut lines = vec![ + point_line(calibration::Direction::Ascending), + point_line(calibration::Direction::Descending), + Series1dLine { + name: "fit".into(), + points: sample_curve(fit.span_volts, fit.offset_volts, fit.inversion()), + }, + ]; + // The configured lobe on the fit's own scale: after applying they + // coincide, and any divergence is the un-applied difference. + let configured = self.lobe_inversion(); + if configured != fit.inversion() { + lines.push(Series1dLine { + name: "configured lobe".into(), + points: sample_curve(fit.span_volts, fit.offset_volts, configured), + }); } - panic!("condition not reached within {timeout:?}"); + lines.push(marker("V_null", fit.v_null_dac, lo, hi)); + lines.push(marker("V_null + Vπ", fit.v_null_dac + fit.v_pi_dac, lo, hi)); + Series1dV1 { + x_label: "DAC code".into(), + y_label: "photodiode [V]".into(), + lines, + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = self.shared.state.lock().expect("device state lock"); + let connection = if state.connected { + format!("connected ({})", state.firmware) + } else if self.connect_requested { + "connecting…".into() + } else { + "disconnected".into() + }; + let board_code = state + .board_code + .map_or_else(|| "—".into(), |code| code.to_string()); + let error = state + .last_error + .clone() + .or_else(|| self.last_error.clone()) + .unwrap_or_default(); + let board_mod = if state.board_mod.is_empty() { + "—".to_owned() + } else { + state.board_mod.clone() + }; + drop(state); + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", connection), + text_column("commanded", self.commanded_summary()), + text_column("board_mod", board_mod), + text_column("board_code", board_code), + text_column("error", error), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("commanded", "Commanded drive"), + column("board_mod", "Board modulation"), + column("board_code", "Board DAC code"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } +} + +/// `YYYYmmdd-HHMMSS` in UTC, from the wall clock alone (no chrono dependency). +fn timestamp_slug() -> String { + let seconds = now_unix_ms() / 1_000; + let (days, time) = (seconds / 86_400, seconds % 86_400); + // Civil-from-days, Howard Hinnant's algorithm, shifted to a 0000-03-01 era. + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_prime = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_prime + 2) / 5 + 1; + let month = if month_prime < 10 { + month_prime + 3 + } else { + month_prime - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + format!( + "{year:04}{month:02}{day:02}-{:02}{:02}{:02}", + time / 3_600, + (time % 3_600) / 60, + time % 60 + ) +} + +fn open_serial(port_hint: &str) -> Result, String> { + if port_hint == "auto" { + // The dual-serial Teensy enumerates two ports and only the command + // port answers HELLO — probe until one does. + let candidates = serial_ports(); + if candidates.is_empty() { + return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + } + let mut failures = Vec::new(); + for path in &candidates { + match probe_command_port(path) { + // Restore the client's default reply timeout after probing. + Ok(client) => return Ok(client.with_reply_timeout(Duration::from_millis(500))), + Err(err) => failures.push(format!("{path}: {err}")), + } + } + return Err(format!( + "no Teensy command port answered HELLO ({})", + failures.join("; ") + )); + } + open_path(port_hint) +} + +fn service_error( + code: ServiceErrorCodeV1, + message: impl Into, + retryable: bool, +) -> ServiceErrorV1 { + ServiceErrorV1 { + code, + message: message.into(), + retryable, + } +} + +fn lease_deadline(ttl_ms: u64) -> u64 { + now_unix_ms().saturating_add(ttl_ms.clamp(MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS)) +} + +/// Wire token for the optical target on the `MOD wave=WARP` command. +fn optical_target_token(target: waveform::OpticalTarget) -> &'static str { + match target { + waveform::OpticalTarget::LogSine => "LOG_SINE", + waveform::OpticalTarget::LinearSine => "LINEAR_SINE", + } +} + +fn waveform_command(waveform: &WaveformV1) -> Command { + match waveform { + WaveformV1::Off => Command::new("MOD").field("wave", "OFF"), + WaveformV1::Constant { level_dac } => Command::new("MOD") + .field("wave", "CONST") + .field("level", *level_dac), + WaveformV1::Periodic { + waveform, + min_dac, + max_dac, + frequency_millihz, + } => Command::new("MOD") + .field( + "wave", + match waveform { + stage_a_plugin_contract::PeriodicWaveformV1::Sine => "SINE", + stage_a_plugin_contract::PeriodicWaveformV1::Square => "SQUARE", + }, + ) + .field("level", *max_dac) + .field("min", *min_dac) + .field("freq_mhz", *frequency_millihz), + } +} + +fn a1_config_command(configuration: &A1AcquisitionConfigV1) -> Command { + Command::new("CONFIG") + .field("mode", "A1") + .field( + "wave", + match configuration.waveform { + stage_a_plugin_contract::PeriodicWaveformV1::Sine => "SINE", + stage_a_plugin_contract::PeriodicWaveformV1::Square => "SQUARE", + }, + ) + .field("freq_mhz", configuration.frequency_millihz) + .field("center_dac", configuration.center_dac) + .field("amplitude_dac", configuration.amplitude_dac) + .field("rate_hz", configuration.sample_rate_hz) + .field("block_samples", configuration.block_samples) + .field("raw", u8::from(configuration.emit_raw_samples)) + .field("summary", u8::from(configuration.emit_summary)) +} + +fn accepted_service_reply( + request: &PluginServiceRequest, + response: &ModulationResponseV1, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Accepted { + payload: serde_json::to_value(response).unwrap_or(Value::Null), + }, + } +} + +fn rejected_service_reply( + request: &PluginServiceRequest, + code: &str, + message: impl Into, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Rejected { + code: code.into(), + message: message.into(), + }, + } +} + +fn open_path(path: &str) -> Result, String> { + let transport = + stage_a_io::SerialTransport::open(path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +/// Opens `path` and sends HELLO with a short timeout: only the Teensy +/// command port replies (the photodiode stream port never answers). +fn probe_command_port(path: &str) -> Result, String> { + let mut client = open_path(path)?.with_reply_timeout(Duration::from_millis(300)); + client + .request(&Command::new("HELLO").field("protocol", 1)) + .map_err(|err| err.to_string())?; + Ok(client) +} + +fn serial_ports() -> Vec { + stage_a_io::transport::available_port_names() + .into_iter() + // macOS lists each device twice; use the callout (cu.*) node only. + .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) + .collect() +} + +/// The exact variant list the settings schema shows for the port enum — the +/// host exchanges enum settings as indices into this list. Real ports carry +/// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; +/// only the leading path is the value. +fn port_variants() -> Vec { + let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; + for (name, label) in stage_a_io::transport::available_ports_with_labels() { + if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { + continue; + } + variants.push(match label { + Some(label) => format!("{name} ({label})"), + None => name, + }); + } + variants +} + +/// The path part of a port variant; the parenthesised USB label is display-only. +fn variant_path(variant: &str) -> &str { + variant.split_whitespace().next().unwrap_or(variant) +} + +/// Host enum widgets send the selected index; string names are also accepted +/// (tests, saved configs). +fn enum_choice(value: &Value, variants: &[String]) -> Result { + if let Some(index) = value.as_u64() { + return variants + .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) + .cloned() + .ok_or_else(|| format!("enum index {index} out of range")); + } + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| "expected an enum index or name".to_owned()) +} + +impl Plugin for StageAModulationPlugin { + fn name(&self) -> &'static str { + "Stage-A Modulation" + } + + fn description(&self) -> &'static str { + "Laser modulation control on the Teensy command port: capped power slider, constant/sine/square with frequency, applied immediately; shows the DAC code the board reports." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.connect_requested = false; + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + self.lease = None; + self.deferred_release_request = None; + } + } + + fn set_runtime_role(&mut self, role: PluginRuntimeRole) { + self.runtime_role = role; + if role != PluginRuntimeRole::LiveWorker { + self.effects_allowed = false; + if self.link.is_some() { + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + } + self.lease = None; + self.deferred_release_request = None; + self.deferred_release_ack_published = false; + } + } + + fn reset(&mut self) {} + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Control is settings-driven and works without camera frames. The + // only frame-pass policy: replaying a recording must never keep a + // hardware connection alive. + if context.execution().mode == ExecutionMode::Replay && self.link.is_some() { + self.connect_requested = false; + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + self.lease = None; + self.last_error = Some("disconnected: replay mode".into()); + } + } + + fn process_control(&mut self, context: &mut PluginControlContext<'_>) { + let execution = context.execution(); + self.apply_execution_context(&execution); + // The photodiode owner broadcasts its summary to every plugin's inbox, + // so a calibration sweep reads the light with no lease and no request. + let level = context + .inbox() + .snapshots + .iter() + .find(|snapshot| { + snapshot.plugin_id == PLUGIN_ID_STAGE_A_PHOTODIODE + && snapshot.topic == CTX_STAGE_A_PHOTODIODE_SUMMARY_V1 + }) + .and_then(|snapshot| { + serde_json::from_value::(snapshot.payload.clone()).ok() + }) + .and_then(|summary| summary.stream.level); + self.drive_calibration(level); + } + + fn handle_service_request( + &mut self, + request: &PluginServiceRequest, + execution: &augur_plugin_api::ExecutionContext, + ) -> PluginServiceReply { + if let Some(index) = self.request_cache.iter().position(|(previous, _)| { + previous.source_plugin_id == request.source_plugin_id + && previous.request_id == request.request_id + }) { + let (previous, cached_reply) = self.request_cache[index].clone(); + if previous != *request { + return rejected_service_reply( + request, + "request_id_conflict", + "request ID was reused for different modulation payload", + ); + } + let cached_in_progress = match &cached_reply.outcome { + PluginServiceOutcome::Accepted { payload } => serde_json::from_value::< + ModulationResponseV1, + >(payload.clone()) + .is_ok_and(|response| response.common.outcome == RequestOutcomeV1::InProgress), + PluginServiceOutcome::Rejected { .. } => false, + }; + if cached_in_progress { + let terminal = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.last_response.clone()) + .filter(|response| { + response.common.request_id.0 == request.request_id + && response.common.outcome != RequestOutcomeV1::InProgress + }); + if let Some(terminal) = terminal { + let upgraded = accepted_service_reply(request, &terminal); + self.request_cache[index].1 = upgraded.clone(); + return upgraded; + } + } + return cached_reply; + } + + let reply = if request.target_plugin_id != PLUGIN_ID_STAGE_A_MODULATION { + rejected_service_reply(request, "wrong_target", "wrong modulation owner target") + } else if request.service != SERVICE_STAGE_A_MODULATION_CONTROL_V1 { + rejected_service_reply( + request, + "unsupported_service", + format!("unsupported modulation service '{}'", request.service), + ) + } else if self.runtime_role != PluginRuntimeRole::LiveWorker + || !execution.hardware_effects_allowed() + { + rejected_service_reply( + request, + "effects_not_allowed", + "modulation effects are allowed only on the active live worker", + ) + } else { + self.effects_allowed = true; + match serde_json::from_value::(request.payload.clone()) { + Err(err) => rejected_service_reply( + request, + "invalid_payload", + format!("invalid modulation request: {err}"), + ), + Ok(payload) + if payload.contract_version != CONTRACT_VERSION_V1 + || payload.request_id.0 != request.request_id + || payload.requester.as_str() != request.source_plugin_id + || payload + .target_owner_instance + .as_ref() + .is_some_and(|owner| owner != &self.owner_instance) => + { + rejected_service_reply( + request, + "identity_mismatch", + "contract version, request, requester, or owner instance mismatch", + ) + } + Ok(payload) + if payload.issued_at_unix_ms != 0 + && (now_unix_ms().saturating_sub(payload.issued_at_unix_ms) > 120_000 + || payload.issued_at_unix_ms.saturating_sub(now_unix_ms()) + > 30_000) => + { + rejected_service_reply(request, "stale_request", "request timestamp is stale") + } + Ok(payload) => match self.handle_modulation_command(&payload) { + Ok(response) => accepted_service_reply(request, &response), + Err(error) => rejected_service_reply( + request, + &format!("{:?}", error.code).to_ascii_lowercase(), + error.message, + ), + }, + } + }; + self.request_cache + .push_back((request.clone(), reply.clone())); + while self.request_cache.len() > REQUEST_CACHE_LIMIT { + self.request_cache.pop_front(); + } + reply + } + + fn control_snapshots(&self) -> Vec { + vec![PluginControlSnapshot { + plugin_id: PLUGIN_ID_STAGE_A_MODULATION.into(), + topic: CTX_STAGE_A_MODULATION_STATE_V1.into(), + revision: self.shared.generation.load(Ordering::Relaxed).max(1), + payload: serde_json::to_value(self.control_state()).unwrap_or(Value::Null), + }] + } + + fn settings_schema(&self) -> SettingsSchema { + let port_variants = port_variants(); + let port_default = port_variants + .iter() + .position(|p| variant_path(p) == self.port_hint) + .unwrap_or(0); + let method_variants: Vec = DriveMethod::VARIANTS + .iter() + .map(|method| method.name().to_owned()) + .collect(); + let method_default = DriveMethod::VARIANTS + .iter() + .position(|method| *method == self.method) + .unwrap_or(0); + let mode_variants: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let mode_default = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + let mut modulation_items = vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "auto (recommended) probes the attached usbmodem ports and picks \ + the one that answers HELLO — the Teensy command port; \ + mock = in-process simulated controller" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the command port. Connecting never changes the \ + output; disconnecting leaves it held (set-and-hold firmware)." + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, + }, + SettingItem { + key: "max_level".into(), + label: "Max limit (DAC code)".into(), + tooltip: Some( + "Hard ceiling for every drive. No manual or calibrated waveform may \ + produce a DAC code above this value at J23." + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.max_level, + }, + }, + SettingItem { + key: "method".into(), + label: "Drive method".into(), + tooltip: Some( + "MANUAL defines the DAC band with Power and Min threshold. CALIBRATED \ + derives it from V_null, Vπ, I_k, and optical depth a." + .into(), + ), + kind: SettingKind::Enum { + variants: method_variants, + default: method_default, + }, + }, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some( + "Selects the waveform that fills the method-defined operating band. \ + All five modes are available with both drive methods." + .into(), + ), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, + }, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Periodic-waveform frequency, 0.01–2000 Hz".into()), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.frequency_hz, + }, + }, + ]; + match self.method { + DriveMethod::Manual => { + modulation_items.push(SettingItem { + key: "level".into(), + label: "Power (DAC code)".into(), + tooltip: Some( + "Manual peak/operating DAC code. CONST holds this value; periodic \ + modes use it as the upper end of the manual band." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.level, + suffix: None, + }, + }); + modulation_items.push(SettingItem { + key: "min_level".into(), + label: "Min threshold (DAC code)".into(), + tooltip: Some( + "Lower end of the manual DAC band. Ignored by CONST, which holds Power." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.min_level, + suffix: None, + }, + }); + } + DriveMethod::Calibrated => { + modulation_items.push(SettingItem { + key: "v_null_dac".into(), + label: "V_null (DAC code at min light)".into(), + tooltip: Some( + "DAC code where excitation light bottoms out (sin² = 0) on one \ + monotonic Pockels lobe. Measure it; do not trust nominal Vπ." + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.v_null_dac, + }, + }); + modulation_items.push(SettingItem { + key: "v_pi_dac".into(), + label: "Vπ (DAC codes, null → max light)".into(), + tooltip: Some( + "DAC-code quarter-wave distance from V_null to the excitation \ + maximum. V_null + Vπ must stay within 0..4095." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: MAX_DAC_CODE, + default: self.v_pi_dac, + }, + }); + modulation_items.push(SettingItem { + key: "operating_point".into(), + label: "Operating point I_k (0..1)".into(), + tooltip: Some( + "Calibrated operating illumination as normalised lobe intensity u_k. \ + CONST holds its DAC code; the calibrated band is derived around it." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 1.0, + speed: 0.01, + default: self.operating_point, + }, + }); + modulation_items.push(SettingItem { + key: "depth_a".into(), + label: "Optical depth a".into(), + tooltip: Some( + "Calibrated log-intensity span a = ln(I_max/I_min). Together with I_k \ + it defines the operating band used by every mode." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 6.0, + speed: 0.01, + default: self.depth_a, + }, + }); + } + } + let geometry_variants: Vec = calibration::DetectorGeometry::VARIANTS + .iter() + .map(|geometry| geometry.name().to_owned()) + .collect(); + let geometry_default = calibration::DetectorGeometry::VARIANTS + .iter() + .position(|geometry| *geometry == self.detector_geometry) + .unwrap_or(0); + SettingsSchema { + sections: vec![ + SettingsSection { + label: "Laser modulation".into(), + description: Some( + "Tick Connect, then every change is sent to the Teensy immediately — no \ + camera required. Method selects the operating band; Mode selects its \ + waveform. Max limit is the hard ceiling. The firmware holds the output \ + when disconnected." + .into(), + ), + default_open: true, + items: modulation_items, + }, + SettingsSection { + label: "Calibration".into(), + description: Some( + "Measures the Pockels/PBS transfer curve: steps settled CONST DAC codes \ + across the range while reading the photodiode, then fits V_null and Vπ. \ + Needs the photodiode plugin connected. The sweep restores your armed \ + drive when it finishes, and the fit is never applied without your \ + confirmation. Watch the transfer-curve view." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "detector_geometry".into(), + label: "Detector port".into(), + tooltip: Some( + "Which way the photodiode moves when the light reaching the \ + sample gets brighter. Stage-A's photodiode sits on the PBS \ + reject port and reads the leftover light, I_pd = I_tot − I_exc, \ + so it goes DOWN as the sample gets brighter — that is REJECT \ + PORT, the default. Pick DIRECT only for a detector that watches \ + the sample beam itself. The sweep cannot work this out: a bright \ + and a dark extremum fit the measured curve equally well, and \ + only the optics say which one is zero light on the sample. \ + Choosing wrong puts V_null a quarter wave off." + .into(), + ), + kind: SettingKind::Enum { + variants: geometry_variants, + default: geometry_default, + }, + }, + SettingItem { + key: "calibrate".into(), + label: "Measure transfer curve".into(), + tooltip: Some( + "Sweeps the full range up and back down (~20 s), then fits the \ + lobe. Press again to abort; the armed drive is restored either \ + way. Progress and the result appear in the status lines below." + .into(), + ), + kind: SettingKind::Button { + enabled: self.calibration_offered(), + }, + }, + SettingItem { + key: "calibrate_apply".into(), + label: "Apply to V_null / Vπ".into(), + tooltip: Some( + "Writes the fitted lobe into the calibrated drive settings. \ + Refused, with the reason in the status lines, until a sweep has \ + produced a fit that is good enough to trust: residual within \ + 2 % of the detector span, at least three quarters of a lobe \ + covered, and no clipped point." + .into(), + ), + kind: SettingKind::Button { + enabled: self.calibration_offered(), + }, + }, + SettingItem { + key: "calibration_dir".into(), + label: "Calibration folder (optional)".into(), + tooltip: Some( + "Where the applied calibration record is archived, with its \ + points and fit. Leave empty to apply without archiving." + .into(), + ), + kind: SettingKind::Path { + dialog: PathDialogKind::Directory, + default: self.calibration_dir.clone(), + }, + }, + ], + }, + SettingsSection { + label: "Protocol".into(), + description: Some( + "Timed sequence of MOD steps from a TOML file: `loops = N` plus \ + [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, \ + min, frequency_hz. Steps run on an absolute schedule; the last \ + step holds after completion (set-and-hold). Stopping never \ + switches the output off by itself." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "protocol_path".into(), + label: "Protocol file".into(), + tooltip: Some("TOML protocol file (validated on start).".into()), + kind: SettingKind::Path { + dialog: PathDialogKind::OpenFile, + default: self.protocol_path.clone(), + }, + }, + SettingItem { + key: "protocol_run".into(), + label: "Run protocol".into(), + tooltip: Some( + "Start/stop the loaded protocol. Requires an open connection; \ + manual drive controls stay live and override the current step \ + until the next one begins." + .into(), + ), + kind: SettingKind::Bool { + default: if self.runtime_role == PluginRuntimeRole::LiveWorker { + self.protocol_active() + } else { + self.protocol_requested + }, + }, + }, + ], + }, + ], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + // Enum settings are exchanged as indices into the schema's + // variant list (see the host settings UI). + "port" => { + let index = port_variants() + .iter() + .position(|p| variant_path(p) == self.port_hint) + .unwrap_or(0); + Some(json!(index)) + } + "connect" => Some(json!(self.connect_requested)), + "level" => Some(json!(self.level)), + "max_level" => Some(json!(self.max_level)), + "method" => { + let index = DriveMethod::VARIANTS + .iter() + .position(|method| *method == self.method) + .unwrap_or(0); + Some(json!(index)) + } + "mode" => { + let index = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + Some(json!(index)) + } + "frequency_hz" => Some(json!(self.frequency_hz)), + "min_level" => Some(json!(self.min_level)), + "depth_a" => Some(json!(self.depth_a)), + "operating_point" => Some(json!(self.operating_point)), + "v_null_dac" => Some(json!(self.v_null_dac)), + "v_pi_dac" => Some(json!(self.v_pi_dac)), + "detector_geometry" => { + let index = calibration::DetectorGeometry::VARIANTS + .iter() + .position(|geometry| *geometry == self.detector_geometry) + .unwrap_or(0); + Some(json!(index)) + } + "calibration_dir" => Some(json!(self.calibration_dir)), + // Momentary buttons export a monotonic press counter so a press on + // the UI mirror reaches the live worker through the settings + // snapshot (ADR 010). + "calibrate" => Some(self.press_measure.value()), + "calibrate_apply" => Some(self.press_apply.value()), + "protocol_path" => Some(json!(self.protocol_path)), + // The live worker reports the actual run state; the UI mirror + // reports the operator's request so the settings snapshot can + // transport the start to the worker (which owns the device link). + "protocol_run" => Some(json!( + if self.runtime_role == PluginRuntimeRole::LiveWorker { + self.protocol_active() + } else { + self.protocol_requested + } + )), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + if self.lease.is_some() { + return Err(format!( + "setting '{key}' is locked while automation holds the modulation lease" + )); + } + match key { + "port" => { + self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); + Ok(()) + } + "connect" => { + let requested = value.as_bool().ok_or("connect must be a boolean")?; + self.connect_requested = requested; + if requested { + self.connect(); + } else { + self.disconnect(); + } + Ok(()) + } + "level" => { + self.level = value + .as_i64() + .ok_or("level must be an integer")? + .clamp(0, self.max_level); + if self.min_level > self.level { + self.min_level = self.level; + } + if self.method == DriveMethod::Manual { + self.send_modulation(); + } + Ok(()) + } + "max_level" => { + self.max_level = value + .as_i64() + .ok_or("max_level must be an integer")? + .clamp(0, MAX_DAC_CODE); + // Lowering the ceiling below the manual peak lowers that peak. + if self.level > self.max_level { + self.level = self.max_level; + } + if self.min_level > self.max_level { + self.min_level = self.max_level; + } + self.send_modulation(); + Ok(()) + } + "method" => { + let method_names: Vec = DriveMethod::VARIANTS + .iter() + .map(|method| method.name().to_owned()) + .collect(); + let name = enum_choice(&value, &method_names)?; + let method = DriveMethod::from_name(&name) + .ok_or_else(|| format!("unknown drive method: {name}"))?; + let previous = self.method; + self.method = method; + if let Err(error) = self.validate_drive() { + self.method = previous; + return Err(error); + } + self.last_error = None; + self.send_modulation(); + Ok(()) + } + "mode" => { + let mode_names: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let name = enum_choice(&value, &mode_names)?; + let mode = Mode::from_name(&name).ok_or_else(|| format!("unknown mode: {name}"))?; + let previous = self.mode; + self.mode = mode; + if let Err(error) = self.validate_drive() { + self.mode = previous; + return Err(error); + } + self.send_modulation(); + Ok(()) + } + "frequency_hz" => { + let hz = value.as_f64().ok_or("frequency_hz must be a number")?; + self.frequency_hz = hz.clamp(0.01, 2_000.0); + if self.mode.is_periodic() { + self.send_modulation(); + } + Ok(()) + } + "min_level" => { + self.min_level = value + .as_i64() + .ok_or("min_level must be an integer")? + .clamp(0, self.level); + if self.method == DriveMethod::Manual { + self.send_modulation(); + } + Ok(()) + } + "depth_a" => { + let depth_a = value + .as_f64() + .ok_or("depth_a must be a number")? + .clamp(0.01, 6.0); + let previous = self.depth_a; + self.depth_a = depth_a; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.depth_a = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "operating_point" => { + let operating_point = value + .as_f64() + .ok_or("operating_point must be a number")? + .clamp(0.01, 1.0); + let previous = self.operating_point; + self.operating_point = operating_point; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.operating_point = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "v_null_dac" => { + let v_null_dac = value + .as_i64() + .ok_or("v_null_dac must be an integer")? + .clamp(0, MAX_DAC_CODE); + let previous = self.v_null_dac; + self.v_null_dac = v_null_dac; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.v_null_dac = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "v_pi_dac" => { + let v_pi_dac = value + .as_i64() + .ok_or("v_pi_dac must be an integer")? + .clamp(1, MAX_DAC_CODE); + let previous = self.v_pi_dac; + self.v_pi_dac = v_pi_dac; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.v_pi_dac = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "detector_geometry" => { + let variants: Vec = calibration::DetectorGeometry::VARIANTS + .iter() + .map(|geometry| geometry.name().to_owned()) + .collect(); + let chosen = enum_choice(&value, &variants)?; + self.detector_geometry = calibration::DetectorGeometry::from_name(&chosen) + .ok_or("unknown detector geometry")?; + // The stored fit was resolved against the old geometry; re-fit + // rather than leave a V_null that is now a quarter wave out. + if let Some(fit) = self.fit.take() { + match calibration::fit_transfer( + &fit.points, + f64::from(self.max_level.clamp(1, MAX_DAC_CODE) as u16), + self.detector_geometry, + ) { + Ok(refitted) => self.fit = Some(refitted), + Err(error) => self.calibration_status = format!("re-fit failed: {error}"), + } + } + Ok(()) + } + "calibration_dir" => { + self.calibration_dir = value + .as_str() + .ok_or("calibration_dir must be a string")? + .to_owned(); + Ok(()) + } + "calibrate" => { + if !self.press_measure.accept(&value) { + return Ok(()); + } + if self.sweep.is_some() { + self.finish_calibration_sweep("sweep stopped".into()); + } else { + self.start_calibration_sweep(); + } + self.shared.bump(); + Ok(()) + } + "calibrate_apply" => { + if !self.press_apply.accept(&value) { + return Ok(()); + } + self.apply_calibration_fit(); + Ok(()) + } + "protocol_path" => { + self.protocol_path = value + .as_str() + .ok_or("protocol_path must be a string")? + .to_owned(); + Ok(()) + } + "protocol_run" => { + let requested = value.as_bool().ok_or("protocol_run must be a boolean")?; + // The host re-applies the full settings snapshot on every + // sync, so only value *transitions* are actions — otherwise a + // finished protocol would silently restart on the next sync. + if requested == self.protocol_requested { + return Ok(()); + } + self.protocol_requested = requested; + if requested { + // Only the live worker owns the device link; the UI mirror + // records the request and the settings snapshot starts the + // protocol on the worker. Failures surface through status + // entries (like `connect`). + if self.runtime_role == PluginRuntimeRole::LiveWorker { + match self.start_protocol() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + } + } else { + self.stop_protocol(); + } + self.shared.bump(); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + let state = self.shared.state.lock().expect("device state lock"); + entries.push(StatusEntry::Text(if state.connected { + format!("Modulation: connected ({})", state.firmware) + } else if self.connect_requested { + "Modulation: connecting…".into() + } else { + "Modulation: disconnected".into() + })); + if let Some(code) = state.board_code { + entries.push(StatusEntry::Text(format!( + "Board: code={code} ({})", + state.board_mod + ))); + } + entries.push(StatusEntry::Text(format!( + "Drive: method={}, mode={}", + self.method.name(), + self.mode.name() + ))); + match self.dac_band() { + Ok((lo, hi, hold)) => entries.push(StatusEntry::Text(format!( + "Resolved DAC band: {lo}..{hi} (hold {hold}, {} codes peak-to-peak)", + hi.saturating_sub(lo) + ))), + Err(error) => entries.push(StatusEntry::Text(format!( + "Resolved DAC band invalid: {error}" + ))), + } + if let Some(target) = self.mode.optical_target() { + match self.optical_warp_table(target) { + Ok(_) => { + let drive = self.optical_drive(target); + entries.push(StatusEntry::Text(format!( + "{}: a={:.2}, I_k={:.2}, V_null={}, Vπ={} @ {:.3} Hz", + self.mode.name(), + drive.depth_a, + drive.operating_point, + self.v_null_dac, + self.v_pi_dac, + self.frequency_hz, + ))); + } + Err(error) => { + entries.push(StatusEntry::Text(format!("Optical drive invalid: {error}"))) + } + } + } + if let Some(run) = &self.protocol { + if let Ok(progress) = run.progress.lock() { + entries.push(StatusEntry::Text(if progress.finished { + if progress.stopped { + "Protocol: stopped (last step holds)".into() + } else { + "Protocol: finished (last step holds)".into() + } + } else { + format!( + "Protocol: loop {}/{} step {}/{} — {}", + progress.loop_index, + progress.loops, + progress.step_index, + progress.total_steps, + progress.summary + ) + })); + } + } + if let Some(sweep) = self.sweep.as_ref() { + entries.push(StatusEntry::Text(format!( + "Calibration: sweeping {}/{}", + sweep.index + 1, + sweep.total() + ))); + } else if !self.calibration_status.is_empty() { + entries.push(StatusEntry::Text(format!( + "Calibration: {}", + self.calibration_status + ))); + } + if let Some(fit) = self.fit.as_ref() { + // The reject-port extremum bounds the anchor from below but is not + // the anchor: the residual transmitted floor is not separable here + // (knowledge base `pockels-waveform-linearisation.md` §4.4). + entries.push(StatusEntry::Text(format!( + "Detector at null: {:.3} V — lower bound on the total-power anchor I_tot, \ + not the anchor itself", + fit.detector_volts_at_null() + ))); + for warning in self.fit_warnings() { + entries.push(StatusEntry::Text(format!("Check: {warning}"))); + } + } + if let Some(calibration_id) = self.calibration_id.as_ref() { + entries.push(StatusEntry::Text(format!( + "Calibration in use: {calibration_id}" + ))); + } + if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { + entries.push(StatusEntry::Text(format!("Error: {error}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Laser modulation".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Modulation control idle.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: CURVE_DATASET_ID.into(), + title: "Pockels transfer curve".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Set V_null and Vπ, or measure a transfer curve.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Laser modulation".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: CURVE_VIEW_ID.into(), + title: "Pockels transfer curve".into(), + dataset_id: CURVE_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::LineSeriesWindow, + }, + ], + actions: Vec::new(), + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + CURVE_DATASET_ID => serde_json::to_vec(&self.curve_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + STATUS_DATASET_ID | CURVE_DATASET_ID => { + self.shared.generation.load(Ordering::Relaxed).max(1) + } + _ => 0, + } + } +} + +impl Drop for StageAModulationPlugin { + fn drop(&mut self) { + self.disconnect(); + } +} + +export_plugin!(StageAModulationPlugin); + +#[cfg(test)] +mod tests { + use super::*; + use augur_plugin_api::{ExecutionContext, ExecutionMode}; + + fn live_execution() -> ExecutionContext { + ExecutionContext { + mode: ExecutionMode::LiveCapture, + effects_allowed: true, + session_id: Some("test".into()), + } + } + + fn service_request( + plugin: &StageAModulationPlugin, + id: u64, + requester: &str, + command: ModulationCommandV1, + revision: Option, + ) -> PluginServiceRequest { + let mut payload = ModulationRequestV1::new( + stage_a_plugin_contract::RequestId(id), + ClientId::from(requester), + command, + ); + payload.target_owner_instance = Some(plugin.owner_instance.clone()); + payload.run_id = Some(RunId::from("run-a")); + payload.lease_id = Some(LeaseId::from("lease-a")); + payload.requested_revision = revision.map(SemanticRevision); + payload.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id: id, + source_plugin_id: requester.into(), + target_plugin_id: PLUGIN_ID_STAGE_A_MODULATION.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + payload: serde_json::to_value(payload).unwrap(), + } + } + + fn live_plugin() -> StageAModulationPlugin { + let mut plugin = StageAModulationPlugin::default(); + plugin.set_runtime_role(PluginRuntimeRole::LiveWorker); + plugin.effects_allowed = true; + plugin + } + + fn wait_until bool>( + plugin: &StageAModulationPlugin, + timeout: Duration, + done: F, + ) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if done(plugin) { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("condition not reached within {timeout:?}"); + } + + fn board_code(plugin: &StageAModulationPlugin) -> Option { + plugin.shared.state.lock().unwrap().board_code + } + + /// Connect checkbox → slider change → MOD sent by the device thread → + /// board echoes the code. No process_frame involved anywhere. + #[test] + fn level_change_transfers_without_frames() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + assert_eq!( + plugin.shared.state.lock().unwrap().firmware, + "0.3.0-mock".to_owned() + ); + + plugin.set_setting("level", json!(1234)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1234) + }); + assert!(plugin.shared.state.lock().unwrap().last_error.is_none()); + + plugin.set_setting("connect", json!(false)).unwrap(); + assert!(!plugin.device_connected()); + } + + /// The max cap bounds the slider, and lowering it re-sends a lower level. + #[test] + fn max_level_caps_the_slider() { + let mut plugin = live_plugin(); + plugin.set_setting("max_level", json!(1000)).unwrap(); + plugin.set_setting("level", json!(4095)).unwrap(); + assert_eq!(plugin.level, 1000, "slider clamps to the cap"); + + plugin.set_setting("max_level", json!(500)).unwrap(); + assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); + + let schema = plugin.settings_schema(); + let level_item = schema.sections[0] + .items + .iter() + .find(|item| item.key == "level") + .expect("level setting exists"); + match &level_item.kind { + SettingKind::I64Slider { max, .. } => assert_eq!(*max, 500), + other => panic!("level must stay a slider, got {other:?}"), + } + } + + /// Square drive with min threshold reaches the mock and starts at min; + /// slider to 0 drives the output to 0. + #[test] + fn square_with_min_threshold_round_trips() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + + plugin.set_setting("level", json!(2000)).unwrap(); + plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); + plugin.set_setting("min_level", json!(500)).unwrap(); + plugin.set_setting("mode", json!("SQUARE")).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(500) + }); + assert!(plugin + .shared + .state + .lock() + .unwrap() + .board_mod + .contains("SQUARE 500..2000")); + + plugin.set_setting("mode", json!("CONST")).unwrap(); + plugin.set_setting("level", json!(0)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(0) + }); + plugin.set_setting("connect", json!(false)).unwrap(); + } + + const TEST_PROTOCOL: &str = r#" +loops = 2 + +[[steps]] +duration_s = 0.03 +wave = "SINE" +level = 2000 +min = 100 +frequency_hz = 100.0 + +[[steps]] +duration_s = 0.03 +wave = "CONST" +level = 750 +"#; + + #[test] + fn protocol_parsing_validates_steps() { + let (steps, loops) = parse_protocol(TEST_PROTOCOL).expect("valid protocol"); + assert_eq!(loops, 2); + assert_eq!(steps.len(), 2); + let encoded = |command: &Command, seq: u32| { + String::from_utf8(command.encode(seq).expect("encodes")).expect("utf8") + }; + assert_eq!( + encoded(&steps[0].command, 1), + "@1 MOD wave=SINE level=2000 min=100 freq_mhz=100000\n" + ); + assert_eq!( + encoded(&steps[1].command, 2), + "@2 MOD wave=CONST level=750\n" + ); + assert!((steps[0].duration.as_secs_f64() - 0.03).abs() < 1e-9); + + assert!(parse_protocol("loops = 1").is_err(), "steps required"); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100").is_err(), + "periodic steps need a frequency" + ); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"CONST\"\nlevel = 9999").is_err(), + "level range enforced" + ); + assert!( + parse_protocol( + "[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100\nmin = 200\nfrequency_hz = 10.0" + ) + .is_err(), + "min above level rejected" + ); + let (off, _) = parse_protocol("[[steps]]\nduration_s = 0.5\nwave = \"OFF\"") + .expect("OFF needs no level"); + assert_eq!(encoded(&off[0].command, 1), "@1 MOD wave=OFF\n"); + } + + /// A protocol against the mock walks every step, holds the last one, and + /// reports finished. + #[test] + fn protocol_runs_to_completion_on_the_mock() { + let dir = std::env::temp_dir().join(format!( + "stage-a-modulation-protocol-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("protocol.toml"); + std::fs::write(&path, TEST_PROTOCOL).unwrap(); + + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + + plugin + .set_setting("protocol_path", json!(path.display().to_string())) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(true))); + + // 2 loops × 2 steps × 30 ms ≈ 120 ms; wait for the final CONST 750. + wait_until(&plugin, Duration::from_secs(3), |p| { + !p.protocol_active() && board_code(p) == Some(750) + }); + assert!(!plugin.protocol_active()); + assert_eq!(board_code(&plugin), Some(750), "last step holds"); + let progress = plugin + .protocol + .as_ref() + .unwrap() + .progress + .lock() + .unwrap() + .clone(); + assert!(progress.finished && !progress.stopped); + assert_eq!((progress.loop_index, progress.step_index), (2, 2)); + + plugin.set_setting("connect", json!(false)).unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn protocol_requires_a_connection() { + let mut plugin = live_plugin(); + plugin + .set_setting("protocol_path", json!("/tmp/x.toml")) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("connect"))); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(false))); + } + + /// The host settings UI exchanges enum values as indices into the + /// schema's variant list (radio buttons send `json!(index)`). + #[test] + fn enum_settings_round_trip_as_indices() { + let mut plugin = live_plugin(); + // Drive method: index 1 = CALIBRATED. + plugin + .set_setting("method", json!(1)) + .expect("method index accepted"); + assert_eq!(plugin.method, DriveMethod::Calibrated); + assert_eq!(plugin.get_setting("method"), Some(json!(1))); + // Mode: index 2 = SQUARE in the schema's variant order. + plugin + .set_setting("mode", json!(2)) + .expect("index accepted"); + assert_eq!(plugin.mode, Mode::Square); + assert_eq!(plugin.get_setting("mode"), Some(json!(2))); + // Port: index 1 = "mock" (variants start with auto, mock). + plugin + .set_setting("port", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.port_hint, "mock"); + assert_eq!(plugin.get_setting("port"), Some(json!(1))); + // Out-of-range indices are visible errors, not silent no-ops. + assert!(plugin.set_setting("mode", json!(99)).is_err()); + assert!(plugin.set_setting("method", json!(99)).is_err()); + // String names keep working (tests, saved configs). + plugin + .set_setting("mode", json!("SINE")) + .expect("name accepted"); + assert_eq!(plugin.mode, Mode::Sine); + plugin + .set_setting("method", json!("MANUAL")) + .expect("method name accepted"); + assert_eq!(plugin.method, DriveMethod::Manual); + } + + #[test] + fn method_switches_only_its_settings_block() { + let mut plugin = live_plugin(); + let keys = |plugin: &StageAModulationPlugin| { + plugin.settings_schema().sections[0] + .items + .iter() + .map(|item| item.key.clone()) + .collect::>() + }; + + let manual = keys(&plugin); + assert_eq!( + &manual[..6], + [ + "port", + "connect", + "max_level", + "method", + "mode", + "frequency_hz" + ] + ); + assert!(manual.iter().any(|key| key == "level")); + assert!(manual.iter().any(|key| key == "min_level")); + assert!(!manual.iter().any(|key| key == "depth_a")); + assert!(!manual.iter().any(|key| key == "operating_point")); + assert!(!manual.iter().any(|key| key == "v_null_dac")); + assert!(!manual.iter().any(|key| key == "v_pi_dac")); + + let schema = plugin.settings_schema(); + let mode = schema.sections[0] + .items + .iter() + .find(|item| item.key == "mode") + .expect("mode setting"); + match &mode.kind { + SettingKind::Enum { variants, .. } => { + assert_eq!(variants.len(), 5, "all modes stay available"); + } + other => panic!("mode must be an enum, got {other:?}"), + } + + plugin.last_error = Some("stale".into()); + plugin.set_setting("method", json!(1)).unwrap(); + assert!( + plugin.last_error.is_none(), + "method change clears stale errors" + ); + let calibrated = keys(&plugin); + assert_eq!( + &calibrated[..6], + [ + "port", + "connect", + "max_level", + "method", + "mode", + "frequency_hz" + ] + ); + assert!(!calibrated.iter().any(|key| key == "level")); + assert!(!calibrated.iter().any(|key| key == "min_level")); + assert!(calibrated.iter().any(|key| key == "depth_a")); + assert!(calibrated.iter().any(|key| key == "operating_point")); + assert!(calibrated.iter().any(|key| key == "v_null_dac")); + assert!(calibrated.iter().any(|key| key == "v_pi_dac")); + } + + #[test] + fn drive_method_resolves_manual_and_calibrated_bands() { + let mut plugin = live_plugin(); + plugin.level = 1_500; + plugin.min_level = 600; + assert_eq!(plugin.dac_band().unwrap(), (600, 1_500, 1_500)); + + plugin.method = DriveMethod::Calibrated; + // The ±a/2 band only exists for modulating modes (Const resolves to a + // pure hold since the full-lobe fix). + plugin.mode = Mode::Sine; + plugin.v_null_dac = 200; + plugin.v_pi_dac = 1_600; + plugin.operating_point = 0.4; + plugin.depth_a = 0.8; + let inversion = plugin.lobe_inversion(); + let expected_lo = inversion + .dac_for_u(plugin.operating_point * (-0.5 * plugin.depth_a).exp()) + .round() as i64; + let expected_hi = inversion + .dac_for_u(plugin.operating_point * (0.5 * plugin.depth_a).exp()) + .round() as i64; + let expected_hold = inversion.dac_for_u(plugin.operating_point).round() as i64; + assert_eq!( + plugin.dac_band().unwrap(), + (expected_lo, expected_hi, expected_hold) + ); + + plugin.max_level = expected_hi - 1; + assert!(plugin + .dac_band() + .unwrap_err() + .contains("exceeds the max limit")); + } + + #[test] + fn manual_optical_drive_is_derived_from_the_slider_band() { + let mut plugin = live_plugin(); + plugin.v_null_dac = 200; + plugin.v_pi_dac = 1_600; + plugin.min_level = 600; + plugin.level = 1_500; + plugin.depth_a = 5.0; + plugin.operating_point = 0.9; + + for target in [ + waveform::OpticalTarget::LogSine, + waveform::OpticalTarget::LinearSine, + ] { + let drive = plugin.optical_drive(target); + let table = plugin + .optical_warp_table(target) + .expect("valid manual band"); + let min = table.iter().copied().min().unwrap(); + let max = table.iter().copied().max().unwrap(); + assert!((i64::from(min) - plugin.min_level).abs() <= 1); + assert!((i64::from(max) - plugin.level).abs() <= 1); + assert_ne!(drive.depth_a, plugin.depth_a); + assert_ne!(drive.operating_point, plugin.operating_point); + } + } + + #[test] + fn every_mode_drives_under_both_methods() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.max_level = MAX_DAC_CODE; + plugin.min_level = 600; + plugin.level = 1_500; + plugin.v_null_dac = 200; + plugin.v_pi_dac = 1_600; + plugin.operating_point = 0.4; + plugin.depth_a = 0.8; + + for method in DriveMethod::VARIANTS { + plugin.method = method; + for mode in Mode::VARIANTS { + plugin.mode = mode; + plugin.shared.state.lock().unwrap().board_mod.clear(); + plugin.send_modulation(); + assert!( + plugin.last_error.is_none(), + "{} {}: {:?}", + method.name(), + mode.name(), + plugin.last_error + ); + let expected_wave = match mode { + Mode::Const => "CONST", + Mode::Sine => "SINE", + Mode::Square => "SQUARE", + Mode::OpticalLogSine | Mode::OpticalLinearSine => "WARP", + }; + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .board_mod + .starts_with(expected_wave) + }); + let board_mod = plugin.shared.state.lock().unwrap().board_mod.clone(); + assert!( + board_mod.starts_with(expected_wave), + "{} {} produced {board_mod}", + method.name(), + mode.name() + ); + } + } + plugin.disconnect(); + } + + /// min_level can never exceed the level. + #[test] + fn min_threshold_is_clamped_to_level() { + let mut plugin = live_plugin(); + plugin.set_setting("level", json!(1000)).unwrap(); + plugin.set_setting("min_level", json!(3000)).unwrap(); + assert_eq!(plugin.min_level, 1000); + plugin.set_setting("level", json!(200)).unwrap(); + assert_eq!(plugin.min_level, 200, "lowering level drags min down"); + } + + #[test] + fn ui_mirror_never_opens_the_command_port() { + let mut plugin = StageAModulationPlugin::default(); + plugin.port_hint = "mock".into(); + plugin.set_setting("connect", json!(true)).unwrap(); + assert!(plugin.link.is_none()); + assert!(!plugin.device_connected()); + assert!(matches!( + plugin + .handle_service_request( + &service_request( + &plugin, + 1, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ), + &live_execution(), + ) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + } + + /// Runs the sweep to completion against the mock board, synthesizing the + /// light the reject-port photodiode *would* report for whatever code the + /// board is actually holding. The fit must then recover the synthetic + /// lobe, which makes this a ground-truth check of the whole loop: + /// commanding, settle gating, point collection, and the fit. + fn run_sweep_against_mock(plugin: &mut StageAModulationPlugin, v_null: f64, v_pi: f64) { + let mut sample_index = 0_u64; + for _ in 0..4_000 { + let Some((code, _)) = plugin.sweep.as_ref().and_then(CalibrationSweep::current) else { + break; + }; + // Only report light once the board actually holds the commanded + // code. On the bench the settle margin covers the serial + // round-trip; here it is asserted, so a level can never be + // attributed to a code the board had not reached. + if plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.commanded_at_sample.is_some()) + { + wait_until(plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(i64::from(code)) + }); + } + let held = board_code(plugin).unwrap_or(0) as f64; + let u = (std::f64::consts::PI * (held - v_null) / (2.0 * v_pi)) + .sin() + .powi(2); + sample_index += SETTLE_SAMPLES; + plugin.drive_calibration(Some(PhotodiodeLevelV1 { + // Reject port: brightest at the excitation null. + mean_volts: 2.4 - 2.2 * u, + peak_to_peak_volts: 0.001, + sample_count: SETTLE_SAMPLES, + end_sample_index: sample_index, + clipped: false, + })); + } + } + + #[test] + fn sweep_recovers_a_synthetic_lobe_and_restores_the_armed_drive() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + // Arm a drive the sweep must put back afterwards. + plugin.set_setting("level", json!(1_234)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1_234) + }); + + plugin.set_setting("calibrate", json!(true)).unwrap(); + assert!(plugin.sweep.is_some(), "sweep started"); + run_sweep_against_mock(&mut plugin, 300.0, 1_600.0); + + assert!(plugin.sweep.is_none(), "sweep ran to completion"); + let fit = plugin.fit.as_ref().expect("produced a fit"); + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null {}", + fit.v_null_dac + ); + assert!((fit.v_pi_dac - 1_600.0).abs() < 10.0, "Vπ {}", fit.v_pi_dac); + assert_eq!(fit.points.len(), SWEEP_POINTS_PER_PASS * 2); + + // The armed drive is back on the board: a calibration sweep must leave + // the bench as it found it. + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1_234) + }); + + // Applying writes the lobe through and publishes a calibration id. + assert!( + plugin.fit_warnings().is_empty(), + "{:?}", + plugin.fit_warnings() + ); + plugin.set_setting("calibrate_apply", json!(true)).unwrap(); + assert_eq!(plugin.v_null_dac, 300); + assert!((plugin.v_pi_dac - 1_600).abs() <= 10); + assert!(plugin.calibration_id.is_some()); + assert!(plugin.control_state().calibration_id.is_some()); + } + + /// The host re-applies the **whole** settings snapshot on every sync, and + /// most drive handlers push to the board unconditionally. Without a guard + /// each sync re-arms the operator's waveform on top of the code the sweep + /// just commanded, so every point measures the armed drive instead of the + /// staircase and the fit sees a flat curve. + #[test] + fn a_settings_sync_during_a_sweep_does_not_re_arm_the_operator_drive() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + // Arm a periodic drive, as an operator would before calibrating. + let sine = Mode::VARIANTS + .iter() + .position(|m| *m == Mode::Sine) + .unwrap(); + plugin.set_setting("level", json!(3_000)).unwrap(); + plugin.set_setting("min_level", json!(1_000)).unwrap(); + plugin.set_setting("mode", json!(sine)).unwrap(); + // Let the device thread drain the armed drive, so anything still queued + // below is something the sync put there. + wait_until(&plugin, Duration::from_secs(2), |p| { + p.shared.pending.lock().unwrap().is_none() + }); + + plugin.set_setting("calibrate", json!(true)).unwrap(); + assert!(plugin.sweep.is_some()); + + // Exactly what `apply_live_plugin_snapshot` does: write every key back. + let resync = |plugin: &mut StageAModulationPlugin| { + for key in [ + "frequency_hz", + "level", + "max_level", + "method", + "min_level", + "mode", + "v_null_dac", + "v_pi_dac", + ] { + let value = plugin.get_setting(key).expect("exported"); + plugin.set_setting(key, value).expect("re-applies"); + } + }; + resync(&mut plugin); + assert!( + plugin.shared.pending.lock().unwrap().is_none(), + "a settings sync queued a drive while the sweep owned the DAC" + ); + + // With the sync fighting it on every tick, the sweep must still see the + // staircase and produce a usable fit. + let mut sample_index = 0_u64; + for _ in 0..4_000 { + let Some((code, _)) = plugin.sweep.as_ref().and_then(CalibrationSweep::current) else { + break; + }; + resync(&mut plugin); + if plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.commanded_at_sample.is_some()) + { + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(i64::from(code)) + }); + } + let held = board_code(&plugin).unwrap_or(0) as f64; + let u = (std::f64::consts::PI * (held - 300.0) / 3_200.0) + .sin() + .powi(2); + sample_index += SETTLE_SAMPLES; + plugin.drive_calibration(Some(PhotodiodeLevelV1 { + mean_volts: 2.4 - 2.2 * u, + peak_to_peak_volts: 0.001, + sample_count: SETTLE_SAMPLES, + end_sample_index: sample_index, + clipped: false, + })); + } + + let fit = plugin + .fit + .as_ref() + .unwrap_or_else(|| panic!("no fit: {}", plugin.calibration_status)); + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null {}", + fit.v_null_dac + ); + + // The armed sine comes back once the sweep releases the DAC. + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p).is_some_and(|code| code != 0) + }); + assert_eq!(plugin.mode, Mode::Sine); + } + + #[test] + fn sweep_waits_for_a_window_measured_after_the_code_was_commanded() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + plugin.set_setting("calibrate", json!(true)).unwrap(); + + let stale = |end_sample_index| PhotodiodeLevelV1 { + mean_volts: 1.0, + peak_to_peak_volts: 0.001, + sample_count: 100, + end_sample_index, + clipped: false, + }; + // First tick commands the point and adopts the sample index. + plugin.drive_calibration(Some(stale(10_000))); + assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 0); + // A window that began before the command must not be accepted, however + // many times it arrives — this is what makes settling provable. + for _ in 0..5 { + plugin.drive_calibration(Some(stale(10_050))); + } + assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 0); + // Once the window starts past the settle margin the point is taken. + plugin.drive_calibration(Some(stale(10_000 + SETTLE_SAMPLES + 100))); + assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 1); + } + + #[test] + fn sweep_is_refused_while_automation_holds_the_lease() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + plugin.handle_service_request( + &service_request( + &plugin, + 1, + "stage-a-a1", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ), + &live_execution(), + ); + assert!(plugin.lease.is_some()); + // Two owners stepping the same DAC would interleave silently. + plugin.start_calibration_sweep(); + assert!(plugin.sweep.is_none()); + assert!( + plugin.calibration_status.contains("leased"), + "{}", + plugin.calibration_status + ); + } + + fn synthetic_fit( + span: f64, + max_code: u16, + edit: impl Fn(&mut calibration::SweepPoint, usize), + ) -> calibration::TransferFit { + let points: Vec = calibration::sweep_codes(max_code, 49, false) + .into_iter() + .enumerate() + .map(|(index, (code, direction))| { + let u = (std::f64::consts::PI * (f64::from(code) - 300.0) / 1_720.0) + .sin() + .powi(2); + let mut point = calibration::SweepPoint { + code, + direction, + volts: 0.098 + span * u, + peak_to_peak_volts: 0.001, + clipped: false, + }; + edit(&mut point, index); + point + }) + .collect(); + calibration::fit_transfer( + &points, + f64::from(max_code), + calibration::DetectorGeometry::RejectedComplement, + ) + .expect("fits") + } + + /// A stray sample inflates the RMS residual several fold while leaving the + /// fitted period accurate. Dropping the wild points keeps the reported + /// residual describing the curve instead of the worst sample. + #[test] + fn a_stray_point_is_dropped_instead_of_ruining_the_fit() { + let clean = synthetic_fit(-0.090, 4_095, |_, _| {}); + let strayed = synthetic_fit(-0.090, 4_095, |point, index| { + if index == 20 { + point.volts += 0.09; + } + }); + + assert_eq!(clean.rejected_points, 0); + assert_eq!(strayed.rejected_points, 1, "the stray should be dropped"); + assert!( + (strayed.v_pi_dac - clean.v_pi_dac).abs() < 5.0, + "Vpi moved from {} to {}", + clean.v_pi_dac, + strayed.v_pi_dac + ); + assert!( + strayed.quality < 0.02, + "residual still dominated by the stray: {:.1}%", + strayed.quality * 100.0 + ); + // The plot still shows every measured point, stray included. + assert_eq!(strayed.points.len(), 49); + } + + /// A poor residual is the operator's call, made against the plot — it warns + /// but never blocks, because a stray sample can inflate it while the fitted + /// lobe stays good. The one genuinely meaningless case, a lobe that does not + /// fit inside the commandable range, is refused by the fit itself. + #[test] + fn a_scattered_or_clipped_fit_warns_but_still_applies() { + let mut plugin = live_plugin(); + plugin.fit = Some(synthetic_fit(-0.090, 4_095, |point, index| { + point.clipped = point.code < 100; + if index % 7 == 0 { + point.volts += 0.004; + } + })); + + let warnings = plugin.fit_warnings().join(" | "); + assert!(warnings.contains("clipped"), "{warnings}"); + + plugin.set_setting("calibrate_apply", json!(true)).unwrap(); + assert!( + plugin.calibration_id.is_some(), + "{}", + plugin.calibration_status + ); + assert!( + (plugin.v_pi_dac - 860).abs() <= 10, + "Vpi {}", + plugin.v_pi_dac + ); + } + + fn calibration_button(plugin: &StageAModulationPlugin, key: &str) -> SettingKind { + plugin + .settings_schema() + .sections + .iter() + .flat_map(|section| section.items.iter()) + .find(|item| item.key == key) + .unwrap_or_else(|| panic!("{key} is in the schema")) + .kind + .clone() + } + + /// The UI mirror renders the settings schema, and it never owns the device + /// link, a lease, a sweep, or a fit. Gating `enabled` on any of those + /// disables the buttons permanently — the operator can never start. + #[test] + fn calibration_buttons_are_offered_on_the_ui_mirror() { + let mut mirror = StageAModulationPlugin::default(); + assert_eq!(mirror.runtime_role, PluginRuntimeRole::UiMirror); + mirror.port_hint = "mock".into(); + + for key in ["calibrate", "calibrate_apply"] { + assert!( + matches!( + calibration_button(&mirror, key), + SettingKind::Button { enabled: false } + ), + "{key} should be off before the operator asks to connect" + ); + } + + mirror.set_setting("connect", json!(true)).unwrap(); + // The mirror deliberately never opens the port... + assert!(mirror.link.is_none()); + // ...but the buttons must still be pressable, because the worker — not + // the mirror — owns the link and enforces the real interlocks. + for key in ["calibrate", "calibrate_apply"] { + assert!( + matches!( + calibration_button(&mirror, key), + SettingKind::Button { enabled: true } + ), + "{key} is disabled on the mirror, so it can never be pressed" + ); + } + } + + /// A press is transported mirror → worker as a monotonic counter. The + /// worker adopts the counter it first sees as a baseline so a reload does + /// not replay old presses — but that baseline must not swallow the + /// operator's first real press. + #[test] + fn a_forwarded_press_reaches_a_freshly_loaded_worker() { + let mut mirror = StageAModulationPlugin::default(); + let mut worker = live_plugin(); + worker.port_hint = "mock".into(); + worker.set_setting("connect", json!(true)).unwrap(); + wait_until(&worker, Duration::from_secs(2), |p| p.device_connected()); + + // The host syncs the settings snapshot before anything is clicked. + let sync = |worker: &mut StageAModulationPlugin, mirror: &StageAModulationPlugin| { + let value = mirror.get_setting("calibrate").expect("exported"); + worker.set_setting("calibrate", value).unwrap(); + }; + sync(&mut worker, &mirror); + assert!( + worker.sweep.is_none(), + "a plain sync must not start a sweep" + ); + + // First real click on the mirror, then the next settings sync. + mirror.set_setting("calibrate", json!(true)).unwrap(); + sync(&mut worker, &mirror); + assert!( + worker.sweep.is_some(), + "the operator's first press never reached the worker" + ); + + // Re-syncing the same counter must not re-trigger. + sync(&mut worker, &mirror); + assert!(worker.sweep.is_some()); + // A second click stops it, proving the toggle survives the transport. + mirror.set_setting("calibrate", json!(true)).unwrap(); + sync(&mut worker, &mirror); + assert!( + worker.sweep.is_none(), + "second press should abort the sweep" + ); } - fn board_code(plugin: &StageAModulationPlugin) -> Option { - plugin.shared.state.lock().unwrap().board_code + #[test] + fn the_curve_view_shows_the_configured_lobe_before_any_measurement() { + let mut plugin = live_plugin(); + plugin.set_setting("v_null_dac", json!(400)).unwrap(); + plugin.set_setting("v_pi_dac", json!(900)).unwrap(); + let curve = plugin.curve_dataset(); + // Normalised until something has actually been measured. + assert!(curve.y_label.contains("normalised")); + let names: Vec<&str> = curve.lines.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["configured lobe", "V_null", "V_null + Vπ"]); + let lobe = &curve.lines[0].points; + // Minimum at V_null, maximum a quarter wave later. + let at = |code: f64| { + lobe.iter() + .min_by(|a, b| (a.x - code).abs().total_cmp(&(b.x - code).abs())) + .expect("sampled") + .y + }; + assert!(at(400.0) < 0.01, "u at V_null = {}", at(400.0)); + assert!(at(1_300.0) > 0.99, "u at V_null+Vπ = {}", at(1_300.0)); } - /// Connect checkbox → slider change → MOD sent by the device thread → - /// board echoes the code. No process_frame involved anywhere. #[test] - fn level_change_transfers_without_frames() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("port", json!("mock")).unwrap(); - plugin.set_setting("connect", json!(true)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); - assert_eq!( - plugin.shared.state.lock().unwrap().firmware, - "0.3.0-mock".to_owned() - ); + fn calibrated_const_hold_spans_the_full_lobe_without_a_headroom() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Const; + plugin.v_null_dac = 1_630; + plugin.v_pi_dac = 860; + plugin.depth_a = 0.5; // must be irrelevant for a constant hold - plugin.set_setting("level", json!(1234)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| { - board_code(p) == Some(1234) - }); - assert!(plugin.shared.state.lock().unwrap().last_error.is_none()); + // I_k = 1 holds exactly at V_null + Vπ (previously rejected because + // the modulated band u_k·e^{a/2} > 1 was demanded even for CONST). + plugin.operating_point = 1.0; + let (lo, hi, hold) = plugin.dac_band().expect("full-lobe hold"); + assert_eq!((lo, hi, hold), (2_490, 2_490, 2_490)); - plugin.set_setting("connect", json!(false)).unwrap(); - assert!(!plugin.device_connected()); + // The user's measured low point: dac_for_u(0.01) ≈ 1685. + plugin.operating_point = 0.01; + let (_, _, hold) = plugin.dac_band().expect("low hold"); + assert_eq!(hold, 1_685); + + // Modulating modes still require the ±a/2 headroom. + plugin.mode = Mode::Sine; + plugin.operating_point = 1.0; + assert!(plugin.dac_band().is_err()); } - /// The max cap bounds the slider, and lowering it re-sends a lower level. #[test] - fn max_level_caps_the_slider() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("max_level", json!(1000)).unwrap(); - plugin.set_setting("level", json!(4095)).unwrap(); - assert_eq!(plugin.level, 1000, "slider clamps to the cap"); + fn rejected_operating_point_does_not_diverge_from_the_board_target() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Sine; + plugin.v_null_dac = 1_630; + plugin.v_pi_dac = 860; + plugin.depth_a = 0.5; + plugin.operating_point = 0.5; - plugin.set_setting("max_level", json!(500)).unwrap(); - assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); + let error = plugin + .set_setting("operating_point", json!(1.0)) + .expect_err("periodic I_k=1 has no modulation headroom"); + assert!(error.contains("lobe ceiling")); + assert_eq!(plugin.operating_point, 0.5); - let schema = plugin.settings_schema(); - let level_item = schema.sections[0] - .items - .iter() - .find(|item| item.key == "level") - .expect("level setting exists"); - match &level_item.kind { - SettingKind::I64Slider { max, .. } => assert_eq!(*max, 500), - other => panic!("level must stay a slider, got {other:?}"), - } + plugin.set_setting("mode", json!(0)).expect("CONST"); + plugin + .set_setting("operating_point", json!(1.0)) + .expect("CONST maps I_k directly"); + assert_eq!(plugin.dac_band().unwrap(), (2_490, 2_490, 2_490)); } - /// Square drive with min threshold reaches the mock and starts at min; - /// slider to 0 drives the output to 0. #[test] - fn square_with_min_threshold_round_trips() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("port", json!("mock")).unwrap(); - plugin.set_setting("connect", json!(true)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + fn calibrated_const_sends_the_expected_codes_to_the_board() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Const; + plugin.v_null_dac = 1_630; + plugin.v_pi_dac = 860; - plugin.set_setting("level", json!(2000)).unwrap(); - plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); - plugin.set_setting("min_level", json!(500)).unwrap(); - plugin.set_setting("mode", json!("SQUARE")).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| { - board_code(p) == Some(500) + plugin + .set_setting("operating_point", json!(1.0)) + .expect("full lobe"); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.shared.state.lock().unwrap().board_code == Some(2_490) }); - assert!(plugin - .shared - .state - .lock() - .unwrap() - .board_mod - .contains("SQUARE 500..2000")); - plugin.set_setting("mode", json!("CONST")).unwrap(); - plugin.set_setting("level", json!(0)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| { - board_code(p) == Some(0) + plugin + .set_setting("operating_point", json!(0.01)) + .expect("low point"); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.shared.state.lock().unwrap().board_code == Some(1_685) }); - plugin.set_setting("connect", json!(false)).unwrap(); + plugin.disconnect(); } - const TEST_PROTOCOL: &str = r#" -loops = 2 - -[[steps]] -duration_s = 0.03 -wave = "SINE" -level = 2000 -min = 100 -frequency_hz = 100.0 - -[[steps]] -duration_s = 0.03 -wave = "CONST" -level = 750 -"#; + #[test] + fn ui_armed_drive_publishes_a_board_echo_acknowledged_target() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + // Arm a sine purely through the operator settings — no lease, no + // service request. Consumers (A1) must still see the frequency. + plugin.set_setting("mode", json!(1)).unwrap(); // Sine + plugin.set_setting("frequency_hz", json!(5.0)).unwrap(); + plugin.set_setting("level", json!(1_000)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .board_mod + .starts_with("SINE") + }); + let snapshot = plugin.control_state(); + let target = snapshot.acknowledged.expect("board-echo target"); + assert_eq!(target.revision, SemanticRevision(0)); + match target.waveform.expect("waveform") { + WaveformV1::Periodic { + frequency_millihz, .. + } => assert_eq!(frequency_millihz, 5_000), + other => panic!("expected periodic waveform, got {other:?}"), + } + plugin.disconnect(); + } #[test] - fn protocol_parsing_validates_steps() { - let (steps, loops) = parse_protocol(TEST_PROTOCOL).expect("valid protocol"); - assert_eq!(loops, 2); - assert_eq!(steps.len(), 2); - let encoded = |command: &Command, seq: u32| { - String::from_utf8(command.encode(seq).expect("encodes")).expect("utf8") - }; - assert_eq!( - encoded(&steps[0].command, 1), - "@1 MOD wave=SINE level=2000 min=100 freq_mhz=100000\n" - ); - assert_eq!( - encoded(&steps[1].command, 2), - "@2 MOD wave=CONST level=750\n" + fn set_optical_depth_requires_lease_and_a_calibrated_drive() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Sine; + + // Without a lease the retarget is refused. + let unleased = service_request( + &plugin, + 30, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_250, + }, + None, ); - assert!((steps[0].duration.as_secs_f64() - 0.03).abs() < 1e-9); + assert!(matches!( + plugin + .handle_service_request(&unleased, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); - assert!(parse_protocol("loops = 1").is_err(), "steps required"); - assert!( - parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100").is_err(), - "periodic steps need a frequency" + let acquire = service_request( + &plugin, + 31, + "stage-a-a1", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, ); - assert!( - parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"CONST\"\nlevel = 9999").is_err(), - "level range enforced" + plugin.handle_service_request(&acquire, &live_execution()); + + let retarget = service_request( + &plugin, + 32, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_250, + }, + None, ); - assert!( - parse_protocol( - "[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100\nmin = 200\nfrequency_hz = 10.0" - ) - .is_err(), - "min above level rejected" + assert!(matches!( + plugin + .handle_service_request(&retarget, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + )); + assert!((plugin.depth_a - 1.25).abs() < 1e-9); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .board_mod + .starts_with("SINE") + }); + + // The manual DAC band cannot express an optical depth. + plugin.method = DriveMethod::Manual; + let manual = service_request( + &plugin, + 33, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_000, + }, + None, ); - let (off, _) = parse_protocol("[[steps]]\nduration_s = 0.5\nwave = \"OFF\"") - .expect("OFF needs no level"); - assert_eq!(encoded(&off[0].command, 1), "@1 MOD wave=OFF\n"); + assert!(matches!( + plugin + .handle_service_request(&manual, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + plugin.disconnect(); } - /// A protocol against the mock walks every step, holds the last one, and - /// reports finished. #[test] - fn protocol_runs_to_completion_on_the_mock() { - let dir = std::env::temp_dir().join(format!( - "stage-a-modulation-protocol-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() + fn lease_acquire_is_idempotent_and_exclusive_without_frames() { + let mut plugin = live_plugin(); + let acquire = service_request( + &plugin, + 10, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + let first = plugin.handle_service_request(&acquire, &live_execution()); + let expiry = plugin.lease.as_ref().unwrap().expires_at_unix_ms; + let duplicate = plugin.handle_service_request(&acquire, &live_execution()); + assert_eq!(first, duplicate); + assert_eq!(plugin.lease.as_ref().unwrap().expires_at_unix_ms, expiry); + assert!(plugin.set_setting("level", json!(1)).is_err()); + + let conflict = service_request( + &plugin, + 11, + "workflow-b", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&conflict, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } )); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join("protocol.toml"); - std::fs::write(&path, TEST_PROTOCOL).unwrap(); + } - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("port", json!("mock")).unwrap(); - plugin.set_setting("connect", json!(true)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + #[test] + fn prepare_safe_off_and_release_publish_terminal_ack_before_lease_loss() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); - plugin - .set_setting("protocol_path", json!(path.display().to_string())) - .unwrap(); - plugin.set_setting("protocol_run", json!(true)).unwrap(); - assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); - assert_eq!(plugin.get_setting("protocol_run"), Some(json!(true))); + let acquire = service_request( + &plugin, + 20, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + let prepare = service_request( + &plugin, + 21, + "workflow-a", + ModulationCommandV1::PrepareA1 { + configuration: A1AcquisitionConfigV1 { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Sine, + frequency_millihz: 10_000, + center_dac: 1_000, + amplitude_dac: 250, + sample_rate_hz: 20_000, + block_samples: 256, + emit_raw_samples: true, + emit_summary: true, + optical_lut_id: None, + }, + }, + Some(1), + ); + let initial = plugin.handle_service_request(&prepare, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = initial.outcome else { + panic!("prepare rejected"); + }; + let response: ModulationResponseV1 = serde_json::from_value(payload).unwrap(); + assert_eq!(response.common.outcome, RequestOutcomeV1::InProgress); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .last_response + .as_ref() + .is_some_and(|response| { + response.common.request_id.0 == 21 + && response.common.outcome == RequestOutcomeV1::Applied + }) + }); + let terminal = plugin.handle_service_request(&prepare, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = terminal.outcome else { + panic!("terminal prepare rejected"); + }; + let response: ModulationResponseV1 = serde_json::from_value(payload).unwrap(); + assert_eq!( + response.common.acknowledged_revision, + Some(SemanticRevision(1)) + ); - // 2 loops × 2 steps × 30 ms ≈ 120 ms; wait for the final CONST 750. - wait_until(&plugin, Duration::from_secs(3), |p| { - !p.protocol_active() && board_code(p) == Some(750) + let safe_off = service_request( + &plugin, + 22, + "workflow-a", + ModulationCommandV1::SafeOff { + reason: "test".into(), + }, + Some(2), + ); + plugin.handle_service_request(&safe_off, &live_execution()); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .acknowledged + .as_ref() + .is_some_and(|target| { + target.revision == SemanticRevision(2) + && target.waveform == Some(WaveformV1::Off) + }) }); - assert!(!plugin.protocol_active()); - assert_eq!(board_code(&plugin), Some(750), "last step holds"); - let progress = plugin - .protocol - .as_ref() - .unwrap() - .progress - .lock() - .unwrap() - .clone(); - assert!(progress.finished && !progress.stopped); - assert_eq!((progress.loop_index, progress.step_index), (2, 2)); - plugin.set_setting("connect", json!(false)).unwrap(); - std::fs::remove_dir_all(dir).unwrap(); + let release = service_request( + &plugin, + 23, + "workflow-a", + ModulationCommandV1::ReleaseLease { + safe_off: true, + reason: "done".into(), + }, + Some(3), + ); + plugin.handle_service_request(&release, &live_execution()); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .last_response + .as_ref() + .is_some_and(|response| { + response.common.request_id.0 == 23 + && response.common.outcome == RequestOutcomeV1::Applied + }) + }); + plugin.apply_execution_context(&live_execution()); + let snapshot = plugin.control_state(); + assert!( + snapshot.lease.is_some(), + "terminal ACK snapshot retains lease" + ); + let duplicate = plugin.handle_service_request(&release, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = duplicate.outcome else { + panic!("release duplicate rejected"); + }; + let response: ModulationResponseV1 = serde_json::from_value(payload).unwrap(); + assert_eq!(response.common.outcome, RequestOutcomeV1::Applied); + plugin.apply_execution_context(&live_execution()); + assert!(plugin.lease.is_none(), "lease clears after ACK publication"); + plugin.disconnect(); } #[test] - fn protocol_requires_a_connection() { - let mut plugin = StageAModulationPlugin::default(); - plugin - .set_setting("protocol_path", json!("/tmp/x.toml")) - .unwrap(); - plugin.set_setting("protocol_run", json!(true)).unwrap(); + fn lease_expiry_and_effect_revocation_fail_closed_without_frames() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + let acquire = service_request( + &plugin, + 30, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + plugin.lease.as_mut().unwrap().expires_at_unix_ms = now_unix_ms().saturating_sub(1); + plugin.apply_execution_context(&live_execution()); + assert!(plugin.lease.is_none()); assert!(plugin .last_error .as_deref() - .is_some_and(|err| err.contains("connect"))); - assert_eq!(plugin.get_setting("protocol_run"), Some(json!(false))); - } - - /// The host settings UI exchanges enum values as indices into the - /// schema's variant list (radio buttons send `json!(index)`). - #[test] - fn enum_settings_round_trip_as_indices() { - let mut plugin = StageAModulationPlugin::default(); - // Mode: index 2 = SQUARE in the schema's variant order. - plugin - .set_setting("mode", json!(2)) - .expect("index accepted"); - assert_eq!(plugin.mode, Mode::Square); - assert_eq!(plugin.get_setting("mode"), Some(json!(2))); - // Port: index 1 = "mock" (variants start with auto, mock). - plugin - .set_setting("port", json!(1)) - .expect("index accepted"); - assert_eq!(plugin.port_hint, "mock"); - assert_eq!(plugin.get_setting("port"), Some(json!(1))); - // Out-of-range indices are visible errors, not silent no-ops. - assert!(plugin.set_setting("mode", json!(99)).is_err()); - // String names keep working (tests, saved configs). - plugin - .set_setting("mode", json!("SINE")) - .expect("name accepted"); - assert_eq!(plugin.mode, Mode::Sine); - } + .is_some_and(|message| message.contains("lease expired"))); - /// min_level can never exceed the level. - #[test] - fn min_threshold_is_clamped_to_level() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("level", json!(1000)).unwrap(); - plugin.set_setting("min_level", json!(3000)).unwrap(); - assert_eq!(plugin.min_level, 1000); - plugin.set_setting("level", json!(200)).unwrap(); - assert_eq!(plugin.min_level, 200, "lowering level drags min down"); + let acquire = service_request( + &plugin, + 31, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + assert!(plugin.lease.is_some()); + plugin.apply_execution_context(&ExecutionContext::fail_closed()); + assert!(plugin.link.is_none()); + assert!(plugin.lease.is_none()); } } diff --git a/plugins/stage-a-modulation/src/waveform.rs b/plugins/stage-a-modulation/src/waveform.rs new file mode 100644 index 0000000..0485f19 --- /dev/null +++ b/plugins/stage-a-modulation/src/waveform.rs @@ -0,0 +1,389 @@ +//! Optical-target DAC warp-table synthesis for the Pockels/PBS modulator. +//! +//! On one monotonic Pockels/PBS lobe the excitation transfer is +//! `I(V) = I_floor + (I_ceil - I_floor) · sin²(α (V - V_null))`, with +//! `α = π / (2 Vπ)`. The manufacturer likewise describes the amplitude +//! modulator as `sin²`; a 50 % bias only *approximately* linearises the small +//! signal. A pure DAC sine therefore does **not** produce a sinusoidal optical +//! target — it must be pre-warped by inverting the transfer: +//! +//! ```text +//! u(t) = (I_d(t) - I_floor) / (I_ceil - I_floor) // normalised target +//! V(u) = V_null + (2 Vπ / π) · arcsin(√u) // increasing lobe +//! ``` +//! +//! Two optical targets are supported (the drive picks one): +//! - [`OpticalTarget::LogSine`] — `ln I_d = ln I_g + (a/2) sin ωt`, the clean A1 +//! input because the event camera responds to changes in `ln I`. +//! - [`OpticalTarget::LinearSine`] — `I_d = I_c (1 + m sin ωt)`, `m = tanh(a/2)`. +//! +//! The inversion parameters `V_null` and `Vπ` are expressed in **DAC codes** and +//! are settable: the engineer should not rely on nominal `Vπ` but sweep settled +//! constant DAC codes, measure the actual optical transfer, and enter the frozen +//! `V_null` / `Vπ` of one monotonic lobe. A fully measured lookup table can +//! replace this analytic inversion later behind the same interface. + +use std::f64::consts::PI; + +/// Warp-table length played back over one modulation period. +pub const WARP_TABLE_LEN: usize = 256; +/// Full-scale DAC code (12-bit). +pub const DAC_FULL_SCALE: u16 = 4_095; + +/// Optical intensity target the drive should reproduce, swung around the +/// operating point `u_k`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OpticalTarget { + /// Recommended A1 log-intensity sine: `ln I = ln I_k + (a/2) sin ωt`. + LogSine, + /// Literal linear-intensity sine: `I = I_k (1 + m sin ωt)`, `m = tanh(a/2)`. + LinearSine, +} + +/// Frozen inversion of one monotonic Pockels/PBS lobe, in DAC codes. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct LobeInversion { + /// DAC code where the excitation light is at its minimum (`sin² = 0`). + pub v_null_dac: f64, + /// DAC-code distance from `v_null` to the excitation maximum (quarter wave). + pub v_pi_dac: f64, +} + +impl LobeInversion { + /// Normalised optical intensity produced by `code` on the configured lobe: + /// `u = sin²(π(code - V_null) / (2 Vπ))`. + pub fn u_for_dac(&self, code: f64) -> f64 { + let alpha = PI / (2.0 * self.v_pi_dac); + (alpha * (code - self.v_null_dac)).sin().powi(2) + } + + /// DAC code producing normalised optical intensity `u ∈ [0, 1]` on the + /// increasing lobe. + pub fn dac_for_u(&self, u: f64) -> f64 { + self.v_null_dac + (2.0 * self.v_pi_dac / PI) * u.clamp(0.0, 1.0).sqrt().asin() + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct OpticalDrive { + pub target: OpticalTarget, + /// Optical log-modulation depth `a = ln(I_max / I_min)`, must be positive. + pub depth_a: f64, + /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0, 1]`: + /// the geometric-mean point the modulation swings around. Held fixed while + /// `a` is swept, so one response curve keeps `I_k` constant. + pub operating_point: f64, + pub inversion: LobeInversion, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum WarpError { + /// `a` is not finite or not positive. + InvalidDepth, + /// The operating point is not in `(0, 1]`. + InvalidOperatingPoint, + /// `Vπ` is not finite or not positive. + InvalidInversion, + /// The peak optical target exceeds the lobe ceiling (`u_k · peak > 1`): the + /// operating point is too bright for this depth and would saturate. + Saturates { peak: f64 }, + /// A computed DAC code falls outside `0..=4095`: the inversion parameters do + /// not fit the requested depth on this lobe. Clamping would silently distort + /// the optical target, so the drive is refused instead. + OutOfRange { index: usize, code: f64 }, +} + +impl std::fmt::Display for WarpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidDepth => f.write_str("optical depth a must be finite and positive"), + Self::InvalidOperatingPoint => f.write_str("operating point must be in (0, 1]"), + Self::InvalidInversion => f.write_str("Vπ must be finite and positive"), + Self::Saturates { peak } => write!( + f, + "peak optical target u = {peak:.3} exceeds the lobe ceiling; lower a or the operating point" + ), + Self::OutOfRange { index, code } => write!( + f, + "warp sample {index} = {code:.1} DAC leaves 0..=4095; reduce a or re-measure the lobe" + ), + } + } +} + +impl std::error::Error for WarpError {} + +impl OpticalDrive { + /// Derives the target-law parameters that make an optical waveform span + /// the intensities produced by the supplied DAC band. + pub fn from_dac_band( + target: OpticalTarget, + inversion: LobeInversion, + lo: f64, + hi: f64, + ) -> Self { + let u_lo = inversion.u_for_dac(lo); + let u_hi = inversion.u_for_dac(hi); + let (operating_point, depth_a) = match target { + OpticalTarget::LogSine => ((u_lo * u_hi).sqrt(), (u_hi / u_lo).ln()), + OpticalTarget::LinearSine => { + let operating_point = 0.5 * (u_lo + u_hi); + let modulation = (u_hi - u_lo) / (u_hi + u_lo); + (operating_point, 2.0 * modulation.atanh()) + } + }; + Self { + target, + depth_a, + operating_point, + inversion, + } + } + + /// Normalised optical target `u(φ)` for phase fraction `φ ∈ [0, 1)`, swung + /// around the operating point `u_k` (not peak-normalised). + pub fn normalised_intensity(&self, phase: f64) -> f64 { + let sine = (2.0 * PI * phase).sin(); + match self.target { + // ln I = ln I_k + (a/2) sin ωt. + OpticalTarget::LogSine => self.operating_point * (0.5 * self.depth_a * sine).exp(), + // I = I_k (1 + m sin ωt), m = tanh(a/2). + OpticalTarget::LinearSine => { + let m = (0.5 * self.depth_a).tanh(); + self.operating_point * (1.0 + m * sine) + } + } + } + + /// Peak normalised optical target over one period. + fn peak_intensity(&self) -> f64 { + match self.target { + OpticalTarget::LogSine => self.operating_point * (0.5 * self.depth_a).exp(), + OpticalTarget::LinearSine => self.operating_point * (1.0 + (0.5 * self.depth_a).tanh()), + } + } + + /// Builds the `WARP_TABLE_LEN`-entry DAC warp table for one period. + pub fn warp_table(&self) -> Result, WarpError> { + if !self.depth_a.is_finite() || self.depth_a <= 0.0 { + return Err(WarpError::InvalidDepth); + } + if !self.operating_point.is_finite() + || !(0.0..=1.0).contains(&self.operating_point) + || self.operating_point <= 0.0 + { + return Err(WarpError::InvalidOperatingPoint); + } + if !self.inversion.v_pi_dac.is_finite() || self.inversion.v_pi_dac <= 0.0 { + return Err(WarpError::InvalidInversion); + } + let peak = self.peak_intensity(); + if peak > 1.0 + 1e-9 { + return Err(WarpError::Saturates { peak }); + } + let mut table = Vec::with_capacity(WARP_TABLE_LEN); + for index in 0..WARP_TABLE_LEN { + let phase = index as f64 / WARP_TABLE_LEN as f64; + let code = self.inversion.dac_for_u(self.normalised_intensity(phase)); + if !code.is_finite() || code < -0.5 || code > f64::from(DAC_FULL_SCALE) + 0.5 { + return Err(WarpError::OutOfRange { index, code }); + } + table.push(code.round().clamp(0.0, f64::from(DAC_FULL_SCALE)) as u16); + } + Ok(table) + } +} + +/// Forward Pockels/PBS transfer used to verify a warp table reproduces the +/// intended optical target: `u = sin²(α (code - V_null))`, `α = π / (2 Vπ)`. +#[cfg(test)] +pub fn lobe_transmission(code: f64, inversion: &LobeInversion) -> f64 { + inversion.u_for_dac(code) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn inversion() -> LobeInversion { + // Null at code 200, quarter wave 1600 codes later (peak light at 1800). + LobeInversion { + v_null_dac: 200.0, + v_pi_dac: 1_600.0, + } + } + + /// Peak-normalised operating point (max light at the lobe ceiling) so the + /// range/round-trip assertions exercise the full swing. + fn peak_operating_point(target: OpticalTarget, depth_a: f64) -> f64 { + match target { + OpticalTarget::LogSine => (-0.5 * depth_a).exp(), + OpticalTarget::LinearSine => 1.0 / (1.0 + (0.5 * depth_a).tanh()), + } + } + + fn drive(target: OpticalTarget, depth_a: f64) -> OpticalDrive { + OpticalDrive { + target, + depth_a, + operating_point: peak_operating_point(target, depth_a), + inversion: inversion(), + } + } + + #[test] + fn tables_stay_inside_the_dac_range_for_both_targets() { + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + let table = drive(target, 1.0).warp_table().expect("in range"); + assert_eq!(table.len(), WARP_TABLE_LEN); + assert!(table.iter().all(|&code| code <= DAC_FULL_SCALE)); + } + } + + #[test] + fn warp_table_reproduces_the_optical_target_through_the_sin2_transfer() { + // Feeding the warp codes back through the sin² lobe must recover the + // intended normalised intensity: that is the whole point of the warp. + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + let drive = drive(target, 0.8); + let table = drive.warp_table().expect("in range"); + for (index, &code) in table.iter().enumerate() { + let phase = index as f64 / WARP_TABLE_LEN as f64; + let recovered = lobe_transmission(f64::from(code), &drive.inversion); + let target_u = drive.normalised_intensity(phase); + assert!( + (recovered - target_u).abs() < 5e-3, + "{target:?} phase {phase}: recovered {recovered} vs target {target_u}" + ); + } + } + } + + #[test] + fn measured_log_contrast_matches_the_requested_depth_for_log_sine() { + // The optical min/max of a log-sine table give back a = ln(max/min). + let drive = drive(OpticalTarget::LogSine, 1.2); + let table = drive.warp_table().expect("in range"); + let intensities: Vec = table + .iter() + .map(|&code| lobe_transmission(f64::from(code), &drive.inversion)) + .collect(); + let max = intensities.iter().cloned().fold(f64::MIN, f64::max); + let min = intensities.iter().cloned().fold(f64::MAX, f64::min); + let measured_a = (max / min).ln(); + assert!((measured_a - 1.2).abs() < 0.05, "measured a = {measured_a}"); + } + + #[test] + fn drive_derived_from_a_dac_band_recovers_that_optical_span() { + let inversion = inversion(); + let lo = 600.0; + let hi = 1_500.0; + let expected_lo = inversion.u_for_dac(lo); + let expected_hi = inversion.u_for_dac(hi); + + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + let drive = OpticalDrive::from_dac_band(target, inversion, lo, hi); + let table = drive.warp_table().expect("manual band is valid"); + let recovered: Vec = table + .iter() + .map(|&code| inversion.u_for_dac(f64::from(code))) + .collect(); + let recovered_lo = recovered.iter().copied().fold(f64::MAX, f64::min); + let recovered_hi = recovered.iter().copied().fold(f64::MIN, f64::max); + assert!( + (recovered_lo - expected_lo).abs() < 5e-3, + "{target:?}: recovered lower intensity {recovered_lo} vs {expected_lo}" + ); + assert!( + (recovered_hi - expected_hi).abs() < 5e-3, + "{target:?}: recovered upper intensity {recovered_hi} vs {expected_hi}" + ); + } + } + + #[test] + fn deeper_depth_gives_more_optical_contrast() { + let contrast = |a: f64| { + let drive = drive(OpticalTarget::LinearSine, a); + let table = drive.warp_table().expect("in range"); + let intensities: Vec = table + .iter() + .map(|&code| lobe_transmission(f64::from(code), &drive.inversion)) + .collect(); + let max = intensities.iter().cloned().fold(f64::MIN, f64::max); + let min = intensities.iter().cloned().fold(f64::MAX, f64::min); + (max / min).ln() + }; + assert!(contrast(1.0) > contrast(0.5)); + } + + #[test] + fn rejects_invalid_depth_and_inversion() { + assert_eq!( + drive(OpticalTarget::LogSine, 0.0).warp_table(), + Err(WarpError::InvalidDepth) + ); + let mut bad = drive(OpticalTarget::LogSine, 1.0); + bad.inversion.v_pi_dac = 0.0; + assert_eq!(bad.warp_table(), Err(WarpError::InvalidInversion)); + } + + #[test] + fn refuses_an_inversion_that_overruns_the_lobe() { + // The reachable optical maximum sits at v_null + Vπ; pushing that past + // the top rail must be refused rather than silently clamped. + let drive = OpticalDrive { + target: OpticalTarget::LogSine, + depth_a: 1.0, + operating_point: peak_operating_point(OpticalTarget::LogSine, 1.0), + inversion: LobeInversion { + v_null_dac: 200.0, + v_pi_dac: 4_000.0, // peak light would land at code 4200 + }, + }; + assert!(matches!( + drive.warp_table(), + Err(WarpError::OutOfRange { .. }) + )); + } + + #[test] + fn refuses_an_operating_point_too_bright_for_the_depth() { + let drive = OpticalDrive { + target: OpticalTarget::LogSine, + depth_a: 1.0, + operating_point: 0.9, // 0.9 * exp(0.5) = 1.48 > 1 -> saturates + inversion: inversion(), + }; + assert!(matches!( + drive.warp_table(), + Err(WarpError::Saturates { .. }) + )); + } + + #[test] + fn fixed_operating_point_keeps_i_k_while_sweeping_a() { + // One response curve: fix u_k, vary a. The geometric-mean intensity at + // phase 0 (sin = 0) stays put; only the contrast grows with a. + let u_k = 0.3; + let drive = |a: f64| OpticalDrive { + target: OpticalTarget::LogSine, + depth_a: a, + operating_point: u_k, + inversion: inversion(), + }; + for a in [0.2, 0.6, 1.0] { + // At phase 0 the log-sine sits exactly at the operating point. + assert!((drive(a).normalised_intensity(0.0) - u_k).abs() < 1e-12); + let table = drive(a).warp_table().expect("in range"); + let intensities: Vec = table + .iter() + .map(|&code| lobe_transmission(f64::from(code), &inversion())) + .collect(); + let max = intensities.iter().cloned().fold(f64::MIN, f64::max); + let min = intensities.iter().cloned().fold(f64::MAX, f64::min); + assert!(((max / min).ln() - a).abs() < 0.05, "a={a}"); + } + } +} diff --git a/plugins/stage-a-photodiode/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml index a838ebb..3e0c8c0 100644 --- a/plugins/stage-a-photodiode/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -14,3 +14,4 @@ augur-plugin-api.workspace = true serde_json.workspace = true serialport.workspace = true stage-a-io = { path = "../../stage-a-io", default-features = false } +stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md index af2e277..44d48d7 100644 --- a/plugins/stage-a-photodiode/README.md +++ b/plugins/stage-a-photodiode/README.md @@ -1,8 +1,8 @@ # Stage-A Photodiode Live readout of the photodiode on **board SMA5 → Teensy pin 18 / analog input A4**, from the -free-running ASCII stream the `stage-a-controller` firmware (0.3.0+) emits on its **second** USB -serial port (`PD code=… n=… t_ms=…` at 50 Hz). The port carries no commands, so this plugin is +free-running PDA1 `SamplesU16` stream the `stage-a-controller` firmware (0.4.0+) emits on its +**second** USB serial port (20 kSa/s default). The port carries no commands, so this plugin is read-only by construction; the command port belongs to `stage-a-modulation`. ## Modes @@ -16,11 +16,29 @@ read-only by construction; the command port belongs to `stage-a-modulation`. ## Views - a live rolling chart (window length settable, 1–120 s) of the value in the selected mode; -- a compact status table with the newest code/value and Connect/Disconnect actions. +- a compact status table with the newest code/value, moving average, integrity, + recording state, and connection state. ## Ports **Use `auto` (default recommendation):** it listens briefly on every attached usbmodem/ttyACM -device and connects to the one actually streaming `PD` lines — that is always the Teensy stream -port. Picking the command port manually by mistake is harmless: its binary frames parse to -nothing (no values appear). `mock` generates a synthetic slow sine for hardware-free testing. +device and connects to the one actually streaming CRC-clean PDA1 sample frames — that is always +the Teensy stream port. `mock` generates a synthetic sine for hardware-free testing. + +## Owner control service + +This plugin is the sole owner of the Teensy photodiode stream port. Workflow +plugins control named recordings through the versioned +`stage_a.photodiode.control.v1` service and consume bounded +`stage_a.photodiode_summary.v1` snapshots. They never open the serial port or +receive raw sample arrays through the control plane; finalized PDQ files remain +the replay and analysis source of truth. + +The snapshot's `stream.level` block carries the settled detector level over the +moving-average window in **raw** detector volts — the ADC map only, never the +RAW/EXCITATION display transform and never the optical geometry transform. It +also reports the window's peak-to-peak spread and the sample index it ends at, +so a consumer can prove a reading was taken *after* it changed something without +a shared clock. Unlike `optical_summary` it never refuses: it stays present +while the window clips (flagged), because the Pockels transfer sweep needs a +reading exactly where the reject-port detector is brightest. diff --git a/plugins/stage-a-photodiode/plugin.toml b/plugins/stage-a-photodiode/plugin.toml index 21b4bad..f773a84 100644 --- a/plugins/stage-a-photodiode/plugin.toml +++ b/plugins/stage-a-photodiode/plugin.toml @@ -1,3 +1,4 @@ +id = "stage-a.photodiode" name = "Stage-A Photodiode" version = "0.4.0" description = "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 1e0cbff..19313c7 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -21,10 +21,10 @@ //! or — for modulated signals — one full period of a user-given frequency, //! which makes the mean independent of the modulation phase. -use std::collections::VecDeque; -use std::fs::File; +use std::collections::{BTreeMap, VecDeque}; +use std::fs::{File, OpenOptions}; use std::io::{BufWriter, Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -34,12 +34,26 @@ use augur_plugin_api::PathDialogKind; use augur_plugin_api::{ export_plugin, EventStoreHandle, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, - PluginFrame, Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, - TableSchema, TableValueType, + PluginControlContext, PluginControlSnapshot, PluginFrame, PluginRuntimeRole, + PluginServiceOutcome, PluginServiceReply, PluginServiceRequest, Series1dLine, Series1dPoint, + Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, + TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{FrameParser, ParseEvent, PdqWriter, StreamIntegrity}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, ContrastGeometry, FrameParser, ParseEvent, PdqWriter, + StreamIntegrity, +}; +use stage_a_plugin_contract::{ + ClientId, ConnectionStateV1, FreshnessV1, LeaseId, LeaseSnapshotV1, OwnerInstanceId, + PdqFinalizedReceiptV1, PdqReceiptV1, PdqStartSpecV1, PdqStartedReceiptV1, PdqTerminationV1, + PhotodiodeCalibrationV1, PhotodiodeCommandV1, PhotodiodeLevelV1, PhotodiodeOpticalSummaryV1, + PhotodiodeRequestV1, PhotodiodeResponseV1, PhotodiodeStreamV1, PhotodiodeSummaryV1, + RequestOutcomeV1, ResponseCommonV1, RunId, SampleRangeV1, SemanticRevision, ServiceErrorCodeV1, + ServiceErrorV1, Sha256V1, StreamIntegrityV1, SynchronizationV1, UnsyncedReasonV1, + CONTRACT_VERSION_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, PLUGIN_ID_STAGE_A_PHOTODIODE, + SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, +}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; const SPECTRUM_DATASET_ID: &str = "stage-a-photodiode.spectrum"; @@ -69,7 +83,22 @@ const SPECTRUM_MIN_SAMPLES: usize = 256; const SPECTRUM_MAX_SAMPLES: usize = 16_384; /// The firmware's default stream rate; the mock mirrors it. const MOCK_RATE_HZ: u32 = 20_000; +/// Trailing samples used for the live optical log-contrast `a`. Sized like the +/// spectrum window so a handful of modulation cycles are always covered. +const CONTRAST_WINDOW_SAMPLES: usize = 16_384; const MOCK_BLOCK_SAMPLES: usize = 256; +/// Cap on retained phase-0 markers (bounds the overlay + frequency window). +const MAX_MARKERS: usize = 4_096; +/// Mock phase-0 marker period in samples (20 kSa/s / 40 = 500 Hz modulation). +const MOCK_MARKER_PERIOD_SAMPLES: u64 = 40; +/// Codes within this margin of an ADC rail mark a level window as clipped; +/// mirrors the estimator's own clip margin. +const CLIP_MARGIN_CODES: u16 = 4; +const REQUEST_CACHE_LIMIT: usize = 256; +const MIN_LEASE_TTL_MS: u64 = 1_000; +const MAX_LEASE_TTL_MS: u64 = 60_000; +const SNAPSHOT_VALID_FOR_MS: u64 = 2_000; +static OWNER_SEQUENCE: AtomicU64 = AtomicU64::new(1); fn code_to_volts(code: f64) -> f64 { code * ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE @@ -138,6 +167,10 @@ struct SharedState { /// chart and moving average never rescan the raw window — at 500 kSa/s a /// full-window rescan per repaint would not be viable. cells: VecDeque, + /// Phase-0 marker sample indices (device clock) still inside the ring, from + /// `Marker` stream frames. Used for the opt-in trigger overlay and to derive + /// the modulation frequency. + markers: VecDeque, latest: Option, /// Cumulative firmware-side drop counter (latest header value). device_dropped: u32, @@ -148,6 +181,7 @@ struct SharedState { /// Monitor-cache length driving ring eviction (user setting). cache_seconds: f64, error: Option, + last_update_unix_ms: u64, } /// min/max/sum over exactly [`SUMMARY_CELL`] consecutive raw samples. @@ -183,6 +217,7 @@ impl Default for SharedState { ring_first_index: 0, samples: VecDeque::new(), cells: VecDeque::new(), + markers: VecDeque::new(), latest: None, device_dropped: 0, crc_failures: 0, @@ -190,6 +225,7 @@ impl Default for SharedState { segments: 0, cache_seconds: DEFAULT_CACHE_SECONDS, error: None, + last_update_unix_ms: 0, } } } @@ -215,12 +251,14 @@ impl SharedState { } self.samples.clear(); self.cells.clear(); + self.markers.clear(); self.ring_first_index = first_index; self.rate_hz = rate_hz; } self.samples.extend(codes.iter().copied()); self.latest = codes.last().copied(); self.device_dropped = device_dropped; + self.last_update_unix_ms = now_unix_ms(); // Summarize every newly completed cell. while (self.cells.len() + 1) * SUMMARY_CELL <= self.samples.len() { @@ -251,6 +289,47 @@ impl SharedState { self.cells.drain(..evict_cells); self.ring_first_index += evict as u64; } + // Drop markers that fell out of the retained ring window. + while self + .markers + .front() + .is_some_and(|&index| index < self.ring_first_index) + { + self.markers.pop_front(); + } + } + + /// Records a phase-0 marker (device sample index) if it sits inside the + /// current ring window. Bounded so a marker storm cannot grow unbounded. + fn push_marker(&mut self, sample_index: u64) { + if sample_index < self.ring_first_index { + return; + } + if self + .markers + .back() + .is_some_and(|&last| last == sample_index) + { + return; // ignore duplicate stamps + } + self.markers.push_back(sample_index); + while self.markers.len() > MAX_MARKERS { + self.markers.pop_front(); + } + self.last_update_unix_ms = now_unix_ms(); + } + + /// Mean marker spacing in samples, i.e. the modulation period on the device + /// clock — the trigger *defining* the frequency. `None` with < 2 markers. + fn marker_period_samples(&self) -> Option { + if self.markers.len() < 2 { + return None; + } + let first = *self.markers.front()?; + let last = *self.markers.back()?; + let spans = (self.markers.len() - 1) as f64; + let period = last.saturating_sub(first) as f64 / spans; + (period > 0.0).then_some(period) } /// min/max/sum over deque offsets `[start, end)`, combining whole @@ -306,6 +385,14 @@ impl SharedState { struct RecordingSink { writer: PdqWriter, pdq_path: PathBuf, + sidecar_path: PathBuf, + pdq_path_label: String, + sidecar_path_label: String, + run_id: RunId, + opened_at_unix_ms: u64, + stream_epoch: u64, + first_sample_index: Option, + metadata: BTreeMap, started_slug: String, samples_written: u64, write_error: Option, @@ -317,6 +404,19 @@ struct RecordingSink { start_segments: u64, } +impl RecordingSink { + fn started_receipt(&self) -> PdqStartedReceiptV1 { + PdqStartedReceiptV1 { + run_id: self.run_id.clone(), + pdq_path: self.pdq_path_label.clone(), + sidecar_path: self.sidecar_path_label.clone(), + opened_at_unix_ms: self.opened_at_unix_ms, + stream_epoch: self.stream_epoch, + first_sample_index: self.first_sample_index, + } + } +} + type SharedRecording = Arc>>; fn record_frame(recording: &SharedRecording, frame: &stage_a_io::Frame, samples: usize) { @@ -422,6 +522,15 @@ impl Reader { sequence = sequence.wrapping_add(1); if let Ok(mut state) = shared.lock() { state.ingest(next_index, MOCK_RATE_HZ, 0, &codes); + // Synthesize phase-0 markers on the device clock so the + // trigger overlay and frequency work without hardware. + let block_end = next_index + MOCK_BLOCK_SAMPLES as u64; + let mut marker = + next_index.next_multiple_of(MOCK_MARKER_PERIOD_SAMPLES); + while marker < block_end { + state.push_marker(marker); + marker += MOCK_MARKER_PERIOD_SAMPLES; + } } next_index += MOCK_BLOCK_SAMPLES as u64; produced = true; @@ -506,6 +615,13 @@ fn read_frames( while let Some(event) = parser.next_event() { match event { ParseEvent::Frame(frame) => { + if let Some(marker) = frame.marker() { + if let Ok(mut state) = shared.lock() { + state.push_marker(marker.sample_index); + } + changed = true; + continue; + } let Some(codes) = frame.samples() else { continue; // Control/summary frames are not expected here. }; @@ -540,6 +656,15 @@ fn read_frames( pub struct StageAPhotodiodePlugin { enabled: bool, + runtime_role: PluginRuntimeRole, + effects_allowed: bool, + owner_instance: OwnerInstanceId, + lease: Option, + request_cache: VecDeque<(PluginServiceRequest, PluginServiceReply)>, + requested_revision: Option, + acknowledged_revision: Option, + last_response: Option, + last_finalized_recording: Option, reader: Option, shared: Arc>, generation: Arc, @@ -556,13 +681,86 @@ pub struct StageAPhotodiodePlugin { avg_samples: usize, avg_sync_freq_hz: f64, time_axis: TimeAxis, + /// Overlay the phase-0 trigger markers on the chart (opt-in). + show_markers: bool, data_dir: String, + // -- momentary-button press forwarding (see PressLatch) -- + press_save_snapshot: PressLatch, + press_record_start: PressLatch, + press_record_stop: PressLatch, +} + +/// Forwards momentary button presses across the host's UI-mirror → live-worker +/// settings snapshot. A click arrives as `true` on the clicked instance; the +/// other instance only ever sees the snapshot value from `get_setting`, so the +/// press is transported as a monotonic counter and a counter advance counts as +/// one press edge. The first counter a fresh instance sees is adopted silently +/// so a reloaded worker does not replay old presses. Without this, an +/// unguarded button `set_setting` fires on every settings sync — the +/// "snapshot files kept appearing" bug. +#[derive(Debug, Default, Clone, Copy)] +struct PressLatch { + counter: u64, + seen: Option, +} + +impl PressLatch { + /// Interprets a settings write to this button; returns true on a press edge. + fn accept(&mut self, value: &Value) -> bool { + if value.as_bool() == Some(true) { + self.counter += 1; + self.seen = Some(self.counter); + return true; + } + let Some(incoming) = value.as_u64() else { + return false; + }; + match self.seen { + None => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + false + } + Some(seen) if incoming > seen => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + true + } + Some(_) => false, + } + } + + fn value(&self) -> Value { + json!(self.counter) + } +} + +#[derive(Clone)] +struct ControlLease { + lease_id: LeaseId, + holder: ClientId, + run_id: Option, + expires_at_unix_ms: u64, } impl Default for StageAPhotodiodePlugin { fn default() -> Self { Self { enabled: false, + runtime_role: PluginRuntimeRole::UiMirror, + effects_allowed: false, + owner_instance: OwnerInstanceId::new(format!( + "photodiode-{}-{}-{}", + std::process::id(), + now_unix_ms(), + OWNER_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )), + lease: None, + request_cache: VecDeque::new(), + requested_revision: None, + acknowledged_revision: None, + last_response: None, + last_finalized_recording: None, reader: None, shared: Arc::new(Mutex::new(SharedState::default())), generation: Arc::new(AtomicU64::new(1)), @@ -577,7 +775,11 @@ impl Default for StageAPhotodiodePlugin { avg_samples: 4, avg_sync_freq_hz: 0.0, time_axis: TimeAxis::BeforeNow, + show_markers: false, data_dir: String::new(), + press_save_snapshot: PressLatch::default(), + press_record_start: PressLatch::default(), + press_record_stop: PressLatch::default(), } } } @@ -591,6 +793,10 @@ impl StageAPhotodiodePlugin { if self.reader.is_some() { return; } + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + self.last_error = Some("connection deferred: hardware effects are not allowed".into()); + return; + } if let Ok(mut state) = self.shared.lock() { *state = SharedState::default(); } @@ -648,15 +854,189 @@ impl StageAPhotodiodePlugin { Ok(PathBuf::from(self.data_dir.trim())) } + /// Resolves a workflow-owned relative evidence path beneath the configured + /// data directory. Existing or newly created parent components must be + /// real directories, never symlinks. + fn resolve_control_path(&self, label: &str, extension: &str) -> Result { + let relative = Path::new(label); + if relative.as_os_str().is_empty() + || relative.is_absolute() + || relative + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err( + "workflow recording paths must be non-empty relative paths without '..'".into(), + ); + } + if relative.extension().and_then(|value| value.to_str()) != Some(extension) { + return Err(format!("workflow path must use the .{extension} extension")); + } + + let root = self.resolved_data_dir()?; + std::fs::create_dir_all(&root) + .map_err(|err| format!("creating {} failed: {err}", root.display()))?; + let root = root + .canonicalize() + .map_err(|err| format!("resolving data directory failed: {err}"))?; + let mut parent = root.clone(); + if let Some(relative_parent) = relative.parent() { + for component in relative_parent.components() { + let Component::Normal(name) = component else { + return Err("invalid workflow recording path".into()); + }; + parent.push(name); + match std::fs::symlink_metadata(&parent) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "workflow path crosses symlink {}", + parent.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(format!("{} is not a directory", parent.display())); + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(&parent).map_err(|err| { + format!("creating {} failed: {err}", parent.display()) + })?; + } + Err(err) => { + return Err(format!("checking {} failed: {err}", parent.display())); + } + } + let canonical = parent + .canonicalize() + .map_err(|err| format!("resolving {} failed: {err}", parent.display()))?; + if !canonical.starts_with(&root) { + return Err("workflow path escapes the configured data directory".into()); + } + } + } + let candidate = root.join(relative); + if let Ok(metadata) = std::fs::symlink_metadata(&candidate) { + if metadata.file_type().is_symlink() { + return Err(format!( + "workflow target is a symlink: {}", + candidate.display() + )); + } + } + Ok(candidate) + } + + fn begin_named_recording( + &mut self, + run_id: RunId, + specification: &PdqStartSpecV1, + ) -> Result { + if !self.connected() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "photodiode stream is not connected", + true, + )); + } + if specification.metadata.len() > 64 + || specification + .metadata + .iter() + .any(|(key, value)| key.len() > 128 || value.len() > 1_024) + { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "recording metadata exceeds owner bounds", + false, + )); + } + let (rate_hz, stream_epoch) = self + .shared + .lock() + .map(|state| (state.rate_hz, state.segments)) + .unwrap_or((0, 0)); + if specification + .expected_sample_rate_hz + .is_some_and(|expected| rate_hz != 0 && expected != rate_hz) + || specification + .expected_stream_epoch + .is_some_and(|expected| expected != stream_epoch) + { + return Err(service_error( + ServiceErrorCodeV1::Integrity, + "live photodiode stream does not match the requested epoch or sample rate", + true, + )); + } + let pdq_path = self + .resolve_control_path(&specification.pdq_path, "pdq") + .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; + let sidecar_path = self + .resolve_control_path(&specification.sidecar_path, "json") + .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; + if pdq_path == sidecar_path { + return Err(service_error( + ServiceErrorCodeV1::InvalidPath, + "PDQ and sidecar paths must differ", + false, + )); + } + self.open_recording( + run_id, + pdq_path, + sidecar_path, + specification.pdq_path.clone(), + specification.sidecar_path.clone(), + specification.metadata.clone(), + true, + ) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false)) + } + fn start_recording(&mut self) -> Result<(), String> { - if self.recording_active() { - return Ok(()); + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + return Err("recording is allowed only on the active live worker".into()); + } + if self.lease.is_some() { + return Err("manual recording is locked while a workflow lease is active".into()); } let dir = self.resolved_data_dir()?; let slug = timestamp_slug(); let pdq_path = dir.join(format!("pd_rec_{slug}.pdq")); - let writer = PdqWriter::create(&pdq_path) - .map_err(|err| format!("creating {} failed: {err}", pdq_path.display()))?; + let sidecar_path = pdq_path.with_extension("json"); + self.open_recording( + RunId::new(format!("manual-{slug}")), + pdq_path.clone(), + sidecar_path.clone(), + pdq_path.to_string_lossy().into_owned(), + sidecar_path.to_string_lossy().into_owned(), + BTreeMap::new(), + false, + )?; + self.last_save_note = Some(format!("recording → {}", pdq_path.display())); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn open_recording( + &mut self, + run_id: RunId, + pdq_path: PathBuf, + sidecar_path: PathBuf, + pdq_path_label: String, + sidecar_path_label: String, + metadata: BTreeMap, + exclusive: bool, + ) -> Result { + if self.recording_active() { + return Err("a photodiode recording is already active".into()); + } + let writer = if exclusive { + PdqWriter::create_new(&pdq_path) + } else { + PdqWriter::create(&pdq_path) + } + .map_err(|err| format!("creating {} failed: {err}", pdq_path.display()))?; let (crc, resync, dropped, segments) = match self.shared.lock() { Ok(state) => ( state.crc_failures, @@ -666,10 +1046,45 @@ impl StageAPhotodiodePlugin { ), Err(_) => (0, 0, 0, 0), }; + let (stream_epoch, first_sample_index) = self + .shared + .lock() + .map(|state| { + ( + state.segments, + (!state.samples.is_empty()) + .then_some(state.ring_first_index + state.samples.len() as u64), + ) + }) + .unwrap_or((0, None)); + let opened_at_unix_ms = now_unix_ms(); + let started_slug = timestamp_slug(); + if exclusive { + let started = json!({ + "kind": "recording_in_progress", + "run_id": run_id.as_str(), + "opened_at_unix_ms": opened_at_unix_ms, + "pdq_path": pdq_path_label, + "metadata": metadata, + }); + if let Err(err) = write_json_new(&sidecar_path, &started) { + drop(writer); + let _ = std::fs::remove_file(&pdq_path); + return Err(err); + } + } let sink = RecordingSink { writer, pdq_path: pdq_path.clone(), - started_slug: slug, + sidecar_path, + pdq_path_label, + sidecar_path_label, + run_id, + opened_at_unix_ms, + stream_epoch, + first_sample_index, + metadata, + started_slug, samples_written: 0, write_error: None, start_crc_failures: crc, @@ -677,16 +1092,25 @@ impl StageAPhotodiodePlugin { start_device_dropped: dropped, start_segments: segments, }; + let receipt = sink.started_receipt(); if let Ok(mut slot) = self.recording.lock() { *slot = Some(sink); } - self.last_save_note = Some(format!("recording → {}", pdq_path.display())); - Ok(()) + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(receipt) } fn stop_recording(&mut self) -> Result<(), String> { + self.finalize_recording(PdqTerminationV1::OperatorStopped) + .map(|_| ()) + } + + fn finalize_recording( + &mut self, + termination: PdqTerminationV1, + ) -> Result, String> { let Some(sink) = self.recording.lock().ok().and_then(|mut slot| slot.take()) else { - return Ok(()); + return Ok(None); }; let (rate_hz, crc, resync, dropped, segments) = match self.shared.lock() { Ok(state) => ( @@ -707,12 +1131,43 @@ impl StageAPhotodiodePlugin { let write_error = sink.write_error.clone(); let started = sink.started_slug.clone(); let samples = sink.samples_written; + let pdq_path = sink.pdq_path.clone(); + let sidecar_path = sink.sidecar_path.clone(); + let run_id = sink.run_id.clone(); + let opened_at_unix_ms = sink.opened_at_unix_ms; + let pdq_path_label = sink.pdq_path_label.clone(); + let sidecar_path_label = sink.sidecar_path_label.clone(); + let metadata = sink.metadata.clone(); let summary = sink .writer .finish(integrity) .map_err(|err| format!("finishing recording failed: {err}"))?; + let contract_integrity = contract_integrity(summary.integrity, summary.sample_segments); + let receipt = PdqFinalizedReceiptV1 { + run_id: run_id.clone(), + pdq_path: pdq_path_label, + sidecar_path: sidecar_path_label, + opened_at_unix_ms, + finalized_at_unix_ms: now_unix_ms(), + file_size_bytes: summary.bytes_written, + sha256: Sha256V1::parse(summary.file_sha256_hex()) + .map_err(|err| format!("invalid recording digest: {err}"))?, + frames_written: summary.frames_written, + sample_frames_written: summary.sample_frames_written, + sample_range: summary.sample_range.map(|range| SampleRangeV1 { + first_sample_index: range.first_sample_index, + end_sample_index_exclusive: range.end_sample_index_exclusive, + sample_count: range.sample_count, + }), + sample_rate_hz: summary.sample_rate_hz, + segment_count: summary.sample_segments, + integrity: contract_integrity, + termination, + valid: summary.valid && write_error.is_none(), + }; let sidecar = json!({ "kind": "recording", + "run_id": run_id, "started_utc": started, "stopped_utc": timestamp_slug(), "port": self.port_hint, @@ -722,6 +1177,9 @@ impl StageAPhotodiodePlugin { "pdq_frames": summary.frames_written, "pdq_bytes": summary.bytes_written, "pdq_crc32": summary.file_crc32, + "pdq_sha256": receipt.sha256.as_str(), + "metadata": metadata, + "termination": receipt.termination, "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), "reference_volts": self.reference_volts, @@ -734,21 +1192,473 @@ impl StageAPhotodiodePlugin { "valid": summary.valid && write_error.is_none(), "write_error": write_error, }); - let sidecar_path = sink.pdq_path.with_extension("json"); write_json(&sidecar_path, &sidecar)?; self.last_save_note = Some(format!( "saved recording {} ({} samples)", - sink.pdq_path.display(), + pdq_path.display(), samples )); + self.last_finalized_recording = Some(receipt.clone()); + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(Some(receipt)) + } + + fn lease_snapshot(&self) -> Option { + self.lease.as_ref().map(|lease| LeaseSnapshotV1 { + lease_id: lease.lease_id.clone(), + holder: lease.holder.clone(), + expires_at_unix_ms: lease.expires_at_unix_ms, + run_id: lease.run_id.clone(), + }) + } + + fn require_lease(&self, request: &PhotodiodeRequestV1) -> Result<(), ServiceErrorV1> { + let lease = self.lease.as_ref().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::LeaseRequired, + "the photodiode owner requires an active automation lease", + false, + ) + })?; + if now_unix_ms() > lease.expires_at_unix_ms { + return Err(service_error( + ServiceErrorCodeV1::LeaseExpired, + "the photodiode automation lease expired", + false, + )); + } + if request.lease_id.as_ref() != Some(&lease.lease_id) + || request.requester != lease.holder + || request.run_id != lease.run_id + { + return Err(service_error( + ServiceErrorCodeV1::LeaseMismatch, + "request lease, holder, or run does not match the active lease", + false, + )); + } Ok(()) } + fn require_new_revision( + &self, + request: &PhotodiodeRequestV1, + ) -> Result { + let revision = request.requested_revision.ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "recording transitions require requested_revision", + false, + ) + })?; + if self + .requested_revision + .is_some_and(|current| revision <= current) + { + return Err(service_error( + ServiceErrorCodeV1::StaleRequest, + "requested_revision must be newer than the current photodiode state", + false, + )); + } + Ok(revision) + } + + fn immediate_response( + &mut self, + request: &PhotodiodeRequestV1, + receipt: Option, + ) -> PhotodiodeResponseV1 { + let response = PhotodiodeResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: request.request_id, + owner_instance: self.owner_instance.clone(), + run_id: request.run_id.clone(), + requested_revision: request.requested_revision, + acknowledged_revision: self.acknowledged_revision, + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(now_unix_ms()), + error: None, + }, + receipt, + }; + self.last_response = Some(response.clone()); + self.generation.fetch_add(1, Ordering::Relaxed); + response + } + + fn handle_photodiode_command( + &mut self, + request: &PhotodiodeRequestV1, + ) -> Result { + match &request.command { + PhotodiodeCommandV1::Connect => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "connection cannot be changed while leased", + false, + )); + } + self.connect_requested = true; + self.connect(); + if !self.connected() { + return Err(service_error( + ServiceErrorCodeV1::Transport, + self.last_error + .clone() + .unwrap_or_else(|| "photodiode connection failed".into()), + true, + )); + } + Ok(self.immediate_response(request, None)) + } + PhotodiodeCommandV1::Disconnect { + finalize_recording, + reason, + } => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "use ReleaseLease while the owner is leased", + false, + )); + } + let receipt = if *finalize_recording { + self.finalize_recording(PdqTerminationV1::OperatorStopped) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .map(PdqReceiptV1::Finalized) + } else { + None + }; + self.connect_requested = false; + self.disconnect(); + self.last_error = Some(format!("disconnected by service: {reason}")); + Ok(self.immediate_response(request, receipt)) + } + PhotodiodeCommandV1::AcquireLease { ttl_ms } => { + let lease_id = request.lease_id.clone().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "AcquireLease requires lease_id", + false, + ) + })?; + if let Some(active) = &self.lease { + if active.lease_id != lease_id || active.holder != request.requester { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "the photodiode owner is already leased", + true, + )); + } + } + self.lease = Some(ControlLease { + lease_id, + holder: request.requester.clone(), + run_id: request.run_id.clone(), + expires_at_unix_ms: lease_deadline(*ttl_ms), + }); + Ok(self.immediate_response(request, None)) + } + PhotodiodeCommandV1::RenewLease { ttl_ms } => { + self.require_lease(request)?; + if let Some(lease) = &mut self.lease { + lease.expires_at_unix_ms = lease_deadline(*ttl_ms); + } + Ok(self.immediate_response(request, None)) + } + PhotodiodeCommandV1::ReleaseLease { + finalize_recording, + reason, + } => { + self.require_lease(request)?; + let receipt = if *finalize_recording { + self.finalize_recording(PdqTerminationV1::OperatorStopped) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .map(PdqReceiptV1::Finalized) + } else if self.recording_active() { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "cannot release a lease with an active recording unless it is finalized", + false, + )); + } else { + None + }; + self.lease = None; + self.last_error = Some(format!("automation lease released: {reason}")); + Ok(self.immediate_response(request, receipt)) + } + PhotodiodeCommandV1::BeginRecording { specification } => { + self.require_lease(request)?; + let revision = self.require_new_revision(request)?; + let run_id = request.run_id.clone().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "BeginRecording requires run_id", + false, + ) + })?; + let started = self.begin_named_recording(run_id, specification)?; + self.requested_revision = Some(revision); + self.acknowledged_revision = Some(revision); + Ok(self.immediate_response(request, Some(PdqReceiptV1::Started(started)))) + } + PhotodiodeCommandV1::FinalizeRecording { termination } => { + self.require_lease(request)?; + let revision = self.require_new_revision(request)?; + let finalized = self + .finalize_recording(*termination) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "no photodiode recording is active", + false, + ) + })?; + self.requested_revision = Some(revision); + self.acknowledged_revision = Some(revision); + Ok(self.immediate_response(request, Some(PdqReceiptV1::Finalized(finalized)))) + } + PhotodiodeCommandV1::AbortRecording { reason } => { + self.require_lease(request)?; + let revision = self.require_new_revision(request)?; + let finalized = self + .finalize_recording(PdqTerminationV1::Aborted) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "no photodiode recording is active", + false, + ) + })?; + self.requested_revision = Some(revision); + self.acknowledged_revision = Some(revision); + self.last_error = Some(format!("recording aborted: {reason}")); + Ok(self.immediate_response(request, Some(PdqReceiptV1::Finalized(finalized)))) + } + } + } + + /// Live optical log-contrast `a` from the trailing ring window. The ADC + /// always measures the rejected diode `I_pd`, so the display mode selects + /// the geometry: RAW reports the raw detector contrast (`Direct`), + /// EXCITATION reports the excitation contrast (`RejectedComplement`) using + /// `reference_volts` as the total-power anchor `I_tot`. `None` when there is + /// no valid window or, in EXCITATION mode, no valid anchor. + fn optical_summary(&self, samples: &VecDeque) -> Option { + let start = samples.len().saturating_sub(CONTRAST_WINDOW_SAMPLES); + let window: Vec = samples.iter().skip(start).copied().collect(); + let calibration = AdcCalibration { + volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, + offset_volts: 0.0, + dark_volts: 0.0, + full_scale_code: ADC_MAX_CODE as u16, + }; + let geometry = match self.mode { + Mode::Raw => ContrastGeometry::Direct, + Mode::Excitation => ContrastGeometry::RejectedComplement { + total_power_volts: self.reference_volts, + }, + }; + let estimate = estimate_contrast(&window, &calibration, geometry).ok()?; + let run_id = self + .lease + .as_ref() + .and_then(|lease| lease.run_id.clone()) + .unwrap_or_else(|| RunId::from("live")); + Some(PhotodiodeOpticalSummaryV1 { + run_id, + calibration: PhotodiodeCalibrationV1 { + adc_calibration_id: "adc-default".into(), + dark_id: "dark-0".into(), + anchor_id: match self.mode { + Mode::Raw => "detector-direct".into(), + Mode::Excitation => "reference-volts".into(), + }, + dark_volts: calibration.dark_volts, + total_power_volts: self.reference_volts, + }, + measured_log_contrast: estimate.a, + log_contrast_stddev: None, + excitation_min_volts: estimate.v_min_volts, + excitation_max_volts: estimate.v_max_volts, + excitation_headroom_volts: estimate.v_min_volts, + low_clip_fraction: estimate.low_clip_fraction, + high_clip_fraction: estimate.high_clip_fraction, + measured_frequency_hz: None, + fundamental_phase_rad: None, + total_harmonic_distortion: None, + }) + } + + /// Locks the ring and returns the current optical log-contrast summary. + fn latest_optical(&self) -> Option { + let state = self.shared.lock().ok()?; + self.optical_summary(&state.samples) + } + + fn control_summary(&self) -> PhotodiodeSummaryV1 { + let (stream, connection, observed_at, optical_summary) = match self.shared.lock() { + Ok(state) => { + let sample_range = (!state.samples.is_empty()).then_some(SampleRangeV1 { + first_sample_index: state.ring_first_index, + end_sample_index_exclusive: state.ring_first_index + state.samples.len() as u64, + sample_count: state.samples.len() as u64, + }); + let optical_summary = self.optical_summary(&state.samples); + let level = self.current_level(&state); + let connection = if self.connected() { + ConnectionStateV1::Connected { + port_label: self.port_hint.clone(), + firmware_version: None, + } + } else if let Some(message) = + state.error.clone().or_else(|| self.last_error.clone()) + { + ConnectionStateV1::Faulted { message } + } else if self.connect_requested { + ConnectionStateV1::Connecting + } else { + ConnectionStateV1::Disconnected + }; + ( + PhotodiodeStreamV1 { + stream_epoch: state.segments, + sample_range, + sample_rate_hz: (state.rate_hz != 0).then_some(state.rate_hz), + latest_adc_code: state.latest, + integrity: StreamIntegrityV1 { + skipped_bytes: state.resync_bytes, + crc_failures: state.crc_failures, + sequence_gaps: state.segments, + dropped_samples: u64::from(state.device_dropped), + segment_restarts: state.segments, + truncated_bytes: 0, + }, + level, + }, + connection, + state.last_update_unix_ms, + optical_summary, + ) + } + Err(_) => ( + PhotodiodeStreamV1 { + stream_epoch: 0, + sample_range: None, + sample_rate_hz: None, + latest_adc_code: None, + integrity: StreamIntegrityV1::default(), + level: None, + }, + ConnectionStateV1::Faulted { + message: "photodiode state lock poisoned".into(), + }, + 0, + None, + ), + }; + let active_recording = self + .recording + .lock() + .ok() + .and_then(|slot| slot.as_ref().map(RecordingSink::started_receipt)); + let synchronization = match ( + self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + self.requested_revision, + self.acknowledged_revision, + ) { + (Some(run_id), Some(requested), Some(acknowledged)) if requested == acknowledged => { + SynchronizationV1::Synced { + run_id, + acknowledged_revision: acknowledged, + stream_epoch: Some(stream.stream_epoch), + } + } + (None, _, _) => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::NoLease, + detail: None, + }, + _ => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::RequestedRevisionNotAcknowledged, + detail: None, + }, + }; + PhotodiodeSummaryV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: self.owner_instance.clone(), + service_revision: self.generation.load(Ordering::Relaxed), + connection, + lease: self.lease_snapshot(), + active_run_id: self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + requested_revision: self.requested_revision, + acknowledged_revision: self.acknowledged_revision, + stream, + active_recording, + last_finalized_recording: self.last_finalized_recording.clone(), + optical_summary, + synchronization, + last_response: self.last_response.clone(), + freshness: FreshnessV1 { + observed_at_unix_ms: if observed_at == 0 { + now_unix_ms() + } else { + observed_at + }, + valid_for_ms: SNAPSHOT_VALID_FOR_MS, + }, + } + } + + fn expire_lease_if_needed(&mut self) { + if self + .lease + .as_ref() + .is_none_or(|lease| now_unix_ms() <= lease.expires_at_unix_ms) + { + return; + } + if let Err(error) = self.finalize_recording(PdqTerminationV1::LeaseExpired) { + self.last_error = Some(error); + } else { + self.last_error = Some("automation lease expired; recording finalized".into()); + } + self.lease = None; + self.generation.fetch_add(1, Ordering::Relaxed); + } + + fn apply_execution_context(&mut self, execution: &augur_plugin_api::ExecutionContext) { + let allowed = self.runtime_role == PluginRuntimeRole::LiveWorker + && execution.hardware_effects_allowed(); + self.effects_allowed = allowed; + if !allowed { + if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { + self.last_error = Some(error); + } + self.connect_requested = false; + self.disconnect(); + self.lease = None; + return; + } + self.expire_lease_if_needed(); + if self.connect_requested && self.reader.is_none() { + self.connect(); + } + } + /// Dumps the current monitor cache (ring) as CSV + JSON sidecar. Raw /// codes and raw volts only — mode/reference land in the sidecar so /// EXCITATION values stay derivable without baking display state into /// the data. fn save_cache_snapshot(&mut self) -> Result<(), String> { + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + return Err("saving is allowed only on the active live worker".into()); + } let dir = self.resolved_data_dir()?; let slug = timestamp_slug(); let csv_path = dir.join(format!("pd_cache_{slug}.csv")); @@ -844,6 +1754,40 @@ impl StageAPhotodiodePlugin { Some(state.range_summary(start, state.samples.len()).mean()) } + /// Settled detector level over the same window, published on the contract + /// in **raw** detector volts — never `display_volts`, so a consumer does + /// not have to know the display mode, and never the optical geometry + /// transform, which needs an anchor this reading must not depend on. + /// + /// Deliberately fail-open where [`Self::optical_summary`] is fail-closed: + /// a transfer-curve sweep needs a level exactly at the excitation null, + /// where the reject-port detector is brightest and may rail. Clipping is + /// reported rather than refused. + fn current_level(&self, state: &SharedState) -> Option { + if state.samples.is_empty() { + return None; + } + let window = self + .avg_window_samples(state.rate_hz) + .min(state.samples.len()); + let start = state.samples.len() - window; + let summary = state.range_summary(start, state.samples.len()); + if summary.count == 0 { + return None; + } + let full_scale = ADC_MAX_CODE as u16; + Some(PhotodiodeLevelV1 { + mean_volts: code_to_volts(summary.mean()), + // `code_to_volts` is a pure scale, so it maps a code difference to + // a voltage difference directly. + peak_to_peak_volts: code_to_volts(f64::from(summary.max - summary.min)), + sample_count: summary.count as u64, + end_sample_index: state.ring_first_index + state.samples.len() as u64, + clipped: summary.min <= CLIP_MARGIN_CODES + || summary.max >= full_scale.saturating_sub(CLIP_MARGIN_CODES), + }) + } + fn series_dataset(&self) -> Series1dV1 { let y_label = match self.mode { Mode::Raw => "photodiode [V]", @@ -952,6 +1896,43 @@ impl StageAPhotodiodePlugin { points: avg_points, }); } + // Opt-in phase-0 trigger overlay: one toggleable line drawing a vertical + // spike at each marker (up then back to a flat baseline between markers). + if self.show_markers && !state.markers.is_empty() { + let first_visible = state.ring_first_index + start as u64; + let y_range = lines + .iter() + .flat_map(|line| line.points.iter()) + .map(|point| point.y) + .fold(None::<(f64, f64)>, |acc, y| { + Some(acc.map_or((y, y), |(lo, hi)| (lo.min(y), hi.max(y)))) + }); + if let Some((y_lo, y_hi)) = y_range { + let x_for = |index: u64| -> f64 { + let device_t = index as f64 / rate; + match self.time_axis { + TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, + TimeAxis::Segment => device_t, + } + }; + let mut points = Vec::with_capacity(state.markers.len() * 3); + for &index in &state.markers { + if index < first_visible || index > latest_x_index { + continue; + } + let x = x_for(index); + points.push(Series1dPoint { x, y: y_lo }); + points.push(Series1dPoint { x, y: y_hi }); + points.push(Series1dPoint { x, y: y_lo }); + } + if !points.is_empty() { + lines.push(Series1dLine { + name: "phase-0 trigger".into(), + points, + }); + } + } + } Series1dV1 { x_label: x_label.into(), y_label: y_label.into(), @@ -1180,6 +2161,85 @@ fn write_json(path: &Path, value: &Value) -> Result<(), String> { std::fs::write(path, bytes).map_err(|err| format!("writing {} failed: {err}", path.display())) } +fn write_json_new(path: &Path, value: &Value) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(value) + .map_err(|err| format!("serializing sidecar failed: {err}"))?; + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|err| format!("creating {} failed: {err}", path.display()))?; + file.write_all(&bytes) + .and_then(|()| file.flush()) + .map_err(|err| format!("writing {} failed: {err}", path.display())) +} + +fn now_unix_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0) +} + +fn lease_deadline(ttl_ms: u64) -> u64 { + now_unix_ms().saturating_add(ttl_ms.clamp(MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS)) +} + +fn service_error( + code: ServiceErrorCodeV1, + message: impl Into, + retryable: bool, +) -> ServiceErrorV1 { + ServiceErrorV1 { + code, + message: message.into(), + retryable, + } +} + +fn contract_integrity(integrity: StreamIntegrity, segments: u64) -> StreamIntegrityV1 { + StreamIntegrityV1 { + skipped_bytes: integrity.skipped_bytes, + crc_failures: integrity.crc_failures, + sequence_gaps: integrity.sequence_gaps, + dropped_samples: integrity.dropped_samples, + segment_restarts: segments.saturating_sub(1), + truncated_bytes: 0, + } +} + +fn accepted_service_reply( + request: &PluginServiceRequest, + response: &PhotodiodeResponseV1, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Accepted { + payload: serde_json::to_value(response).unwrap_or(Value::Null), + }, + } +} + +fn rejected_service_reply( + request: &PluginServiceRequest, + code: impl Into, + message: impl Into, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Rejected { + code: code.into(), + message: message.into(), + }, + } +} + fn serial_ports() -> Vec { serialport::available_ports() .map(|ports| { @@ -1348,10 +2408,29 @@ impl Plugin for StageAPhotodiodePlugin { self.connect_requested = false; // Finalize an active recording so the .pdq/.json pair is complete // even when the plugin is disabled mid-run. - if let Err(err) = self.stop_recording() { + let termination = if self.lease.is_some() { + PdqTerminationV1::Aborted + } else { + PdqTerminationV1::OperatorStopped + }; + if let Err(err) = self.finalize_recording(termination) { self.last_error = Some(err); } self.disconnect(); + self.lease = None; + } + } + + fn set_runtime_role(&mut self, role: PluginRuntimeRole) { + self.runtime_role = role; + if role != PluginRuntimeRole::LiveWorker { + if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { + self.last_error = Some(error); + } + self.connect_requested = false; + self.disconnect(); + self.lease = None; + self.effects_allowed = false; } } @@ -1366,12 +2445,116 @@ impl Plugin for StageAPhotodiodePlugin { &mut self, _frame: &PluginFrame<'_>, _output: &mut HostOutput<'_>, - _context: &mut HostContext<'_>, + context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - // Reading is settings-driven (connect checkbox) and works without - // camera frames; the stream port carries no commands, so no replay - // teardown is needed either. + if context.execution().mode == augur_plugin_api::ExecutionMode::Replay { + if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { + self.last_error = Some(error); + } + self.connect_requested = false; + self.disconnect(); + self.lease = None; + } + } + + fn process_control(&mut self, context: &mut PluginControlContext<'_>) { + let execution = context.execution(); + self.apply_execution_context(&execution); + } + + fn handle_service_request( + &mut self, + request: &PluginServiceRequest, + execution: &augur_plugin_api::ExecutionContext, + ) -> PluginServiceReply { + if let Some((previous, reply)) = self.request_cache.iter().find(|(previous, _)| { + previous.source_plugin_id == request.source_plugin_id + && previous.request_id == request.request_id + }) { + return if previous == request { + reply.clone() + } else { + rejected_service_reply( + request, + "request_id_conflict", + "request ID was reused for a different photodiode payload", + ) + }; + } + + let reply = if request.target_plugin_id != PLUGIN_ID_STAGE_A_PHOTODIODE { + rejected_service_reply(request, "wrong_target", "wrong photodiode owner target") + } else if request.service != SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1 { + rejected_service_reply( + request, + "unsupported_service", + format!("unsupported photodiode service '{}'", request.service), + ) + } else if self.runtime_role != PluginRuntimeRole::LiveWorker + || !execution.hardware_effects_allowed() + { + rejected_service_reply( + request, + "effects_not_allowed", + "photodiode effects are allowed only on the active live worker", + ) + } else { + self.effects_allowed = true; + match serde_json::from_value::(request.payload.clone()) { + Err(error) => rejected_service_reply( + request, + "invalid_payload", + format!("invalid photodiode request: {error}"), + ), + Ok(payload) + if payload.contract_version != CONTRACT_VERSION_V1 + || payload.request_id.0 != request.request_id + || payload.requester.as_str() != request.source_plugin_id + || payload + .target_owner_instance + .as_ref() + .is_some_and(|owner| owner != &self.owner_instance) => + { + rejected_service_reply( + request, + "identity_mismatch", + "contract version, request, requester, or owner instance mismatch", + ) + } + Ok(payload) + if payload.issued_at_unix_ms != 0 + && (now_unix_ms().saturating_sub(payload.issued_at_unix_ms) > 120_000 + || payload.issued_at_unix_ms.saturating_sub(now_unix_ms()) + > 30_000) => + { + rejected_service_reply(request, "stale_request", "request timestamp is stale") + } + Ok(payload) => match self.handle_photodiode_command(&payload) { + Ok(response) => accepted_service_reply(request, &response), + Err(error) => rejected_service_reply( + request, + format!("{:?}", error.code).to_ascii_lowercase(), + error.message, + ), + }, + } + }; + self.request_cache + .push_back((request.clone(), reply.clone())); + while self.request_cache.len() > REQUEST_CACHE_LIMIT { + self.request_cache.pop_front(); + } + reply + } + + fn control_snapshots(&self) -> Vec { + vec![PluginControlSnapshot { + plugin_id: PLUGIN_ID_STAGE_A_PHOTODIODE.into(), + topic: CTX_STAGE_A_PHOTODIODE_SUMMARY_V1.into(), + revision: self.generation.load(Ordering::Relaxed).max(1), + payload: serde_json::to_value(self.control_summary()).unwrap_or(Value::Null), + }] } fn settings_schema(&self) -> SettingsSchema { @@ -1516,6 +2699,19 @@ impl Plugin for StageAPhotodiodePlugin { .unwrap_or(0), }, }, + SettingItem { + key: "show_markers".into(), + label: "Show phase-0 trigger markers".into(), + tooltip: Some( + "Overlay the firmware phase-0 markers (device-clock MARKER frames) \ + as a toggleable vertical curve. Also defines the modulation \ + frequency from the marker spacing." + .into(), + ), + kind: SettingKind::Bool { + default: self.show_markers, + }, + }, ], }, SettingsSection { @@ -1561,26 +2757,37 @@ impl Plugin for StageAPhotodiodePlugin { }, }, SettingItem { - key: "record".into(), - label: "Record to disk".into(), + key: "record_start".into(), + label: "Start recording".into(), tooltip: Some( - "Start/stop appending every incoming sample frame to \ - pd_rec_.pdq; stopping writes the JSON sidecar." + "Start appending every incoming sample frame to \ + pd_rec_.pdq. Disabled until a data directory \ + is selected." .into(), ), - kind: SettingKind::Bool { - default: self.recording_active(), + kind: SettingKind::Button { + enabled: !self.data_dir.trim().is_empty(), }, }, + SettingItem { + key: "record_stop".into(), + label: "Stop recording".into(), + tooltip: Some( + "Stop the disk recording and write the JSON sidecar.".into(), + ), + kind: SettingKind::Button { enabled: true }, + }, SettingItem { key: "save_snapshot".into(), label: "Save cache snapshot".into(), tooltip: Some( - "Write the current cache as pd_cache_.csv \ - (+ JSON sidecar)." + "Write the current cache once as pd_cache_.csv \ + (+ JSON sidecar). Disabled until a data directory is selected." .into(), ), - kind: SettingKind::Button, + kind: SettingKind::Button { + enabled: !self.data_dir.trim().is_empty(), + }, }, ], }, @@ -1610,6 +2817,7 @@ impl Plugin for StageAPhotodiodePlugin { "reference_volts" => Some(json!(self.reference_volts)), "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), + "show_markers" => Some(json!(self.show_markers)), "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), "time_axis" => { let index = TimeAxis::VARIANTS @@ -1624,14 +2832,25 @@ impl Plugin for StageAPhotodiodePlugin { .lock() .map(|state| state.cache_seconds) .unwrap_or(DEFAULT_CACHE_SECONDS))), + // Kept for compatibility (tests, external tooling); not in the + // schema anymore, so it is never synced across instances. "record" => Some(json!(self.recording_active())), - // Momentary trigger: never reports as pressed. - "save_snapshot" => Some(json!(false)), + // Button presses are exported as monotonic counters so the host's + // settings snapshot transports them from the UI mirror to the + // live worker (see PressLatch). + "record_start" => Some(self.press_record_start.value()), + "record_stop" => Some(self.press_record_stop.value()), + "save_snapshot" => Some(self.press_save_snapshot.value()), _ => None, } } fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + if self.lease.is_some() { + return Err(format!( + "manual setting '{key}' is locked while the photodiode owner is leased" + )); + } match key { "port" => { self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); @@ -1660,6 +2879,10 @@ impl Plugin for StageAPhotodiodePlugin { self.reference_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); Ok(()) } + "show_markers" => { + self.show_markers = value.as_bool().ok_or("show_markers must be a boolean")?; + Ok(()) + } "window_s" => { let seconds = value.as_f64().ok_or("window_s must be a number")?; self.window_s = seconds.clamp(0.01, 120.0); @@ -1700,9 +2923,9 @@ impl Plugin for StageAPhotodiodePlugin { Ok(()) } "record" => { + // Compatibility alias (not in the schema): direct boolean + // start/stop with the same edge-free semantics as before. let requested = value.as_bool().ok_or("record must be a boolean")?; - // Failures surface through status entries (like `connect`), - // so a missing data directory doesn't read as a broken UI. let result = if requested { self.start_recording() } else { @@ -1716,12 +2939,39 @@ impl Plugin for StageAPhotodiodePlugin { self.generation.fetch_add(1, Ordering::Relaxed); Ok(()) } + "record_start" => { + // Failures surface through status entries (like `connect`), + // so a missing data directory doesn't read as a broken UI. + if self.press_record_start.accept(&value) { + match self.start_recording() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } + "record_stop" => { + if self.press_record_stop.accept(&value) { + match self.stop_recording() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } "save_snapshot" => { - match self.save_cache_snapshot() { - Ok(()) => self.last_error = None, - Err(err) => self.last_error = Some(err), + // Edge-guarded: the host re-applies the full settings snapshot + // on every sync, and an unguarded arm wrote one cache file per + // sync of *any* plugin's settings. + if self.press_save_snapshot.accept(&value) { + match self.save_cache_snapshot() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); } - self.generation.fetch_add(1, Ordering::Relaxed); Ok(()) } _ => Err(format!("unknown setting: {key}")), @@ -1771,6 +3021,27 @@ impl Plugin for StageAPhotodiodePlugin { ))); } } + if let Some(optical) = self.latest_optical() { + let label = match self.mode { + Mode::Raw => "a_raw (detector)", + Mode::Excitation => "a (excitation)", + }; + entries.push(StatusEntry::Text(format!( + "{label} = {:.3} (I {:.4}..{:.4} V)", + optical.measured_log_contrast, + optical.excitation_min_volts, + optical.excitation_max_volts + ))); + } + if let Ok(state) = self.shared.lock() { + if let Some(period_samples) = state.marker_period_samples() { + let hz = f64::from(state.rate_hz.max(1)) / period_samples; + entries.push(StatusEntry::Text(format!( + "Trigger: {} markers, f = {hz:.3} Hz", + state.markers.len() + ))); + } + } if self.recording_active() { let (samples, path) = self .recording @@ -1877,8 +3148,50 @@ export_plugin!(StageAPhotodiodePlugin); #[cfg(test)] mod tests { use super::*; + use augur_plugin_api::{ExecutionContext, ExecutionMode}; use stage_a_io::{Frame, FrameHeader, FrameType}; + fn live_execution() -> ExecutionContext { + ExecutionContext { + mode: ExecutionMode::LiveCapture, + effects_allowed: true, + session_id: Some("test".into()), + } + } + + fn live_plugin() -> StageAPhotodiodePlugin { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_runtime_role(PluginRuntimeRole::LiveWorker); + plugin.effects_allowed = true; + plugin + } + + fn service_request( + plugin: &StageAPhotodiodePlugin, + id: u64, + requester: &str, + command: PhotodiodeCommandV1, + revision: Option, + ) -> PluginServiceRequest { + let mut payload = PhotodiodeRequestV1::new( + stage_a_plugin_contract::RequestId(id), + ClientId::from(requester), + command, + ); + payload.target_owner_instance = Some(plugin.owner_instance.clone()); + payload.run_id = Some(RunId::from("run-a")); + payload.lease_id = Some(LeaseId::from("lease-a")); + payload.requested_revision = revision.map(SemanticRevision); + payload.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id: id, + source_plugin_id: requester.into(), + target_plugin_id: PLUGIN_ID_STAGE_A_PHOTODIODE.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + payload: serde_json::to_value(payload).unwrap(), + } + } + fn sample_frame(sequence: u32, first_index: u64, rate_hz: u32, codes: &[u16]) -> Vec { let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); Frame::build( @@ -1941,6 +3254,31 @@ mod tests { assert_eq!(state.segments, 2); } + #[test] + fn phase0_markers_define_frequency_and_evict_with_the_ring() { + // Ring holds 1 s = 20_000 samples at 20 kSa/s. + let mut state = SharedState { + cache_seconds: 1.0, + ..SharedState::default() + }; + // 500 Hz modulation: markers every 40 samples. + ingest_bytes(&mut state, &sample_frame(0, 0, 20_000, &[100; 40])); + state.push_marker(0); + state.push_marker(40); + state.push_marker(80); + assert_eq!(state.markers.len(), 3); + let period = state.marker_period_samples().expect("period"); + assert!((period - 40.0).abs() < 1e-9); + let hz = f64::from(state.rate_hz) / period; + assert!((hz - 500.0).abs() < 1e-6, "hz={hz}"); + + // Duplicate stamps are ignored, and markers before the ring start too. + state.push_marker(80); + state.ring_first_index = 60; + state.push_marker(40); // now below the ring start + assert_eq!(state.markers.len(), 3); + } + #[test] fn ring_is_bounded_by_duration() { let mut state = SharedState::default(); @@ -2001,6 +3339,35 @@ mod tests { assert!((average - 250.0).abs() < 1e-9); } + #[test] + fn published_level_is_raw_volts_and_survives_clipping() { + let mut plugin = StageAPhotodiodePlugin::default(); // window = 4 samples + let mut state = SharedState::default(); + state.ingest(0, 20_000, 0, &[0, 0, 0, 0, 100, 200, 300, 400]); + + let level = plugin.current_level(&state).expect("has samples"); + assert!((level.mean_volts - code_to_volts(250.0)).abs() < 1e-9); + assert!((level.peak_to_peak_volts - code_to_volts(300.0)).abs() < 1e-9); + assert_eq!(level.sample_count, 4); + // The window is the newest 4 of 8 ingested samples. + assert_eq!(level.end_sample_index, 8); + assert!(!level.clipped); + + // EXCITATION display must not leak into the published level: it stays + // the raw detector reading whatever the operator is looking at. + plugin.set_setting("mode", json!(1)).expect("excitation"); + let raw_again = plugin.current_level(&state).expect("has samples"); + assert_eq!(raw_again.mean_volts, level.mean_volts); + + // At the rail the optical summary refuses; the level must not, because + // that is exactly where a transfer sweep needs a reading. + let mut railed = SharedState::default(); + railed.ingest(0, 20_000, 0, &[4_095; 8]); + let clipped = plugin.current_level(&railed).expect("still reports"); + assert!(clipped.clipped); + assert!(plugin.optical_summary(&railed.samples).is_none()); + } + #[test] fn series_dataset_decimates_with_envelope_and_average() { let mut plugin = StageAPhotodiodePlugin::default(); @@ -2065,7 +3432,7 @@ mod tests { fn mock_reader_fills_the_ring_and_series() { let mut plugin = StageAPhotodiodePlugin { port_hint: "mock".into(), - ..Default::default() + ..live_plugin() }; plugin.connect(); let deadline = Instant::now() + Duration::from_secs(2); @@ -2212,7 +3579,7 @@ mod tests { #[test] fn cache_snapshot_writes_csv_and_sidecar() { let dir = temp_dir("snapshot"); - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = live_plugin(); plugin .set_setting("data_dir", json!(dir.display().to_string())) .unwrap(); @@ -2247,9 +3614,59 @@ mod tests { std::fs::remove_dir_all(dir).unwrap(); } + #[test] + fn forwarded_snapshot_counter_saves_exactly_once() { + let dir = temp_dir("snapshot-forwarded"); + let mut plugin = live_plugin(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(10, 20_000, 0, &[100, 200, 300]); + } + let csv_count = |dir: &std::path::Path| { + std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|ext| ext == "csv")) + .count() + }; + // First forwarded counter is the baseline a fresh worker adopts. + plugin.set_setting("save_snapshot", json!(2)).unwrap(); + assert_eq!(csv_count(&dir), 0, "baseline must not save"); + // One press on the mirror advances the counter by one → one file. + plugin.set_setting("save_snapshot", json!(3)).unwrap(); + assert_eq!(csv_count(&dir), 1); + // The host re-applies the same snapshot on every settings sync of any + // plugin — this used to write one file per sync. + plugin.set_setting("save_snapshot", json!(3)).unwrap(); + plugin.set_setting("save_snapshot", json!(3)).unwrap(); + assert_eq!(csv_count(&dir), 1, "re-applied snapshots must not save"); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn record_buttons_start_and_stop_the_disk_recording() { + let dir = temp_dir("record-buttons"); + let mut plugin = live_plugin(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + plugin.set_setting("record_start", json!(true)).unwrap(); + assert!(plugin.recording_active()); + // Idle stop is a no-op, an active stop finalizes. + plugin.set_setting("record_stop", json!(true)).unwrap(); + assert!(!plugin.recording_active()); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + plugin.set_setting("record_stop", json!(true)).unwrap(); + assert!(plugin.last_error.is_none()); + std::fs::remove_dir_all(dir).unwrap(); + } + #[test] fn snapshot_without_data_dir_reports_an_error() { - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = live_plugin(); plugin.set_setting("save_snapshot", json!(true)).unwrap(); assert!(plugin .last_error @@ -2260,7 +3677,7 @@ mod tests { #[test] fn recording_tees_frames_to_pdq_and_writes_a_sidecar() { let dir = temp_dir("recording"); - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = live_plugin(); plugin .set_setting("data_dir", json!(dir.display().to_string())) .unwrap(); @@ -2342,4 +3759,217 @@ mod tests { .expect("name accepted"); assert_eq!(plugin.mode, Mode::Raw); } + + #[test] + fn ui_mirror_never_opens_the_stream_or_writes_recordings() { + let dir = temp_dir("ui-mirror"); + let mut plugin = StageAPhotodiodePlugin { + port_hint: "mock".into(), + data_dir: dir.display().to_string(), + ..Default::default() + }; + plugin.set_setting("connect", json!(true)).unwrap(); + plugin.set_setting("record", json!(true)).unwrap(); + assert!(!plugin.connected()); + assert!(!plugin.recording_active()); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn service_is_idempotent_and_enforces_exclusive_leases_without_frames() { + let mut plugin = live_plugin(); + let acquire = service_request( + &plugin, + 1, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + let first = plugin.handle_service_request(&acquire, &live_execution()); + let expiry = plugin.lease.as_ref().unwrap().expires_at_unix_ms; + let duplicate = plugin.handle_service_request(&acquire, &live_execution()); + assert_eq!(first, duplicate); + assert_eq!(plugin.lease.as_ref().unwrap().expires_at_unix_ms, expiry); + + let conflict = service_request( + &plugin, + 2, + "workflow-b", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&conflict, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + assert!(plugin.set_setting("mode", json!("RAW")).is_err()); + } + + #[test] + fn named_recording_rejects_unsafe_paths_and_returns_final_receipt() { + let dir = temp_dir("named"); + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.data_dir = dir.display().to_string(); + plugin.connect(); + let acquire = service_request( + &plugin, + 10, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&acquire, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + )); + + let unsafe_begin = service_request( + &plugin, + 11, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "../escape.pdq".into(), + sidecar_path: "run/escape.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + }, + }, + Some(1), + ); + assert!(matches!( + plugin + .handle_service_request(&unsafe_begin, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + + let begin = service_request( + &plugin, + 12, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "A1/run-a_pd.pdq".into(), + sidecar_path: "A1/run-a_pd.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::from([("workflow".into(), "A1".into())]), + }, + }, + Some(1), + ); + let begin_reply = plugin.handle_service_request(&begin, &live_execution()); + assert!(matches!( + begin_reply.outcome, + PluginServiceOutcome::Accepted { .. } + )); + assert_eq!( + plugin.handle_service_request(&begin, &live_execution()), + begin_reply, + "duplicate begin must not open a second file" + ); + record_frame(&plugin.recording, &mock_sample_frame(9, 0, &[1, 2, 3]), 3); + + let finalize = service_request( + &plugin, + 13, + "workflow-a", + PhotodiodeCommandV1::FinalizeRecording { + termination: PdqTerminationV1::Completed, + }, + Some(2), + ); + let reply = plugin.handle_service_request(&finalize, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = reply.outcome else { + panic!("finalize rejected"); + }; + let response: PhotodiodeResponseV1 = serde_json::from_value(payload).unwrap(); + let Some(PdqReceiptV1::Finalized(receipt)) = response.receipt else { + panic!("missing finalized receipt"); + }; + assert_eq!(receipt.sha256.as_str().len(), 64); + assert!(receipt.file_size_bytes > 0); + assert!(dir.join(&receipt.pdq_path).is_file()); + assert!(dir.join(&receipt.sidecar_path).is_file()); + + let collision = service_request( + &plugin, + 14, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: receipt.pdq_path.clone(), + sidecar_path: receipt.sidecar_path.clone(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + }, + }, + Some(3), + ); + assert!(matches!( + plugin + .handle_service_request(&collision, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + plugin.disconnect(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn effects_revocation_finalizes_and_disconnects_without_a_frame() { + let dir = temp_dir("revoked"); + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.data_dir = dir.display().to_string(); + plugin.connect(); + let acquire = service_request( + &plugin, + 20, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + let begin = service_request( + &plugin, + 21, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "revoked/run.pdq".into(), + sidecar_path: "revoked/run.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + }, + }, + Some(1), + ); + plugin.handle_service_request(&begin, &live_execution()); + assert!(plugin.recording_active()); + + plugin.apply_execution_context(&ExecutionContext::fail_closed()); + assert!(!plugin.connected()); + assert!(!plugin.recording_active()); + assert!(plugin.lease.is_none()); + assert_eq!( + plugin + .last_finalized_recording + .as_ref() + .unwrap() + .termination, + PdqTerminationV1::Aborted + ); + std::fs::remove_dir_all(dir).unwrap(); + } } diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs index 809786a..1c87f44 100644 --- a/stage-a-io/src/estimator.rs +++ b/stage-a-io/src/estimator.rs @@ -1,11 +1,22 @@ //! Calibrated optical log-contrast estimator. //! -//! `a = ln(I_max / I_min)` is defined by the *measured light*, never by the -//! commanded DAC excursion: the Pockels-cell V→T response is non-linear, so -//! the photodiode ADC trace is the only valid source of `a` +//! `a = ln(I_exc,max / I_exc,min)` is defined by the *excitation light*, never +//! by the commanded DAC excursion: the Pockels-cell V→T response is non-linear, +//! so the photodiode ADC trace is the only valid source of `a` //! (knowledge base: `methodology/camera-calibration.md`, "define `a` from //! the light, not the drive"). //! +//! The detector geometry matters. When the photodiode sits behind the PBS +//! reject port it measures the *rejected complement* `I_pd = I_tot - I_exc`, +//! so the peak detector ratio is **not** the excitation contrast. The caller +//! selects the geometry via [`ContrastGeometry`]: +//! - [`ContrastGeometry::Direct`] — the detector already sees the excitation +//! intensity (e.g. the plugin's EXCITATION display, `I_tot - I_pd`), so +//! `a = ln(v_max / v_min)`. +//! - [`ContrastGeometry::RejectedComplement`] — the detector sees the rejected +//! light (the plugin's RAW display), so +//! `a = ln((I_tot - v_min) / (I_tot - v_max))`. +//! //! The estimator therefore: //! - converts ADC codes to volts through a characterised affine calibration, //! - subtracts the dark level (the detector is DC-coupled; `a` needs true @@ -13,8 +24,8 @@ //! - takes robust percentile extrema rather than raw min/max so single-code //! noise spikes do not bias the contrast, //! - refuses to produce a value at all when the window clips (top/bottom of -//! the ADC range) or has no headroom above dark — a wrong `a` is worse -//! than no `a`. +//! the ADC range), has no headroom above dark, or the total-power anchor is +//! below the measured signal — a wrong `a` is worse than no `a`. use serde::{Deserialize, Serialize}; @@ -48,10 +59,25 @@ impl AdcCalibration { } } +/// Optical geometry of the detector relative to the excitation beam. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub enum ContrastGeometry { + /// The detector already measures the excitation intensity, so + /// `a = ln(v_max / v_min)`. + Direct, + /// The detector sits behind the PBS reject port and measures the rejected + /// complement `I_pd = I_tot - I_exc`. `total_power_volts` is the + /// dark-corrected total power `I_tot`; the excitation contrast is + /// `a = ln((I_tot - v_min) / (I_tot - v_max))`. + RejectedComplement { total_power_volts: f64 }, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ContrastEstimate { - /// Peak-to-peak log-contrast `a = ln(V_max / V_min)` (dark-corrected). + /// Peak-to-peak excitation log-contrast `a = ln(I_exc,max / I_exc,min)` + /// (dark-corrected, geometry-resolved). pub a: f64, + /// Excitation intensity extrema in volts after the geometry transform. pub v_min_volts: f64, pub v_max_volts: f64, /// Fraction of samples at or below code 0 + margin. @@ -61,7 +87,7 @@ pub struct ContrastEstimate { pub sample_count: usize, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum EstimateError { /// Fewer samples than the estimator can use robustly. TooFewSamples { count: usize, minimum: usize }, @@ -72,6 +98,13 @@ pub enum EstimateError { }, /// The dark-corrected minimum is not positive: no optical headroom. NoHeadroomAboveDark, + /// The rejected-complement total-power anchor `I_tot` is not above the + /// measured detector maximum, so the excitation minimum would be + /// non-positive: the anchor is wrong or the light is not the complement. + TotalPowerBelowSignal { + total_power_volts: f64, + detector_max_volts: f64, + }, } impl std::fmt::Display for EstimateError { @@ -90,6 +123,14 @@ impl std::fmt::Display for EstimateError { Self::NoHeadroomAboveDark => { f.write_str("dark-corrected minimum is not positive; a is undefined") } + Self::TotalPowerBelowSignal { + total_power_volts, + detector_max_volts, + } => write!( + f, + "total-power anchor {total_power_volts:.4} V is not above the detector \ + maximum {detector_max_volts:.4} V; a is undefined" + ), } } } @@ -105,12 +146,15 @@ pub const MAX_CLIP_FRACTION: f64 = 0.001; const LOW_PERCENTILE: f64 = 0.01; const HIGH_PERCENTILE: f64 = 0.99; -/// Estimates the optical log-contrast from one settled, phase-attributed +/// Estimates the excitation log-contrast from one settled, phase-attributed /// ADC window. The window must span at least a few full modulation cycles; -/// enforcing that is the caller's job (it knows the drive frequency). +/// enforcing that is the caller's job (it knows the drive frequency). The +/// `geometry` selects whether the codes are the excitation intensity directly +/// or the rejected complement measured behind the PBS reject port. pub fn estimate_contrast( codes: &[u16], calibration: &AdcCalibration, + geometry: ContrastGeometry, ) -> Result { if codes.len() < MIN_SAMPLES { return Err(EstimateError::TooFewSamples { @@ -139,16 +183,38 @@ pub fn estimate_contrast( let low_code = percentile(&sorted, LOW_PERCENTILE); let high_code = percentile(&sorted, HIGH_PERCENTILE); - let v_min = calibration.code_to_volts(low_code) - calibration.dark_volts; - let v_max = calibration.code_to_volts(high_code) - calibration.dark_volts; - if v_min <= 0.0 || v_max <= 0.0 { - return Err(EstimateError::NoHeadroomAboveDark); - } + // Dark-corrected detector volts at the robust extrema. + let detector_low = calibration.code_to_volts(low_code) - calibration.dark_volts; + let detector_high = calibration.code_to_volts(high_code) - calibration.dark_volts; + + // Resolve the excitation extrema from the detector geometry. + let (exc_min, exc_max) = match geometry { + ContrastGeometry::Direct => { + if detector_low <= 0.0 { + return Err(EstimateError::NoHeadroomAboveDark); + } + (detector_low, detector_high) + } + ContrastGeometry::RejectedComplement { total_power_volts } => { + // The most transmitted excitation coincides with the least rejected + // light (detector_low), and vice versa. + if total_power_volts <= detector_high { + return Err(EstimateError::TotalPowerBelowSignal { + total_power_volts, + detector_max_volts: detector_high, + }); + } + ( + total_power_volts - detector_high, + total_power_volts - detector_low, + ) + } + }; Ok(ContrastEstimate { - a: (v_max / v_min).ln(), - v_min_volts: v_min, - v_max_volts: v_max, + a: (exc_max / exc_min).ln(), + v_min_volts: exc_min, + v_max_volts: exc_max, low_clip_fraction, high_clip_fraction, sample_count: codes.len(), @@ -183,7 +249,8 @@ mod tests { }; // center 2048, amplitude 900 -> dark-corrected V ratio: let codes = sine_codes(2_048.0, 900.0, 4_096); - let estimate = estimate_contrast(&codes, &calibration).expect("clean window estimates"); + let estimate = estimate_contrast(&codes, &calibration, ContrastGeometry::Direct) + .expect("clean window estimates"); let expected = ((2_048.0_f64 + 900.0 - 40.0) / (2_048.0 - 900.0 - 40.0)).ln(); assert!( @@ -194,11 +261,60 @@ mod tests { assert!(estimate.low_clip_fraction == 0.0 && estimate.high_clip_fraction == 0.0); } + #[test] + fn direct_and_rejected_complement_recover_the_same_excitation_contrast() { + // Excitation is a clean sine between exc_min and exc_max; the reject + // port sees the complement I_tot - I_exc. Both geometries must recover + // the same excitation log-contrast a = ln(exc_max / exc_min). + let calibration = AdcCalibration::default(); + let volts_per_code = calibration.volts_per_code; + let total_power_volts = 3_600.0 * volts_per_code; + let exc_center = 1_600.0; + let exc_amplitude = 900.0; + + let excitation_codes = sine_codes(exc_center, exc_amplitude, 4_096); + let rejected_codes: Vec = excitation_codes.iter().map(|&code| 3_600 - code).collect(); + + let direct = estimate_contrast(&excitation_codes, &calibration, ContrastGeometry::Direct) + .expect("direct excitation window"); + let rejected = estimate_contrast( + &rejected_codes, + &calibration, + ContrastGeometry::RejectedComplement { total_power_volts }, + ) + .expect("rejected complement window"); + + let expected = ((exc_center + exc_amplitude) / (exc_center - exc_amplitude)).ln(); + assert!((direct.a - expected).abs() < 0.01, "direct a={}", direct.a); + assert!( + (rejected.a - direct.a).abs() < 0.01, + "rejected a={} direct a={}", + rejected.a, + direct.a + ); + } + + #[test] + fn rejected_complement_rejects_a_total_power_anchor_below_the_signal() { + let calibration = AdcCalibration::default(); + let codes = sine_codes(2_048.0, 900.0, 2_048); + // Anchor far below the detector maximum (~2948 codes). + let err = estimate_contrast( + &codes, + &calibration, + ContrastGeometry::RejectedComplement { + total_power_volts: 1_000.0 * calibration.volts_per_code, + }, + ) + .expect_err("anchor below signal must be rejected"); + assert!(matches!(err, EstimateError::TotalPowerBelowSignal { .. })); + } + #[test] fn rejects_clipped_windows() { // Amplitude pushes past full scale -> clipping at the top rail. let codes = sine_codes(3_500.0, 900.0, 2_048); - let err = estimate_contrast(&codes, &AdcCalibration::default()) + let err = estimate_contrast(&codes, &AdcCalibration::default(), ContrastGeometry::Direct) .expect_err("clipped window must be rejected"); assert!(matches!(err, EstimateError::Clipped { .. })); } @@ -211,15 +327,19 @@ mod tests { }; // Minimum (2048-900=1148) sits below the dark level (1300). let codes = sine_codes(2_048.0, 900.0, 2_048); - let err = estimate_contrast(&codes, &calibration) + let err = estimate_contrast(&codes, &calibration, ContrastGeometry::Direct) .expect_err("no headroom above dark must be rejected"); assert_eq!(err, EstimateError::NoHeadroomAboveDark); } #[test] fn rejects_short_windows() { - let err = estimate_contrast(&[100; 10], &AdcCalibration::default()) - .expect_err("short window rejected"); + let err = estimate_contrast( + &[100; 10], + &AdcCalibration::default(), + ContrastGeometry::Direct, + ) + .expect_err("short window rejected"); assert!(matches!(err, EstimateError::TooFewSamples { .. })); } @@ -230,9 +350,12 @@ mod tests { let clean = estimate_contrast( &sine_codes(2_048.0, 500.0, 4_096), &AdcCalibration::default(), + ContrastGeometry::Direct, ) .expect("clean"); - let spiked = estimate_contrast(&codes, &AdcCalibration::default()).expect("spiked"); + let spiked = + estimate_contrast(&codes, &AdcCalibration::default(), ContrastGeometry::Direct) + .expect("spiked"); assert!((clean.a - spiked.a).abs() < 0.005); } } diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs index 2a9eabd..bf726e7 100644 --- a/stage-a-io/src/lib.rs +++ b/stage-a-io/src/lib.rs @@ -12,7 +12,8 @@ //! ADC overruns — any of which invalidates a measurement point), //! - a bounded background I/O worker so plugin `process_frame()` never //! blocks on serial, -//! - the `.pdq` raw-frame writer and the JSON run sidecar, +//! - streaming `.pdq` write/replay with CRC32, SHA-256, byte/frame counts, +//! contiguous sample-range receipts, plus the JSON run sidecar, //! - the calibrated optical log-contrast estimator (`a` is measured light, //! never the commanded DAC excursion), //! - a mock controller for tests and hardware-free development. @@ -26,20 +27,29 @@ pub mod estimator; pub mod mock; pub mod pdq; pub mod protocol; +mod sha256; pub mod sidecar; pub mod transport; pub mod wire; pub use client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; -pub use estimator::{estimate_contrast, AdcCalibration, ContrastEstimate, EstimateError}; +pub use estimator::{ + estimate_contrast, AdcCalibration, ContrastEstimate, ContrastGeometry, EstimateError, +}; pub use mock::{MockController, MockState, MockWave}; -pub use pdq::{PdqSummary, PdqWriter}; +pub use pdq::{ + inspect_pdq, PdqReadEvent, PdqReadSummary, PdqReader, PdqSampleRange, PdqSummary, PdqWriter, +}; pub use protocol::{Command, ControlMessage, ProtocolError}; +pub use sha256::Sha256Digest; pub use sidecar::{DetectorLoad, IntegrityRecord, RunSidecar, TriggerSource}; #[cfg(feature = "hardware")] pub use transport::SerialTransport; pub use transport::{MockLink, MockTransport, Transport}; -pub use wire::{Frame, FrameHeader, FrameParser, FrameType, ParseEvent, SummaryPayload}; +pub use wire::{ + Frame, FrameHeader, FrameParser, FrameType, MarkerPayload, ParseEvent, SummaryPayload, + MARKER_SOURCE_PHASE0, +}; pub use worker::{IoWorker, WorkerOutput, WorkerRequest}; pub mod worker; diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs index 203733c..e3aabb0 100644 --- a/stage-a-io/src/mock.rs +++ b/stage-a-io/src/mock.rs @@ -70,6 +70,17 @@ impl MockWave { } } +/// Optical warp parameters accepted on `MOD wave=WARP` (firmware rebuilds the +/// DAC table from these; the mock only validates them). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct WarpParams { + target: Option, + a_milli: u32, + u_k_milli: u32, + v_null: u32, + v_pi: u32, +} + #[derive(Debug, Clone, PartialEq)] struct MockConfig { mode: String, @@ -435,6 +446,7 @@ impl MockController { let mut min_level = 0_u32; let mut freq_mhz = 0_u32; let mut saw_freq = false; + let mut warp: WarpParams = WarpParams::default(); for (key, value) in fields { match key.as_str() { "wave" => { @@ -443,6 +455,7 @@ impl MockController { "CONST" => "CONST", "SINE" => "SINE", "SQUARE" => "SQUARE", + "WARP" => "WARP", _ => return err("RANGE", "invalid_wave"), }); } @@ -464,12 +477,64 @@ impl MockController { } _ => return err("RANGE", "invalid_freq_mhz"), }, + // Optical warp parameters (wave=WARP): the firmware rebuilds the + // 256-entry DAC table from these; the mock only validates them. + "target" => match value.as_str() { + "LOG_SINE" | "LINEAR_SINE" => warp.target = Some(value.clone()), + _ => return err("RANGE", "invalid_target"), + }, + "a_milli" => match value.parse::() { + Ok(parsed) if parsed > 0 => warp.a_milli = parsed, + _ => return err("RANGE", "invalid_a"), + }, + "u_k_milli" => match value.parse::() { + Ok(parsed) if (1..=1_000).contains(&parsed) => warp.u_k_milli = parsed, + _ => return err("RANGE", "invalid_u_k"), + }, + "v_null" => match value.parse::() { + Ok(parsed) if parsed <= 4_095 => warp.v_null = parsed, + _ => return err("RANGE", "invalid_v_null"), + }, + "v_pi" => match value.parse::() { + Ok(parsed) if (1..=4_095).contains(&parsed) => warp.v_pi = parsed, + _ => return err("RANGE", "invalid_v_pi"), + }, _ => return err("SYNTAX", "unknown_mod_field"), } } let Some(wave) = wave else { return err("SYNTAX", "wave_required"); }; + if wave == "WARP" { + if !saw_freq { + return err("SYNTAX", "freq_mhz_required"); + } + if warp.target.is_none() { + return err("SYNTAX", "target_required"); + } + if warp.u_k_milli == 0 { + return err("SYNTAX", "u_k_required"); + } + if warp.v_null + warp.v_pi > 4_095 { + return err("RANGE", "warp_exceeds_range"); + } + if !(MOCK_MOD_MIN_FREQ_MHZ..=MOCK_MOD_MAX_FREQ_MHZ).contains(&freq_mhz) { + return err("RANGE", "mod_rejected"); + } + self.mod_wave = "WARP"; + self.mod_min = warp.v_null; + self.mod_level = warp.v_null + warp.v_pi; + self.mod_code = warp.v_null; + self.mod_freq_mhz = freq_mhz; + return format!( + "+{sequence} OK mod_wave=WARP mod_level={} mod_min={} mod_freq_mhz={} code={} target={}", + self.mod_level, + self.mod_min, + self.mod_freq_mhz, + self.mod_code, + warp.target.unwrap_or_default() + ); + } let periodic = wave == "SINE" || wave == "SQUARE"; if wave != "OFF" && !saw_level { return err("SYNTAX", "level_required"); @@ -777,6 +842,41 @@ mod tests { .contains("mod_wave=OFF mod_level=0 mod_min=0 mod_freq_mhz=0 code=0")); } + #[test] + fn mod_warp_validates_optical_parameters_and_reports_the_lobe_range() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + request(&mut controller, "@1 MOD wave=WARP freq_mhz=10000"); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=target_required")); + + // Operating point is required. + request( + &mut controller, + "@2 MOD wave=WARP freq_mhz=10000 target=LOG_SINE a_milli=800 v_null=200 v_pi=1600", + ); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=u_k_required")); + + // V_null + Vπ overruns the DAC top rail. + request( + &mut controller, + "@3 MOD wave=WARP freq_mhz=10000 target=LOG_SINE a_milli=800 u_k_milli=500 v_null=200 v_pi=4000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=warp_exceeds_range")); + + // A valid log-sine warp holds and reports the reachable code range. + request( + &mut controller, + "@4 MOD wave=WARP freq_mhz=10000 target=LOG_SINE a_milli=800 u_k_milli=500 v_null=200 v_pi=1600", + ); + let ok = last_control_text(&mut host); + assert!( + ok.contains("mod_wave=WARP mod_level=1800 mod_min=200 mod_freq_mhz=10000 code=200 target=LOG_SINE"), + "{ok}" + ); + } + #[test] fn waveform_extension_validates_drive_bounds() { let link = MockLink::new(); @@ -846,6 +946,7 @@ mod tests { dark_volts: 40.0 * 3.3 / 4_095.0, ..Default::default() }, + crate::estimator::ContrastGeometry::Direct, ) .expect("clean synthetic window"); estimate.a diff --git a/stage-a-io/src/pdq.rs b/stage-a-io/src/pdq.rs index 1eb4d1c..164ebf9 100644 --- a/stage-a-io/src/pdq.rs +++ b/stage-a-io/src/pdq.rs @@ -1,71 +1,387 @@ -//! `.pdq` writer: preserves every valid PDA1 frame verbatim on disk and -//! tracks run validity. +//! Streaming `.pdq` persistence, replay, and evidence receipts. //! -//! Raw ADC waveforms belong in the PDQ file, never in `HostContext` JSON or -//! per-frame plugin output. A CRC error, frame-sequence gap, or nonzero -//! dropped-sample counter invalidates the run — the file is still written -//! (evidence), but the sidecar must record `valid = false`. +//! Writers preserve every clean PDA1 frame verbatim and finalize the file +//! with CRC32, SHA-256, byte/frame counts, and a contiguous device sample +//! range when one exists. Readers incrementally recover the same frames, +//! report corruption/truncated tails, and produce an independently computed +//! summary suitable for replay verification. -use std::fs::File; -use std::io::{BufWriter, Write}; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; use crate::client::StreamIntegrity; -use crate::wire::{Crc32, Frame}; +use crate::sha256::{Sha256, Sha256Digest}; +use crate::wire::{Crc32, Frame, FrameParser, FrameType, ParseEvent}; -pub struct PdqWriter { - path: PathBuf, - file: BufWriter, - frames_written: u64, - bytes_written: u64, - running_crc: Crc32, +const READ_BUFFER_BYTES: usize = 64 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PdqSampleRange { + pub first_sample_index: u64, + pub end_sample_index_exclusive: u64, + pub sample_count: u64, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct PdqSummary { pub path: PathBuf, pub frames_written: u64, + pub sample_frames_written: u64, + pub samples_written: u64, pub bytes_written: u64, - /// CRC32 over the whole file contents, recorded in the sidecar. + /// CRC32 over the complete file contents, retained for compatibility + /// with existing sidecars and quick local checks. pub file_crc32: u32, + /// SHA-256 over the complete file contents for immutable run receipts. + pub file_sha256: Sha256Digest, + /// Present only when every sample frame belongs to one contiguous, + /// constant-rate device-index segment. + pub sample_range: Option, + pub sample_rate_hz: Option, + pub sample_segments: u64, pub integrity: StreamIntegrity, pub valid: bool, } +impl PdqSummary { + pub fn file_sha256_hex(&self) -> String { + self.file_sha256.to_hex() + } +} + +pub struct PdqWriter { + path: PathBuf, + file: BufWriter, + frames_written: u64, + bytes_written: u64, + running_crc: Crc32, + running_sha256: Sha256, + tracker: FrameTracker, +} + impl PdqWriter { - pub fn create(path: impl AsRef) -> std::io::Result { - let path = path.as_ref().to_owned(); + pub fn create(path: impl AsRef) -> io::Result { + Self::create_with(path.as_ref(), false) + } + + /// Creates a new evidence file without replacing an existing run. + pub fn create_new(path: impl AsRef) -> io::Result { + Self::create_with(path.as_ref(), true) + } + + fn create_with(path: &Path, exclusive: bool) -> io::Result { + let path = path.to_owned(); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } + let file = OpenOptions::new() + .write(true) + .create(true) + .create_new(exclusive) + .truncate(!exclusive) + .open(&path)?; Ok(Self { - file: BufWriter::new(File::create(&path)?), + file: BufWriter::new(file), path, frames_written: 0, bytes_written: 0, running_crc: Crc32::default(), + running_sha256: Sha256::default(), + tracker: FrameTracker::default(), }) } - pub fn write_frame(&mut self, frame: &Frame) -> std::io::Result<()> { + pub fn write_frame(&mut self, frame: &Frame) -> io::Result<()> { let bytes = frame.to_bytes(); self.file.write_all(&bytes)?; self.frames_written += 1; self.bytes_written += bytes.len() as u64; self.running_crc.update(&bytes); + self.running_sha256.update(&bytes); + self.tracker.observe(frame); Ok(()) } - /// Flushes and closes the file, returning the summary for the sidecar. - pub fn finish(mut self, integrity: StreamIntegrity) -> std::io::Result { + /// Flushes and closes the file, returning everything needed for a named + /// finalized receipt. Integrity observed by the live transport is merged + /// fail-closed with discontinuities inferable from the written frames. + pub fn finish(mut self, mut integrity: StreamIntegrity) -> io::Result { self.file.flush()?; + integrity.sequence_gaps = integrity + .sequence_gaps + .max(self.tracker.frame_sequence_gaps); + integrity.dropped_samples = integrity + .dropped_samples + .max(self.tracker.dropped_samples_delta()); + let sample_range = self.tracker.contiguous_sample_range(); + let valid = integrity.is_clean() + && self.tracker.malformed_sample_frames == 0 + && self.tracker.sample_segments <= 1; Ok(PdqSummary { file_crc32: self.running_crc.finalize(), + file_sha256: self.running_sha256.finalize(), path: self.path, frames_written: self.frames_written, + sample_frames_written: self.tracker.sample_frames, + samples_written: self.tracker.samples, bytes_written: self.bytes_written, - valid: integrity.is_clean(), + sample_range, + sample_rate_hz: self.tracker.uniform_sample_rate(), + sample_segments: self.tracker.sample_segments, integrity, + valid, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PdqReadEvent { + Frame(Frame), + Corruption { + skipped_bytes: usize, + crc_failures: usize, + }, + TruncatedTail { + bytes: usize, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PdqReadSummary { + pub frames_read: u64, + pub sample_frames_read: u64, + pub samples_read: u64, + pub bytes_read: u64, + pub file_crc32: u32, + pub file_sha256: Sha256Digest, + pub sample_range: Option, + pub sample_rate_hz: Option, + pub sample_segments: u64, + pub malformed_sample_frames: u64, + pub truncated_bytes: u64, + pub integrity: StreamIntegrity, + pub valid: bool, +} + +impl PdqReadSummary { + pub fn file_sha256_hex(&self) -> String { + self.file_sha256.to_hex() + } +} + +/// Incremental PDA1 file reader. `next_event` preserves corruption notices +/// instead of silently skipping them, allowing replay to continue while the +/// final summary remains invalid. +pub struct PdqReader { + reader: R, + parser: FrameParser, + buffer: Vec, + eof: bool, + tail_reported: bool, + frames_read: u64, + bytes_read: u64, + running_crc: Crc32, + running_sha256: Sha256, + tracker: FrameTracker, + integrity: StreamIntegrity, + truncated_bytes: u64, +} + +impl PdqReader { + pub fn open(path: impl AsRef) -> io::Result { + File::open(path).map(Self::new) + } +} + +impl PdqReader { + pub fn new(reader: R) -> Self { + Self { + reader, + parser: FrameParser::default(), + buffer: vec![0; READ_BUFFER_BYTES], + eof: false, + tail_reported: false, + frames_read: 0, + bytes_read: 0, + running_crc: Crc32::default(), + running_sha256: Sha256::default(), + tracker: FrameTracker::default(), + integrity: StreamIntegrity::default(), + truncated_bytes: 0, + } + } + + pub fn next_event(&mut self) -> io::Result> { + loop { + if let Some(event) = self.parser.next_event() { + return Ok(Some(match event { + ParseEvent::Frame(frame) => { + self.frames_read += 1; + self.tracker.observe(&frame); + PdqReadEvent::Frame(frame) + } + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + self.integrity.skipped_bytes += skipped_bytes as u64; + self.integrity.crc_failures += crc_failures as u64; + PdqReadEvent::Corruption { + skipped_bytes, + crc_failures, + } + } + })); + } + + if self.eof { + if !self.tail_reported && self.parser.buffered_len() > 0 { + self.tail_reported = true; + let bytes = self.parser.discard_buffered(); + self.truncated_bytes += bytes as u64; + self.integrity.skipped_bytes += bytes as u64; + return Ok(Some(PdqReadEvent::TruncatedTail { bytes })); + } + return Ok(None); + } + + let read = match self.reader.read(&mut self.buffer) { + Ok(read) => read, + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, + Err(err) => return Err(err), + }; + if read == 0 { + self.eof = true; + continue; + } + let bytes = &self.buffer[..read]; + self.bytes_read += read as u64; + self.running_crc.update(bytes); + self.running_sha256.update(bytes); + self.parser.extend(bytes); + } + } + + /// Drains the remaining file and returns an independently verified + /// summary. This can follow any number of prior `next_event` calls. + pub fn finish(mut self) -> io::Result { + while self.next_event()?.is_some() {} + self.integrity.sequence_gaps = self.tracker.frame_sequence_gaps; + self.integrity.dropped_samples = self.tracker.dropped_samples_delta(); + let sample_range = self.tracker.contiguous_sample_range(); + let valid = self.integrity.is_clean() + && self.truncated_bytes == 0 + && self.tracker.malformed_sample_frames == 0 + && self.tracker.sample_segments <= 1; + Ok(PdqReadSummary { + frames_read: self.frames_read, + sample_frames_read: self.tracker.sample_frames, + samples_read: self.tracker.samples, + bytes_read: self.bytes_read, + file_crc32: self.running_crc.finalize(), + file_sha256: self.running_sha256.finalize(), + sample_range, + sample_rate_hz: self.tracker.uniform_sample_rate(), + sample_segments: self.tracker.sample_segments, + malformed_sample_frames: self.tracker.malformed_sample_frames, + truncated_bytes: self.truncated_bytes, + integrity: self.integrity, + valid, + }) + } +} + +pub fn inspect_pdq(path: impl AsRef) -> io::Result { + PdqReader::open(path)?.finish() +} + +#[derive(Default)] +struct FrameTracker { + previous_frame_sequence: Option, + frame_sequence_gaps: u64, + sample_frames: u64, + samples: u64, + first_sample_index: Option, + last_sample_end: Option, + previous_sample_end: Option, + first_sample_rate_hz: Option, + previous_sample_rate_hz: Option, + sample_rate_changed: bool, + sample_segments: u64, + malformed_sample_frames: u64, + first_dropped_samples: Option, + last_dropped_samples: Option, +} + +impl FrameTracker { + fn observe(&mut self, frame: &Frame) { + if let Some(previous) = self.previous_frame_sequence { + if frame.header.sequence != previous.wrapping_add(1) { + self.frame_sequence_gaps += 1; + } + } + self.previous_frame_sequence = Some(frame.header.sequence); + self.first_dropped_samples + .get_or_insert(frame.header.dropped_samples); + self.last_dropped_samples = Some(frame.header.dropped_samples); + + if frame.header.frame_type != FrameType::SamplesU16 { + return; + } + if !frame.payload.len().is_multiple_of(2) { + self.malformed_sample_frames += 1; + return; + } + let count = (frame.payload.len() / 2) as u64; + if count == 0 { + return; + } + let first = frame.header.first_sample_index; + let end = first.saturating_add(count); + let starts_new_segment = self.previous_sample_end.is_none() + || self.previous_sample_end != Some(first) + || self.previous_sample_rate_hz != Some(frame.header.sample_rate_hz); + if starts_new_segment { + self.sample_segments += 1; + } + if self + .first_sample_rate_hz + .is_some_and(|rate| rate != frame.header.sample_rate_hz) + { + self.sample_rate_changed = true; + } + self.first_sample_rate_hz + .get_or_insert(frame.header.sample_rate_hz); + self.previous_sample_rate_hz = Some(frame.header.sample_rate_hz); + self.first_sample_index.get_or_insert(first); + self.last_sample_end = Some(end); + self.previous_sample_end = Some(end); + self.sample_frames += 1; + self.samples += count; + } + + fn dropped_samples_delta(&self) -> u64 { + match (self.first_dropped_samples, self.last_dropped_samples) { + (Some(first), Some(last)) => u64::from(last.saturating_sub(first)), + _ => 0, + } + } + + fn uniform_sample_rate(&self) -> Option { + (!self.sample_rate_changed) + .then_some(self.first_sample_rate_hz) + .flatten() + } + + fn contiguous_sample_range(&self) -> Option { + if self.sample_segments != 1 || self.malformed_sample_frames > 0 { + return None; + } + Some(PdqSampleRange { + first_sample_index: self.first_sample_index?, + end_sample_index_exclusive: self.last_sample_end?, + sample_count: self.samples, }) } } @@ -73,9 +389,10 @@ impl PdqWriter { #[cfg(test)] mod tests { use super::*; - use crate::wire::{FrameHeader, FrameType, PROTOCOL_VERSION}; + use crate::wire::{FrameHeader, PROTOCOL_VERSION}; + use std::io::Cursor; - fn frame(sequence: u32) -> Frame { + fn control_frame(sequence: u32) -> Frame { Frame::build( FrameHeader { version: PROTOCOL_VERSION, @@ -92,50 +409,139 @@ mod tests { ) } - #[test] - fn writes_frames_verbatim_and_reports_validity() { + fn sample_frame(sequence: u32, first_index: u64, rate_hz: u32, codes: &[u16]) -> Frame { + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::SamplesU16, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: first_index, + sample_rate_hz: rate_hz, + dropped_samples: 0, + crc32: 0, + }, + codes.iter().flat_map(|code| code.to_le_bytes()).collect(), + ) + } + + fn temp_dir(tag: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!( - "stage-a-io-pdq-{}", + "stage-a-io-pdq-{tag}-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .unwrap() + .expect("clock") .as_nanos() )); - let path = dir.join("run.pdq"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + #[test] + fn writer_and_reader_agree_on_digest_size_and_sample_range() { + let dir = temp_dir("receipt"); + let path = dir.join("run.pdq"); + let frames = [ + sample_frame(10, 1_000, 20_000, &[1, 2, 3]), + sample_frame(11, 1_003, 20_000, &[4, 5]), + ]; let mut writer = PdqWriter::create(&path).expect("create pdq"); - let first = frame(1); - let second = frame(2); - writer.write_frame(&first).expect("write"); - writer.write_frame(&second).expect("write"); - let summary = writer + for frame in &frames { + writer.write_frame(frame).expect("write frame"); + } + let written = writer .finish(StreamIntegrity::default()) - .expect("finish pdq"); + .expect("finish writer"); + let read = inspect_pdq(&path).expect("inspect pdq"); - assert!(summary.valid); - assert_eq!(summary.frames_written, 2); - let on_disk = std::fs::read(&path).expect("read back"); - let mut expected = first.to_bytes(); - expected.extend_from_slice(&second.to_bytes()); - assert_eq!(on_disk, expected); - assert_eq!(summary.file_crc32, crate::wire::crc32(&expected)); + assert!(written.valid && read.valid); + assert_eq!(written.frames_written, 2); + assert_eq!(written.sample_frames_written, 2); + assert_eq!(written.samples_written, 5); + assert_eq!(written.bytes_written, read.bytes_read); + assert_eq!(written.file_crc32, read.file_crc32); + assert_eq!(written.file_sha256, read.file_sha256); + assert_eq!(written.file_sha256_hex().len(), 64); + assert_eq!( + written.sample_range, + Some(PdqSampleRange { + first_sample_index: 1_000, + end_sample_index_exclusive: 1_005, + sample_count: 5, + }) + ); + assert_eq!(written.sample_range, read.sample_range); + assert_eq!(written.sample_rate_hz, Some(20_000)); std::fs::remove_dir_all(dir).expect("cleanup"); } #[test] - fn integrity_faults_invalidate_the_run_but_keep_the_file() { - let dir = std::env::temp_dir().join(format!( - "stage-a-io-pdq-invalid-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let path = dir.join("run.pdq"); + fn reader_streams_frames_and_reports_crc_corruption() { + let first = control_frame(1).to_bytes(); + let mut corrupt = control_frame(2).to_bytes(); + let last = corrupt.len() - 1; + corrupt[last] ^= 0x80; + let third = control_frame(3).to_bytes(); + let bytes: Vec = first.into_iter().chain(corrupt).chain(third).collect(); + let mut reader = PdqReader::new(Cursor::new(bytes)); + let mut frames = Vec::new(); + let mut saw_corruption = false; + while let Some(event) = reader.next_event().expect("read event") { + match event { + PdqReadEvent::Frame(frame) => frames.push(frame.header.sequence), + PdqReadEvent::Corruption { crc_failures, .. } => { + saw_corruption |= crc_failures > 0; + } + PdqReadEvent::TruncatedTail { .. } => {} + } + } + assert_eq!(frames, [1, 3]); + assert!(saw_corruption); + let summary = reader.finish().expect("finish after iteration"); + assert!(!summary.valid); + assert_eq!(summary.integrity.crc_failures, 1); + assert_eq!(summary.integrity.sequence_gaps, 1); + } + + #[test] + fn truncated_tail_is_visible_and_invalid() { + let mut bytes = sample_frame(1, 0, 20_000, &[1, 2, 3]).to_bytes(); + bytes.extend_from_slice(b"PDA"); + let mut reader = PdqReader::new(Cursor::new(bytes)); + let mut truncated = 0; + while let Some(event) = reader.next_event().expect("read") { + if let PdqReadEvent::TruncatedTail { bytes } = event { + truncated += bytes; + } + } + assert_eq!(truncated, 3); + let summary = reader.finish().expect("finish"); + assert_eq!(summary.truncated_bytes, 3); + assert!(!summary.valid); + } + #[test] + fn discontinuous_samples_have_no_contiguous_range() { + let first = sample_frame(4, 100, 20_000, &[1, 2]).to_bytes(); + let second = sample_frame(5, 900, 50_000, &[3, 4]).to_bytes(); + let bytes: Vec = first.into_iter().chain(second).collect(); + let summary = PdqReader::new(Cursor::new(bytes)) + .finish() + .expect("inspect"); + assert_eq!(summary.sample_segments, 2); + assert_eq!(summary.sample_range, None); + assert_eq!(summary.sample_rate_hz, None); + assert!(!summary.valid); + } + + #[test] + fn explicit_integrity_faults_invalidate_writer_but_keep_the_file() { + let dir = temp_dir("invalid"); + let path = dir.join("run.pdq"); let mut writer = PdqWriter::create(&path).expect("create pdq"); - writer.write_frame(&frame(1)).expect("write"); + writer.write_frame(&control_frame(1)).expect("write frame"); let summary = writer .finish(StreamIntegrity { dropped_samples: 5, diff --git a/stage-a-io/src/sha256.rs b/stage-a-io/src/sha256.rs new file mode 100644 index 0000000..8447d31 --- /dev/null +++ b/stage-a-io/src/sha256.rs @@ -0,0 +1,259 @@ +//! Small streaming SHA-256 implementation used to finalize PDQ evidence. +//! +//! Keeping this implementation local avoids adding a crypto dependency to +//! the hardware-facing crate. It implements only unkeyed SHA-256 and is +//! tested against the FIPS 180-4 example vectors. + +use std::fmt; + +const INITIAL_STATE: [u32; 8] = [ + 0x6a09_e667, + 0xbb67_ae85, + 0x3c6e_f372, + 0xa54f_f53a, + 0x510e_527f, + 0x9b05_688c, + 0x1f83_d9ab, + 0x5be0_cd19, +]; + +const ROUND_CONSTANTS: [u32; 64] = [ + 0x428a_2f98, + 0x7137_4491, + 0xb5c0_fbcf, + 0xe9b5_dba5, + 0x3956_c25b, + 0x59f1_11f1, + 0x923f_82a4, + 0xab1c_5ed5, + 0xd807_aa98, + 0x1283_5b01, + 0x2431_85be, + 0x550c_7dc3, + 0x72be_5d74, + 0x80de_b1fe, + 0x9bdc_06a7, + 0xc19b_f174, + 0xe49b_69c1, + 0xefbe_4786, + 0x0fc1_9dc6, + 0x240c_a1cc, + 0x2de9_2c6f, + 0x4a74_84aa, + 0x5cb0_a9dc, + 0x76f9_88da, + 0x983e_5152, + 0xa831_c66d, + 0xb003_27c8, + 0xbf59_7fc7, + 0xc6e0_0bf3, + 0xd5a7_9147, + 0x06ca_6351, + 0x1429_2967, + 0x27b7_0a85, + 0x2e1b_2138, + 0x4d2c_6dfc, + 0x5338_0d13, + 0x650a_7354, + 0x766a_0abb, + 0x81c2_c92e, + 0x9272_2c85, + 0xa2bf_e8a1, + 0xa81a_664b, + 0xc24b_8b70, + 0xc76c_51a3, + 0xd192_e819, + 0xd699_0624, + 0xf40e_3585, + 0x106a_a070, + 0x19a4_c116, + 0x1e37_6c08, + 0x2748_774c, + 0x34b0_bcb5, + 0x391c_0cb3, + 0x4ed8_aa4a, + 0x5b9c_ca4f, + 0x682e_6ff3, + 0x748f_82ee, + 0x78a5_636f, + 0x84c8_7814, + 0x8cc7_0208, + 0x90be_fffa, + 0xa450_6ceb, + 0xbef9_a3f7, + 0xc671_78f2, +]; + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct Sha256Digest([u8; 32]); + +impl Sha256Digest { + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub fn to_hex(self) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(64); + for byte in self.0 { + output.push(char::from(HEX[usize::from(byte >> 4)])); + output.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + output + } +} + +impl fmt::Display for Sha256Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_hex()) + } +} + +impl fmt::Debug for Sha256Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Sha256Digest({self})") + } +} + +pub(crate) struct Sha256 { + state: [u32; 8], + buffer: [u8; 64], + buffer_len: usize, + bytes_seen: u64, +} + +impl Default for Sha256 { + fn default() -> Self { + Self { + state: INITIAL_STATE, + buffer: [0; 64], + buffer_len: 0, + bytes_seen: 0, + } + } +} + +impl Sha256 { + pub(crate) fn update(&mut self, mut bytes: &[u8]) { + self.bytes_seen = self.bytes_seen.wrapping_add(bytes.len() as u64); + if self.buffer_len > 0 { + let fill = (64 - self.buffer_len).min(bytes.len()); + self.buffer[self.buffer_len..self.buffer_len + fill].copy_from_slice(&bytes[..fill]); + self.buffer_len += fill; + bytes = &bytes[fill..]; + if self.buffer_len == 64 { + let block = self.buffer; + self.compress(&block); + self.buffer_len = 0; + } else { + return; + } + } + + while bytes.len() >= 64 { + let block: &[u8; 64] = bytes[..64].try_into().expect("exact SHA-256 block"); + self.compress(block); + bytes = &bytes[64..]; + } + self.buffer[..bytes.len()].copy_from_slice(bytes); + self.buffer_len = bytes.len(); + } + + pub(crate) fn finalize(mut self) -> Sha256Digest { + let bit_len = self.bytes_seen.wrapping_mul(8); + self.buffer[self.buffer_len] = 0x80; + self.buffer_len += 1; + if self.buffer_len > 56 { + self.buffer[self.buffer_len..].fill(0); + let block = self.buffer; + self.compress(&block); + self.buffer = [0; 64]; + } else { + self.buffer[self.buffer_len..56].fill(0); + } + self.buffer[56..64].copy_from_slice(&bit_len.to_be_bytes()); + let block = self.buffer; + self.compress(&block); + + let mut digest = [0; 32]; + for (chunk, word) in digest.chunks_exact_mut(4).zip(self.state) { + chunk.copy_from_slice(&word.to_be_bytes()); + } + Sha256Digest(digest) + } + + fn compress(&mut self, block: &[u8; 64]) { + let mut schedule = [0_u32; 64]; + for (word, bytes) in schedule[..16].iter_mut().zip(block.chunks_exact(4)) { + *word = u32::from_be_bytes(bytes.try_into().expect("four-byte word")); + } + for index in 16..64 { + let x = schedule[index - 15]; + let y = schedule[index - 2]; + let sigma0 = x.rotate_right(7) ^ x.rotate_right(18) ^ (x >> 3); + let sigma1 = y.rotate_right(17) ^ y.rotate_right(19) ^ (y >> 10); + schedule[index] = schedule[index - 16] + .wrapping_add(sigma0) + .wrapping_add(schedule[index - 7]) + .wrapping_add(sigma1); + } + + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = self.state; + for (word, constant) in schedule.into_iter().zip(ROUND_CONSTANTS) { + let sum1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let choose = (e & f) ^ (!e & g); + let temp1 = h + .wrapping_add(sum1) + .wrapping_add(choose) + .wrapping_add(constant) + .wrapping_add(word); + let sum0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let majority = (a & b) ^ (a & c) ^ (b & c); + let temp2 = sum0.wrapping_add(majority); + h = g; + g = f; + f = e; + e = d.wrapping_add(temp1); + d = c; + c = b; + b = a; + a = temp1.wrapping_add(temp2); + } + for (state, value) in self.state.iter_mut().zip([a, b, c, d, e, f, g, h]) { + *state = state.wrapping_add(value); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn digest(chunks: &[&[u8]]) -> String { + let mut sha = Sha256::default(); + for chunk in chunks { + sha.update(chunk); + } + sha.finalize().to_hex() + } + + #[test] + fn matches_fips_vectors_and_fragmentation() { + assert_eq!( + digest(&[b""]), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + digest(&[b"a", b"b", b"c"]), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!( + digest(&[b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"]), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + } +} diff --git a/stage-a-io/src/sidecar.rs b/stage-a-io/src/sidecar.rs index 0b591bc..0ebd036 100644 --- a/stage-a-io/src/sidecar.rs +++ b/stage-a-io/src/sidecar.rs @@ -191,8 +191,18 @@ mod tests { let pdq = PdqSummary { path: PathBuf::from("/data/A1-20260713-01.pdq"), frames_written: 128, + sample_frames_written: 128, + samples_written: 32_768, bytes_written: 65_536, file_crc32: 0xDEAD_BEEF, + file_sha256: crate::Sha256Digest::from_bytes([0xAB; 32]), + sample_range: Some(crate::PdqSampleRange { + first_sample_index: 0, + end_sample_index_exclusive: 32_768, + sample_count: 32_768, + }), + sample_rate_hz: Some(20_000), + sample_segments: 1, integrity: StreamIntegrity::default(), valid: true, }; diff --git a/stage-a-io/src/wire.rs b/stage-a-io/src/wire.rs index c42a249..bb90b5a 100644 --- a/stage-a-io/src/wire.rs +++ b/stage-a-io/src/wire.rs @@ -139,7 +139,8 @@ impl Frame { /// Decodes the payload of a `SamplesU16` frame into ADC codes. pub fn samples(&self) -> Option> { - if self.header.frame_type != FrameType::SamplesU16 || self.payload.len() % 2 != 0 { + if self.header.frame_type != FrameType::SamplesU16 || !self.payload.len().is_multiple_of(2) + { return None; } Some( @@ -157,6 +158,21 @@ impl Frame { } std::str::from_utf8(&self.payload).ok() } + + /// Decodes the payload of a `Marker` frame (phase-0 fiducial on the device + /// clock). + pub fn marker(&self) -> Option { + if self.header.frame_type != FrameType::Marker || self.payload.len() != 16 { + return None; + } + let p = &self.payload; + Some(MarkerPayload { + sample_index: u64::from_le_bytes(p[0..8].try_into().ok()?), + tick_us: u32::from_le_bytes(p[8..12].try_into().ok()?), + level: p[12], + source: p[13], + }) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -189,6 +205,32 @@ impl SummaryPayload { } } +/// A `Marker` frame payload: a phase-0 fiducial stamped on the device clock +/// (matches the firmware `MarkerPayload`; `source = 1` is a modulation phase-0). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MarkerPayload { + /// ADC sample index at the fiducial — aligns the marker with the stream. + pub sample_index: u64, + pub tick_us: u32, + pub level: u8, + pub source: u8, +} + +/// `source` value the firmware stamps on a modulation phase-0 marker. +pub const MARKER_SOURCE_PHASE0: u8 = 1; + +impl MarkerPayload { + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(16); + out.extend_from_slice(&self.sample_index.to_le_bytes()); + out.extend_from_slice(&self.tick_us.to_le_bytes()); + out.push(self.level); + out.push(self.source); + out.extend_from_slice(&[0_u8, 0_u8]); // reserved[2] + out + } +} + /// CRC32 (IEEE, reflected, init/final 0xFFFF_FFFF) — identical to the /// firmware's `crc32Update` loop. pub fn crc32(data: &[u8]) -> u32 { @@ -270,6 +312,22 @@ impl FrameParser { self.buffer.extend_from_slice(bytes); } + /// Bytes retained while waiting for a complete header or payload. + /// Primarily useful at a finite-file EOF, where a nonzero value means + /// the PDQ ends with a truncated frame or garbage tail. + pub fn buffered_len(&self) -> usize { + self.buffer.len() + } + + /// Discards and returns the number of bytes still buffered. Live serial + /// readers normally never need this; finite-file readers use it once at + /// EOF to report a truncated tail without exposing parser internals. + pub fn discard_buffered(&mut self) -> usize { + let len = self.buffer.len(); + self.buffer.clear(); + len + } + pub fn next_event(&mut self) -> Option { loop { // Scan to the next plausible magic. @@ -443,6 +501,33 @@ mod tests { assert!((summary.mean_code() - 1953.125).abs() < 1e-9); } + #[test] + fn marker_payload_round_trips() { + let marker = MarkerPayload { + sample_index: 1_234_567, + tick_us: 987_654, + level: 1, + source: MARKER_SOURCE_PHASE0, + }; + let frame = Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Marker, + flags: 0, + sequence: 9, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 20_000, + dropped_samples: 0, + crc32: 0, + }, + marker.encode(), + ); + assert_eq!(frame.marker(), Some(marker)); + // A samples decode must not accept a marker frame. + assert_eq!(frame.samples(), None); + } + #[test] fn samples_frame_decodes_codes() { let codes = [1_u16, 2, 4_095]; diff --git a/stage-a-plugin-contract/Cargo.toml b/stage-a-plugin-contract/Cargo.toml new file mode 100644 index 0000000..9446f85 --- /dev/null +++ b/stage-a-plugin-contract/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "stage-a-plugin-contract" +version = "0.1.0" +edition = "2021" +license = "MIT" +authors = ["Mika Uthmann "] +description = "Serde-only inter-plugin control and status contract for the Stage-A device-owner plugins" + +[dependencies] +serde = { version = "1", features = ["derive"] } + +[dev-dependencies] +serde_json = "1" + +[lints.rust] +unsafe_code = "forbid" + diff --git a/stage-a-plugin-contract/README.md b/stage-a-plugin-contract/README.md new file mode 100644 index 0000000..735434c --- /dev/null +++ b/stage-a-plugin-contract/README.md @@ -0,0 +1,47 @@ +# Stage-A Plugin Contract + +This crate is the serde-only control-plane contract between the Stage-A workflow plugins and the +two plugins that permanently own the Teensy ports: + +- `stage-a-modulation` owns and controls the command port; +- `stage-a-photodiode` owns and reads the PDA1 stream port; +- experiment plugins such as `stage-a-a1` orchestrate those owners without opening either port. + +The crate deliberately has no Augur, serial, filesystem, or thread dependency. Its payloads can be +serialized through the host's persistent plugin context. Every context key and payload is +explicitly versioned. A request carries a unique request ID, the target owner instance, an +optional lease and run ID, and an optional requested semantic revision. Responses echo those +identities and report the ACKed revision. + +## Mailboxes + +| Direction | Context key | +|---|---| +| orchestrator → modulation owner | `stage_a.modulation_request.v1` | +| modulation owner → orchestrator | `stage_a.modulation_response.v1` | +| modulation owner snapshot | `stage_a.modulation_state.v1` | +| orchestrator → photodiode owner | `stage_a.photodiode_request.v1` | +| photodiode owner → orchestrator | `stage_a.photodiode_response.v1` | +| photodiode owner snapshot | `stage_a.photodiode_summary.v1` | + +Persistent context is a last-writer-wins mailbox, not a queue. An orchestrator must keep at most +one outstanding request per owner, retain it until its request ID is acknowledged, and never +reuse a request ID. Owners must make duplicate delivery idempotent by returning the original +result without repeating the effect. + +## Safety and data boundaries + +Control commands are semantic (`PrepareA1`, `SafeOff`, `BeginRecording`, and so on), not raw +firmware strings or remote setting changes. Automated mutations require an owner-issued lease; +leases expire unless renewed. Owner snapshots carry an instance ID and freshness deadline so an +orchestrator can detect reloads and stale state. + +Photodiode messages contain only bounded summaries and named PDQ receipts. Raw ADC arrays never +cross the JSON context. The finalized receipt names the PDQ/sidecar, SHA-256, byte and frame +counts, contiguous sample range, stream integrity, and validity. Analysis reads the finalized PDQ +through `stage-a-io`. + +`SynchronizationV1::Unsynced` is a first-class state. Missing firmware configuration revisions, +owner restarts, stream-epoch changes, stale snapshots, or run-ID mismatches must be reported as +UNSYNCED rather than inferred away. + diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs new file mode 100644 index 0000000..e7e1638 --- /dev/null +++ b/stage-a-plugin-contract/src/lib.rs @@ -0,0 +1,853 @@ +//! Versioned, serde-only messages shared by Stage-A experiment workflows and +//! the two persistent Teensy device-owner plugins. +//! +//! This crate contains semantic control-plane types only. It intentionally +//! contains no Augur ABI types, serial transports, filesystem access, raw ADC +//! arrays, or experiment state machines. + +#![forbid(unsafe_code)] + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt; + +pub const CONTRACT_VERSION_V1: u16 = 1; + +pub const PLUGIN_ID_STAGE_A_MODULATION: &str = "stage-a.modulation"; +pub const PLUGIN_ID_STAGE_A_PHOTODIODE: &str = "stage-a.photodiode"; +pub const SERVICE_STAGE_A_MODULATION_CONTROL_V1: &str = "stage_a.modulation.control.v1"; +pub const SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1: &str = "stage_a.photodiode.control.v1"; + +pub const CTX_STAGE_A_MODULATION_REQUEST_V1: &str = "stage_a.modulation_request.v1"; +pub const CTX_STAGE_A_MODULATION_RESPONSE_V1: &str = "stage_a.modulation_response.v1"; +pub const CTX_STAGE_A_MODULATION_STATE_V1: &str = "stage_a.modulation_state.v1"; +pub const CTX_STAGE_A_PHOTODIODE_REQUEST_V1: &str = "stage_a.photodiode_request.v1"; +pub const CTX_STAGE_A_PHOTODIODE_RESPONSE_V1: &str = "stage_a.photodiode_response.v1"; +pub const CTX_STAGE_A_PHOTODIODE_SUMMARY_V1: &str = "stage_a.photodiode_summary.v1"; + +macro_rules! string_id { + ($name:ident) => { + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] + #[serde(transparent)] + pub struct $name(pub String); + + impl $name { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl From<&str> for $name { + fn from(value: &str) -> Self { + Self(value.to_owned()) + } + } + + impl From for $name { + fn from(value: String) -> Self { + Self(value) + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } + } + }; +} + +string_id!(ClientId); +string_id!(LeaseId); +string_id!(OwnerInstanceId); +string_id!(RunId); + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default, +)] +#[serde(transparent)] +pub struct RequestId(pub u64); + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default, +)] +#[serde(transparent)] +pub struct SemanticRevision(pub u64); + +/// Wall-clock freshness information transferable between dynamic plugins. +/// The consumer determines staleness against its current Unix time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct FreshnessV1 { + pub observed_at_unix_ms: u64, + pub valid_for_ms: u64, +} + +impl FreshnessV1 { + pub fn is_stale_at(self, now_unix_ms: u64) -> bool { + now_unix_ms.saturating_sub(self.observed_at_unix_ms) > self.valid_for_ms + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum ConnectionStateV1 { + Disconnected, + Connecting, + Connected { + port_label: String, + firmware_version: Option, + }, + Faulted { + message: String, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseSnapshotV1 { + pub lease_id: LeaseId, + pub holder: ClientId, + pub expires_at_unix_ms: u64, + pub run_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UnsyncedReasonV1 { + NoOwnerSnapshot, + OwnerRestarted, + StaleSnapshot, + NoLease, + LeaseMismatch, + RunMismatch, + RequestedRevisionNotAcknowledged, + FirmwareRevisionUnavailable, + StreamEpochChanged, + DeviceFault, + Other(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum SynchronizationV1 { + Synced { + run_id: RunId, + acknowledged_revision: SemanticRevision, + stream_epoch: Option, + }, + Unsynced { + reason: UnsyncedReasonV1, + detail: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ServiceErrorCodeV1 { + ContractVersion, + WrongOwnerInstance, + StaleRequest, + DuplicateRequestConflict, + NotConnected, + LeaseRequired, + LeaseBusy, + LeaseMismatch, + LeaseExpired, + UnsafeExecutionContext, + InvalidCommand, + InvalidPath, + DeviceRejected, + Transport, + Io, + Integrity, + Internal, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ServiceErrorV1 { + pub code: ServiceErrorCodeV1, + pub message: String, + pub retryable: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RequestOutcomeV1 { + InProgress, + Applied, + Rejected, +} + +/// Common request envelope. The command-specific aliases below are the +/// public mailbox payloads. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RequestEnvelopeV1 { + pub contract_version: u16, + pub request_id: RequestId, + pub requester: ClientId, + /// `None` is allowed only for discovery/connect or first lease acquire. + pub target_owner_instance: Option, + pub lease_id: Option, + pub run_id: Option, + pub requested_revision: Option, + pub issued_at_unix_ms: u64, + pub command: C, +} + +impl RequestEnvelopeV1 { + pub fn new(request_id: RequestId, requester: ClientId, command: C) -> Self { + Self { + contract_version: CONTRACT_VERSION_V1, + request_id, + requester, + target_owner_instance: None, + lease_id: None, + run_id: None, + requested_revision: None, + issued_at_unix_ms: 0, + command, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseCommonV1 { + pub contract_version: u16, + pub request_id: RequestId, + pub owner_instance: OwnerInstanceId, + pub run_id: Option, + pub requested_revision: Option, + pub acknowledged_revision: Option, + pub outcome: RequestOutcomeV1, + pub completed_at_unix_ms: Option, + pub error: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PeriodicWaveformV1 { + Sine, + Square, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum WaveformV1 { + Off, + Constant { + level_dac: u16, + }, + Periodic { + waveform: PeriodicWaveformV1, + min_dac: u16, + max_dac: u16, + frequency_millihz: u64, + }, +} + +/// Complete semantic configuration for one A1 controller acquisition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct A1AcquisitionConfigV1 { + pub waveform: PeriodicWaveformV1, + pub frequency_millihz: u64, + pub center_dac: u16, + pub amplitude_dac: u16, + pub sample_rate_hz: u32, + pub block_samples: u32, + pub emit_raw_samples: bool, + pub emit_summary: bool, + pub optical_lut_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ModulationCommandV1 { + Connect, + Disconnect { + safe_off: bool, + reason: String, + }, + AcquireLease { + ttl_ms: u64, + }, + RenewLease { + ttl_ms: u64, + }, + ReleaseLease { + safe_off: bool, + reason: String, + }, + SetWaveform { + waveform: WaveformV1, + }, + /// Retarget the owner's *calibrated optical drive* to a new modulation + /// depth `a` (log contrast, in milli-units) without changing anything else + /// about the armed drive: waveform shape, frequency, operating point and + /// calibration stay whatever the operator armed in the modulation plugin. + /// This is the scoped amplitude-sweep path (A1 automation): the owner + /// rejects the command when its current drive cannot express `a` + /// (manual DAC method or constant mode) or the device link is closed. + SetOpticalDepth { + depth_a_milli: u32, + }, + PrepareA1 { + configuration: A1AcquisitionConfigV1, + }, + StartAcquisition, + StopAcquisition { + reason: String, + }, + SafeOff { + reason: String, + }, +} + +pub type ModulationRequestV1 = RequestEnvelopeV1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ControllerStateV1 { + Unknown, + SafeIdle, + Configured, + Running, + Faulted, +} + +/// The full desired or board-acknowledged command-port state at one semantic +/// revision. Owners never infer an ACK from the requested state. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModulationTargetV1 { + pub revision: SemanticRevision, + pub waveform: Option, + pub a1_configuration: Option, + pub acquisition_running: bool, + pub board_dac_code: Option, + pub firmware_configuration_revision: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ModulationResponseV1 { + #[serde(flatten)] + pub common: ResponseCommonV1, + pub controller_state: ControllerStateV1, + pub acknowledged_target: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ModulationStateV1 { + pub contract_version: u16, + pub owner_instance: OwnerInstanceId, + pub service_revision: u64, + pub connection: ConnectionStateV1, + pub capabilities: Vec, + pub lease: Option, + pub controller_state: ControllerStateV1, + pub active_run_id: Option, + pub requested: Option, + pub acknowledged: Option, + pub synchronization: SynchronizationV1, + pub last_response: Option, + pub freshness: FreshnessV1, + /// Identifier of the measured Pockels transfer calibration currently + /// applied to `V_null`/`Vπ`, so a consumer's sidecar can cite which + /// inversion produced a run's optical depth. `None` means the operator + /// entered the lobe parameters by hand. Additive in V1. + #[serde(default)] + pub calibration_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct StreamIntegrityV1 { + pub skipped_bytes: u64, + pub crc_failures: u64, + pub sequence_gaps: u64, + pub dropped_samples: u64, + pub segment_restarts: u64, + pub truncated_bytes: u64, +} + +impl StreamIntegrityV1 { + pub fn is_clean(self) -> bool { + self == Self::default() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct SampleRangeV1 { + pub first_sample_index: u64, + pub end_sample_index_exclusive: u64, + pub sample_count: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct Sha256V1(String); + +impl Sha256V1 { + pub fn parse(value: impl Into) -> Result { + let value = value.into(); + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("SHA-256 must be exactly 64 hexadecimal characters".into()); + } + Ok(Self(value.to_ascii_lowercase())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for Sha256V1 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for Sha256V1 { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::parse(value).map_err(serde::de::Error::custom) + } +} + +/// The exact file to open at a recording boundary. Metadata is deliberately +/// string-valued and bounded by the owner; scientific sidecars remain the +/// canonical rich metadata record. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PdqStartSpecV1 { + pub pdq_path: String, + pub sidecar_path: String, + pub expected_sample_rate_hz: Option, + pub expected_stream_epoch: Option, + pub metadata: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PdqStartedReceiptV1 { + pub run_id: RunId, + pub pdq_path: String, + pub sidecar_path: String, + pub opened_at_unix_ms: u64, + pub stream_epoch: u64, + pub first_sample_index: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PdqTerminationV1 { + Completed, + OperatorStopped, + LeaseExpired, + SafeOff, + DeviceFault, + IoFault, + Aborted, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PdqFinalizedReceiptV1 { + pub run_id: RunId, + pub pdq_path: String, + pub sidecar_path: String, + pub opened_at_unix_ms: u64, + pub finalized_at_unix_ms: u64, + pub file_size_bytes: u64, + pub sha256: Sha256V1, + pub frames_written: u64, + pub sample_frames_written: u64, + pub sample_range: Option, + pub sample_rate_hz: Option, + pub segment_count: u64, + pub integrity: StreamIntegrityV1, + pub termination: PdqTerminationV1, + pub valid: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PdqReceiptV1 { + Started(PdqStartedReceiptV1), + Finalized(PdqFinalizedReceiptV1), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PhotodiodeCommandV1 { + Connect, + Disconnect { + finalize_recording: bool, + reason: String, + }, + AcquireLease { + ttl_ms: u64, + }, + RenewLease { + ttl_ms: u64, + }, + ReleaseLease { + finalize_recording: bool, + reason: String, + }, + BeginRecording { + specification: PdqStartSpecV1, + }, + FinalizeRecording { + termination: PdqTerminationV1, + }, + AbortRecording { + reason: String, + }, +} + +pub type PhotodiodeRequestV1 = RequestEnvelopeV1; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeResponseV1 { + #[serde(flatten)] + pub common: ResponseCommonV1, + pub receipt: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeCalibrationV1 { + pub adc_calibration_id: String, + pub dark_id: String, + pub anchor_id: String, + pub dark_volts: f64, + /// Named full-extinction anchor after dark subtraction. + pub total_power_volts: f64, +} + +/// Bounded optical result for one named run. It contains no raw or decimated +/// waveform samples; the finalized PDQ remains the source for replay. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeOpticalSummaryV1 { + pub run_id: RunId, + pub calibration: PhotodiodeCalibrationV1, + pub measured_log_contrast: f64, + pub log_contrast_stddev: Option, + pub excitation_min_volts: f64, + pub excitation_max_volts: f64, + pub excitation_headroom_volts: f64, + pub low_clip_fraction: f64, + pub high_clip_fraction: f64, + pub measured_frequency_hz: Option, + pub fundamental_phase_rad: Option, + pub total_harmonic_distortion: Option, +} + +/// Settled detector level over the newest averaging window, in **raw detector +/// volts**: the ADC affine map only, before dark subtraction and before any +/// [`PhotodiodeOpticalSummaryV1`] geometry transform. Unlike the optical +/// summary this never refuses — it stays present while the window clips (see +/// `clipped`), because a consumer sweeping a static transfer curve needs a +/// level exactly where the detector is brightest. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeLevelV1 { + pub mean_volts: f64, + /// Spread over the averaged window. A settled `CONST` point has a small + /// peak-to-peak; a drifting or still-slewing one does not. + pub peak_to_peak_volts: f64, + pub sample_count: u64, + /// Exclusive end of the averaged window on the device sample clock. The + /// window covers `[end_sample_index - sample_count, end_sample_index)`, so + /// a consumer can prove a level was measured *after* it commanded a + /// change without needing a shared wall clock. + pub end_sample_index: u64, + /// The window touches an ADC rail; `mean_volts` is a truncated estimate. + pub clipped: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeStreamV1 { + pub stream_epoch: u64, + pub sample_range: Option, + pub sample_rate_hz: Option, + pub latest_adc_code: Option, + pub integrity: StreamIntegrityV1, + /// Additive in V1: absent from older owners, and older consumers ignore it. + #[serde(default)] + pub level: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeSummaryV1 { + pub contract_version: u16, + pub owner_instance: OwnerInstanceId, + pub service_revision: u64, + pub connection: ConnectionStateV1, + pub lease: Option, + pub active_run_id: Option, + pub requested_revision: Option, + pub acknowledged_revision: Option, + pub stream: PhotodiodeStreamV1, + pub active_recording: Option, + pub last_finalized_recording: Option, + pub optical_summary: Option, + pub synchronization: SynchronizationV1, + pub last_response: Option, + pub freshness: FreshnessV1, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn common(request_id: u64) -> ResponseCommonV1 { + ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: RequestId(request_id), + owner_instance: OwnerInstanceId::from("owner-7"), + run_id: Some(RunId::from("A1-20260721-003")), + requested_revision: Some(SemanticRevision(4)), + acknowledged_revision: Some(SemanticRevision(4)), + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(1_721_000_001_000), + error: None, + } + } + + #[test] + fn context_keys_are_stable_and_versioned() { + assert_eq!(PLUGIN_ID_STAGE_A_MODULATION, "stage-a.modulation"); + assert_eq!(PLUGIN_ID_STAGE_A_PHOTODIODE, "stage-a.photodiode"); + assert_eq!( + SERVICE_STAGE_A_MODULATION_CONTROL_V1, + "stage_a.modulation.control.v1" + ); + assert_eq!( + SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, + "stage_a.photodiode.control.v1" + ); + assert_eq!( + CTX_STAGE_A_MODULATION_REQUEST_V1, + "stage_a.modulation_request.v1" + ); + assert_eq!( + CTX_STAGE_A_MODULATION_RESPONSE_V1, + "stage_a.modulation_response.v1" + ); + assert_eq!( + CTX_STAGE_A_MODULATION_STATE_V1, + "stage_a.modulation_state.v1" + ); + assert_eq!( + CTX_STAGE_A_PHOTODIODE_REQUEST_V1, + "stage_a.photodiode_request.v1" + ); + assert_eq!( + CTX_STAGE_A_PHOTODIODE_RESPONSE_V1, + "stage_a.photodiode_response.v1" + ); + assert_eq!( + CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, + "stage_a.photodiode_summary.v1" + ); + } + + #[test] + fn modulation_request_round_trips_with_semantic_discriminants() { + let mut request = ModulationRequestV1::new( + RequestId(12), + ClientId::from("stage-a-a1"), + ModulationCommandV1::PrepareA1 { + configuration: A1AcquisitionConfigV1 { + waveform: PeriodicWaveformV1::Sine, + frequency_millihz: 10_000, + center_dac: 2_048, + amplitude_dac: 512, + sample_rate_hz: 20_000, + block_samples: 256, + emit_raw_samples: true, + emit_summary: true, + optical_lut_id: Some("lut-2026-07".into()), + }, + }, + ); + request.target_owner_instance = Some(OwnerInstanceId::from("mod-owner-1")); + request.lease_id = Some(LeaseId::from("lease-a1")); + request.run_id = Some(RunId::from("run-3")); + request.requested_revision = Some(SemanticRevision(9)); + request.issued_at_unix_ms = 42; + + let json = serde_json::to_value(&request).expect("serializes"); + assert_eq!(json["command"]["kind"], "prepare_a1"); + assert_eq!( + json["command"]["configuration"]["frequency_millihz"], + 10_000 + ); + let decoded: ModulationRequestV1 = serde_json::from_value(json).expect("deserializes"); + assert_eq!(decoded, request); + } + + #[test] + fn set_optical_depth_round_trips_in_milli_units() { + let request = ModulationRequestV1::new( + RequestId(7), + ClientId::from("stage-a-a1"), + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_250, + }, + ); + let json = serde_json::to_value(&request).expect("serializes"); + assert_eq!(json["command"]["kind"], "set_optical_depth"); + assert_eq!(json["command"]["depth_a_milli"], 1_250); + let decoded: ModulationRequestV1 = serde_json::from_value(json).expect("deserializes"); + assert_eq!(decoded, request); + } + + #[test] + fn snapshots_keep_requested_and_acknowledged_revisions_distinct() { + let requested = ModulationTargetV1 { + revision: SemanticRevision(5), + waveform: Some(WaveformV1::Constant { level_dac: 900 }), + a1_configuration: None, + acquisition_running: false, + board_dac_code: None, + firmware_configuration_revision: None, + }; + let acknowledged = ModulationTargetV1 { + revision: SemanticRevision(4), + waveform: Some(WaveformV1::Constant { level_dac: 800 }), + board_dac_code: Some(800), + ..requested.clone() + }; + let snapshot = ModulationStateV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::from("mod-owner-1"), + service_revision: 17, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: Some("0.4.0".into()), + }, + capabilities: vec!["MOD".into(), "PDSTREAM".into()], + lease: None, + controller_state: ControllerStateV1::SafeIdle, + active_run_id: None, + requested: Some(requested), + acknowledged: Some(acknowledged), + synchronization: SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::RequestedRevisionNotAcknowledged, + detail: Some("requested 5, acknowledged 4".into()), + }, + last_response: None, + freshness: FreshnessV1 { + observed_at_unix_ms: 100, + valid_for_ms: 500, + }, + calibration_id: Some("pockels-20260724-120000".into()), + }; + let encoded = serde_json::to_vec(&snapshot).expect("serializes"); + let decoded: ModulationStateV1 = serde_json::from_slice(&encoded).expect("deserializes"); + assert_eq!(decoded.requested.unwrap().revision, SemanticRevision(5)); + assert_eq!(decoded.acknowledged.unwrap().revision, SemanticRevision(4)); + assert!(matches!( + decoded.synchronization, + SynchronizationV1::Unsynced { .. } + )); + } + + #[test] + fn finalized_pdq_receipt_round_trips_without_raw_samples() { + let receipt = PdqFinalizedReceiptV1 { + run_id: RunId::from("run-3"), + pdq_path: "/data/run-3_pd.pdq".into(), + sidecar_path: "/data/run-3.toml".into(), + opened_at_unix_ms: 1_000, + finalized_at_unix_ms: 2_000, + file_size_bytes: 8_192, + sha256: Sha256V1::parse("ab".repeat(32)).expect("digest"), + frames_written: 32, + sample_frames_written: 30, + sample_range: Some(SampleRangeV1 { + first_sample_index: 10_000, + end_sample_index_exclusive: 17_680, + sample_count: 7_680, + }), + sample_rate_hz: Some(20_000), + segment_count: 1, + integrity: StreamIntegrityV1::default(), + termination: PdqTerminationV1::Completed, + valid: true, + }; + let response = PhotodiodeResponseV1 { + common: common(22), + receipt: Some(PdqReceiptV1::Finalized(receipt.clone())), + }; + let json = serde_json::to_value(&response).expect("serializes"); + assert_eq!(json["receipt"]["kind"], "finalized"); + assert!(json.to_string().len() < 2_048, "receipt stays bounded"); + let decoded: PhotodiodeResponseV1 = serde_json::from_value(json).expect("deserializes"); + assert_eq!(decoded.receipt, Some(PdqReceiptV1::Finalized(receipt))); + } + + #[test] + fn sha256_and_freshness_validate_boundaries() { + assert!(Sha256V1::parse("0".repeat(64)).is_ok()); + assert!( + Sha256V1::parse("A".repeat(64)).is_ok_and(|digest| digest.as_str() == "a".repeat(64)) + ); + assert!(Sha256V1::parse("0".repeat(63)).is_err()); + assert!(Sha256V1::parse("z".repeat(64)).is_err()); + assert!(serde_json::from_str::(&format!("\"{}\"", "z".repeat(64))).is_err()); + + let freshness = FreshnessV1 { + observed_at_unix_ms: 1_000, + valid_for_ms: 500, + }; + assert!(!freshness.is_stale_at(1_500)); + assert!(freshness.is_stale_at(1_501)); + assert!(!freshness.is_stale_at(900), "clock rollback saturates"); + } + + #[test] + fn stream_integrity_is_fail_closed() { + assert!(StreamIntegrityV1::default().is_clean()); + assert!(!StreamIntegrityV1 { + segment_restarts: 1, + ..StreamIntegrityV1::default() + } + .is_clean()); + } + + #[test] + fn additive_v1_fields_decode_from_payloads_that_predate_them() { + // An older owner's stream block carries no `level`, and an older + // modulation state no `calibration_id`. Both must still decode. + let stream: PhotodiodeStreamV1 = serde_json::from_value(json!({ + "stream_epoch": 3, + "sample_range": null, + "sample_rate_hz": 20_000, + "latest_adc_code": 1_024, + "integrity": StreamIntegrityV1::default(), + })) + .expect("stream without level decodes"); + assert!(stream.level.is_none()); + + let level = PhotodiodeLevelV1 { + mean_volts: 1.5, + peak_to_peak_volts: 0.01, + sample_count: 4_096, + end_sample_index: 1_000_000, + clipped: false, + }; + let round_tripped: PhotodiodeLevelV1 = + serde_json::from_value(serde_json::to_value(level).expect("serializes")) + .expect("deserializes"); + assert_eq!(round_tripped, level); + // The window is identified without a wall clock: it ends at + // `end_sample_index` and spans `sample_count` samples. + assert_eq!( + level.end_sample_index - level.sample_count, + 1_000_000 - 4_096 + ); + } +} From c0e091a1010ac24a8e251a6adc2d1a6a8dc8903e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:02:05 +0200 Subject: [PATCH 22/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20publish=20?= =?UTF-8?q?the=20excitation=20contrast=20independently=20of=20the=20displa?= =?UTF-8?q?y=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The photodiode sits behind the PBS reject port and measures the complement I_pd = I_tot - I_exc — a property of the bench, not of what the operator chose to plot. `optical_summary` picked its geometry from the display mode, so leaving the chart on RAW published the raw detector contrast as `measured_log_contrast`. A1's amplitude sweep settles on that value against a target `a`: it would never settle, time out at every point, and write a wrong `measured_a` into each sweep sidecar. The geometry is now always the rejected complement; the display mode is presentational. A withheld `a` now reports which gate rejected the window instead of silently showing nothing. Also in the photodiode plugin: - dark level is a measured setting with a capture action, applied to both the detector samples and the I_tot anchor so it cancels out of the complement instead of biasing it; `dark_id` names it honestly - phase-0 marker frames are written into the .pdq, so a recorded run stays phase-attributable offline - `save_cache_snapshot` copies the ring and releases the lock before writing the CSV, instead of blocking the reader across millions of writes - the UI mirror keeps the operator's connect intent rather than clearing it every control tick - a 0-byte read backs off instead of spinning a core - the spectrum max-hold seeds each bucket with its own first bin --- plugins/stage-a-photodiode/src/lib.rs | 603 +++++++++++++++++++++----- 1 file changed, 503 insertions(+), 100 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 19313c7..1e845eb 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -41,8 +41,8 @@ use augur_plugin_api::{ }; use serde_json::{json, Value}; use stage_a_io::{ - estimate_contrast, AdcCalibration, ContrastGeometry, FrameParser, ParseEvent, PdqWriter, - StreamIntegrity, + estimate_contrast, AdcCalibration, ContrastGeometry, EstimateError, FrameParser, ParseEvent, + PdqWriter, StreamIntegrity, }; use stage_a_plugin_contract::{ ClientId, ConnectionStateV1, FreshnessV1, LeaseId, LeaseSnapshotV1, OwnerInstanceId, @@ -598,7 +598,14 @@ fn read_frames( let mut buf = [0_u8; 4_096]; while !stop.load(Ordering::Relaxed) { let read = match port.read(&mut buf) { - Ok(0) => continue, + // A 0-byte read is EOF (e.g. a yanked USB device before the OS + // surfaces an error). Spinning here burns a core while the UI + // still says "reading", so back off and let the timeout path + // report the stall. + Ok(0) => { + std::thread::sleep(Duration::from_millis(5)); + continue; + } Ok(read) => read, Err(err) if err.kind() == std::io::ErrorKind::TimedOut => continue, Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, @@ -613,40 +620,7 @@ fn read_frames( parser.extend(&buf[..read]); let mut changed = false; while let Some(event) = parser.next_event() { - match event { - ParseEvent::Frame(frame) => { - if let Some(marker) = frame.marker() { - if let Ok(mut state) = shared.lock() { - state.push_marker(marker.sample_index); - } - changed = true; - continue; - } - let Some(codes) = frame.samples() else { - continue; // Control/summary frames are not expected here. - }; - record_frame(recording, &frame, codes.len()); - if let Ok(mut state) = shared.lock() { - state.ingest( - frame.header.first_sample_index, - frame.header.sample_rate_hz, - frame.header.dropped_samples, - &codes, - ); - } - changed = true; - } - ParseEvent::Corruption { - skipped_bytes, - crc_failures, - } => { - if let Ok(mut state) = shared.lock() { - state.resync_bytes += skipped_bytes as u64; - state.crc_failures += crc_failures as u64; - } - changed = true; - } - } + changed |= ingest_parse_event(event, shared, recording); } if changed { generation.fetch_add(1, Ordering::Relaxed); @@ -654,6 +628,54 @@ fn read_frames( } } +/// Applies one parsed stream event to the ring and to any active recording. +/// Split out of [`read_frames`] so the recording/ingest contract is testable +/// without a serial port. Returns whether anything observable changed. +fn ingest_parse_event( + event: ParseEvent, + shared: &Mutex, + recording: &SharedRecording, +) -> bool { + match event { + ParseEvent::Frame(frame) => { + if let Some(marker) = frame.marker() { + // Record before the early return: the phase-0 marker is what + // makes a recorded run phase-attributable offline, so it has + // to reach the .pdq as well as the live ring. It carries no + // samples, hence a sample count of 0. + record_frame(recording, &frame, 0); + if let Ok(mut state) = shared.lock() { + state.push_marker(marker.sample_index); + } + return true; + } + let Some(codes) = frame.samples() else { + return false; // Control/summary frames are not expected here. + }; + record_frame(recording, &frame, codes.len()); + if let Ok(mut state) = shared.lock() { + state.ingest( + frame.header.first_sample_index, + frame.header.sample_rate_hz, + frame.header.dropped_samples, + &codes, + ); + } + true + } + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + if let Ok(mut state) = shared.lock() { + state.resync_bytes += skipped_bytes as u64; + state.crc_failures += crc_failures as u64; + } + true + } + } +} + pub struct StageAPhotodiodePlugin { enabled: bool, runtime_role: PluginRuntimeRole, @@ -677,6 +699,12 @@ pub struct StageAPhotodiodePlugin { port_hint: String, mode: Mode, reference_volts: f64, + /// Measured dark level in photodiode volts (beam blocked). Applied to both + /// the detector samples and the `reference_volts` anchor, so it cancels out + /// of the rejected-complement contrast rather than biasing it — its job is + /// to keep the two sides consistent and to record the calibration that the + /// reading was taken under. Captured via the "Capture dark" action. + dark_volts: f64, window_s: f64, avg_samples: usize, avg_sync_freq_hz: f64, @@ -688,6 +716,7 @@ pub struct StageAPhotodiodePlugin { press_save_snapshot: PressLatch, press_record_start: PressLatch, press_record_stop: PressLatch, + press_capture_dark: PressLatch, } /// Forwards momentary button presses across the host's UI-mirror → live-worker @@ -771,6 +800,7 @@ impl Default for StageAPhotodiodePlugin { port_hint: "auto".into(), mode: Mode::Raw, reference_volts: 3.3, + dark_volts: 0.0, window_s: 10.0, avg_samples: 4, avg_sync_freq_hz: 0.0, @@ -780,6 +810,7 @@ impl Default for StageAPhotodiodePlugin { press_save_snapshot: PressLatch::default(), press_record_start: PressLatch::default(), press_record_stop: PressLatch::default(), + press_capture_dark: PressLatch::default(), } } } @@ -789,6 +820,44 @@ impl StageAPhotodiodePlugin { self.reader.is_some() } + /// The ADC calibration handed to the contrast estimator, including the + /// measured dark level. + fn adc_calibration(&self) -> AdcCalibration { + AdcCalibration { + volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, + offset_volts: 0.0, + dark_volts: self.dark_volts, + full_scale_code: ADC_MAX_CODE as u16, + } + } + + /// Captures the dark level as the mean of the current ring: the operator + /// blocks the beam, presses the button, and every later contrast is + /// dark-corrected against it. + fn capture_dark(&mut self) -> Result<(), String> { + let mean = { + let state = self + .shared + .lock() + .map_err(|_| "photodiode state lock poisoned".to_owned())?; + if state.samples.is_empty() { + return Err("no samples cached yet — connect and stream first".into()); + } + let sum: u64 = state.samples.iter().map(|&code| u64::from(code)).sum(); + code_to_volts(sum as f64 / state.samples.len() as f64) + }; + if mean >= self.reference_volts { + return Err(format!( + "dark level {mean:.4} V is not below the I_tot reference \ + {:.4} V — is the beam actually blocked?", + self.reference_volts + )); + } + self.dark_volts = mean; + self.last_save_note = Some(format!("dark level captured: {mean:.4} V")); + Ok(()) + } + fn connect(&mut self) { if self.reader.is_some() { return; @@ -1444,42 +1513,64 @@ impl StageAPhotodiodePlugin { } } - /// Live optical log-contrast `a` from the trailing ring window. The ADC - /// always measures the rejected diode `I_pd`, so the display mode selects - /// the geometry: RAW reports the raw detector contrast (`Direct`), - /// EXCITATION reports the excitation contrast (`RejectedComplement`) using - /// `reference_volts` as the total-power anchor `I_tot`. `None` when there is - /// no valid window or, in EXCITATION mode, no valid anchor. + /// Live optical log-contrast `a` from the trailing ring window. + /// + /// The detector sits behind the PBS reject port and measures the rejected + /// complement `I_pd = I_tot - I_exc` — that is a property of the optical + /// bench, settled by construction (knowledge base: + /// `setup/optical-path.md`), not of what the operator chose to plot. So the + /// geometry is always [`ContrastGeometry::RejectedComplement`] anchored on + /// `reference_volts`, and `measured_log_contrast` is always the *excitation* + /// contrast `a = ln(I_exc,max / I_exc,min)`. + /// + /// The display [`Mode`] is presentational only. It must never reach this + /// function: A1's amplitude sweep settles on this value against a target + /// `a`, so letting a display toggle change its meaning would silently + /// retarget the sweep and write a wrong `measured_a` into every sidecar. + /// + /// `None` when there is no valid window or no valid total-power anchor. fn optical_summary(&self, samples: &VecDeque) -> Option { + self.optical_summary_result(samples).ok() + } + + /// [`Self::optical_summary`], keeping the rejection reason so the status + /// readout can explain *why* `a` is being withheld instead of silently + /// showing nothing. + fn optical_summary_result( + &self, + samples: &VecDeque, + ) -> Result { let start = samples.len().saturating_sub(CONTRAST_WINDOW_SAMPLES); let window: Vec = samples.iter().skip(start).copied().collect(); - let calibration = AdcCalibration { - volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, - offset_volts: 0.0, - dark_volts: 0.0, - full_scale_code: ADC_MAX_CODE as u16, - }; - let geometry = match self.mode { - Mode::Raw => ContrastGeometry::Direct, - Mode::Excitation => ContrastGeometry::RejectedComplement { - total_power_volts: self.reference_volts, - }, + let calibration = self.adc_calibration(); + // `ContrastGeometry::RejectedComplement` wants the *dark-corrected* + // I_tot, and the estimator dark-corrects the detector samples. The + // reference is a reading from the same DC-coupled detector, so it + // carries the same dark offset and has to be corrected the same way. + // Correcting only one side is what would bias `a`; corrected on both, + // the dark term cancels out of the complement exactly (it is a + // difference of two readings), which is the physically right answer. + let geometry = ContrastGeometry::RejectedComplement { + total_power_volts: self.reference_volts - self.dark_volts, }; - let estimate = estimate_contrast(&window, &calibration, geometry).ok()?; + let estimate = estimate_contrast(&window, &calibration, geometry)?; let run_id = self .lease .as_ref() .and_then(|lease| lease.run_id.clone()) .unwrap_or_else(|| RunId::from("live")); - Some(PhotodiodeOpticalSummaryV1 { + Ok(PhotodiodeOpticalSummaryV1 { run_id, calibration: PhotodiodeCalibrationV1 { adc_calibration_id: "adc-default".into(), - dark_id: "dark-0".into(), - anchor_id: match self.mode { - Mode::Raw => "detector-direct".into(), - Mode::Excitation => "reference-volts".into(), + // Name the dark level honestly: consumers must be able to tell + // a measured dark from the un-measured zero default. + dark_id: if self.dark_volts > 0.0 { + "dark-measured".into() + } else { + "dark-none".into() }, + anchor_id: "reference-volts".into(), dark_volts: calibration.dark_volts, total_power_volts: self.reference_volts, }, @@ -1487,6 +1578,10 @@ impl StageAPhotodiodePlugin { log_contrast_stddev: None, excitation_min_volts: estimate.v_min_volts, excitation_max_volts: estimate.v_max_volts, + // Both geometries are dark-referenced (`reference_volts` is the + // dark-corrected `I_tot`), so the excitation minimum *is* the + // margin above dark. Same number as `excitation_min_volts` by + // construction; kept because the contract publishes both. excitation_headroom_volts: estimate.v_min_volts, low_clip_fraction: estimate.low_clip_fraction, high_clip_fraction: estimate.high_clip_fraction, @@ -1496,10 +1591,11 @@ impl StageAPhotodiodePlugin { }) } - /// Locks the ring and returns the current optical log-contrast summary. - fn latest_optical(&self) -> Option { + /// Locks the ring and returns the current optical log-contrast summary, + /// keeping the rejection reason so the caller can explain a withheld `a`. + fn latest_optical_result(&self) -> Option> { let state = self.shared.lock().ok()?; - self.optical_summary(&state.samples) + (!state.samples.is_empty()).then(|| self.optical_summary_result(&state.samples)) } fn control_summary(&self) -> PhotodiodeSummaryV1 { @@ -1640,7 +1736,12 @@ impl StageAPhotodiodePlugin { if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { self.last_error = Some(error); } - self.connect_requested = false; + // Deliberately keep `connect_requested`: it is the operator's + // *intent*, and this branch is what the UI mirror runs on every + // control tick. Clearing it there resets the checkbox before the + // host can sample it, so the live worker never sees the request. + // `connect()` is already guarded on the role, so the intent alone + // is inert here; the worker acts on it below. self.disconnect(); self.lease = None; return; @@ -1662,23 +1763,45 @@ impl StageAPhotodiodePlugin { let dir = self.resolved_data_dir()?; let slug = timestamp_slug(); let csv_path = dir.join(format!("pd_cache_{slug}.csv")); - let state = self - .shared - .lock() - .map_err(|_| "photodiode state lock poisoned".to_owned())?; - if state.samples.is_empty() || state.rate_hz == 0 { - return Err("no samples cached yet".into()); - } + // Copy the ring out under the lock and release it before touching the + // filesystem: holding it across up to RING_MAX_SAMPLES writeln! calls + // blocks the reader thread, overruns the serial input buffer and shows + // up as dropped samples plus a segment restart in any recording that is + // in flight. + let (samples, rate_hz, ring_first_index, cache_seconds, integrity) = { + let state = self + .shared + .lock() + .map_err(|_| "photodiode state lock poisoned".to_owned())?; + if state.samples.is_empty() || state.rate_hz == 0 { + return Err("no samples cached yet".into()); + } + let samples: Vec = state.samples.iter().copied().collect(); + let integrity = json!({ + "resync_bytes": state.resync_bytes, + "crc_failures": state.crc_failures, + "segment_restarts": state.segments, + "device_dropped_samples": state.device_dropped, + }); + ( + samples, + state.rate_hz, + state.ring_first_index, + state.cache_seconds, + integrity, + ) + }; + std::fs::create_dir_all(&dir) .map_err(|err| format!("creating {} failed: {err}", dir.display()))?; let file = File::create(&csv_path) .map_err(|err| format!("creating {} failed: {err}", csv_path.display()))?; let mut writer = BufWriter::new(file); - let rate = f64::from(state.rate_hz); + let rate = f64::from(rate_hz); writeln!(writer, "sample_index,t_s,code,volts") .map_err(|err| format!("writing CSV failed: {err}"))?; - for (offset, &code) in state.samples.iter().enumerate() { - let index = state.ring_first_index + offset as u64; + for (offset, &code) in samples.iter().enumerate() { + let index = ring_first_index + offset as u64; writeln!( writer, "{index},{:.9},{code},{:.6}", @@ -1691,28 +1814,23 @@ impl StageAPhotodiodePlugin { .flush() .map_err(|err| format!("writing CSV failed: {err}"))?; + let sample_count = samples.len(); let sidecar = json!({ "kind": "cache_snapshot", "created_utc": slug, "port": self.port_hint, - "sample_rate_hz": state.rate_hz, - "samples": state.samples.len(), - "first_sample_index": state.ring_first_index, - "cache_seconds": state.cache_seconds, + "sample_rate_hz": rate_hz, + "samples": sample_count, + "first_sample_index": ring_first_index, + "cache_seconds": cache_seconds, "csv_path": csv_path, "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), "reference_volts": self.reference_volts, + "dark_volts": self.dark_volts, "time_base": "t_s = sample_index / sample_rate_hz, device clock, segment-relative", - "integrity": { - "resync_bytes": state.resync_bytes, - "crc_failures": state.crc_failures, - "segment_restarts": state.segments, - "device_dropped_samples": state.device_dropped, - }, + "integrity": integrity, }); - let sample_count = state.samples.len(); - drop(state); write_json(&csv_path.with_extension("json"), &sidecar)?; self.last_save_note = Some(format!( "saved cache {} ({sample_count} samples)", @@ -2006,7 +2124,10 @@ impl StageAPhotodiodePlugin { y: peak, }); peak = 0.0; - peak_freq = freq; + // Seed the *next* bucket with its own first bin. Seeding with + // `freq` (the bin that just closed this bucket) put a flat + // bucket's point one bucket to the left. + peak_freq = (k + 1) as f64 * rate / n as f64; in_bucket = 0; } } @@ -2427,7 +2548,8 @@ impl Plugin for StageAPhotodiodePlugin { if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { self.last_error = Some(error); } - self.connect_requested = false; + // Demoting to the UI mirror drops the hardware, not the operator's + // connect intent — see `apply_execution_context`. self.disconnect(); self.lease = None; self.effects_allowed = false; @@ -2452,7 +2574,8 @@ impl Plugin for StageAPhotodiodePlugin { if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { self.last_error = Some(error); } - self.connect_requested = false; + // Runs every replayed frame, so it must not clear the intent + // either — the port stays closed because `connect()` is guarded. self.disconnect(); self.lease = None; } @@ -2634,6 +2757,32 @@ impl Plugin for StageAPhotodiodePlugin { default: self.reference_volts, }, }, + SettingItem { + key: "dark_volts".into(), + label: "Dark level".into(), + tooltip: Some( + "Measured dark level in photodiode volts (beam blocked). The \ + detector is DC-coupled, so the published contrast a is biased low \ + while this is 0." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: ADC_FULL_SCALE_VOLTS, + speed: 0.001, + default: self.dark_volts, + }, + }, + SettingItem { + key: "capture_dark".into(), + label: "Capture dark".into(), + tooltip: Some( + "Block the beam, then press: takes the mean of the current cache \ + as the dark level." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, SettingItem { key: "window_s".into(), label: "Chart window".into(), @@ -2840,6 +2989,8 @@ impl Plugin for StageAPhotodiodePlugin { // live worker (see PressLatch). "record_start" => Some(self.press_record_start.value()), "record_stop" => Some(self.press_record_stop.value()), + "dark_volts" => Some(json!(self.dark_volts)), + "capture_dark" => Some(self.press_capture_dark.value()), "save_snapshot" => Some(self.press_save_snapshot.value()), _ => None, } @@ -2961,6 +3112,23 @@ impl Plugin for StageAPhotodiodePlugin { } Ok(()) } + "dark_volts" => { + let volts = value.as_f64().ok_or("dark_volts must be a number")?; + self.dark_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); + Ok(()) + } + "capture_dark" => { + // Edge-guarded like every other effectful arm: the host + // re-applies the whole settings snapshot on each sync. + if self.press_capture_dark.accept(&value) { + match self.capture_dark() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } "save_snapshot" => { // Edge-guarded: the host re-applies the full settings snapshot // on every sync, and an unguarded arm wrote one cache file per @@ -3021,17 +3189,28 @@ impl Plugin for StageAPhotodiodePlugin { ))); } } - if let Some(optical) = self.latest_optical() { - let label = match self.mode { - Mode::Raw => "a_raw (detector)", - Mode::Excitation => "a (excitation)", - }; - entries.push(StatusEntry::Text(format!( - "{label} = {:.3} (I {:.4}..{:.4} V)", - optical.measured_log_contrast, - optical.excitation_min_volts, - optical.excitation_max_volts - ))); + match self.latest_optical_result() { + // Always the excitation contrast: the geometry follows the bench, + // not the display mode. + Some(Ok(optical)) => { + entries.push(StatusEntry::Text(format!( + "a (excitation) = {:.3} (I {:.4}..{:.4} V)", + optical.measured_log_contrast, + optical.excitation_min_volts, + optical.excitation_max_volts + ))); + if self.dark_volts <= 0.0 { + entries.push(StatusEntry::Text( + "a is uncorrected for dark — capture a dark level".into(), + )); + } + } + // A withheld `a` is a fail-closed refusal, not an absence of data: + // say which gate rejected the window so the operator can fix it. + Some(Err(error)) => { + entries.push(StatusEntry::Text(format!("a unavailable: {error}"))); + } + None => {} } if let Ok(state) = self.shared.lock() { if let Some(period_samples) = state.marker_period_samples() { @@ -3166,6 +3345,141 @@ mod tests { plugin } + /// A clean rejected-port sine: the detector swings around `center` while + /// the excitation is its complement against `I_tot`. + fn rejected_port_samples(center: f64, amplitude: f64, count: usize) -> VecDeque { + (0..count) + .map(|i| { + let phase = 2.0 * std::f64::consts::PI * (i as f64) * 8.0 / count as f64; + (center + amplitude * phase.sin()).round().clamp(0.0, 4_095.0) as u16 + }) + .collect() + } + + #[test] + fn published_contrast_is_the_excitation_contrast_in_both_display_modes() { + // The detector sits behind the PBS reject port whatever the operator + // is plotting, so a display toggle must not move a published + // scientific quantity. A1's amplitude sweep settles on this value. + let mut plugin = live_plugin(); + plugin.reference_volts = 3.0; + let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + + plugin.mode = Mode::Raw; + let raw = plugin.optical_summary(&samples).expect("raw display"); + plugin.mode = Mode::Excitation; + let excitation = plugin + .optical_summary(&samples) + .expect("excitation display"); + + assert_eq!(raw.measured_log_contrast, excitation.measured_log_contrast); + assert_eq!(raw.calibration.anchor_id, "reference-volts"); + assert_eq!(excitation.calibration.anchor_id, "reference-volts"); + + // And it really is the complement contrast, not ln(v_max/v_min) of the + // detector trace. + let detector_direct = ((1_600.0_f64 + 700.0) / (1_600.0 - 700.0)).ln(); + assert!( + (raw.measured_log_contrast - detector_direct).abs() > 0.1, + "published a={} collapsed to the detector-direct contrast", + raw.measured_log_contrast + ); + } + + #[test] + fn captured_dark_level_reaches_the_estimator_and_is_named() { + let mut plugin = live_plugin(); + plugin.reference_volts = 3.0; + let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + + let undarkened = plugin.optical_summary(&samples).expect("no dark yet"); + assert_eq!(undarkened.calibration.dark_id, "dark-none"); + assert_eq!(undarkened.calibration.dark_volts, 0.0); + + plugin.dark_volts = 0.05; + let darkened = plugin.optical_summary(&samples).expect("with dark"); + assert_eq!(darkened.calibration.dark_id, "dark-measured"); + assert_eq!(darkened.calibration.dark_volts, 0.05); + // A DC dark offset is common to the detector samples and to the + // reference reading, so it cancels out of the complement. Anything + // else means one of the two sides is being corrected without the + // other — which is what would actually bias `a`. + assert!( + (darkened.measured_log_contrast - undarkened.measured_log_contrast).abs() < 1e-9, + "dark did not cancel: {} vs {}", + darkened.measured_log_contrast, + undarkened.measured_log_contrast + ); + } + + #[test] + fn a_dark_offset_on_only_one_side_would_bias_the_contrast() { + // Guards the invariance above against a regression that dark-corrects + // the detector but leaves the anchor raw (or vice versa): that is the + // asymmetry the estimator contract warns about. + let calibration = AdcCalibration { + volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, + offset_volts: 0.0, + dark_volts: 0.05, + full_scale_code: ADC_MAX_CODE as u16, + }; + let samples: Vec = rejected_port_samples(1_600.0, 700.0, 4_096) + .into_iter() + .collect(); + let consistent = estimate_contrast( + &samples, + &calibration, + ContrastGeometry::RejectedComplement { + total_power_volts: 3.0 - 0.05, + }, + ) + .expect("consistent"); + let asymmetric = estimate_contrast( + &samples, + &calibration, + ContrastGeometry::RejectedComplement { + total_power_volts: 3.0, + }, + ) + .expect("anchor left raw"); + assert!( + (consistent.a - asymmetric.a).abs() > 1e-3, + "the asymmetry must be observable, else this test proves nothing" + ); + } + + #[test] + fn capture_dark_refuses_a_level_at_or_above_the_anchor() { + let mut plugin = live_plugin(); + plugin.reference_volts = 0.5; + if let Ok(mut state) = plugin.shared.lock() { + state.ingest(0, 20_000, 0, &[4_000; 256]); + } + let err = plugin.capture_dark().expect_err("beam clearly not blocked"); + assert!(err.contains("is not below the I_tot reference"), "{err}"); + assert_eq!(plugin.dark_volts, 0.0); + } + + #[test] + fn the_ui_mirror_keeps_the_operators_connect_intent() { + // The mirror runs `apply_execution_context` every control tick. If it + // clears the intent, the host samples `connect` as false and the live + // worker never opens the port. + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_runtime_role(PluginRuntimeRole::UiMirror); + plugin.set_setting("connect", json!(true)).expect("connect"); + assert!(plugin.connect_requested); + + plugin.apply_execution_context(&live_execution()); + assert!( + plugin.connect_requested, + "the mirror cleared the connect intent" + ); + assert_eq!(plugin.get_setting("connect"), Some(json!(true))); + // ...but it must not have actually opened anything. + assert!(!plugin.connected()); + } + fn service_request( plugin: &StageAPhotodiodePlugin, id: u64, @@ -3576,6 +3890,95 @@ mod tests { dir } + fn marker_frame(sequence: u32, sample_index: u64) -> Frame { + let mut payload = Vec::with_capacity(16); + payload.extend_from_slice(&sample_index.to_le_bytes()); + payload.extend_from_slice(&0_u32.to_le_bytes()); // tick_us + payload.push(1); // level + payload.push(0); // source + payload.extend_from_slice(&[0, 0]); // reserved + Frame::build( + FrameHeader { + version: stage_a_io::wire::PROTOCOL_VERSION, + frame_type: FrameType::Marker, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: sample_index, + sample_rate_hz: MOCK_RATE_HZ, + dropped_samples: 0, + crc32: 0, + }, + payload, + ) + } + + #[test] + fn phase_zero_markers_are_written_into_the_recording() { + // Without the marker frames a recorded run cannot be phase-attributed + // offline, which is the whole point of the .pdq evidence file. + let dir = temp_dir("marker-record"); + let pdq_path = dir.join("run.pdq"); + let shared = Arc::new(Mutex::new(SharedState::default())); + let recording: SharedRecording = Arc::new(Mutex::new(Some(RecordingSink { + writer: PdqWriter::create(&pdq_path).expect("create pdq"), + pdq_path: pdq_path.clone(), + sidecar_path: dir.join("run.json"), + pdq_path_label: "run.pdq".into(), + sidecar_path_label: "run.json".into(), + run_id: RunId::from("test"), + opened_at_unix_ms: 0, + stream_epoch: 0, + first_sample_index: None, + metadata: BTreeMap::new(), + started_slug: "slug".into(), + samples_written: 0, + write_error: None, + start_crc_failures: 0, + start_resync_bytes: 0, + start_device_dropped: 0, + start_segments: 0, + }))); + + let codes = [100_u16, 200, 300, 400]; + assert!(ingest_parse_event( + ParseEvent::Frame(mock_sample_frame(0, 0, &codes)), + &shared, + &recording, + )); + assert!(ingest_parse_event( + ParseEvent::Frame(marker_frame(1, 2)), + &shared, + &recording, + )); + + // The marker still reaches the live ring... + assert_eq!( + shared.lock().unwrap().markers.iter().copied().last(), + Some(2) + ); + // ...and the sample count is unaffected by the marker frame. + let sink = recording.lock().unwrap().take().expect("sink"); + assert_eq!(sink.samples_written, codes.len() as u64); + sink.writer + .finish(StreamIntegrity::default()) + .expect("finish pdq"); + + let mut reader = stage_a_io::PdqReader::open(&pdq_path).expect("open pdq"); + let mut frame_types = Vec::new(); + while let Some(event) = reader.next_event().expect("read event") { + if let stage_a_io::PdqReadEvent::Frame(frame) = event { + frame_types.push(frame.header.frame_type); + } + } + assert!( + frame_types.contains(&FrameType::Marker), + "the .pdq holds no marker frame: {frame_types:?}" + ); + assert!(frame_types.contains(&FrameType::SamplesU16)); + std::fs::remove_dir_all(dir).unwrap(); + } + #[test] fn cache_snapshot_writes_csv_and_sidecar() { let dir = temp_dir("snapshot"); From 029638a4f97878de8d34b52af937aafb7d432004 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:06:08 +0200 Subject: [PATCH 23/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20stop=20set?= =?UTF-8?q?tings=20syncs=20from=20overwriting=20a=20running=20protocol=20s?= =?UTF-8?q?tep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `send_modulation` was silenced for an automation lease and a calibration sweep but not for a running protocol, which queues its steps into the same `pending` slot. Because the host re-applies the whole settings snapshot on every sync, any settings change — from this plugin or another — dropped the operator's armed drive on top of the protocol's queued step, and the board held it until the next step boundary. Also in the modulation plugin: - `mod_freq_mhz` is parsed once as f64 and rounded to millihertz; the second u64 parse returned None as soon as the firmware echoed a decimal, which published frequency_millihz: 0 and cost A1 its fallback modulation period - a leased `SetOpticalDepth` now parks the operator's armed depth and `end_lease` restores it, so the board no longer holds the last sweep point's depth after an A1 amplitude sweep finishes --- plugins/stage-a-modulation/src/lib.rs | 186 +++++++++++++++++++++++--- 1 file changed, 169 insertions(+), 17 deletions(-) diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index a77c6f2..2dc3e51 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -561,10 +561,16 @@ fn apply_reply_fields(state: &mut DeviceState, fields: &BTreeMap if let Some(wave) = fields.get("mod_wave") { let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); - let freq_mhz = fields + // Parse the echoed frequency exactly once, as f64. Parsing it a second + // time as u64 silently yielded None the moment the firmware echoed a + // decimal ("10000.0"): `board_echo_target` then published + // frequency_millihz: 0, A1 rejected it, and A1 lost its only fallback + // modulation period whenever the EXT_TRIGGER markers were absent. + let freq_millihz = fields .get("mod_freq_mhz") .and_then(|v| v.parse::().ok()) - .unwrap_or(0.0); + .filter(|hz| hz.is_finite() && *hz >= 0.0); + let freq_mhz = freq_millihz.unwrap_or(0.0); state.board_mod = if wave == "SINE" || wave == "SQUARE" { format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) } else { @@ -573,7 +579,7 @@ fn apply_reply_fields(state: &mut DeviceState, fields: &BTreeMap state.board_wave = Some(wave.clone()); state.board_level = fields.get("mod_level").and_then(|v| v.parse().ok()); state.board_min = fields.get("mod_min").and_then(|v| v.parse().ok()); - state.board_freq_millihz = fields.get("mod_freq_mhz").and_then(|v| v.parse().ok()); + state.board_freq_millihz = freq_millihz.map(|hz| hz.round() as u64); } } @@ -900,6 +906,9 @@ pub struct StageAModulationPlugin { // -- optical drive inversion (OPTICAL_* modes) -- /// Requested optical log-modulation depth `a = ln(I_max / I_min)`. depth_a: f64, + /// The operator's armed `depth_a`, parked while a lease drives the optical + /// depth (A1's amplitude sweep) and restored by [`Self::end_lease`]. + armed_depth_a: Option, /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0,1]`. /// Held fixed while `a` is swept, so one response curve keeps `I_k` constant. operating_point: f64, @@ -973,6 +982,7 @@ impl Default for StageAModulationPlugin { mode: Mode::Const, frequency_hz: 10.0, depth_a: 0.5, + armed_depth_a: None, operating_point: 0.5, v_null_dac: 0, v_pi_dac: 2_048, @@ -1240,14 +1250,20 @@ impl StageAModulationPlugin { /// Queues one MOD command carrying the complete current drive settings; /// newer changes overwrite queued ones (drag coalescing). /// - /// Silent while another owner holds the DAC. Besides an automation lease - /// that now includes a calibration sweep: the host re-applies the *whole* - /// settings snapshot on every sync, and most handlers here call this - /// unconditionally, so without the guard every sync would re-arm the - /// operator's drive on top of the code the sweep just commanded — the - /// sweep would measure the armed waveform instead of its own staircase. + /// Silent while another owner holds the DAC: an automation lease, a + /// calibration sweep, or a running protocol. The host re-applies the + /// *whole* settings snapshot on every sync, and most handlers here call + /// this unconditionally, so without the guard every sync would re-arm the + /// operator's drive on top of the code the current owner just commanded — + /// the sweep would measure the armed waveform instead of its own + /// staircase, and a protocol step would be overwritten mid-step and held + /// until the next step boundary. fn send_modulation(&mut self) { - if self.link.is_none() || self.lease.is_some() || self.sweep.is_some() { + if self.link.is_none() + || self.lease.is_some() + || self.sweep.is_some() + || self.protocol_active() + { return; } let command = match self.drive_command() { @@ -1887,7 +1903,7 @@ impl StageAModulationPlugin { self.deferred_release_ack_published = false; return Ok(response); } - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.shared .fail_closed_on_stop @@ -1976,6 +1992,10 @@ impl StageAModulationPlugin { )); } }; + // Remember what the operator had armed before the first + // sweep point, so `end_lease` can hand it back. Only the + // first one: later points must not overwrite the original. + self.armed_depth_a.get_or_insert(previous); *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { commands: vec![command], purpose: "MOD", @@ -2131,6 +2151,23 @@ impl StageAModulationPlugin { } } + /// Ends the current lease and gives the operator their armed drive back. + /// + /// A leased `SetOpticalDepth` (A1's amplitude sweep) writes straight into + /// `depth_a`. Without this the modulation UI kept showing — and the board + /// kept holding — the last sweep point's depth after the sweep finished, + /// rather than what the operator had armed. The calibration sweep already + /// restores through `Sweep::restore`; this is the leased equivalent. + fn end_lease(&mut self) { + self.lease = None; + if let Some(depth) = self.armed_depth_a.take() { + self.depth_a = depth; + // Re-arm the board only if nobody else now owns the DAC; + // `send_modulation` is itself guarded. + self.send_modulation(); + } + } + fn expire_lease_if_needed(&mut self) { let expired = self .lease @@ -2152,7 +2189,7 @@ impl StageAModulationPlugin { purpose: "LEASE_EXPIRED_SAFE_OFF", meta: None, }); - self.lease = None; + self.end_lease(); self.last_error = Some("automation lease expired; queued STOP + output off".into()); self.shared.bump(); } @@ -2175,7 +2212,7 @@ impl StageAModulationPlugin { return; } if self.deferred_release_ack_published { - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.deferred_release_ack_published = false; self.shared @@ -2201,7 +2238,7 @@ impl StageAModulationPlugin { .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); } - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.deferred_release_ack_published = false; return; @@ -2647,7 +2684,7 @@ impl Plugin for StageAModulationPlugin { .fail_closed_on_stop .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); - self.lease = None; + self.end_lease(); self.deferred_release_request = None; } } @@ -2662,7 +2699,7 @@ impl Plugin for StageAModulationPlugin { .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); } - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.deferred_release_ack_published = false; } @@ -2686,7 +2723,7 @@ impl Plugin for StageAModulationPlugin { .fail_closed_on_stop .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); - self.lease = None; + self.end_lease(); self.last_error = Some("disconnected: replay mode".into()); } } @@ -3337,6 +3374,13 @@ impl Plugin for StageAModulationPlugin { } self.send_modulation(); } + // An edit made while a lease drives the depth is withheld from + // the board (`send_modulation` is guarded), so it has to land + // in the parked value or it would be lost when the lease ends + // — same rule the calibration sweep follows. + if self.armed_depth_a.is_some() { + self.armed_depth_a = Some(self.depth_a); + } Ok(()) } "operating_point" => { @@ -4718,6 +4762,114 @@ level = 750 plugin.disconnect(); } + #[test] + fn ending_a_lease_restores_the_operators_armed_optical_depth() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Sine; + plugin.depth_a = 0.4; // what the operator armed + + let acquire = service_request( + &plugin, + 60, + "stage-a-a1", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + + // Two sweep points: only the first must be remembered as "armed". + for (id, milli) in [(61_u64, 900_u32), (62, 1_250)] { + let point = service_request( + &plugin, + id, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: milli, + }, + None, + ); + let reply = plugin.handle_service_request(&point, &live_execution()); + assert!( + matches!(reply.outcome, PluginServiceOutcome::Accepted { .. }), + "sweep point {milli} rejected: {:?}", + reply.outcome + ); + } + assert!((plugin.depth_a - 1.25).abs() < 1e-9, "sweep drives the depth"); + + plugin.end_lease(); + assert!( + (plugin.depth_a - 0.4).abs() < 1e-9, + "armed depth not restored: {}", + plugin.depth_a + ); + assert!(plugin.armed_depth_a.is_none()); + plugin.disconnect(); + } + + #[test] + fn a_running_protocol_owns_the_pending_slot() { + // The host re-applies the whole settings snapshot on every sync. An + // unguarded `send_modulation` would drop the operator's armed drive + // into the slot the protocol step is queued in, and the board would + // hold it until the next step boundary. + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + + let progress = Arc::new(Mutex::new(ProtocolProgress::default())); + plugin.protocol = Some(ProtocolRun { + progress: Arc::clone(&progress), + stop: Arc::new(AtomicBool::new(false)), + join: None, + }); + assert!(plugin.protocol_active()); + + *plugin.shared.pending.lock().unwrap() = None; + plugin.set_setting("max_level", json!(3_000)).expect("set"); + assert!( + plugin.shared.pending.lock().unwrap().is_none(), + "a settings sync overwrote the protocol's pending slot" + ); + + // Once the protocol finishes, the operator's drive gets through again. + progress.lock().unwrap().finished = true; + assert!(!plugin.protocol_active()); + plugin.set_setting("max_level", json!(3_100)).expect("set"); + assert!(plugin.shared.pending.lock().unwrap().is_some()); + plugin.disconnect(); + } + + #[test] + fn a_decimal_frequency_echo_still_yields_millihertz() { + // Firmware echoing "10000.0" used to parse as u64 -> None, which + // published frequency_millihz: 0 and cost A1 its fallback period. + let mut state = DeviceState::default(); + let mut fields = BTreeMap::new(); + fields.insert("mod_wave".to_owned(), "SINE".to_owned()); + fields.insert("mod_level".to_owned(), "2000".to_owned()); + fields.insert("mod_min".to_owned(), "100".to_owned()); + fields.insert("mod_freq_mhz".to_owned(), "10000.0".to_owned()); + apply_reply_fields(&mut state, &fields); + assert_eq!(state.board_freq_millihz, Some(10_000)); + + // The integer form keeps working. + fields.insert("mod_freq_mhz".to_owned(), "7500".to_owned()); + apply_reply_fields(&mut state, &fields); + assert_eq!(state.board_freq_millihz, Some(7_500)); + } + #[test] fn set_optical_depth_requires_lease_and_a_calibrated_drive() { let mut plugin = live_plugin(); From c1303d77901b376b73ee8d428c15bd8dd38e9249 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:10:35 +0200 Subject: [PATCH 24/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20normalise?= =?UTF-8?q?=20the=20A1=20rolling=20response=20over=20the=20ROI,=20and=20me?= =?UTF-8?q?moise=20the=20fold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `S_p(t)` divided by the whole sensor (`width * height`) while `q_p` on the same screen divided by ROI area minus masked pixels, and the rolling numerator counted events from outside the ROI and from masked pixels. With a small ROI that under-reported `S_p` by the ROI/frame ratio, and the status readout printed both numbers under the same "valid pixels" label. The fold is now built from ROI-filtered events and both quantities divide by `valid_pixel_count()`. `current_fold()` is memoised on a fingerprint of its inputs. It is called from `rolling_dataset`, `latest_rolling` (twice), `current_windows` and `current_response`, each allocating a `Vec` over up to MAX_EVENTS — a single repaint could allocate and discard hundreds of megabytes at bench event rates. Also in the A1 plugin: - the no-EventStore fallback trims by the analysis window instead of growing to MAX_EVENTS and then freezing on a stale buffer while the plots still looked live - a failed pilot-window freeze clears the previously loaded windows, so the sidecar cannot record an earlier pilot's windows as this run's --- plugins/stage-a-a1/src/runtime.rs | 242 ++++++++++++++++++++++++++++-- 1 file changed, 228 insertions(+), 14 deletions(-) diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index b1baedd..64ca65b 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -24,6 +24,7 @@ //! `q_p(a, f)` fit is computed offline from the recordings; the live plot is a //! quicklook. +use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -299,9 +300,10 @@ pub struct StageAA1Plugin { /// anchor the fold to the drive on the camera clock; empty falls back to the /// free-running fold on `T`. camera_markers_us: Vec, - valid_pixels: usize, frame_width: u16, frame_height: u16, + /// Memoised [`StageAA1Plugin::current_fold`], keyed on its inputs. + fold_cache: RefCell)>>, // -- host camera ROI/mask, mirrored from CTX_GLOBAL_SETTINGS -- host_roi: Option, masked_pixels: HashSet<(u16, u16)>, @@ -367,7 +369,7 @@ impl Default for StageAA1Plugin { event_scratch: Vec::new(), analysis_window_ms: DEFAULT_ANALYSIS_WINDOW_MS, camera_markers_us: Vec::new(), - valid_pixels: 0, + fold_cache: RefCell::new(None), frame_width: 0, frame_height: 0, host_roi: None, @@ -466,6 +468,22 @@ impl Recording { } } +/// Fingerprint of everything the phase fold is computed from. Cheap to build +/// (no scan of the event buffer) and exact enough that a stale fold cannot +/// survive a change to any input. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct FoldKey { + period_us_bits: u64, + event_count: usize, + first_event_us: Option, + last_event_us: Option, + marker_count: usize, + first_marker_us: Option, + last_marker_us: Option, + roi: Option, + masked_count: usize, +} + impl StageAA1Plugin { fn bump(&mut self) { self.dataset_generation = self.dataset_generation.wrapping_add(1); @@ -523,12 +541,59 @@ impl StageAA1Plugin { } } + /// The current phase fold, memoised. + /// + /// Called several times per repaint (`rolling_dataset`, `latest_rolling` + /// from both `status_dataset` and `status_entries`, `current_windows`, + /// `current_response`). Each fold allocates a `Vec` over up to + /// `MAX_EVENTS` events, so refolding per call threw away hundreds of + /// megabytes per repaint at bench event rates. The cache is keyed on a + /// cheap fingerprint of everything the fold reads, so it invalidates + /// exactly when the inputs move rather than on every `bump()`. fn current_fold(&self) -> Option { + let key = self.fold_key()?; + if let Ok(cache) = self.fold_cache.try_borrow() { + if let Some((cached_key, fold)) = cache.as_ref() { + if *cached_key == key { + return fold.clone(); + } + } + } + let fold = self.compute_fold(); + if let Ok(mut cache) = self.fold_cache.try_borrow_mut() { + *cache = Some((key, fold.clone())); + } + fold + } + + /// Fingerprint of every input [`Self::compute_fold`] reads. `None` when + /// there is no period, i.e. no fold to compute. + fn fold_key(&self) -> Option { + let period_us = self.period_us()?; + Some(FoldKey { + period_us_bits: period_us.to_bits(), + event_count: self.camera_events.len(), + first_event_us: self.camera_events.first().map(|event| event.timestamp_us), + last_event_us: self.camera_events.last().map(|event| event.timestamp_us), + marker_count: self.camera_markers_us.len(), + first_marker_us: self.camera_markers_us.first().copied(), + last_marker_us: self.camera_markers_us.last().copied(), + roi: self.roi(), + masked_count: self.masked_pixels.len(), + }) + } + + fn compute_fold(&self) -> Option { let period_us = self.period_us()?; + // Fold only the events the analysis is normalised over. `q_p` already + // restricts to ROI minus masked pixels; the rolling response divides by + // the same count, so its numerator has to be restricted too or it + // counts events from outside the ROI against an ROI-sized denominator. + let events = self.roi_filtered_events(); let marker_fold = self.is_marker_anchored().then(|| { let expected_hz = 1_000_000.0 / period_us; fold_events( - &self.camera_events, + &events, &self.camera_markers_us, MarkerValidationConfig { expected_frequency_hz: expected_hz, @@ -545,7 +610,27 @@ impl StageAA1Plugin { // blank the live plots — fall back to the free-running fold on T. marker_fold .flatten() - .or_else(|| fold_events_free_running(&self.camera_events, period_us)) + .or_else(|| fold_events_free_running(&events, period_us)) + } + + /// The analysis-window events restricted to the ROI, masked pixels removed. + /// Without an ROI the whole frame is the ROI, so this is a clone. + fn roi_filtered_events(&self) -> Vec { + let Some(roi) = self.roi() else { + return self.camera_events.clone(); + }; + if roi.area() == usize::from(self.frame_width) * usize::from(self.frame_height) + && self.masked_pixels.is_empty() + { + return self.camera_events.clone(); + } + self.camera_events + .iter() + .filter(|event| { + roi.contains(event.x, event.y) && !self.masked_pixels.contains(&(event.x, event.y)) + }) + .copied() + .collect() } /// Optical modulation depth `a` published by the photodiode plugin. @@ -622,6 +707,10 @@ impl StageAA1Plugin { self.note("Pilot windows frozen from the live signal"); } None => { + // Drop whatever was loaded for this measurement: leaving it in + // place let `write_sidecar` record windows from an *earlier* + // pilot as if they had just been frozen from this run. + self.pilot_windows = None; self.note("No live signal to freeze windows — enable Live analysis first"); } } @@ -716,8 +805,13 @@ impl StageAA1Plugin { let sample_times: Vec = (0..samples) .map(|index| first + (last - first) * index / (samples - 1)) .collect(); + // Same denominator as `q_p` (ROI minus masked), against the ROI-filtered + // fold — the two are shown side by side and must mean the same thing. + let Some(valid_pixels) = self.valid_pixel_count() else { + return empty(); + }; let line = |polarity: Polarity| { - rolling_half_period_response(&fold, polarity, self.valid_pixels, &sample_times, None) + rolling_half_period_response(&fold, polarity, valid_pixels, &sample_times, None) .map(|points| points_for(&points, first)) .unwrap_or_default() }; @@ -741,8 +835,9 @@ impl StageAA1Plugin { fn latest_rolling(&self) -> Option<(f64, f64)> { let fold = self.current_fold()?; let at = [fold.validation.last_marker_us]; + let valid_pixels = self.valid_pixel_count()?; let value = |polarity| { - rolling_half_period_response(&fold, polarity, self.valid_pixels, &at, None) + rolling_half_period_response(&fold, polarity, valid_pixels, &at, None) .ok() .and_then(|points| points.first().map(|point| point.run_per_pixel)) }; @@ -2123,7 +2218,6 @@ impl Plugin for StageAA1Plugin { self.camera_events.clear(); self.event_scratch.clear(); self.camera_markers_us.clear(); - self.valid_pixels = 0; self.response_points.clear(); self.pilot_windows = None; self.background_floor = None; @@ -2179,7 +2273,6 @@ impl Plugin for StageAA1Plugin { if !self.live { return; } - self.valid_pixels = usize::from(frame.width()) * usize::from(frame.height()); // Markers (phase-0 sync) only exist on the preview frame, so accumulate // the rising EXT_TRIGGER edges here regardless of the event source. @@ -2216,11 +2309,30 @@ impl Plugin for StageAA1Plugin { // Keep the marker set on the same window as the events. self.camera_markers_us .retain(|&marker| marker >= window_start); - } else if self.camera_events.len() < MAX_EVENTS { + } else { // Fallback (no retained history available): accumulate the - // best-effort preview-frame events. + // best-effort preview-frame events, then trim to the same analysis + // window the exact path uses. Without the trim the buffer grew to + // MAX_EVENTS and then stopped accepting anything at all, so the + // fold silently spanned an ever-widening window and finally froze + // on a stale 4M-event buffer while the plots still looked live. self.camera_events .extend(frame.events().iter().map(ffi_to_camera_event)); + let window_start = window_end.saturating_sub(window_us); + let keep_from = self + .camera_events + .partition_point(|event| event.timestamp_us < window_start); + if keep_from > 0 { + self.camera_events.drain(..keep_from); + } + // Hard ceiling as well: a window longer than the event buffer can + // hold must drop the oldest events, not stop taking new ones. + if self.camera_events.len() > MAX_EVENTS { + let excess = self.camera_events.len() - MAX_EVENTS; + self.camera_events.drain(..excess); + } + self.camera_markers_us + .retain(|&marker| marker >= window_start); } self.bump(); } @@ -2673,7 +2785,6 @@ impl Plugin for StageAA1Plugin { self.camera_events.clear(); self.event_scratch.clear(); self.camera_markers_us.clear(); - self.valid_pixels = 0; } } "window_floor" => { @@ -2759,7 +2870,7 @@ impl Plugin for StageAA1Plugin { entries.push(StatusEntry::Text(format!( "{} events, {} valid pixels; {anchor}", self.camera_events.len(), - self.valid_pixels + self.valid_pixel_count().unwrap_or(0) ))); entries.push(StatusEntry::Text(match self.measured_a() { Some(a) => format!("a = {a:.3} (photodiode)"), @@ -2988,7 +3099,9 @@ mod tests { /// A plugin whose period comes from marker spacing (no fallback frequency). fn plugin_with_markers() -> StageAA1Plugin { StageAA1Plugin { - valid_pixels: 10, + // 10 x 1 sensor, no host ROI => valid_pixel_count() == 10. + frame_width: 10, + frame_height: 1, camera_markers_us: vec![0, 1_000, 2_000, 3_000], ..StageAA1Plugin::default() } @@ -3077,6 +3190,106 @@ mod tests { assert!(plugin.record_response_point().is_err()); } + #[test] + fn the_rolling_response_is_normalised_over_the_roi_not_the_sensor() { + // `q_p` counts ROI-minus-masked pixels; the rolling half-period rate is + // plotted next to it and must agree. Normalising by the whole sensor + // under-reported S_p by the ROI/frame ratio *and* counted events from + // outside the ROI. + let mut plugin = StageAA1Plugin { + frame_width: 10, + frame_height: 10, + camera_markers_us: vec![0, 1_000, 2_000, 3_000], + ..StageAA1Plugin::default() + }; + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 2, + height: 2, + }); + let event = |x: u16, y: u16, timestamp_us: u64| CameraEvent { + timestamp_us, + x, + y, + polarity: Polarity::On, + }; + // Two ON events inside the 2x2 ROI, five well outside it, all inside + // the trailing half period the status readout samples. + plugin.camera_events.push(event(0, 0, 2_800)); + plugin.camera_events.push(event(1, 1, 2_850)); + for x in 5..10_u16 { + plugin.camera_events.push(event(x, 9, 2_900)); + } + + let (on_rate, _) = plugin.latest_rolling().expect("rolling value"); + assert!( + (on_rate - 0.5).abs() < 1e-9, + "expected 2 ROI events over 4 valid pixels, got {on_rate}" + ); + } + + #[test] + fn the_fold_cache_tracks_its_inputs() { + let mut plugin = plugin_with_markers(); + for cycle in 0..8 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + let first = plugin.current_fold().expect("fold"); + // Repeated calls within a repaint must be identical, not merely equal + // to a fresh recomputation. + assert_eq!(plugin.current_fold().as_ref(), Some(&first)); + assert_eq!(plugin.compute_fold().as_ref(), Some(&first)); + + // ...and adding an event inside the marker span must invalidate it. + plugin.camera_events.push(on(2_500)); + plugin.camera_events.sort_by_key(|event| event.timestamp_us); + let second = plugin.current_fold().expect("fold"); + assert_eq!(second.events.len(), first.events.len() + 1); + + // A changed ROI also invalidates, even at identical event counts. + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 1, + height: 1, + }); + let third = plugin.current_fold().expect("fold"); + assert_eq!(third.events.len(), second.events.len()); + plugin.host_roi = Some(RoiV1 { + x: 5, + y: 0, + width: 1, + height: 1, + }); + let fourth = plugin.current_fold().expect("fold"); + assert!( + fourth.events.is_empty(), + "ROI moved off the events but the cache served a stale fold" + ); + } + + #[test] + fn a_failed_pilot_freeze_clears_stale_windows() { + // `scan_measurement_folder` may have loaded windows from an earlier + // pilot for this measurement. If the freeze then fails, the sidecar + // must not record those as if they had come from this run. + let mut plugin = plugin_with_markers(); + plugin.pilot_windows = Some(( + PhaseWindow { start: 0.0, end: 0.2 }, + PhaseWindow { start: 0.5, end: 0.7 }, + )); + // No events => the fold carries no signal => the freeze cannot pick + // windows and must not leave the loaded ones in place. + assert!(plugin.camera_events.is_empty()); + plugin.freeze_pilot_windows(); + assert!( + plugin.pilot_windows.is_none(), + "stale pilot windows survived a failed freeze" + ); + assert!(!plugin.windows_are_frozen()); + } + #[test] fn press_latch_distinguishes_clicks_baselines_and_advances() { let mut latch = PressLatch::default(); @@ -3300,7 +3513,8 @@ mod tests { // marker validation rejects the fold, but the quicklook must fall back // to the free-running fold instead of blanking. let mut plugin = StageAA1Plugin { - valid_pixels: 10, + frame_width: 10, + frame_height: 1, camera_markers_us: vec![0, 1_000, 2_000, 10_000], ..StageAA1Plugin::default() }; From f22ce9a716fe823864e75e79277571ae059d742a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:12:58 +0200 Subject: [PATCH 25/46] =?UTF-8?q?docs(stage-a):=20=F0=9F=93=9D=20record=20?= =?UTF-8?q?the=20contrast-geometry=20decision=20as=20ADR=20012?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The photodiode's display mode used to select the optical geometry the published log-contrast was computed in, so a UI toggle changed a scientific quantity that A1's amplitude sweep settles against. ADR 012 records that geometry follows the bench, the measured dark level is applied to both sides of the complement (where it cancels), and a withheld `a` states its reason. Also documents the A1 `N_valid` definition (ROI area minus masked pixels, the same denominator `q_p` uses) and generalises the rule in architecture.md: a published field's meaning must not depend on the publisher's UI state. --- ...-contrast-geometry-is-bench-not-display.md | 82 +++++++++++++++++++ docs/architecture.md | 7 ++ docs/features/README.md | 2 +- docs/features/stage-a-a1.md | 14 +++- docs/features/stage-a-photodiode.md | 22 +++++ 5 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md diff --git a/docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md b/docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md new file mode 100644 index 0000000..7a39528 --- /dev/null +++ b/docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md @@ -0,0 +1,82 @@ +# ADR 012 — The contrast geometry follows the bench, not the display mode + +- **Status:** Accepted +- **Date:** 2026-07-27 +- **Relates to:** ADR 006 (two-plugin split), ADR 008 (optical waveform + inversion), ADR 010 (amplitude sweep), ADR 011 (Pockels transfer + calibration), + [Stage-A Photodiode](../features/stage-a-photodiode.md), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +The photodiode plugin has a display toggle: **RAW** plots the detector volts as +measured, **EXCITATION** plots `I_tot − I_pd`. `optical_summary` picked the +estimator's [`ContrastGeometry`] from that toggle — `Direct` under RAW, +`RejectedComplement` under EXCITATION — and published the result as +`PhotodiodeOpticalSummaryV1::measured_log_contrast`. + +That made a *published scientific quantity* depend on what the operator +happened to be looking at. It is wrong on the physics and it breaks A1: + +- On this bench the detector sits behind the PBS reject port and measures the + complement `I_pd = I_tot − I_exc`. That is settled by construction + (`knowledge base: setup/optical-path.md`), not a display choice. Under RAW the + published value was `ln(I_pd,max / I_pd,min)` — the *detector* contrast, not + the excitation contrast `a` that every A1 estimand is defined against. +- RAW is the default. A1's amplitude sweep settles `measured_a` against a target + `a` (`drive_sweep`): with the display left on its default the sweep compares + the wrong quantity, never settles, times out at 30 s per point, and writes a + wrong `measured_a` into every sweep sidecar. + +The same function also passed `dark_volts: 0.0` and a raw `reference_volts` +anchor, i.e. it dark-corrected one side of the complement and not the other. + +## Decision + +### 1. Geometry is a property of the optical configuration + +`optical_summary` always uses `ContrastGeometry::RejectedComplement`, anchored on +`reference_volts`. `measured_log_contrast` is always the excitation contrast. +The display `Mode` is presentational and never reaches the estimator; the status +readout is labelled `a (excitation)` unconditionally. + +If a future bench puts the detector in the excitation path, that is a new +optical configuration ID and a code change here — not a UI toggle. + +### 2. The dark level is measured, and applied to both sides + +`dark_volts` is a plugin setting with a **Capture dark** action (block the beam, +press; the mean of the current ring becomes the dark level, refused if it is not +below the `I_tot` reference). It is applied to the detector samples *and* +subtracted from the `reference_volts` anchor. + +Applied consistently, the DC dark term **cancels** out of the complement — the +excitation is a difference of two readings from the same DC-coupled detector, so +a common offset drops out. Correcting only one side is what would bias `a`, and +that is what the code did. `dark_id` reports `dark-measured` or `dark-none` so a +consumer can tell a real dark measurement from the un-measured default. + +### 3. A withheld `a` states its reason + +The estimator is deliberately fail-closed (clipping, no headroom, anchor below +signal). Those refusals now surface in the status readout as +`a unavailable: ` instead of the row silently disappearing. This matters +more under the new geometry: with an un-measured anchor left at ADC full scale, +`TotalPowerBelowSignal` is the expected first-run outcome, and the operator has +to be told to set `reference_volts`. + +## Consequences + +- `measured_log_contrast` is comparable across runs and independent of operator + UI state; A1's sweep settles against the quantity it targets. +- Runs recorded before this change that were taken with the display on RAW + carry a detector contrast in `measured_a`. They are distinguishable: their + sidecar has `anchor_id: "detector-direct"`. Those points must not be mixed + with `reference-volts` points. +- `excitation_headroom_volts` is, by construction, equal to + `excitation_min_volts` (both geometries are dark-referenced). The field is + kept because the contract publishes it, and is now documented as redundant + rather than silently duplicated. +- First use on a fresh bench requires setting `reference_volts` before any `a` + is published at all. This is intended: a wrong `a` is worse than no `a`. diff --git a/docs/architecture.md b/docs/architecture.md index dcba238..a46c4c1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,6 +93,13 @@ calibration reads photodiode levels this way while driving only its own DAC: [`docs/adr/011-stage-a-pockels-transfer-calibration.md`](./adr/011-stage-a-pockels-transfer-calibration.md). Reserve the leased service path for *commanding* hardware someone else owns. +A published field is part of that contract, so its **meaning must not depend on +the publisher's UI state**. The photodiode plugin's display toggle used to +select the optical geometry the published log-contrast was computed in, which +silently retargeted A1's amplitude sweep whenever the chart was left on its +default. Geometry follows the bench, not the display: +[`docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md`](./adr/012-stage-a-contrast-geometry-is-bench-not-display.md). + ## Host Views Plugins declare host-rendered datasets and views through: diff --git a/docs/features/README.md b/docs/features/README.md index 2ff4830..56f1ed2 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -8,7 +8,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC so the *optical* output is a log- or linear-intensity sine, inverting the Pockels `sin²` transfer from settable `V_null`/`Vπ`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`Vπ` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`, plus the geometry-corrected optical depth `a`. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). - [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 0b72be2..edb0405 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -133,8 +133,14 @@ quicklook falls back to the free-running fold on `T` instead of going empty. events per valid pixel in the trailing half-cycle, ON and OFF. A live indicator: are events appearing, does the ON/OFF timing look sane, is the response - saturating? It counts *every* event, so a noisy pixel weighs heavily — it is a - quicklook, not the response metric. + saturating? It counts *every* event in the ROI, so a noisy pixel weighs heavily + — it is a quicklook, not the response metric. + + `N_valid` is **ROI area minus masked pixels**, the same denominator `q_p` uses, + and the numerator counts only events inside that same region. The two are shown + side by side and have to mean the same thing; normalising `S_p` over the whole + sensor under-reported it by the ROI/frame ratio while counting events from + outside the ROI. 2. **Response probability** `q_p` @@ -186,10 +192,10 @@ still reset everything. | Input | Source | |---|---| -| camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()` | +| camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()`, trimmed to the same window | | phase-0 markers | rising `frame.external_triggers()` — the host **banks trigger edges from dropped preview frames** into the next processed frame (drain-to-newest and the preview throttle drop whole frames; at low modulation frequencies the survivors alone rarely held 2 markers inside the analysis window) | | modulation period `T` | measured from the `EXT_TRIGGER` marker spacing; else the modulation plugin's acknowledged waveform — which, since the board-echo fallback, includes the **operator-armed UI drive**, not only service-path (leased) targets | -| optical modulation depth `a` | photodiode plugin's optical summary (`measured_log_contrast`) | +| optical modulation depth `a` | photodiode plugin's optical summary (`measured_log_contrast`) — always the *excitation* contrast, independent of that plugin's display mode (ADR 012) | | ROI, masked pixels | augur-rs camera config (`CTX_GLOBAL_SETTINGS`) | ## Tests diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 122e6ff..07cb165 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -28,11 +28,33 @@ source (the camera `EXT_TRIGGER` belongs to A1's camera-clock analysis, not here ## Modes +The mode is a **display** choice only. It selects what the chart and the sample readout show; it +never changes a published quantity (ADR 012). + - **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). - **EXCITATION** — the diode sits behind the PBS in the excitation path and measures the light removed from the beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. +## Optical log-contrast `a` + +`measured_log_contrast` in the published `PhotodiodeOpticalSummaryV1` is **always** the excitation +contrast `a = ln(I_exc,max / I_exc,min)`, in **both** display modes. The detector sits behind the +PBS reject port and measures the complement — that is a property of the bench, not of the display — +so the estimator always runs the `RejectedComplement` geometry against `reference_volts`. A1's +amplitude sweep settles on this value, so a display toggle must not be able to move it (ADR 012). + +- **Reference I_tot** (`reference_volts`) is the total-power anchor: the PD reading with the full + beam diverted into the diode. Until it is set to a real measurement, `a` is withheld. +- **Dark level** (`dark_volts`) + the **Capture dark** button: block the beam and press; the mean of + the current cache becomes the dark level. It is applied to the detector samples *and* to the + `I_tot` anchor, so it cancels out of the complement rather than biasing `a` — its job is to keep + the two sides consistent and to record the calibration the reading was taken under. `dark_id` in + the sidecar reads `dark-measured` or `dark-none` accordingly. +- The estimator is **fail-closed**: it refuses on ADC clipping, on no headroom above dark, and when + the anchor is not above the measured signal. A refusal is shown as `a unavailable: ` + rather than a missing row — a wrong `a` is worse than no `a`. + ## Chart - The visible window is decimated into at most 1 000 buckets; when a bucket covers more than one From 361fc676f4c22eb177ffbf8ade270e1e3f92e13c Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:41:43 +0200 Subject: [PATCH 26/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20bring=20the?= =?UTF-8?q?=20a=E2=82=80=20depth=20lock=20onto=20the=20fixed=20contrast=20?= =?UTF-8?q?geometry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the per-frequency exact-event-count depth lock (ADR 013, renumbered from 012 to clear the contrast-geometry ADR) onto the branch that carries the photodiode and modulation fixes, and repairs what that combination exposes. The lock closes `a_cmd ← a_cmd · a₀/a_measured` against the photodiode-measured log contrast. On its own branch that value's geometry followed the photodiode's *display toggle*, so under the default RAW mode it was the reject-port detector contrast rather than the excitation contrast — every locked depth would have been wrong. It is correct only together with the fixed geometry. Then the measurement itself: `a = ln(I_exc,max / I_exc,min)` is peak-to-peak, but the photodiode estimated it over a fixed 16 384-sample window — 0.82 s at 20 kSa/s, less than one cycle for every f < 1.2 Hz. Below one cycle the robust extrema see an arc of the waveform, so `a` comes out low and phase-dependent. That is exactly the sub-hertz plateau reference the A1 protocol normalises |H(f)| against, and the lock *divides* by it: a truncated estimate does not add noise, it drives the commanded depth up trial after trial until it rails at 6.0 or the detector clips. The contrast window is now sized from the phase-0 markers to cover CONTRAST_WINDOW_CYCLES whole cycles, floored at the old fixed window and capped by what the ring retains, and `a` is withheld outright below one cycle. The retained markers cannot measure a period longer than the ring — once the ring holds under a cycle it holds at most one marker — so the interval is remembered as markers go past instead of recovered from what survived eviction. `window_seconds` and `covered_cycles` join the optical summary (additive in V1). In the lock: - Find a₀ refuses up front when the published window is under one cycle at the current frequency, naming the cache length to raise. A1 always knows f, so this also covers an owner whose own markers cannot prove it. - the per-trial dwell is at least one estimator window, so a trial cannot average the depth it just replaced; the deadline grows with it - readings are spaced by half a window instead of per service_revision. Consecutive revisions share nearly their whole window, so three of them said no more than one - the trial value is the median and the spread is a stability gate: readings straddling a₀ abort the lock instead of locking onto a drifting drive - the clip warning threshold sits below the estimator's own refusal, where it can actually fire, instead of above it where it never could - a lock-table save failure is appended to the result instead of being overwritten by it --- .../013-stage-a-a1-event-count-depth-lock.md | 120 + docs/features/README.md | 1 + docs/features/stage-a-a1-automation.md | 11 + docs/features/stage-a-a1-event-count.md | 168 ++ docs/features/stage-a-a1.md | 20 +- plugins/stage-a-a1/README.md | 26 +- plugins/stage-a-a1/src/runtime.rs | 2167 +++++++++++++++-- plugins/stage-a-modulation/src/lib.rs | 5 +- plugins/stage-a-photodiode/src/lib.rs | 235 +- stage-a-io/src/estimator.rs | 20 + stage-a-plugin-contract/src/lib.rs | 12 + 11 files changed, 2577 insertions(+), 208 deletions(-) create mode 100644 docs/adr/013-stage-a-a1-event-count-depth-lock.md create mode 100644 docs/features/stage-a-a1-event-count.md diff --git a/docs/adr/013-stage-a-a1-event-count-depth-lock.md b/docs/adr/013-stage-a-a1-event-count-depth-lock.md new file mode 100644 index 0000000..7ee1c2e --- /dev/null +++ b/docs/adr/013-stage-a-a1-event-count-depth-lock.md @@ -0,0 +1,120 @@ +# ADR 013 — Stage-A A1 exact-event-count depth lock (`a₀`) + +- **Status:** accepted (2026-07-25) +- **Relates to:** ADR 009 (recording coordinator), ADR 010 (amplitude sweep via + leased `SetOpticalDepth`), ADR 011 (measured Pockels transfer calibration), + ADR 012 (the contrast geometry the measured `a` comes from), + [Stage-A A1 Exact Event Count](../features/stage-a-a1-event-count.md) + +## Context + +The minimum-depth workflow sweeps the depth `a` at one frequency and fits `a50`. +The **exact-event-count** workflow is the complement: freeze **one** depth + +```math +a_0=\ln\!\left(\frac{I_{\mathrm{exc,max}}}{I_{\mathrm{exc,min}}}\right), +\qquad I_\mathrm{exc}=I_\mathrm{tot}-I_\mathrm{pd} +``` + +and hold **that measured value** constant while the frequency varies, so the +event count per half-cycle is compared across `f` at equal optical contrast. + +`a₀` is defined on the **photodiode-measured** log contrast, never on a DAC +excursion. That is exactly where the existing sweep path stops short: ADR 010 +drives `SetOpticalDepth { depth_a_milli }` **open-loop**, trusting the measured +Pockels inversion (ADR 011) to turn a commanded depth into an optical one, and +then only *waits* for the measured `a` to arrive. The inversion is static, so as +`f` rises the drive electronics and the crystal response roll off and the +delivered depth falls short of the commanded one. Waiting cannot fix a +systematic gain error: the sweep would hit its 30 s settle cap and record a +point at the wrong depth (with the measured value honestly in the sidecar, but +the run wasted). + +A second, subtler problem: ADR 010 already notes that "the operator's own +`depth a` re-applies on the next modulation settings sync after release". A +depth found in one operator action and recorded in a *later* one can therefore be +silently overwritten between the two. + +## Decision + +**1. A closed-loop `a₀` lock in A1, separate from recording.** A *Find a₀* +button runs a small state machine — `AcquiringLease → (per trial) SettingDepth → +Measuring → …release` — that iterates + +```math +a_\text{cmd} \leftarrow a_\text{cmd}\cdot\frac{a_0}{a_\text{measured}} +``` + +until the photodiode-measured `a` is within an absolute tolerance of `a₀` +(default ±0.02), at most 8 trials, each correction capped at ×2/÷2 and clamped +to the owner's `0.01..=6.0`. The delivered depth is proportional to the commanded +one to first order, so this converges in two or three trials while absorbing +whatever roll-off the frequency introduces. It reuses the ADR 010 contract +command unchanged — no new modulation command, no optical math outside its owner. + +The lock **records nothing** and releases the lease with `safe_off = false`, so +the drive stays exactly where the lock left it. + +Measurement hygiene: readings are only taken after the operator's settle dwell +has passed, and one reading per **fresh** photodiode `service_revision` (three +per trial), so a slow publisher is not averaged once per control tick. A +measured `a ≤ 0`, a missing optical summary, or an owner rejection ends the lock +with the owner's own wording — a refused depth *is* the "`a₀` unreachable at this +operating point" answer. + +**2. The result is data, not a transient.** Each finished lock is stored as one +row per frequency — `frequency_hz`, `target_a`, `commanded_a`, `measured_a`, +`trials`, `converged`, clip fractions, timestamp — replacing any earlier row +within 1 % of the same frequency, shown in an `a₀ lock table` host view, and +mirrored to `a0_locks.json` in the output folder so the found depths survive a +restart and can be cited offline. Non-converged attempts are kept for the record +but never arm a recording. + +**3. Recording replays the locked depth under the lease.** *Record a₀ point* +does **not** simply record at whatever the drive currently is. It runs the ADR 010 +sweep machinery as a **one-point sweep of a new kind**: lease → command the +locked `a_cmd` → confirm the measured `a` holds `a₀` within the lock tolerance → +record through the unchanged coordinator → release. This gives three things at +once: the depth is re-asserted (immune to an intervening settings sync), the +lease locks the operator's modulation settings out for the whole point, so +"never change amplitude during the recorded interval" is enforced rather than +trusted, and the point is one button press. + +To express this, a sweep point became a pair — what the drive is **commanded** +to, and the depth it is **expected to measure**. The amplitude sweep sets both +equal (it trusts the calibration); an event-count point deliberately does not, +and the difference *is* the absorbed roll-off. + +**4. Naming and provenance.** Event-count points take the role suffix `_ec` and +carry their **frequency** in the stem (`…_ec_f50Hz`, `…_ec_f0p5Hz`) instead of a +sweep-point index, because one measurement id spans the whole frequency sweep at +the single frozen depth. The sidecar gains `sweep.commanded_a` and an +`[a0_lock]` section (target, commanded, measured-at-lock, frequency-at-lock, +trials, converged, locked-at), and both recorders' own sidecars carry the same +values as string metadata. + +**5. What stays the operator's.** The flux point, camera configuration, ROI/mask, +pedestal, bias set, gates, reference epoch, the frequency itself, the +`I_tot` anchor, the zero-depth background and the pilot (already separate +buttons), the randomised frequency order, the interleaved low-frequency +reference, and the repeated blocks. A1 adds exactly two buttons per frequency — +*Find a₀* and *Record a₀ point* — because the protocol's ordering and +randomisation decisions are scientific, not mechanical. + +## Consequences + +- A1's scoped hardware reach is unchanged in kind (still only the armed drive's + depth, still only while leased) but now closed-loop: it reads the photodiode + to decide what to command. +- The recorded amplitude is provably the measured `a₀`, not a calibrated guess, + at every frequency — including frequencies where the static Pockels inversion + is no longer accurate. +- `a₀` itself is **not** frozen numerically in this repository: it is an operator + input, to be chosen from the low-frequency scout (several events per + pixel-half-cycle, still proportional, refractory-safe at the top frequency). + The plugin default is a placeholder. +- The refractory condition `2 f a₀/C ≪ 1/τ_refr` is *not* checked in the plugin; + it is a choice made once when `a₀` is picked, and stays with the operator. +- Re-locking after changing the flux point, the calibration or `a₀` is required: + a stored lock is only armed for a matching frequency **and** a matching `a₀`, + and *Clear a₀ lock table* exists for the rest. diff --git a/docs/features/README.md b/docs/features/README.md index 56f1ed2..8606c60 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -11,6 +11,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). - [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. +- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀`, a per-frequency lock table on disk, and a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-a1-automation.md b/docs/features/stage-a-a1-automation.md index 529a10d..bcbeafc 100644 --- a/docs/features/stage-a-a1-automation.md +++ b/docs/features/stage-a-a1-automation.md @@ -25,6 +25,17 @@ > §3 coordinator with `sweep.requested_a` / `point_index` / `point_total` in > the sidecar. Remaining below: scout/randomized order, multi-`f`/`I_k` > iteration, the `UNIDENTIFIABLE` rule, and the `a50` fit. +> +> **Update (2026-07-25):** the *second* workflow — **exact event count**, holding +> one measured depth `a₀` across the frequency sweep — now has its own blocks +> (ADR 013, [brief](./stage-a-a1-event-count.md)): a closed-loop **Find a₀** per +> frequency (§1–§2 applied the other way round — measure, then correct the +> *commanded* depth) and a **Record a₀ point** that replays the locked depth under +> the lease through the §3 coordinator. The multi-`f` iteration below stays +> deliberately manual there: randomising the frequency order, interleaving a +> low-frequency reference and repeating independent blocks are scientific ordering +> decisions, so A1 exposes them as per-frequency button presses rather than one +> opaque run. ## Goal diff --git a/docs/features/stage-a-a1-event-count.md b/docs/features/stage-a-a1-event-count.md new file mode 100644 index 0000000..f9c4d7e --- /dev/null +++ b/docs/features/stage-a-a1-event-count.md @@ -0,0 +1,168 @@ +# Stage-A A1 Exact Event Count — the `a₀` depth lock + +- **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) +- **Status:** Built — per-frequency `a₀` lock + one-button event-count point +- **Design:** [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md); builds + on [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (leased + `SetOpticalDepth`) and [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) + (the RAW + PDQ + sidecar coordinator) +- **Relates to:** [Stage-A A1 Analysis](./stage-a-a1.md), + [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md), + [Stage-A Photodiode](./stage-a-photodiode.md) + +## Purpose + +The minimum-depth workflow sweeps `a` at one frequency to fit `a50`. The +**exact-event-count** workflow is the complement: freeze **one** depth + +```math +a_0=\ln\!\left(\frac{I_{\mathrm{exc,max}}}{I_{\mathrm{exc,min}}}\right), +\qquad I_\mathrm{exc}=I_\mathrm{tot}-I_\mathrm{pd} +``` + +and hold that **photodiode-measured** value constant while the frequency varies, +so event counts per half-cycle are comparable across `f` at equal optical +contrast. `a₀` is a measured log contrast — **never** a DAC-code excursion. + +## Why a lock is needed at all + +`ModulationCommandV1::SetOpticalDepth` commands a depth through the *measured* +Pockels inversion (`V_null`, `Vπ`, `u_k` — see the calibration brief). That +inversion is static, so at higher frequencies the drive electronics and crystal +response roll off and the delivered optical depth falls short of the commanded +one. The amplitude sweep (ADR 010) only *waits* for the measured `a`, which +cannot correct a systematic gain error — it would hit the 30 s settle cap and +record at the wrong depth. + +The lock closes that loop: it commands, measures, and corrects until the +photodiode reports `a₀`. + +## The workflow, one frequency at a time + +Everything up to the references is unchanged and stays the operator's: freeze the +flux point and camera configuration, reuse the same film position, ROI/mask, +optical pedestal, bias set, gates and reference epoch as the minimum-depth +measurement, keep ON and OFF separate, and per frequency record the full-extinction +`I_tot` anchor, the zero-depth background and the high non-saturating pilot (the +existing **Record pilot** / **Record background** buttons; background reuse +across frequencies is not automated, i.e. off by default). Then: + +1. Set the frequency `f` in the modulation plugin (yours — the drive is armed + there, A1 only reads it). +2. Enter **a₀** once for the whole sweep, and press **Find a₀**. A1 leases the + modulation owner and trims the commanded depth until the photodiode measures + `a₀` at *this* frequency. Nothing is recorded; the drive is left at the depth + it found and the result is stored for `f`. +3. Press **Record a₀ point (event-count)**. A1 re-applies the found depth under a + modulation lease, waits for the measured `a` to hold `a₀`, and records one + atomic camera RAW + photodiode PDQ + sidecar under one run id. +4. Repeat for the next frequency. Randomising the frequency order, interleaving + the low-frequency reference and repeating independent blocks (three where + practical) are yours — every point is one button press. + +## Controls + +| Control | Meaning | +|---|---| +| a₀ (measured log contrast) | the one photodiode-measured depth held across the whole frequency sweep | +| a₀ tolerance (absolute) | convergence band on `|measured a − a₀|`; also the settle band an event-count point must hold before it records (default ±0.02) | +| Find a₀ (lock the drive depth) | closed-loop trim of the commanded depth at the current frequency; records nothing, stores the result, leaves the drive there | +| Record a₀ point (event-count) | re-applies the locked depth under the lease and records one atomic frequency point (`…_ec_fHz`) | +| Clear a₀ lock table | drops every stored lock and rewrites `a0_locks.json` | +| Stop (abort recording / sweep) | also aborts a running lock | + +The **Sweep settle (s)** value in the Recording section is reused as the +per-trial dwell before the lock starts averaging. + +## The lock loop + +```math +a_\text{cmd} \leftarrow a_\text{cmd}\cdot\frac{a_0}{a_\text{measured}} +``` + +- Starts from an earlier lock at the same frequency when one exists, otherwise + from `a₀` itself (the calibrated open-loop guess). +- Converges when `|measured − a₀| ≤ tolerance`; at most **8 trials**, each + correction capped at ×2/÷2 and clamped to the owner's `0.01..=6.0`. +- Per trial it waits the settle dwell, then averages **three fresh** photodiode + optical summaries (one per new `service_revision`, so a slow publisher is not + averaged once per control tick); it evaluates early with fewer readings only if + the 30 s measurement deadline hits first. +- Ends with the owner's own wording when a commanded depth is **rejected** (lobe + ceiling, DAC limit) — that is the "`a₀` is unreachable at this operating point, + lower `a₀` or `I_k`" answer — or reports the drivable limit when the correction + rails at `0.01`/`6.0`. +- Photodiode clipping above 1 % is called out in the result message and stored + with the lock: a clipped window makes the measured `a` a truncated estimate. +- Releases the lease with `safe_off = false`, so the drive holds the found depth. + +## The lock table + +One row per frequency (a re-lock within 1 % of a stored frequency replaces it): +frequency, target `a₀`, commanded `a`, measured `a`, trials, state, locked-at. +Visible as the **A1 a₀ locks** host view and mirrored to +`/a0_locks.json`, so the found depths survive a restart and can be +cited offline. A non-converged row is kept for the record but **never** arms a +recording; a stored lock only arms an event-count point when both its frequency +**and** its `a₀` still match the current settings. + +## Why recording re-applies the depth + +*Record a₀ point* does not simply record at whatever the drive currently is. It +runs the ADR 010 sweep machinery as a one-point sweep of a new kind — lease → +command the locked depth → confirm the measured `a` holds `a₀` → record → release +— which buys three things: + +- the depth is **re-asserted**, so an intervening modulation settings sync (which + re-applies the operator's own `depth a`) cannot silently spoil the point; +- the lease **locks the operator's modulation settings out** for the whole point, + so *"never change amplitude during the recorded interval"* is enforced rather + than trusted; +- it stays one button press. + +Internally a sweep point is now a pair: the depth the drive is **commanded** to +and the depth it is **expected to measure**. The amplitude sweep sets both equal; +an event-count point deliberately does not, and the difference is the roll-off +the lock absorbed. + +## Naming and sidecar + +Event-count points use the role suffix `_ec` and carry the **frequency** in the +stem instead of a sweep-point index — one measurement id spans the whole +frequency sweep at the single frozen depth: + +- `/__ec_f50Hz.raw` (+ the host's own `.toml`) +- `/__ec_f50Hz_pd.pdq` + `_pd.json` +- `/__ec_f50Hz_config.toml` + +Sub-hertz frequencies keep the decimal as `p` (`f0p5Hz`). The A1 sidecar adds +`sweep.commanded_a` and an `[a0_lock]` section (`target_a`, `commanded_a`, +`measured_a_at_lock`, `frequency_hz_at_lock`, `trials`, `converged`, +`locked_at_utc`); both recorders' own sidecars carry `a0_target`, +`a0_commanded_a`, `a0_lock_measured_a` and `a0_lock_frequency_hz` as metadata. +The measured `a` of the recording itself stays in `[optical]` as for every run. + +## Choosing `a₀` (still an operator decision) + +No numerical `a₀` is frozen in this repository — the plugin default is a +placeholder. Pick it from the low-frequency scout so that + +- the low-frequency response gives **several** events, not the one-event floor; +- the event count is still **proportional** to depth and has not saturated; +- the refractory condition `2 f a₀/C ≪ 1/τ_refr` holds at the **highest** + frequency (checked once by you when picking `a₀`; the plugin does not test it); +- the same measured `a₀` is **reachable at every frequency** in the sweep — the + lock reports when it is not, before any data is recorded. + +Today's low-frequency `a50` result is a sensible starting point; targeting +several plateau events per pixel per half-cycle is a good scout criterion. + +## Tests + +`cargo test -p augur-plugin-stage-a-a1` covers the lock converging against a +simulated 60 %-gain bench (and leaving the drive at the found depth with a +`safe_off = false` release), the unreachable-depth case railing at the drive +limit without arming a recording, an owner rejection surfacing verbatim, an +event-count point commanding the **locked** depth rather than `a₀`, the +`_ec_fHz` stem and `[a0_lock]` sidecar section, file-safe frequency tags, and +the one-row-per-frequency lock table round-tripping through `a0_locks.json`. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index edb0405..dec9534 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -1,11 +1,15 @@ # Stage-A A1 Analysis - **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) -- **Status:** Recording coordinator + live quicklooks + amplitude sweep +- **Status:** Recording coordinator + live quicklooks + amplitude sweep + `a₀` lock - **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button - press forwarding) + press forwarding), + [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (exact-event-count + `a₀` lock) - **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) +- **Second workflow:** [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) + — hold one *measured* depth `a₀` across the frequency sweep ## Purpose @@ -43,6 +47,7 @@ folder. A1 makes each recording one button press: | Record pilot | records a bright reference (`…_pilot`) **and** freezes the ON/OFF windows for the row from the live signal | | Record background | records an unmodulated reference (`…_background`) **and** captures the false-response floor `q0` | | Stop (abort recording / sweep) | finalize the current recording early; during a sweep also aborts the remaining points | +| a₀ / Find a₀ / Record a₀ point | the **exact-event-count** workflow: hold one *measured* depth `a₀` across the frequency sweep — see [its brief](./stage-a-a1-event-count.md) | The record and sweep buttons stay **disabled until an output folder is selected**. @@ -63,10 +68,12 @@ require `min a > 0` — record `a≈0` with the background button instead. Sidec of sweep recordings additionally carry `sweep.requested_a`, `sweep.point_index` and `sweep.point_total`. After the sweep releases the lease, the drive holds the last sweep amplitude until the operator's own `depth a` setting is re-applied -(any modulation settings change re-sends it). +(any modulation settings change re-sends it) — which is exactly why an +event-count point re-applies its locked depth under the lease instead of trusting +the drive to still be where a previous action left it (ADR 013). **Naming.** Files share an `_[_role]` stem under an `/` subfolder -(`_pilot` / `_background` tag the reference runs): +(`_pilot` / `_background` tag the reference runs, `_ec_fHz` an event-count point): - `/_.raw` — camera RAW, under the **host output root**, with the host's own `.toml` sidecar (camera biases, ROI) written next to it. @@ -206,5 +213,6 @@ path, file-safe id generation, UTC timestamp formatting, the config-sidecar buil the pilot-window round-trip through the measurement folder, press-latch edge/baseline semantics, the jittery-marker free-running fallback, sweep-point spacing, the sweep-point sidecar fields, the ordered camera → PDQ → PDQ finalize → camera -finalize lifecycle (including envelope identity/revision and save location), and -the selective discontinuity reset. +finalize lifecycle (including envelope identity/revision and save location), the +selective discontinuity reset, and the `a₀`-lock set listed in the +[exact-event-count brief](./stage-a-a1-event-count.md). diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 50d98a6..4ab0638 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -24,6 +24,28 @@ optical drive in the modulation plugin; A1 only reads its published settings. one normal recording. Sidecars carry `sweep.requested_a` / `point_index` / `point_total`. - The record/sweep buttons are disabled until an output folder is selected. +## Exact event count (`a₀` lock) + +The second Stage-A workflow holds **one** photodiode-measured depth +`a₀ = ln(I_exc,max / I_exc,min)` constant while the frequency varies. Because the measured Pockels +inversion is static, the delivered depth rolls off with frequency — so the depth must be found by +measurement, not calculated. + +- **a₀** / **a₀ tolerance** — the frozen measured depth and its convergence band (default ±0.02). +- **Find a₀** — per frequency: leases the modulation owner and iterates + `commanded a ← commanded a · a₀/measured a` (≤ 8 trials, averaging three fresh photodiode summaries + per trial after **Sweep settle (s)**) until the photodiode measures `a₀`. Records nothing, leaves the + drive at the depth it found, and stores one row per frequency — shown in the **A1 a₀ locks** view and + mirrored to `a0_locks.json`. An unreachable `a₀` is reported (drive limit or the owner's own + rejection) before any data is recorded. +- **Record a₀ point (event-count)** — re-applies the locked depth under the lease (so the amplitude + cannot change during the recorded interval), waits for the measured `a` to hold `a₀`, and records + one atomic frequency point named `…_ec_fHz` with an `[a0_lock]` sidecar section. +- **Clear a₀ lock table** — after changing the flux point, the calibration or `a₀` itself. + +Frequency order, the interleaved low-frequency reference and the repeated blocks stay yours — every +point is one button press. + Files share an `_` stem: `/_.raw` (camera, under the host output root), `/__pd.pdq` + `.json` (photodiode, under its data root), and `/__config.toml` (A1, under the chosen folder). Point all three roots at the same @@ -48,6 +70,8 @@ the frequency), falling back to the modulation plugin's acknowledged waveform. T pixels come from the augur-rs camera config. See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the full brief, -[ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, and +[ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, +[docs/features/stage-a-a1-event-count.md](../../docs/features/stage-a-a1-event-count.md) plus +[ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, and [docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the planned amplitude-sweep automation on top of this. diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 64ca65b..e32636e 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -14,7 +14,12 @@ //! settings into the sidecar. The **amplitude sweep** (ADR 010) is the one scoped //! exception: per sweep point it retargets the armed drive's *depth* through the //! leased modulation service (`SetOpticalDepth`), waits for the photodiode-measured -//! `a` to settle, and records the point through the same coordinator. +//! `a` to settle, and records the point through the same coordinator. The +//! **exact-event-count workflow** (ADR 013) reuses that path the other way round: +//! the `a₀` **lock** trims the *commanded* depth closed-loop until the photodiode +//! *measures* the one frozen depth `a₀`, and an **event-count point** replays that +//! trimmed depth under the same lease so one atomic frequency point is recorded at +//! exactly `a₀`. //! //! 2. **Live sanity quicklooks.** Folding the camera event stream on the modulation //! period `T` (defined by the firmware phase-0 `EXT_TRIGGER`), it renders the @@ -66,6 +71,8 @@ const ROLLING_DATASET_ID: &str = "stage-a-a1.rolling-response"; const ROLLING_VIEW_ID: &str = "stage-a-a1.rolling-response.view"; const RESPONSE_CURVE_DATASET_ID: &str = "stage-a-a1.response-curve"; const RESPONSE_CURVE_VIEW_ID: &str = "stage-a-a1.response-curve.view"; +const A0_LOCK_DATASET_ID: &str = "stage-a-a1.a0-locks"; +const A0_LOCK_VIEW_ID: &str = "stage-a-a1.a0-locks.view"; /// Camera events retained for the live fold. At the bench event rates this is a /// few seconds of history and keeps the fold cost bounded. @@ -85,11 +92,84 @@ const MAX_MARKERS: usize = 65_536; /// after this long and record anyway (the sidecar stores the measured value). const SWEEP_SETTLE_TIMEOUT_MS: u64 = 30_000; +/// Closed-loop trials the `a₀` lock spends on one frequency before it gives up +/// and reports the best commanded depth it reached. +const A0_LOCK_MAX_TRIALS: u32 = 8; +/// Per-trial cap on the multiplicative correction of the commanded depth, so one +/// noisy photodiode reading cannot slam the drive across its whole range. +const A0_LOCK_MAX_STEP_RATIO: f64 = 2.0; +/// Independent photodiode readings taken per trial (fewer only when the +/// measurement deadline hits first). Their *median* is the trial's value and +/// their spread is the stability check — one estimator window already averages +/// many cycles, so repeating it is about catching drift, not reducing noise. +const A0_LOCK_SAMPLES: usize = 3; +/// Fraction of one estimator window that must pass between two readings for +/// them to count as independent. Consecutive `service_revision`s share almost +/// their whole window, so sampling per revision alone measures the publisher's +/// tick rate rather than the drive. +const A0_LOCK_SAMPLE_SPACING: f64 = 0.5; +/// Spread across a trial's readings, relative to its tolerance, above which the +/// operating point is called unstable instead of locked. A drifting `a` that +/// happens to cross the target on one reading is not a lock. +const A0_LOCK_MAX_SPREAD_TOLERANCES: f64 = 2.0; +/// Clipping fraction above which a lock's measured `a` is called out as +/// unreliable in the operator message. +/// +/// Deliberately far below the estimator's own `MAX_CLIP_FRACTION` (1 ‰, above +/// which it withholds `a` altogether): a threshold at or above that one could +/// never fire, because a published summary has already passed it. +const A0_LOCK_CLIP_WARNING: f64 = 0.000_2; +/// Closed range of commanded optical depths the modulation owner accepts. +const COMMANDED_A_MIN: f64 = 0.01; +const COMMANDED_A_MAX: f64 = 6.0; +/// Relative distance within which two frequencies are the same sweep point. +const FREQUENCY_MATCH_FRACTION: f64 = 0.01; +/// Lock table persisted in the output folder, so found depths survive a restart. +const A0_LOCK_FILE: &str = "a0_locks.json"; + /// Absolute/relative tolerance for "the measured `a` reached the sweep target". fn sweep_tolerance(target_a: f64) -> f64 { (target_a * 0.10).max(0.05) } +/// Commanded optical depth clamped to what the modulation owner accepts. +fn clamp_commanded_a(depth_a: f64) -> f64 { + if depth_a.is_finite() { + depth_a.clamp(COMMANDED_A_MIN, COMMANDED_A_MAX) + } else { + COMMANDED_A_MIN + } +} + +/// Wire encoding of a commanded optical depth for `SetOpticalDepth`. +fn depth_a_milli(depth_a: f64) -> u32 { + (depth_a * 1_000.0).round().clamp(0.0, u32::MAX as f64) as u32 +} + +/// Whether two frequencies name the same sweep point (drive vs trigger readback +/// never agree to the last digit). +fn same_frequency(left: f64, right: f64) -> bool { + let scale = left.abs().max(right.abs()); + (left - right).abs() <= (scale * FREQUENCY_MATCH_FRACTION).max(1e-6) +} + +fn frequency_label(hz: f64) -> String { + format!("{hz:.3} Hz") +} + +/// Compact file-safe frequency tag for an event-count point's stem: +/// `50 Hz → f50Hz`, `0.5 Hz → f0p5Hz`. +fn frequency_tag(hz: f64) -> String { + let mut text = format!("{hz:.3}"); + while text.ends_with('0') { + text.pop(); + } + if text.ends_with('.') { + text.pop(); + } + format!("f{}Hz", text.replace('.', "p")) +} + trait RecordingControl { fn request_service(&mut self, request: &PluginServiceRequest); fn request_host(&mut self, request: &HostCommandRequest); @@ -177,6 +257,9 @@ enum RecRole { Pilot, /// Unmodulated (`a≈0`) reference that gives the false-response floor. Background, + /// One atomic frequency point of the exact-event-count workflow, recorded at + /// the one frozen depth `a₀` the lock found for that frequency. + EventCount, } impl RecRole { @@ -186,6 +269,7 @@ impl RecRole { RecRole::Normal => "", RecRole::Pilot => "_pilot", RecRole::Background => "_background", + RecRole::EventCount => "_ec", } } @@ -194,6 +278,7 @@ impl RecRole { RecRole::Normal => "point", RecRole::Pilot => "pilot", RecRole::Background => "background", + RecRole::EventCount => "event-count point", } } } @@ -249,13 +334,47 @@ enum SweepPhase { Recording, } +/// What a leased sweep is for: the amplitude sweep of one `(I_k, f)` row, or one +/// atomic frequency point of the exact-event-count workflow. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SweepKind { + Amplitude, + EventCount, +} + +impl SweepKind { + fn role(self) -> RecRole { + match self { + SweepKind::Amplitude => RecRole::Normal, + SweepKind::EventCount => RecRole::EventCount, + } + } +} + +/// One sweep point: what the drive is *commanded* to, and the +/// photodiode-measured `a` that point is supposed to produce. +/// +/// The amplitude sweep asks for its own value open-loop, trusting the Pockels +/// calibration, so both are equal. An event-count point replays a commanded +/// depth the `a₀` lock already trimmed closed-loop against the *measured* depth, +/// so there its commanded depth is deliberately **not** the depth it expects to +/// measure — that difference is the drive roll-off the lock absorbed. +#[derive(Debug, Clone, Copy, PartialEq)] +struct SweepPoint { + commanded_a: f64, + expected_a: f64, +} + /// One "record every point of the amplitude range" run: per point the sweep /// retargets the leased modulation drive, waits for the photodiode-measured /// `a` to settle, and hands off to the normal recording coordinator. struct Sweep { phase: SweepPhase, - /// Requested `a` per point, ascending over `[min_a, max_a]`. - points: Vec, + kind: SweepKind, + /// The points to record, in order. + points: Vec, + /// The `a₀` lock an event-count point replays; `None` for the amplitude sweep. + lock: Option, index: usize, lease_id: LeaseId, lease_granted: bool, @@ -274,8 +393,21 @@ struct Sweep { } impl Sweep { + fn point(&self) -> SweepPoint { + self.points.get(self.index).copied().unwrap_or(SweepPoint { + commanded_a: 0.0, + expected_a: 0.0, + }) + } + + /// The photodiode-measured `a` this point must settle at. fn target_a(&self) -> f64 { - self.points.get(self.index).copied().unwrap_or(0.0) + self.point().expected_a + } + + /// The depth the drive is commanded to for this point. + fn commanded_a(&self) -> f64 { + self.point().commanded_a } fn total(&self) -> usize { @@ -283,6 +415,81 @@ impl Sweep { } } +/// Where the `a₀` lock is within its current closed-loop trial. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum A0LockPhase { + /// AcquireLease sent to the modulation owner; waiting for the grant. + AcquiringLease, + /// SetOpticalDepth for the current trial sent; waiting for Applied. + SettingDepth, + /// Settling, then averaging fresh photodiode readings for this trial. + Measuring, +} + +/// One "find the commanded depth that makes the photodiode measure `a₀` at this +/// frequency" run. Iterates `commanded ← commanded · a₀/measured` under a +/// modulation lease and never records anything itself. +struct A0Lock { + phase: A0LockPhase, + /// The photodiode-measured log contrast the operator froze for the sweep. + target_a: f64, + /// Convergence band on `|measured − target|`. + tolerance: f64, + /// The depth the current trial commands. + commanded_a: f64, + /// Frequency this lock belongs to, captured when it started. + frequency_hz: f64, + /// 1-based trial counter, bounded by `A0_LOCK_MAX_TRIALS`. + trial: u32, + /// Independent photodiode readings collected for the current trial. + samples: Vec, + /// `service_revision` of the newest photodiode summary already sampled, so a + /// slow publisher is not sampled once per control tick. + sampled_revision: Option, + /// Earliest instant the next reading may be taken: the settle dwell before + /// the first, then one sample spacing after each. + measure_from_ms: u64, + /// Estimator window length (ms) the photodiode reported when this trial + /// commanded its depth. Both the dwell and the sample spacing derive from + /// it, because a reading taken sooner still contains the previous depth. + window_ms: u64, + /// Give-up deadline for the current trial's measurement. + deadline_ms: u64, + lease_id: LeaseId, + lease_granted: bool, + lease_req: u64, + depth_req: u64, + depth_applied: bool, + last_activity_ms: u64, + stop_requested: bool, +} + +/// The result of one lock: the commanded depth that produced the frozen `a₀` at +/// one frequency. Persisted in `a0_locks.json` and replayed by event-count points. +#[derive(Debug, Clone, Serialize, serde::Deserialize)] +struct A0LockPoint { + frequency_hz: f64, + /// The frozen `a₀` the lock aimed at. + target_a: f64, + /// What the drive must be commanded to in order to *measure* `target_a`. + commanded_a: f64, + /// The photodiode-measured `a` averaged over the final trial. + measured_a: f64, + trials: u32, + /// False when the lock ran out of trials or hit a drive limit; such a row is + /// kept for the record but never arms an event-count recording. + converged: bool, + locked_at_unix_ms: u64, + low_clip_fraction: Option, + high_clip_fraction: Option, +} + +/// On-disk form of the per-frequency lock table. +#[derive(Debug, Clone, Default, Serialize, serde::Deserialize)] +struct A0LockTable { + locks: Vec, +} + pub struct StageAA1Plugin { enabled: bool, runtime_role: PluginRuntimeRole, @@ -346,6 +553,23 @@ pub struct StageAA1Plugin { /// Latched by the Start sweep button, consumed next control tick. sweep_pending: bool, sweep: Option, + // -- exact event-count depth a₀ (ADR 013) -- + /// The one photodiode-measured log contrast held across the frequency sweep. + a0_target: f64, + /// Convergence band on `|measured a − a₀|` for the lock and for an + /// event-count point's settle check. + a0_tolerance: f64, + /// Latched by the Find a₀ button, consumed next control tick. + a0_lock_pending: bool, + /// Latched by the Record a₀ point button, consumed next control tick. + a0_point_pending: bool, + a0_lock: Option, + /// One converged (or attempted) lock per frequency, newest per frequency + /// wins; mirrored to `a0_locks.json` in the output folder. + a0_locks: Vec, + /// Output folder the lock table was last read for, so it is re-read only + /// when the experiment folder changes. + loaded_locks_folder: Option, // -- momentary-button press forwarding (see PressLatch) -- press_start: PressLatch, press_pilot: PressLatch, @@ -355,6 +579,9 @@ pub struct StageAA1Plugin { press_clear: PressLatch, press_record_point: PressLatch, press_clear_curve: PressLatch, + press_find_a0: PressLatch, + press_record_a0: PressLatch, + press_clear_a0: PressLatch, } impl Default for StageAA1Plugin { @@ -395,6 +622,15 @@ impl Default for StageAA1Plugin { settle_s: 2.0, sweep_pending: false, sweep: None, + // No numerical a₀ is frozen in the repository: this default is a + // placeholder the operator replaces with the scout result. + a0_target: 0.5, + a0_tolerance: 0.02, + a0_lock_pending: false, + a0_point_pending: false, + a0_lock: None, + a0_locks: Vec::new(), + loaded_locks_folder: None, press_start: PressLatch::default(), press_pilot: PressLatch::default(), press_background: PressLatch::default(), @@ -403,6 +639,9 @@ impl Default for StageAA1Plugin { press_clear: PressLatch::default(), press_record_point: PressLatch::default(), press_clear_curve: PressLatch::default(), + press_find_a0: PressLatch::default(), + press_record_a0: PressLatch::default(), + press_clear_a0: PressLatch::default(), } } } @@ -506,6 +745,11 @@ impl StageAA1Plugin { (hz > 0.0).then(|| 1_000_000.0 / hz) } + /// Modulation frequency implied by [`Self::period_us`]. + fn frequency_hz(&self) -> Option { + self.period_us().map(|period| 1_000_000.0 / period) + } + /// Modulation period measured from the phase-0 markers (mean spacing). fn measured_period_us(&self) -> Option { if self.camera_markers_us.len() < 2 { @@ -996,9 +1240,25 @@ impl StageAA1Plugin { "sweep_requested_a".into(), format!("{:.6}", sweep.target_a()), ); + meta.insert( + "sweep_commanded_a".into(), + format!("{:.6}", sweep.commanded_a()), + ); meta.insert("sweep_point_index".into(), (sweep.index + 1).to_string()); meta.insert("sweep_point_total".into(), sweep.total().to_string()); } + if let Some(lock) = self.sweep.as_ref().and_then(|sweep| sweep.lock.as_ref()) { + meta.insert("a0_target".into(), format!("{:.6}", lock.target_a)); + meta.insert("a0_commanded_a".into(), format!("{:.6}", lock.commanded_a)); + meta.insert( + "a0_lock_measured_a".into(), + format!("{:.6}", lock.measured_a), + ); + meta.insert( + "a0_lock_frequency_hz".into(), + format!("{:.6}", lock.frequency_hz), + ); + } if let Some(a) = self.measured_a() { meta.insert("measured_a".into(), format!("{a:.6}")); } @@ -1037,12 +1297,26 @@ impl StageAA1Plugin { let now_ms = now_unix_ms(); let id = sanitize_stem(self.measurement_id.trim()); // Sweep points get a stable per-point tag so the row's files sort by - // sweep order as well as by timestamp. + // sweep order as well as by timestamp. Event-count points instead carry + // their frequency, because one measurement id spans the whole frequency + // sweep at the single frozen depth a₀. + let live_hz = self.frequency_hz(); let sweep_tag = self .sweep .as_ref() .filter(|sweep| sweep.phase == SweepPhase::Recording) - .map(|sweep| format!("_p{:02}", sweep.index + 1)) + .map(|sweep| match sweep.kind { + SweepKind::Amplitude => format!("_p{:02}", sweep.index + 1), + SweepKind::EventCount => { + let hz = sweep + .lock + .as_ref() + .map(|lock| lock.frequency_hz) + .or(live_hz) + .unwrap_or_default(); + format!("_{}", frequency_tag(hz)) + } + }) .unwrap_or_default(); let stem = format!( "{id}_{}{}{sweep_tag}", @@ -1069,7 +1343,8 @@ impl StageAA1Plugin { match role { RecRole::Pilot => self.freeze_pilot_windows(), RecRole::Background => self.capture_background_floor(), - RecRole::Normal => {} + // Both keep the row's pilot-frozen windows and background floor. + RecRole::Normal | RecRole::EventCount => {} } self.start_camera(context); @@ -1283,11 +1558,19 @@ impl StageAA1Plugin { } /// The requested `a` per sweep point, ascending and inclusive of both ends. - fn sweep_points(&self) -> Vec { + /// The amplitude sweep trusts the calibration, so each point commands the + /// very depth it expects to measure. + fn sweep_points(&self) -> Vec { let count = self.sweep_count.clamp(2, 64) as usize; let span = self.max_a - self.min_a; (0..count) - .map(|index| self.min_a + span * index as f64 / (count - 1) as f64) + .map(|index| { + let depth_a = self.min_a + span * index as f64 / (count - 1) as f64; + SweepPoint { + commanded_a: depth_a, + expected_a: depth_a, + } + }) .collect() } @@ -1303,44 +1586,67 @@ impl StageAA1Plugin { } /// Kick off the amplitude sweep: validate, then lease the modulation owner. - fn begin_sweep(&mut self, context: &mut PluginControlContext<'_>) { - if self.recording.is_active() || self.sweep.is_some() { - self.message = "A recording or sweep is already running".into(); + fn begin_sweep(&mut self, context: &mut impl RecordingControl) { + if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { + self.message = + "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); + return; + } + if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { + self.message = "Sweep needs max a > min a".into(); + return; + } + let points = self.sweep_points(); + let message = format!( + "Sweep: acquiring modulation lease for {} points…", + points.len() + ); + self.begin_leased_sweep(context, SweepKind::Amplitude, points, None, message); + } + + /// Shared entry point for both leased recording runs (amplitude sweep and + /// single event-count point): validate the destination and the owner, then + /// acquire the modulation lease that holds the drive for the whole run. + fn begin_leased_sweep( + &mut self, + context: &mut impl RecordingControl, + kind: SweepKind, + points: Vec, + lock: Option, + message: String, + ) { + if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { + self.message = "A recording, sweep or a₀ lock is already running".into(); return; } if self.output_folder.trim().is_empty() { - self.message = "Set an output folder before sweeping".into(); + self.message = "Set an output folder before recording".into(); return; } if self.measurement_id.trim().is_empty() { - self.message = "Set a measurement id before sweeping".into(); + self.message = "Set a measurement id before recording".into(); return; } if !self.modulation_connected() { - self.message = "Modulation owner is not connected — cannot sweep".into(); + self.message = "Modulation owner is not connected — cannot drive the depth".into(); return; } - if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { - self.message = - "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); - return; - } - if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { - self.message = "Sweep needs max a > min a".into(); + if points.is_empty() { + self.message = "Nothing to record: the run has no points".into(); return; } - let points = self.sweep_points(); let now_ms = now_unix_ms(); let lease_id = LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))); let ttl_ms = self.sweep_lease_ttl_ms(points.len()); let request = self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); let lease_req = request.request_id; - let _ = context.request_service(&request); - let total = points.len(); + context.request_service(&request); self.sweep = Some(Sweep { phase: SweepPhase::AcquiringLease, + kind, points, + lock, index: 0, lease_id, lease_granted: false, @@ -1353,11 +1659,42 @@ impl StageAA1Plugin { last_activity_ms: now_ms, stop_requested: false, }); - self.message = format!("Sweep: acquiring modulation lease for {total} points…"); + self.message = message; + } + + /// Record one atomic frequency point of the exact-event-count workflow. + /// + /// The armed lock's commanded depth is re-applied under a modulation lease — + /// which also locks the operator's drive settings out for the whole point, so + /// the amplitude provably cannot change during the recorded interval — and the + /// point is then recorded through the same coordinator as every other run. + fn begin_a0_point(&mut self, context: &mut impl RecordingControl) { + let Some(hz) = self.frequency_hz() else { + self.message = "No modulation frequency yet — arm the drive first".into(); + return; + }; + let Some(lock) = self.armed_lock().cloned() else { + self.message = format!( + "No converged a₀ lock for {} — press Find a₀ at this frequency first", + frequency_label(hz) + ); + return; + }; + let points = vec![SweepPoint { + commanded_a: lock.commanded_a, + expected_a: lock.target_a, + }]; + let message = format!( + "Event-count point at {}: leasing the drive at commanded a = {:.3} (a₀ = {:.3})…", + frequency_label(lock.frequency_hz), + lock.commanded_a, + lock.target_a + ); + self.begin_leased_sweep(context, SweepKind::EventCount, points, Some(lock), message); } /// Release the modulation lease (if held) and clear the sweep. - fn finish_sweep(&mut self, context: &mut PluginControlContext<'_>, message: String) { + fn finish_sweep(&mut self, context: &mut impl RecordingControl, message: String) { if let Some(sweep) = self.sweep.take() { if sweep.lease_granted { let request = self.modulation_request( @@ -1367,35 +1704,36 @@ impl StageAA1Plugin { }, &sweep.lease_id, ); - let _ = context.request_service(&request); + context.request_service(&request); } } self.message = message; } /// Renew the modulation lease and retarget the drive at the current point. - fn send_sweep_depth(&mut self, context: &mut PluginControlContext<'_>) { + fn send_sweep_depth(&mut self, context: &mut impl RecordingControl) { let Some(sweep) = self.sweep.as_ref() else { return; }; let lease_id = sweep.lease_id.clone(); let remaining = sweep.total().saturating_sub(sweep.index); + let commanded_a = sweep.commanded_a(); let target_a = sweep.target_a(); let index = sweep.index; let total = sweep.total(); let ttl_ms = self.sweep_lease_ttl_ms(remaining); let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); - let _ = context.request_service(&renew); + context.request_service(&renew); let depth = self.modulation_request( ModulationCommandV1::SetOpticalDepth { - depth_a_milli: (target_a * 1_000.0).round().clamp(0.0, u32::MAX as f64) as u32, + depth_a_milli: depth_a_milli(commanded_a), }, &lease_id, ); let depth_req = depth.request_id; - let _ = context.request_service(&depth); + context.request_service(&depth); let now_ms = now_unix_ms(); if let Some(sweep) = self.sweep.as_mut() { @@ -1406,29 +1744,46 @@ impl StageAA1Plugin { sweep.point_started = false; sweep.last_activity_ms = now_ms; } - self.message = format!( - "Sweep point {}/{}: retargeting drive to a = {:.3}…", - index + 1, - total, - target_a - ); + self.message = if commanded_a == target_a { + format!( + "Sweep point {}/{total}: retargeting drive to a = {target_a:.3}…", + index + 1 + ) + } else { + format!( + "Event-count point: commanding a = {commanded_a:.3} for a measured a₀ = {target_a:.3}…" + ) + }; } /// Advance the amplitude sweep one control tick. Runs before /// `drive_recording`, so a point's recording starts on the same tick. - fn drive_sweep(&mut self, context: &mut PluginControlContext<'_>) { + fn drive_sweep(&mut self, context: &mut impl RecordingControl) { if self.sweep.is_none() { if std::mem::take(&mut self.sweep_pending) { self.begin_sweep(context); + } else if std::mem::take(&mut self.a0_point_pending) { + self.begin_a0_point(context); } return; } self.sweep_pending = false; + self.a0_point_pending = false; let now_ms = now_unix_ms(); - let (phase, stop_requested, lease_granted, depth_applied, last_activity_ms, index, total) = { + let ( + phase, + kind, + stop_requested, + lease_granted, + depth_applied, + last_activity_ms, + index, + total, + ) = { let sweep = self.sweep.as_ref().expect("sweep checked above"); ( sweep.phase, + sweep.kind, sweep.stop_requested, sweep.lease_granted, sweep.depth_applied, @@ -1483,9 +1838,16 @@ impl StageAA1Plugin { } SweepPhase::Settling => { let target = self.sweep.as_ref().map(Sweep::target_a).unwrap_or_default(); + // The amplitude sweep drives open-loop and accepts the coarse + // calibration band; an event-count point replays a depth that was + // already trimmed against `a₀`, so it holds the lock's band. + let tolerance = match kind { + SweepKind::Amplitude => sweep_tolerance(target), + SweepKind::EventCount => self.a0_tolerance.max(1e-3), + }; let settled = self .measured_a() - .is_some_and(|measured| (measured - target).abs() <= sweep_tolerance(target)); + .is_some_and(|measured| (measured - target).abs() <= tolerance); let dwell_ms = (self.settle_s.max(0.0) * 1_000.0) as u64; let mut start_recording = false; let mut settle_timed_out = false; @@ -1509,10 +1871,14 @@ impl StageAA1Plugin { } } if start_recording { - self.pending_role = Some(RecRole::Normal); + self.pending_role = Some(kind.role()); if settle_timed_out { + let measured = self + .measured_a() + .map_or_else(|| "—".into(), |value| format!("{value:.3}")); self.message = format!( - "Sweep point {}/{}: a did not settle at {target:.3} — recording anyway", + "Sweep point {}/{}: a did not settle at {target:.3} (measured {measured}) \ + — recording anyway", index + 1, total, ); @@ -1541,7 +1907,11 @@ impl StageAA1Plugin { let message = format!("Sweep aborted: {}", self.message); self.finish_sweep(context, message); } else if index + 1 >= total { - self.finish_sweep(context, format!("Sweep complete: {total} points recorded")); + let message = match kind { + SweepKind::Amplitude => format!("Sweep complete: {total} points recorded"), + SweepKind::EventCount => self.message.clone(), + }; + self.finish_sweep(context, message); } else { if let Some(sweep) = self.sweep.as_mut() { sweep.index += 1; @@ -1605,124 +1975,753 @@ impl StageAA1Plugin { } } - fn on_host_reply(&mut self, reply: &HostCommandReply) { - if reply.request_id == self.recording.cam_start_req { - match &reply.outcome { - HostCommandOutcome::RecordingStarted { - actual_raw_path, .. - } => { - self.recording.cam_raw_path = Some(actual_raw_path.clone()); - self.recording.last_activity_ms = now_unix_ms(); - } - HostCommandOutcome::Rejected { code, message } => { - // Stop the rest of the recording; drive_recording resolves the - // abort from the current phase on the next tick. - self.note(format!("Camera recording rejected ({code}): {message}")); - self.recording.cam_rejected = true; - self.recording.stop_requested = true; - } - _ => {} - } - } else if reply.request_id == self.recording.cam_stop_req { - match &reply.outcome { - HostCommandOutcome::RecordingFinalized { - actual_raw_path, .. - } => { - self.recording.cam_finalized_path = Some(actual_raw_path.clone()); - self.recording.cam_complete = true; - self.recording.last_activity_ms = now_unix_ms(); - } - HostCommandOutcome::RecordingPartial { - actual_raw_path, .. - } => { - self.recording.cam_finalized_path = Some(actual_raw_path.clone()); - self.recording.last_activity_ms = now_unix_ms(); - } - HostCommandOutcome::Rejected { code, message } => { - self.message = format!("Camera stop failed ({code}): {message}"); - self.recording.cam_rejected = true; - self.recording.last_activity_ms = now_unix_ms(); + // ---- exact event-count depth a₀ (ADR 013) ------------------------------ + + /// The lock stored for `hz`, whether or not it converged. + fn lock_for_frequency(&self, hz: f64) -> Option<&A0LockPoint> { + self.a0_locks + .iter() + .find(|lock| same_frequency(lock.frequency_hz, hz)) + } + + /// The lock that applies to the drive right now: same frequency, converged, + /// and aimed at the `a₀` currently entered. + fn armed_lock(&self) -> Option<&A0LockPoint> { + let hz = self.frequency_hz()?; + self.lock_for_frequency(hz) + .filter(|lock| lock.converged && (lock.target_a - self.a0_target).abs() <= 1e-6) + } + + fn a0_locks_path(&self) -> Option { + let folder = self.output_folder.trim(); + (!folder.is_empty()).then(|| Path::new(folder).join(A0_LOCK_FILE)) + } + + /// Store a finished lock, replacing any earlier one at the same frequency, + /// and mirror the table to disk. Returns a save failure for the caller to + /// append to its own message. + fn store_lock(&mut self, lock: A0LockPoint) -> Result<(), String> { + self.a0_locks + .retain(|existing| !same_frequency(existing.frequency_hz, lock.frequency_hz)); + self.a0_locks.push(lock); + self.a0_locks + .sort_by(|left, right| left.frequency_hz.total_cmp(&right.frequency_hz)); + self.save_a0_locks() + } + + /// Persist the lock table next to the recordings, so the found depths survive + /// a restart and can be cited offline. + /// + /// Returns the failure so the caller can append it to its own message: a + /// lock the operator can see on screen but that never reached disk is a + /// lock they will not have after a restart. + fn save_a0_locks(&mut self) -> Result<(), String> { + let Some(path) = self.a0_locks_path() else { + return Ok(()); + }; + let table = A0LockTable { + locks: self.a0_locks.clone(), + }; + let written = serde_json::to_string_pretty(&table) + .map_err(|error| error.to_string()) + .and_then(|text| { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; } - _ => {} - } - } + std::fs::write(&path, text).map_err(|error| error.to_string()) + }); + written.map_err(|error| format!("a₀ lock table save failed: {error}")) } - fn on_service_reply(&mut self, reply: &PluginServiceReply) { - if self.on_sweep_reply(reply) { + /// Re-read the lock table when the experiment folder changes. + fn load_a0_locks(&mut self) { + let folder = self.output_folder.trim().to_string(); + if self.loaded_locks_folder.as_deref() == Some(folder.as_str()) { return; } - let response = match &reply.outcome { - PluginServiceOutcome::Accepted { payload } => { - serde_json::from_value::(payload.clone()).ok() - } - PluginServiceOutcome::Rejected { code, message } => { - if reply.request_id == self.recording.connect_req - || reply.request_id == self.recording.lease_req - || reply.request_id == self.recording.pd_begin_req - { - self.note(format!("Photodiode start failed ({code}): {message}")); - self.recording.pd_rejected = true; - self.recording.stop_requested = true; - } else if reply.request_id == self.recording.pd_finalize_req { - self.note(format!("Photodiode save failed ({code}): {message}")); - self.recording.pd_rejected = true; - self.recording.lease_granted = false; - self.recording.last_activity_ms = now_unix_ms(); - } - None - } - }; - let Some(response) = response else { + self.loaded_locks_folder = Some(folder); + self.a0_locks.clear(); + let Some(path) = self.a0_locks_path() else { return; }; - if reply.request_id == self.recording.connect_req { - self.recording.connect_accepted = true; - self.recording.last_activity_ms = now_unix_ms(); - } else if reply.request_id == self.recording.lease_req { - self.recording.lease_granted = true; - self.recording.last_activity_ms = now_unix_ms(); - } else if reply.request_id == self.recording.pd_begin_req { - if let Some(PdqReceiptV1::Started(started)) = &response.receipt { - self.recording.pd_pdq_path = Some(started.pdq_path.clone()); - self.recording.pd_sidecar_path = Some(started.sidecar_path.clone()); - self.recording.last_activity_ms = now_unix_ms(); - } - } else if reply.request_id == self.recording.pd_finalize_req { - self.recording.pd_finalized = true; - if let Some(PdqReceiptV1::Finalized(finalized)) = &response.receipt { - self.recording.pd_pdq_path = Some(finalized.pdq_path.clone()); - self.recording.pd_sidecar_path = Some(finalized.sidecar_path.clone()); - self.recording.pd_valid = finalized.valid; - } - self.recording.lease_granted = false; - self.recording.last_activity_ms = now_unix_ms(); + if let Some(table) = std::fs::read_to_string(&path) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + { + self.a0_locks = table.locks; } } - /// Advance the recording state machine one control tick. - fn drive_recording(&mut self, context: &mut impl RecordingControl) { + /// Worst-case lock duration, used as the modulation lease TTL. + fn a0_lock_lease_ttl_ms(&self) -> u64 { + let per_trial_ms = (self.settle_s.max(0.0) * 1_000.0) as u64 + SWEEP_SETTLE_TIMEOUT_MS; + u64::from(A0_LOCK_MAX_TRIALS) + .saturating_mul(per_trial_ms) + .saturating_add(60_000) + } + + /// Kick off the closed-loop `a₀` lock at the current frequency. + fn begin_a0_lock(&mut self, context: &mut impl RecordingControl) { + if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { + self.message = "A recording, sweep or a₀ lock is already running".into(); + return; + } + if !self.modulation_connected() { + self.message = "Modulation owner is not connected — cannot find a₀".into(); + return; + } + // The lock table belongs to the experiment folder, and it is re-read + // whenever that folder changes: without one, a lock found now would be + // dropped the moment the operator picks the destination. + if self.output_folder.trim().is_empty() { + self.message = "Set an output folder before finding a₀".into(); + return; + } + let Some(hz) = self.frequency_hz() else { + self.message = "No modulation frequency yet — arm the drive before finding a₀".into(); + return; + }; + if self.measured_a().is_none() { + self.message = + "No photodiode-measured a — connect the photodiode and anchor I_tot first".into(); + return; + } + // Refuse before touching the drive, not after eight trials of chasing a + // truncated estimate upwards. + if let Err(reason) = self.optical_window_covers_a_cycle(hz) { + self.message = format!("Cannot find a₀ at {}: {reason}", frequency_label(hz)); + return; + } + let target = self.a0_target; + if !(COMMANDED_A_MIN..=COMMANDED_A_MAX).contains(&target) { + self.message = format!( + "a₀ = {target:.3} is outside the drivable {COMMANDED_A_MIN}..={COMMANDED_A_MAX}" + ); + return; + } + // Warm start from an earlier lock at this frequency; otherwise trust the + // Pockels calibration for the first guess (command exactly `a₀`). + let start = self + .lock_for_frequency(hz) + .map(|lock| lock.commanded_a) + .unwrap_or(target); let now_ms = now_unix_ms(); - match self.recording.phase { - RecPhase::Idle => { - if let Some(role) = self.pending_role.take() { - self.begin_recording(context, role); - } - } - RecPhase::StartingCamera => { - if self.recording.cam_rejected { - let message = self.message.clone(); - self.release_and_idle(context, message); - } else if self.recording.cam_raw_path.is_some() { - if self.recording.stop_requested { - self.stop_camera(context); - } else { - self.connect_photodiode(context); - } - } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS - { - self.recording.cam_rejected = true; - self.release_and_idle(context, "Timed out starting camera recording".into()); + let lease_id = LeaseId::new(format!("a1-a0-{}", format_compact_utc(now_ms / 1_000))); + let ttl_ms = self.a0_lock_lease_ttl_ms(); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + let lease_req = request.request_id; + context.request_service(&request); + self.a0_lock = Some(A0Lock { + phase: A0LockPhase::AcquiringLease, + target_a: target, + tolerance: self.a0_tolerance.max(1e-3), + commanded_a: clamp_commanded_a(start), + frequency_hz: hz, + trial: 1, + samples: Vec::new(), + sampled_revision: None, + measure_from_ms: 0, + window_ms: 0, + deadline_ms: 0, + lease_id, + lease_granted: false, + lease_req, + depth_req: 0, + depth_applied: false, + last_activity_ms: now_ms, + stop_requested: false, + }); + self.message = format!( + "a₀ lock at {}: acquiring the modulation lease…", + frequency_label(hz) + ); + } + + /// Renew the lease and command the current trial's depth. + fn send_a0_depth(&mut self, context: &mut impl RecordingControl) { + let Some(lock) = self.a0_lock.as_ref() else { + return; + }; + let lease_id = lock.lease_id.clone(); + let commanded = lock.commanded_a; + let trial = lock.trial; + let target = lock.target_a; + + let ttl_ms = self.a0_lock_lease_ttl_ms(); + let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + context.request_service(&renew); + let depth = self.modulation_request( + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: depth_a_milli(commanded), + }, + &lease_id, + ); + let depth_req = depth.request_id; + context.request_service(&depth); + + let now_ms = now_unix_ms(); + if let Some(lock) = self.a0_lock.as_mut() { + lock.phase = A0LockPhase::SettingDepth; + lock.depth_req = depth_req; + lock.depth_applied = false; + lock.samples.clear(); + lock.sampled_revision = None; + lock.last_activity_ms = now_ms; + } + self.message = format!( + "a₀ lock trial {trial}/{A0_LOCK_MAX_TRIALS}: commanding a = {commanded:.3} for a \ + measured a₀ = {target:.3}…" + ); + } + + /// Release the modulation lease and clear the lock. + /// + /// Never `safe_off`: the drive must stay exactly where the lock left it, so + /// the event-count point that follows records at `a₀`. + fn finish_a0_lock(&mut self, context: &mut impl RecordingControl, message: String) { + if let Some(lock) = self.a0_lock.take() { + if lock.lease_granted { + let request = self.modulation_request( + ModulationCommandV1::ReleaseLease { + safe_off: false, + reason: "a1 a0 lock finished".into(), + }, + &lock.lease_id, + ); + context.request_service(&request); + } + } + self.message = message; + } + + /// Length of the photodiode's contrast estimator window, in milliseconds. + /// + /// This is the time a commanded depth needs to fully replace the previous + /// one inside the estimate. Owners that predate the field do not publish + /// it; then only the operator's settle dwell is available. + fn optical_window_seconds(&self) -> Option { + self.photodiode + .as_ref()? + .optical_summary + .as_ref()? + .window_seconds + .filter(|seconds| seconds.is_finite() && *seconds > 0.0) + } + + /// [`Self::optical_window_seconds`] rounded up to the millisecond the lock's + /// timers work in. + fn optical_window_ms(&self) -> Option { + self.optical_window_seconds() + .map(|seconds| (seconds * 1_000.0).ceil() as u64) + } + + /// Whether the photodiode's estimator window spans at least one full + /// modulation cycle at `hz`, i.e. whether the published `a` can be a + /// peak-to-peak measurement at all. + /// + /// The owner refuses on its own when its markers can prove the window is + /// too short. It cannot when it has no marker stream — but A1 always knows + /// the frequency, from its own phase-0 triggers or the armed drive, so the + /// check is repeated here where the knowledge is. Getting this wrong is not + /// a small error: a sub-cycle window *under*-reports `a`, and the lock + /// divides by it, so it would drive the depth up until it rails. + fn optical_window_covers_a_cycle(&self, hz: f64) -> Result<(), String> { + let Some(window_seconds) = self.optical_window_seconds() else { + return Ok(()); + }; + let cycles = window_seconds * hz; + if cycles >= 1.0 { + return Ok(()); + } + Err(format!( + "the photodiode estimates a over {window_seconds:.4} s, only {cycles:.2} cycles at \ + {} — a is a peak-to-peak quantity and would be under-reported. Raise the photodiode \ + cache length to at least {:.0} s", + frequency_label(hz), + (2.0 / hz).ceil().max(1.0), + )) + } + + /// Take one reading per *independent* photodiode window. + /// + /// Two constraints, both about the estimator window rather than the + /// publisher: a reading must come from a summary that did not exist when + /// the depth was commanded (`sampled_revision`), and consecutive readings + /// must be at least [`A0_LOCK_SAMPLE_SPACING`] of a window apart — + /// otherwise they share nearly all their samples and three of them say no + /// more than one. + fn sample_a0_measurement(&mut self, now_ms: u64) { + let Some((revision, measured)) = self.photodiode.as_ref().and_then(|summary| { + summary + .optical_summary + .as_ref() + .map(|optical| (summary.service_revision, optical.measured_log_contrast)) + }) else { + return; + }; + let spacing_ms = self.a0_sample_spacing_ms(); + let Some(lock) = self.a0_lock.as_mut() else { + return; + }; + if now_ms < lock.measure_from_ms || lock.sampled_revision == Some(revision) { + return; + } + lock.sampled_revision = Some(revision); + lock.samples.push(measured); + lock.measure_from_ms = now_ms.saturating_add(spacing_ms); + } + + /// Minimum gap between two readings of one trial. + fn a0_sample_spacing_ms(&self) -> u64 { + let window_ms = self + .a0_lock + .as_ref() + .map(|lock| lock.window_ms) + .unwrap_or_default(); + ((window_ms as f64) * A0_LOCK_SAMPLE_SPACING).ceil() as u64 + } + + /// Photodiode clipping note for a lock message, empty when the windows are clean. + fn clip_warning(&self) -> String { + let Some(optical) = self + .photodiode + .as_ref() + .and_then(|summary| summary.optical_summary.as_ref()) + else { + return String::new(); + }; + if optical.low_clip_fraction.max(optical.high_clip_fraction) <= A0_LOCK_CLIP_WARNING { + return String::new(); + } + format!( + " — warning: photodiode clipping (low {:.1} %, high {:.1} %), the measured a is a \ + truncated estimate", + optical.low_clip_fraction * 100.0, + optical.high_clip_fraction * 100.0 + ) + } + + /// Close out one trial: converged, out of trials, at a drive limit, or one + /// more multiplicative correction. + fn evaluate_a0_trial(&mut self, context: &mut impl RecordingControl) { + let Some(lock) = self.a0_lock.as_ref() else { + return; + }; + let (target, tolerance, commanded, trial, hz) = ( + lock.target_a, + lock.tolerance, + lock.commanded_a, + lock.trial, + lock.frequency_hz, + ); + let mut readings = lock.samples.clone(); + if readings.is_empty() { + // The owner withholds `a` for a stated reason (clipping, no + // headroom, a bad `I_tot` anchor, a sub-cycle window). It does not + // publish the reason on the contract, so name the likely ones + // rather than leave the operator with "nothing happened". + self.finish_a0_lock( + context, + "a₀ lock aborted: the photodiode published no a while measuring — it withholds \ + one when the window clips, has no headroom above dark, the I_tot anchor is \ + below the signal, or the window is shorter than one modulation cycle" + .into(), + ); + return; + } + readings.sort_by(f64::total_cmp); + let measured = readings[readings.len() / 2]; + let spread = readings[readings.len() - 1] - readings[0]; + if measured <= 0.0 { + self.finish_a0_lock( + context, + format!( + "a₀ lock aborted: the photodiode measured a = {measured:.3} — check the I_tot \ + anchor and that the drive is modulating" + ), + ); + return; + } + // A drifting `a` that happens to cross the target on one reading is not + // a lock: the next action would record at whatever it drifted to. + if readings.len() > 1 && spread > tolerance * A0_LOCK_MAX_SPREAD_TOLERANCES { + self.finish_a0_lock( + context, + format!( + "a₀ lock aborted at {}: the measured a is not settled — {} readings spread \ + {spread:.3} across {}× the ±{tolerance:.3} tolerance (median {measured:.3}). \ + Increase Sweep settle (s) or check the drive and the I_tot anchor", + frequency_label(hz), + readings.len(), + A0_LOCK_MAX_SPREAD_TOLERANCES, + ), + ); + return; + } + + let converged = (measured - target).abs() <= tolerance; + // The delivered optical depth is proportional to the commanded one to + // first order, so one gain correction per trial converges in a couple of + // steps even where the drive rolls off at high frequency. + let ratio = (target / measured).clamp(1.0 / A0_LOCK_MAX_STEP_RATIO, A0_LOCK_MAX_STEP_RATIO); + let next = clamp_commanded_a(commanded * ratio); + let railed = !converged && (next - commanded).abs() < 1e-9; + let exhausted = trial >= A0_LOCK_MAX_TRIALS; + + if !converged && !railed && !exhausted { + if let Some(lock) = self.a0_lock.as_mut() { + lock.commanded_a = next; + lock.trial += 1; + } + self.message = format!( + "a₀ lock trial {trial}: measured a = {measured:.3} vs a₀ = {target:.3} — \ + correcting the commanded depth to {next:.3}" + ); + self.send_a0_depth(context); + return; + } + + let optical = self + .photodiode + .as_ref() + .and_then(|summary| summary.optical_summary.as_ref()); + let saved = self.store_lock(A0LockPoint { + frequency_hz: hz, + target_a: target, + commanded_a: commanded, + measured_a: measured, + trials: trial, + converged, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: optical.map(|optical| optical.low_clip_fraction), + high_clip_fraction: optical.map(|optical| optical.high_clip_fraction), + }); + let label = frequency_label(hz); + let message = if converged { + format!( + "a₀ locked at {label}: commanded a = {commanded:.3} measures a = {measured:.3} \ + (a₀ = {target:.3}, {trial} trial(s)){}", + self.clip_warning() + ) + } else if railed { + format!( + "a₀ lock stopped at {label}: commanded a = {commanded:.3} is at the drivable limit \ + and only measures a = {measured:.3} — lower a₀ or the operating point I_k" + ) + } else { + format!( + "a₀ lock did not converge at {label}: best commanded a = {commanded:.3} measures \ + a = {measured:.3} after {trial} trials — widen the tolerance or check the drive" + ) + }; + // A lock the operator can see but that never reached disk is a lock + // they will not have after a restart — say so on the same line. + let message = match saved { + Ok(()) => message, + Err(error) => format!("{message} — {error}"), + }; + self.finish_a0_lock(context, message); + } + + /// Advance the `a₀` lock one control tick. + fn drive_a0_lock(&mut self, context: &mut impl RecordingControl) { + if self.a0_lock.is_none() { + if std::mem::take(&mut self.a0_lock_pending) { + self.begin_a0_lock(context); + } + return; + } + self.a0_lock_pending = false; + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, depth_applied, last_activity_ms) = { + let lock = self.a0_lock.as_ref().expect("lock checked above"); + ( + lock.phase, + lock.stop_requested, + lock.lease_granted, + lock.depth_applied, + lock.last_activity_ms, + ) + }; + if stop_requested { + let message = if self.message.is_empty() { + "a₀ lock stopped".into() + } else { + self.message.clone() + }; + self.finish_a0_lock(context, message); + return; + } + match phase { + A0LockPhase::AcquiringLease => { + if lease_granted { + self.send_a0_depth(context); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_a0_lock( + context, + "a₀ lock aborted: timed out acquiring the modulation lease".into(), + ); + } + } + A0LockPhase::SettingDepth => { + if depth_applied { + // The drive settles for the operator's dwell, and the + // photodiode's own estimator window has to roll over before + // the published `a` is free of the previous depth. Waiting + // for only the shorter of the two silently measures a + // mixture — with the 0.82 s default window that is every + // settle below ~1 s, and it gets worse at low frequency + // where the window grows to cover whole cycles. + let window_ms = self.optical_window_ms().unwrap_or_default(); + let dwell_ms = ((self.settle_s.max(0.0) * 1_000.0) as u64).max(window_ms); + // Only summaries published *after* this depth was commanded + // count, so the trial never averages the previous depth. + let published = self + .photodiode + .as_ref() + .map(|summary| summary.service_revision); + if let Some(lock) = self.a0_lock.as_mut() { + lock.phase = A0LockPhase::Measuring; + lock.window_ms = window_ms; + lock.measure_from_ms = now_ms.saturating_add(dwell_ms); + // The deadline has to outlast the readings it is + // waiting for, or a low-frequency point times out + // before its first independent sample can exist. + let sampling_ms = + (window_ms as f64 * A0_LOCK_SAMPLE_SPACING * A0_LOCK_SAMPLES as f64) + .ceil() as u64; + lock.deadline_ms = lock + .measure_from_ms + .saturating_add(SWEEP_SETTLE_TIMEOUT_MS.max(sampling_ms * 2)); + lock.samples.clear(); + lock.sampled_revision = published; + } + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_a0_lock( + context, + "a₀ lock aborted: timed out retargeting the modulation drive".into(), + ); + } + } + A0LockPhase::Measuring => { + self.sample_a0_measurement(now_ms); + let ready = self.a0_lock.as_ref().is_some_and(|lock| { + lock.samples.len() >= A0_LOCK_SAMPLES || now_ms >= lock.deadline_ms + }); + if ready { + self.evaluate_a0_trial(context); + } + } + } + } + + /// Routes modulation-service replies belonging to the `a₀` lock. Returns true + /// when the reply was consumed. + fn on_a0_lock_reply(&mut self, reply: &PluginServiceReply) -> bool { + let Some((lease_req, depth_req)) = self + .a0_lock + .as_ref() + .map(|lock| (lock.lease_req, lock.depth_req)) + else { + return false; + }; + let abort = |this: &mut Self, message: String| { + this.message = message; + if let Some(lock) = this.a0_lock.as_mut() { + lock.stop_requested = true; + } + }; + if reply.request_id == lease_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(lock) = self.a0_lock.as_mut() { + lock.lease_granted = true; + lock.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("a₀ lock aborted: modulation lease rejected: {message}"), + ), + } + true + } else if reply.request_id == depth_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(lock) = self.a0_lock.as_mut() { + lock.depth_applied = true; + lock.last_activity_ms = now_unix_ms(); + } + } + // The owner refuses a depth its calibrated drive cannot express + // (lobe ceiling, DAC limit) — that *is* the "a₀ unreachable at + // this operating point" answer, so surface its wording verbatim. + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("a₀ lock aborted: the drive rejected the commanded depth: {message}"), + ), + } + true + } else { + false + } + } + + fn a0_locks_dataset(&self) -> TableDatasetV1 { + let column = |id: &str, values: Vec| TableColumnData { + column_id: id.into(), + values: TableColumnValues::String(values), + }; + let map = |select: fn(&A0LockPoint) -> String| { + self.a0_locks.iter().map(select).collect::>() + }; + TableDatasetV1 { + columns: vec![ + column("frequency", map(|lock| frequency_label(lock.frequency_hz))), + column("target_a", map(|lock| format!("{:.3}", lock.target_a))), + column( + "commanded_a", + map(|lock| format!("{:.3}", lock.commanded_a)), + ), + column("measured_a", map(|lock| format!("{:.3}", lock.measured_a))), + column("trials", map(|lock| lock.trials.to_string())), + column( + "state", + map(|lock| { + if lock.converged { + "locked".into() + } else { + "not converged".into() + } + }), + ), + column( + "locked_at", + map(|lock| format_iso_utc(lock.locked_at_unix_ms / 1_000)), + ), + ], + } + } + + fn on_host_reply(&mut self, reply: &HostCommandReply) { + if reply.request_id == self.recording.cam_start_req { + match &reply.outcome { + HostCommandOutcome::RecordingStarted { + actual_raw_path, .. + } => { + self.recording.cam_raw_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + // Stop the rest of the recording; drive_recording resolves the + // abort from the current phase on the next tick. + self.note(format!("Camera recording rejected ({code}): {message}")); + self.recording.cam_rejected = true; + self.recording.stop_requested = true; + } + _ => {} + } + } else if reply.request_id == self.recording.cam_stop_req { + match &reply.outcome { + HostCommandOutcome::RecordingFinalized { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.cam_complete = true; + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::RecordingPartial { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + self.message = format!("Camera stop failed ({code}): {message}"); + self.recording.cam_rejected = true; + self.recording.last_activity_ms = now_unix_ms(); + } + _ => {} + } + } + } + + fn on_service_reply(&mut self, reply: &PluginServiceReply) { + if self.on_sweep_reply(reply) || self.on_a0_lock_reply(reply) { + return; + } + let response = match &reply.outcome { + PluginServiceOutcome::Accepted { payload } => { + serde_json::from_value::(payload.clone()).ok() + } + PluginServiceOutcome::Rejected { code, message } => { + if reply.request_id == self.recording.connect_req + || reply.request_id == self.recording.lease_req + || reply.request_id == self.recording.pd_begin_req + { + self.note(format!("Photodiode start failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.stop_requested = true; + } else if reply.request_id == self.recording.pd_finalize_req { + self.note(format!("Photodiode save failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + None + } + }; + let Some(response) = response else { + return; + }; + if reply.request_id == self.recording.connect_req { + self.recording.connect_accepted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.lease_req { + self.recording.lease_granted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.pd_begin_req { + if let Some(PdqReceiptV1::Started(started)) = &response.receipt { + self.recording.pd_pdq_path = Some(started.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(started.sidecar_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + } else if reply.request_id == self.recording.pd_finalize_req { + self.recording.pd_finalized = true; + if let Some(PdqReceiptV1::Finalized(finalized)) = &response.receipt { + self.recording.pd_pdq_path = Some(finalized.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(finalized.sidecar_path.clone()); + self.recording.pd_valid = finalized.valid; + } + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + } + + /// Advance the recording state machine one control tick. + fn drive_recording(&mut self, context: &mut impl RecordingControl) { + let now_ms = now_unix_ms(); + match self.recording.phase { + RecPhase::Idle => { + if let Some(role) = self.pending_role.take() { + self.begin_recording(context, role); + } + } + RecPhase::StartingCamera => { + if self.recording.cam_rejected { + let message = self.message.clone(); + self.release_and_idle(context, message); + } else if self.recording.cam_raw_path.is_some() { + if self.recording.stop_requested { + self.stop_camera(context); + } else { + self.connect_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.cam_rejected = true; + self.release_and_idle(context, "Timed out starting camera recording".into()); } } RecPhase::ConnectingPhotodiode => { @@ -1853,10 +2852,24 @@ impl StageAA1Plugin { min_a: self.min_a, max_a: self.max_a, requested_a: point.map(Sweep::target_a), + commanded_a: point.map(Sweep::commanded_a), point_index: point.map(|sweep| sweep.index + 1), point_total: point.map(Sweep::total), } }, + a0_lock: self + .sweep + .as_ref() + .and_then(|sweep| sweep.lock.as_ref()) + .map(|lock| A0LockSidecar { + target_a: lock.target_a, + commanded_a: lock.commanded_a, + measured_a_at_lock: lock.measured_a, + frequency_hz_at_lock: lock.frequency_hz, + trials: lock.trials, + converged: lock.converged, + locked_at_utc: format_iso_utc(lock.locked_at_unix_ms / 1_000), + }), pilot: (self.recording.role == RecRole::Pilot) .then_some(self.pilot_windows) .flatten() @@ -1926,6 +2939,9 @@ struct SidecarDoc { finalized_at_utc: String, duration_s: u64, sweep: SweepSidecar, + /// Present on **event-count** points: the `a₀` lock this point replayed. + #[serde(skip_serializing_if = "Option::is_none")] + a0_lock: Option, #[serde(skip_serializing_if = "Option::is_none")] pilot: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1945,6 +2961,11 @@ struct SweepSidecar { /// `[optical]`); absent on manual recordings. #[serde(skip_serializing_if = "Option::is_none")] requested_a: Option, + /// The depth the drive was *commanded* to for this point. Equal to + /// `requested_a` on the amplitude sweep; on an event-count point it is the + /// `a₀`-locked depth, which differs by the drive roll-off at that frequency. + #[serde(skip_serializing_if = "Option::is_none")] + commanded_a: Option, /// 1-based point position within the sweep; absent on manual recordings. #[serde(skip_serializing_if = "Option::is_none")] point_index: Option, @@ -1952,6 +2973,19 @@ struct SweepSidecar { point_total: Option, } +/// The `a₀` lock an **event-count** point replayed: the closed-loop trim that +/// made the photodiode measure the frozen `a₀` at this frequency. +#[derive(Serialize)] +struct A0LockSidecar { + target_a: f64, + commanded_a: f64, + measured_a_at_lock: f64, + frequency_hz_at_lock: f64, + trials: u32, + converged: bool, + locked_at_utc: String, +} + /// Frozen ON/OFF windows written into a **pilot** recording's sidecar and read /// back to reuse them across the row. #[derive(Serialize, serde::Deserialize)] @@ -2350,8 +3384,11 @@ impl Plugin for StageAA1Plugin { // the folder or id changes, look them up in the folder. if !self.recording.is_active() { self.scan_measurement_folder(); + self.load_a0_locks(); } - // The sweep runs first so a point's recording starts on the same tick. + // The lock and the sweep run first so a point's recording starts on the + // same tick. They are mutually exclusive, guarded when they begin. + self.drive_a0_lock(context); self.drive_sweep(context); self.drive_recording(context); // The fold reflects live snapshots (T, a) even between frames. @@ -2512,19 +3549,115 @@ impl Plugin for StageAA1Plugin { selected." .into(), ), - kind: SettingKind::Button { - enabled: can_record, + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_pilot".into(), + label: "Record pilot (freeze ON/OFF windows)".into(), + tooltip: Some( + "Records a bright reference for this row into the same folder \ + (…_pilot) and freezes the ON/OFF windows from the current live \ + signal. Set a high, non-saturating a in the modulation plugin \ + first. The frozen windows are reused for the whole row's q_p. \ + Disabled until an output folder is selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_background".into(), + label: "Record background (a≈0 floor)".into(), + tooltip: Some( + "Records an unmodulated reference (…_background) and captures the \ + false-response floor q0 in the current windows. Set a≈0 in the \ + modulation plugin first. Disabled until an output folder is \ + selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "stop_recording".into(), + label: "Stop (abort recording / sweep)".into(), + tooltip: Some( + "Stop and finalize the current recording before the duration \ + ends; during a sweep this also aborts the remaining points." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + SettingsSection { + label: "Exact event-count depth a₀".into(), + description: Some( + "Second Stage-A workflow, on top of the minimum-depth sweep above: hold \ + ONE photodiode-measured depth a₀ = ln(I_exc,max / I_exc,min) constant \ + across the frequency sweep. Freeze the flux point, camera configuration \ + and references first (pilot and background are recorded above), then per \ + frequency: set f in the modulation plugin, press Find a₀ — A1 leases the \ + drive and trims the *commanded* depth until the photodiode *measures* a₀ \ + — and then press Record a₀ point, which re-applies that depth under the \ + same lease (so the amplitude cannot change during the recorded interval) \ + and records one atomic RAW + PDQ + sidecar point named …_ec_fHz. The \ + found depths are kept per frequency, listed in the a₀ lock table view and \ + mirrored to a0_locks.json in the output folder. Randomising the frequency \ + order, interleaving the low-frequency reference and repeating blocks stay \ + yours — every point is one button press." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "a0_target".into(), + label: "a₀ (measured log contrast)".into(), + tooltip: Some( + "The one photodiode-measured depth held across the whole frequency \ + sweep — never a DAC excursion. Pick it from the low-frequency \ + scout: high enough for several events per pixel per half-cycle, \ + still proportional (not saturated), and refractory-safe at the \ + highest frequency." + .into(), + ), + kind: SettingKind::F64Drag { + min: COMMANDED_A_MIN, + max: COMMANDED_A_MAX, + speed: 0.01, + default: self.a0_target, + }, + }, + SettingItem { + key: "a0_tolerance".into(), + label: "a₀ tolerance (absolute)".into(), + tooltip: Some( + "Convergence band on |measured a − a₀| for the lock, and the \ + settle band an event-count point must hold before it records." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.002, + max: 0.5, + speed: 0.002, + default: self.a0_tolerance, }, }, SettingItem { - key: "record_pilot".into(), - label: "Record pilot (freeze ON/OFF windows)".into(), + key: "find_a0".into(), + label: "Find a₀ (lock the drive depth)".into(), tooltip: Some( - "Records a bright reference for this row into the same folder \ - (…_pilot) and freezes the ON/OFF windows from the current live \ - signal. Set a high, non-saturating a in the modulation plugin \ - first. The frozen windows are reused for the whole row's q_p. \ - Disabled until an output folder is selected." + "Leases the modulation owner and iterates commanded a ← commanded \ + a · a₀/measured a until the photodiode-measured depth is a₀ at the \ + current frequency (up to 8 trials, waiting Sweep settle (s) per \ + trial). Records nothing, leaves the drive at the depth it found, \ + and stores it for this frequency. Requires a calibrated \ + periodic/optical drive armed in the modulation plugin and a \ + photodiode-measured a. Disabled until an output folder is selected." .into(), ), kind: SettingKind::Button { @@ -2532,13 +3665,14 @@ impl Plugin for StageAA1Plugin { }, }, SettingItem { - key: "record_background".into(), - label: "Record background (a≈0 floor)".into(), + key: "record_a0_point".into(), + label: "Record a₀ point (event-count)".into(), tooltip: Some( - "Records an unmodulated reference (…_background) and captures the \ - false-response floor q0 in the current windows. Set a≈0 in the \ - modulation plugin first. Disabled until an output folder is \ - selected." + "Records one atomic frequency point at the locked depth: re-applies \ + the found commanded a under a modulation lease, waits for the \ + measured a to hold a₀, then records camera RAW + photodiode PDQ + \ + sidecar under one run id (…_ec_fHz). Needs a converged lock for \ + the current frequency and an output folder." .into(), ), kind: SettingKind::Button { @@ -2546,11 +3680,12 @@ impl Plugin for StageAA1Plugin { }, }, SettingItem { - key: "stop_recording".into(), - label: "Stop (abort recording / sweep)".into(), + key: "clear_a0_locks".into(), + label: "Clear a₀ lock table".into(), tooltip: Some( - "Stop and finalize the current recording before the duration \ - ends; during a sweep this also aborts the remaining points." + "Drops every stored per-frequency lock and rewrites \ + a0_locks.json. Use it after changing the flux point, the \ + calibration or a₀ itself." .into(), ), kind: SettingKind::Button { enabled: true }, @@ -2684,6 +3819,11 @@ impl Plugin for StageAA1Plugin { "clear" => Some(self.press_clear.value()), "record_point" => Some(self.press_record_point.value()), "clear_curve" => Some(self.press_clear_curve.value()), + "a0_target" => Some(json!(self.a0_target)), + "a0_tolerance" => Some(json!(self.a0_tolerance)), + "find_a0" => Some(self.press_find_a0.value()), + "record_a0_point" => Some(self.press_record_a0.value()), + "clear_a0_locks" => Some(self.press_clear_a0.value()), // New id regenerates the measurement id locally; the id itself is // what synchronizes, so the press must not be forwarded (both // instances would generate different ids). @@ -2768,7 +3908,13 @@ impl Plugin for StageAA1Plugin { sweep.stop_requested = true; self.message = "Sweep stop requested".into(); } + if let Some(lock) = self.a0_lock.as_mut() { + lock.stop_requested = true; + self.message = "a₀ lock stop requested".into(); + } self.sweep_pending = false; + self.a0_lock_pending = false; + self.a0_point_pending = false; } } "live" => { @@ -2808,6 +3954,37 @@ impl Plugin for StageAA1Plugin { self.response_points.clear(); } } + "a0_target" => { + self.a0_target = value + .as_f64() + .ok_or("a0_target must be a number")? + .clamp(COMMANDED_A_MIN, COMMANDED_A_MAX); + } + "a0_tolerance" => { + self.a0_tolerance = value + .as_f64() + .ok_or("a0_tolerance must be a number")? + .clamp(0.002, 0.5); + } + "find_a0" => { + if self.press_find_a0.accept(&value) { + self.a0_lock_pending = true; + } + } + "record_a0_point" => { + if self.press_record_a0.accept(&value) { + self.a0_point_pending = true; + } + } + "clear_a0_locks" => { + if self.press_clear_a0.accept(&value) { + self.a0_locks.clear(); + self.message = match self.save_a0_locks() { + Ok(()) => "a₀ lock table cleared".into(), + Err(error) => error, + }; + } + } "new_id" => return Ok(()), _ => return Err(format!("unknown setting '{key}'")), } @@ -2836,13 +4013,34 @@ impl Plugin for StageAA1Plugin { SweepPhase::Settling => "settling", SweepPhase::Recording => "recording", }; + let label = match sweep.kind { + SweepKind::Amplitude => "Sweep", + SweepKind::EventCount => "Event-count point", + }; entries.push(StatusEntry::Text(format!( - "Sweep: point {}/{} at a → {:.3} ({phase})", + "{label}: point {}/{} commanding a = {:.3} for a measured {:.3} ({phase})", sweep.index + 1, sweep.total(), + sweep.commanded_a(), sweep.target_a() ))); } + if let Some(lock) = &self.a0_lock { + let phase = match lock.phase { + A0LockPhase::AcquiringLease => "leasing modulation", + A0LockPhase::SettingDepth => "commanding depth", + A0LockPhase::Measuring => "measuring", + }; + entries.push(StatusEntry::Text(format!( + "a₀ lock at {}: trial {}/{A0_LOCK_MAX_TRIALS} commanding a = {:.3} for a₀ = {:.3} \ + ({phase}, {} sample(s))", + frequency_label(lock.frequency_hz), + lock.trial, + lock.commanded_a, + lock.target_a, + lock.samples.len() + ))); + } if !self.message.is_empty() { entries.push(StatusEntry::Text(self.message.clone())); } @@ -2914,6 +4112,21 @@ impl Plugin for StageAA1Plugin { "Background floor: q0_on = {q0_on:.3}, q0_off = {q0_off:.3}" ))); } + entries.push(StatusEntry::Text(match self.armed_lock() { + Some(lock) => format!( + "a₀ = {:.3} armed at {}: commanded a = {:.3} (measured {:.3}); {} lock(s) stored", + lock.target_a, + frequency_label(lock.frequency_hz), + lock.commanded_a, + lock.measured_a, + self.a0_locks.len() + ), + None => format!( + "a₀ = {:.3}: no lock for this frequency — press Find a₀; {} lock(s) stored", + self.a0_target, + self.a0_locks.len() + ), + })); entries } @@ -2964,6 +4177,25 @@ impl Plugin for StageAA1Plugin { display: None, relations: Vec::new(), }, + HostDatasetDescriptor { + id: A0_LOCK_DATASET_ID.into(), + title: "A1 a₀ locks — commanded depth per frequency".into(), + kind: HostDatasetKind::TableV1(TableSchema { + columns: vec![ + column("frequency", "Frequency"), + column("target_a", "a₀ (target)"), + column("commanded_a", "Commanded a"), + column("measured_a", "Measured a"), + column("trials", "Trials"), + column("state", "State"), + column("locked_at", "Locked at (UTC)"), + ], + ..TableSchema::default() + }), + empty_message: "No a₀ lock yet — set a₀ and press Find a₀ per frequency".into(), + display: None, + relations: Vec::new(), + }, ], views: vec![ HostViewDescriptor { @@ -2987,6 +4219,13 @@ impl Plugin for StageAA1Plugin { placement: HostViewPlacement::Window, kind: HostViewKind::LineSeriesWindow, }, + HostViewDescriptor { + id: A0_LOCK_VIEW_ID.into(), + title: "A1 a₀ locks (commanded depth per frequency)".into(), + dataset_id: A0_LOCK_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, ], actions: Vec::new(), } @@ -2997,6 +4236,7 @@ impl Plugin for StageAA1Plugin { STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), ROLLING_DATASET_ID => serde_json::to_vec(&self.rolling_dataset()).ok(), RESPONSE_CURVE_DATASET_ID => serde_json::to_vec(&self.response_curve_dataset()).ok(), + A0_LOCK_DATASET_ID => serde_json::to_vec(&self.a0_locks_dataset()).ok(), _ => None, } } @@ -3004,7 +4244,7 @@ impl Plugin for StageAA1Plugin { fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { matches!( dataset_id, - STATUS_DATASET_ID | ROLLING_DATASET_ID | RESPONSE_CURVE_DATASET_ID + STATUS_DATASET_ID | ROLLING_DATASET_ID | RESPONSE_CURVE_DATASET_ID | A0_LOCK_DATASET_ID ) .then_some(self.dataset_generation) .unwrap_or(0) @@ -3047,6 +4287,7 @@ mod tests { } } + /// Mirrors the ordering of [`StageAA1Plugin::process_control`]. fn control_tick( plugin: &mut StageAA1Plugin, inbox: PluginControlInbox, @@ -3058,9 +4299,210 @@ mod tests { for reply in &inbox.service_replies { plugin.on_service_reply(reply); } + plugin.drive_a0_lock(sink); + plugin.drive_sweep(sink); plugin.drive_recording(sink); } + /// Bare `Accepted` reply, as the modulation owner answers a lease or depth + /// command (only the outcome variant is routed). + fn accepted(request_id: u64) -> PluginServiceReply { + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: MODULATION_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Accepted { + payload: Value::Null, + }, + } + } + + fn rejected(request_id: u64, message: &str) -> PluginServiceReply { + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: MODULATION_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Rejected { + code: "invalid_command".into(), + message: message.into(), + }, + } + } + + fn connected_modulation() -> ModulationStateV1 { + ModulationStateV1 { + contract_version: stage_a_plugin_contract::CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("mod-test"), + service_revision: 1, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: Some("0.4.0".into()), + }, + capabilities: Vec::new(), + lease: None, + controller_state: stage_a_plugin_contract::ControllerStateV1::Configured, + active_run_id: None, + requested: None, + acknowledged: None, + synchronization: stage_a_plugin_contract::SynchronizationV1::Unsynced { + reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: stage_a_plugin_contract::FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 5_000, + }, + calibration_id: Some("pockels-test".into()), + } + } + + /// A photodiode snapshot reporting `measured_a`, published at `revision`. + fn photodiode_measuring(revision: u64, measured_a: f64) -> PhotodiodeSummaryV1 { + PhotodiodeSummaryV1 { + contract_version: stage_a_plugin_contract::CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("pd-test"), + service_revision: revision, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: None, + }, + lease: None, + active_run_id: None, + requested_revision: None, + acknowledged_revision: None, + stream: stage_a_plugin_contract::PhotodiodeStreamV1 { + stream_epoch: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + latest_adc_code: None, + integrity: StreamIntegrityV1::default(), + level: None, + }, + active_recording: None, + last_finalized_recording: None, + optical_summary: Some(stage_a_plugin_contract::PhotodiodeOpticalSummaryV1 { + run_id: RunId::new("pd-run"), + calibration: stage_a_plugin_contract::PhotodiodeCalibrationV1 { + adc_calibration_id: "adc".into(), + dark_id: "dark".into(), + anchor_id: "anchor".into(), + dark_volts: 0.0, + total_power_volts: 1.0, + }, + measured_log_contrast: measured_a, + log_contrast_stddev: None, + excitation_min_volts: 0.1, + excitation_max_volts: 0.9, + excitation_headroom_volts: 0.1, + low_clip_fraction: 0.0, + high_clip_fraction: 0.0, + measured_frequency_hz: None, + fundamental_phase_rad: None, + total_harmonic_distortion: None, + // A short window, so the lock's dwell and sample spacing stay + // in the millisecond range the tests tick at. + window_seconds: Some(0.001), + covered_cycles: Some(8.0), + }), + synchronization: stage_a_plugin_contract::SynchronizationV1::Unsynced { + reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: stage_a_plugin_contract::FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 5_000, + }, + } + } + + /// The depth carried by the newest `SetOpticalDepth` the plugin emitted. + fn last_commanded_depth(sink: &ControlSink) -> Option { + sink.services.iter().rev().find_map(|request| { + let envelope: ModulationRequestV1 = + serde_json::from_value(request.payload.clone()).ok()?; + match envelope.command { + ModulationCommandV1::SetOpticalDepth { depth_a_milli } => { + Some(f64::from(depth_a_milli) / 1_000.0) + } + _ => None, + } + }) + } + + /// A unique scratch directory for a test's lock table and sidecars. + fn temp_folder(tag: &str) -> PathBuf { + std::env::temp_dir().join(format!("a1-{tag}-{}", now_unix_ms())) + } + + /// A plugin wired to a connected drive at 1 kHz (marker-anchored) whose + /// photodiode reports a bench that delivers `gain ×` the commanded depth. + fn plugin_locking(gain: f64, folder: &Path) -> StageAA1Plugin { + let mut plugin = plugin_with_markers(); + plugin.modulation = Some(connected_modulation()); + plugin.photodiode = Some(photodiode_measuring(1, gain)); + plugin.output_folder = folder.display().to_string(); + plugin.measurement_id = "A1-ec".into(); + plugin.settle_s = 0.0; + plugin.a0_tolerance = 0.02; + plugin + } + + /// Answers the lock's outstanding lease/depth request and publishes the + /// photodiode readings the commanded depth produces, until the lock ends. + fn run_lock_to_completion( + plugin: &mut StageAA1Plugin, + sink: &mut ControlSink, + gain: f64, + max_ticks: usize, + ) -> usize { + let mut revision = 1; + // The first tick consumes the latched press and starts the lock. + control_tick(plugin, PluginControlInbox::default(), sink); + for tick in 0..max_ticks { + if plugin.a0_lock.is_none() { + return tick + 1; + } + let (lease_req, depth_req, granted, applied) = { + let lock = plugin.a0_lock.as_ref().expect("lock"); + ( + lock.lease_req, + lock.depth_req, + lock.lease_granted, + lock.depth_applied, + ) + }; + let mut replies = Vec::new(); + if !granted { + replies.push(accepted(lease_req)); + } else if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + // Measuring: publish what the bench delivers for the commanded + // depth as a fresh summary. + let commanded = plugin.a0_lock.as_ref().expect("lock").commanded_a; + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, commanded * gain)); + // The lock spaces its readings by a fraction of the photodiode's + // estimator window (1 ms in these fixtures), so a tick loop that + // never advances the wall clock would collect exactly one. + std::thread::sleep(std::time::Duration::from_millis(1)); + } + control_tick( + plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + sink, + ); + } + max_ticks + } + fn pd_reply(request_id: u64, receipt: Option) -> PluginServiceReply { let response = PhotodiodeResponseV1 { common: ResponseCommonV1 { @@ -3276,8 +4718,14 @@ mod tests { // must not record those as if they had come from this run. let mut plugin = plugin_with_markers(); plugin.pilot_windows = Some(( - PhaseWindow { start: 0.0, end: 0.2 }, - PhaseWindow { start: 0.5, end: 0.7 }, + PhaseWindow { + start: 0.0, + end: 0.2, + }, + PhaseWindow { + start: 0.5, + end: 0.7, + }, )); // No events => the fold carries no signal => the freeze cannot pick // windows and must not leave the loaded ones in place. @@ -3536,9 +4984,13 @@ mod tests { }; let points = plugin.sweep_points(); assert_eq!(points.len(), 5); - assert!((points[0] - 0.5).abs() < 1e-12); - assert!((points[4] - 2.5).abs() < 1e-12); - assert!((points[2] - 1.5).abs() < 1e-12); + assert!((points[0].expected_a - 0.5).abs() < 1e-12); + assert!((points[4].expected_a - 2.5).abs() < 1e-12); + assert!((points[2].expected_a - 1.5).abs() < 1e-12); + // The amplitude sweep trusts the calibration: it commands what it expects. + assert!(points + .iter() + .all(|point| point.commanded_a == point.expected_a)); } #[test] @@ -3546,9 +4998,12 @@ mod tests { let mut plugin = plugin_with_markers(); plugin.min_a = 0.5; plugin.max_a = 1.5; + plugin.sweep_count = 3; plugin.sweep = Some(Sweep { phase: SweepPhase::Recording, - points: vec![0.5, 1.0, 1.5], + kind: SweepKind::Amplitude, + points: plugin.sweep_points(), + lock: None, index: 1, lease_id: LeaseId::new("a1-sweep-test"), lease_granted: true, @@ -3574,6 +5029,360 @@ mod tests { let _ = std::fs::remove_file(&path); } + #[test] + fn a0_lock_trims_the_commanded_depth_until_the_photodiode_measures_a0() { + // A bench that delivers 60 % of the commanded depth (drive roll-off): + // commanding a₀ directly would record a = 0.30 instead of 0.50. + let folder = temp_folder("lock"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + + let ticks = run_lock_to_completion(&mut plugin, &mut sink, 0.6, 64); + assert!(ticks < 64, "lock never finished"); + + let lock = plugin + .a0_locks + .first() + .expect("the converged lock is stored"); + assert!(lock.converged, "message: {}", plugin.message); + assert!( + (lock.measured_a - 0.5).abs() <= plugin.a0_tolerance, + "measured {}", + lock.measured_a + ); + assert!( + (lock.commanded_a - 0.5 / 0.6).abs() < 0.01, + "commanded {}", + lock.commanded_a + ); + assert!(lock.trials >= 2, "trials {}", lock.trials); + assert!((lock.frequency_hz - 1_000.0).abs() < 1.0); + // The drive is left at the depth the lock found, and the lease is + // released without a safe-off so it stays there for the recording. + assert!((last_commanded_depth(&sink).expect("depth") - lock.commanded_a).abs() < 0.002); + let release: ModulationRequestV1 = + serde_json::from_value(sink.services.last().expect("release").payload.clone()) + .expect("envelope"); + assert!(matches!( + release.command, + ModulationCommandV1::ReleaseLease { + safe_off: false, + .. + } + )); + // The lock arms the event-count recording for this frequency. + assert!(plugin.armed_lock().is_some()); + // …and the table is on disk next to the recordings. + assert!(folder.join(A0_LOCK_FILE).exists()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_refuses_a_photodiode_window_shorter_than_one_cycle() { + // `a` is peak-to-peak. Under one cycle the photodiode under-reports it, + // and the lock divides by it — so it would inflate the drive until it + // railed. Refuse before touching the drive, and say what to change. + let folder = temp_folder("subcycle"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + // 1 kHz markers give the plugin its frequency; make the estimator + // window 0.4 ms, i.e. 0.4 of a cycle. + if let Some(summary) = plugin.photodiode.as_mut() { + if let Some(optical) = summary.optical_summary.as_mut() { + optical.window_seconds = Some(0.000_4); + } + } + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert!(plugin.a0_lock.is_none(), "the lock must not start"); + assert!(sink.services.is_empty(), "no lease may be requested"); + assert!( + plugin.message.contains("0.40 cycles") && plugin.message.contains("cache length"), + "message: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_refuses_to_lock_onto_an_unsettled_operating_point() { + // Readings that walk across the target are not a lock: the next action + // would record at wherever the drive drifted to, not at a₀. + let folder = temp_folder("unsettled"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + let mut revision = 1; + let mut drift = 0.30; + for _ in 0..64 { + if plugin.a0_lock.is_none() { + break; + } + let (lease_req, depth_req, granted, applied) = { + let lock = plugin.a0_lock.as_ref().expect("lock"); + ( + lock.lease_req, + lock.depth_req, + lock.lease_granted, + lock.depth_applied, + ) + }; + let mut replies = Vec::new(); + if !granted { + replies.push(accepted(lease_req)); + } else if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + drift += 0.20; // 0.50, 0.70, 0.90 — straddling a₀ = 0.50 + plugin.photodiode = Some(photodiode_measuring(revision, drift)); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + &mut sink, + ); + } + + assert!(plugin.a0_lock.is_none(), "the lock must end"); + assert!( + plugin.message.contains("not settled"), + "message: {}", + plugin.message + ); + // Nothing is stored, so nothing can arm a recording. + assert!(plugin.a0_locks.is_empty()); + assert!(plugin.armed_lock().is_none()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_reports_an_unreachable_depth_instead_of_arming_a_recording() { + // The bench delivers 5 % of the commanded depth: a₀ = 0.5 would need a + // commanded depth far beyond what the owner accepts. + let folder = temp_folder("unreachable"); + let mut plugin = plugin_locking(0.05, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + + assert!(run_lock_to_completion(&mut plugin, &mut sink, 0.05, 256) < 256); + let lock = plugin.a0_locks.first().expect("the attempt is recorded"); + assert!(!lock.converged); + assert!((lock.commanded_a - COMMANDED_A_MAX).abs() < 1e-9); + assert!( + plugin.message.contains("drivable limit") + || plugin.message.contains("did not converge"), + "message: {}", + plugin.message + ); + // A non-converged lock must never arm an event-count recording. + assert!(plugin.armed_lock().is_none()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_surfaces_a_drive_rejection_verbatim() { + let folder = temp_folder("reject"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + // Tick 1: begin and lease. + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = plugin.a0_lock.as_ref().expect("lock").lease_req; + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![accepted(lease_req)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let depth_req = plugin.a0_lock.as_ref().expect("lock").depth_req; + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![rejected( + depth_req, + "calibrated optical peak u = 1.2 exceeds the lobe ceiling", + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert!(plugin.a0_lock.is_none(), "the lock must not keep trying"); + assert!( + plugin.message.contains("lobe ceiling"), + "message: {}", + plugin.message + ); + assert!(plugin.a0_locks.is_empty(), "a rejected lock stores nothing"); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn event_count_point_commands_the_locked_depth_not_a0() { + let folder = temp_folder("ecpoint"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.a0_target = 0.5; + plugin.a0_locks.push(A0LockPoint { + frequency_hz: 1_000.0, + target_a: 0.5, + commanded_a: 0.8333, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: Some(0.0), + high_clip_fraction: Some(0.0), + }); + let mut sink = ControlSink::default(); + + plugin + .set_setting("record_a0_point", json!(true)) + .expect("press"); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let sweep = plugin.sweep.as_ref().expect("event-count sweep"); + assert_eq!(sweep.kind, SweepKind::EventCount); + assert_eq!(sweep.total(), 1); + let lease_req = sweep.lease_req; + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![accepted(lease_req)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + // The drive is commanded to the locked depth, *not* to a₀ itself. + let commanded = last_commanded_depth(&sink).expect("commanded depth"); + assert!((commanded - 0.833).abs() < 0.002, "commanded {commanded}"); + assert!((plugin.sweep.as_ref().expect("sweep").target_a() - 0.5).abs() < 1e-9); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn event_count_stems_and_sidecars_carry_the_frequency_and_the_lock() { + let folder = temp_folder("ecstem"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.measurement_id = "A1-ecrow".into(); + plugin.frame_width = 4; + plugin.frame_height = 1; + let lock = A0LockPoint { + frequency_hz: 1_000.0, + target_a: 0.5, + commanded_a: 0.8333, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: 1_784_764_800_000, + low_clip_fraction: Some(0.0), + high_clip_fraction: Some(0.0), + }; + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + kind: SweepKind::EventCount, + points: vec![SweepPoint { + commanded_a: 0.8333, + expected_a: 0.5, + }], + lock: Some(lock), + index: 0, + lease_id: LeaseId::new("a1-sweep-test"), + lease_granted: true, + lease_req: 0, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: true, + last_activity_ms: 0, + stop_requested: false, + }); + + // The stem carries the frequency instead of a sweep-point index. + let mut sink = ControlSink::default(); + plugin.begin_recording(&mut sink, RecRole::EventCount); + let stem = plugin.recording.stem.clone(); + assert!(stem.ends_with("_ec_f1000Hz"), "stem: {stem}"); + + plugin.recording.duration_s = 5; + plugin.recording.start_unix_ms = 1_784_764_800_000; + let path = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&path).expect("read sidecar"); + assert!(text.contains("role = \"event-count point\""), "{text}"); + assert!(text.contains("[a0_lock]"), "{text}"); + assert!(text.contains("target_a = 0.5"), "{text}"); + assert!(text.contains("commanded_a = 0.8333"), "{text}"); + assert!(text.contains("converged = true"), "{text}"); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn frequency_tags_are_file_safe() { + assert_eq!(frequency_tag(50.0), "f50Hz"); + assert_eq!(frequency_tag(0.5), "f0p5Hz"); + assert_eq!(frequency_tag(1_200.0), "f1200Hz"); + assert_eq!(frequency_tag(12.345), "f12p345Hz"); + assert_eq!(sanitize_stem(&frequency_tag(0.5)), frequency_tag(0.5)); + } + + #[test] + fn locks_are_one_per_frequency_and_round_trip_through_the_folder() { + let dir = std::env::temp_dir().join(format!("a1-a0-{}", now_unix_ms())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let folder = dir.display().to_string(); + + let mut plugin = StageAA1Plugin { + output_folder: folder.clone(), + ..StageAA1Plugin::default() + }; + let point = |hz: f64, commanded_a: f64| A0LockPoint { + frequency_hz: hz, + target_a: 0.5, + commanded_a, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: None, + high_clip_fraction: None, + }; + plugin.store_lock(point(1_000.0, 0.83)).expect("saved"); + plugin.store_lock(point(50.0, 0.52)).expect("saved"); + // Re-locking the same frequency replaces the row rather than appending. + plugin.store_lock(point(1_000.5, 0.86)).expect("saved"); + assert_eq!(plugin.a0_locks.len(), 2); + assert!( + (plugin.a0_locks[0].frequency_hz - 50.0).abs() < 1e-9, + "sorted by frequency" + ); + + let mut other = StageAA1Plugin { + output_folder: folder.clone(), + ..StageAA1Plugin::default() + }; + other.load_a0_locks(); + assert_eq!(other.a0_locks.len(), 2); + let reloaded = other.lock_for_frequency(1_000.0).expect("reloaded lock"); + assert!((reloaded.commanded_a - 0.86).abs() < 1e-9); + assert_eq!(other.a0_locks_dataset().columns.len(), 7); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn settings_discontinuities_keep_the_response_curve() { let mut plugin = StageAA1Plugin::default(); diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 2dc3e51..f60ac8f 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -4802,7 +4802,10 @@ level = 750 reply.outcome ); } - assert!((plugin.depth_a - 1.25).abs() < 1e-9, "sweep drives the depth"); + assert!( + (plugin.depth_a - 1.25).abs() < 1e-9, + "sweep drives the depth" + ); plugin.end_lease(); assert!( diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 1e845eb..95cbc59 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -83,9 +83,20 @@ const SPECTRUM_MIN_SAMPLES: usize = 256; const SPECTRUM_MAX_SAMPLES: usize = 16_384; /// The firmware's default stream rate; the mock mirrors it. const MOCK_RATE_HZ: u32 = 20_000; -/// Trailing samples used for the live optical log-contrast `a`. Sized like the -/// spectrum window so a handful of modulation cycles are always covered. +/// Floor on the trailing samples used for the live optical log-contrast `a`, +/// and the fallback window when no phase-0 markers give a period. Sized like +/// the spectrum window: 16 384 samples ≈ 0.82 s at 20 kSa/s. const CONTRAST_WINDOW_SAMPLES: usize = 16_384; +/// Whole modulation cycles the contrast window is sized to cover. +/// +/// `a` is a *peak-to-peak* quantity, so a window shorter than one cycle sees +/// only an arc of the waveform and under-reports it — and a consumer that +/// divides by the measured `a` (A1's `a₀` lock) then inflates its drive against +/// that bias. A fixed 0.82 s window is below one cycle for every `f < 1.2 Hz`, +/// i.e. exactly the sub-hertz plateau reference the A1 protocol needs. The +/// markers give the period on the same sample clock, so size the window from +/// them instead. +const CONTRAST_WINDOW_CYCLES: f64 = 8.0; const MOCK_BLOCK_SAMPLES: usize = 256; /// Cap on retained phase-0 markers (bounds the overlay + frequency window). const MAX_MARKERS: usize = 4_096; @@ -171,6 +182,16 @@ struct SharedState { /// `Marker` stream frames. Used for the opt-in trigger overlay and to derive /// the modulation frequency. markers: VecDeque, + /// Newest phase-0 marker index seen, retained or already evicted, and the + /// spacing to the one before it. + /// + /// The retained markers alone cannot measure a period longer than the ring: + /// once the ring holds less than one cycle it holds at most one marker, so + /// the mean spacing is undefined exactly where knowing the period matters + /// most. Markers arrive one at a time, so remember the interval as it goes + /// past instead of trying to recover it from what survived eviction. + last_marker_index: Option, + marker_period_estimate: Option, latest: Option, /// Cumulative firmware-side drop counter (latest header value). device_dropped: u32, @@ -218,6 +239,8 @@ impl Default for SharedState { samples: VecDeque::new(), cells: VecDeque::new(), markers: VecDeque::new(), + last_marker_index: None, + marker_period_estimate: None, latest: None, device_dropped: 0, crc_failures: 0, @@ -252,6 +275,10 @@ impl SharedState { self.samples.clear(); self.cells.clear(); self.markers.clear(); + // The sample clock restarts with the segment, so a spacing + // measured across the discontinuity is meaningless. + self.last_marker_index = None; + self.marker_period_estimate = None; self.ring_first_index = first_index; self.rate_hz = rate_hz; } @@ -312,6 +339,12 @@ impl SharedState { { return; // ignore duplicate stamps } + if let Some(previous) = self.last_marker_index { + if sample_index > previous { + self.marker_period_estimate = Some((sample_index - previous) as f64); + } + } + self.last_marker_index = Some(sample_index); self.markers.push_back(sample_index); while self.markers.len() > MAX_MARKERS { self.markers.pop_front(); @@ -319,11 +352,31 @@ impl SharedState { self.last_update_unix_ms = now_unix_ms(); } + /// How many trailing samples the optical log-contrast is estimated over, + /// with the whole modulation cycles that window covers. + /// + /// `a` is peak-to-peak, so the window has to span whole cycles: sized to + /// [`CONTRAST_WINDOW_CYCLES`] of the marker-measured period, floored at + /// [`CONTRAST_WINDOW_SAMPLES`] so nothing gets shorter than today at high + /// `f`, and capped by what the ring actually retains. `covered_cycles` is + /// `None` when there is no period to measure against — then the caller can + /// only fall back to the fixed window and say so. + fn contrast_window(&self) -> (usize, Option) { + let available = self.samples.len(); + let Some(period) = self.marker_period_samples() else { + return (available.min(CONTRAST_WINDOW_SAMPLES), None); + }; + let wanted = (period * CONTRAST_WINDOW_CYCLES).ceil() as usize; + let window = wanted.max(CONTRAST_WINDOW_SAMPLES).min(available); + (window, Some(window as f64 / period)) + } + /// Mean marker spacing in samples, i.e. the modulation period on the device /// clock — the trigger *defining* the frequency. `None` with < 2 markers. fn marker_period_samples(&self) -> Option { if self.markers.len() < 2 { - return None; + // Below one retained cycle only the remembered interval is left. + return self.marker_period_estimate; } let first = *self.markers.front()?; let last = *self.markers.back()?; @@ -1529,8 +1582,8 @@ impl StageAPhotodiodePlugin { /// retarget the sweep and write a wrong `measured_a` into every sidecar. /// /// `None` when there is no valid window or no valid total-power anchor. - fn optical_summary(&self, samples: &VecDeque) -> Option { - self.optical_summary_result(samples).ok() + fn optical_summary(&self, state: &SharedState) -> Option { + self.optical_summary_result(state).ok() } /// [`Self::optical_summary`], keeping the rejection reason so the status @@ -1538,9 +1591,23 @@ impl StageAPhotodiodePlugin { /// showing nothing. fn optical_summary_result( &self, - samples: &VecDeque, + state: &SharedState, ) -> Result { - let start = samples.len().saturating_sub(CONTRAST_WINDOW_SAMPLES); + let samples = &state.samples; + let (window_samples, covered_cycles) = state.contrast_window(); + let rate_hz = f64::from(state.rate_hz.max(1)); + let window_seconds = window_samples as f64 / rate_hz; + // Fail closed below one full cycle: the robust extrema would see an arc + // of the waveform, and `a` would be a phase-dependent under-estimate. A + // consumer that divides by the measured `a` — A1's `a₀` lock — would + // then drive itself up against a bias it cannot see. + if let Some(cycles) = covered_cycles.filter(|cycles| *cycles < 1.0) { + return Err(EstimateError::WindowShorterThanCycle { + covered_cycles: cycles, + window_seconds, + }); + } + let start = samples.len().saturating_sub(window_samples); let window: Vec = samples.iter().skip(start).copied().collect(); let calibration = self.adc_calibration(); // `ContrastGeometry::RejectedComplement` wants the *dark-corrected* @@ -1585,9 +1652,13 @@ impl StageAPhotodiodePlugin { excitation_headroom_volts: estimate.v_min_volts, low_clip_fraction: estimate.low_clip_fraction, high_clip_fraction: estimate.high_clip_fraction, - measured_frequency_hz: None, + // The phase-0 markers are the trigger that *defines* the frequency, + // on the same sample clock as the codes above. + measured_frequency_hz: state.marker_period_samples().map(|period| rate_hz / period), fundamental_phase_rad: None, total_harmonic_distortion: None, + window_seconds: Some(window_seconds), + covered_cycles, }) } @@ -1595,7 +1666,7 @@ impl StageAPhotodiodePlugin { /// keeping the rejection reason so the caller can explain a withheld `a`. fn latest_optical_result(&self) -> Option> { let state = self.shared.lock().ok()?; - (!state.samples.is_empty()).then(|| self.optical_summary_result(&state.samples)) + (!state.samples.is_empty()).then(|| self.optical_summary_result(&state)) } fn control_summary(&self) -> PhotodiodeSummaryV1 { @@ -1606,7 +1677,7 @@ impl StageAPhotodiodePlugin { end_sample_index_exclusive: state.ring_first_index + state.samples.len() as u64, sample_count: state.samples.len() as u64, }); - let optical_summary = self.optical_summary(&state.samples); + let optical_summary = self.optical_summary(&state); let level = self.current_level(&state); let connection = if self.connected() { ConnectionStateV1::Connected { @@ -3347,15 +3418,139 @@ mod tests { /// A clean rejected-port sine: the detector swings around `center` while /// the excitation is its complement against `I_tot`. - fn rejected_port_samples(center: f64, amplitude: f64, count: usize) -> VecDeque { + fn rejected_port_samples(center: f64, amplitude: f64, count: usize) -> Vec { (0..count) .map(|i| { let phase = 2.0 * std::f64::consts::PI * (i as f64) * 8.0 / count as f64; - (center + amplitude * phase.sin()).round().clamp(0.0, 4_095.0) as u16 + (center + amplitude * phase.sin()) + .round() + .clamp(0.0, 4_095.0) as u16 }) .collect() } + /// [`rejected_port_samples`] ingested into a ring, with one phase-0 marker + /// per cycle when `mark_cycles` — the estimator sizes its window from them. + fn rejected_port_state( + center: f64, + amplitude: f64, + count: usize, + mark_cycles: bool, + ) -> SharedState { + let mut state = SharedState::default(); + state.ingest( + 0, + 20_000, + 0, + &rejected_port_samples(center, amplitude, count), + ); + if mark_cycles { + // `rejected_port_samples` puts 8 whole cycles in `count` samples. + let period = (count / 8) as u64; + for cycle in 0..8 { + state.push_marker(cycle * period); + } + } + state + } + + /// A slow sine streamed for `total` samples into a ring that only retains + /// `retained` of them, with one phase-0 marker per cycle delivered as the + /// stream goes past — so markers are evicted exactly as they are on the + /// bench when the period outgrows the monitor cache. + fn slow_sine_state(period_samples: u64, retained: usize, total: usize) -> SharedState { + let mut state = SharedState { + cache_seconds: retained as f64 / 20_000.0, + ..Default::default() + }; + let block = 4_000; + let mut index = 0usize; + while index < total { + let end = (index + block).min(total); + let codes: Vec = (index..end) + .map(|i| { + let phase = 2.0 * std::f64::consts::PI * (i as f64) / period_samples as f64; + (1_600.0 + 700.0 * phase.sin()).round() as u16 + }) + .collect(); + state.ingest(index as u64, 20_000, 0, &codes); + let mut marker = index.next_multiple_of(period_samples as usize) as u64; + while (marker as usize) < end { + state.push_marker(marker); + marker += period_samples; + } + index = end; + } + state + } + + #[test] + fn a_window_shorter_than_one_cycle_withholds_a_instead_of_under_reporting_it() { + // `a` is peak-to-peak. Below one full cycle the robust extrema see an + // arc of the sine, so `a` comes out low — and A1's a₀ lock divides by + // it, inflating its drive against a bias it cannot see. Fail closed. + let mut plugin = live_plugin(); + plugin.reference_volts = 3.0; + + // 0.5 Hz at 20 kSa/s = 40 000 samples per cycle; retain 0.6 of one. + let partial = slow_sine_state(40_000, 24_000, 200_000); + let error = plugin + .optical_summary_result(&partial) + .expect_err("a partial cycle must not publish an a"); + assert!( + matches!( + error, + EstimateError::WindowShorterThanCycle { covered_cycles, .. } + if (covered_cycles - 0.6).abs() < 0.05 + ), + "unexpected rejection: {error:?}" + ); + + // Two whole cycles of the same drive: published, and the window is + // reported so a consumer can wait it out before trusting a re-read. + let whole = slow_sine_state(40_000, 80_000, 200_000); + let summary = plugin + .optical_summary(&whole) + .expect("two whole cycles estimate"); + let expected = ((3.0_f64 - (1_600.0 - 700.0) * (3.3 / 4_095.0)) + / (3.0 - (1_600.0 + 700.0) * (3.3 / 4_095.0))) + .ln(); + assert!( + (summary.measured_log_contrast - expected).abs() < 0.02, + "a={} expected~{expected}", + summary.measured_log_contrast + ); + assert!((summary.measured_frequency_hz.expect("markers") - 0.5).abs() < 0.01); + assert!((summary.window_seconds.expect("window") - 4.0).abs() < 0.01); + assert!(summary.covered_cycles.expect("cycles") >= 1.0); + } + + #[test] + fn the_contrast_window_grows_to_cover_whole_cycles_at_low_frequency() { + // A fixed 16 384-sample window is 0.82 s: below one cycle for every + // f < 1.2 Hz, which is where the A1 plateau reference lives. + let fast = rejected_port_state(1_600.0, 700.0, 4_096, true); + let (window, cycles) = fast.contrast_window(); + assert_eq!(window, 4_096, "high f keeps the whole retained ring"); + assert!(cycles.expect("markers") >= 8.0); + + let slow = slow_sine_state(40_000, 400_000, 400_000); + let (window, cycles) = slow.contrast_window(); + assert_eq!( + window, + (CONTRAST_WINDOW_CYCLES as usize) * 40_000, + "the window is sized from the marker period, not fixed" + ); + assert!((cycles.expect("markers") - CONTRAST_WINDOW_CYCLES).abs() < 0.01); + + // Without markers there is no period to size against: fall back to the + // fixed window and report no cycle count rather than guess one. + let mut unmarked = slow_sine_state(40_000, 400_000, 400_000); + unmarked.markers.clear(); + unmarked.marker_period_estimate = None; + assert_eq!(unmarked.contrast_window(), (CONTRAST_WINDOW_SAMPLES, None)); + } + #[test] fn published_contrast_is_the_excitation_contrast_in_both_display_modes() { // The detector sits behind the PBS reject port whatever the operator @@ -3363,14 +3558,12 @@ mod tests { // scientific quantity. A1's amplitude sweep settles on this value. let mut plugin = live_plugin(); plugin.reference_volts = 3.0; - let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + let state = rejected_port_state(1_600.0, 700.0, 4_096, true); plugin.mode = Mode::Raw; - let raw = plugin.optical_summary(&samples).expect("raw display"); + let raw = plugin.optical_summary(&state).expect("raw display"); plugin.mode = Mode::Excitation; - let excitation = plugin - .optical_summary(&samples) - .expect("excitation display"); + let excitation = plugin.optical_summary(&state).expect("excitation display"); assert_eq!(raw.measured_log_contrast, excitation.measured_log_contrast); assert_eq!(raw.calibration.anchor_id, "reference-volts"); @@ -3390,14 +3583,14 @@ mod tests { fn captured_dark_level_reaches_the_estimator_and_is_named() { let mut plugin = live_plugin(); plugin.reference_volts = 3.0; - let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + let state = rejected_port_state(1_600.0, 700.0, 4_096, true); - let undarkened = plugin.optical_summary(&samples).expect("no dark yet"); + let undarkened = plugin.optical_summary(&state).expect("no dark yet"); assert_eq!(undarkened.calibration.dark_id, "dark-none"); assert_eq!(undarkened.calibration.dark_volts, 0.0); plugin.dark_volts = 0.05; - let darkened = plugin.optical_summary(&samples).expect("with dark"); + let darkened = plugin.optical_summary(&state).expect("with dark"); assert_eq!(darkened.calibration.dark_id, "dark-measured"); assert_eq!(darkened.calibration.dark_volts, 0.05); // A DC dark offset is common to the detector samples and to the @@ -3679,7 +3872,7 @@ mod tests { railed.ingest(0, 20_000, 0, &[4_095; 8]); let clipped = plugin.current_level(&railed).expect("still reports"); assert!(clipped.clipped); - assert!(plugin.optical_summary(&railed.samples).is_none()); + assert!(plugin.optical_summary(&railed).is_none()); } #[test] diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs index 1c87f44..cfb445f 100644 --- a/stage-a-io/src/estimator.rs +++ b/stage-a-io/src/estimator.rs @@ -105,6 +105,18 @@ pub enum EstimateError { total_power_volts: f64, detector_max_volts: f64, }, + /// The window is shorter than one full modulation cycle, so the robust + /// extrema only see an arc of the waveform and `a` would be a + /// phase-dependent *under*-estimate. + /// + /// Constructed by the caller — [`estimate_contrast`] is given codes, not a + /// period, and sizing the window against the drive is the caller's job. + /// It is in this enum because it belongs with the other fail-closed + /// reasons a consumer has to render. + WindowShorterThanCycle { + covered_cycles: f64, + window_seconds: f64, + }, } impl std::fmt::Display for EstimateError { @@ -131,6 +143,14 @@ impl std::fmt::Display for EstimateError { "total-power anchor {total_power_volts:.4} V is not above the detector \ maximum {detector_max_volts:.4} V; a is undefined" ), + Self::WindowShorterThanCycle { + covered_cycles, + window_seconds, + } => write!( + f, + "the {window_seconds:.2} s window covers only {covered_cycles:.2} modulation \ + cycles; a needs at least one full cycle — raise the cache length" + ), } } } diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index e7e1638..545644b 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -541,6 +541,18 @@ pub struct PhotodiodeOpticalSummaryV1 { pub measured_frequency_hz: Option, pub fundamental_phase_rad: Option, pub total_harmonic_distortion: Option, + /// Duration of the trailing window `measured_log_contrast` was estimated + /// over. A consumer that *commands* a depth and then reads this value back + /// has to wait at least this long, or it averages the previous depth in. + /// Additive in V1: absent from older owners, ignored by older consumers. + #[serde(default)] + pub window_seconds: Option, + /// Whole modulation cycles that window covered, from the phase-0 markers. + /// `a` is peak-to-peak, so below one cycle the owner withholds it entirely + /// rather than publish a phase-dependent under-estimate. `None` when there + /// is no marker period to measure against. + #[serde(default)] + pub covered_cycles: Option, } /// Settled detector level over the newest averaging window, in **raw detector From 3fee3d68493664defae2d90c39cf661a37eead42 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 21:49:44 +0200 Subject: [PATCH 27/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20run=20the=20?= =?UTF-8?q?A1=20a=E2=82=80=20frequency=20ladder=20unattended=20on=20one=20?= =?UTF-8?q?lease?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An A1 event-count block is 7-12 frequencies over two or three decades, each one a Find a₀ and a Record a₀ point, repeated over three blocks. ADR 013 left that manual because the ordering decisions are scientific — but they are also expressible, and the checklist wants them frozen in the session plan anyway. Every gap between the two presses was also a gap in which a modulation settings sync could re-apply the operator's own depth on top of the found one. Start frequency sweep runs the whole ladder on ONE modulation lease: per point it retargets the drive frequency, waits for the phase-0 trigger to confirm the new period, runs the unchanged a₀ lock, and records the unchanged event-count point. Both children gained an inherited-lease mode, so they run on the ladder's lease instead of taking their own — which is the substantive guarantee: the operator's drive settings are locked out from the first frequency to the last, so the amplitude provably cannot move between a lock and the point that replays it. `ModulationCommandV1::SetDriveFrequency` is the frequency counterpart of SetOpticalDepth (additive in V1, same scoping: leased only, re-derived through the owner's own drive_command, refused for a manual DAC or constant drive). The owner parks the operator's armed frequency on the first retarget and restores it with the depth when the lease ends, so a finished ladder does not leave the bench on its last point. Robustness, which is most of the work: - the trigger confirms the frequency, not the firmware ACK. A point starts only once enough phase-0 markers at the *new* period agree with the commanded one - retained markers and events are dropped on every frequency change: the measured period is their mean spacing, so keeping them would confirm the new frequency against a mixture of the old drive and the new - pilot windows are dropped with them. Windows frozen at one period are a phase interval of that period; carrying them across would score a point in the wrong window, silently, because a fold always produces something - the plan is validated before the drive moves. The photodiode estimates `a` over one window for the whole ladder, so its lowest frequency decides whether the ladder is measurable at all — checked at the button press, not at the ninth point two hours in - an unreachable a₀, an unconfirmed frequency or a failed recording skips that point and names it in the summary; the remaining decades are still recorded. A refused frequency carries the owner's own wording into the skip The schedule is data: log spacing (|H(f)| is read per decade), ascending / descending / alternating / seeded-random order, and an optional low-frequency reference interleaved every N points. Every point's sidecar gains a [frequency_sweep] section with the executed position, the order and the seed, so a block is interpretable from its files rather than from a notebook. See ADR 014. --- docs/adr/014-stage-a-a1-frequency-ladder.md | 109 ++ docs/features/README.md | 2 +- docs/features/stage-a-a1-event-count.md | 106 +- docs/features/stage-a-a1.md | 3 +- plugins/stage-a-a1/README.md | 4 +- plugins/stage-a-a1/src/runtime.rs | 1511 ++++++++++++++++++- plugins/stage-a-modulation/src/lib.rs | 70 +- stage-a-plugin-contract/src/lib.rs | 13 + 8 files changed, 1769 insertions(+), 49 deletions(-) create mode 100644 docs/adr/014-stage-a-a1-frequency-ladder.md diff --git a/docs/adr/014-stage-a-a1-frequency-ladder.md b/docs/adr/014-stage-a-a1-frequency-ladder.md new file mode 100644 index 0000000..3b1f99e --- /dev/null +++ b/docs/adr/014-stage-a-a1-frequency-ladder.md @@ -0,0 +1,109 @@ +# ADR 014 — Stage-A A1 unattended frequency ladder + +- **Status:** accepted (2026-07-27) +- **Relates to:** ADR 009 (recording coordinator), ADR 010 (amplitude sweep via + leased `SetOpticalDepth`), ADR 012 (the contrast geometry the measured `a` is + defined in), ADR 013 (the per-frequency `a₀` lock), + [Stage-A A1 Exact Event Count](../features/stage-a-a1-event-count.md) + +## Context + +ADR 013 gave the operator two buttons per frequency — *Find a₀* and *Record a₀ +point* — and deliberately left the ladder manual, because "the protocol's +ordering and randomisation decisions are scientific, not mechanical". + +In practice an A1 event-count block is 7–12 frequencies over two or three +decades, each one a lock plus a recording, repeated over three independent +blocks. That is an hour of pressing two buttons in the right order while +watching a status line — and every gap between the two presses is a gap in which +a modulation settings sync can re-apply the operator's own `depth a` on top of +the found one (ADR 013 §3 exists precisely because of this hazard, and only +closes it *within* one point). + +The ordering decisions are scientific, but they are also **expressible**: a +seeded schedule and an interleaved reference cadence are exactly what the A1 +checklist asks to be frozen in the session plan before the block starts. Freezing +them as settings and recording them per point is stronger than leaving them to +be executed by hand and written down afterwards. + +Three things blocked automation: + +1. **A1 could not change the frequency.** The contract exposed `SetOpticalDepth` + but no frequency equivalent, and A1's scoped reach (ADR 007/010) was "the + armed drive's depth while leased". +2. **Nothing could confirm a frequency had arrived.** The firmware ACKs a table + it accepted; the light modulating at that rate is a different claim. +3. **The measured `a` was not trustworthy at the bottom of a ladder.** The + photodiode estimated the peak-to-peak contrast over a fixed 0.82 s window, + under one cycle for every `f < 1.2 Hz` — and the `a₀` lock divides by that + value, so a truncated estimate drives the depth up until it rails. + +## Decision + +**1. `ModulationCommandV1::SetDriveFrequency { frequency_millihz }`** (contract +addition, additive to V1) — the frequency counterpart of `SetOpticalDepth`, with +the same scoping: leased only, re-derived through the same `drive_command()` +builder, rejected when the link is closed or the armed drive has no frequency to +retarget. The owner **parks the operator's armed frequency** on the first +retarget and restores it in `end_lease`, exactly as it already does for the +depth, so a finished ladder does not leave the bench on its last point. + +**2. The ladder is a supervisor, not a third state machine.** `FreqSweep` runs +`AcquiringLease → (per point) SettingFrequency → ConfirmingFrequency → Locking → +Recording → …release`, where *Locking* and *Recording* are the **unchanged** +ADR 013 lock and ADR 010/013 point. Both gained an inherited-lease mode +(`owns_lease: false`): when the ladder starts them they neither acquire nor +release, they run on its lease. + +That is the substantive guarantee: **one lease spans the whole ladder**, so the +operator's drive settings are locked out from the first frequency to the last, +and the "the amplitude cannot change during the recorded interval" property +ADR 013 established for one point now holds across the gap between a lock and +the point that replays it. + +**3. The trigger confirms the frequency.** A point does not start until enough +phase-0 markers *at the new period* agree with the commanded frequency. On every +frequency change the retained markers and events are dropped: the measured +period is their mean spacing, so keeping them would confirm the new frequency +against a mixture of the old drive and the new one. + +**Pilot windows are dropped with them.** Windows frozen at one period are a +phase interval of *that* period; carrying them into another frequency would +score the point in the wrong window — silently, because a fold always produces +something. Re-freezing a pilot per frequency stays the operator's call; the +ladder only guarantees it never reuses a stale one. + +**4. The schedule is data.** Log spacing (a Bode ladder is read per decade), +four orders — ascending, descending, alternating, seeded random — and an +optional low-frequency reference interleaved every N points. The executed +position, the order and the seed go into every point's sidecar +(`[frequency_sweep]`), so a block is interpretable from its files rather than +from a notebook. + +**5. A bad point is skipped, not fatal.** An unreachable `a₀`, an unconfirmed +frequency, or a failed recording skips that frequency and names it in the final +summary; the lock table keeps the failed attempt. The remaining decades are +worth more than a clean abort. Only losing the lease ends the ladder. + +**6. The plan is validated before the drive moves.** The photodiode estimates +`a` over one window for the whole ladder, so its *lowest* frequency decides +whether the ladder is measurable. That, the drivability of `a₀`, the presence of +a trigger, and the destination are all checked at the button press. + +## Consequences + +- A1's scoped hardware reach widens by one parameter: it may retarget the armed + drive's **frequency** as well as its depth, still only while leased, still + through the owner's own builder and validation. Everything else about the + drive remains the operator's. +- The lease is now held for the length of a whole block rather than a point, so + its TTL is sized from the ladder (renewed per point). A lost lease ends the + run — which is the correct failure: without it the drive is no longer + provably A1's. +- Pilot-frozen windows no longer survive a frequency change. A workflow that + relied on freezing one pilot and recording several frequencies against it was + producing wrongly-scored `q_p`; it now falls back to per-fold auto-windows and + says so. +- `a₀` is still not frozen numerically here, the refractory bound is still not + checked, and references are still the operator's. The ladder automates the + mechanical repetition, not the scientific choices. diff --git a/docs/features/README.md b/docs/features/README.md index 8606c60..43305e3 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -11,7 +11,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). - [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. -- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀`, a per-frequency lock table on disk, and a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease. +- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-a1-event-count.md b/docs/features/stage-a-a1-event-count.md index f9c4d7e..b10ac64 100644 --- a/docs/features/stage-a-a1-event-count.md +++ b/docs/features/stage-a-a1-event-count.md @@ -1,11 +1,15 @@ # Stage-A A1 Exact Event Count — the `a₀` depth lock - **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) -- **Status:** Built — per-frequency `a₀` lock + one-button event-count point -- **Design:** [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md); builds - on [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (leased - `SetOpticalDepth`) and [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) - (the RAW + PDQ + sidecar coordinator) +- **Status:** Built — per-frequency `a₀` lock, one-button event-count point, and + an unattended frequency ladder that does both at every planned `f` +- **Design:** [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (the + lock) and [ADR 014](../adr/014-stage-a-a1-frequency-ladder.md) (the ladder); + builds on [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (leased + `SetOpticalDepth`), [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) + (the RAW + PDQ + sidecar coordinator) and + [ADR 012](../adr/012-stage-a-contrast-geometry-is-bench-not-display.md) (the + geometry the measured `a` is defined in) - **Relates to:** [Stage-A A1 Analysis](./stage-a-a1.md), [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md), [Stage-A Photodiode](./stage-a-photodiode.md) @@ -22,7 +26,8 @@ a_0=\ln\!\left(\frac{I_{\mathrm{exc,max}}}{I_{\mathrm{exc,min}}}\right), and hold that **photodiode-measured** value constant while the frequency varies, so event counts per half-cycle are comparable across `f` at equal optical -contrast. `a₀` is a measured log contrast — **never** a DAC-code excursion. +contrast. `a₀` is a measured log contrast — **never** a DAC-code excursion, and +never the reject-port detector's own contrast (ADR 012). ## Why a lock is needed at all @@ -37,6 +42,31 @@ record at the wrong depth. The lock closes that loop: it commands, measures, and corrects until the photodiode reports `a₀`. +## What the measured `a` needs to be worth dividing by + +The lock divides by the measured `a`, so a *biased* measurement is not noise — +it is a systematic push on the drive. Two properties of the photodiode estimate +therefore matter more here than anywhere else, and both are enforced: + +- **Whole cycles.** `a` is peak-to-peak, so its window has to span at least one + full modulation cycle. The photodiode sizes its contrast window from the + phase-0 markers to cover several cycles, and **withholds `a` entirely** below + one. A fixed 0.82 s window — what it used before — is under one cycle for + every `f < 1.2 Hz`, exactly where the A1 plateau reference lives, and would + have under-reported `a` and driven the depth up until it railed. Its length + and cycle count are published as `window_seconds` / `covered_cycles`. +- **A window that has turned over.** A reading taken sooner than one window + after a depth change still contains the old depth. The lock's per-trial dwell + is therefore at least one window (never less than **Sweep settle (s)**), and + its three readings are spaced by half a window so they are not three views of + the same samples. The trial value is their **median**; if they spread by more + than twice the tolerance the operating point is called unsettled and the lock + aborts rather than latching onto a drifting drive. + +If the ladder's lowest frequency needs a longer window than the photodiode's +ring holds, raise its **Cache length**; the refusal says so and names the +seconds needed. + ## The workflow, one frequency at a time Everything up to the references is unchanged and stays the operator's: freeze the @@ -56,9 +86,67 @@ across frequencies is not automated, i.e. off by default). Then: 3. Press **Record a₀ point (event-count)**. A1 re-applies the found depth under a modulation lease, waits for the measured `a` to hold `a₀`, and records one atomic camera RAW + photodiode PDQ + sidecar under one run id. -4. Repeat for the next frequency. Randomising the frequency order, interleaving - the low-frequency reference and repeating independent blocks (three where - practical) are yours — every point is one button press. +4. Repeat for the next frequency. Repeating independent blocks (three where + practical) is yours — every point is one button press. + +Steps 1–4 are what **Start frequency sweep** automates; see below. + +## The frequency ladder (unattended) + +**Start frequency sweep (find a₀ + record per f)** runs the whole ladder on +**one** modulation lease. Per point it retargets the drive's frequency +(`ModulationCommandV1::SetDriveFrequency`), waits for the phase-0 trigger to +actually report the new period, runs the `a₀` lock, and records one atomic +event-count point — then moves on. + +| Control | Meaning | +|---|---| +| Sweep min f / max f (Hz) | ends of the ladder, both included | +| Frequency points | how many, **log-spaced** — `\|H(f)\|` is read per decade | +| Frequency order | ascending / descending / alternating / random (seeded) | +| Random order seed | makes the random schedule reproducible; recorded per point | +| Low-f reference every N points | re-visit the lowest frequency every N points | +| Start frequency sweep | run the ladder | +| Stop | aborts the ladder and whichever child is mid-flight | + +What it guarantees: + +- **One lease for the whole ladder.** The lock and the recording run on the + ladder's lease instead of taking their own, so the operator's drive settings + are locked out from the first frequency to the last — the amplitude provably + cannot move between a lock and the point that replays it. The owner parks the + operator's frequency *and* depth on the first retarget and hands both back + when the lease is released. +- **The trigger confirms the frequency, not the firmware.** An ACK says a table + was accepted; the phase-0 markers say the light is modulating at that rate. + A point only starts once enough markers at the *new* period agree with the + commanded frequency. +- **Nothing from the previous frequency survives.** Retained markers and events + are dropped on every frequency change — the measured period is their mean + spacing, so keeping them would confirm the new frequency against a mixture. + **Pilot windows are dropped too**: windows frozen at one period do not + transfer to another, and scoring a point in the wrong window is a silent + error. Re-freeze a pilot per frequency if you need pilot-frozen windows. +- **A bad point is skipped, not fatal.** A frequency whose `a₀` is unreachable, + whose trigger never confirms, or whose recording fails is skipped and named in + the final summary; the remaining decades are still recorded. The lock table + keeps the failed attempt. +- **The plan is checked before the drive moves.** The lowest planned frequency + decides whether the photodiode can measure `a` at all, so it is checked up + front — not at the ninth point, two hours in. + +Every point's sidecar gains a `[frequency_sweep]` section: `min_f`, `max_f`, +`planned_points`, the position in the **executed** order, the order name, the +seed, whether the point is an interleaved reference, and the requested +frequency (`[trigger]` carries what the markers measured). + +### What the ladder still does not do + +The flux point `I_k`, the camera configuration, ROI/mask, pedestal, bias set, +gates, the `I_tot` anchor, the zero-depth background, the pilot, and repeating +independent blocks stay the operator's. `a₀` itself is an operator input, and +the refractory condition `2·f·a₀/C ≪ 1/τ_refr` is **not** checked — verify it at +your highest planned frequency when you pick `a₀`. ## Controls diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index dec9534..b230c53 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -5,6 +5,7 @@ - **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button press forwarding), + [ADR 014](../adr/014-stage-a-a1-frequency-ladder.md) (the unattended ladder), [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (exact-event-count `a₀` lock) - **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) @@ -47,7 +48,7 @@ folder. A1 makes each recording one button press: | Record pilot | records a bright reference (`…_pilot`) **and** freezes the ON/OFF windows for the row from the live signal | | Record background | records an unmodulated reference (`…_background`) **and** captures the false-response floor `q0` | | Stop (abort recording / sweep) | finalize the current recording early; during a sweep also aborts the remaining points | -| a₀ / Find a₀ / Record a₀ point | the **exact-event-count** workflow: hold one *measured* depth `a₀` across the frequency sweep — see [its brief](./stage-a-a1-event-count.md) | +| a₀ / Find a₀ / Record a₀ point / Start frequency sweep | the **exact-event-count** workflow: hold one *measured* depth `a₀` across the frequency sweep, by hand or as an unattended ladder — see [its brief](./stage-a-a1-event-count.md) | The record and sweep buttons stay **disabled until an output folder is selected**. diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 4ab0638..11e3e4d 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -72,6 +72,8 @@ pixels come from the augur-rs camera config. See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the full brief, [ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, [docs/features/stage-a-a1-event-count.md](../../docs/features/stage-a-a1-event-count.md) plus -[ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, and +[ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, +[ADR 014](../../docs/adr/014-stage-a-a1-frequency-ladder.md) for the unattended +frequency ladder, and [docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the planned amplitude-sweep automation on top of this. diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index e32636e..209bdb9 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -127,6 +127,19 @@ const FREQUENCY_MATCH_FRACTION: f64 = 0.01; /// Lock table persisted in the output folder, so found depths survive a restart. const A0_LOCK_FILE: &str = "a0_locks.json"; +/// Frequency points a single run may visit, before the interleaved references. +const FREQ_SWEEP_MAX_POINTS: usize = 64; +/// How long the frequency sweep waits for the phase-0 trigger to report the +/// frequency it just commanded, before it gives that point up. +/// +/// The drive is a firmware table rebuild plus however long the camera takes to +/// deliver two markers at the new period — at 0.1 Hz that is 20 s on its own. +const FREQ_CONFIRM_BASE_MS: u64 = 20_000; +/// Marker periods that must elapse at the *new* frequency before the sweep +/// believes the measured period. Below this the mean spacing is still a mixture +/// of the old and the new drive. +const FREQ_CONFIRM_CYCLES: f64 = 4.0; + /// Absolute/relative tolerance for "the measured `a` reached the sweep target". fn sweep_tolerance(target_a: f64) -> f64 { (target_a * 0.10).max(0.05) @@ -379,6 +392,11 @@ struct Sweep { lease_id: LeaseId, lease_granted: bool, lease_req: u64, + /// False when the lease belongs to an enclosing run (the frequency sweep): + /// then this run neither acquires nor releases it, so the operator's drive + /// settings stay locked out across the whole ladder rather than only + /// between its points. + owns_lease: bool, depth_req: u64, depth_applied: bool, /// Instant the measured `a` first satisfied the tolerance, for the dwell. @@ -458,6 +476,8 @@ struct A0Lock { lease_id: LeaseId, lease_granted: bool, lease_req: u64, + /// See [`Sweep::owns_lease`]. + owns_lease: bool, depth_req: u64, depth_applied: bool, last_activity_ms: u64, @@ -484,6 +504,121 @@ struct A0LockPoint { high_clip_fraction: Option, } +/// Order the planned frequencies are actually visited in. +/// +/// A Bode ladder recorded strictly low-to-high confounds frequency with +/// everything that drifts monotonically during the block — bleaching, thermal +/// drift of the Pockels bias, source ageing. The A1 checklist therefore asks +/// for a randomised or alternating schedule, and for the seed to be part of the +/// frozen session plan; both are reproduced in the sidecar. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum FreqOrder { + #[default] + Ascending, + Descending, + /// Lowest, highest, second lowest, second highest, … — a deterministic + /// alternation that decorrelates frequency from time without a seed. + Alternating, + /// Seeded shuffle; the seed is an operator setting and is recorded. + Random, +} + +impl FreqOrder { + fn from_index(index: u64) -> Self { + match index { + 1 => Self::Descending, + 2 => Self::Alternating, + 3 => Self::Random, + _ => Self::Ascending, + } + } + + fn index(self) -> u64 { + match self { + Self::Ascending => 0, + Self::Descending => 1, + Self::Alternating => 2, + Self::Random => 3, + } + } + + fn label(self) -> &'static str { + match self { + Self::Ascending => "ascending", + Self::Descending => "descending", + Self::Alternating => "alternating", + Self::Random => "random", + } + } +} + +/// One stop of the frequency sweep. +#[derive(Debug, Clone, Copy, PartialEq)] +struct FreqSweepPoint { + frequency_hz: f64, + /// True for the interleaved low-frequency reference repeats, which exist to + /// expose drift across the block rather than to add a new frequency. + is_reference: bool, +} + +/// Where the multi-frequency run is within its per-point cycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FreqSweepPhase { + /// AcquireLease sent to the modulation owner; waiting for the grant. + AcquiringLease, + /// SetDriveFrequency for the current point sent; waiting for Applied. + SettingFrequency, + /// Waiting for the phase-0 trigger to actually report the new period. + ConfirmingFrequency, + /// The `a₀` lock owns this phase. + Locking, + /// The one-point event-count sweep owns this phase. + Recording, +} + +/// One "find a₀ and record a point at every frequency" run. +/// +/// It is a supervisor, not a third copy of the machinery: per point it +/// retargets the leased drive's frequency, waits for the trigger to confirm it, +/// then hands off to the unchanged `a₀` lock and the unchanged event-count +/// point — both running on *this* run's lease, so the operator's drive settings +/// stay locked out from the first frequency to the last. +struct FreqSweep { + phase: FreqSweepPhase, + points: Vec, + index: usize, + lease_id: LeaseId, + lease_granted: bool, + lease_req: u64, + freq_req: u64, + freq_applied: bool, + /// Give-up deadline for the trigger to confirm the commanded frequency. + confirm_deadline_ms: u64, + /// Why the current point is being given up, when that was decided in a + /// service reply rather than in the tick. Carries the owner's own wording + /// through to the skip message instead of replacing it with a timeout. + skip_reason: Option, + /// Points whose `a₀` could not be locked or whose recording failed. Kept + /// and reported rather than aborting the ladder: the remaining frequencies + /// are still worth having, and the lock table already carries the detail. + failed: Vec, + recorded: usize, + order: FreqOrder, + seed: u64, + last_activity_ms: u64, + stop_requested: bool, +} + +impl FreqSweep { + fn point(&self) -> Option { + self.points.get(self.index).copied() + } + + fn frequency_hz(&self) -> f64 { + self.point().map(|point| point.frequency_hz).unwrap_or(0.0) + } +} + /// On-disk form of the per-frequency lock table. #[derive(Debug, Clone, Default, Serialize, serde::Deserialize)] struct A0LockTable { @@ -564,6 +699,20 @@ pub struct StageAA1Plugin { /// Latched by the Record a₀ point button, consumed next control tick. a0_point_pending: bool, a0_lock: Option, + // -- multi-frequency run over the a₀ ladder -- + /// Frequency range and resolution of the planned ladder. Log-spaced: a Bode + /// ladder is read per decade, not per hertz. + min_f: f64, + max_f: f64, + freq_count: u32, + freq_order: FreqOrder, + freq_seed: u64, + /// Insert the lowest planned frequency again after every N points, so drift + /// across the block shows up as a disagreement between its repeats. 0 = off. + freq_reference_every: u32, + /// Latched by the Start frequency sweep button, consumed next control tick. + freq_sweep_pending: bool, + freq_sweep: Option, /// One converged (or attempted) lock per frequency, newest per frequency /// wins; mirrored to `a0_locks.json` in the output folder. a0_locks: Vec, @@ -580,6 +729,7 @@ pub struct StageAA1Plugin { press_record_point: PressLatch, press_clear_curve: PressLatch, press_find_a0: PressLatch, + press_freq_sweep: PressLatch, press_record_a0: PressLatch, press_clear_a0: PressLatch, } @@ -629,6 +779,14 @@ impl Default for StageAA1Plugin { a0_lock_pending: false, a0_point_pending: false, a0_lock: None, + min_f: 1.0, + max_f: 100.0, + freq_count: 7, + freq_order: FreqOrder::Alternating, + freq_seed: 1, + freq_reference_every: 0, + freq_sweep_pending: false, + freq_sweep: None, a0_locks: Vec::new(), loaded_locks_folder: None, press_start: PressLatch::default(), @@ -640,6 +798,7 @@ impl Default for StageAA1Plugin { press_record_point: PressLatch::default(), press_clear_curve: PressLatch::default(), press_find_a0: PressLatch::default(), + press_freq_sweep: PressLatch::default(), press_record_a0: PressLatch::default(), press_clear_a0: PressLatch::default(), } @@ -1601,7 +1760,7 @@ impl StageAA1Plugin { "Sweep: acquiring modulation lease for {} points…", points.len() ); - self.begin_leased_sweep(context, SweepKind::Amplitude, points, None, message); + self.begin_leased_sweep(context, SweepKind::Amplitude, points, None, None, message); } /// Shared entry point for both leased recording runs (amplitude sweep and @@ -1613,6 +1772,7 @@ impl StageAA1Plugin { kind: SweepKind, points: Vec, lock: Option, + inherited_lease: Option, message: String, ) { if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { @@ -1636,12 +1796,18 @@ impl StageAA1Plugin { return; } let now_ms = now_unix_ms(); - let lease_id = LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))); - let ttl_ms = self.sweep_lease_ttl_ms(points.len()); - let request = - self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); - let lease_req = request.request_id; - context.request_service(&request); + let owns_lease = inherited_lease.is_none(); + let lease_id = inherited_lease.unwrap_or_else(|| { + LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))) + }); + let mut lease_req = 0; + if owns_lease { + let ttl_ms = self.sweep_lease_ttl_ms(points.len()); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + lease_req = request.request_id; + context.request_service(&request); + } self.sweep = Some(Sweep { phase: SweepPhase::AcquiringLease, kind, @@ -1649,8 +1815,11 @@ impl StageAA1Plugin { lock, index: 0, lease_id, - lease_granted: false, + // An inherited lease is already granted; the first tick goes + // straight to retargeting the depth. + lease_granted: !owns_lease, lease_req, + owns_lease, depth_req: 0, depth_applied: false, settled_since_ms: None, @@ -1668,7 +1837,11 @@ impl StageAA1Plugin { /// which also locks the operator's drive settings out for the whole point, so /// the amplitude provably cannot change during the recorded interval — and the /// point is then recorded through the same coordinator as every other run. - fn begin_a0_point(&mut self, context: &mut impl RecordingControl) { + fn begin_a0_point( + &mut self, + context: &mut impl RecordingControl, + inherited_lease: Option, + ) { let Some(hz) = self.frequency_hz() else { self.message = "No modulation frequency yet — arm the drive first".into(); return; @@ -1690,13 +1863,20 @@ impl StageAA1Plugin { lock.commanded_a, lock.target_a ); - self.begin_leased_sweep(context, SweepKind::EventCount, points, Some(lock), message); + self.begin_leased_sweep( + context, + SweepKind::EventCount, + points, + Some(lock), + inherited_lease, + message, + ); } /// Release the modulation lease (if held) and clear the sweep. fn finish_sweep(&mut self, context: &mut impl RecordingControl, message: String) { if let Some(sweep) = self.sweep.take() { - if sweep.lease_granted { + if sweep.owns_lease && sweep.lease_granted { let request = self.modulation_request( ModulationCommandV1::ReleaseLease { safe_off: false, @@ -1763,7 +1943,7 @@ impl StageAA1Plugin { if std::mem::take(&mut self.sweep_pending) { self.begin_sweep(context); } else if std::mem::take(&mut self.a0_point_pending) { - self.begin_a0_point(context); + self.begin_a0_point(context, None); } return; } @@ -2061,7 +2241,11 @@ impl StageAA1Plugin { } /// Kick off the closed-loop `a₀` lock at the current frequency. - fn begin_a0_lock(&mut self, context: &mut impl RecordingControl) { + fn begin_a0_lock( + &mut self, + context: &mut impl RecordingControl, + inherited_lease: Option, + ) { if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { self.message = "A recording, sweep or a₀ lock is already running".into(); return; @@ -2106,12 +2290,18 @@ impl StageAA1Plugin { .map(|lock| lock.commanded_a) .unwrap_or(target); let now_ms = now_unix_ms(); - let lease_id = LeaseId::new(format!("a1-a0-{}", format_compact_utc(now_ms / 1_000))); - let ttl_ms = self.a0_lock_lease_ttl_ms(); - let request = - self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); - let lease_req = request.request_id; - context.request_service(&request); + let owns_lease = inherited_lease.is_none(); + let lease_id = inherited_lease.unwrap_or_else(|| { + LeaseId::new(format!("a1-a0-{}", format_compact_utc(now_ms / 1_000))) + }); + let mut lease_req = 0; + if owns_lease { + let ttl_ms = self.a0_lock_lease_ttl_ms(); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + lease_req = request.request_id; + context.request_service(&request); + } self.a0_lock = Some(A0Lock { phase: A0LockPhase::AcquiringLease, target_a: target, @@ -2125,17 +2315,25 @@ impl StageAA1Plugin { window_ms: 0, deadline_ms: 0, lease_id, - lease_granted: false, + lease_granted: !owns_lease, lease_req, + owns_lease, depth_req: 0, depth_applied: false, last_activity_ms: now_ms, stop_requested: false, }); - self.message = format!( - "a₀ lock at {}: acquiring the modulation lease…", - frequency_label(hz) - ); + self.message = if owns_lease { + format!( + "a₀ lock at {}: acquiring the modulation lease…", + frequency_label(hz) + ) + } else { + format!( + "a₀ lock at {}: trimming the drive depth…", + frequency_label(hz) + ) + }; } /// Renew the lease and command the current trial's depth. @@ -2181,7 +2379,7 @@ impl StageAA1Plugin { /// the event-count point that follows records at `a₀`. fn finish_a0_lock(&mut self, context: &mut impl RecordingControl, message: String) { if let Some(lock) = self.a0_lock.take() { - if lock.lease_granted { + if lock.owns_lease && lock.lease_granted { let request = self.modulation_request( ModulationCommandV1::ReleaseLease { safe_off: false, @@ -2428,7 +2626,7 @@ impl StageAA1Plugin { fn drive_a0_lock(&mut self, context: &mut impl RecordingControl) { if self.a0_lock.is_none() { if std::mem::take(&mut self.a0_lock_pending) { - self.begin_a0_lock(context); + self.begin_a0_lock(context, None); } return; } @@ -2554,13 +2752,584 @@ impl StageAA1Plugin { lock.last_activity_ms = now_unix_ms(); } } - // The owner refuses a depth its calibrated drive cannot express - // (lobe ceiling, DAC limit) — that *is* the "a₀ unreachable at - // this operating point" answer, so surface its wording verbatim. - PluginServiceOutcome::Rejected { message, .. } => abort( - self, - format!("a₀ lock aborted: the drive rejected the commanded depth: {message}"), - ), + // The owner refuses a depth its calibrated drive cannot express + // (lobe ceiling, DAC limit) — that *is* the "a₀ unreachable at + // this operating point" answer, so surface its wording verbatim. + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("a₀ lock aborted: the drive rejected the commanded depth: {message}"), + ), + } + true + } else { + false + } + } + + // ---- multi-frequency a₀ ladder ----------------------------------------- + + /// The planned frequency ladder, log-spaced and inclusive of both ends. + /// + /// Log spacing because `|H(f)|` is read per decade: a linear ladder spends + /// most of its points where the response is flat and none where it rolls + /// off. + fn planned_frequencies(&self) -> Vec { + let count = self.freq_count.clamp(1, FREQ_SWEEP_MAX_POINTS as u32) as usize; + if count == 1 { + return vec![self.min_f]; + } + let (low, high) = (self.min_f.ln(), self.max_f.ln()); + (0..count) + .map(|index| (low + (high - low) * index as f64 / (count - 1) as f64).exp()) + .collect() + } + + /// The planned ladder in the order it will actually be visited, with the + /// interleaved low-frequency reference repeats inserted. + fn freq_sweep_points(&self) -> Vec { + let mut ladder = self.planned_frequencies(); + match self.freq_order { + FreqOrder::Ascending => {} + FreqOrder::Descending => ladder.reverse(), + FreqOrder::Alternating => { + // Lowest, highest, second lowest, second highest, … + let mut out = Vec::with_capacity(ladder.len()); + let (mut low, mut high) = (0usize, ladder.len()); + while low < high { + out.push(ladder[low]); + low += 1; + if low < high { + high -= 1; + out.push(ladder[high]); + } + } + ladder = out; + } + FreqOrder::Random => { + // A seeded Fisher-Yates with a small xorshift, so the executed + // order is reproducible from the seed recorded in the sidecar. + let mut state = self.freq_seed.max(1); + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for index in (1..ladder.len()).rev() { + ladder.swap(index, (next() % (index as u64 + 1)) as usize); + } + } + } + let reference_hz = self.planned_frequencies().first().copied(); + let every = self.freq_reference_every as usize; + let mut points = Vec::with_capacity(ladder.len() * 2); + for (visited, frequency_hz) in ladder.into_iter().enumerate() { + points.push(FreqSweepPoint { + frequency_hz, + is_reference: false, + }); + // Interleave the low-frequency reference so drift across the block + // shows up as a disagreement between its repeats (A1 checklist, + // "interleave a low-frequency reference to expose drift"). + if let Some(reference_hz) = reference_hz.filter(|_| every > 0) { + if (visited + 1) % every == 0 { + points.push(FreqSweepPoint { + frequency_hz: reference_hz, + is_reference: true, + }); + } + } + } + points + } + + /// Lease TTL for the whole ladder: every point pays a lock and a recording. + fn freq_sweep_lease_ttl_ms(&self, remaining_points: usize) -> u64 { + let per_point_ms = self + .a0_lock_lease_ttl_ms() + .saturating_add(self.sweep_lease_ttl_ms(1)) + .saturating_add(FREQ_CONFIRM_BASE_MS); + (remaining_points as u64) + .saturating_mul(per_point_ms) + .saturating_add(60_000) + } + + /// Kick off the multi-frequency run: validate the whole plan, then lease. + /// + /// Everything checkable is checked *here*, before the drive moves: a plan + /// that cannot work at its lowest frequency should say so in a message, not + /// two hours into a block. + fn begin_freq_sweep(&mut self, context: &mut impl RecordingControl) { + if self.recording.is_active() + || self.sweep.is_some() + || self.a0_lock.is_some() + || self.freq_sweep.is_some() + { + self.message = "A recording, sweep or a₀ lock is already running".into(); + return; + } + if self.output_folder.trim().is_empty() { + self.message = "Set an output folder before sweeping the frequency".into(); + return; + } + if self.measurement_id.trim().is_empty() { + self.message = "Set a measurement id before sweeping the frequency".into(); + return; + } + if !self.modulation_connected() { + self.message = "Modulation owner is not connected — cannot drive the frequency".into(); + return; + } + // Written through `partial_cmp` so a NaN from the settings drag is + // rejected rather than silently passing a negated comparison. + let range_ok = self.min_f.partial_cmp(&0.0) == Some(std::cmp::Ordering::Greater) + && matches!( + self.max_f.partial_cmp(&self.min_f), + Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) + ); + if !range_ok { + self.message = "Frequency sweep needs 0 < min f ≤ max f".into(); + return; + } + if self.measured_a().is_none() { + self.message = + "No photodiode-measured a — connect the photodiode and anchor I_tot first".into(); + return; + } + let target = self.a0_target; + if !(COMMANDED_A_MIN..=COMMANDED_A_MAX).contains(&target) { + self.message = format!( + "a₀ = {target:.3} is outside the drivable {COMMANDED_A_MIN}..={COMMANDED_A_MAX}" + ); + return; + } + // The photodiode estimates `a` over one window for all frequencies, so + // the *lowest* planned frequency decides whether the ladder is + // measurable at all. Refuse the plan, not its 9th point. + if let Err(reason) = self.optical_window_covers_a_cycle(self.min_f) { + self.message = format!("Frequency sweep refused at its lowest point: {reason}"); + return; + } + if !self.is_marker_anchored() { + // Without the phase-0 trigger there is nothing that can confirm the + // drive actually reached a commanded frequency, and the fold has no + // anchor either. + self.message = "No phase-0 trigger markers — the sweep cannot confirm a commanded \ + frequency. Enable Live analysis and check EXT_TRIGGER" + .into(); + return; + } + let points = self.freq_sweep_points(); + if points.is_empty() { + self.message = "Nothing to sweep: the frequency ladder has no points".into(); + return; + } + let now_ms = now_unix_ms(); + let lease_id = LeaseId::new(format!("a1-fsweep-{}", format_compact_utc(now_ms / 1_000))); + let ttl_ms = self.freq_sweep_lease_ttl_ms(points.len()); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + let lease_req = request.request_id; + context.request_service(&request); + let total = points.len(); + self.freq_sweep = Some(FreqSweep { + phase: FreqSweepPhase::AcquiringLease, + points, + index: 0, + lease_id, + lease_granted: false, + lease_req, + freq_req: 0, + freq_applied: false, + confirm_deadline_ms: 0, + skip_reason: None, + failed: Vec::new(), + recorded: 0, + order: self.freq_order, + seed: self.freq_seed, + last_activity_ms: now_ms, + stop_requested: false, + }); + self.message = format!( + "Frequency sweep: acquiring the modulation lease for {total} points ({} order)…", + self.freq_order.label() + ); + } + + /// Release the ladder's lease (if this run holds it) and clear the sweep. + /// + /// `safe_off = false` as everywhere else: stopping the drive is the owner's + /// lease-expiry job, not a sweep's. Releasing does hand the operator's own + /// frequency and depth back, because the owner parks them on the first + /// retarget. + fn finish_freq_sweep(&mut self, context: &mut impl RecordingControl, message: String) { + if let Some(sweep) = self.freq_sweep.take() { + if sweep.lease_granted { + let request = self.modulation_request( + ModulationCommandV1::ReleaseLease { + safe_off: false, + reason: "a1 frequency sweep finished".into(), + }, + &sweep.lease_id, + ); + context.request_service(&request); + } + } + self.message = message; + } + + /// Renew the ladder's lease and retarget the drive at the current point. + fn send_freq_sweep_frequency(&mut self, context: &mut impl RecordingControl) { + let Some(sweep) = self.freq_sweep.as_ref() else { + return; + }; + let lease_id = sweep.lease_id.clone(); + let remaining = sweep.points.len().saturating_sub(sweep.index); + let hz = sweep.frequency_hz(); + let (index, total) = (sweep.index, sweep.points.len()); + let is_reference = sweep.point().is_some_and(|point| point.is_reference); + + let ttl_ms = self.freq_sweep_lease_ttl_ms(remaining); + let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + context.request_service(&renew); + let request = self.modulation_request( + ModulationCommandV1::SetDriveFrequency { + frequency_millihz: (hz * 1_000.0).round().max(0.0) as u64, + }, + &lease_id, + ); + let freq_req = request.request_id; + context.request_service(&request); + + // The retained markers and events belong to the *previous* frequency: + // the measured period is their mean spacing, so leaving them in place + // would confirm the new frequency against a mixture of the two. The + // pilot windows are frozen at a phase of the old period and are not + // transferable either — a point recorded against them would be scored + // in the wrong window. + self.camera_markers_us.clear(); + self.camera_events.clear(); + self.fold_cache.replace(None); + self.pilot_windows = None; + + let now_ms = now_unix_ms(); + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::SettingFrequency; + sweep.freq_req = freq_req; + sweep.freq_applied = false; + sweep.skip_reason = None; + sweep.last_activity_ms = now_ms; + } + self.message = format!( + "Frequency sweep {}/{total}: retargeting the drive to {}{}…", + index + 1, + frequency_label(hz), + if is_reference { " (reference)" } else { "" }, + ); + } + + /// Give up on the current point and move to the next one. + /// + /// A frequency that cannot be locked or recorded does not end the ladder: + /// the remaining points are still worth having, and the failure is already + /// in the lock table. It is reported in the final summary. + fn fail_freq_sweep_point(&mut self, context: &mut impl RecordingControl, reason: String) { + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.failed.push(hz); + } + self.message = format!( + "Frequency sweep: skipping {} — {reason}", + frequency_label(hz) + ); + self.advance_freq_sweep(context); + } + + /// Move to the next ladder point, or finish with a summary. + fn advance_freq_sweep(&mut self, context: &mut impl RecordingControl) { + let done = match self.freq_sweep.as_mut() { + Some(sweep) => { + sweep.index += 1; + sweep.index >= sweep.points.len() + } + None => return, + }; + if !done { + self.send_freq_sweep_frequency(context); + return; + } + let (recorded, failed, total, order, seed) = self + .freq_sweep + .as_ref() + .map(|sweep| { + ( + sweep.recorded, + sweep.failed.clone(), + sweep.points.len(), + sweep.order, + sweep.seed, + ) + }) + .unwrap_or_default(); + let mut message = format!( + "Frequency sweep complete: {recorded}/{total} points recorded ({} order, seed {seed})", + order.label() + ); + if !failed.is_empty() { + let list = failed + .iter() + .map(|hz| frequency_label(*hz)) + .collect::>() + .join(", "); + message.push_str(&format!( + " — {} skipped: {list}. See the a₀ lock table", + failed.len() + )); + } + self.finish_freq_sweep(context, message); + } + + /// Advance the multi-frequency run one control tick. Runs before the lock + /// and the point sweep, so a child it starts runs on the same tick. + fn drive_freq_sweep(&mut self, context: &mut impl RecordingControl) { + if self.freq_sweep.is_none() { + if std::mem::take(&mut self.freq_sweep_pending) { + self.begin_freq_sweep(context); + } + return; + } + self.freq_sweep_pending = false; + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, freq_applied, last_activity_ms, index, total) = { + let sweep = self.freq_sweep.as_ref().expect("sweep checked above"); + ( + sweep.phase, + sweep.stop_requested, + sweep.lease_granted, + sweep.freq_applied, + sweep.last_activity_ms, + sweep.index, + sweep.points.len(), + ) + }; + // A stop propagates into whichever child is running; the ladder ends + // once that child has let go. + if stop_requested { + if let Some(lock) = self.a0_lock.as_mut() { + lock.stop_requested = true; + return; + } + if let Some(sweep) = self.sweep.as_mut() { + sweep.stop_requested = true; + return; + } + let message = if self.message.is_empty() { + "Frequency sweep stopped".into() + } else { + self.message.clone() + }; + self.finish_freq_sweep(context, message); + return; + } + match phase { + FreqSweepPhase::AcquiringLease => { + if lease_granted { + self.send_freq_sweep_frequency(context); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_freq_sweep( + context, + "Frequency sweep aborted: timed out acquiring the modulation lease".into(), + ); + } + } + FreqSweepPhase::SettingFrequency => { + // A refused frequency is a property of this point, not of the + // ladder; the owner already said why. + if let Some(reason) = self + .freq_sweep + .as_mut() + .and_then(|sweep| sweep.skip_reason.take()) + { + self.fail_freq_sweep_point(context, reason); + } else if freq_applied { + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + // Confirming needs whole cycles at the *new* period, so the + // budget has to scale with it: 4 cycles at 0.1 Hz is 40 s. + let cycles_ms = if hz > 0.0 { + (FREQ_CONFIRM_CYCLES / hz * 1_000.0).ceil() as u64 + } else { + 0 + }; + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::ConfirmingFrequency; + sweep.confirm_deadline_ms = + now_ms.saturating_add(FREQ_CONFIRM_BASE_MS.max(cycles_ms * 3)); + } + self.message = format!( + "Frequency sweep {}/{total}: waiting for the trigger to report {}…", + index + 1, + frequency_label(hz), + ); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_freq_sweep( + context, + "Frequency sweep aborted: timed out retargeting the drive frequency".into(), + ); + } + } + FreqSweepPhase::ConfirmingFrequency => { + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + // The trigger *defines* the frequency, so the point only starts + // once the markers say the drive is really there — an ACK from + // the firmware says the table was accepted, not that the light + // is modulating at that rate. Enough markers must have arrived + // at the new period for their mean spacing to mean anything. + let enough_markers = self.camera_markers_us.len() as f64 >= FREQ_CONFIRM_CYCLES; + let confirmed = enough_markers + && self + .frequency_hz() + .is_some_and(|measured| same_frequency(measured, hz)); + let deadline = self + .freq_sweep + .as_ref() + .map(|sweep| sweep.confirm_deadline_ms) + .unwrap_or_default(); + if confirmed { + if let Err(reason) = self.optical_window_covers_a_cycle(hz) { + self.fail_freq_sweep_point(context, reason); + return; + } + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::Locking; + } + let lease = self.freq_sweep.as_ref().map(|sweep| sweep.lease_id.clone()); + self.begin_a0_lock(context, lease); + if self.a0_lock.is_none() { + // `begin_a0_lock` refused and said why; keep its wording. + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + } + } else if now_ms >= deadline { + let measured = self + .frequency_hz() + .map_or_else(|| "—".into(), frequency_label); + self.fail_freq_sweep_point( + context, + format!( + "the trigger never reported it (measured {measured} from {} markers)", + self.camera_markers_us.len() + ), + ); + } + } + FreqSweepPhase::Locking => { + if self.a0_lock.is_some() { + return; + } + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + // A non-converged lock is stored but never arms a recording, so + // `armed_lock` is the single question worth asking here. + if self.armed_lock().is_none() { + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + return; + } + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::Recording; + } + let lease = self.freq_sweep.as_ref().map(|sweep| sweep.lease_id.clone()); + self.begin_a0_point(context, lease); + if self.sweep.is_none() { + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + return; + } + self.message = format!( + "Frequency sweep {}/{total}: recording the a₀ point at {}…", + index + 1, + frequency_label(hz), + ); + } + FreqSweepPhase::Recording => { + if self.sweep.is_some() || self.recording.is_active() { + return; + } + if self.recording_completed_ok { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.recorded += 1; + } + self.advance_freq_sweep(context); + } else { + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + } + } + } + } + + /// Routes modulation-service replies belonging to the frequency sweep. + fn on_freq_sweep_reply(&mut self, reply: &PluginServiceReply) -> bool { + let Some((lease_req, freq_req)) = self + .freq_sweep + .as_ref() + .map(|sweep| (sweep.lease_req, sweep.freq_req)) + else { + return false; + }; + let abort = |this: &mut Self, message: String| { + this.message = message; + if let Some(sweep) = this.freq_sweep.as_mut() { + sweep.stop_requested = true; + } + }; + if reply.request_id == lease_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.lease_granted = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("Frequency sweep aborted: modulation lease rejected: {message}"), + ), + } + true + } else if reply.request_id == freq_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.freq_applied = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + // A refused frequency is a property of this point, not of the + // ladder: skip it and keep the remaining decades. The skip runs + // on the next tick, through the one path that advances the + // ladder, carrying the owner's wording. + PluginServiceOutcome::Rejected { message, .. } => { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.skip_reason = + Some(format!("the drive rejected the frequency: {message}")); + } + } } true } else { @@ -2648,7 +3417,10 @@ impl StageAA1Plugin { } fn on_service_reply(&mut self, reply: &PluginServiceReply) { - if self.on_sweep_reply(reply) || self.on_a0_lock_reply(reply) { + if self.on_sweep_reply(reply) + || self.on_a0_lock_reply(reply) + || self.on_freq_sweep_reply(reply) + { return; } let response = match &reply.outcome { @@ -2870,6 +3642,19 @@ impl StageAA1Plugin { converged: lock.converged, locked_at_utc: format_iso_utc(lock.locked_at_unix_ms / 1_000), }), + frequency_sweep: self.freq_sweep.as_ref().and_then(|sweep| { + sweep.point().map(|point| FreqSweepSidecar { + min_f: self.min_f, + max_f: self.max_f, + planned_points: self.freq_count as usize, + point_index: sweep.index + 1, + point_total: sweep.points.len(), + order: sweep.order.label().into(), + seed: sweep.seed, + is_reference: point.is_reference, + requested_frequency_hz: point.frequency_hz, + }) + }), pilot: (self.recording.role == RecRole::Pilot) .then_some(self.pilot_windows) .flatten() @@ -2942,6 +3727,9 @@ struct SidecarDoc { /// Present on **event-count** points: the `a₀` lock this point replayed. #[serde(skip_serializing_if = "Option::is_none")] a0_lock: Option, + /// Present on points recorded by the automatic frequency ladder. + #[serde(skip_serializing_if = "Option::is_none")] + frequency_sweep: Option, #[serde(skip_serializing_if = "Option::is_none")] pilot: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -2986,6 +3774,29 @@ struct A0LockSidecar { locked_at_utc: String, } +/// The automatic frequency ladder this point belongs to. +/// +/// The executed order and its seed are part of the frozen session schedule the +/// A1 checklist asks for, so they belong in every point rather than only in an +/// operator's notebook: a block is only interpretable if you can tell which +/// frequency was recorded when. +#[derive(Serialize)] +struct FreqSweepSidecar { + min_f: f64, + max_f: f64, + planned_points: usize, + /// Position in the *executed* order, references included. + point_index: usize, + point_total: usize, + order: String, + seed: u64, + /// True for the interleaved low-frequency reference repeats. + is_reference: bool, + /// The ladder asked for this frequency; `[trigger] measured_frequency_hz` + /// is what the phase-0 markers reported when the point was recorded. + requested_frequency_hz: f64, +} + /// Frozen ON/OFF windows written into a **pilot** recording's sidecar and read /// back to reuse them across the row. #[derive(Serialize, serde::Deserialize)] @@ -3386,8 +4197,11 @@ impl Plugin for StageAA1Plugin { self.scan_measurement_folder(); self.load_a0_locks(); } - // The lock and the sweep run first so a point's recording starts on the - // same tick. They are mutually exclusive, guarded when they begin. + // Outermost first: the frequency sweep starts the lock or the point it + // supervises, and each of those starts its own next stage, so one tick + // carries a hand-off all the way down. They are mutually exclusive at + // the top, guarded where they begin. + self.drive_freq_sweep(context); self.drive_a0_lock(context); self.drive_sweep(context); self.drive_recording(context); @@ -3679,6 +4493,125 @@ impl Plugin for StageAA1Plugin { enabled: can_record, }, }, + SettingItem { + key: "min_f".into(), + label: "Sweep min f (Hz)".into(), + tooltip: Some( + "Lowest frequency of the automatic ladder. It decides whether the \ + ladder is measurable at all: the photodiode needs a contrast \ + window of at least one cycle at this frequency, so raise its \ + Cache length if the sweep refuses to start." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 0.1, + default: self.min_f, + }, + }, + SettingItem { + key: "max_f".into(), + label: "Sweep max f (Hz)".into(), + tooltip: Some( + "Highest frequency of the automatic ladder. Check the refractory \ + condition 2·f·a₀/C ≪ 1/τ_refr here — the plugin does not." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.max_f, + }, + }, + SettingItem { + key: "freq_count".into(), + label: "Frequency points".into(), + tooltip: Some( + "Points on the ladder, log-spaced and inclusive of both ends: \ + |H(f)| is read per decade, so a linear ladder would spend most of \ + its points on the flat part." + .into(), + ), + kind: SettingKind::I64Slider { + min: 1, + max: FREQ_SWEEP_MAX_POINTS as i64, + default: i64::from(self.freq_count), + suffix: None, + }, + }, + SettingItem { + key: "freq_order".into(), + label: "Frequency order".into(), + tooltip: Some( + "Order the ladder is visited in. Low-to-high confounds frequency \ + with anything that drifts through the block (bleaching, thermal \ + bias drift), so prefer alternating or a seeded random order — \ + both are recorded in the sidecar." + .into(), + ), + kind: SettingKind::Enum { + variants: vec![ + "ascending".into(), + "descending".into(), + "alternating".into(), + "random (seeded)".into(), + ], + default: self.freq_order.index() as usize, + }, + }, + SettingItem { + key: "freq_seed".into(), + label: "Random order seed".into(), + tooltip: Some( + "Seed for the random order, so the executed schedule is \ + reproducible and can be frozen in the session plan. Recorded in \ + every point's sidecar." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 9_999, + default: self.freq_seed as i64, + }, + }, + SettingItem { + key: "freq_reference_every".into(), + label: "Low-f reference every N points".into(), + tooltip: Some( + "Re-visit the lowest planned frequency after every N points, so \ + drift across the block shows up as a disagreement between its \ + repeats. 0 disables it." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: 10, + default: i64::from(self.freq_reference_every), + suffix: None, + }, + }, + SettingItem { + key: "start_freq_sweep".into(), + label: "Start frequency sweep (find a₀ + record per f)".into(), + tooltip: Some( + "Runs the whole ladder unattended on one modulation lease: per \ + frequency it retargets the drive, waits for the phase-0 trigger \ + to confirm the new period, locks a₀ closed-loop, and records one \ + atomic RAW + PDQ + sidecar point. A frequency whose a₀ cannot be \ + reached is skipped and named in the summary rather than ending \ + the ladder. The operator's own frequency and depth come back when \ + the lease is released. References (pilot, background, I_tot \ + anchor) and the flux point stay yours — and pilot windows are \ + dropped at every frequency change, because windows frozen at one \ + period do not transfer to another." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, SettingItem { key: "clear_a0_locks".into(), label: "Clear a₀ lock table".into(), @@ -3824,6 +4757,13 @@ impl Plugin for StageAA1Plugin { "find_a0" => Some(self.press_find_a0.value()), "record_a0_point" => Some(self.press_record_a0.value()), "clear_a0_locks" => Some(self.press_clear_a0.value()), + "min_f" => Some(json!(self.min_f)), + "max_f" => Some(json!(self.max_f)), + "freq_count" => Some(json!(self.freq_count)), + "freq_order" => Some(json!(self.freq_order.index())), + "freq_seed" => Some(json!(self.freq_seed)), + "freq_reference_every" => Some(json!(self.freq_reference_every)), + "start_freq_sweep" => Some(self.press_freq_sweep.value()), // New id regenerates the measurement id locally; the id itself is // what synchronizes, so the press must not be forwarded (both // instances would generate different ids). @@ -3912,9 +4852,16 @@ impl Plugin for StageAA1Plugin { lock.stop_requested = true; self.message = "a₀ lock stop requested".into(); } + // Last, so its wording wins: a stop during a ladder is a + // stop of the ladder, whatever child was mid-flight. + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.stop_requested = true; + self.message = "Frequency sweep stop requested".into(); + } self.sweep_pending = false; self.a0_lock_pending = false; self.a0_point_pending = false; + self.freq_sweep_pending = false; } } "live" => { @@ -3976,6 +4923,37 @@ impl Plugin for StageAA1Plugin { self.a0_point_pending = true; } } + "min_f" => { + self.min_f = value.as_f64().ok_or("min_f must be a number")?.max(0.01); + } + "max_f" => { + self.max_f = value.as_f64().ok_or("max_f must be a number")?.max(0.01); + } + "freq_count" => { + self.freq_count = value + .as_u64() + .ok_or("freq_count must be an integer")? + .clamp(1, FREQ_SWEEP_MAX_POINTS as u64) + as u32; + } + "freq_order" => { + self.freq_order = + FreqOrder::from_index(value.as_u64().ok_or("freq_order must be an index")?); + } + "freq_seed" => { + self.freq_seed = value.as_u64().ok_or("freq_seed must be an integer")?.max(1); + } + "freq_reference_every" => { + self.freq_reference_every = value + .as_u64() + .ok_or("freq_reference_every must be an integer")? + .min(10) as u32; + } + "start_freq_sweep" => { + if self.press_freq_sweep.accept(&value) { + self.freq_sweep_pending = true; + } + } "clear_a0_locks" => { if self.press_clear_a0.accept(&value) { self.a0_locks.clear(); @@ -4006,6 +4984,29 @@ impl Plugin for StageAA1Plugin { ))); } } + if let Some(sweep) = &self.freq_sweep { + let phase = match sweep.phase { + FreqSweepPhase::AcquiringLease => "leasing modulation", + FreqSweepPhase::SettingFrequency => "retargeting frequency", + FreqSweepPhase::ConfirmingFrequency => "confirming from the trigger", + FreqSweepPhase::Locking => "locking a₀", + FreqSweepPhase::Recording => "recording", + }; + let point = sweep.point(); + entries.push(StatusEntry::Text(format!( + "Frequency sweep {}/{} at {}{} — {phase} ({} recorded, {} skipped)", + sweep.index + 1, + sweep.points.len(), + frequency_label(sweep.frequency_hz()), + if point.is_some_and(|point| point.is_reference) { + " (reference)" + } else { + "" + }, + sweep.recorded, + sweep.failed.len(), + ))); + } if let Some(sweep) = &self.sweep { let phase = match sweep.phase { SweepPhase::AcquiringLease => "leasing modulation", @@ -4299,6 +5300,9 @@ mod tests { for reply in &inbox.service_replies { plugin.on_service_reply(reply); } + // Same order as `process_control`: outermost supervisor first, so one + // tick can carry a hand-off from the ladder down into a recording. + plugin.drive_freq_sweep(sink); plugin.drive_a0_lock(sink); plugin.drive_sweep(sink); plugin.drive_recording(sink); @@ -5008,6 +6012,7 @@ mod tests { lease_id: LeaseId::new("a1-sweep-test"), lease_granted: true, lease_req: 0, + owns_lease: true, depth_req: 0, depth_applied: true, settled_since_ms: None, @@ -5079,6 +6084,439 @@ mod tests { let _ = std::fs::remove_dir_all(&folder); } + /// Widens the fixture photodiode's contrast window, so it covers a whole + /// cycle at every frequency a ladder test visits (the lock refuses below + /// one cycle, which is the point of a different test). + fn photodiode_window(plugin: &mut StageAA1Plugin, seconds: f64) { + if let Some(summary) = plugin.photodiode.as_mut() { + if let Some(optical) = summary.optical_summary.as_mut() { + optical.window_seconds = Some(seconds); + } + } + } + + /// Rewrites the plugin's phase-0 markers so the trigger reports `hz`, the + /// way the camera would once the drive has really moved. + fn trigger_reports(plugin: &mut StageAA1Plugin, hz: f64) { + let period_us = (1_000_000.0 / hz).round() as u64; + plugin.camera_markers_us = (0..8).map(|index| index * period_us).collect(); + plugin.fold_cache.replace(None); + } + + /// Drives a whole frequency ladder to completion against a bench that + /// delivers `gain ×` the commanded depth, answering every lease/depth/ + /// frequency request and letting the trigger confirm each commanded + /// frequency. Returns the frequencies whose points were recorded, in order. + fn run_freq_sweep_to_completion( + plugin: &mut StageAA1Plugin, + sink: &mut ControlSink, + gain: f64, + max_ticks: usize, + ) -> Vec { + let mut revision = 1; + let mut recorded = Vec::new(); + control_tick(plugin, PluginControlInbox::default(), sink); + for _ in 0..max_ticks { + let Some((phase, target_hz, lease_req, freq_req, granted, applied)) = + plugin.freq_sweep.as_ref().map(|sweep| { + ( + sweep.phase, + sweep.frequency_hz(), + sweep.lease_req, + sweep.freq_req, + sweep.lease_granted, + sweep.freq_applied, + ) + }) + else { + break; + }; + let mut replies = Vec::new(); + match phase { + FreqSweepPhase::AcquiringLease if !granted => replies.push(accepted(lease_req)), + FreqSweepPhase::SettingFrequency if !applied && freq_req != 0 => { + replies.push(accepted(freq_req)); + } + FreqSweepPhase::ConfirmingFrequency => trigger_reports(plugin, target_hz), + FreqSweepPhase::Locking => { + if let Some(lock) = plugin.a0_lock.as_ref() { + let (depth_req, applied, commanded) = + (lock.depth_req, lock.depth_applied, lock.commanded_a); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = + Some(photodiode_measuring(revision, commanded * gain)); + photodiode_window(plugin, 0.02); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + } + FreqSweepPhase::Recording => { + if let Some(sweep) = plugin.sweep.as_ref() { + let (depth_req, applied, expected) = + (sweep.depth_req, sweep.depth_applied, sweep.target_a()); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, expected)); + photodiode_window(plugin, 0.02); + } + } + // Short-circuit the recording coordinator once the sweep + // has seen the point start: this test is about the ladder, + // and the coordinator has tests of its own. + if plugin.recording.is_active() + && plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.point_started) + { + plugin.recording = Recording::idle(); + plugin.recording_completed_ok = true; + recorded.push(target_hz); + } + } + _ => {} + } + control_tick( + plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + sink, + ); + } + recorded + } + + #[test] + fn the_frequency_ladder_is_log_spaced_and_ordered_reproducibly() { + let mut plugin = plugin_with_markers(); + plugin.min_f = 1.0; + plugin.max_f = 100.0; + plugin.freq_count = 3; + + plugin.freq_order = FreqOrder::Ascending; + let ladder = plugin.planned_frequencies(); + // Log-spaced: |H(f)| is read per decade, so a decade per step. + assert_eq!(ladder.len(), 3); + assert!((ladder[0] - 1.0).abs() < 1e-9); + assert!((ladder[1] - 10.0).abs() < 1e-6, "middle {}", ladder[1]); + assert!((ladder[2] - 100.0).abs() < 1e-6); + + // Alternating decorrelates frequency from time without a seed. + plugin.freq_order = FreqOrder::Alternating; + let order: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + assert!((order[0] - 1.0).abs() < 1e-9 && (order[1] - 100.0).abs() < 1e-6); + assert!((order[2] - 10.0).abs() < 1e-6); + + // A seeded random order is reproducible — the seed is in the sidecar. + plugin.freq_order = FreqOrder::Random; + plugin.freq_count = 8; + plugin.freq_seed = 42; + let first: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + let again: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + assert_eq!(first, again, "the seeded order must be reproducible"); + plugin.freq_seed = 43; + let other: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + assert_ne!(first, other, "a different seed must shuffle differently"); + let mut sorted = first.clone(); + sorted.sort_by(f64::total_cmp); + let mut planned = plugin.planned_frequencies(); + planned.sort_by(f64::total_cmp); + assert_eq!(sorted.len(), planned.len(), "the shuffle is a permutation"); + } + + #[test] + fn the_low_frequency_reference_is_interleaved_into_the_ladder() { + let mut plugin = plugin_with_markers(); + plugin.min_f = 1.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 4; + plugin.freq_order = FreqOrder::Ascending; + plugin.freq_reference_every = 2; + + let points = plugin.freq_sweep_points(); + let flags: Vec = points.iter().map(|point| point.is_reference).collect(); + assert_eq!(flags, [false, false, true, false, false, true]); + for point in points.iter().filter(|point| point.is_reference) { + assert!( + (point.frequency_hz - 1.0).abs() < 1e-9, + "the reference repeats the lowest planned frequency" + ); + } + } + + #[test] + fn the_frequency_sweep_locks_and_records_every_point_on_one_lease() { + let folder = temp_folder("fsweep"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.a0_target = 0.5; + plugin.min_f = 100.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 2; + plugin.freq_order = FreqOrder::Ascending; + photodiode_window(&mut plugin, 0.02); + plugin.freq_sweep_pending = true; + let mut sink = ControlSink::default(); + + let recorded = run_freq_sweep_to_completion(&mut plugin, &mut sink, 0.6, 4_000); + + assert_eq!(recorded.len(), 2, "message: {}", plugin.message); + assert!((recorded[0] - 100.0).abs() < 1.0 && (recorded[1] - 1_000.0).abs() < 10.0); + assert!(plugin.freq_sweep.is_none(), "the ladder must finish"); + assert!( + plugin.message.contains("2/2 points recorded"), + "message: {}", + plugin.message + ); + + // One lease for the whole ladder: the operator's drive settings are + // locked out from the first frequency to the last, so the amplitude + // provably cannot move between a lock and the point that replays it. + let commands: Vec = sink + .services + .iter() + .filter_map(|request| { + serde_json::from_value::(request.payload.clone()) + .ok() + .map(|envelope| envelope.command) + }) + .collect(); + let acquired = commands + .iter() + .filter(|command| matches!(command, ModulationCommandV1::AcquireLease { .. })) + .count(); + let released = commands + .iter() + .filter(|command| matches!(command, ModulationCommandV1::ReleaseLease { .. })) + .count(); + assert_eq!(acquired, 1, "one lease for the ladder, not one per child"); + assert_eq!(released, 1, "released exactly once, at the end"); + assert!(commands + .iter() + .any(|command| matches!(command, ModulationCommandV1::SetDriveFrequency { .. }))); + + // Both frequencies are locked, each at the depth its own roll-off needs. + assert_eq!(plugin.a0_locks.len(), 2); + for lock in &plugin.a0_locks { + assert!( + lock.converged, + "lock at {} did not converge", + lock.frequency_hz + ); + assert!((lock.measured_a - 0.5).abs() <= plugin.a0_tolerance); + } + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a_frequency_the_trigger_never_confirms_is_skipped_not_fatal() { + // The firmware ACKs a table it accepted, not light that is modulating. + // A point whose trigger never reports the commanded period is skipped + // and named; the rest of the ladder is still worth having. + let folder = temp_folder("fskip"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + plugin.min_f = 100.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 2; + plugin.freq_order = FreqOrder::Ascending; + photodiode_window(&mut plugin, 0.02); + plugin.freq_sweep_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + let mut revision = 1; + let mut recorded = Vec::new(); + for _ in 0..4_000 { + let Some((phase, target_hz, lease_req, freq_req, granted, applied)) = + plugin.freq_sweep.as_ref().map(|sweep| { + ( + sweep.phase, + sweep.frequency_hz(), + sweep.lease_req, + sweep.freq_req, + sweep.lease_granted, + sweep.freq_applied, + ) + }) + else { + break; + }; + let mut replies = Vec::new(); + match phase { + FreqSweepPhase::AcquiringLease if !granted => replies.push(accepted(lease_req)), + FreqSweepPhase::SettingFrequency if !applied && freq_req != 0 => { + replies.push(accepted(freq_req)); + } + FreqSweepPhase::ConfirmingFrequency => { + // The trigger confirms 100 Hz but never moves to 1 kHz. + if target_hz < 500.0 { + trigger_reports(&mut plugin, target_hz); + } else if let Some(sweep) = plugin.freq_sweep.as_mut() { + sweep.confirm_deadline_ms = 1; + } + } + FreqSweepPhase::Locking => { + if let Some(lock) = plugin.a0_lock.as_ref() { + let (depth_req, applied, commanded) = + (lock.depth_req, lock.depth_applied, lock.commanded_a); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, commanded)); + photodiode_window(&mut plugin, 0.02); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + } + FreqSweepPhase::Recording => { + if let Some(sweep) = plugin.sweep.as_ref() { + let (depth_req, applied, expected) = + (sweep.depth_req, sweep.depth_applied, sweep.target_a()); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, expected)); + photodiode_window(&mut plugin, 0.02); + } + } + if plugin.recording.is_active() + && plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.point_started) + { + plugin.recording = Recording::idle(); + plugin.recording_completed_ok = true; + recorded.push(target_hz); + } + } + _ => {} + } + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + &mut sink, + ); + } + + assert_eq!(recorded.len(), 1, "message: {}", plugin.message); + assert!(plugin.freq_sweep.is_none()); + assert!( + plugin.message.contains("1/2 points recorded") && plugin.message.contains("1 skipped"), + "message: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn the_frequency_sweep_refuses_a_ladder_its_photodiode_cannot_measure() { + // The estimator window is one window for the whole ladder, so the + // *lowest* point decides measurability. Refuse the plan, not its + // ninth point two hours in. + let folder = temp_folder("fladder"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + plugin.min_f = 0.1; + plugin.max_f = 100.0; + plugin.freq_count = 4; + if let Some(summary) = plugin.photodiode.as_mut() { + if let Some(optical) = summary.optical_summary.as_mut() { + optical.window_seconds = Some(1.0); // 0.1 cycles at 0.1 Hz + } + } + plugin.freq_sweep_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert!(plugin.freq_sweep.is_none(), "the ladder must not start"); + assert!(sink.services.is_empty(), "no lease may be requested"); + assert!( + plugin.message.contains("lowest point") && plugin.message.contains("cache length"), + "message: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn changing_frequency_drops_the_previous_period_s_markers_and_windows() { + // The measured period is the mean marker spacing, so markers from the + // old drive would confirm the new frequency against a mixture. Pilot + // windows are frozen at a phase of the old period and do not transfer. + let folder = temp_folder("fflush"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.pilot_windows = Some(( + PhaseWindow { + start: 0.0, + end: 0.2, + }, + PhaseWindow { + start: 0.5, + end: 0.7, + }, + )); + plugin.freq_sweep = Some(FreqSweep { + phase: FreqSweepPhase::AcquiringLease, + points: vec![FreqSweepPoint { + frequency_hz: 50.0, + is_reference: false, + }], + index: 0, + lease_id: LeaseId::new("a1-fsweep-test"), + lease_granted: true, + lease_req: 0, + freq_req: 0, + freq_applied: false, + confirm_deadline_ms: 0, + skip_reason: None, + failed: Vec::new(), + recorded: 0, + order: FreqOrder::Ascending, + seed: 1, + last_activity_ms: now_unix_ms(), + stop_requested: false, + }); + let mut sink = ControlSink::default(); + plugin.send_freq_sweep_frequency(&mut sink); + + assert!(plugin.camera_markers_us.is_empty()); + assert!(plugin.camera_events.is_empty()); + assert!( + plugin.pilot_windows.is_none(), + "windows frozen at another period must not carry over" + ); + let _ = std::fs::remove_dir_all(&folder); + } + #[test] fn a0_lock_refuses_a_photodiode_window_shorter_than_one_cycle() { // `a` is peak-to-peak. Under one cycle the photodiode under-reports it, @@ -5303,6 +6741,7 @@ mod tests { lease_id: LeaseId::new("a1-sweep-test"), lease_granted: true, lease_req: 0, + owns_lease: true, depth_req: 0, depth_applied: true, settled_since_ms: None, diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index f60ac8f..563b655 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -909,6 +909,9 @@ pub struct StageAModulationPlugin { /// The operator's armed `depth_a`, parked while a lease drives the optical /// depth (A1's amplitude sweep) and restored by [`Self::end_lease`]. armed_depth_a: Option, + /// The operator's armed `frequency_hz`, parked while a lease drives the + /// frequency (A1's frequency sweep) and restored by [`Self::end_lease`]. + armed_frequency_hz: Option, /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0,1]`. /// Held fixed while `a` is swept, so one response curve keeps `I_k` constant. operating_point: f64, @@ -983,6 +986,7 @@ impl Default for StageAModulationPlugin { frequency_hz: 10.0, depth_a: 0.5, armed_depth_a: None, + armed_frequency_hz: None, operating_point: 0.5, v_null_dac: 0, v_pi_dac: 2_048, @@ -2004,6 +2008,63 @@ impl StageAModulationPlugin { self.shared.bump(); self.immediate_response(request, RequestOutcomeV1::Applied, None) } + ModulationCommandV1::SetDriveFrequency { frequency_millihz } => { + self.require_lease(request)?; + if self.link.is_none() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "the modulation owner is not connected to the device", + false, + )); + } + let frequency_hz = *frequency_millihz as f64 / 1_000.0; + // The same band `drive_command` clamps to; refuse rather than + // silently record a different frequency than the one asked for. + if !(0.01..=2_000.0).contains(&frequency_hz) { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + format!( + "frequency {frequency_hz:.3} Hz outside the supported \ + 0.01..=2000 Hz" + ), + false, + )); + } + // A constant hold has no frequency, and the manual DAC band is + // not the calibrated drive this path retargets. + if self.method == DriveMethod::Manual || self.mode == Mode::Const { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "arm a calibrated periodic/optical drive in the modulation plugin \ + before sweeping the frequency", + false, + )); + } + let previous = self.frequency_hz; + self.frequency_hz = frequency_hz; + let command = match self.drive_command() { + Ok(command) => command, + Err(error) => { + self.frequency_hz = previous; + return Err(service_error( + ServiceErrorCodeV1::DeviceRejected, + format!("frequency {frequency_hz:.3} Hz rejected: {error}"), + false, + )); + } + }; + // As for the depth: park the operator's own frequency on the + // first retarget only, so `end_lease` hands back what they + // armed rather than the sweep's last point. + self.armed_frequency_hz.get_or_insert(previous); + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); + self.shared.bump(); + self.immediate_response(request, RequestOutcomeV1::Applied, None) + } ModulationCommandV1::PrepareA1 { configuration } => { self.require_lease(request)?; let revision = self.requested_revision(request)?; @@ -2160,8 +2221,15 @@ impl StageAModulationPlugin { /// restores through `Sweep::restore`; this is the leased equivalent. fn end_lease(&mut self) { self.lease = None; - if let Some(depth) = self.armed_depth_a.take() { + let depth = self.armed_depth_a.take(); + let frequency = self.armed_frequency_hz.take(); + if let Some(depth) = depth { self.depth_a = depth; + } + if let Some(frequency) = frequency { + self.frequency_hz = frequency; + } + if depth.is_some() || frequency.is_some() { // Re-arm the board only if nobody else now owns the DAC; // `send_modulation` is itself guarded. self.send_modulation(); diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index 545644b..29b2b33 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -293,6 +293,19 @@ pub enum ModulationCommandV1 { SetOpticalDepth { depth_a_milli: u32, }, + /// Retarget the armed drive's *frequency*, leaving everything else — the + /// waveform shape, the depth, the operating point and the calibration — as + /// the operator armed it. The frequency counterpart of + /// [`ModulationCommandV1::SetOpticalDepth`], and the same scoping rules + /// apply: leased only, rejected when the link is closed or the armed drive + /// has no frequency to retarget (manual DAC method, constant mode). + /// + /// A1's frequency sweep drives this. The owner parks the operator's armed + /// frequency on the first one and restores it when the lease ends, so a + /// finished sweep does not leave the bench on its last point. + SetDriveFrequency { + frequency_millihz: u64, + }, PrepareA1 { configuration: A1AcquisitionConfigV1, }, From 5011dac9c1951fa282c5a1ab4491c4df300da8e8 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Sat, 25 Jul 2026 17:22:50 +0200 Subject: [PATCH 28/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20keep=20A1?= =?UTF-8?q?=20recordings=20full-length=20and=20in=20one=20folder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starting an A1 recording appeared to succeed and then reported a finished run immediately. What landed was a config sidecar in the chosen output folder, a truncated camera .raw in the host process's working directory, and no .pdq at all. Three defects combined: - Every photodiode-leg failure jumped straight to stop_camera, so the host had been recording for a few hundred milliseconds and produced a stub RAW that still carried a complete finalization receipt. Photodiode faults also set stop_requested, conflating them with an operator stop. The camera now runs its full duration and closes as a camera-only recording instead. - The specific cause ('set the data directory first') was overwritten by the generic 'was incomplete' on the way out. The first cause is now preserved and named in the closing message. - The RAW, the PDQ, and the sidecar are written by three owners against three roots, and the host's relative output path resolved to its working directory. Once both recorders report finalization their files are closed and hashed, so A1 now gathers them into // and records the final paths. PDQ receipts report a label relative to the photodiode data directory, which the owner now publishes in its summary so the path can be resolved. Also pre-flights the photodiode before starting the camera (not reporting, not connected, no data directory, leased elsewhere), surfaces the blocker in the status view while idle, and stops A1's own pipeline restart from wiping the row's pilot windows, background floor, and collected response points. Cherry-picked from fix/stage-a-a1-recording onto this branch, because the frequency ladder records every one of its points through this coordinator: without these fixes an unattended ladder would write a folder of truncated RAWs and no PDQ at all. Its ADR is renumbered 012 to 015 — the third branch to have claimed 012 independently. Refs ADR 015, revises ADR 009 decision 3. --- .../009-stage-a-a1-recording-coordinator.md | 10 +- .../015-stage-a-a1-recording-robustness.md | 112 ++++ docs/features/stage-a-a1.md | 66 +- plugins/stage-a-a1/src/runtime.rs | 567 +++++++++++++++++- plugins/stage-a-photodiode/src/lib.rs | 4 + stage-a-plugin-contract/src/lib.rs | 5 + 6 files changed, 713 insertions(+), 51 deletions(-) create mode 100644 docs/adr/015-stage-a-a1-recording-robustness.md diff --git a/docs/adr/009-stage-a-a1-recording-coordinator.md b/docs/adr/009-stage-a-a1-recording-coordinator.md index c56c977..3b446f1 100644 --- a/docs/adr/009-stage-a-a1-recording-coordinator.md +++ b/docs/adr/009-stage-a-a1-recording-coordinator.md @@ -1,6 +1,8 @@ # ADR 009 — Stage-A A1 as a focused recording coordinator -- **Status:** Accepted +- **Status:** Accepted — decision 3 revised by + [ADR 015](015-stage-a-a1-recording-robustness.md), which makes A1's output + folder authoritative and gathers the RAW/PDQ into it after finalization - **Date:** 2026-07-23 - **Relates to:** ADR 005 (device ownership), ADR 006 (two-plugin split), ADR 007 (owner orchestration — the earlier, broader orchestrator), @@ -100,8 +102,10 @@ phase-anchoring knobs (event latency, self-align) that the now-reliable not a rolling internal log. - The host returns to Preview before delivering its final receipt, which keeps repeated recordings and automated sweeps live without an extra operator step. -- True single-directory co-location is a **configuration** convention (align the +- ~~True single-directory co-location is a **configuration** convention (align the recorder roots), not something A1 enforces. Enforcing it would require host and - photodiode path changes and is out of scope. + photodiode path changes and is out of scope.~~ **Revised by ADR 015:** A1 moves + the finalized files into its own measurement folder, which needs no host or + photodiode path changes because both files are already closed and hashed. - The contract and ABI are unchanged: every message used already exists (`HostCommand`, `PhotodiodeCommandV1` lease/begin/finalize). diff --git a/docs/adr/015-stage-a-a1-recording-robustness.md b/docs/adr/015-stage-a-a1-recording-robustness.md new file mode 100644 index 0000000..15af6f3 --- /dev/null +++ b/docs/adr/015-stage-a-a1-recording-robustness.md @@ -0,0 +1,112 @@ +# ADR 015 — Stage-A A1 recording: one folder, full duration, named failures + +- **Status:** Accepted +- **Date:** 2026-07-25 +- **Relates to:** ADR 009 (A1 as a recording coordinator — revises decision 3 and + its co-location consequence), ADR 005 (device ownership), + ADR 006 (two-plugin split), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +On the bench, *Start recording* looked like it worked and then reported a +finished run almost immediately. What actually landed on disk was: + +- an A1 `_config.toml` in the chosen output folder, +- a **truncated** camera `.raw` (plus the host's bias `.toml`) in an unrelated + directory — the host process's working directory, +- **no `.pdq` and no photodiode sidecar at all**, +- and a status message that said only "was incomplete". + +Three separate defects produced that outcome. + +1. **A photodiode failure cut the camera recording short.** The coordinator + starts the camera first, then connects, leases, and opens the PDQ. Every + photodiode-leg failure — a rejected `Connect`, a refused lease, or a + `BeginRecording` rejected because the photodiode's *Data directory* was unset + — jumped straight to `stop_camera`. The host had been recording for a few + hundred milliseconds, so the RAW was a stub that nevertheless carried a + complete finalization receipt. `stop_requested` was also set on photodiode + faults, conflating "the operator asked to stop" with "the photodiode broke". + +2. **The failure reason was discarded.** Each failure wrote a specific message + (`Photodiode start failed (invalid_path): set the data directory first`), and + `finish_recording` then overwrote it with the generic "was incomplete". + The one piece of information the operator needed was destroyed on the way out. + +3. **One measurement scattered across up to three roots.** Per ADR 009 decision 3 + each recorder confines its own writes: the host resolves plugin recording paths + below *its* output directory and **rejects absolute paths**; the photodiode + resolves PDQ paths below *its* data directory; A1 writes its sidecar below + *its* output folder. ADR 009 accepted this and called physical co-location a + configuration convention. In practice the host's output path was relative, so + its parent resolved to the process working directory, and the RAW landed in a + source checkout — nowhere near the experiment folder. + +## Decision + +1. **The camera RAW always runs its full duration.** A photodiode failure while + the camera is already recording no longer stops it. The run continues to the + requested duration and closes normally, with the sidecar and message marking + it camera-only. A complete camera-only recording is a usable measurement; a + truncated file that reports itself as finalized is a trap. `stop_requested` + now means only what its name says — an operator stop — and photodiode faults + travel in `pd_rejected`. + +2. **The photodiode is pre-flighted before the camera starts.** A recording is + refused, with nothing recorded and an actionable message, when the photodiode + is not reporting status, is not connected, has no data directory, or is leased + by another client. These were exactly the conditions that used to surface as a + PDQ rejection *after* the host was already recording. The same check feeds the + A1 status view while idle, so the blocker is visible **before** the operator + presses Record rather than after a wasted run. + + This needs the owner's data directory, so `PhotodiodeSummaryV1` gains an + additive `data_dir: Option` field (`#[serde(default)]`, absent from + older owners, ignored by older consumers — the contract version is unchanged). + +3. **The first failure is preserved and named.** `Recording::failure` keeps the + first, most specific cause; later fallout cannot overwrite it. The closing + message reads `Recording incomplete: — metadata saved to `. + +4. **A1's output folder becomes authoritative for the whole measurement.** After + both recorders report finalization, A1 moves the RAW, the host's bias sidecar, + and the PDQ and its sidecar into `//`, then writes the + config sidecar with the final paths. This reverses ADR 009's "co-location is a + configuration convention" without touching the host or photodiode path rules: + both files are closed and hashed by the time their receipts arrive, so moving + them afterwards is safe and stays inside each owner's contract. + + The move is a `rename` on one volume and a size-verified copy-then-delete + across volumes. It never overwrites an existing destination and never removes + a source it has not verified; if a move fails, the file stays put and the + sidecar records where it actually is. + + PDQ receipts report the path **label** A1 requested — relative to the + photodiode's data directory — not an absolute path, so A1 resolves it against + the `data_dir` from decision 2 before locating the file. The sidecar records + the resolved absolute path either way, which also fixes the previous ambiguity + of storing a bare relative label under `[files]`. + +5. **Self-inflicted pipeline restarts no longer wipe the row.** Starting and + stopping the host recorder restarts the capture pipeline, which the host + reports as `SourceChanged` — twice per recording, caused by A1 itself. That + used to clear the pilot windows, the background floor, and every response + point collected across a sweep. While a recording or sweep is in flight the + boundary now resets only the event fold, whose timeline genuinely did restart. + +## Consequences + +- A recording can now end as *camera-only*: `recording_completed_ok` stays false, + so an amplitude sweep still stops rather than silently collecting points with + no measured `a`. The RAW is complete and reusable. +- A misconfigured bench refuses to record instead of producing a stub. This is a + deliberate behaviour change: pressing Record with a disconnected photodiode + now yields a message and no files, where it previously yielded a junk RAW. +- The measurement folder is the single place to look. Files are no longer where + the host and photodiode settings happen to point, so operators do not have to + keep three roots aligned by hand. Aligning them is still harmless — a file + already in the destination is left alone. +- Moving a large RAW across volumes copies it. On one volume (the normal case) + the move is a metadata operation regardless of file size. +- The contract addition is additive and backward compatible; no ABI change. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index b230c53..8b0f152 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -76,16 +76,21 @@ the drive to still be where a previous action left it (ADR 013). **Naming.** Files share an `_[_role]` stem under an `/` subfolder (`_pilot` / `_background` tag the reference runs, `_ec_fHz` an event-count point): -- `/_.raw` — camera RAW, under the **host output root**, with the host's - own `.toml` sidecar (camera biases, ROI) written next to it. -- `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar, under the - **photodiode data root**. -- `/__config.toml` — the A1 sidecar, under the chosen output folder. - -Each recorder confines its writes to its own root, so A1 cannot force one absolute -directory (see ADR 009). Point the host output root and the photodiode data root -at the same experiment directory to co-locate everything; the A1 sidecar records -the *resolved* paths so the set stays linked either way. +- `/_.raw` — camera RAW, with the host's own `.toml` sidecar + (camera biases, ROI) next to it. +- `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar. +- `/__config.toml` — the A1 sidecar. + +**Everything lands under `//`.** The two recorders each +write below their own root while recording — the host resolves plugin recording +paths below *its* output directory and rejects absolute ones, the photodiode +resolves PDQ paths below *its* data directory — so once both files are finalized +(closed and hashed) A1 moves them into the measurement folder and records the +final paths in the sidecar (ADR 015). The A1 output folder is therefore the only +setting that decides where a measurement ends up; the host and photodiode roots +no longer have to be kept aligned by hand. A move is a rename on one volume and a +size-verified copy across volumes; a file that cannot be moved stays where it is +and the sidecar points at it there. **A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the @@ -111,15 +116,32 @@ one concise result or error message; it does not render an internal event log. A1 declares `host_commands = ["start_recording", "stop_recording"]` in its manifest. Every role uses this same lifecycle. +**When something is wrong** (ADR 015): + +- **Before the camera starts**, A1 refuses the recording — writing nothing — if + the photodiode is not reporting status, not connected, has no data directory, + or is leased by someone else. The same hint fills the status `message` cell + while idle, so it is visible before the button is pressed. +- **If the photodiode fails once the camera is running**, the camera keeps + recording for the full requested duration and closes normally. The run is + marked camera-only: `recording_completed_ok` stays false (so a sweep stops), + but the RAW is complete rather than a truncated stub. +- **The first, most specific failure is what you see.** The closing message is + `Recording incomplete: — metadata saved to `; later fallout + cannot overwrite the original cause. +- **Starting and stopping the host recorder restarts the capture pipeline**, which + the host reports as a `SourceChanged` discontinuity — twice per recording. While + a recording or sweep is in flight that boundary resets only the event fold, not + the row's pilot windows, background floor, or collected response points. + **Host-side note.** The camera RAW leg restarts the host pipeline into Recording mode and stops it again at finalize. After the file is finalized, the host restores Preview before returning the receipt, so a sweep or another button press can start the next recording automatically. -**File locations** (three roots, point them at the same experiment directory): -`//.raw` (+ host `.toml`), -`//_pd.pdq` + `_pd.json`, and -`//_config.toml`. +**File locations.** One place: `//` holds `.raw` +(+ the host's `.toml`), `_pd.pdq` + `_pd.json`, and +`_config.toml`. ## The two live plots @@ -193,8 +215,9 @@ used to do nothing — the presses died on the mirror. Related: A1 overrides `on_discontinuity` to ignore `SettingsChanged` (raised on *every* settings sync of any plugin), so the response curve, pilot windows and -background floor survive ordinary UI interaction; source changes and seeks -still reset everything. +background floor survive ordinary UI interaction. Source changes and seeks reset +everything **unless** a recording or sweep is in flight, in which case the +boundary is A1's own pipeline restart and only the event fold resets (ADR 015). ## Where the inputs come from @@ -215,5 +238,12 @@ the pilot-window round-trip through the measurement folder, press-latch edge/bas semantics, the jittery-marker free-running fallback, sweep-point spacing, the sweep-point sidecar fields, the ordered camera → PDQ → PDQ finalize → camera finalize lifecycle (including envelope identity/revision and save location), the -selective discontinuity reset, and the `a₀`-lock set listed in the -[exact-event-count brief](./stage-a-a1-event-count.md). +selective discontinuity reset, and the `a₀`-lock and frequency-ladder sets listed +in the [exact-event-count brief](./stage-a-a1-event-count.md). + +Three of them guard the recording defects fixed in ADR 015: a photodiode leg that +cannot start is refused before any host command is sent; a photodiode failure +mid-run keeps the camera recording for the full duration, names the cause in the +closing message, and still gathers the RAW and its bias sidecar into the +measurement folder; and a self-inflicted `SourceChanged` during a recording keeps +the row's response points and pilot windows while still resetting the event fold. diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 209bdb9..5133f6f 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -332,6 +332,9 @@ struct Recording { /// The photodiode rejected BeginRecording — skip the finalize and don't /// wait for its receipt. pd_rejected: bool, + /// First thing that went wrong, kept verbatim so the closing message names + /// the cause instead of only reporting that the run was incomplete. + failure: Option, } /// Where the amplitude sweep is within its per-point cycle. @@ -835,6 +838,15 @@ impl Recording { pd_finalized: false, pd_valid: false, pd_rejected: false, + failure: None, + } + } + + /// Records the first failure only: later fallout (a stop that finds nothing + /// to finalize) must not mask the reason the run went wrong. + fn fail(&mut self, reason: impl Into) { + if self.failure.is_none() { + self.failure = Some(reason.into()); } } @@ -893,6 +905,18 @@ impl StageAA1Plugin { self.bump(); } + /// Reports a failure and remembers it as the run's cause, so the closing + /// message can name it after the coordinator has unwound. The first cause + /// wins: it is the specific one, and later arms only see the fallout. + fn note_failure(&mut self, message: impl Into) { + if self.recording.failure.is_some() { + return; + } + let message = message.into(); + self.recording.fail(message.clone()); + self.note(message); + } + /// The modulation period `T` in microseconds: measured from the phase-0 /// markers when present (the trigger *defines* the frequency, latency- /// invariant), otherwise the modulation plugin's acknowledged waveform. @@ -1296,10 +1320,13 @@ impl StageAA1Plugin { cell("events", self.camera_events.len().to_string()), cell( "message", - if self.message.is_empty() { - "—".into() - } else { - self.message.clone() + // While idle, anything that would refuse the next recording + // is worth more than the previous run's result: the operator + // sees it before pressing Record, not after. + match (self.recording.is_active(), self.photodiode_blocker()) { + (false, Some(blocker)) => blocker, + _ if self.message.is_empty() => "—".into(), + _ => self.message.clone(), }, ), ], @@ -1439,6 +1466,42 @@ impl StageAA1Plugin { meta } + /// Why the photodiode cannot record right now, phrased as the operator + /// action that fixes it. `None` means the PDQ leg is expected to succeed. + fn photodiode_blocker(&self) -> Option { + let Some(photodiode) = self.photodiode.as_ref() else { + return Some( + "The photodiode plugin is not reporting status — enable it before recording".into(), + ); + }; + if !matches!(photodiode.connection, ConnectionStateV1::Connected { .. }) { + return Some(format!( + "The photodiode is {} — connect it before recording", + connection_label(&photodiode.connection) + )); + } + if photodiode + .data_dir + .as_ref() + .is_none_or(|folder| folder.trim().is_empty()) + { + return Some( + "Set the photodiode Data directory before recording — the PDQ has nowhere to go" + .into(), + ); + } + // A lease held by anyone else means the PDQ is already committed. + if let Some(lease) = photodiode.lease.as_ref() { + if lease.holder.as_str() != A1_PLUGIN_ID { + return Some(format!( + "The photodiode is leased by {} — release it before recording", + lease.holder.as_str() + )); + } + } + None + } + /// Kick off a coordinated recording by starting the camera first. Called /// on the control tick after a record button is pressed. fn begin_recording(&mut self, context: &mut impl RecordingControl, role: RecRole) { @@ -1453,6 +1516,13 @@ impl StageAA1Plugin { self.note("Set a measurement id before recording"); return; } + // Checked before the camera starts: every one of these used to surface + // as a PDQ rejection *after* the host was already recording, which left + // a stub RAW behind and no photodiode data. + if let Some(blocker) = self.photodiode_blocker() { + self.note(blocker); + return; + } let now_ms = now_unix_ms(); let id = sanitize_stem(self.measurement_id.trim()); // Sweep points get a stable per-point tag so the row's files sort by @@ -1630,6 +1700,31 @@ impl StageAA1Plugin { )); } + /// The photodiode leg failed while the camera was already recording. The + /// camera RAW is the primary measurement, so it keeps running for its full + /// duration instead of being cut short — a truncated file that reports + /// itself as finalized is worse than a complete camera-only one. Any lease + /// still held is released by the normal stop path at the end. + fn continue_without_photodiode(&mut self, context: &mut impl RecordingControl) { + let camera_running = self.recording.cam_raw_path.is_some() && !self.recording.cam_rejected; + if !camera_running || self.recording.stop_requested { + self.stop_camera(context); + return; + } + self.recording.phase = RecPhase::Running; + self.recording.start_unix_ms = now_unix_ms(); + self.recording.last_activity_ms = self.recording.start_unix_ms; + let reason = self + .recording + .failure + .clone() + .unwrap_or_else(|| "the photodiode did not start".into()); + self.note(format!( + "{reason} — recording camera only for {} s", + self.recording.duration_s + )); + } + /// Stop the host recorder after the PDQ has been safely finalized. fn stop_camera(&mut self, context: &mut impl RecordingControl) { if self.recording.cam_raw_path.is_some() && !self.recording.cam_rejected { @@ -1656,11 +1751,19 @@ impl StageAA1Plugin { && self.recording.pd_valid && self.recording.pd_pdq_path.is_some() && self.recording.pd_sidecar_path.is_some(); + // Gather the RAW/PDQ next to the sidecar before writing it, so the + // recorded paths are the final ones. + self.gather_into_measurement_folder(); let sidecar = self.write_sidecar(); + let reason = self + .recording + .failure + .clone() + .unwrap_or_else(|| "not every file was finalized".into()); let message = match (sidecar, clean) { (Ok(path), true) => format!("Saved recording {} → {path}", self.recording.id), (Ok(path), false) => format!( - "Recording {} was incomplete — metadata saved to {path}", + "Recording {} incomplete: {reason} — metadata saved to {path}", self.recording.id ), (Err(err), _) => format!( @@ -1672,6 +1775,67 @@ impl StageAA1Plugin { self.release_and_idle(context, message); } + /// Collects the finalized artifacts into `//`. + /// + /// The camera RAW and the PDQ are written by two other owners against their + /// own roots — the host resolves plugin recording paths below *its* output + /// directory and rejects absolute ones, and the photodiode resolves PDQ + /// paths below *its* data directory. Left alone, one measurement scatters + /// across up to three unrelated folders. Both files are closed and hashed + /// by the time their receipts arrive, so moving them here is safe and makes + /// this plugin's output folder authoritative for the whole measurement. + fn gather_into_measurement_folder(&mut self) { + let dir = PathBuf::from(&self.recording.folder).join(&self.recording.id); + if std::fs::create_dir_all(&dir).is_err() { + return; + } + let raw = self + .recording + .cam_finalized_path + .clone() + .or_else(|| self.recording.cam_raw_path.clone()); + // The host writes the bias/config sidecar as a sibling of the RAW; it + // travels with it so the recording stays self-describing. + if let Some(raw) = raw { + if let Some(moved) = move_into(&dir, &raw) { + if self.recording.cam_finalized_path.is_some() { + self.recording.cam_finalized_path = Some(moved.clone()); + } + self.recording.cam_raw_path = Some(moved); + } + if let Some(bias) = sibling_toml(&raw) { + move_into(&dir, &bias); + } + } + // PDQ receipts report the *label* A1 asked for, which is relative to the + // photodiode's data directory — resolve it before touching the file, and + // record the absolute path either way. + if let Some(pdq) = self.resolved_photodiode_path(self.recording.pd_pdq_path.as_deref()) { + self.recording.pd_pdq_path = Some(move_into(&dir, &pdq).unwrap_or(pdq)); + } + if let Some(sidecar) = + self.resolved_photodiode_path(self.recording.pd_sidecar_path.as_deref()) + { + self.recording.pd_sidecar_path = Some(move_into(&dir, &sidecar).unwrap_or(sidecar)); + } + } + + /// Absolute location of a photodiode-reported recording path. The owner + /// reports paths relative to its own data directory, which it publishes in + /// its summary; an already-absolute path is taken as given. + fn resolved_photodiode_path(&self, reported: Option<&str>) -> Option { + let reported = reported?; + let path = Path::new(reported); + if path.is_absolute() { + return Some(reported.to_owned()); + } + let root = self + .photodiode + .as_ref() + .and_then(|photodiode| photodiode.data_dir.as_deref())?; + Some(Path::new(root).join(path).display().to_string()) + } + /// Release the photodiode lease (only if we actually hold it) and return to idle. fn release_and_idle(&mut self, context: &mut impl RecordingControl, message: String) { if self.recording.lease_granted { @@ -3385,7 +3549,7 @@ impl StageAA1Plugin { HostCommandOutcome::Rejected { code, message } => { // Stop the rest of the recording; drive_recording resolves the // abort from the current phase on the next tick. - self.note(format!("Camera recording rejected ({code}): {message}")); + self.note_failure(format!("Camera recording rejected ({code}): {message}")); self.recording.cam_rejected = true; self.recording.stop_requested = true; } @@ -3407,7 +3571,7 @@ impl StageAA1Plugin { self.recording.last_activity_ms = now_unix_ms(); } HostCommandOutcome::Rejected { code, message } => { - self.message = format!("Camera stop failed ({code}): {message}"); + self.note_failure(format!("Camera stop failed ({code}): {message}")); self.recording.cam_rejected = true; self.recording.last_activity_ms = now_unix_ms(); } @@ -3432,11 +3596,13 @@ impl StageAA1Plugin { || reply.request_id == self.recording.lease_req || reply.request_id == self.recording.pd_begin_req { - self.note(format!("Photodiode start failed ({code}): {message}")); + // Not `stop_requested`: that flag means the operator asked + // to stop. A photodiode fault leaves the camera running to + // its full duration (see `continue_without_photodiode`). + self.note_failure(format!("Photodiode start failed ({code}): {message}")); self.recording.pd_rejected = true; - self.recording.stop_requested = true; } else if reply.request_id == self.recording.pd_finalize_req { - self.note(format!("Photodiode save failed ({code}): {message}")); + self.note_failure(format!("Photodiode save failed ({code}): {message}")); self.recording.pd_rejected = true; self.recording.lease_granted = false; self.recording.last_activity_ms = now_unix_ms(); @@ -3493,11 +3659,15 @@ impl StageAA1Plugin { } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.cam_rejected = true; - self.release_and_idle(context, "Timed out starting camera recording".into()); + self.note_failure("Timed out starting camera recording"); + let message = self.message.clone(); + self.release_and_idle(context, message); } } RecPhase::ConnectingPhotodiode => { - if self.recording.stop_requested && !self.recording.connect_accepted { + if self.recording.pd_rejected { + self.continue_without_photodiode(context); + } else if self.recording.stop_requested && !self.recording.connect_accepted { self.stop_camera(context); } else if self.recording.connect_accepted { if self.recording.stop_requested { @@ -3508,13 +3678,13 @@ impl StageAA1Plugin { } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.pd_rejected = true; - self.note("Timed out connecting the photodiode"); - self.stop_camera(context); + self.note_failure("Timed out connecting the photodiode"); + self.continue_without_photodiode(context); } } RecPhase::AcquiringLease => { if self.recording.pd_rejected { - self.stop_camera(context); + self.continue_without_photodiode(context); } else if self.recording.lease_granted { if self.recording.stop_requested { self.stop_photodiode(context); @@ -3524,8 +3694,8 @@ impl StageAA1Plugin { } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.pd_rejected = true; - self.note("Timed out preparing the photodiode"); - self.stop_camera(context); + self.note_failure("Timed out preparing the photodiode"); + self.continue_without_photodiode(context); } } RecPhase::StartingPhotodiode => { @@ -3545,7 +3715,8 @@ impl StageAA1Plugin { || now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.pd_rejected = true; - self.stop_photodiode(context); + self.note_failure("The photodiode did not open its PDQ file"); + self.continue_without_photodiode(context); } } RecPhase::Running => { @@ -3562,7 +3733,7 @@ impl StageAA1Plugin { { self.recording.pd_rejected = true; self.recording.lease_granted = false; - self.note("Timed out saving photodiode data"); + self.note_failure("Timed out saving photodiode data"); self.stop_camera(context); } } @@ -3573,7 +3744,7 @@ impl StageAA1Plugin { { if self.recording.cam_finalized_path.is_none() && !self.recording.cam_rejected { self.recording.cam_rejected = true; - self.note("Timed out saving camera data"); + self.note_failure("Timed out saving camera data"); } self.finish_recording(context); } @@ -3921,6 +4092,39 @@ fn waveform_label(waveform: &WaveformV1) -> String { } } +/// Moves `source` into `dir`, returning the new path when it now lives there. +/// +/// A rename covers the common case (one volume) at zero cost; a cross-volume +/// move falls back to copy-then-delete, and the copy is size-checked before the +/// original goes away so a failed move never loses measurement data. `None` +/// means the file stayed where it was — callers keep the original path. +fn move_into(dir: &Path, source: &str) -> Option { + let source = Path::new(source); + let name = source.file_name()?; + if source.parent() == Some(dir) { + return None; + } + if !source.is_file() { + return None; + } + let destination = dir.join(name); + if destination.exists() { + return None; + } + if std::fs::rename(source, &destination).is_ok() { + return Some(destination.display().to_string()); + } + let copied = std::fs::copy(source, &destination).ok()?; + let expected = source.metadata().ok()?.len(); + if copied != expected { + let _ = std::fs::remove_file(&destination); + return None; + } + // Keeping the original after a verified copy is harmless; losing it is not. + let _ = std::fs::remove_file(source); + Some(destination.display().to_string()) +} + fn sibling_toml(raw_path: &str) -> Option { let path = Path::new(raw_path); let stem = path.file_stem()?.to_string_lossy(); @@ -4080,7 +4284,22 @@ impl Plugin for StageAA1Plugin { PluginDiscontinuity::SettingsChanged => {} PluginDiscontinuity::Seek | PluginDiscontinuity::SourceChanged - | PluginDiscontinuity::HistoryEvicted => self.reset(), + | PluginDiscontinuity::HistoryEvicted => { + // Starting and stopping the host recorder restarts the capture + // pipeline, and the host reports that as SourceChanged. Those + // boundaries are self-inflicted — twice per recording — so they + // must not wipe the row's pilot windows, background floor, or + // the response points collected across a sweep. The event fold + // still resets: that timeline really did restart. + if self.recording.is_active() || self.sweep.is_some() { + self.camera_events.clear(); + self.event_scratch.clear(); + self.camera_markers_us.clear(); + self.bump(); + } else { + self.reset(); + } + } } } @@ -4220,11 +4439,12 @@ impl Plugin for StageAA1Plugin { description: Some( "Records the camera RAW stream and the photodiode PDQ stream together for \ a fixed duration and writes an A1 config sidecar (.toml) linking them. \ - Files are grouped under the measurement id and share an _ \ - stem. Arm the optical drive in the modulation plugin first; A1 only reads \ - its settings — it never drives the Teensy. For everything to land in one \ - place, point the host output folder and the photodiode data folder at the \ - same experiment directory as this folder." + Everything lands under // and shares an \ + _ stem: the RAW and PDQ are gathered here once both are \ + finalized, wherever their own recorders wrote them. Arm the optical \ + drive in the modulation plugin first; A1 only reads its settings — it \ + never drives the Teensy. The photodiode must be connected and have a \ + data directory set, otherwise the recording is refused before it starts." .into(), ), default_open: true, @@ -4233,8 +4453,9 @@ impl Plugin for StageAA1Plugin { key: "output_folder".into(), label: "Output folder".into(), tooltip: Some( - "Directory where the A1 config sidecar is written. Also the \ - recommended shared experiment root for the RAW/PDQ files." + "Experiment directory for this measurement. The config sidecar is \ + written here, and the camera RAW and photodiode PDQ are moved \ + here once finalized, so one measurement is one folder." .into(), ), kind: SettingKind::Path { @@ -5266,8 +5487,9 @@ export_plugin!(StageAA1Plugin); #[cfg(test)] mod tests { use stage_a_plugin_contract::{ - OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, RequestOutcomeV1, - ResponseCommonV1, Sha256V1, StreamIntegrityV1, CONTRACT_VERSION_V1, + FreshnessV1, OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, + PhotodiodeStreamV1, RequestOutcomeV1, ResponseCommonV1, Sha256V1, StreamIntegrityV1, + SynchronizationV1, CONTRACT_VERSION_V1, }; use super::*; @@ -5387,6 +5609,7 @@ mod tests { }, active_recording: None, last_finalized_recording: None, + data_dir: Some(std::env::temp_dir().display().to_string()), optical_summary: Some(stage_a_plugin_contract::PhotodiodeOpticalSummaryV1 { run_id: RunId::new("pd-run"), calibration: stage_a_plugin_contract::PhotodiodeCalibrationV1 { @@ -5533,6 +5756,45 @@ mod tests { } } + /// A photodiode summary that passes the pre-flight: connected, unleased, + /// and with somewhere to put the PDQ. + fn ready_photodiode() -> PhotodiodeSummaryV1 { + PhotodiodeSummaryV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("pd-test"), + service_revision: 1, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: None, + }, + lease: None, + active_run_id: None, + requested_revision: None, + acknowledged_revision: None, + stream: PhotodiodeStreamV1 { + stream_epoch: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + latest_adc_code: Some(1_000), + integrity: StreamIntegrityV1::default(), + level: None, + }, + data_dir: Some("/pd".into()), + active_recording: None, + last_finalized_recording: None, + optical_summary: None, + synchronization: SynchronizationV1::Unsynced { + reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 60_000, + }, + } + } + fn on(timestamp_us: u64) -> CameraEvent { CameraEvent { timestamp_us, @@ -5790,6 +6052,7 @@ mod tests { measurement_id: "A1-row".into(), duration_s: 1, pending_role: Some(RecRole::Normal), + photodiode: Some(ready_photodiode()), ..StageAA1Plugin::default() }; let mut sink = ControlSink::default(); @@ -6919,4 +7182,248 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + + fn pd_rejection(request_id: u64, code: &str, message: &str) -> PluginServiceReply { + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Rejected { + code: code.into(), + message: message.into(), + }, + } + } + + /// A photodiode that cannot record is caught before the host is recording, + /// so a misconfigured bench no longer leaves a stub RAW behind. + #[test] + fn a_photodiode_without_a_data_directory_is_refused_before_the_camera_starts() { + let mut photodiode = ready_photodiode(); + photodiode.data_dir = None; + let mut plugin = StageAA1Plugin { + output_folder: "/tmp/a1-preflight".into(), + measurement_id: "A1-row".into(), + duration_s: 10, + pending_role: Some(RecRole::Normal), + photodiode: Some(photodiode), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert_eq!(plugin.recording.phase, RecPhase::Idle); + assert!( + sink.hosts.is_empty(), + "the camera must not start when the PDQ has nowhere to go" + ); + assert!( + plugin.message.contains("Data directory"), + "message={}", + plugin.message + ); + } + + /// The regression this whole coordinator exists for: a photodiode failure + /// used to stop the host recorder immediately, leaving a RAW that was a + /// fraction of the requested duration but reported itself as finalized. + #[test] + fn a_photodiode_failure_keeps_the_camera_recording_for_the_full_duration() { + let folder = std::env::temp_dir().join(format!("a1-camera-only-{}", now_unix_ms())); + let host_dir = folder.join("host-output"); + std::fs::create_dir_all(&host_dir).expect("host dir"); + let mut plugin = StageAA1Plugin { + output_folder: folder.display().to_string(), + measurement_id: "A1-row".into(), + duration_s: 10, + pending_role: Some(RecRole::Normal), + photodiode: Some(ready_photodiode()), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let cam_start_req = sink.hosts[0].request_id; + let raw_path = host_dir.join(format!("{}.raw", plugin.recording.stem)); + std::fs::write(&raw_path, b"raw-events").expect("raw file"); + std::fs::write(raw_path.with_extension("toml"), b"biases = true").expect("bias sidecar"); + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_start_req, + outcome: HostCommandOutcome::RecordingStarted { + actual_raw_path: raw_path.display().to_string(), + started_at: "2026-07-25T00:00:00Z".into(), + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let connect = sink.services.last().expect("connect request").clone(); + + // The photodiode refuses to open the stream. + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_rejection( + connect.request_id, + "transport", + "photodiode connection failed", + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + + assert_eq!( + plugin.recording.phase, + RecPhase::Running, + "the camera must keep recording without the photodiode" + ); + assert_eq!( + sink.hosts.len(), + 1, + "no StopRecording may be sent before the duration elapses" + ); + + // Nothing happens until the fixed duration is actually over. + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!(sink.hosts.len(), 1, "the run is still inside its window"); + + plugin.recording.start_unix_ms = now_unix_ms().saturating_sub(10_000); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!(plugin.recording.phase, RecPhase::StoppingCamera); + let cam_stop_req = sink.hosts[1].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_stop_req, + outcome: HostCommandOutcome::RecordingFinalized { + actual_raw_path: raw_path.display().to_string(), + size: 10, + sha256: "cd".repeat(32), + duration_us: 10_000_000, + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + + assert_eq!(plugin.recording.phase, RecPhase::Idle); + assert!(!plugin.recording_completed_ok, "the PDQ is missing"); + // The closing message names the cause instead of only "incomplete". + assert!( + plugin.message.contains("photodiode connection failed"), + "message={}", + plugin.message + ); + // Camera RAW, its bias sidecar, and the config all land together. + let measurement_dir = folder.join("A1-row"); + let mut names: Vec = std::fs::read_dir(&measurement_dir) + .expect("measurement folder") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + assert_eq!(names.len(), 3, "names={names:?}"); + assert!(names.iter().any(|name| name.ends_with(".raw"))); + assert!(names.iter().any(|name| name.ends_with("_config.toml"))); + assert!( + !raw_path.exists(), + "the RAW must be moved out of the host output folder" + ); + + let _ = std::fs::remove_dir_all(&folder); + } + + /// PDQ receipts name the path *relative to the photodiode's data directory*, + /// so gathering has to resolve it against the owner's published root before + /// the file can be found and moved. + #[test] + fn a_relative_pdq_label_is_resolved_against_the_photodiode_data_directory() { + let root = std::env::temp_dir().join(format!("a1-gather-{}", now_unix_ms())); + let pd_root = root.join("pd-data"); + std::fs::create_dir_all(pd_root.join("A1-row")).expect("pd dirs"); + std::fs::write(pd_root.join("A1-row/run_pd.pdq"), b"pdq").expect("pdq"); + std::fs::write(pd_root.join("A1-row/run_pd.json"), b"{}").expect("pd sidecar"); + + let mut photodiode = ready_photodiode(); + photodiode.data_dir = Some(pd_root.display().to_string()); + let mut plugin = StageAA1Plugin { + output_folder: root.display().to_string(), + photodiode: Some(photodiode), + ..StageAA1Plugin::default() + }; + plugin.recording.id = "A1-row".into(); + plugin.recording.stem = "run".into(); + plugin.recording.folder = root.display().to_string(); + // Exactly what the owner reports: a label, not a path. + plugin.recording.pd_pdq_path = Some("A1-row/run_pd.pdq".into()); + plugin.recording.pd_sidecar_path = Some("A1-row/run_pd.json".into()); + + plugin.gather_into_measurement_folder(); + + let measurement_dir = root.join("A1-row"); + assert!(measurement_dir.join("run_pd.pdq").is_file()); + assert!(measurement_dir.join("run_pd.json").is_file()); + assert!(!pd_root.join("A1-row/run_pd.pdq").exists()); + // The sidecar records where the file actually ended up. + assert_eq!( + plugin.recording.pd_pdq_path.as_deref(), + Some(measurement_dir.join("run_pd.pdq").display().to_string()).as_deref() + ); + + let _ = std::fs::remove_dir_all(&root); + } + + /// The host restarts the pipeline when A1 starts its own recording and + /// reports it as SourceChanged. That must not wipe the row's science state. + #[test] + fn a_self_inflicted_source_change_keeps_the_rows_science_state() { + let mut plugin = StageAA1Plugin { + response_points: vec![ResponsePoint { + measured_a: 1.0, + q_on: 0.5, + q_off: 0.4, + cycles: 20, + valid_pixels: 10, + }], + pilot_windows: Some(( + PhaseWindow { + start: 0.1, + end: 0.4, + }, + PhaseWindow { + start: 0.6, + end: 0.9, + }, + )), + camera_markers_us: vec![0, 1_000], + ..StageAA1Plugin::default() + }; + plugin.recording.phase = RecPhase::Running; + + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + + assert_eq!(plugin.response_points.len(), 1, "sweep points were wiped"); + assert!(plugin.pilot_windows.is_some(), "pilot windows were wiped"); + assert!( + plugin.camera_markers_us.is_empty(), + "the event timeline really did restart and must reset" + ); + + // Outside a recording the boundary still resets everything. + plugin.recording = Recording::idle(); + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + assert!(plugin.response_points.is_empty()); + assert!(plugin.pilot_windows.is_none()); + } } diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 95cbc59..3e39e79 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -1766,6 +1766,10 @@ impl StageAPhotodiodePlugin { requested_revision: self.requested_revision, acknowledged_revision: self.acknowledged_revision, stream, + // Automation clients need this to refuse a coordinated run before + // it starts the camera, instead of failing at BeginRecording. + data_dir: Some(self.data_dir.trim().to_owned()) + .filter(|folder| !folder.is_empty()), active_recording, last_finalized_recording: self.last_finalized_recording.clone(), optical_summary, diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index 29b2b33..5028931 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -613,6 +613,11 @@ pub struct PhotodiodeSummaryV1 { pub requested_revision: Option, pub acknowledged_revision: Option, pub stream: PhotodiodeStreamV1, + /// Directory the owner resolves relative PDQ/sidecar paths against. `None` + /// when it is unset, in which case every recording command is rejected — + /// automation clients check this before they start a coordinated run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_dir: Option, pub active_recording: Option, pub last_finalized_recording: Option, pub optical_summary: Option, From 7b78432f3ee78277d2512f505db1e82aa6ffaaf5 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 28 Jul 2026 09:11:42 +0200 Subject: [PATCH 29/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20write=20th?= =?UTF-8?q?e=20A1=20PDQ=20straight=20into=20the=20measurement=20folder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gather-after-finalization from the previous commit left the photodiode's own Data directory in the critical path: an A1 run still failed when it was unset, and changing it mid-experiment could move files out from under a measurement. `PdqStartSpecV1` gains an additive `root_dir: Option` — an absolute directory the client wants the recording written below, replacing the owner's configured data directory for that run. Every safety rule the owner already had survives below the new root (the path stays relative, `..` and non-normal components refused, parent components must be real directories rather than symlinks, the resolved target must stay below the root), plus the root itself must be absolute. An A1-driven run therefore no longer depends on the photodiode's Data directory at all, and the pre-flight stops checking it. The camera RAW still has to be gathered after finalization: the host resolves plugin recording paths below its own output directory and rejects absolute ones, and that rule lives in the other repository. The gather now also runs over the PDQ, where it is normally a no-op because the file was opened in place — which means a run that dies before finalization still leaves its PDQ in the measurement folder. PDQ receipts report the label the client requested, so A1 resolves it against the root it named, falling back to the owner's published `data_dir` and preferring whichever exists — an owner too old to honour `root_dir` still yields a correct path. Cherry-picked from fix/stage-a-a1-recording; revises ADR 015 decision 4. --- .../015-stage-a-a1-recording-robustness.md | 68 +++++--- docs/features/stage-a-a1.md | 31 ++-- plugins/stage-a-a1/src/runtime.rs | 123 ++++++++++++--- plugins/stage-a-photodiode/src/lib.rs | 147 ++++++++++++++++-- stage-a-plugin-contract/src/lib.rs | 8 + 5 files changed, 314 insertions(+), 63 deletions(-) diff --git a/docs/adr/015-stage-a-a1-recording-robustness.md b/docs/adr/015-stage-a-a1-recording-robustness.md index 15af6f3..5981e15 100644 --- a/docs/adr/015-stage-a-a1-recording-robustness.md +++ b/docs/adr/015-stage-a-a1-recording-robustness.md @@ -69,24 +69,39 @@ Three separate defects produced that outcome. first, most specific cause; later fallout cannot overwrite it. The closing message reads `Recording incomplete: — metadata saved to `. -4. **A1's output folder becomes authoritative for the whole measurement.** After - both recorders report finalization, A1 moves the RAW, the host's bias sidecar, - and the PDQ and its sidecar into `//`, then writes the - config sidecar with the final paths. This reverses ADR 009's "co-location is a - configuration convention" without touching the host or photodiode path rules: - both files are closed and hashed by the time their receipts arrive, so moving - them afterwards is safe and stays inside each owner's contract. - - The move is a `rename` on one volume and a size-verified copy-then-delete - across volumes. It never overwrites an existing destination and never removes - a source it has not verified; if a move fails, the file stays put and the - sidecar records where it actually is. - - PDQ receipts report the path **label** A1 requested — relative to the - photodiode's data directory — not an absolute path, so A1 resolves it against - the `data_dir` from decision 2 before locating the file. The sidecar records - the resolved absolute path either way, which also fixes the previous ambiguity - of storing a bare relative label under `[files]`. +4. **A1's output folder is the destination for the whole measurement**, reversing + ADR 009's "co-location is a configuration convention". A recording started in + A1 puts every file under `//`, by two mechanisms — chosen + per recorder by how much control that owner grants a client: + + **The PDQ is written there directly.** `PdqStartSpecV1` gains an additive + `root_dir: Option`: an absolute directory the client wants the + recording written below, replacing the owner's configured data directory for + that run. The owner keeps every safety rule it already had below the new root + — the path stays relative, `..` and non-normal components are refused, parent + components must be real directories rather than symlinks, and the resolved + target must stay below the root — and additionally requires the root itself to + be absolute. Consequently an A1-driven run **does not depend on the + photodiode's own Data directory at all**, which is what removed the failure + mode in context item 1; the pre-flight in decision 2 no longer checks it. + + **The camera RAW is moved there after finalization.** The host resolves plugin + recording paths below *its* output directory and rejects absolute paths, and + it lives in the other repository, so A1 cannot name the destination up front. + Instead, once the host reports finalization — at which point the file is closed + and hashed — A1 moves the RAW and the host's bias sidecar into the measurement + folder. A `rename` on one volume, a size-verified copy-then-delete across + volumes; it never overwrites an existing destination and never removes a source + it has not verified. If a move fails the file stays put and the sidecar records + where it actually is. The same gather runs over the PDQ, which is normally a + no-op because it is already in place. + + PDQ receipts report the path **label** the client requested, not an absolute + path, so A1 resolves it against the root it named — falling back to the owner's + published `data_dir` (decision 2) and preferring whichever exists, so an owner + too old to honour `root_dir` still yields a correct path. The sidecar records + the resolved absolute path, which also fixes the previous ambiguity of storing + a bare relative label under `[files]`. 5. **Self-inflicted pipeline restarts no longer wipe the row.** Starting and stopping the host recorder restarts the capture pipeline, which the host @@ -107,6 +122,21 @@ Three separate defects produced that outcome. the host and photodiode settings happen to point, so operators do not have to keep three roots aligned by hand. Aligning them is still harmless — a file already in the destination is left alone. +- The photodiode's Data directory now governs only its *own* manual saves (cache + snapshots, operator-started recordings). A workflow-driven run overrides it, so + changing it mid-experiment cannot move A1's files out from under a measurement. +- A crash mid-run leaves the PDQ in the measurement folder, because it was opened + there. Only the camera RAW depends on surviving to finalization to be gathered; + if a run dies before that, the RAW is left in the host's output directory and + the sidecar (if written) names it there. - Moving a large RAW across volumes copies it. On one volume (the normal case) the move is a metadata operation regardless of file size. -- The contract addition is additive and backward compatible; no ABI change. +- Both contract additions (`data_dir`, `root_dir`) are additive `#[serde(default)]` + fields, backward compatible in both directions; no ABI change and the contract + version stays at 1. Letting a client name an absolute root is a deliberate + widening of what a workflow may ask the owner to do — bounded by keeping every + traversal and symlink check, and by the owner still refusing anything it cannot + resolve below that root. +- Making the camera RAW land directly in the measurement folder would need the + host to accept a plugin-declared recording root. That belongs to `augur-rs` and + is deliberately left out of scope here; the gather makes it unnecessary. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 8b0f152..6266165 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -81,16 +81,20 @@ the drive to still be where a previous action left it (ADR 013). - `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar. - `/__config.toml` — the A1 sidecar. -**Everything lands under `//`.** The two recorders each -write below their own root while recording — the host resolves plugin recording -paths below *its* output directory and rejects absolute ones, the photodiode -resolves PDQ paths below *its* data directory — so once both files are finalized -(closed and hashed) A1 moves them into the measurement folder and records the -final paths in the sidecar (ADR 015). The A1 output folder is therefore the only -setting that decides where a measurement ends up; the host and photodiode roots -no longer have to be kept aligned by hand. A move is a rename on one volume and a -size-verified copy across volumes; a file that cannot be moved stays where it is -and the sidecar points at it there. +**Everything lands under `//`** (ADR 015). That folder is +the only setting deciding where a measurement ends up — the host output root and +the photodiode Data directory no longer have to be kept aligned by hand: + +- **The PDQ and its sidecar are written there directly.** A1 names the + destination root in the start spec (`PdqStartSpecV1::root_dir`), which replaces + the photodiode's own Data directory for that run. An A1-driven recording + therefore does not depend on the photodiode's folder setting at all. +- **The camera RAW and the host's bias `.toml` are moved there after + finalization.** The host resolves plugin recording paths below *its* output + directory and rejects absolute ones, so A1 cannot name the destination up + front; instead it gathers the file once the host reports it closed and hashed. + A rename on one volume, a size-verified copy across volumes. A file that cannot + be moved stays where it is and the sidecar points at it there. **A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the @@ -119,9 +123,10 @@ manifest. Every role uses this same lifecycle. **When something is wrong** (ADR 015): - **Before the camera starts**, A1 refuses the recording — writing nothing — if - the photodiode is not reporting status, not connected, has no data directory, - or is leased by someone else. The same hint fills the status `message` cell - while idle, so it is visible before the button is pressed. + the photodiode is not reporting status, is not connected, or is leased by + someone else. The same hint fills the status `message` cell while idle, so it + is visible before the button is pressed. (The photodiode's *Data directory* is + deliberately not among these: A1 supplies the destination itself.) - **If the photodiode fails once the camera is running**, the camera keeps recording for the full requested duration and closes normally. The run is marked camera-only: `recording_completed_ok` stays false (so a sweep stops), diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 5133f6f..845f6c6 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -1480,16 +1480,10 @@ impl StageAA1Plugin { connection_label(&photodiode.connection) )); } - if photodiode - .data_dir - .as_ref() - .is_none_or(|folder| folder.trim().is_empty()) - { - return Some( - "Set the photodiode Data directory before recording — the PDQ has nowhere to go" - .into(), - ); - } + // The photodiode's own Data directory is deliberately *not* checked: A1 + // names the destination root in the start spec, so a recording started + // here does not depend on the owner's folder setting at all. + // // A lease held by anyone else means the PDQ is already committed. if let Some(lease) = photodiode.lease.as_ref() { if lease.holder.as_str() != A1_PLUGIN_ID { @@ -1664,6 +1658,10 @@ impl StageAA1Plugin { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: self.recording_metadata(), + // Write the PDQ straight into this measurement's folder rather than + // the photodiode's own data directory: for a recording started here, + // this plugin's output folder is the one that decides where files go. + root_dir: Some(self.recording.folder.clone()), }; let pd_request = self.photodiode_request(PhotodiodeCommandV1::BeginRecording { specification: spec, @@ -1820,20 +1818,30 @@ impl StageAA1Plugin { } } - /// Absolute location of a photodiode-reported recording path. The owner - /// reports paths relative to its own data directory, which it publishes in - /// its summary; an already-absolute path is taken as given. + /// Absolute location of a photodiode-reported recording path. Receipts name + /// the *label* A1 asked for, which is relative to whichever root the owner + /// used: the folder A1 named in the start spec, or — for an owner too old to + /// honour it — the owner's own data directory. Resolve against both and + /// prefer the one that exists. fn resolved_photodiode_path(&self, reported: Option<&str>) -> Option { let reported = reported?; let path = Path::new(reported); if path.is_absolute() { return Some(reported.to_owned()); } - let root = self + let requested = Path::new(&self.recording.folder).join(path); + if requested.exists() { + return Some(requested.display().to_string()); + } + let owner_root = self .photodiode .as_ref() - .and_then(|photodiode| photodiode.data_dir.as_deref())?; - Some(Path::new(root).join(path).display().to_string()) + .and_then(|photodiode| photodiode.data_dir.as_deref()) + .map(|root| Path::new(root).join(path)); + match owner_root { + Some(owner_root) if owner_root.exists() => Some(owner_root.display().to_string()), + _ => Some(requested.display().to_string()), + } } /// Release the photodiode lease (only if we actually hold it) and return to idle. @@ -7199,9 +7207,9 @@ mod tests { /// A photodiode that cannot record is caught before the host is recording, /// so a misconfigured bench no longer leaves a stub RAW behind. #[test] - fn a_photodiode_without_a_data_directory_is_refused_before_the_camera_starts() { + fn a_disconnected_photodiode_is_refused_before_the_camera_starts() { let mut photodiode = ready_photodiode(); - photodiode.data_dir = None; + photodiode.connection = ConnectionStateV1::Disconnected; let mut plugin = StageAA1Plugin { output_folder: "/tmp/a1-preflight".into(), measurement_id: "A1-row".into(), @@ -7217,15 +7225,90 @@ mod tests { assert_eq!(plugin.recording.phase, RecPhase::Idle); assert!( sink.hosts.is_empty(), - "the camera must not start when the PDQ has nowhere to go" + "the camera must not start when the PDQ cannot follow" ); assert!( - plugin.message.contains("Data directory"), + plugin.message.contains("connect it"), "message={}", plugin.message ); } + /// The photodiode's own Data directory is irrelevant to a recording started + /// from A1: A1 names the destination root, so the run proceeds and the PDQ + /// is written into A1's measurement folder. + #[test] + fn the_pdq_start_spec_points_at_the_a1_output_folder() { + let mut photodiode = ready_photodiode(); + photodiode.data_dir = None; + let mut plugin = StageAA1Plugin { + output_folder: "/tmp/a1-destination".into(), + measurement_id: "A1-row".into(), + duration_s: 10, + pending_role: Some(RecRole::Normal), + photodiode: Some(photodiode), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!( + plugin.recording.phase, + RecPhase::StartingCamera, + "an unset owner data directory must not block an A1-driven run" + ); + let cam_start_req = sink.hosts[0].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_start_req, + outcome: HostCommandOutcome::RecordingStarted { + actual_raw_path: "/camera/A1-row/run.raw".into(), + started_at: "2026-07-25T00:00:00Z".into(), + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let connect = sink.services.last().expect("connect").clone(); + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(connect.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let acquire = sink.services.last().expect("lease").clone(); + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(acquire.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + + let begin: PhotodiodeRequestV1 = + serde_json::from_value(sink.services.last().expect("begin").payload.clone()) + .expect("begin envelope"); + let PhotodiodeCommandV1::BeginRecording { specification } = begin.command else { + panic!("expected BeginRecording"); + }; + assert_eq!( + specification.root_dir.as_deref(), + Some("/tmp/a1-destination"), + "the PDQ must be written below the A1 output folder" + ); + assert_eq!( + specification.pdq_path, + format!("A1-row/{}_pd.pdq", plugin.recording.stem) + ); + } + /// The regression this whole coordinator exists for: a photodiode failure /// used to stop the host recorder immediately, leaving a RAW that was a /// fraction of the requested duration but reported itself as finalized. diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 3e39e79..f722815 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -976,10 +976,16 @@ impl StageAPhotodiodePlugin { Ok(PathBuf::from(self.data_dir.trim())) } - /// Resolves a workflow-owned relative evidence path beneath the configured - /// data directory. Existing or newly created parent components must be - /// real directories, never symlinks. - fn resolve_control_path(&self, label: &str, extension: &str) -> Result { + /// Resolves a workflow-owned relative evidence path beneath `root_override` + /// when the client named one, else beneath the configured data directory. + /// Existing or newly created parent components must be real directories, + /// never symlinks — that holds for either root. + fn resolve_control_path( + &self, + label: &str, + extension: &str, + root_override: Option<&str>, + ) -> Result { let relative = Path::new(label); if relative.as_os_str().is_empty() || relative.is_absolute() @@ -995,12 +1001,24 @@ impl StageAPhotodiodePlugin { return Err(format!("workflow path must use the .{extension} extension")); } - let root = self.resolved_data_dir()?; + // A client-named root replaces the data directory entirely: a + // coordinated run keeps every file of one measurement together, and + // the owner's own Data section then has no bearing on it. + let root = match root_override.map(str::trim).filter(|root| !root.is_empty()) { + Some(root) => { + let root = PathBuf::from(root); + if !root.is_absolute() { + return Err("workflow recording root must be an absolute path".into()); + } + root + } + None => self.resolved_data_dir()?, + }; std::fs::create_dir_all(&root) .map_err(|err| format!("creating {} failed: {err}", root.display()))?; let root = root .canonicalize() - .map_err(|err| format!("resolving data directory failed: {err}"))?; + .map_err(|err| format!("resolving recording directory failed: {err}"))?; let mut parent = root.clone(); if let Some(relative_parent) = relative.parent() { for component in relative_parent.components() { @@ -1032,7 +1050,7 @@ impl StageAPhotodiodePlugin { .canonicalize() .map_err(|err| format!("resolving {} failed: {err}", parent.display()))?; if !canonical.starts_with(&root) { - return Err("workflow path escapes the configured data directory".into()); + return Err("workflow path escapes the recording directory".into()); } } } @@ -1090,11 +1108,12 @@ impl StageAPhotodiodePlugin { true, )); } + let root = specification.root_dir.as_deref(); let pdq_path = self - .resolve_control_path(&specification.pdq_path, "pdq") + .resolve_control_path(&specification.pdq_path, "pdq", root) .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; let sidecar_path = self - .resolve_control_path(&specification.sidecar_path, "json") + .resolve_control_path(&specification.sidecar_path, "json", root) .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; if pdq_path == sidecar_path { return Err(service_error( @@ -1768,8 +1787,7 @@ impl StageAPhotodiodePlugin { stream, // Automation clients need this to refuse a coordinated run before // it starts the camera, instead of failing at BeginRecording. - data_dir: Some(self.data_dir.trim().to_owned()) - .filter(|folder| !folder.is_empty()), + data_dir: Some(self.data_dir.trim().to_owned()).filter(|folder| !folder.is_empty()), active_recording, last_finalized_recording: self.last_finalized_recording.clone(), optical_summary, @@ -4408,6 +4426,109 @@ mod tests { assert!(plugin.set_setting("mode", json!("RAW")).is_err()); } + /// A workflow client that names its own recording root gets the PDQ written + /// there, and the owner's Data directory stops being involved at all — that + /// is what lets one coordinated run keep every file in one folder. + #[test] + fn a_client_named_root_overrides_the_data_directory() { + let dir = temp_dir("client-root"); + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + // Deliberately unset: it must not be consulted. + plugin.data_dir = String::new(); + plugin.connect(); + let acquire = service_request( + &plugin, + 20, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&acquire, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + )); + + let begin = service_request( + &plugin, + 21, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "A1-row/run_pd.pdq".into(), + sidecar_path: "A1-row/run_pd.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + root_dir: Some(dir.display().to_string()), + }, + }, + Some(1), + ); + assert!( + matches!( + plugin + .handle_service_request(&begin, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + ), + "a client-named root must not need the owner's data directory" + ); + assert!(dir.join("A1-row/run_pd.pdq").is_file()); + + // Traversal is still refused below a client-named root. + let escape = service_request( + &plugin, + 22, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "../escape.pdq".into(), + sidecar_path: "A1-row/escape.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + root_dir: Some(dir.display().to_string()), + }, + }, + Some(2), + ); + assert!(matches!( + plugin + .handle_service_request(&escape, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + + // A relative root is refused outright. + let relative_root = service_request( + &plugin, + 23, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "A1-row/other_pd.pdq".into(), + sidecar_path: "A1-row/other_pd.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + root_dir: Some("relative/root".into()), + }, + }, + Some(3), + ); + assert!(matches!( + plugin + .handle_service_request(&relative_root, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn named_recording_rejects_unsafe_paths_and_returns_final_receipt() { let dir = temp_dir("named"); @@ -4440,6 +4561,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::new(), + root_dir: None, }, }, Some(1), @@ -4462,6 +4584,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::from([("workflow".into(), "A1".into())]), + root_dir: None, }, }, Some(1), @@ -4511,6 +4634,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::new(), + root_dir: None, }, }, Some(3), @@ -4551,6 +4675,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::new(), + root_dir: None, }, }, Some(1), diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index 5028931..97b7aca 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -440,6 +440,14 @@ pub struct PdqStartSpecV1 { pub expected_sample_rate_hz: Option, pub expected_stream_epoch: Option, pub metadata: BTreeMap, + /// Absolute directory the workflow client wants this recording written + /// below, so a coordinated run can put every file in one measurement + /// folder instead of the owner's own data directory. `None` keeps the + /// owner's configured data directory. `pdq_path`/`sidecar_path` stay + /// relative to whichever root applies, and the owner still refuses + /// traversal and symlinked path components below it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root_dir: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] From bcd0e6609bb29fe2cbb578bf460492ed768b71d8 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 28 Jul 2026 09:12:36 +0200 Subject: [PATCH 30/46] =?UTF-8?q?docs(stage-a):=20=F0=9F=93=9D=20index=20t?= =?UTF-8?q?he=20recording-robustness=20ADR=20after=20its=20renumber=20to?= =?UTF-8?q?=20015?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/features/stage-a-a1.md | 3 +++ plugins/stage-a-a1/README.md | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 6266165..9a187b4 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -2,9 +2,12 @@ - **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) - **Status:** Recording coordinator + live quicklooks + amplitude sweep + `a₀` lock + + unattended frequency ladder - **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button press forwarding), + [ADR 015](../adr/015-stage-a-a1-recording-robustness.md) (one folder, full + duration, named failures), [ADR 014](../adr/014-stage-a-a1-frequency-ladder.md) (the unattended ladder), [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (exact-event-count `a₀` lock) diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 11e3e4d..322ff7d 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -74,6 +74,8 @@ See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the ful [docs/features/stage-a-a1-event-count.md](../../docs/features/stage-a-a1-event-count.md) plus [ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, [ADR 014](../../docs/adr/014-stage-a-a1-frequency-ladder.md) for the unattended -frequency ladder, and +frequency ladder, +[ADR 015](../../docs/adr/015-stage-a-a1-recording-robustness.md) for the +recording coordinator's one-folder/full-duration guarantees, and [docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the planned amplitude-sweep automation on top of this. From fc6eaabc18c8de1190235dfe2e6ef02037b628ab Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 29 Jul 2026 14:41:44 +0200 Subject: [PATCH 31/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20align=20Po?= =?UTF-8?q?ckels=20and=20contrast=20semantics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../008-stage-a-optical-waveform-inversion.md | 23 +- docs/adr/010-stage-a-a1-amplitude-sweep.md | 26 +- ...11-stage-a-pockels-transfer-calibration.md | 2 +- docs/features/README.md | 6 +- docs/features/stage-a-a1-automation.md | 10 +- docs/features/stage-a-a1.md | 25 +- docs/features/stage-a-modulation.md | 24 +- docs/features/stage-a-optical-waveform.md | 85 ++++-- docs/features/stage-a-photodiode.md | 17 +- docs/features/stage-a-pockels-calibration.md | 4 +- plugins/stage-a-a1/README.md | 6 +- plugins/stage-a-a1/src/runtime.rs | 250 ++++++++++++++++-- plugins/stage-a-modulation/README.md | 23 +- plugins/stage-a-modulation/src/calibration.rs | 16 +- plugins/stage-a-modulation/src/lib.rs | 221 ++++++++++++---- plugins/stage-a-modulation/src/waveform.rs | 92 +++++-- plugins/stage-a-photodiode/src/lib.rs | 214 +++++++++++++-- stage-a-io/src/estimator.rs | 20 ++ stage-a-plugin-contract/src/lib.rs | 67 ++++- 19 files changed, 921 insertions(+), 210 deletions(-) diff --git a/docs/adr/008-stage-a-optical-waveform-inversion.md b/docs/adr/008-stage-a-optical-waveform-inversion.md index fc599a7..f60a2e0 100644 --- a/docs/adr/008-stage-a-optical-waveform-inversion.md +++ b/docs/adr/008-stage-a-optical-waveform-inversion.md @@ -42,16 +42,31 @@ capped at 192 bytes, too small to upload a 256-code table inline. commanded value. 6. **Keep drive method orthogonal to waveform mode (2026-07-23 amendment).** `MANUAL` defines a DAC band from Power + Min threshold; `CALIBRATED` derives - one from `V_null`, `Vπ`, `I_k`, and `a`. All five waveform modes remain + one from `V_null`, `Vπ`, normalized `u`, and `a`. All five waveform modes remain available with both methods. Manual optical modes pass their DAC endpoints - through the forward `sin²` transfer to derive `(I_k, a)`, then reuse the same + through the forward `sin²` transfer to derive `(u, a)`, then reuse the same inversion path. The separate `max_level` setting is the hard ceiling for every drive; it is no longer merely the upper bound of the Power slider. 7. **Treat constant hold separately from modulation headroom (2026-07-23 - amendment).** `CONST` maps `I_k` directly through the inverse lobe and ignores - `a`; periodic modes retain the `I_k·exp(a/2) ≤ 1` ceiling. A rejected + amendment).** `CONST` maps `u` directly through the inverse lobe and ignores + `a`; periodic modes retain target-specific headroom. A rejected calibrated setting is rolled back so displayed settings always describe the command that can actually be sent. +8. **Separate normalized transfer coordinate from physical flux and preserve + its mean (2026-07-28 amendment).** The UI setting is the normalized + cycle mean `ū`; it is never called the physical A1 flux `I_k`. Log-sine + generation derives `u_g=ū/I_0(a/2)` before sending the existing WARP + parameters. The internal value is quantized to the existing `u_k_milli` + wire field, and the acknowledged state publishes both requested and resolved + means, so sweeping `a` removes the analytic mean shift and makes the small + quantization residual explicit. Finite `I_floor` + still makes physical contrast smaller than the requested floor-subtracted + contrast. A1 therefore records a separate physical `flux_point_id`, + measured photodiode `a` remains authoritative, and the measured finite-floor + LUT remains required when the analytic residual exceeds the error budget. + `ModulationStateV1.optical_drive` publishes requested and resolved `ū`, + internal `u_g`/`u_c`, requested `a`, target and lobe codes as an additive V1 + provenance field. ## Consequences diff --git a/docs/adr/010-stage-a-a1-amplitude-sweep.md b/docs/adr/010-stage-a-a1-amplitude-sweep.md index ab036fb..97edfea 100644 --- a/docs/adr/010-stage-a-a1-amplitude-sweep.md +++ b/docs/adr/010-stage-a-a1-amplitude-sweep.md @@ -18,7 +18,7 @@ Two structural gaps blocked this: 1. **No semantic "set depth" command.** The modulation service only exposed `SetWaveform` (raw DAC band) and `PrepareA1`. Sweeping `a` through raw DAC values would duplicate the optical-inversion math (ADR 008) and the - calibration state (`V_null`, `Vπ`, `u_k`) outside their owner. + calibration state (`V_null`, `Vπ`, requested mean `ū`) outside their owner. 2. **Momentary buttons never reached the live worker.** The host runs a UI mirror and a live worker per plugin; button presses land on the mirror via `set_setting(key, true)`, while the worker only receives the settings @@ -31,10 +31,11 @@ Two structural gaps blocked this: addition, additive to V1). Under an automation lease the modulation owner re-derives its armed drive with the new depth through the same `drive_command()` builder the operator path uses; everything else (waveform -shape, frequency, `u_k`, calibration, power cap) stays as armed. The owner -rejects the command when no device link is open, when the armed drive cannot -express a depth (manual DAC method, constant mode), or when the derived drive -violates its own safety validation. The command is applied immediately +shape, frequency, requested `ū`, calibration, power cap) stays as armed. For +each depth, the owner derives and wire-quantizes the internal +`u_g=ū/I_0(a/2)`. The owner rejects the command when no device link is open, +when the armed drive is not calibrated `OPTICAL_LOG_SINE`, or when the derived +drive violates its own safety validation. The command is applied immediately (`Applied`), not revision-tracked: the sweep's ground truth for "the drive is really there" is the photodiode-measured `a`, not a firmware ACK. @@ -42,13 +43,24 @@ really there" is the photodiode-measured `a`, not a firmware ACK. ADR 009 coordinator: `AcquiringLease → (per point) SettingDepth → Settling → Recording → …release`. Per point it renews the modulation lease, retargets the depth, waits until the photodiode-measured `a` holds the target tolerance -(±10 %, at least ±0.05) for the configured dwell (30 s cap, then it records -anyway — the sidecar stores the measured `a`), and hands off to the unchanged +(±10 %, at least ±0.05) for the configured dwell and hands off to the unchanged recording coordinator (`…_pNN` stem tag, `sweep.requested_a` / `point_index` / `point_total` in the sidecar). Any rejection, timeout, or failed point aborts the sweep and releases the lease (`safe_off = false` — the drive holds; safety remains the owner's lease-expiry job). +**2026-07-28 amendment:** `SetOpticalDepth` is accepted only for an applied +measured calibration and `OPTICAL_LOG_SINE`; accepting DAC, square, linear, or +unidentified hand-entered lobe parameters under the same semantic command made +`a` ambiguous. A1 also requires a connected, fresh photodiode optical summary +from complete marker-bounded cycles and a confirmed named `I_tot` anchor. +Failure to settle before the deadline now aborts the sweep instead of recording +an invalid point. A required `flux_point_id` carries the physical cycle-mean +`I_k` provenance separately from the Bessel-normalized modulation mean `ū`; +the config sidecar records requested/resolved `ū`, quantized internal `u_g`, +requested `a`, target, `V_null`, `Vπ`, and transfer calibration ID from the +additive modulation snapshot. + **3. Press counters for momentary buttons.** Every A1 button exports a monotonic press counter from `get_setting`; `set_setting` interprets `true` as a local click and a counter advance as one forwarded press edge, adopting the diff --git a/docs/adr/011-stage-a-pockels-transfer-calibration.md b/docs/adr/011-stage-a-pockels-transfer-calibration.md index 30070f1..de1d7e7 100644 --- a/docs/adr/011-stage-a-pockels-transfer-calibration.md +++ b/docs/adr/011-stage-a-pockels-transfer-calibration.md @@ -51,7 +51,7 @@ the port. It cannot. Since `sin²` is symmetric about its peak, *identically* — the data cannot say which extremum is zero excitation. This is a fact about the optics, so it is asked (`Detector port`, default `REJECT PORT`, which `setup/optical-path.md` settles by construction) and the fit selects the -matching representation. Guessing would place `V_null` a quarter wave off and +matching representation. Guessing would place `V_null` one half-wave-voltage span off and silently run the drive on the inverted branch. ### 4. One-dimensional harmonic fit, not a nonlinear solve diff --git a/docs/features/README.md b/docs/features/README.md index 56f1ed2..358a4e5 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -6,10 +6,10 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. - [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. -- [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC so the *optical* output is a log- or linear-intensity sine, inverting the Pockels `sin²` transfer from settable `V_null`/`Vπ`. +- [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC from measured `V_null`/`Vπ`, with target-specific headroom, Bessel-normalized cycle mean `ū`, and an explicit separation from physical flux `I_k`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`Vπ` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). -- [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry only after a named `I_tot` anchor is explicitly confirmed. +- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with physical `flux_point_id`, transfer/anchor provenance, and live response quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. diff --git a/docs/features/stage-a-a1-automation.md b/docs/features/stage-a-a1-automation.md index 529a10d..d68648b 100644 --- a/docs/features/stage-a-a1-automation.md +++ b/docs/features/stage-a-a1-automation.md @@ -45,7 +45,8 @@ naming and full parameters. Later, repeat over `f` and over `I_k`. `N_valid = |ROI| − |masked|`. - Photodiode-measured **`a`** (rejected-complement geometry) published and surfaced in A1. -- Optical drive with a **fixed operating point `I_k`** and swept `a` +- Optical drive with a **fixed normalized cycle mean `ū`** and swept `a`; + physical cycle-mean flux `I_k` is a separately calibrated/verified row quantity (`OPTICAL_LOG_SINE`/`OPTICAL_LINEAR_SINE`), power-capped. - Events sourced exactly from the retained **EventStore** over a sliding window. @@ -54,8 +55,11 @@ naming and full parameters. Later, repeat over `f` and over `I_k`. ### 1. A1 → modulation control path (scoped) Re-introduce a *focused* control path (the contract + modulation plugin still support it): acquire a modulation lease, set the optical drive -`(target, I_k, a, f, V_null, Vπ)`, start, stop, release. No full workflow zoo — -just set-amplitude / start / stop. Fixed `I_k`, only `a` varies within a curve. +`(target, ū, a, f, V_null, Vπ)`, start, stop, release. No full workflow zoo — +just set-amplitude / start / stop. The plugin's Bessel normalization preserves +`ū` to the controller's milli-unit resolution and publishes the resolved mean; +the independently calibrated physical `I_k` still needs bench feedback and a +flux-point ID. ### 2. Settle detection Before collecting a point, wait until the photodiode confirms the optical diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index edb0405..3802aef 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -34,9 +34,10 @@ folder. A1 makes each recording one button press: |---|---| | Output folder | where the A1 config sidecar is written (recommended shared experiment root) | | Measurement id | one per `(I_k, f)` row; auto-generated default, editable, or press **New id** | +| Physical `I_k` flux point id | required canonical id of the cycle-mean local flux calibration/map point; never inferred from the modulator's normalized mean `ū` | | Sweep min a / max a | the `a`-range for this row; the **Start sweep** button records it, and it is stored in every sidecar | | Sweep points (count) | how many amplitudes Start sweep records, spaced evenly over `[min a, max a]` | -| Sweep settle (s) | dwell the photodiode-measured `a` must hold the target (±10 %, ≥±0.05) before each sweep recording; 30 s cap, then it records anyway | +| Sweep settle (s) | dwell the fresh photodiode-measured `a` must hold the target (±10 %, ≥±0.05) before each sweep recording; timeout aborts the sweep | | Duration (s) | each recording auto-stops and finalizes after this | | Start recording (sweep point) | start camera RAW → connect and lease photodiode → start PDQ → auto-stop and save both → sidecar | | Start sweep (record all points) | per point: lease the modulation owner → retarget the calibrated drive to `a_i` → settle → one recording (`…_pNN`) → next point | @@ -54,10 +55,12 @@ matching button — the recording captures whatever `a` is currently set. **The sweep is the one scoped exception.** Start sweep leases the modulation owner (`SERVICE_STAGE_A_MODULATION_CONTROL_V1`) and, per point, issues `ModulationCommandV1::SetOpticalDepth` — which only retargets the *depth* of the -drive the operator already armed (waveform, frequency, operating point `I_k`, -and calibration stay untouched; the owner refuses when a manual-DAC or constant -drive is armed). It renews the lease per point, waits for the -photodiode-measured `a` to settle, hands the point to the normal recording +drive the operator already armed (frequency, normalized cycle mean `ū`, and +calibration stay untouched). The modulation owner accepts this command only +with an applied measured calibration and `OPTICAL_LOG_SINE`; manual, constant, +DAC-sine, square, and optical-linear modes are rejected. It renews the lease per +point, waits for a fresh, marker-bounded photodiode `a` from a confirmed `I_tot` +anchor to settle, hands the point to the normal recording coordinator, and releases the lease at the end or on abort. Sweep points require `min a > 0` — record `a≈0` with the background button instead. Sidecars of sweep recordings additionally carry `sweep.requested_a`, `sweep.point_index` @@ -79,10 +82,14 @@ directory (see ADR 009). Point the host output root and the photodiode data root at the same experiment directory to co-locate everything; the A1 sidecar records the *resolved* paths so the set stays linked either way. -**A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize +**A1 config sidecar** captures: `measurement_id`, physical `flux_point_id`, file +stem, role, start/finalize timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the -acknowledged snapshot (frequency, center/amplitude DAC, waveform); the -photodiode-measured `a` (`measured_log_contrast`) and clip fractions; ROI + +acknowledged snapshot (frequency, center/amplitude DAC, waveform, transfer +`calibration_id`, optical target, requested and resolved normalized mean `ū`, +internal `u_g`/`u_c`, requested `a`, `V_null` and `Vπ`); the +photodiode-measured `a`, extrema, geometric pedestal, +headroom, clip fractions, dark/ADC ids, and named dark-corrected `I_tot` anchor; ROI + masked-pixel count + `N_valid`; trigger info (marker-anchored, marker count, measured period); and the resolved paths of the RAW (+ its camera-config sidecar) and the PDQ (+ its sidecar). The **pilot** run additionally records the frozen @@ -195,7 +202,7 @@ still reset everything. | camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()`, trimmed to the same window | | phase-0 markers | rising `frame.external_triggers()` — the host **banks trigger edges from dropped preview frames** into the next processed frame (drain-to-newest and the preview throttle drop whole frames; at low modulation frequencies the survivors alone rarely held 2 markers inside the analysis window) | | modulation period `T` | measured from the `EXT_TRIGGER` marker spacing; else the modulation plugin's acknowledged waveform — which, since the board-echo fallback, includes the **operator-armed UI drive**, not only service-path (leased) targets | -| optical modulation depth `a` | photodiode plugin's optical summary (`measured_log_contrast`) — always the *excitation* contrast, independent of that plugin's display mode (ADR 012) | +| optical modulation depth `a` | fresh photodiode optical summary (`measured_log_contrast`) from complete marker-bounded cycles and a confirmed `I_tot` anchor — always the *excitation* contrast, independent of display mode (ADR 012) | | ROI, masked pixels | augur-rs camera config (`CTX_GLOBAL_SETTINGS`) | ## Tests diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index e9077f9..aed1ff0 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -10,7 +10,7 @@ Laser-modulation control for the Stage-A bench with two orthogonal axes: - **Drive method** defines the DAC operating band. `MANUAL` uses Power + Min threshold; - `CALIBRATED` derives it from `V_null`, `Vπ`, `I_k`, and optical depth `a`. + `CALIBRATED` derives it from `V_null`, `Vπ`, normalized cycle mean `ū`, and optical depth `a`. - **Mode** defines the shape that fills the band: `CONST`, `DAC_SINE`, `SQUARE`, `OPTICAL_LOG_SINE`, or `OPTICAL_LINEAR_SINE`. All five remain available under both methods. @@ -18,17 +18,18 @@ The always-visible **max limit** is the hard DAC ceiling for every manual and ca The settings schema shows only the selected method's parameter block and refreshes when Method changes; Manual is the default. -| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `Vπ` | +| Mode | Manual band `[min, power]` | Calibrated band from `ū`, `a`, `V_null`, `Vπ` | |---|---|---| -| `CONST` | hold `power` | hold the DAC code for `I_k` | +| `CONST` | hold `power` | hold the DAC code for `ū` | | `DAC_SINE` | DAC sine across the band | DAC sine across the band | | `SQUARE` | DAC square across the band | DAC square across the band | -| `OPTICAL_LOG_SINE` | intensity log-sine across the band | intensity log-sine about `I_k` | -| `OPTICAL_LINEAR_SINE` | intensity linear-sine across the band | intensity linear-sine about `I_k` | +| `OPTICAL_LOG_SINE` | intensity log-sine across the band | mean `ū`, converted to `u_g=ū/I_0(a/2)` | +| `OPTICAL_LINEAR_SINE` | intensity linear-sine across the band | centre/mean `u_c=ū` | Manual optical modes reuse the persisted `V_null`/`Vπ` lobe parameters and derive effective -`(I_k, a)` from the manual DAC band through the forward `sin²` transfer. Both optical modes then +`(u, a)` from the manual DAC band through the forward `sin²` transfer. Both optical modes then use the same inversion path described in [Optical waveform drive](./stage-a-optical-waveform.md). +`ū` is dimensionless and must not be confused with physical cycle-mean A1 flux `I_k`. `V_null`/`Vπ` are measured, not typed: the Calibration section sweeps settled `CONST` codes against the photodiode and fits the lobe — see @@ -53,13 +54,18 @@ board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded val - Safety invariants enforced plugin-side: `min_level ≤ level ≤ max_level` for Manual and every resolved calibrated/optical peak must be `≤ max_level`; invalid drives are refused. - Status and commanded summaries include Method and the resolved `(lo, hi, hold)` DAC band. +- `ModulationStateV1.optical_drive` publishes the exact resolved optical + target, requested and resolved normalized mean `ū`, internal `u_g`/`u_c`, + requested `a`, `V_null`, and `Vπ` as an additive V1 field; A1 sidecars no + longer have to infer these from DAC endpoints. - `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. - The workflow-owner service and `WaveformV1` automation path remain exact-waveform contracts and do not use the UI Drive method. - **`SetOpticalDepth`** (ADR 010): under an automation lease the service can retarget the *depth* - `a` of the drive the operator armed — same `drive_command()` builder as the UI path, everything - else untouched. Refused with no device link, a manual-DAC method, or a constant mode; the derived - drive still passes all safety validation. Used by the A1 amplitude sweep. + `a` through the same `drive_command()` builder as the UI path. It is accepted only with an + applied transfer calibration and armed `OPTICAL_LOG_SINE`; no device link, manual/constant, + DAC/square/linear modes, or an unidentified hand-entered lobe are refused. Used by the A1 + amplitude sweep. - **Link watchdog**: the device thread exits after 5 consecutive serial failures (marking the device disconnected/faulted), and the control tick reaps a finished device thread and auto-reconnects with a 2 s backoff while `connect` stays requested. Previously a wedged or dead diff --git a/docs/features/stage-a-optical-waveform.md b/docs/features/stage-a-optical-waveform.md index c0ab98b..dffc19a 100644 --- a/docs/features/stage-a-optical-waveform.md +++ b/docs/features/stage-a-optical-waveform.md @@ -24,38 +24,59 @@ u(t) = \frac{I_d(t)-I_\text{floor}}{I_\text{ceil}-I_\text{floor}},\qquad V(u) = V_\text{null} + \frac{2 V_\pi}{\pi}\,\arcsin\!\sqrt{u}. ``` +The installed cell is the Excelitas **LM 0202**, part `84502049000`: four +KD*P crystals, 3×3 mm aperture, 400–850 nm, 5 W, nominal half-wave voltage +`210 V ±10 %` at 633 nm. The catalog value is a hardware plausibility check, +not a drive calibration; the plugin uses the measured DAC-domain +`V_null`/`Vπ`. + ## Targets -- **`OPTICAL_LOG_SINE`** (recommended A1 input): `ln I_d = ln I_g + (a/2)\sin\omega t`. - The event camera responds to changes in `ln I`, so this is the clean input. -- **`OPTICAL_LINEAR_SINE`**: `I_d = I_c(1 + m\sin\omega t)`, `m = \tanh(a/2)`. +- **`OPTICAL_LOG_SINE`** (recommended A1 input): + `ln u = ln u_g + (a/2)\sin\omega t`. +- **`OPTICAL_LINEAR_SINE`**: + `u = u_c(1 + m\sin\omega t)`, `m = \tanh(a/2)`. Both operate around an explicit operating point and are refused if their optical -maximum exceeds the lobe ceiling. `DAC_SINE` remains the pure-DAC sine. +maximum exceeds the lobe ceiling. Their headroom tests use their own target law; +the linear target is not tested against the log-sine endpoints. `DAC_SINE` +remains the pure-DAC sine. ## Inversion parameters (settable — you do not need a rig to start) | Setting | Meaning | |---|---| | `V_null` | DAC code at the excitation minimum (`sin² = 0`) | -| `Vπ` | DAC-code quarter-wave distance from `V_null` to the excitation maximum | -| `a` | requested optical log-modulation depth `ln(I_max/I_min)` | -| `I_k` | operating illumination as a normalised lobe intensity `u_k ∈ (0,1]` | +| `Vπ` | DAC-code **half-wave-voltage span** from `V_null` to the excitation maximum | +| `a` | requested peak-to-trough natural-log contrast `ln(I_max/I_min)` | +| `ū` | requested dimensionless floor-subtracted **cycle mean** in `(0,1]` | Get `V_null`/`Vπ` from a two-point check (code giving min light, code giving max light on one lobe) or from nominal `Vπ ÷ driver volts-per-code`. The drive is refused (never silently clamped) if `V_null + Vπ` overruns `0..4095`. -### Fixed operating point `I_k`, swept depth `a` - -`I_k` is the geometric-mean point the modulation swings around: -`u(t) = u_k·exp[(a/2) sin ωt]` (log) or `u_k·(1 + m sin ωt)` (linear). **Hold -`I_k` fixed and sweep `a`** for one response curve. The drive is refused -(`Saturates`) when the peak `u_k·exp(a/2) > 1` — lower `I_k` or `a`. - -`CONST` is the exception because it does not modulate: it maps only `I_k` +### `u` is not the physical flux point `I_k` + +The modulation setting `ū` is a normalized cycle mean, not photons per pixel +per second. For a log target the plugin derives the geometric pedestal +`u_g=ū/I_0(a/2)` before generating/sending the WARP parameters; for a linear +target the arithmetic centre is already `u_c=ū`. The physical A1 quantity +`I_k` is the cycle-mean local excitation flux after all optics and +sample/spatial mapping. A1 therefore records a separate, required +`flux_point_id`; it never derives absolute `I_k` from `ū`. + +For the log target, `u(t)=u_g exp[(a/2)sin ωt]` and +`⟨u⟩=u_g I_0(a/2)=ū`. The implemented Bessel normalization therefore holds +the normalized—and, for a stable affine floor/span, physical—cycle mean while +`a` is swept, to the existing `u_k_milli` wire resolution. The acknowledged +state and A1 sidecar publish both the requested mean and the resolved mean after +that milli-unit quantization, plus the internal `u_g`. The independent flux +calibration still supplies the absolute local `I_k` and verifies it on the +bench. + +`CONST` maps only `u` through the inverse lobe and ignores `a`. For example, `V_null=1630`, -`Vπ=860` gives DAC `2490` at `I_k=1` and DAC `1685` at `I_k=0.01`. +`Vπ=860` gives DAC `2490` at `u=1` and DAC `1685` at `u=0.01`. Periodic modes still require the headroom above. Invalid setting changes are rejected transactionally, so the UI retains the last applied value instead of showing a target that the board never received. Photodiode RAW/EXCITATION mode @@ -63,15 +84,15 @@ does not participate in this DAC calculation. ### Drive method and hard ceiling -Under `CALIBRATED`, `V_null`/`Vπ`/`I_k`/`a` define the operating band directly. +Under `CALIBRATED`, `V_null`/`Vπ`/`ū`/`a` define the operating band directly. Under `MANUAL`, the Power + Min-threshold DAC endpoints are passed through the -forward `sin²` transfer and converted to the target law's effective `(I_k, a)`; +forward `sin²` transfer and converted to the target law's effective `(u, a)`; the same inverse-warp implementation then fills that band. Warp codes are absolute lobe codes and cannot be rescaled without distorting the target. The plugin therefore **refuses** any drive whose peak exceeds the always-visible `max_level` hard ceiling. Raise the max limit, or lower the -operating band / `I_k` / `a` / `Vπ`, to fit. +operating band / `ū` / `a` / `Vπ`, to fit. ### Modulation reference range @@ -92,6 +113,22 @@ form, dropping in behind the same `warp_table` interface and superseding `V_null`/`Vπ` entirely. The calibration record already archives the points such a table would need. +This matters when the floor is finite. The implemented drive makes `u` obey the +requested target, so with `I=I_floor+(I_ceil-I_floor)u` the realised physical +contrast is + +```math +a_\mathrm{phys} = +\ln\frac{I_\mathrm{floor}+S u_g e^{a/2}} + {I_\mathrm{floor}+S u_g e^{-a/2}}, +\qquad S=I_\mathrm{ceil}-I_\mathrm{floor}. +``` + +It equals the requested `a` only for zero floor (or if contrast is explicitly +defined on floor-subtracted intensity). The measured photodiode `a` is therefore +the authority, and quantitative A1 acquisition requires the residual/floor +validation or the measured-LUT extension. + ## Wire form (firmware line limit) The command line is capped at 192 bytes, too small for a 256-code table, so the @@ -99,9 +136,11 @@ plugin computes and validates the warp table locally (for the operator preview and range guard) but sends the compact **parameters**: ``` -MOD wave=WARP freq_mhz= target= a_milli= u_k_milli= v_null= v_pi= +MOD wave=WARP freq_mhz= target= a_milli= u_k_milli= v_null= v_pi= ``` +For log-sine, the plugin first converts the UI's `ū` to +`u_g=ū/I_0(a/2)` and transmits that backward-compatible `u_k_milli` field. The firmware rebuilds the identical 256-entry DAC table with the same formula (`stimulus_mod::normalisedIntensity` + `dacForU`) and plays it back at the drive frequency. A chunked **table upload** command is the natural extension for the @@ -122,4 +161,8 @@ the requested `a`, that a manual DAC band round-trips through `OpticalDrive::from_dac_band`, and that invalid depth/inversion and lobe overruns are refused. `cargo test -p stage-a-io mod_warp` covers the mock command surface. The modulation-plugin tests also pin the full-lobe `CONST` values above and -verify that a rejected periodic `I_k` change cannot diverge from the board target. +verify that a rejected periodic `ū` change cannot diverge from the board +target, that Bessel normalization preserves the log-sine cycle mean, and that +linear-sine headroom uses the linear target law. The resolved optical-drive +provenance test also checks the exact milli-unit value sent to the controller +and the corresponding resolved cycle mean. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 07cb165..d8321e9 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -32,8 +32,8 @@ The mode is a **display** choice only. It selects what the chart and the sample never changes a published quantity (ADR 012). - **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). -- **EXCITATION** — the diode sits behind the PBS in the excitation path and measures the light - removed from the beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set +- **EXCITATION** — the diode sits at the PBS reject port and measures the light + removed from the sample beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. ## Optical log-contrast `a` @@ -45,14 +45,21 @@ so the estimator always runs the `RejectedComplement` geometry against `referenc amplitude sweep settles on this value, so a display toggle must not be able to move it (ADR 012). - **Reference I_tot** (`reference_volts`) is the total-power anchor: the PD reading with the full - beam diverted into the diode. Until it is set to a real measurement, `a` is withheld. + beam diverted into the diode. A non-empty **anchor id** and explicit + **measured and current** confirmation are required. Changing the value or id + clears confirmation; until all three agree, `a` is withheld. - **Dark level** (`dark_volts`) + the **Capture dark** button: block the beam and press; the mean of the current cache becomes the dark level. It is applied to the detector samples *and* to the `I_tot` anchor, so it cancels out of the complement rather than biasing `a` — its job is to keep the two sides consistent and to record the calibration the reading was taken under. `dark_id` in the sidecar reads `dark-measured` or `dark-none` accordingly. -- The estimator is **fail-closed**: it refuses on ADC clipping, on no headroom above dark, and when - the anchor is not above the measured signal. A refusal is shown as `a unavailable: ` +- The estimator uses only marker-bounded windows containing at least **two + complete modulation cycles**, ending on phase 0. It no longer estimates + extrema from an arbitrary trailing sample count; a low-frequency trace that + does not fit the bounded window is withheld rather than phase biased. +- The estimator is **fail-closed**: it refuses on a missing/unconfirmed anchor, + incomplete cycles, ADC clipping, no headroom above dark, and when the anchor + is not above the measured signal. A refusal is shown as `a unavailable: ` rather than a missing row — a wrong `a` is worse than no `a`. ## Chart diff --git a/docs/features/stage-a-pockels-calibration.md b/docs/features/stage-a-pockels-calibration.md index d1a9eca..142d694 100644 --- a/docs/features/stage-a-pockels-calibration.md +++ b/docs/features/stage-a-pockels-calibration.md @@ -46,8 +46,8 @@ gets brighter, does the photodiode reading go up or down?* Stage-A's photodiode sits on the PBS **reject** port and reads the light the sample does not get, `I_pd = I_tot − I_exc`, so it falls as the sample brightens — and reads its **maximum** at `V_null`. That is `REJECT PORT`, the default. `DIRECT` is for a -detector watching the sample beam itself. Declaring it wrong places `V_null` a -quarter wave off and runs the drive on the inverted branch. +detector watching the sample beam itself. Declaring it wrong places `V_null` +one half-wave-voltage span off and runs the drive on the inverted branch. **The shape needs no dark measurement and no anchor.** `p_0` absorbs the dark level and any DC offset; `p_1` absorbs the front-end gain. `V_null` and `Vπ` diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 50d98a6..f684a48 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -14,13 +14,17 @@ optical drive in the modulation plugin; A1 only reads its published settings. - **Output folder** — where the A1 config sidecar is written (recommended shared experiment root). - **Measurement id** — one per `(I_k, f)` pair; auto-generated default, editable, or press **New id**. +- **Physical I_k flux point id** — required canonical id of the cycle-mean local flux + calibration/map point; separate from normalized modulation mean `ū`. - **Duration (s)** — each recording auto-stops and finalizes after this. - **Start recording** — starts camera RAW, then connects/leases the photodiode and starts PDQ; the timer begins after both acknowledge. It auto-finalizes PDQ first, camera second, then writes the sidecar. **Stop** saves the current recording early (and aborts a running sweep). - **Start sweep** — records **Sweep points (count)** amplitudes spanning `[Sweep min a, Sweep max a]` (min > 0): per point it renews the modulation lease, issues `SetOpticalDepth`, waits for the - measured `a` to hold the target for **Sweep settle (s)** (30 s cap, then records anyway), and runs + fresh marker-bounded measured `a` to hold the target for **Sweep settle (s)**; timeout aborts + rather than recording an unsettled point. The modulation owner also requires an applied + transfer calibration and `OPTICAL_LOG_SINE`, and runs one normal recording. Sidecars carry `sweep.requested_a` / `point_index` / `point_total`. - The record/sweep buttons are disabled until an output folder is selected. diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 64ca65b..f531666 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -45,10 +45,11 @@ use serde::Serialize; use serde_json::{json, Value}; use stage_a_plugin_contract::{ ClientId, ConnectionStateV1, LeaseId, ModulationCommandV1, ModulationRequestV1, - ModulationStateV1, PdqReceiptV1, PdqStartSpecV1, PhotodiodeCommandV1, PhotodiodeRequestV1, - PhotodiodeResponseV1, PhotodiodeSummaryV1, RequestId, RunId, SemanticRevision, WaveformV1, - CTX_STAGE_A_MODULATION_STATE_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, - SERVICE_STAGE_A_MODULATION_CONTROL_V1, SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, + ModulationStateV1, OpticalTargetV1, PdqReceiptV1, PdqStartSpecV1, PhotodiodeCommandV1, + PhotodiodeOpticalSummaryV1, PhotodiodeRequestV1, PhotodiodeResponseV1, PhotodiodeSummaryV1, + RequestId, RunId, SemanticRevision, WaveformV1, CTX_STAGE_A_MODULATION_STATE_V1, + CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, SERVICE_STAGE_A_MODULATION_CONTROL_V1, + SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, }; use crate::phase::{fold_events, fold_events_free_running, MarkerValidationConfig, PhaseFold}; @@ -320,6 +321,9 @@ pub struct StageAA1Plugin { // -- recording coordinator -- output_folder: String, measurement_id: String, + /// Canonical identifier of the physical cycle-mean local flux point + /// `I_k` (for example a row in the illumination calibration/map). + flux_point_id: String, /// Sweep range `[min_a, max_a]` for this `(I_k, f)` row (automation template). min_a: f64, max_a: f64, @@ -380,6 +384,7 @@ impl Default for StageAA1Plugin { background_floor: None, output_folder: String::new(), measurement_id: generate_measurement_id(), + flux_point_id: String::new(), min_a: 0.0, max_a: 2.0, duration_s: 10, @@ -633,12 +638,20 @@ impl StageAA1Plugin { .collect() } - /// Optical modulation depth `a` published by the photodiode plugin. + /// Fresh optical summary from a connected photodiode owner. + fn fresh_optical_summary(&self) -> Option<&PhotodiodeOpticalSummaryV1> { + let state = self.photodiode.as_ref()?; + if !matches!(state.connection, ConnectionStateV1::Connected { .. }) + || state.freshness.is_stale_at(now_unix_ms()) + { + return None; + } + state.optical_summary.as_ref() + } + + /// Fresh optical modulation depth `a` published by the photodiode plugin. fn measured_a(&self) -> Option { - self.photodiode - .as_ref()? - .optical_summary - .as_ref() + self.fresh_optical_summary() .map(|summary| summary.measured_log_contrast) } @@ -979,6 +992,7 @@ impl StageAA1Plugin { fn recording_metadata(&self) -> BTreeMap { let mut meta = BTreeMap::new(); meta.insert("a1_measurement_id".into(), self.recording.id.clone()); + meta.insert("a1_flux_point_id".into(), self.flux_point_id.clone()); meta.insert("a1_stem".into(), self.recording.stem.clone()); meta.insert("a1_role".into(), self.recording.role.label().into()); meta.insert( @@ -1034,6 +1048,10 @@ impl StageAA1Plugin { self.note("Set a measurement id before recording"); return; } + if self.flux_point_id.trim().is_empty() { + self.note("Set the physical I_k flux point id before recording"); + return; + } let now_ms = now_unix_ms(); let id = sanitize_stem(self.measurement_id.trim()); // Sweep points get a stable per-point tag so the row's files sort by @@ -1316,10 +1334,30 @@ impl StageAA1Plugin { self.message = "Set a measurement id before sweeping".into(); return; } + if self.flux_point_id.trim().is_empty() { + self.message = "Set the physical I_k flux point id before sweeping".into(); + return; + } if !self.modulation_connected() { self.message = "Modulation owner is not connected — cannot sweep".into(); return; } + if self + .modulation + .as_ref() + .and_then(|state| state.calibration_id.as_deref()) + .is_none() + { + self.message = + "Apply a measured Pockels transfer calibration before starting an A1 sweep".into(); + return; + } + if self.fresh_optical_summary().is_none() { + self.message = "Connect the photodiode and obtain a fresh, marker-bounded optical \ + summary from a confirmed I_tot anchor before sweeping" + .into(); + return; + } if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { self.message = "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); @@ -1499,24 +1537,26 @@ impl StageAA1Plugin { sweep.settled_since_ms = None; } if !start_recording && now_ms >= sweep.settle_deadline_ms { - // Record anyway: the sidecar stores the *measured* a, - // so an unsettled point is still a usable sample. - start_recording = true; settle_timed_out = true; } if start_recording { sweep.phase = SweepPhase::Recording; } } + if settle_timed_out { + self.finish_sweep( + context, + format!( + "Sweep aborted at point {}/{}: no fresh, settled optical a at \ + target {target:.3} before timeout", + index + 1, + total + ), + ); + return; + } if start_recording { self.pending_role = Some(RecRole::Normal); - if settle_timed_out { - self.message = format!( - "Sweep point {}/{}: a did not settle at {target:.3} — recording anyway", - index + 1, - total, - ); - } } } SweepPhase::Recording => { @@ -1818,10 +1858,18 @@ impl StageAA1Plugin { .as_ref() .and_then(|s| s.acknowledged.as_ref()); let a1_config = modulation.and_then(|t| t.a1_configuration.as_ref()); - let optical = self - .photodiode + let mod_optical = self + .modulation .as_ref() - .and_then(|s| s.optical_summary.as_ref()); + .and_then(|state| state.optical_drive.as_ref()); + let optical = self.fresh_optical_summary(); + if optical.is_none() { + return Err( + "cannot write a quantitative A1 sidecar without a fresh photodiode optical \ + summary from a confirmed I_tot anchor" + .into(), + ); + } let roi = self.host_roi.unwrap_or_default(); let raw_path = self @@ -1833,6 +1881,7 @@ impl StageAA1Plugin { let doc = SidecarDoc { measurement_id: self.recording.id.clone(), + flux_point_id: self.flux_point_id.trim().to_owned(), file_stem: self.recording.stem.clone(), role: self.recording.role.label().into(), recorded_at_utc: format_iso_utc( @@ -1873,6 +1922,22 @@ impl StageAA1Plugin { modulation: ModulationSidecar { frequency_hz: self.period_us().map(|t| 1_000_000.0 / t), frequency_source: self.frequency_source().into(), + calibration_id: self + .modulation + .as_ref() + .and_then(|state| state.calibration_id.clone()), + optical_target: mod_optical.map(|drive| match drive.target { + OpticalTargetV1::LogSine => "log_sine".into(), + OpticalTargetV1::LinearSine => "linear_sine".into(), + }), + requested_mean_u: mod_optical + .map(|drive| f64::from(drive.requested_mean_u_milli) / 1_000.0), + resolved_mean_u: mod_optical + .map(|drive| f64::from(drive.resolved_mean_u_milli) / 1_000.0), + internal_u: mod_optical.map(|drive| f64::from(drive.internal_u_milli) / 1_000.0), + requested_a: mod_optical.map(|drive| f64::from(drive.depth_a_milli) / 1_000.0), + v_null_dac: mod_optical.map(|drive| drive.v_null_dac), + v_pi_dac: mod_optical.map(|drive| drive.v_pi_dac), center_dac: a1_config.map(|c| c.center_dac), amplitude_dac: a1_config.map(|c| c.amplitude_dac), waveform: modulation @@ -1881,9 +1946,19 @@ impl StageAA1Plugin { }, optical: OpticalSidecar { measured_a: optical.map(|o| o.measured_log_contrast), + geometric_mean_excitation_volts: optical + .map(|o| (o.excitation_min_volts * o.excitation_max_volts).sqrt()), + excitation_min_volts: optical.map(|o| o.excitation_min_volts), + excitation_max_volts: optical.map(|o| o.excitation_max_volts), + excitation_headroom_volts: optical.map(|o| o.excitation_headroom_volts), low_clip_fraction: optical.map(|o| o.low_clip_fraction), high_clip_fraction: optical.map(|o| o.high_clip_fraction), measured_frequency_hz: optical.and_then(|o| o.measured_frequency_hz), + adc_calibration_id: optical.map(|o| o.calibration.adc_calibration_id.clone()), + dark_id: optical.map(|o| o.calibration.dark_id.clone()), + total_power_anchor_id: optical.map(|o| o.calibration.anchor_id.clone()), + dark_volts: optical.map(|o| o.calibration.dark_volts), + total_power_volts: optical.map(|o| o.calibration.total_power_volts), }, camera: CameraSidecar { roi_x: roi.x, @@ -1920,6 +1995,7 @@ impl StageAA1Plugin { #[derive(Serialize)] struct SidecarDoc { measurement_id: String, + flux_point_id: String, file_stem: String, role: String, recorded_at_utc: String, @@ -1985,6 +2061,22 @@ struct ModulationSidecar { frequency_hz: Option, frequency_source: String, #[serde(skip_serializing_if = "Option::is_none")] + calibration_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + optical_target: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_mean_u: Option, + #[serde(skip_serializing_if = "Option::is_none")] + resolved_mean_u: Option, + #[serde(skip_serializing_if = "Option::is_none")] + internal_u: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_a: Option, + #[serde(skip_serializing_if = "Option::is_none")] + v_null_dac: Option, + #[serde(skip_serializing_if = "Option::is_none")] + v_pi_dac: Option, + #[serde(skip_serializing_if = "Option::is_none")] center_dac: Option, #[serde(skip_serializing_if = "Option::is_none")] amplitude_dac: Option, @@ -1997,11 +2089,29 @@ struct OpticalSidecar { #[serde(skip_serializing_if = "Option::is_none")] measured_a: Option, #[serde(skip_serializing_if = "Option::is_none")] + geometric_mean_excitation_volts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + excitation_min_volts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + excitation_max_volts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + excitation_headroom_volts: Option, + #[serde(skip_serializing_if = "Option::is_none")] low_clip_fraction: Option, #[serde(skip_serializing_if = "Option::is_none")] high_clip_fraction: Option, #[serde(skip_serializing_if = "Option::is_none")] measured_frequency_hz: Option, + #[serde(skip_serializing_if = "Option::is_none")] + adc_calibration_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dark_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + total_power_anchor_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dark_volts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + total_power_volts: Option, } #[derive(Serialize)] @@ -2403,6 +2513,19 @@ impl Plugin for StageAA1Plugin { default: self.measurement_id.clone(), }, }, + SettingItem { + key: "flux_point_id".into(), + label: "Physical I_k flux point id".into(), + tooltip: Some( + "Canonical id of the cycle-mean local flux calibration/map point. \ + This is physical photons/pixel/s provenance, not the modulation \ + plugin's dimensionless lobe coordinate u." + .into(), + ), + kind: SettingKind::Text { + default: self.flux_point_id.clone(), + }, + }, SettingItem { key: "new_id".into(), label: "New id".into(), @@ -2665,6 +2788,7 @@ impl Plugin for StageAA1Plugin { match key { "output_folder" => Some(json!(self.output_folder)), "measurement_id" => Some(json!(self.measurement_id)), + "flux_point_id" => Some(json!(self.flux_point_id)), "min_a" => Some(json!(self.min_a)), "max_a" => Some(json!(self.max_a)), "sweep_count" => Some(json!(self.sweep_count)), @@ -2706,6 +2830,12 @@ impl Plugin for StageAA1Plugin { .ok_or("measurement_id must be a string")? .to_string(); } + "flux_point_id" => { + self.flux_point_id = value + .as_str() + .ok_or("flux_point_id must be a string")? + .to_string(); + } "new_id" if value.as_bool() == Some(true) => { self.measurement_id = generate_measurement_id(); } @@ -3025,8 +3155,9 @@ export_plugin!(StageAA1Plugin); #[cfg(test)] mod tests { use stage_a_plugin_contract::{ - OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, RequestOutcomeV1, - ResponseCommonV1, Sha256V1, StreamIntegrityV1, CONTRACT_VERSION_V1, + FreshnessV1, OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, + PhotodiodeCalibrationV1, PhotodiodeStreamV1, RequestOutcomeV1, ResponseCommonV1, Sha256V1, + StreamIntegrityV1, SynchronizationV1, UnsyncedReasonV1, CONTRACT_VERSION_V1, }; use super::*; @@ -3096,6 +3227,61 @@ mod tests { } } + fn fresh_photodiode_summary() -> PhotodiodeSummaryV1 { + PhotodiodeSummaryV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("pd-test"), + service_revision: 1, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: Some("test".into()), + }, + lease: None, + active_run_id: None, + requested_revision: None, + acknowledged_revision: None, + stream: PhotodiodeStreamV1 { + stream_epoch: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + latest_adc_code: Some(1_000), + integrity: StreamIntegrityV1::default(), + level: None, + }, + active_recording: None, + last_finalized_recording: None, + optical_summary: Some(PhotodiodeOpticalSummaryV1 { + run_id: RunId::from("test-run"), + calibration: PhotodiodeCalibrationV1 { + adc_calibration_id: "adc-test".into(), + dark_id: "dark-test".into(), + anchor_id: "itot-test".into(), + dark_volts: 0.05, + total_power_volts: 3.0, + }, + measured_log_contrast: 1.0, + log_contrast_stddev: None, + excitation_min_volts: 0.8, + excitation_max_volts: 0.8 * std::f64::consts::E, + excitation_headroom_volts: 0.8, + low_clip_fraction: 0.0, + high_clip_fraction: 0.0, + measured_frequency_hz: Some(1_000.0), + fundamental_phase_rad: None, + total_harmonic_distortion: None, + }), + synchronization: SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 60_000, + }, + } + } + /// A plugin whose period comes from marker spacing (no fallback frequency). fn plugin_with_markers() -> StageAA1Plugin { StageAA1Plugin { @@ -3103,6 +3289,8 @@ mod tests { frame_width: 10, frame_height: 1, camera_markers_us: vec![0, 1_000, 2_000, 3_000], + flux_point_id: "flux-test".into(), + photodiode: Some(fresh_photodiode_summary()), ..StageAA1Plugin::default() } } @@ -3186,6 +3374,7 @@ mod tests { assert_eq!(valid, 4); assert!(q_on > 0.9 && q_off > 0.9, "q_on={q_on} q_off={q_off}"); // Recording a point is still refused without a photodiode-measured a. + plugin.photodiode = None; assert!(plugin.measured_a().is_none()); assert!(plugin.record_response_point().is_err()); } @@ -3276,8 +3465,14 @@ mod tests { // must not record those as if they had come from this run. let mut plugin = plugin_with_markers(); plugin.pilot_windows = Some(( - PhaseWindow { start: 0.0, end: 0.2 }, - PhaseWindow { start: 0.5, end: 0.7 }, + PhaseWindow { + start: 0.0, + end: 0.2, + }, + PhaseWindow { + start: 0.5, + end: 0.7, + }, )); // No events => the fold carries no signal => the freeze cannot pick // windows and must not leave the loaded ones in place. @@ -3336,6 +3531,8 @@ mod tests { let mut plugin = StageAA1Plugin { output_folder: folder.display().to_string(), measurement_id: "A1-row".into(), + flux_point_id: "flux-row-1".into(), + photodiode: Some(fresh_photodiode_summary()), duration_s: 1, pending_role: Some(RecRole::Normal), ..StageAA1Plugin::default() @@ -3625,6 +3822,7 @@ mod tests { let doc = plugin.write_sidecar().expect("sidecar path"); let text = std::fs::read_to_string(&doc).expect("read sidecar"); assert!(text.contains("measurement_id = \"A1-test\"")); + assert!(text.contains("flux_point_id = \"flux-test\"")); assert!(text.contains("[modulation]")); assert!(text.contains("[camera]")); assert!(text.contains("[files]")); diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index bb29cab..2bedf3a 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -2,12 +2,14 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy **command port** (the first of the two USB serial ports enumerated by `stage-a-controller` firmware 0.3.0+). +The optical load is an Excelitas **LM 0202** (`84502049000`), four-crystal +KD*P, 3×3 mm, 400–850 nm, 5 W. ## What it does - **Drive method** selects how the DAC operating band is defined: - `MANUAL`: **Power** is the peak/operating code and **Min threshold** is the lower endpoint. - - `CALIBRATED`: `V_null`, `Vπ`, `I_k`, and optical depth `a` determine the endpoints. + - `CALIBRATED`: `V_null`, `Vπ`, normalized cycle mean `ū`, and optical depth `a` determine the endpoints. Measure `V_null`/`Vπ` with the built-in transfer sweep — see [Calibration](#calibration--measuring-v_null-and-vπ). - **Mode** independently selects the waveform that fills that band. All five modes are available under both methods. @@ -17,21 +19,22 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** - The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and a 2 Hz `STATUS` poll), plus the selected method and resolved DAC band. -| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `Vπ` | +| Mode | Manual band `[min, power]` | Calibrated band from `ū`, `a`, `V_null`, `Vπ` | |---|---|---| -| `CONST` | hold `power` | hold the DAC code for `I_k` | +| `CONST` | hold `power` | hold the DAC code for `ū` | | `DAC_SINE` | DAC sine across the band | DAC sine across the band | | `SQUARE` | DAC square across the band | DAC square across the band | -| `OPTICAL_LOG_SINE` | optical log-sine across the band | optical log-sine about `I_k` | -| `OPTICAL_LINEAR_SINE` | optical linear-sine across the band | optical linear-sine about `I_k` | +| `OPTICAL_LOG_SINE` | optical log-sine across the band | mean `ū`, converted to `u_g=ū/I_0(a/2)` | +| `OPTICAL_LINEAR_SINE` | optical linear-sine across the band | centre/mean `u_c=ū` | -Manual optical modes reuse the stored `V_null`/`Vπ` lobe and derive their effective `(I_k, a)` +Manual optical modes reuse the stored `V_null`/`Vπ` lobe and derive their effective `(u, a)` from the slider band through the forward optical transfer. In calibrated `CONST`, `a` is irrelevant: the hold is -`V_null + (2Vπ/π)·asin(sqrt(I_k))`. With `V_null=1630` and `Vπ=860`, this is -2490 at `I_k=1` and 1685 at `I_k=0.01`. Periodic modes still need optical -headroom and reject impossible `I_k`/`a` combinations without changing the +`V_null + (2Vπ/π)·asin(sqrt(u))`. With `V_null=1630` and `Vπ=860`, this is +2490 at `ū=1` and 1685 at `ū=0.01`. This dimensionless `ū` is not the physical +A1 flux point `I_k`. Periodic modes still need optical +headroom and reject impossible `ū`/`a` combinations without changing the displayed setting or leaving it out of sync with the board. ## Calibration — measuring `V_null` and `Vπ` @@ -44,7 +47,7 @@ gain, temperature, and the actual electrical load all enter the realised map, so 2. Set **Detector port**. Stage-A watches the PBS *reject* port, where the detector is **brightest** at `V_null` — the default. This cannot be inferred from the sweep: a bright and a dark extremum fit the measured curve equally well, and only the optics say which one is zero - excitation. Getting it wrong puts `V_null` a quarter wave out. + excitation. Getting it wrong puts `V_null` one half-wave-voltage span out. 3. Press **Measure transfer curve**. It steps settled `CONST` codes across `0..max limit`, up and back down (~20 s), and fits the lobe. Your armed drive is restored afterwards, on every exit path. diff --git a/plugins/stage-a-modulation/src/calibration.rs b/plugins/stage-a-modulation/src/calibration.rs index 9bddcfe..900ffd1 100644 --- a/plugins/stage-a-modulation/src/calibration.rs +++ b/plugins/stage-a-modulation/src/calibration.rs @@ -22,7 +22,7 @@ //! excitation null. That is a physical fact about the port, not a fit //! parameter, so [`fit_transfer`] takes the geometry as an **input** and picks //! the matching representation. Getting it wrong would place `V_null` a -//! quarter wave off and run the drive on the inverted branch, so it is asked +//! one half-wave-voltage span off and run the drive on the inverted branch, so it is asked //! rather than guessed. //! //! Two consequences worth stating, because they remove procedure rather than @@ -117,7 +117,7 @@ impl DetectorGeometry { pub struct TransferFit { /// DAC code at the excitation minimum. pub v_null_dac: f64, - /// DAC codes from `v_null` to the excitation maximum (quarter wave). + /// DAC codes from `v_null` to the excitation maximum (one half-wave-voltage span). pub v_pi_dac: f64, /// Detector volts at the excitation null (`p0`). pub offset_volts: f64, @@ -131,7 +131,7 @@ pub struct TransferFit { /// span. `None` when the sweep ran in one direction only. pub hysteresis: Option, /// Fraction of one full lobe (`Vπ` codes) the sweep actually covered. - /// Below ~1 the quarter-wave distance is extrapolated, not measured. + /// Below ~1 the half-wave-voltage span is extrapolated, not measured. pub lobe_coverage: f64, /// Points discarded as wild before the final fit. A couple is ordinary; a /// large share means the sweep, not the model, is the problem. @@ -201,7 +201,7 @@ pub const MIN_POINTS: usize = 16; /// A detector span below this is treated as noise rather than a lobe. const MIN_SPAN_VOLTS: f64 = 0.01; -/// Least-squares solution for one candidate quarter wave `w`. +/// Least-squares solution for one candidate half-wave-voltage span `w`. struct Harmonic { /// Mean level `A`, and the quadrature amplitudes of `cos`/`sin(πc/w)`. mean: f64, @@ -387,7 +387,7 @@ fn hysteresis_fraction(points: &[SweepPoint], span: f64) -> Option { Some(differences.iter().sum::() / differences.len() as f64 / span.abs()) } -/// Scans the quarter wave over every period the sweep could resolve, then +/// Scans the half-wave-voltage span over every period the sweep could resolve, then /// refines. Returns the best `(Vπ, harmonic)`. fn fit_period(points: &[SweepPoint], swept_span: f64) -> Option<(f64, Harmonic)> { // From four samples per lobe (below that the lobe is aliased) out to a @@ -489,7 +489,7 @@ pub fn fit_transfer( // `A + R·cos(θ − φ)` with `θ = πc/w` is the same curve as // `p0 + p1·sin²(π(c − v)/(2w))` with `|p1| = 2R`. Which of the two signs // of `p1` applies — and therefore whether the null sits at the phase or a - // quarter wave past it — is the geometry question the data cannot answer. + // one half-wave-voltage span past it — is the geometry question the data cannot answer. let radius = harmonic.amplitude; let (v, p0, p1) = match geometry { DetectorGeometry::RejectedComplement => ( @@ -636,7 +636,7 @@ mod tests { #[test] fn geometry_selects_between_the_two_equivalent_representations() { // One curve, two readings. Declaring the wrong port must move V_null by - // exactly a quarter wave — the failure this input exists to prevent. + // exactly one half-wave-voltage span — the failure this input exists to prevent. let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, false); let reject = fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); @@ -758,7 +758,7 @@ mod tests { } #[test] - fn lobe_coverage_flags_an_extrapolated_quarter_wave() { + fn lobe_coverage_flags_an_extrapolated_half_wave_span() { // Sweeping only to code 800 with Vπ = 1600 sees half a lobe. let points = synthetic_sweep(0.0, 1_600.0, 2.4, -2.2, 800, 0.001, false); let fit = diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 2dc3e51..e4ef492 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -43,11 +43,11 @@ use stage_a_io::{Command, DeviceEvent, MockController, StageAClient, Transport}; use stage_a_plugin_contract::{ A1AcquisitionConfigV1, ClientId, ConnectionStateV1, ControllerStateV1, FreshnessV1, LeaseId, LeaseSnapshotV1, ModulationCommandV1, ModulationRequestV1, ModulationResponseV1, - ModulationStateV1, ModulationTargetV1, OwnerInstanceId, PhotodiodeLevelV1, PhotodiodeSummaryV1, - RequestOutcomeV1, ResponseCommonV1, RunId, SemanticRevision, ServiceErrorCodeV1, - ServiceErrorV1, SynchronizationV1, UnsyncedReasonV1, WaveformV1, CONTRACT_VERSION_V1, - CTX_STAGE_A_MODULATION_STATE_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, - PLUGIN_ID_STAGE_A_MODULATION, PLUGIN_ID_STAGE_A_PHOTODIODE, + ModulationStateV1, ModulationTargetV1, OpticalDriveStateV1, OpticalTargetV1, OwnerInstanceId, + PhotodiodeLevelV1, PhotodiodeSummaryV1, RequestOutcomeV1, ResponseCommonV1, RunId, + SemanticRevision, ServiceErrorCodeV1, ServiceErrorV1, SynchronizationV1, UnsyncedReasonV1, + WaveformV1, CONTRACT_VERSION_V1, CTX_STAGE_A_MODULATION_STATE_V1, + CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, PLUGIN_ID_STAGE_A_MODULATION, PLUGIN_ID_STAGE_A_PHOTODIODE, SERVICE_STAGE_A_MODULATION_CONTROL_V1, }; @@ -909,12 +909,12 @@ pub struct StageAModulationPlugin { /// The operator's armed `depth_a`, parked while a lease drives the optical /// depth (A1's amplitude sweep) and restored by [`Self::end_lease`]. armed_depth_a: Option, - /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0,1]`. - /// Held fixed while `a` is swept, so one response curve keeps `I_k` constant. + /// Dimensionless, floor-subtracted **cycle-mean** lobe coordinate + /// `ū ∈ (0,1]`; not the physical A1 flux point `I_k`. operating_point: f64, /// DAC code at the excitation minimum of one monotonic Pockels lobe. v_null_dac: i64, - /// DAC-code quarter-wave distance from `v_null` to the excitation maximum. + /// DAC-code half-wave-voltage span from `v_null` to the excitation maximum. v_pi_dac: i64, // -- measured transfer calibration -- /// Bench detector geometry. Not inferable from a sweep — see @@ -1123,6 +1123,24 @@ impl StageAModulationPlugin { } } + /// Resolves the calibrated UI setting (cycle-mean normalized lobe + /// coordinate) to the target law's internal pedestal/centre. + fn periodic_lobe_point(&self) -> f64 { + let point = if self.mode == Mode::OpticalLogSine { + waveform::log_sine_geometric_pedestal(self.operating_point, self.depth_a) + } else { + self.operating_point + }; + if matches!(self.mode, Mode::OpticalLogSine | Mode::OpticalLinearSine) { + // The firmware contract carries this coordinate in milli-units. + // Validate and preview the value the board will actually rebuild, + // not a higher-precision local table. + (point * 1_000.0).round() / 1_000.0 + } else { + point + } + } + /// Resolves the selected method into the DAC band used by every waveform. /// The third value is the constant-mode operating code. fn dac_band(&self) -> Result<(i64, i64, i64), String> { @@ -1131,9 +1149,9 @@ impl StageAModulationPlugin { return Ok((self.min_level.clamp(0, hi), hi, hi)); } - let u_k = self.operating_point; + let mean_u = self.operating_point; let a = self.depth_a; - if !u_k.is_finite() || u_k <= 0.0 || u_k > 1.0 { + if !mean_u.is_finite() || mean_u <= 0.0 || mean_u > 1.0 { return Err("operating point must be in (0, 1]".into()); } let inversion = self.lobe_inversion(); @@ -1141,12 +1159,10 @@ impl StageAModulationPlugin { return Err("Vπ must be finite and positive".into()); } - // Constant hold at I_k modulates nothing: no ±a/2 headroom applies, so - // the full (0, 1] range of u_k is expressible (I_k = 1 holds exactly at - // V_null + Vπ). Requiring the modulated band here silently froze the - // drive at the last accepted code whenever u_k·e^{a/2} exceeded 1. + // A constant hold modulates nothing: no depth headroom applies, so the + // full (0, 1] range of normalized lobe coordinate u is expressible. if self.mode == Mode::Const { - let hold = inversion.dac_for_u(u_k).round() as i64; + let hold = inversion.dac_for_u(mean_u).round() as i64; if hold < 0 { return Err(format!( "calibrated hold code {hold} is below 0; re-measure V_null/Vπ" @@ -1154,7 +1170,7 @@ impl StageAModulationPlugin { } if hold > self.max_level { return Err(format!( - "calibrated hold {hold} exceeds the max limit {}; raise the max limit or lower I_k / Vπ", + "calibrated hold {hold} exceeds the max limit {}; raise the max limit or lower u / Vπ", self.max_level )); } @@ -1164,17 +1180,22 @@ impl StageAModulationPlugin { if !a.is_finite() || a <= 0.0 { return Err("optical depth a must be finite and positive".into()); } - let u_lo = u_k * (-0.5 * a).exp(); - let u_hi = u_k * (0.5 * a).exp(); + let target_u = self.periodic_lobe_point(); + let (u_lo, u_hi) = if self.mode == Mode::OpticalLinearSine { + let m = (0.5 * a).tanh(); + (target_u * (1.0 - m), target_u * (1.0 + m)) + } else { + (target_u * (-0.5 * a).exp(), target_u * (0.5 * a).exp()) + }; if u_hi > 1.0 { return Err(format!( - "calibrated optical peak u = {u_hi:.3} exceeds the lobe ceiling; lower a or I_k" + "calibrated optical peak u = {u_hi:.3} exceeds the lobe ceiling; lower a or u" )); } let lo = inversion.dac_for_u(u_lo).round() as i64; let hi = inversion.dac_for_u(u_hi).round() as i64; - let hold = inversion.dac_for_u(u_k).round() as i64; + let hold = inversion.dac_for_u(target_u).round() as i64; if lo < 0 { return Err(format!( "calibrated lower DAC code {lo} is below 0; re-measure V_null/Vπ" @@ -1182,7 +1203,7 @@ impl StageAModulationPlugin { } if hi > self.max_level { return Err(format!( - "calibrated peak {hi} exceeds the max limit {}; raise the max limit or lower a / I_k / Vπ", + "calibrated peak {hi} exceeds the max limit {}; raise the max limit or lower a / u / Vπ", self.max_level )); } @@ -1200,12 +1221,43 @@ impl StageAModulationPlugin { DriveMethod::Calibrated => waveform::OpticalDrive { target, depth_a: self.depth_a, - operating_point: self.operating_point, + operating_point: self.periodic_lobe_point(), inversion, }, } } + fn optical_drive_state(&self) -> Option { + if self.method != DriveMethod::Calibrated { + return None; + } + let (target, contract_target) = match self.mode { + Mode::OpticalLogSine => (waveform::OpticalTarget::LogSine, OpticalTargetV1::LogSine), + Mode::OpticalLinearSine => ( + waveform::OpticalTarget::LinearSine, + OpticalTargetV1::LinearSine, + ), + _ => return None, + }; + self.dac_band().ok()?; + let drive = self.optical_drive(target); + Some(OpticalDriveStateV1 { + target: contract_target, + requested_mean_u_milli: (self.operating_point * 1_000.0).round() as u32, + resolved_mean_u_milli: (match target { + waveform::OpticalTarget::LogSine => { + waveform::log_sine_cycle_mean(drive.operating_point, self.depth_a) + } + waveform::OpticalTarget::LinearSine => drive.operating_point, + } * 1_000.0) + .round() as u32, + internal_u_milli: (drive.operating_point * 1_000.0).round() as u32, + depth_a_milli: (self.depth_a * 1_000.0).round() as u32, + v_null_dac: u16::try_from(self.v_null_dac).ok()?, + v_pi_dac: u16::try_from(self.v_pi_dac).ok()?, + }) + } + /// Builds the single MOD command carrying the complete current drive /// settings (mode, method, band, frequency). Shared by the operator path /// (`send_modulation`) and the leased `SetOpticalDepth` service command. @@ -1308,7 +1360,7 @@ impl StageAModulationPlugin { if i64::from(peak) > self.max_level { return Err(format!( "optical peak {peak} exceeds the max limit {}; raise the max limit or lower the \ - operating band / a / I_k / Vπ", + operating band / a / u / Vπ", self.max_level )); } @@ -1969,13 +2021,17 @@ impl StageAModulationPlugin { false, )); } - // Only the calibrated drive expresses an optical depth; the - // manual DAC band and the constant hold do not. - if self.method == DriveMethod::Manual || self.mode == Mode::Const { + // A1's depth command has one scientific meaning: a calibrated + // log-intensity sine. Reject every other mode instead of + // silently sweeping a DAC or linear-intensity waveform. + if self.method != DriveMethod::Calibrated + || self.mode != Mode::OpticalLogSine + || self.calibration_id.is_none() + { return Err(service_error( ServiceErrorCodeV1::InvalidCommand, - "arm a calibrated periodic/optical drive in the modulation plugin \ - before sweeping the optical depth", + "apply a calibration and arm OPTICAL_LOG_SINE in the modulation plugin \ + before sweeping optical depth a", false, )); } @@ -2148,6 +2204,7 @@ impl StageAModulationPlugin { valid_for_ms: 1_500, }, calibration_id: self.calibration_id.clone(), + optical_drive: self.optical_drive_state(), } } @@ -2932,7 +2989,7 @@ impl Plugin for StageAModulationPlugin { label: "Drive method".into(), tooltip: Some( "MANUAL defines the DAC band with Power and Min threshold. CALIBRATED \ - derives it from V_null, Vπ, I_k, and optical depth a." + derives it from V_null, Vπ, normalized u, and optical depth a." .into(), ), kind: SettingKind::Enum { @@ -3016,7 +3073,7 @@ impl Plugin for StageAModulationPlugin { key: "v_pi_dac".into(), label: "Vπ (DAC codes, null → max light)".into(), tooltip: Some( - "DAC-code quarter-wave distance from V_null to the excitation \ + "DAC-code half-wave-voltage span from V_null to the excitation \ maximum. V_null + Vπ must stay within 0..4095." .into(), ), @@ -3028,10 +3085,11 @@ impl Plugin for StageAModulationPlugin { }); modulation_items.push(SettingItem { key: "operating_point".into(), - label: "Operating point I_k (0..1)".into(), + label: "Normalized mean lobe point ū (0..1)".into(), tooltip: Some( - "Calibrated operating illumination as normalised lobe intensity u_k. \ - CONST holds its DAC code; the calibrated band is derived around it." + "Dimensionless floor-subtracted cycle mean, not physical A1 flux I_k. \ + OPTICAL_LOG_SINE converts it to u_g = ū/I₀(a/2), so sweeping a keeps \ + the normalized mean fixed; OPTICAL_LINEAR_SINE uses it as its centre." .into(), ), kind: SettingKind::F64Drag { @@ -3045,8 +3103,8 @@ impl Plugin for StageAModulationPlugin { key: "depth_a".into(), label: "Optical depth a".into(), tooltip: Some( - "Calibrated log-intensity span a = ln(I_max/I_min). Together with I_k \ - it defines the operating band used by every mode." + "Peak-to-trough natural-log contrast a = ln(I_max/I_min). Together with \ + the normalized lobe point u it defines the requested optical band." .into(), ), kind: SettingKind::F64Drag { @@ -3104,7 +3162,7 @@ impl Plugin for StageAModulationPlugin { the sample beam itself. The sweep cannot work this out: a bright \ and a dark extremum fit the measured curve equally well, and \ only the optics say which one is zero light on the sample. \ - Choosing wrong puts V_null a quarter wave off." + Choosing wrong puts V_null one half-wave-voltage span off." .into(), ), kind: SettingKind::Enum { @@ -3440,7 +3498,7 @@ impl Plugin for StageAModulationPlugin { self.detector_geometry = calibration::DetectorGeometry::from_name(&chosen) .ok_or("unknown detector geometry")?; // The stored fit was resolved against the old geometry; re-fit - // rather than leave a V_null that is now a quarter wave out. + // rather than leave a V_null that is now one half-wave-voltage span out. if let Some(fit) = self.fit.take() { match calibration::fit_transfer( &fit.points, @@ -3551,9 +3609,10 @@ impl Plugin for StageAModulationPlugin { Ok(_) => { let drive = self.optical_drive(target); entries.push(StatusEntry::Text(format!( - "{}: a={:.2}, I_k={:.2}, V_null={}, Vπ={} @ {:.3} Hz", + "{}: a={:.2}, ū={:.2}, internal u={:.2}, V_null={}, Vπ={} @ {:.3} Hz", self.mode.name(), drive.depth_a, + self.operating_point, drive.operating_point, self.v_null_dac, self.v_pi_dac, @@ -4637,7 +4696,7 @@ level = 750 let names: Vec<&str> = curve.lines.iter().map(|l| l.name.as_str()).collect(); assert_eq!(names, ["configured lobe", "V_null", "V_null + Vπ"]); let lobe = &curve.lines[0].points; - // Minimum at V_null, maximum a quarter wave later. + // Minimum at V_null, maximum one half-wave-voltage span later. let at = |code: f64| { lobe.iter() .min_by(|a, b| (a.x - code).abs().total_cmp(&(b.x - code).abs())) @@ -4657,7 +4716,7 @@ level = 750 plugin.v_pi_dac = 860; plugin.depth_a = 0.5; // must be irrelevant for a constant hold - // I_k = 1 holds exactly at V_null + Vπ (previously rejected because + // u = 1 holds exactly at V_null + Vπ (previously rejected because // the modulated band u_k·e^{a/2} > 1 was demanded even for CONST). plugin.operating_point = 1.0; let (lo, hi, hold) = plugin.dac_band().expect("full-lobe hold"); @@ -4674,6 +4733,58 @@ level = 750 assert!(plugin.dac_band().is_err()); } + #[test] + fn optical_linear_sine_uses_linear_not_logarithmic_headroom() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::OpticalLinearSine; + plugin.v_null_dac = 400; + plugin.v_pi_dac = 900; + plugin.operating_point = 0.4; + plugin.depth_a = 2.0; + + // Linear target: u_hi = 0.4 * (1 + tanh(1)) ≈ 0.705, which fits. + // Reusing log-sine endpoints would incorrectly test + // 0.4 * exp(1) ≈ 1.087 and reject this valid drive. + let (_, hi, _) = plugin.dac_band().expect("linear target fits the lobe"); + let expected = plugin + .lobe_inversion() + .dac_for_u(0.4 * (1.0 + 1.0_f64.tanh())) + .round() as i64; + assert_eq!(hi, expected); + } + + #[test] + fn control_state_publishes_exact_optical_drive_provenance() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::OpticalLogSine; + plugin.v_null_dac = 400; + plugin.v_pi_dac = 900; + plugin.operating_point = 0.4; + plugin.depth_a = 1.0; + plugin.calibration_id = Some("cal-test".into()); + + let state = plugin.control_state(); + let drive = state.optical_drive.expect("optical provenance"); + assert_eq!(drive.target, OpticalTargetV1::LogSine); + assert_eq!(drive.requested_mean_u_milli, 400); + assert_eq!( + drive.internal_u_milli, + (waveform::log_sine_geometric_pedestal(0.4, 1.0) * 1_000.0).round() as u32 + ); + assert_eq!( + drive.resolved_mean_u_milli, + (waveform::log_sine_cycle_mean(f64::from(drive.internal_u_milli) / 1_000.0, 1.0,) + * 1_000.0) + .round() as u32 + ); + assert_eq!(drive.depth_a_milli, 1_000); + assert_eq!(drive.v_null_dac, 400); + assert_eq!(drive.v_pi_dac, 900); + assert_eq!(state.calibration_id.as_deref(), Some("cal-test")); + } + #[test] fn rejected_operating_point_does_not_diverge_from_the_board_target() { let mut plugin = live_plugin(); @@ -4686,14 +4797,14 @@ level = 750 let error = plugin .set_setting("operating_point", json!(1.0)) - .expect_err("periodic I_k=1 has no modulation headroom"); + .expect_err("periodic u=1 has no modulation headroom"); assert!(error.contains("lobe ceiling")); assert_eq!(plugin.operating_point, 0.5); plugin.set_setting("mode", json!(0)).expect("CONST"); plugin .set_setting("operating_point", json!(1.0)) - .expect("CONST maps I_k directly"); + .expect("CONST maps u directly"); assert_eq!(plugin.dac_band().unwrap(), (2_490, 2_490, 2_490)); } @@ -4772,7 +4883,8 @@ level = 750 owner.device_connected() }); plugin.method = DriveMethod::Calibrated; - plugin.mode = Mode::Sine; + plugin.mode = Mode::OpticalLogSine; + plugin.calibration_id = Some("cal-test".into()); plugin.depth_a = 0.4; // what the operator armed let acquire = service_request( @@ -4802,7 +4914,10 @@ level = 750 reply.outcome ); } - assert!((plugin.depth_a - 1.25).abs() < 1e-9, "sweep drives the depth"); + assert!( + (plugin.depth_a - 1.25).abs() < 1e-9, + "sweep drives the depth" + ); plugin.end_lease(); assert!( @@ -4880,7 +4995,8 @@ level = 750 owner.device_connected() }); plugin.method = DriveMethod::Calibrated; - plugin.mode = Mode::Sine; + plugin.mode = Mode::OpticalLogSine; + plugin.calibration_id = Some("cal-test".into()); // Without a lease the retarget is refused. let unleased = service_request( @@ -4917,12 +5033,13 @@ level = 750 }, None, ); - assert!(matches!( - plugin - .handle_service_request(&retarget, &live_execution()) - .outcome, - PluginServiceOutcome::Accepted { .. } - )); + let outcome = plugin + .handle_service_request(&retarget, &live_execution()) + .outcome; + assert!( + matches!(outcome, PluginServiceOutcome::Accepted { .. }), + "retarget outcome: {outcome:?}" + ); assert!((plugin.depth_a - 1.25).abs() < 1e-9); wait_until(&plugin, Duration::from_secs(2), |owner| { owner @@ -4931,7 +5048,7 @@ level = 750 .lock() .unwrap() .board_mod - .starts_with("SINE") + .starts_with("WARP") }); // The manual DAC band cannot express an optical depth. diff --git a/plugins/stage-a-modulation/src/waveform.rs b/plugins/stage-a-modulation/src/waveform.rs index 0485f19..859b7ef 100644 --- a/plugins/stage-a-modulation/src/waveform.rs +++ b/plugins/stage-a-modulation/src/waveform.rs @@ -30,13 +30,45 @@ pub const WARP_TABLE_LEN: usize = 256; /// Full-scale DAC code (12-bit). pub const DAC_FULL_SCALE: u16 = 4_095; +/// Modified Bessel function `I₀(x)` for the Stage-A depth range (`|x| ≤ 3`). +/// +/// The positive power series converges rapidly here and avoids adding a +/// special-functions dependency to the plugin/firmware parameter path. +fn modified_bessel_i0(x: f64) -> f64 { + let y = 0.25 * x * x; + let mut sum = 1.0; + let mut term = 1.0; + for k in 1..=32 { + term *= y / (k as f64 * k as f64); + sum += term; + if term <= f64::EPSILON * sum { + break; + } + } + sum +} + +/// Geometric pedestal that makes a log-sine's cycle-mean normalized lobe +/// coordinate equal `mean_u`: +/// +/// `u(t) = u_g exp[(a/2) sin(ωt)]`, `u_g = mean_u / I₀(a/2)`. +pub fn log_sine_geometric_pedestal(mean_u: f64, depth_a: f64) -> f64 { + mean_u / modified_bessel_i0(0.5 * depth_a) +} + +pub fn log_sine_cycle_mean(pedestal_u: f64, depth_a: f64) -> f64 { + pedestal_u * modified_bessel_i0(0.5 * depth_a) +} + /// Optical intensity target the drive should reproduce, swung around the -/// operating point `u_k`. +/// dimensionless lobe point `u`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OpticalTarget { - /// Recommended A1 log-intensity sine: `ln I = ln I_k + (a/2) sin ωt`. + /// Recommended A1 log-intensity sine in normalized, floor-subtracted lobe + /// coordinate: `ln u = ln u_g + (a/2) sin ωt`. LogSine, - /// Literal linear-intensity sine: `I = I_k (1 + m sin ωt)`, `m = tanh(a/2)`. + /// Literal linear-intensity sine: `u = u_c (1 + m sin ωt)`, + /// `m = tanh(a/2)`. LinearSine, } @@ -45,7 +77,7 @@ pub enum OpticalTarget { pub struct LobeInversion { /// DAC code where the excitation light is at its minimum (`sin² = 0`). pub v_null_dac: f64, - /// DAC-code distance from `v_null` to the excitation maximum (quarter wave). + /// DAC-code half-wave-voltage span from `v_null` to the excitation maximum. pub v_pi_dac: f64, } @@ -69,9 +101,10 @@ pub struct OpticalDrive { pub target: OpticalTarget, /// Optical log-modulation depth `a = ln(I_max / I_min)`, must be positive. pub depth_a: f64, - /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0, 1]`: - /// the geometric-mean point the modulation swings around. Held fixed while - /// `a` is swept, so one response curve keeps `I_k` constant. + /// Dimensionless, floor-subtracted lobe point in `(0, 1]`. It is the + /// geometric pedestal `u_g` for [`OpticalTarget::LogSine`] and the + /// arithmetic centre `u_c` for [`OpticalTarget::LinearSine`]. This is not + /// the physical A1 flux point `I_k`. pub operating_point: f64, pub inversion: LobeInversion, } @@ -84,8 +117,8 @@ pub enum WarpError { InvalidOperatingPoint, /// `Vπ` is not finite or not positive. InvalidInversion, - /// The peak optical target exceeds the lobe ceiling (`u_k · peak > 1`): the - /// operating point is too bright for this depth and would saturate. + /// The peak optical target exceeds the lobe ceiling: the internal + /// pedestal/centre is too bright for this depth and would saturate. Saturates { peak: f64 }, /// A computed DAC code falls outside `0..=4095`: the inversion parameters do /// not fit the requested depth on this lobe. Clamping would silently distort @@ -141,13 +174,13 @@ impl OpticalDrive { } /// Normalised optical target `u(φ)` for phase fraction `φ ∈ [0, 1)`, swung - /// around the operating point `u_k` (not peak-normalised). + /// around the internal `u_g`/`u_c` point (not peak-normalised). pub fn normalised_intensity(&self, phase: f64) -> f64 { let sine = (2.0 * PI * phase).sin(); match self.target { - // ln I = ln I_k + (a/2) sin ωt. + // ln u = ln u_g + (a/2) sin ωt. OpticalTarget::LogSine => self.operating_point * (0.5 * self.depth_a * sine).exp(), - // I = I_k (1 + m sin ωt), m = tanh(a/2). + // u = u_c (1 + m sin ωt), m = tanh(a/2). OpticalTarget::LinearSine => { let m = (0.5 * self.depth_a).tanh(); self.operating_point * (1.0 + m * sine) @@ -206,7 +239,7 @@ mod tests { use super::*; fn inversion() -> LobeInversion { - // Null at code 200, quarter wave 1600 codes later (peak light at 1800). + // Null at code 200, one half-wave-voltage span later at peak light. LobeInversion { v_null_dac: 200.0, v_pi_dac: 1_600.0, @@ -363,19 +396,21 @@ mod tests { } #[test] - fn fixed_operating_point_keeps_i_k_while_sweeping_a() { - // One response curve: fix u_k, vary a. The geometric-mean intensity at - // phase 0 (sin = 0) stays put; only the contrast grows with a. - let u_k = 0.3; + fn fixed_internal_log_pedestal_stays_at_phase_zero_while_sweeping_a() { + // The low-level OpticalDrive takes the geometric pedestal u_g. At + // phase 0 (sin = 0), that pedestal stays put while contrast grows. + // The plugin wrapper adjusts u_g with I₀(a/2) when its requested + // cycle-mean ū is held fixed. + let u_g = 0.3; let drive = |a: f64| OpticalDrive { target: OpticalTarget::LogSine, depth_a: a, - operating_point: u_k, + operating_point: u_g, inversion: inversion(), }; for a in [0.2, 0.6, 1.0] { // At phase 0 the log-sine sits exactly at the operating point. - assert!((drive(a).normalised_intensity(0.0) - u_k).abs() < 1e-12); + assert!((drive(a).normalised_intensity(0.0) - u_g).abs() < 1e-12); let table = drive(a).warp_table().expect("in range"); let intensities: Vec = table .iter() @@ -386,4 +421,23 @@ mod tests { assert!(((max / min).ln() - a).abs() < 0.05, "a={a}"); } } + + #[test] + fn bessel_normalization_keeps_the_log_sine_cycle_mean() { + for depth_a in [0.2, 0.4, 1.0, 2.0, 6.0] { + let mean_u = 0.2; + let pedestal = log_sine_geometric_pedestal(mean_u, depth_a); + let sample_mean = (0..65_536) + .map(|index| { + let phase = 2.0 * PI * index as f64 / 65_536.0; + pedestal * (0.5 * depth_a * phase.sin()).exp() + }) + .sum::() + / 65_536.0; + assert!( + (sample_mean - mean_u).abs() < 1e-12, + "a={depth_a}: mean={sample_mean}" + ); + } + } } diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 1e845eb..772665b 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -699,6 +699,12 @@ pub struct StageAPhotodiodePlugin { port_hint: String, mode: Mode, reference_volts: f64, + /// Stable provenance identifier for the measured full-extinction + /// total-power reading in `reference_volts`. + reference_anchor_id: String, + /// Explicit operator confirmation that `reference_volts` is a measured + /// full-extinction anchor for the current optical configuration. + reference_confirmed: bool, /// Measured dark level in photodiode volts (beam blocked). Applied to both /// the detector samples and the `reference_volts` anchor, so it cancels out /// of the rejected-complement contrast rather than biasing it — its job is @@ -800,6 +806,8 @@ impl Default for StageAPhotodiodePlugin { port_hint: "auto".into(), mode: Mode::Raw, reference_volts: 3.3, + reference_anchor_id: String::new(), + reference_confirmed: false, dark_volts: 0.0, window_s: 10.0, avg_samples: 4, @@ -1252,6 +1260,8 @@ impl StageAPhotodiodePlugin { "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), "reference_volts": self.reference_volts, + "reference_anchor_id": self.reference_anchor_id, + "reference_confirmed": self.reference_confirmed, "integrity": { "resync_bytes": summary.integrity.skipped_bytes, "crc_failures": summary.integrity.crc_failures, @@ -1513,7 +1523,7 @@ impl StageAPhotodiodePlugin { } } - /// Live optical log-contrast `a` from the trailing ring window. + /// Live optical log-contrast `a` from a marker-bounded ring window. /// /// The detector sits behind the PBS reject port and measures the rejected /// complement `I_pd = I_tot - I_exc` — that is a property of the optical @@ -1528,9 +1538,10 @@ impl StageAPhotodiodePlugin { /// `a`, so letting a display toggle change its meaning would silently /// retarget the sweep and write a wrong `measured_a` into every sidecar. /// - /// `None` when there is no valid window or no valid total-power anchor. - fn optical_summary(&self, samples: &VecDeque) -> Option { - self.optical_summary_result(samples).ok() + /// `None` when there is no valid whole-cycle window or no explicitly + /// confirmed total-power anchor. + fn optical_summary(&self, state: &SharedState) -> Option { + self.optical_summary_result(state).ok() } /// [`Self::optical_summary`], keeping the rejection reason so the status @@ -1538,10 +1549,47 @@ impl StageAPhotodiodePlugin { /// showing nothing. fn optical_summary_result( &self, - samples: &VecDeque, + state: &SharedState, ) -> Result { - let start = samples.len().saturating_sub(CONTRAST_WINDOW_SAMPLES); - let window: Vec = samples.iter().skip(start).copied().collect(); + if !self.reference_confirmed || self.reference_anchor_id.trim().is_empty() { + return Err(EstimateError::MissingTotalPowerAnchor); + } + + let ring_end = state.ring_first_index + state.samples.len() as u64; + let markers: Vec = state + .markers + .iter() + .copied() + .filter(|index| *index >= state.ring_first_index && *index <= ring_end) + .collect(); + if markers.len() < 3 { + return Err(EstimateError::IncompleteModulationCycles { + marker_count: markers.len(), + max_samples: CONTRAST_WINDOW_SAMPLES, + }); + } + + // End on the newest complete phase-0 boundary. Start at least two + // complete cycles earlier, then include as many older whole cycles as + // fit in the bounded estimator window. + let end_index = *markers.last().expect("three markers checked"); + let mut start_marker = markers.len() - 3; + if end_index.saturating_sub(markers[start_marker]) as usize > CONTRAST_WINDOW_SAMPLES { + return Err(EstimateError::IncompleteModulationCycles { + marker_count: markers.len(), + max_samples: CONTRAST_WINDOW_SAMPLES, + }); + } + while start_marker > 0 + && end_index.saturating_sub(markers[start_marker - 1]) as usize + <= CONTRAST_WINDOW_SAMPLES + { + start_marker -= 1; + } + let start_index = markers[start_marker]; + let start = start_index.saturating_sub(state.ring_first_index) as usize; + let end = end_index.saturating_sub(state.ring_first_index) as usize; + let window: Vec = state.samples.range(start..end).copied().collect(); let calibration = self.adc_calibration(); // `ContrastGeometry::RejectedComplement` wants the *dark-corrected* // I_tot, and the estimator dark-corrects the detector samples. The @@ -1570,9 +1618,9 @@ impl StageAPhotodiodePlugin { } else { "dark-none".into() }, - anchor_id: "reference-volts".into(), + anchor_id: self.reference_anchor_id.clone(), dark_volts: calibration.dark_volts, - total_power_volts: self.reference_volts, + total_power_volts: self.reference_volts - self.dark_volts, }, measured_log_contrast: estimate.a, log_contrast_stddev: None, @@ -1585,7 +1633,11 @@ impl StageAPhotodiodePlugin { excitation_headroom_volts: estimate.v_min_volts, low_clip_fraction: estimate.low_clip_fraction, high_clip_fraction: estimate.high_clip_fraction, - measured_frequency_hz: None, + measured_frequency_hz: (state.rate_hz > 0).then(|| { + let cycles = markers.len() - 1 - start_marker; + let period_samples = end_index.saturating_sub(start_index) as f64 / cycles as f64; + f64::from(state.rate_hz) / period_samples + }), fundamental_phase_rad: None, total_harmonic_distortion: None, }) @@ -1595,7 +1647,7 @@ impl StageAPhotodiodePlugin { /// keeping the rejection reason so the caller can explain a withheld `a`. fn latest_optical_result(&self) -> Option> { let state = self.shared.lock().ok()?; - (!state.samples.is_empty()).then(|| self.optical_summary_result(&state.samples)) + (!state.samples.is_empty()).then(|| self.optical_summary_result(&state)) } fn control_summary(&self) -> PhotodiodeSummaryV1 { @@ -1606,7 +1658,7 @@ impl StageAPhotodiodePlugin { end_sample_index_exclusive: state.ring_first_index + state.samples.len() as u64, sample_count: state.samples.len() as u64, }); - let optical_summary = self.optical_summary(&state.samples); + let optical_summary = self.optical_summary(&state); let level = self.current_level(&state); let connection = if self.connected() { ConnectionStateV1::Connected { @@ -1827,6 +1879,8 @@ impl StageAPhotodiodePlugin { "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), "reference_volts": self.reference_volts, + "reference_anchor_id": self.reference_anchor_id, + "reference_confirmed": self.reference_confirmed, "dark_volts": self.dark_volts, "time_base": "t_s = sample_index / sample_rate_hz, device clock, segment-relative", "integrity": integrity, @@ -2757,6 +2811,30 @@ impl Plugin for StageAPhotodiodePlugin { default: self.reference_volts, }, }, + SettingItem { + key: "reference_anchor_id".into(), + label: "I_tot anchor id".into(), + tooltip: Some( + "Stable identifier for the measured full-extinction reference \ + (for example the calibration/run id)." + .into(), + ), + kind: SettingKind::Text { + default: self.reference_anchor_id.clone(), + }, + }, + SettingItem { + key: "reference_confirmed".into(), + label: "I_tot measured and current".into(), + tooltip: Some( + "Confirm only after measuring I_tot for the current optical \ + configuration. Changing the value or anchor id clears this." + .into(), + ), + kind: SettingKind::Bool { + default: self.reference_confirmed, + }, + }, SettingItem { key: "dark_volts".into(), label: "Dark level".into(), @@ -2964,6 +3042,8 @@ impl Plugin for StageAPhotodiodePlugin { Some(json!(index)) } "reference_volts" => Some(json!(self.reference_volts)), + "reference_anchor_id" => Some(json!(self.reference_anchor_id)), + "reference_confirmed" => Some(json!(self.reference_confirmed)), "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), "show_markers" => Some(json!(self.show_markers)), @@ -3027,7 +3107,36 @@ impl Plugin for StageAPhotodiodePlugin { } "reference_volts" => { let volts = value.as_f64().ok_or("reference_volts must be a number")?; - self.reference_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); + let volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); + if self.reference_volts != volts { + self.reference_volts = volts; + self.reference_confirmed = false; + } + Ok(()) + } + "reference_anchor_id" => { + let anchor_id = value + .as_str() + .ok_or("reference_anchor_id must be a string")? + .trim() + .to_owned(); + if self.reference_anchor_id != anchor_id { + self.reference_anchor_id = anchor_id; + self.reference_confirmed = false; + } + Ok(()) + } + "reference_confirmed" => { + let confirmed = value + .as_bool() + .ok_or("reference_confirmed must be a boolean")?; + if confirmed && self.reference_anchor_id.trim().is_empty() { + return Err("set a non-empty I_tot anchor id before confirming".into()); + } + if confirmed && self.reference_volts <= self.dark_volts { + return Err("I_tot must be above the measured dark level".into()); + } + self.reference_confirmed = confirmed; Ok(()) } "show_markers" => { @@ -3342,6 +3451,8 @@ mod tests { let mut plugin = StageAPhotodiodePlugin::default(); plugin.set_runtime_role(PluginRuntimeRole::LiveWorker); plugin.effects_allowed = true; + plugin.reference_anchor_id = "test-itot".into(); + plugin.reference_confirmed = true; plugin } @@ -3351,11 +3462,26 @@ mod tests { (0..count) .map(|i| { let phase = 2.0 * std::f64::consts::PI * (i as f64) * 8.0 / count as f64; - (center + amplitude * phase.sin()).round().clamp(0.0, 4_095.0) as u16 + (center + amplitude * phase.sin()) + .round() + .clamp(0.0, 4_095.0) as u16 }) .collect() } + fn state_with_cycles(samples: VecDeque, cycles: usize) -> SharedState { + let count = samples.len(); + let mut state = SharedState { + rate_hz: 20_000, + samples, + ..SharedState::default() + }; + state.markers = (0..=cycles) + .map(|cycle| (cycle * count / cycles) as u64) + .collect(); + state + } + #[test] fn published_contrast_is_the_excitation_contrast_in_both_display_modes() { // The detector sits behind the PBS reject port whatever the operator @@ -3363,18 +3489,18 @@ mod tests { // scientific quantity. A1's amplitude sweep settles on this value. let mut plugin = live_plugin(); plugin.reference_volts = 3.0; - let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + let state = state_with_cycles(rejected_port_samples(1_600.0, 700.0, 4_096), 8); plugin.mode = Mode::Raw; - let raw = plugin.optical_summary(&samples).expect("raw display"); + let raw = plugin.optical_summary(&state).expect("raw display"); plugin.mode = Mode::Excitation; - let excitation = plugin - .optical_summary(&samples) - .expect("excitation display"); + let excitation = plugin.optical_summary(&state).expect("excitation display"); assert_eq!(raw.measured_log_contrast, excitation.measured_log_contrast); - assert_eq!(raw.calibration.anchor_id, "reference-volts"); - assert_eq!(excitation.calibration.anchor_id, "reference-volts"); + assert_eq!(raw.calibration.anchor_id, "test-itot"); + assert_eq!(excitation.calibration.anchor_id, "test-itot"); + assert!((raw.calibration.total_power_volts - 3.0).abs() < 1e-12); + assert!((raw.measured_frequency_hz.expect("marker frequency") - 39.0625).abs() < 1e-12); // And it really is the complement contrast, not ln(v_max/v_min) of the // detector trace. @@ -3386,18 +3512,56 @@ mod tests { ); } + #[test] + fn optical_contrast_requires_a_confirmed_anchor_and_complete_cycles() { + let mut plugin = live_plugin(); + plugin.reference_volts = 3.0; + let mut state = state_with_cycles(rejected_port_samples(1_600.0, 700.0, 4_096), 8); + + plugin.reference_confirmed = false; + assert_eq!( + plugin.optical_summary_result(&state), + Err(EstimateError::MissingTotalPowerAnchor) + ); + + plugin.reference_confirmed = true; + state.markers = VecDeque::from([0, 512]); + assert_eq!( + plugin.optical_summary_result(&state), + Err(EstimateError::IncompleteModulationCycles { + marker_count: 2, + max_samples: CONTRAST_WINDOW_SAMPLES, + }) + ); + } + + #[test] + fn changing_the_total_power_anchor_invalidates_confirmation() { + let mut plugin = live_plugin(); + plugin + .set_setting("reference_volts", json!(2.9)) + .expect("reference"); + assert!(!plugin.reference_confirmed); + + plugin.reference_confirmed = true; + plugin + .set_setting("reference_anchor_id", json!("itot-next")) + .expect("anchor id"); + assert!(!plugin.reference_confirmed); + } + #[test] fn captured_dark_level_reaches_the_estimator_and_is_named() { let mut plugin = live_plugin(); plugin.reference_volts = 3.0; - let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + let state = state_with_cycles(rejected_port_samples(1_600.0, 700.0, 4_096), 8); - let undarkened = plugin.optical_summary(&samples).expect("no dark yet"); + let undarkened = plugin.optical_summary(&state).expect("no dark yet"); assert_eq!(undarkened.calibration.dark_id, "dark-none"); assert_eq!(undarkened.calibration.dark_volts, 0.0); plugin.dark_volts = 0.05; - let darkened = plugin.optical_summary(&samples).expect("with dark"); + let darkened = plugin.optical_summary(&state).expect("with dark"); assert_eq!(darkened.calibration.dark_id, "dark-measured"); assert_eq!(darkened.calibration.dark_volts, 0.05); // A DC dark offset is common to the detector samples and to the @@ -3679,7 +3843,7 @@ mod tests { railed.ingest(0, 20_000, 0, &[4_095; 8]); let clipped = plugin.current_level(&railed).expect("still reports"); assert!(clipped.clipped); - assert!(plugin.optical_summary(&railed.samples).is_none()); + assert!(plugin.optical_summary(&railed).is_none()); } #[test] diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs index 1c87f44..fe2ffe3 100644 --- a/stage-a-io/src/estimator.rs +++ b/stage-a-io/src/estimator.rs @@ -89,6 +89,15 @@ pub struct ContrastEstimate { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum EstimateError { + /// The rejected-complement geometry has no explicitly confirmed, + /// traceable total-power anchor. + MissingTotalPowerAnchor, + /// No marker-bounded window containing at least two complete modulation + /// cycles fits inside the retained sample budget. + IncompleteModulationCycles { + marker_count: usize, + max_samples: usize, + }, /// Fewer samples than the estimator can use robustly. TooFewSamples { count: usize, minimum: usize }, /// The window touches the ADC rails — `a` would be silently wrong. @@ -110,6 +119,17 @@ pub enum EstimateError { impl std::fmt::Display for EstimateError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Self::MissingTotalPowerAnchor => f.write_str( + "no confirmed, named total-power anchor; excitation contrast a is withheld", + ), + Self::IncompleteModulationCycles { + marker_count, + max_samples, + } => write!( + f, + "no marker-bounded window with at least two complete cycles fits in \ + {max_samples} samples ({marker_count} usable markers)" + ), Self::TooFewSamples { count, minimum } => { write!(f, "only {count} samples (minimum {minimum})") } diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index e7e1638..ed05dd3 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -285,11 +285,13 @@ pub enum ModulationCommandV1 { }, /// Retarget the owner's *calibrated optical drive* to a new modulation /// depth `a` (log contrast, in milli-units) without changing anything else - /// about the armed drive: waveform shape, frequency, operating point and - /// calibration stay whatever the operator armed in the modulation plugin. + /// about the armed drive: waveform shape, frequency, requested normalized + /// cycle mean and calibration stay whatever the operator armed in the + /// modulation plugin. /// This is the scoped amplitude-sweep path (A1 automation): the owner /// rejects the command when its current drive cannot express `a` - /// (manual DAC method or constant mode) or the device link is closed. + /// (anything other than calibrated `OPTICAL_LOG_SINE` with an identified + /// transfer calibration) or the device link is closed. SetOpticalDepth { depth_a_milli: u32, }, @@ -337,6 +339,32 @@ pub struct ModulationResponseV1 { pub acknowledged_target: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpticalTargetV1 { + LogSine, + LinearSine, +} + +/// Exact optical-inversion parameters currently resolved by the modulation +/// owner. Additive in V1 so A1 sidecars can reproduce the requested drive +/// without misusing physical flux `I_k` for the normalized lobe coordinate. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OpticalDriveStateV1 { + pub target: OpticalTargetV1, + /// Requested normalized cycle-mean lobe coordinate `ū`. + pub requested_mean_u_milli: u32, + /// Mean reconstructed from the quantized internal wire coordinate. + pub resolved_mean_u_milli: u32, + /// Internal target pedestal/centre sent in the wire's legacy `u_k_milli` + /// field (`u_g` for log-sine, `u_c` for linear-sine). + pub internal_u_milli: u32, + pub depth_a_milli: u32, + pub v_null_dac: u16, + /// Null-to-maximum half-wave-voltage span in DAC codes. + pub v_pi_dac: u16, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ModulationStateV1 { pub contract_version: u16, @@ -358,6 +386,9 @@ pub struct ModulationStateV1 { /// entered the lobe parameters by hand. Additive in V1. #[serde(default)] pub calibration_id: Option, + /// Additive V1 optical-drive provenance. + #[serde(default)] + pub optical_drive: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] @@ -744,6 +775,15 @@ mod tests { valid_for_ms: 500, }, calibration_id: Some("pockels-20260724-120000".into()), + optical_drive: Some(OpticalDriveStateV1 { + target: OpticalTargetV1::LogSine, + requested_mean_u_milli: 400, + resolved_mean_u_milli: 399, + internal_u_milli: 355, + depth_a_milli: 1_000, + v_null_dac: 1_630, + v_pi_dac: 860, + }), }; let encoded = serde_json::to_vec(&snapshot).expect("serializes"); let decoded: ModulationStateV1 = serde_json::from_slice(&encoded).expect("deserializes"); @@ -753,6 +793,24 @@ mod tests { decoded.synchronization, SynchronizationV1::Unsynced { .. } )); + assert_eq!( + decoded + .optical_drive + .as_ref() + .unwrap() + .requested_mean_u_milli, + 400 + ); + assert_eq!(decoded.optical_drive.unwrap().resolved_mean_u_milli, 399); + + let mut legacy = serde_json::to_value(&snapshot).expect("serializes"); + let object = legacy.as_object_mut().expect("state object"); + object.remove("calibration_id"); + object.remove("optical_drive"); + let decoded_legacy: ModulationStateV1 = + serde_json::from_value(legacy).expect("pre-provenance state decodes"); + assert!(decoded_legacy.calibration_id.is_none()); + assert!(decoded_legacy.optical_drive.is_none()); } #[test] @@ -820,8 +878,7 @@ mod tests { #[test] fn additive_v1_fields_decode_from_payloads_that_predate_them() { - // An older owner's stream block carries no `level`, and an older - // modulation state no `calibration_id`. Both must still decode. + // An older owner's stream block carries no `level`. let stream: PhotodiodeStreamV1 = serde_json::from_value(json!({ "stream_epoch": 3, "sample_range": null, From f9b7dde05a6c78fc87cc92fe8b83fb81828f8221 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 29 Jul 2026 14:49:53 +0200 Subject: [PATCH 32/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20configure?= =?UTF-8?q?=20Pockels=20lobes=20by=20observed=20endpoints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../008-stage-a-optical-waveform-inversion.md | 4 +- ...6-stage-a-lobe-endpoints-not-a-distance.md | 121 ++++++ docs/features/README.md | 4 +- docs/features/stage-a-optical-waveform.md | 53 ++- docs/features/stage-a-pockels-calibration.md | 12 +- plugins/stage-a-modulation/README.md | 27 +- plugins/stage-a-modulation/src/calibration.rs | 6 + plugins/stage-a-modulation/src/lib.rs | 376 +++++++++++++----- plugins/stage-a-modulation/src/waveform.rs | 197 ++++++++- 9 files changed, 654 insertions(+), 146 deletions(-) create mode 100644 docs/adr/016-stage-a-lobe-endpoints-not-a-distance.md diff --git a/docs/adr/008-stage-a-optical-waveform-inversion.md b/docs/adr/008-stage-a-optical-waveform-inversion.md index fc599a7..3287fad 100644 --- a/docs/adr/008-stage-a-optical-waveform-inversion.md +++ b/docs/adr/008-stage-a-optical-waveform-inversion.md @@ -22,7 +22,9 @@ capped at 192 bytes, too small to upload a 256-code table inline. 1. **Own the inversion in the modulation plugin.** `waveform.rs` computes a 256-entry DAC warp table from an `OpticalTarget` (`LogSine`/`LinearSine`), the - requested depth `a`, and a `LobeInversion { v_null_dac, v_pi_dac }`. It refuses + requested depth `a`, and a `LobeInversion { v_null_dac, v_pi_dac }` (built from + the two observed endpoint codes since + [ADR 016](016-stage-a-lobe-endpoints-not-a-distance.md)). It refuses (never clamps) a drive whose codes leave `0..4095`. 2. **Keep the inversion parameters settable.** `V_null` and `Vπ` are entered in DAC codes; no measurement rig is required to start. The scientifically clean diff --git a/docs/adr/016-stage-a-lobe-endpoints-not-a-distance.md b/docs/adr/016-stage-a-lobe-endpoints-not-a-distance.md new file mode 100644 index 0000000..498cd47 --- /dev/null +++ b/docs/adr/016-stage-a-lobe-endpoints-not-a-distance.md @@ -0,0 +1,121 @@ +# ADR 016 — The Pockels lobe is two observed codes, not a code and a distance + +- **Status:** Accepted +- **Date:** 2026-07-28 +- **Relates to:** ADR 008 (optical waveform inversion), ADR 011 (measured + Pockels transfer calibration), + [Stage-A Optical Waveform Drive](../features/stage-a-optical-waveform.md), + [Stage-A Pockels Transfer Calibration](../features/stage-a-pockels-calibration.md) + +## Context + +ADR 008 made the lobe settable as `V_null` (a DAC **code**) plus `Vπ` (a DAC +**distance** from it). On the bench on 2026-07-28 the drive behaved backwards: +excitation was brightest at `I_k = 0.5` and returned to the null at both +`I_k = 0` and `I_k = 1`, with the reject-port photodiode reading its maximum at +both ends. + +Nothing was wrong with the arithmetic. Host (`waveform.rs`), firmware +(`stage-a-controller/src/stimulus_mod.cpp`) and the knowledge base +(`methodology/pockels-waveform-linearisation.md` §3) all implement the same +inverse, `V(u) = V_null + (2Vπ/π)·arcsin(√u)`, with maximum light at +`V_null + Vπ`. + +What was wrong was the question the settings pane asked. `V_null (DAC code at +min light)` and `Vπ (DAC codes, null → max light)` render one above the other, +both labelled in DAC codes, and only the second is a distance. The operator +entered the **code** where the light was brightest. With a true null `N`, a true +peak `P` and `Vπ` set to `P`, the realised light is + +``` +I(I_k) = sin²( (P/(P−N)) · arcsin(√I_k) ) +``` + +which peaks at `I_k = sin²((π/2)(1 − N/P))` — one half when `N ≈ P/2` — and +falls back to the null at `I_k = 1`. That reproduces the observed curve exactly, +including the symmetry, and is asserted as a regression witness in +`waveform::tests::the_brightest_code_typed_as_v_pi_is_what_used_to_peak_at_half`. + +A distance is not an observable. Sweeping the DAC yields two *codes* — where the +light is dimmest and where it is brightest — and the pane asked the operator to +subtract them in their head, silently, with no way for the software to check the +result. The failure is silent by construction: any positive `Vπ` produces a +valid-looking drive, so the mistake only shows up as light that does the wrong +thing. + +## Decision + +### 1. The two settings are both absolute codes + +`v_null_dac` (code at minimum light) and `v_peak_dac` (code at maximum light). +`Vπ = |V_peak − V_null|` is derived, never typed. Both fields are read straight +off a sweep or off the transfer-curve plot, so there is nothing to subtract and +nothing to confuse. + +The mis-entry that caused this ADR cannot be expressed in the new form: the code +of the brightest point **is** what `V_peak` asks for. + +### 2. `LobeInversion::resolve` is the single place a pair becomes a lobe + +It returns the ascending lobe the drive inverts, or an error. Two cases beyond +the obvious one: + +- **A pair measured running downward** (`V_peak < V_null`) is now expressible, + where before it simply could not be entered — `Vπ` was constrained positive + and the inverse only ever climbs from `V_null`. `sin²` repeats every `2Vπ`, so + the branch one full period below the observed null rises into the very maximum + that was measured; that branch is used and the status pane says so, because + the codes driven are not the ones that were typed. +- **A degenerate or unreachable pair** is refused with the measurement to redo, + rather than accepted into a drive that cannot be armed. + +### 3. The lobe is resolved against the DAC, the ceiling checks emitted codes + +Where the crystal nulls and peaks is a fact about the bench, so `resolve` bounds +the lobe by the DAC range (`0..=4095`) and not by the operator's `max_level` +safety ceiling. Resolving against the ceiling would have refused a perfectly +drivable `MANUAL` band merely because the lobe it is interpreted against extends +past the ceiling. + +What the ceiling constrains is the codes actually emitted. The four floor/ceiling +guards in `dac_band` collapse to one — the floor cannot be breached now that +every emitted code lies between two in-range endpoints — and it names the +settings that still exist: *"the modulation peak needs DAC code 2600, above the +max limit 2400; raise the max limit or lower I_k / a"*. + +### 4. The wire format and the fit are unchanged + +`MOD wave=WARP … v_null=… v_pi=…` still carries the quarter wave, because that +is what the firmware rebuilds the table from; the derived distance goes on the +wire. `fit_transfer` still reports `v_pi_dac`, because a fitted period *is* a +distance — the endpoint form is about what an operator types, not about how the +model is expressed internally. Applying a fit writes `V_peak = V_null + Vπ`. + +### 5. `v_pi_dac` remains settable, and only settable + +It is absent from `settings_schema` but still accepted by `set_setting`, where +it is converted to `V_peak = V_null + Vπ`. Stored configs keep loading; nothing +new can be authored against the form that caused the mix-up. + +### 6. The status pane states where `I_k` lands + +`Lobe: Vπ = 860 codes — I_k 0 → 1630 (min light), 0.5 → 2060, 1 → 2490 (max +light)`. The parameters are only meaningful as the codes they produce, and a +wrong endpoint is visible there without running a sweep or looking at the light. + +## Consequences + +- `I_k = 1` holds exactly at the measured maximum, by construction rather than + by arithmetic that has to come out right. +- Drive rejection narrows to one honest case: the max-limit ceiling cutting the + requested `I_k`/`a` short. The DAC floor can no longer be breached at all. +- A descending branch is expressible for the first time. +- **Breaking:** `v_pi_dac` no longer appears in the settings schema. Stored + values still load through the compatibility path above, but a saved value that + was *wrong* in the old sense (a code entered as a distance) migrates to an + equally wrong `V_peak` — the bench pair must be re-entered once, or a + calibration sweep re-applied. +- This does not make the calibration self-checking. Nothing yet compares the + applied lobe against the light; the measured sweep of ADR 011 remains the way + to establish the two codes, and this ADR only makes hand-entering them + unambiguous. diff --git a/docs/features/README.md b/docs/features/README.md index 43305e3..fd15983 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -6,8 +6,8 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. - [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. -- [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC so the *optical* output is a log- or linear-intensity sine, inverting the Pockels `sin²` transfer from settable `V_null`/`Vπ`. -- [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`Vπ` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. +- [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC so the *optical* output is a log- or linear-intensity sine, inverting the Pockels `sin²` transfer from a settable `V_null`/`V_peak` pair of observed DAC codes. +- [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). - [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. diff --git a/docs/features/stage-a-optical-waveform.md b/docs/features/stage-a-optical-waveform.md index c0ab98b..3786a05 100644 --- a/docs/features/stage-a-optical-waveform.md +++ b/docs/features/stage-a-optical-waveform.md @@ -2,10 +2,12 @@ - **Crate:** `plugins/stage-a-modulation` (`waveform.rs`) - **Firmware:** `stage-a-controller` — `MOD wave=WARP` (`stimulus_mod::configureWarp`) -- **Status:** Analytic inversion, fed by a measured `V_null`/`Vπ` +- **Status:** Analytic inversion, fed by a measured `V_null`/`V_peak` pair ([Pockels transfer calibration](./stage-a-pockels-calibration.md)); a fully measured LUT remains a documented follow-up -- **ADR:** [ADR 008](../adr/008-stage-a-optical-waveform-inversion.md) +- **ADR:** [ADR 008](../adr/008-stage-a-optical-waveform-inversion.md), + [ADR 016](../adr/016-stage-a-lobe-endpoints-not-a-distance.md) (the lobe is two + observed codes, not a code and a distance) ## Why @@ -37,14 +39,32 @@ maximum exceeds the lobe ceiling. `DAC_SINE` remains the pure-DAC sine. | Setting | Meaning | |---|---| -| `V_null` | DAC code at the excitation minimum (`sin² = 0`) | -| `Vπ` | DAC-code quarter-wave distance from `V_null` to the excitation maximum | +| `V_null` | DAC code at the excitation **minimum** (`sin² = 0`) | +| `V_peak` | DAC code at the excitation **maximum**, on the same lobe | | `a` | requested optical log-modulation depth `ln(I_max/I_min)` | | `I_k` | operating illumination as a normalised lobe intensity `u_k ∈ (0,1]` | -Get `V_null`/`Vπ` from a two-point check (code giving min light, code giving max -light on one lobe) or from nominal `Vπ ÷ driver volts-per-code`. The drive is -refused (never silently clamped) if `V_null + Vπ` overruns `0..4095`. +Both endpoints are **absolute codes you observe** — sweep the DAC and read off +where the light is dimmest and where it is brightest. The quarter wave +`Vπ = |V_peak − V_null|` is derived, never typed: an earlier form asked for +`Vπ` as a *distance* directly beneath `V_null` as a code, and entering the +brightest code there puts maximum light at `I_k ≈ 0.5` with a null back at +`I_k = 1` (ADR 016). `I_k = 1` now holds exactly at `V_peak` by construction. + +The pair is resolved against the **DAC** range, not the `max_level` ceiling — +where the crystal nulls and peaks is a fact about the bench. A ceiling that cuts +the lobe short is reported against the code it actually blocks (*"the modulation +peak needs DAC code 2600, above the max limit 2400"*), not against the lobe. + +A pair entered running downward in code (`V_peak < V_null`) is accepted: the +transfer repeats every `2Vπ`, so the drive uses the ascending branch one period +below, which rises into the same measured maximum, and says so in the status +pane. A degenerate pair, or one whose lobe fits nowhere inside `0..max_level`, +is refused rather than armed. + +The status pane spells the resolved lobe out in codes — +`Lobe: Vπ = 860 codes — I_k 0 → 1630 (min light), 0.5 → 2060, 1 → 2490 (max +light)` — so a wrong endpoint is visible without measuring anything. ### Fixed operating point `I_k`, swept depth `a` @@ -55,7 +75,7 @@ refused (never silently clamped) if `V_null + Vπ` overruns `0..4095`. `CONST` is the exception because it does not modulate: it maps only `I_k` through the inverse lobe and ignores `a`. For example, `V_null=1630`, -`Vπ=860` gives DAC `2490` at `I_k=1` and DAC `1685` at `I_k=0.01`. +`V_peak=2490` gives DAC `2490` at `I_k=1` and DAC `1685` at `I_k=0.01`. Periodic modes still require the headroom above. Invalid setting changes are rejected transactionally, so the UI retains the last applied value instead of showing a target that the board never received. Photodiode RAW/EXCITATION mode @@ -63,7 +83,7 @@ does not participate in this DAC calculation. ### Drive method and hard ceiling -Under `CALIBRATED`, `V_null`/`Vπ`/`I_k`/`a` define the operating band directly. +Under `CALIBRATED`, `V_null`/`V_peak`/`I_k`/`a` define the operating band directly. Under `MANUAL`, the Power + Min-threshold DAC endpoints are passed through the forward `sin²` transfer and converted to the target law's effective `(I_k, a)`; the same inverse-warp implementation then fills that band. @@ -71,7 +91,9 @@ the same inverse-warp implementation then fills that band. Warp codes are absolute lobe codes and cannot be rescaled without distorting the target. The plugin therefore **refuses** any drive whose peak exceeds the always-visible `max_level` hard ceiling. Raise the max limit, or lower the -operating band / `I_k` / `a` / `Vπ`, to fit. +operating band / `I_k` / `a`, to fit. Under `CALIBRATED` the DAC *floor* can no +longer be breached — every emitted code lies between the two endpoints — so only +the ceiling is ever reported. ### Modulation reference range @@ -80,7 +102,7 @@ upper endpoint, constant hold code, and peak-to-peak swing. ### Measured parameters (built) and the measured LUT (still future) -`V_null`/`Vπ` are no longer typed in from a datasheet: the +`V_null`/`V_peak` are no longer typed in from a datasheet: the [Pockels transfer calibration](./stage-a-pockels-calibration.md) sweeps settled constant DAC codes, reads the photodiode level at each, and fits the lobe those two parameters describe. The analytic `sin²` inversion above is unchanged — it is @@ -89,7 +111,7 @@ now fed measured parameters. The fully measured **LUT** remains open: keep the swept `(code → optical level)` table for one monotonic lobe and invert it directly instead of the analytic form, dropping in behind the same `warp_table` interface and superseding -`V_null`/`Vπ` entirely. The calibration record already archives the points such +`V_null`/`V_peak` entirely. The calibration record already archives the points such a table would need. ## Wire form (firmware line limit) @@ -120,6 +142,11 @@ stay in the DAC range, that feeding the warp table back through the `sin²` lobe recovers the intended optical intensity, that the recovered log-contrast matches the requested `a`, that a manual DAC band round-trips through `OpticalDrive::from_dac_band`, and that invalid depth/inversion and lobe overruns -are refused. `cargo test -p stage-a-io mod_warp` covers the mock command surface. +are refused. It also pins the endpoint form: that `I_k` rises monotonically to +the measured maximum for any observed pair, that a pair measured downward folds +onto the branch into the same peak, and — as a regression witness for the bench +report of 2026-07-28 — that entering the brightest *code* where the quarter-wave +*distance* belongs is what peaked the light at `I_k = 0.5`. +`cargo test -p stage-a-io mod_warp` covers the mock command surface. The modulation-plugin tests also pin the full-lobe `CONST` values above and verify that a rejected periodic `I_k` change cannot diverge from the board target. diff --git a/docs/features/stage-a-pockels-calibration.md b/docs/features/stage-a-pockels-calibration.md index d1a9eca..987c782 100644 --- a/docs/features/stage-a-pockels-calibration.md +++ b/docs/features/stage-a-pockels-calibration.md @@ -3,13 +3,15 @@ - **Crate:** `plugins/stage-a-modulation` (`calibration.rs`) - **Depends on:** `stage-a-photodiode` publishing `PhotodiodeStreamV1.level` - **Status:** built -- **ADR:** [ADR 011](../adr/011-stage-a-pockels-transfer-calibration.md) +- **ADR:** [ADR 011](../adr/011-stage-a-pockels-transfer-calibration.md), + [ADR 016](../adr/016-stage-a-lobe-endpoints-not-a-distance.md) (what the two + settings ask for) - **Knowledge base:** `methodology/pockels-waveform-linearisation.md` §4, `setup/optical-path.md` ## Why -`V_null` and `Vπ` drive every calibrated waveform through the optical inversion +`V_null` and `V_peak` drive every calibrated waveform through the optical inversion ([Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md)), but they were two bare number fields whose tooltip said *"measure it; do not trust nominal Vπ"* — with no way to measure it. Nothing in the UI connected a DAC code to an @@ -158,9 +160,11 @@ measured rather than extrapolated by the time a fit exists. A `LineSeriesWindow` host view, `Pockels transfer curve`: -- **before any sweep** — the lobe the *configured* `V_null`/`Vπ` claim, on a - normalised `u` axis, with markers at `V_null` and `V_null + Vπ`. This works +- **before any sweep** — the lobe the *configured* `V_null`/`V_peak` claim, on a + normalised `u` axis, with markers at `V_null` and `V_peak`. This works with no hardware attached and is the answer to "what are these two numbers". + The markers are the settings themselves: both are absolute DAC codes, so the + plot can be read straight back into the two fields (ADR 016). - **after a fit** — `measured ↑`, `measured ↓`, the fitted curve, and (while they differ) the configured lobe on the fit's own scale, in detector volts. diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index bb29cab..0c7210d 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -7,8 +7,11 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** - **Drive method** selects how the DAC operating band is defined: - `MANUAL`: **Power** is the peak/operating code and **Min threshold** is the lower endpoint. - - `CALIBRATED`: `V_null`, `Vπ`, `I_k`, and optical depth `a` determine the endpoints. - Measure `V_null`/`Vπ` with the built-in transfer sweep — see [Calibration](#calibration--measuring-v_null-and-vπ). + - `CALIBRATED`: `V_null`, `V_peak`, `I_k`, and optical depth `a` determine the endpoints. + Both lobe fields are **absolute DAC codes** — where the light is dimmest and where it is + brightest — and the quarter wave `Vπ = |V_peak − V_null|` is derived, never typed + ([ADR 016](../../docs/adr/016-stage-a-lobe-endpoints-not-a-distance.md)). + Measure them with the built-in transfer sweep — see [Calibration](#calibration--measuring-v_null-and-v_peak). - **Mode** independently selects the waveform that fills that band. All five modes are available under both methods. - **Max limit** is always visible and is the hard DAC ceiling for every drive. @@ -17,7 +20,7 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** - The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and a 2 Hz `STATUS` poll), plus the selected method and resolved DAC band. -| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `Vπ` | +| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `V_peak` | |---|---|---| | `CONST` | hold `power` | hold the DAC code for `I_k` | | `DAC_SINE` | DAC sine across the band | DAC sine across the band | @@ -25,16 +28,16 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** | `OPTICAL_LOG_SINE` | optical log-sine across the band | optical log-sine about `I_k` | | `OPTICAL_LINEAR_SINE` | optical linear-sine across the band | optical linear-sine about `I_k` | -Manual optical modes reuse the stored `V_null`/`Vπ` lobe and derive their effective `(I_k, a)` +Manual optical modes reuse the stored `V_null`/`V_peak` lobe and derive their effective `(I_k, a)` from the slider band through the forward optical transfer. In calibrated `CONST`, `a` is irrelevant: the hold is -`V_null + (2Vπ/π)·asin(sqrt(I_k))`. With `V_null=1630` and `Vπ=860`, this is -2490 at `I_k=1` and 1685 at `I_k=0.01`. Periodic modes still need optical +`V_null + (2Vπ/π)·asin(sqrt(I_k))`. With `V_null=1630` and `V_peak=2490`, this is +2490 at `I_k=1` and 1685 at `I_k=0.01` — `I_k=1` lands on `V_peak` by construction. Periodic modes still need optical headroom and reject impossible `I_k`/`a` combinations without changing the displayed setting or leaving it out of sync with the board. -## Calibration — measuring `V_null` and `Vπ` +## Calibration — measuring `V_null` and `V_peak` Do not type these in from a datasheet. Static birefringence, alignment, PBS extinction, driver gain, temperature, and the actual electrical load all enter the realised map, so measure them: @@ -49,17 +52,19 @@ gain, temperature, and the actual electrical load all enter the realised map, so back down (~20 s), and fits the lobe. Your armed drive is restored afterwards, on every exit path. 4. Read the result in the **Pockels transfer curve** view and the status line, then press - **Apply to V_null / Vπ**. Anything questionable — a high residual, dropped points, + **Apply to V_null / Vπ**, which writes both endpoint codes. Anything questionable — a high residual, dropped points, hysteresis, clipping — appears as a `Check:` line but does not block the apply: the plot is the arbiter, and a single stray sample can inflate the residual fivefold while leaving `Vπ` accurate to a few codes. Wild points are dropped from the fit automatically. -The view also works *before* any measurement: it draws the lobe your current `V_null`/`Vπ` claim, -on a normalised axis, with markers at `V_null` and `V_null + Vπ`. +The view also works *before* any measurement: it draws the lobe your current `V_null`/`V_peak` +claim, on a normalised axis, with markers at `V_null` and `V_peak` — the two settings themselves, +so the plot reads straight back into the two fields. The status pane states the same thing in +codes: `Lobe: Vπ = 860 codes — I_k 0 → 1630 (min light), 0.5 → 2060, 1 → 2490 (max light)`. Two properties worth knowing: -- `V_null`/`Vπ` need **no** dark measurement and **no** total-power anchor — the fitted offset and +- `V_null`/`V_peak` need **no** dark measurement and **no** total-power anchor — the fitted offset and amplitude absorb the dark level and the front-end gain. - The detector level at the null is reported as a **lower bound** on the total-power anchor `I_tot`, *not* as the anchor. On the reject port the residual transmitted floor is not separable diff --git a/plugins/stage-a-modulation/src/calibration.rs b/plugins/stage-a-modulation/src/calibration.rs index 9bddcfe..e9f44df 100644 --- a/plugins/stage-a-modulation/src/calibration.rs +++ b/plugins/stage-a-modulation/src/calibration.rs @@ -149,6 +149,12 @@ impl TransferFit { } } + /// DAC code at the excitation maximum — the second of the two codes the + /// drive is configured with. + pub fn v_peak_dac(&self) -> f64 { + self.v_null_dac + self.v_pi_dac + } + /// Detector extremum at the excitation null. On the reject port this is the /// detector *maximum* and a **lower bound** on the total-power anchor /// `I_tot` — not the anchor itself, because the residual transmitted floor diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 563b655..36c7d8a 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -917,8 +917,10 @@ pub struct StageAModulationPlugin { operating_point: f64, /// DAC code at the excitation minimum of one monotonic Pockels lobe. v_null_dac: i64, - /// DAC-code quarter-wave distance from `v_null` to the excitation maximum. - v_pi_dac: i64, + /// DAC code at the excitation maximum of that lobe. An absolute code like + /// `v_null_dac`, not a distance: the quarter wave `Vπ` is derived from the + /// pair (see [`waveform::LobeInversion::resolve`]). + v_peak_dac: i64, // -- measured transfer calibration -- /// Bench detector geometry. Not inferable from a sweep — see /// [`calibration::DetectorGeometry`]. @@ -927,7 +929,7 @@ pub struct StageAModulationPlugin { sweep: Option, /// Last completed fit, awaiting review and an explicit apply. fit: Option, - /// Set once a fit has been applied to `v_null_dac`/`v_pi_dac`; published on + /// Set once a fit has been applied to `v_null_dac`/`v_peak_dac`; published on /// the contract so a consumer's sidecar can cite the inversion in use. calibration_id: Option, /// Directory for the archived calibration record; empty means "apply the @@ -989,7 +991,7 @@ impl Default for StageAModulationPlugin { armed_frequency_hz: None, operating_point: 0.5, v_null_dac: 0, - v_pi_dac: 2_048, + v_peak_dac: 2_048, detector_geometry: calibration::DetectorGeometry::RejectedComplement, sweep: None, fit: None, @@ -1120,11 +1122,25 @@ impl StageAModulationPlugin { self.shared.bump(); } - fn lobe_inversion(&self) -> waveform::LobeInversion { - waveform::LobeInversion { - v_null_dac: self.v_null_dac as f64, - v_pi_dac: self.v_pi_dac as f64, - } + /// The lobe the two configured codes name, or why they name none. + /// + /// Resolved against the **DAC's** range, not the operator's `max_level` + /// ceiling: where the crystal nulls and peaks is a fact about the bench, and + /// a ceiling that cuts the lobe short still leaves the codes below it + /// perfectly drivable (a MANUAL band inside the ceiling must keep working). + /// What the ceiling constrains is the codes actually emitted, which + /// [`Self::dac_band`] checks. + fn resolved_lobe(&self) -> Result { + waveform::LobeInversion::resolve( + self.v_null_dac as f64, + self.v_peak_dac as f64, + MAX_DAC_CODE as f64, + ) + .map_err(|error| error.to_string()) + } + + fn lobe_inversion(&self) -> Result { + self.resolved_lobe().map(|lobe| lobe.inversion) } /// Resolves the selected method into the DAC band used by every waveform. @@ -1140,28 +1156,29 @@ impl StageAModulationPlugin { if !u_k.is_finite() || u_k <= 0.0 || u_k > 1.0 { return Err("operating point must be in (0, 1]".into()); } - let inversion = self.lobe_inversion(); - if !inversion.v_pi_dac.is_finite() || inversion.v_pi_dac <= 0.0 { - return Err("Vπ must be finite and positive".into()); - } - - // Constant hold at I_k modulates nothing: no ±a/2 headroom applies, so - // the full (0, 1] range of u_k is expressible (I_k = 1 holds exactly at - // V_null + Vπ). Requiring the modulated band here silently froze the - // drive at the last accepted code whenever u_k·e^{a/2} exceeded 1. - if self.mode == Mode::Const { - let hold = inversion.dac_for_u(u_k).round() as i64; - if hold < 0 { + let inversion = self.lobe_inversion()?; + + // Every code the inversion emits lies between the two endpoints, and + // `resolve` has already placed both inside the DAC range — so the floor + // can no longer be breached and only the operator's ceiling is left to + // check. One guard, naming the settings that actually exist. + let under_ceiling = |code: i64, what: &str| -> Result { + if code > self.max_level { return Err(format!( - "calibrated hold code {hold} is below 0; re-measure V_null/Vπ" - )); - } - if hold > self.max_level { - return Err(format!( - "calibrated hold {hold} exceeds the max limit {}; raise the max limit or lower I_k / Vπ", + "{what} needs DAC code {code}, above the max limit {}; raise the max limit \ + or lower I_k / a", self.max_level )); } + Ok(code) + }; + + // Constant hold at I_k modulates nothing: no ±a/2 headroom applies, so + // the full (0, 1] range of u_k is expressible (I_k = 1 holds exactly at + // V_peak). Requiring the modulated band here silently froze the drive + // at the last accepted code whenever u_k·e^{a/2} exceeded 1. + if self.mode == Mode::Const { + let hold = under_ceiling(inversion.dac_for_u(u_k).round() as i64, "the constant hold")?; return Ok((hold, hold, hold)); } @@ -1177,25 +1194,20 @@ impl StageAModulationPlugin { } let lo = inversion.dac_for_u(u_lo).round() as i64; - let hi = inversion.dac_for_u(u_hi).round() as i64; + let hi = under_ceiling( + inversion.dac_for_u(u_hi).round() as i64, + "the modulation peak", + )?; let hold = inversion.dac_for_u(u_k).round() as i64; - if lo < 0 { - return Err(format!( - "calibrated lower DAC code {lo} is below 0; re-measure V_null/Vπ" - )); - } - if hi > self.max_level { - return Err(format!( - "calibrated peak {hi} exceeds the max limit {}; raise the max limit or lower a / I_k / Vπ", - self.max_level - )); - } Ok((lo, hi, hold)) } - fn optical_drive(&self, target: waveform::OpticalTarget) -> waveform::OpticalDrive { - let inversion = self.lobe_inversion(); - match self.method { + fn optical_drive( + &self, + target: waveform::OpticalTarget, + ) -> Result { + let inversion = self.lobe_inversion()?; + Ok(match self.method { DriveMethod::Manual => { let hi = self.level.clamp(0, self.max_level); let lo = self.min_level.clamp(0, hi); @@ -1207,7 +1219,7 @@ impl StageAModulationPlugin { operating_point: self.operating_point, inversion, }, - } + }) } /// Builds the single MOD command carrying the complete current drive @@ -1235,7 +1247,11 @@ impl StageAModulationPlugin { // fit on one command line. self.optical_warp_table(target) .map_err(|error| format!("optical drive: {error}"))?; - let drive = self.optical_drive(target); + let drive = self.optical_drive(target)?; + // The firmware rebuilds the table from `v_null` and the *quarter + // wave*, so the derived distance goes on the wire, not the peak + // code the operator configures. + let inversion = drive.inversion; Command::new("MOD") .field("wave", "WARP") .field("freq_mhz", freq_mhz) @@ -1245,8 +1261,8 @@ impl StageAModulationPlugin { "u_k_milli", (drive.operating_point * 1_000.0).round() as i64, ) - .field("v_null", self.v_null_dac) - .field("v_pi", self.v_pi_dac) + .field("v_null", inversion.v_null_dac.round() as i64) + .field("v_pi", inversion.v_pi_dac.round() as i64) } }) } @@ -1305,7 +1321,7 @@ impl StageAModulationPlugin { /// without distorting the target, so an over-limit drive is refused. fn optical_warp_table(&self, target: waveform::OpticalTarget) -> Result, String> { let table = self - .optical_drive(target) + .optical_drive(target)? .warp_table() .map_err(|error| error.to_string())?; let peak = table.iter().copied().max().unwrap_or(0); @@ -1573,14 +1589,16 @@ impl StageAModulationPlugin { self.calibration_status = "nothing to apply: measure a transfer curve first".into(); return; }; - let previous = (self.v_null_dac, self.v_pi_dac); + let previous = (self.v_null_dac, self.v_peak_dac); self.v_null_dac = fit.v_null_dac.round().clamp(0.0, MAX_DAC_CODE as f64) as i64; - self.v_pi_dac = fit.v_pi_dac.round().clamp(1.0, MAX_DAC_CODE as f64) as i64; + // The fit reports the quarter wave; the settings hold the peak code the + // operator can see on the plot. + self.v_peak_dac = fit.v_peak_dac().round().clamp(0.0, MAX_DAC_CODE as f64) as i64; // The applied lobe must still produce a legal drive; a calibration that // cannot be armed is not an improvement. if let Err(error) = self.validate_drive() { self.v_null_dac = previous.0; - self.v_pi_dac = previous.1; + self.v_peak_dac = previous.1; self.calibration_status = format!("not applied: {error}"); return; } @@ -1592,8 +1610,10 @@ impl StageAModulationPlugin { }; self.calibration_id = Some(calibration_id); self.calibration_status = format!( - "applied V_null {} / Vπ {}{archived}", - self.v_null_dac, self.v_pi_dac + "applied V_null {} / V_peak {} (Vπ {} codes){archived}", + self.v_null_dac, + self.v_peak_dac, + self.v_peak_dac - self.v_null_dac ); self.send_modulation(); self.shared.bump(); @@ -2397,18 +2417,15 @@ impl StageAModulationPlugin { }; let Some(fit) = self.fit.as_ref() else { - let inversion = self.lobe_inversion(); - let mut lines = vec![Series1dLine { - name: "configured lobe".into(), - points: sample_curve(1.0, 0.0, inversion), - }]; - lines.push(marker("V_null", inversion.v_null_dac, 0.0, 1.0)); - lines.push(marker( - "V_null + Vπ", - inversion.v_null_dac + inversion.v_pi_dac, - 0.0, - 1.0, - )); + let mut lines = Vec::new(); + if let Ok(inversion) = self.lobe_inversion() { + lines.push(Series1dLine { + name: "configured lobe".into(), + points: sample_curve(1.0, 0.0, inversion), + }); + lines.push(marker("V_null", inversion.v_null_dac, 0.0, 1.0)); + lines.push(marker("V_peak", inversion.v_peak_dac(), 0.0, 1.0)); + } return Series1dV1 { x_label: "DAC code".into(), y_label: "normalised transmission u (not yet measured)".into(), @@ -2444,15 +2461,16 @@ impl StageAModulationPlugin { ]; // The configured lobe on the fit's own scale: after applying they // coincide, and any divergence is the un-applied difference. - let configured = self.lobe_inversion(); - if configured != fit.inversion() { - lines.push(Series1dLine { - name: "configured lobe".into(), - points: sample_curve(fit.span_volts, fit.offset_volts, configured), - }); + if let Ok(configured) = self.lobe_inversion() { + if configured != fit.inversion() { + lines.push(Series1dLine { + name: "configured lobe".into(), + points: sample_curve(fit.span_volts, fit.offset_volts, configured), + }); + } } lines.push(marker("V_null", fit.v_null_dac, lo, hi)); - lines.push(marker("V_null + Vπ", fit.v_null_dac + fit.v_pi_dac, lo, hi)); + lines.push(marker("V_peak", fit.v_peak_dac(), lo, hi)); Series1dV1 { x_label: "DAC code".into(), y_label: "photodiode [V]".into(), @@ -3068,7 +3086,7 @@ impl Plugin for StageAModulationPlugin { DriveMethod::Calibrated => { modulation_items.push(SettingItem { key: "v_null_dac".into(), - label: "V_null (DAC code at min light)".into(), + label: "V_null (DAC code at MIN light)".into(), tooltip: Some( "DAC code where excitation light bottoms out (sin² = 0) on one \ monotonic Pockels lobe. Measure it; do not trust nominal Vπ." @@ -3081,17 +3099,19 @@ impl Plugin for StageAModulationPlugin { }, }); modulation_items.push(SettingItem { - key: "v_pi_dac".into(), - label: "Vπ (DAC codes, null → max light)".into(), + key: "v_peak_dac".into(), + label: "V_peak (DAC code at MAX light)".into(), tooltip: Some( - "DAC-code quarter-wave distance from V_null to the excitation \ - maximum. V_null + Vπ must stay within 0..4095." + "DAC code where excitation light is brightest, on the same lobe as \ + V_null. Both fields are codes you read off a sweep — the quarter \ + wave Vπ = |V_peak − V_null| is derived, never typed. I_k = 1 holds \ + exactly here and I_k = 0 at V_null." .into(), ), kind: SettingKind::I64Drag { - min: 1, + min: 0, max: MAX_DAC_CODE, - default: self.v_pi_dac, + default: self.v_peak_dac, }, }); modulation_items.push(SettingItem { @@ -3300,7 +3320,7 @@ impl Plugin for StageAModulationPlugin { "depth_a" => Some(json!(self.depth_a)), "operating_point" => Some(json!(self.operating_point)), "v_null_dac" => Some(json!(self.v_null_dac)), - "v_pi_dac" => Some(json!(self.v_pi_dac)), + "v_peak_dac" => Some(json!(self.v_peak_dac)), "detector_geometry" => { let index = calibration::DetectorGeometry::VARIANTS .iter() @@ -3483,16 +3503,25 @@ impl Plugin for StageAModulationPlugin { } Ok(()) } - "v_pi_dac" => { - let v_pi_dac = value + // `v_pi_dac` is the pre-endpoint key: a *distance* from V_null. Kept + // settable so a stored config still loads, converted on the way in. + // It is deliberately absent from `settings_schema`, so nothing new + // can be authored against the form that caused the mix-up. + "v_peak_dac" | "v_pi_dac" => { + let entered = value .as_i64() - .ok_or("v_pi_dac must be an integer")? - .clamp(1, MAX_DAC_CODE); - let previous = self.v_pi_dac; - self.v_pi_dac = v_pi_dac; + .ok_or("v_peak_dac must be an integer")? + .clamp(0, MAX_DAC_CODE); + let v_peak_dac = if key == "v_pi_dac" { + (self.v_null_dac + entered).clamp(0, MAX_DAC_CODE) + } else { + entered + }; + let previous = self.v_peak_dac; + self.v_peak_dac = v_peak_dac; if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { if let Err(error) = self.validate_drive() { - self.v_pi_dac = previous; + self.v_peak_dac = previous; return Err(error); } self.send_modulation(); @@ -3605,6 +3634,36 @@ impl Plugin for StageAModulationPlugin { self.method.name(), self.mode.name() ))); + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + // Spell the lobe out in the operator's own units. A wrong endpoint + // shows up here immediately — the code I_k = 1 maps to is the code + // where the light should be brightest, and nothing in between may + // overshoot it. + entries.push(StatusEntry::Text(match self.resolved_lobe() { + Ok(lobe) => { + let inversion = lobe.inversion; + format!( + "Lobe: Vπ = {:.0} codes — I_k 0 → {:.0} (min light), 0.5 → {:.0}, \ + 1 → {:.0} (max light){}", + inversion.v_pi_dac, + inversion.dac_for_u(0.0), + inversion.dac_for_u(0.5), + inversion.dac_for_u(1.0), + if lobe.folded { + format!( + ", folded onto the ascending branch V_null {:.0} → V_peak {:.0} \ + (the pair was entered running downward in code)", + inversion.v_null_dac, + inversion.v_peak_dac() + ) + } else { + String::new() + } + ) + } + Err(error) => format!("Lobe invalid: {error}"), + })); + } match self.dac_band() { Ok((lo, hi, hold)) => entries.push(StatusEntry::Text(format!( "Resolved DAC band: {lo}..{hi} (hold {hold}, {} codes peak-to-peak)", @@ -3615,20 +3674,19 @@ impl Plugin for StageAModulationPlugin { ))), } if let Some(target) = self.mode.optical_target() { - match self.optical_warp_table(target) { - Ok(_) => { - let drive = self.optical_drive(target); + match (self.optical_warp_table(target), self.optical_drive(target)) { + (Ok(_), Ok(drive)) => { entries.push(StatusEntry::Text(format!( - "{}: a={:.2}, I_k={:.2}, V_null={}, Vπ={} @ {:.3} Hz", + "{}: a={:.2}, I_k={:.2}, V_null={:.0}, V_peak={:.0} @ {:.3} Hz", self.mode.name(), drive.depth_a, drive.operating_point, - self.v_null_dac, - self.v_pi_dac, + drive.inversion.v_null_dac, + drive.inversion.v_peak_dac(), self.frequency_hz, ))); } - Err(error) => { + (Err(error), _) | (_, Err(error)) => { entries.push(StatusEntry::Text(format!("Optical drive invalid: {error}"))) } } @@ -4080,7 +4138,7 @@ level = 750 assert!(!manual.iter().any(|key| key == "depth_a")); assert!(!manual.iter().any(|key| key == "operating_point")); assert!(!manual.iter().any(|key| key == "v_null_dac")); - assert!(!manual.iter().any(|key| key == "v_pi_dac")); + assert!(!manual.iter().any(|key| key == "v_peak_dac")); let schema = plugin.settings_schema(); let mode = schema.sections[0] @@ -4118,7 +4176,7 @@ level = 750 assert!(calibrated.iter().any(|key| key == "depth_a")); assert!(calibrated.iter().any(|key| key == "operating_point")); assert!(calibrated.iter().any(|key| key == "v_null_dac")); - assert!(calibrated.iter().any(|key| key == "v_pi_dac")); + assert!(calibrated.iter().any(|key| key == "v_peak_dac")); } #[test] @@ -4133,10 +4191,10 @@ level = 750 // pure hold since the full-lobe fix). plugin.mode = Mode::Sine; plugin.v_null_dac = 200; - plugin.v_pi_dac = 1_600; + plugin.v_peak_dac = 1_800; plugin.operating_point = 0.4; plugin.depth_a = 0.8; - let inversion = plugin.lobe_inversion(); + let inversion = plugin.lobe_inversion().expect("a real lobe"); let expected_lo = inversion .dac_for_u(plugin.operating_point * (-0.5 * plugin.depth_a).exp()) .round() as i64; @@ -4149,18 +4207,26 @@ level = 750 (expected_lo, expected_hi, expected_hold) ); + // The ceiling still bites the emitted codes — but the lobe itself stays + // valid, so a MANUAL band under the ceiling keeps working. plugin.max_level = expected_hi - 1; assert!(plugin .dac_band() .unwrap_err() - .contains("exceeds the max limit")); + .contains("above the max limit")); + plugin.method = DriveMethod::Manual; + plugin.level = plugin.max_level; + assert!( + plugin.dac_band().is_ok(), + "a manual band inside the ceiling" + ); } #[test] fn manual_optical_drive_is_derived_from_the_slider_band() { let mut plugin = live_plugin(); plugin.v_null_dac = 200; - plugin.v_pi_dac = 1_600; + plugin.v_peak_dac = 1_800; plugin.min_level = 600; plugin.level = 1_500; plugin.depth_a = 5.0; @@ -4170,7 +4236,7 @@ level = 750 waveform::OpticalTarget::LogSine, waveform::OpticalTarget::LinearSine, ] { - let drive = plugin.optical_drive(target); + let drive = plugin.optical_drive(target).expect("a real lobe"); let table = plugin .optical_warp_table(target) .expect("valid manual band"); @@ -4196,7 +4262,7 @@ level = 750 plugin.min_level = 600; plugin.level = 1_500; plugin.v_null_dac = 200; - plugin.v_pi_dac = 1_600; + plugin.v_peak_dac = 1_800; plugin.operating_point = 0.4; plugin.depth_a = 0.8; @@ -4355,7 +4421,7 @@ level = 750 ); plugin.set_setting("calibrate_apply", json!(true)).unwrap(); assert_eq!(plugin.v_null_dac, 300); - assert!((plugin.v_pi_dac - 1_600).abs() <= 10); + assert!((plugin.v_peak_dac - plugin.v_null_dac - 1_600).abs() <= 10); assert!(plugin.calibration_id.is_some()); assert!(plugin.control_state().calibration_id.is_some()); } @@ -4398,7 +4464,7 @@ level = 750 "min_level", "mode", "v_null_dac", - "v_pi_dac", + "v_peak_dac", ] { let value = plugin.get_setting(key).expect("exported"); plugin.set_setting(key, value).expect("re-applies"); @@ -4598,9 +4664,9 @@ level = 750 plugin.calibration_status ); assert!( - (plugin.v_pi_dac - 860).abs() <= 10, + (plugin.v_peak_dac - plugin.v_null_dac - 860).abs() <= 10, "Vpi {}", - plugin.v_pi_dac + plugin.v_peak_dac - plugin.v_null_dac ); } @@ -4698,12 +4764,12 @@ level = 750 fn the_curve_view_shows_the_configured_lobe_before_any_measurement() { let mut plugin = live_plugin(); plugin.set_setting("v_null_dac", json!(400)).unwrap(); - plugin.set_setting("v_pi_dac", json!(900)).unwrap(); + plugin.set_setting("v_peak_dac", json!(1_300)).unwrap(); let curve = plugin.curve_dataset(); // Normalised until something has actually been measured. assert!(curve.y_label.contains("normalised")); let names: Vec<&str> = curve.lines.iter().map(|l| l.name.as_str()).collect(); - assert_eq!(names, ["configured lobe", "V_null", "V_null + Vπ"]); + assert_eq!(names, ["configured lobe", "V_null", "V_peak"]); let lobe = &curve.lines[0].points; // Minimum at V_null, maximum a quarter wave later. let at = |code: f64| { @@ -4716,13 +4782,103 @@ level = 750 assert!(at(1_300.0) > 0.99, "u at V_null+Vπ = {}", at(1_300.0)); } + /// The bench failure of 2026-07-28: brightest light at `I_k = 0.5` and a + /// null at `I_k = 1`, because the *code* of the maximum was entered where a + /// distance from `V_null` was expected. With both endpoints being codes, + /// `I_k` cannot turn over — `I_k = 1` lands on the measured maximum. + #[test] + fn i_k_rises_all_the_way_to_the_measured_maximum() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Const; + // Codes read off the bench: dimmest at 1600, brightest at 3200. + plugin.v_null_dac = 1_600; + plugin.v_peak_dac = 3_200; + + let truth = waveform::LobeInversion { + v_null_dac: 1_600.0, + v_pi_dac: 1_600.0, + }; + let mut previous = f64::MIN; + for step in 1..=100 { + plugin.operating_point = f64::from(step) / 100.0; + let (_, _, hold) = plugin.dac_band().expect("every I_k is drivable"); + let light = truth.u_for_dac(hold as f64); + assert!( + light >= previous - 1e-6, + "light turned over at I_k = {}: {light}", + plugin.operating_point + ); + previous = light; + } + assert!(previous > 0.999, "I_k = 1 is not the maximum: {previous}"); + plugin.operating_point = 1.0; + assert_eq!(plugin.dac_band().unwrap().2, 3_200); + } + + #[test] + fn a_stored_quarter_wave_migrates_to_the_peak_code() { + // Configs written before the endpoint form hold `v_pi_dac`, a distance. + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.v_null_dac = 1_630; + plugin + .set_setting("v_pi_dac", json!(860)) + .expect("migrates"); + assert_eq!(plugin.v_peak_dac, 2_490); + // And it is gone from the schema, so nothing new is authored against it. + let keys: Vec = plugin + .settings_schema() + .sections + .iter() + .flat_map(|section| section.items.iter().map(|item| item.key.clone())) + .collect(); + assert!(!keys.iter().any(|key| key == "v_pi_dac")); + } + + #[test] + fn the_status_pane_spells_out_where_i_k_lands() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.v_null_dac = 1_630; + plugin.v_peak_dac = 2_490; + let status = plugin + .status_entries() + .iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) => Some(text.clone()), + _ => None, + }) + .collect::>() + .join(" | "); + assert!(status.contains("Vπ = 860 codes"), "{status}"); + assert!(status.contains("0 → 1630"), "{status}"); + assert!(status.contains("1 → 2490"), "{status}"); + + // A pair entered running downward is reported as folded, not silently + // driven on a branch the operator did not name. + plugin.v_null_dac = 3_000; + plugin.v_peak_dac = 2_000; + let status = plugin + .status_entries() + .iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) => Some(text.clone()), + _ => None, + }) + .collect::>() + .join(" | "); + assert!(status.contains("folded"), "{status}"); + assert!(status.contains("1 → 2000"), "{status}"); + } + #[test] fn calibrated_const_hold_spans_the_full_lobe_without_a_headroom() { let mut plugin = live_plugin(); plugin.method = DriveMethod::Calibrated; plugin.mode = Mode::Const; plugin.v_null_dac = 1_630; - plugin.v_pi_dac = 860; + plugin.v_peak_dac = 2_490; plugin.depth_a = 0.5; // must be irrelevant for a constant hold // I_k = 1 holds exactly at V_null + Vπ (previously rejected because @@ -4748,7 +4904,7 @@ level = 750 plugin.method = DriveMethod::Calibrated; plugin.mode = Mode::Sine; plugin.v_null_dac = 1_630; - plugin.v_pi_dac = 860; + plugin.v_peak_dac = 2_490; plugin.depth_a = 0.5; plugin.operating_point = 0.5; @@ -4777,7 +4933,7 @@ level = 750 plugin.method = DriveMethod::Calibrated; plugin.mode = Mode::Const; plugin.v_null_dac = 1_630; - plugin.v_pi_dac = 860; + plugin.v_peak_dac = 2_490; plugin .set_setting("operating_point", json!(1.0)) diff --git a/plugins/stage-a-modulation/src/waveform.rs b/plugins/stage-a-modulation/src/waveform.rs index 0485f19..54eb7c2 100644 --- a/plugins/stage-a-modulation/src/waveform.rs +++ b/plugins/stage-a-modulation/src/waveform.rs @@ -17,11 +17,13 @@ //! input because the event camera responds to changes in `ln I`. //! - [`OpticalTarget::LinearSine`] — `I_d = I_c (1 + m sin ωt)`, `m = tanh(a/2)`. //! -//! The inversion parameters `V_null` and `Vπ` are expressed in **DAC codes** and -//! are settable: the engineer should not rely on nominal `Vπ` but sweep settled -//! constant DAC codes, measure the actual optical transfer, and enter the frozen -//! `V_null` / `Vπ` of one monotonic lobe. A fully measured lookup table can -//! replace this analytic inversion later behind the same interface. +//! The lobe is configured as the two **DAC codes an operator can observe** — +//! where the light is dimmest (`V_null`) and where it is brightest (`V_peak`) — +//! and `Vπ` is derived from the pair by [`LobeInversion::resolve`]. The engineer +//! should not rely on nominal `Vπ` but sweep settled constant DAC codes, measure +//! the actual optical transfer, and freeze those two codes. A fully measured +//! lookup table can replace this analytic inversion later behind the same +//! interface. use std::f64::consts::PI; @@ -41,6 +43,10 @@ pub enum OpticalTarget { } /// Frozen inversion of one monotonic Pockels/PBS lobe, in DAC codes. +/// +/// Built from the two codes an operator can actually observe on the bench via +/// [`LobeInversion::resolve`], never from a typed-in distance — see the error +/// type for why. #[derive(Debug, Clone, Copy, PartialEq)] pub struct LobeInversion { /// DAC code where the excitation light is at its minimum (`sin² = 0`). @@ -49,7 +55,110 @@ pub struct LobeInversion { pub v_pi_dac: f64, } +/// One monotonic lobe resolved from a measured `(min, max)` pair of codes. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ResolvedLobe { + pub inversion: LobeInversion, + /// The observed pair ran *downward* in code, so the drive uses the + /// equivalent ascending branch — the one that rises into the very maximum + /// that was measured. Worth reporting: the codes driven are not the ones + /// the operator typed. + pub folded: bool, +} + +/// Why two observed codes do not name a drivable lobe. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum LobeError { + /// The two codes coincide: no measurable lobe, so nothing to invert. + Degenerate { code: f64 }, + /// Neither the observed branch nor its ascending equivalent fits inside + /// `0..=max_code`. + Unreachable { + v_null: f64, + v_peak: f64, + max_code: f64, + }, +} + +impl std::fmt::Display for LobeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Degenerate { code } => write!( + f, + "V_null and V_peak are both {code:.0}: sweep the DAC and read off the codes where \ + the light is dimmest and brightest" + ), + Self::Unreachable { + v_null, + v_peak, + max_code, + } => write!( + f, + "no monotonic lobe between V_null {v_null:.0} and V_peak {v_peak:.0} fits inside \ + 0..={max_code:.0}; raise the max limit or pick a lobe further down the range" + ), + } + } +} + +impl std::error::Error for LobeError {} + impl LobeInversion { + /// Resolves the two codes an operator can *observe* — the DAC code at the + /// excitation minimum and the one at the excitation maximum — into the + /// ascending lobe the drive inverts. + /// + /// Both inputs are absolute codes, deliberately. The earlier form paired an + /// absolute `V_null` with `Vπ` as a *distance* from it, and a distance is + /// not what an operator reads off a sweep: entering the brightest **code** + /// as `Vπ` doubles the half wave whenever the null sits near half the peak + /// code, which puts maximum light at `I_k ≈ 0.5` and a null back at + /// `I_k = 1`. Two observed codes cannot be mixed up that way, and they make + /// `I_k = 1` land exactly on the measured maximum by construction. + pub fn resolve(v_null: f64, v_peak: f64, max_code: f64) -> Result { + if !v_null.is_finite() || !v_peak.is_finite() { + return Err(LobeError::Degenerate { code: v_null }); + } + let span = v_peak - v_null; + // Sub-code separation is meaningless on a 12-bit DAC. + if span.abs() < 1.0 { + return Err(LobeError::Degenerate { code: v_null }); + } + let v_pi_dac = span.abs(); + let fits = |null: f64| null >= -0.5 && null + v_pi_dac <= max_code + 0.5; + // `sin²` repeats every `2Vπ` and every branch is a mirror of its + // neighbour, so a pair measured running downward in code names the same + // physical lobe as the ascending branch one full period below — which + // ends on the maximum that was actually measured. Prefer that one; fall + // back to the branch rising out of the observed null only if it is what + // fits inside the commandable range. + let (v_null_dac, folded) = if span > 0.0 && fits(v_null) { + (v_null, false) + } else if fits(v_peak - v_pi_dac) { + (v_peak - v_pi_dac, true) + } else if fits(v_null) { + (v_null, true) + } else { + return Err(LobeError::Unreachable { + v_null, + v_peak, + max_code, + }); + }; + Ok(ResolvedLobe { + inversion: Self { + v_null_dac: v_null_dac.clamp(0.0, (max_code - v_pi_dac).max(0.0)), + v_pi_dac, + }, + folded, + }) + } + + /// DAC code at the excitation maximum: where `I_k = 1` lands. + pub fn v_peak_dac(&self) -> f64 { + self.v_null_dac + self.v_pi_dac + } + /// Normalised optical intensity produced by `code` on the configured lobe: /// `u = sin²(π(code - V_null) / (2 Vπ))`. pub fn u_for_dac(&self, code: f64) -> f64 { @@ -231,6 +340,84 @@ mod tests { } } + #[test] + fn two_observed_codes_put_the_light_maximum_at_i_k_one() { + // The property the endpoint form exists to guarantee: whatever pair of + // codes was measured, I_k = 1 lands on the measured maximum, I_k = 0 on + // the measured minimum, and nothing turns over in between. + for (null, peak) in [(200.0, 1_800.0), (0.0, 4_095.0), (1_600.0, 3_200.0)] { + let lobe = LobeInversion::resolve(null, peak, 4_095.0).expect("a real lobe"); + assert!(!lobe.folded); + let inversion = lobe.inversion; + assert!((inversion.dac_for_u(1.0) - peak).abs() < 1e-9); + assert!((inversion.dac_for_u(0.0) - null).abs() < 1e-9); + assert!((inversion.u_for_dac(peak) - 1.0).abs() < 1e-9); + let mut previous = f64::MIN; + for step in 0..=100 { + let u = f64::from(step) / 100.0; + let light = inversion.u_for_dac(inversion.dac_for_u(u)); + assert!(light >= previous - 1e-9, "light turned over at u = {u}"); + previous = light; + } + } + } + + #[test] + fn the_brightest_code_typed_as_v_pi_is_what_used_to_peak_at_half() { + // Regression witness for the bench report of 2026-07-28. With the null + // at half the brightest code, feeding the *absolute* brightest code in + // as the quarter-wave distance peaks the light at I_k = 0.5 and returns + // it to the null at I_k = 1 — exactly what was observed. + let (null, peak) = (1_600.0, 3_200.0); + let truth = LobeInversion::resolve(null, peak, 4_095.0) + .expect("a real lobe") + .inversion; + let mistake = LobeInversion { + v_null_dac: null, + v_pi_dac: peak, // the distance field filled with a code + }; + let light = |u: f64| truth.u_for_dac(mistake.dac_for_u(u)); + assert!(light(0.5) > 0.99, "peak light at I_k = 0.5: {}", light(0.5)); + assert!(light(1.0) < 0.01, "null light at I_k = 1: {}", light(1.0)); + // And the endpoint form is immune to the same typo, because there is no + // distance to type: the brightest code *is* the field. + assert!((truth.u_for_dac(truth.dac_for_u(1.0)) - 1.0).abs() < 1e-9); + } + + #[test] + fn a_pair_measured_downward_folds_onto_the_branch_into_the_same_peak() { + // Peak below null: the same physical lobe, approached from below. The + // ascending equivalent must end on the measured maximum. + let lobe = LobeInversion::resolve(3_000.0, 2_000.0, 4_095.0).expect("a real lobe"); + assert!(lobe.folded); + assert_eq!(lobe.inversion.v_pi_dac, 1_000.0); + assert!((lobe.inversion.v_peak_dac() - 2_000.0).abs() < 1e-9); + assert!(lobe.inversion.v_null_dac >= 0.0); + } + + #[test] + fn a_downward_pair_with_no_room_below_rises_out_of_the_observed_null() { + // 500 → 100 would fold to a null at −300; the branch above the observed + // null is the one that fits. + let lobe = LobeInversion::resolve(500.0, 100.0, 4_095.0).expect("a real lobe"); + assert!(lobe.folded); + assert_eq!(lobe.inversion.v_null_dac, 500.0); + assert_eq!(lobe.inversion.v_peak_dac(), 900.0); + } + + #[test] + fn refuses_a_degenerate_or_unreachable_pair() { + assert!(matches!( + LobeInversion::resolve(1_000.0, 1_000.0, 4_095.0), + Err(LobeError::Degenerate { .. }) + )); + // A lobe wider than the commandable range fits nowhere. + assert!(matches!( + LobeInversion::resolve(0.0, 3_000.0, 2_000.0), + Err(LobeError::Unreachable { .. }) + )); + } + #[test] fn tables_stay_inside_the_dac_range_for_both_targets() { for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { From e1ca7cfa88b6842ef152f2c40c6f3fba315d3e0e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 29 Jul 2026 14:50:07 +0200 Subject: [PATCH 33/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20calibrate?= =?UTF-8?q?=20millivolt-scale=20photodiode=20signals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...11-stage-a-pockels-transfer-calibration.md | 23 +++-- docs/features/stage-a-pockels-calibration.md | 18 +++- plugins/stage-a-modulation/src/calibration.rs | 98 ++++++++++++++++--- 3 files changed, 113 insertions(+), 26 deletions(-) diff --git a/docs/adr/011-stage-a-pockels-transfer-calibration.md b/docs/adr/011-stage-a-pockels-transfer-calibration.md index de1d7e7..de95420 100644 --- a/docs/adr/011-stage-a-pockels-transfer-calibration.md +++ b/docs/adr/011-stage-a-pockels-transfer-calibration.md @@ -107,13 +107,13 @@ The first cut refused to apply a fit whose residual exceeded 2 % of the detector span. On the bench that gate fired at 20.8 % on a sweep whose plot looked correct, and withheld a usable calibration. -Measuring the failure modes on a realistic small-signal sweep settled it: 5 mV -of noise gives 3.1 %, drift 3.2 %, hysteresis 5.5 % — but a **single stray -point gives 9.9 % while leaving `Vπ` accurate to three codes**. Residual and -correctness are not the same axis, so a residual threshold is the wrong thing to -block on. (A genuinely wrong fit — an amplifier compressing the top of the -range — gives 15.2 % *and* a `Vπ` off by 250 codes, which the plot shows -plainly.) +Synthetic failure-mode sweeps show the distinction: 5 mV of injected noise +gives 3.1 %, drift 3.2 %, hysteresis 5.5 % — but a **single stray point gives +9.9 % while leaving `Vπ` accurate to three codes**. These are test-model +outputs, not bench measurements. Residual and correctness are not the same +axis, so a residual threshold is the wrong thing to block on. (A compressed +synthetic waveform gives 15.2 % *and* a `Vπ` off by 250 codes, which the plot +shows plainly.) Two changes follow. The fit now runs twice, dropping points beyond `6 × median` absolute residual before refitting — a median cut, because mean and standard @@ -126,6 +126,15 @@ Applying still re-validates the resulting drive and rolls back if it cannot be armed. The sweep restores the pre-sweep drive on every exit path, and refuses to run while a lease or protocol owns the DAC. +The original fit also rejected every detector swing below a fixed 10 mV. That +is incompatible with the observed Stage-A operating range of roughly +0.5–15 mV and confuses small absolute scale with absence of information. The +absolute threshold is removed. The only signal gate is now relative: the +between-code sweep span and the fitted lobe span must exceed the median raw +peak-to-peak excursion measured inside the settled CONST windows. This accepts +repeatable millivolt-scale transfers while still refusing structure that is no +larger than the acquisition noise witness. + ### 8. `ModulationStateV1.calibration_id` — one additive V1 field Set when a measured fit is applied, `None` when the lobe was typed in by hand, diff --git a/docs/features/stage-a-pockels-calibration.md b/docs/features/stage-a-pockels-calibration.md index 142d694..d4e9e33 100644 --- a/docs/features/stage-a-pockels-calibration.md +++ b/docs/features/stage-a-pockels-calibration.md @@ -120,8 +120,8 @@ deviation, because those are themselves dragged out by the very points being looked for. `6 × median` is roughly 4σ for Gaussian noise, so ordinary scatter survives untouched. -This matters because of how the numbers actually behave on a bench. Measured on -a realistic small-signal sweep (90 mV span, `Vπ = 860`, 2.4 lobes): +This matters because of how the numbers behave in synthetic stress tests. The +following table is test-model output, not a bench measurement: | Condition | Residual | Fitted `Vπ` | |---|---|---| @@ -139,6 +139,15 @@ bad reason. A residual that stays high after rejection, with a visibly poor overlay, is the real signal — and as the last row shows, it comes with a `Vπ` that is wrong in a way the plot makes obvious. +There is deliberately no absolute minimum voltage. The earlier implementation +rejected every detector span below **10 mV**, while the real Stage-A +photodiode commonly reads only about **0.5–15 mV**. The fit now compares the +between-code sweep span with the median `peak_to_peak_volts` measured inside +the settled CONST windows. A repeatable millivolt-scale lobe is accepted; a +putative lobe no larger than the detector's own typical within-window +excursion is rejected as unresolved. The regression suite includes a 4 mV +transfer that the old threshold always refused. + The fit is **never** applied automatically, and applying re-validates the resulting drive: a calibration that cannot be armed is rolled back rather than stored. Warnings surface as `Check:` lines in the status: @@ -197,8 +206,9 @@ host does take from the worker. - `calibration.rs` unit tests recover a known lobe from **both** ports, across a multi-lobe sweep, and with a null at code 0; they check the geometry input - selects between the two equivalent representations, and that flat sweeps, - short sweeps, and out-of-range lobes are refused. + selects between the two equivalent representations, accept a resolved + sub-10-mV transfer, and ensure flat/noise-level sweeps, short sweeps, and + out-of-range lobes are refused. - An end-to-end test runs the sweep against the mock board, synthesizing the light the reject-port detector *would* report for whatever code the board is actually holding — ground truth for commanding, settle gating, point diff --git a/plugins/stage-a-modulation/src/calibration.rs b/plugins/stage-a-modulation/src/calibration.rs index 900ffd1..890a924 100644 --- a/plugins/stage-a-modulation/src/calibration.rs +++ b/plugins/stage-a-modulation/src/calibration.rs @@ -167,9 +167,12 @@ impl TransferFit { pub enum FitError { /// Fewer points than parameters can be resolved from. TooFewPoints { count: usize, minimum: usize }, - /// The detector never moved: no lobe to fit (light blocked, no drive - /// reaching the cell, or the sweep span sits in a flat region). - NoModulation, + /// The between-code signal span is not larger than the detector's typical + /// within-window excursion, so the sweep does not resolve a lobe. + NoModulation { + signal_span_volts: f64, + noise_span_volts: f64, + }, /// A fitted lobe exists but no `[V_null, V_null+Vπ]` fits inside the /// commandable range, so no monotonic branch is usable. NoLobeInRange { v_pi_dac: f64 }, @@ -181,9 +184,14 @@ impl std::fmt::Display for FitError { Self::TooFewPoints { count, minimum } => { write!(f, "only {count} sweep points (minimum {minimum})") } - Self::NoModulation => f.write_str( - "the detector level did not change across the sweep; check the light path, \ - the HV amplifier, and that the photodiode is connected", + Self::NoModulation { + signal_span_volts, + noise_span_volts, + } => write!( + f, + "detector sweep span {signal_span_volts:.6} V does not exceed the typical \ + within-window excursion {noise_span_volts:.6} V; check the light path and HV \ + amplifier, or reduce detector noise / increase averaging" ), Self::NoLobeInRange { v_pi_dac } => write!( f, @@ -198,8 +206,24 @@ impl std::error::Error for FitError {} /// Smallest usable sweep: four points per fitted parameter. pub const MIN_POINTS: usize = 16; -/// A detector span below this is treated as noise rather than a lobe. -const MIN_SPAN_VOLTS: f64 = 0.01; + +/// Median raw peak-to-peak excursion inside one settled CONST window. +/// +/// This is the scale a between-code transfer curve has to beat. Unlike the +/// former absolute 10 mV cut, it follows the detector gain and acquisition +/// noise, so millivolt-scale but repeatable Pockels sweeps remain usable. +fn typical_window_noise(points: &[SweepPoint]) -> f64 { + let mut spans: Vec = points + .iter() + .map(|point| point.peak_to_peak_volts) + .filter(|span| span.is_finite() && *span >= 0.0) + .collect(); + if spans.is_empty() { + return 0.0; + } + spans.sort_by(f64::total_cmp); + spans[spans.len() / 2] +} /// Least-squares solution for one candidate half-wave-voltage span `w`. struct Harmonic { @@ -461,8 +485,16 @@ pub fn fit_transfer( let profile = smoothed_profile(points); let min_volts = profile.iter().map(|(_, v)| *v).fold(f64::MAX, f64::min); let max_volts = profile.iter().map(|(_, v)| *v).fold(f64::MIN, f64::max); - if max_volts - min_volts < MIN_SPAN_VOLTS { - return Err(FitError::NoModulation); + let observed_span = max_volts - min_volts; + let noise_span = typical_window_noise(points); + if !observed_span.is_finite() + || observed_span <= f64::EPSILON + || (noise_span > 0.0 && observed_span <= noise_span) + { + return Err(FitError::NoModulation { + signal_span_volts: observed_span.max(0.0), + noise_span_volts: noise_span, + }); } let swept_lo = profile.first().map(|(code, _)| *code).unwrap_or(0.0); @@ -475,7 +507,10 @@ pub fn fit_transfer( // what is left, so the reported residual describes the curve rather than // the worst sample. let (w, harmonic, rejected_points) = { - let first = fit_period(points, swept_span).ok_or(FitError::NoModulation)?; + let first = fit_period(points, swept_span).ok_or(FitError::NoModulation { + signal_span_volts: observed_span, + noise_span_volts: noise_span, + })?; let kept = without_outliers(points, first.0, &first.1); if kept.len() < points.len() && kept.len() >= MIN_POINTS { match fit_period(&kept, swept_span) { @@ -503,8 +538,11 @@ pub fn fit_transfer( 2.0 * radius, ), }; - if p1.abs() < MIN_SPAN_VOLTS { - return Err(FitError::NoModulation); + if !p1.is_finite() || p1.abs() <= f64::EPSILON || (noise_span > 0.0 && p1.abs() <= noise_span) { + return Err(FitError::NoModulation { + signal_span_volts: p1.abs(), + noise_span_volts: noise_span, + }); } let v_null = select_lobe(v, w, max_code).ok_or(FitError::NoLobeInRange { v_pi_dac: w })?; @@ -716,10 +754,40 @@ mod tests { clipped: false, }) .collect(); - assert_eq!( + assert!(matches!( fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement), - Err(FitError::NoModulation) + Err(FitError::NoModulation { .. }) + )); + } + + #[test] + fn accepts_a_repeatable_sub_10mv_transfer() { + // The real detector commonly operates between roughly 0.5 and 15 mV. + // A repeatable 4 mV lobe was rejected by the former absolute 10 mV + // threshold even though it is twice the measured window excursion. + let points = synthetic_sweep(300.0, 1_600.0, 0.010, -0.004, 4_095, 0.000_05, true); + let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement) + .expect("a resolved millivolt-scale lobe must fit"); + + assert!( + (fit.v_pi_dac - 1_600.0).abs() < 10.0, + "Vπ = {}", + fit.v_pi_dac ); + assert!(fit.span_volts.abs() < 0.010); + assert!(fit.span_volts.abs() > 0.003); + } + + #[test] + fn refuses_apparent_modulation_below_the_window_noise() { + let points = synthetic_sweep(300.0, 1_600.0, 0.008, 0.0, 4_095, 0.000_4, false); + assert!(matches!( + fit_transfer(&points, 4_095.0, DetectorGeometry::Direct), + Err(FitError::NoModulation { + noise_span_volts, + .. + }) if (noise_span_volts - 0.002).abs() < 1e-12 + )); } #[test] From f80c16032dba78f2d40838fcb1964c5eff813994 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 29 Jul 2026 15:14:51 +0200 Subject: [PATCH 34/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20allow=20mi?= =?UTF-8?q?crovolt=20photodiode=20inputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/features/stage-a-photodiode.md | 14 ++++-- plugins/stage-a-photodiode/src/lib.rs | 67 ++++++++++++++++++++++----- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index d8321e9..a576df8 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -45,14 +45,18 @@ so the estimator always runs the `RejectedComplement` geometry against `referenc amplitude sweep settles on this value, so a display toggle must not be able to move it (ADR 012). - **Reference I_tot** (`reference_volts`) is the total-power anchor: the PD reading with the full - beam diverted into the diode. A non-empty **anchor id** and explicit + beam diverted into the diode. The input accepts 1 µV steps (six decimal places in volts), which + covers the usual 0.0005–0.015 V detector range. A non-empty **anchor id** and explicit **measured and current** confirmation are required. Changing the value or id clears confirmation; until all three agree, `a` is withheld. - **Dark level** (`dark_volts`) + the **Capture dark** button: block the beam and press; the mean of - the current cache becomes the dark level. It is applied to the detector samples *and* to the - `I_tot` anchor, so it cancels out of the complement rather than biasing `a` — its job is to keep - the two sides consistent and to record the calibration the reading was taken under. `dark_id` in - the sidecar reads `dark-measured` or `dark-none` accordingly. + every sample currently retained in the monitor cache becomes the dark level. This is not a new + fixed-duration acquisition and it does not measure or modify `I_tot`: after blocking the beam, + wait at least one configured cache duration so earlier illuminated samples have aged out. The + dark input also accepts 1 µV steps for a manual correction. The value is applied to the detector + samples *and* to the `I_tot` anchor, so it cancels out of the complement rather than biasing `a` + — its job is to keep the two sides consistent and to record the calibration the reading was + taken under. `dark_id` in the sidecar reads `dark-measured` or `dark-none` accordingly. - The estimator uses only marker-bounded windows containing at least **two complete modulation cycles**, ending on phase 0. It no longer estimates extrema from an arbitrary trailing sample count; a low-frequency trace that diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 5ad2d20..f6b24d9 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -64,6 +64,12 @@ const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; +/// Drag increment for manually entered photodiode calibration voltages. +/// +/// The host derives the displayed decimal precision from this increment. The +/// Stage-A detector normally operates around 0.0005–0.015 V, so the former +/// 10 mV / 1 mV increments hid physically relevant values. +const VOLTAGE_INPUT_STEP_VOLTS: f64 = 0.000_001; /// Default monitor cache, in seconds of samples at the active stream rate /// (user-settable 1–130 s). const DEFAULT_CACHE_SECONDS: f64 = 20.0; @@ -911,13 +917,13 @@ impl StageAPhotodiodePlugin { }; if mean >= self.reference_volts { return Err(format!( - "dark level {mean:.4} V is not below the I_tot reference \ - {:.4} V — is the beam actually blocked?", + "dark level {mean:.6} V is not below the I_tot reference \ + {:.6} V — is the beam actually blocked?", self.reference_volts )); } self.dark_volts = mean; - self.last_save_note = Some(format!("dark level captured: {mean:.4} V")); + self.last_save_note = Some(format!("dark level captured: {mean:.6} V")); Ok(()) } @@ -2873,13 +2879,14 @@ impl Plugin for StageAPhotodiodePlugin { label: "Reference I_tot".into(), tooltip: Some( "Total power reference for EXCITATION mode, in photodiode volts: \ - the PD reading with the full beam diverted into the diode" + the PD reading with the full beam diverted into the diode. The field \ + accepts 1 µV increments; Capture dark does not set this value." .into(), ), kind: SettingKind::F64Drag { min: 0.0, max: ADC_FULL_SCALE_VOLTS, - speed: 0.01, + speed: VOLTAGE_INPUT_STEP_VOLTS, default: self.reference_volts, }, }, @@ -2911,15 +2918,15 @@ impl Plugin for StageAPhotodiodePlugin { key: "dark_volts".into(), label: "Dark level".into(), tooltip: Some( - "Measured dark level in photodiode volts (beam blocked). The \ - detector is DC-coupled, so the published contrast a is biased low \ - while this is 0." + "Measured detector offset in photodiode volts with the beam \ + blocked. The field accepts 1 µV increments; Capture dark can fill \ + it from the current sample cache." .into(), ), kind: SettingKind::F64Drag { min: 0.0, max: ADC_FULL_SCALE_VOLTS, - speed: 0.001, + speed: VOLTAGE_INPUT_STEP_VOLTS, default: self.dark_volts, }, }, @@ -2927,8 +2934,10 @@ impl Plugin for StageAPhotodiodePlugin { key: "capture_dark".into(), label: "Capture dark".into(), tooltip: Some( - "Block the beam, then press: takes the mean of the current cache \ - as the dark level." + "Block the beam and wait until earlier illuminated samples have \ + left the cache, then press. Uses the mean of every sample currently \ + retained in the cache as Dark level; it does not measure I_tot or \ + start a separate acquisition." .into(), ), kind: SettingKind::Button { enabled: true }, @@ -3801,6 +3810,42 @@ mod tests { assert_eq!(plugin.dark_volts, 0.0); } + #[test] + fn capture_dark_uses_the_mean_of_the_retained_cache() { + let mut plugin = live_plugin(); + plugin.reference_volts = 0.015; + if let Ok(mut state) = plugin.shared.lock() { + state.ingest(0, 20_000, 0, &[4, 6, 8]); + } + + plugin.capture_dark().expect("blocked-beam cache accepted"); + + let expected = code_to_volts(6.0); + assert!((plugin.dark_volts - expected).abs() < f64::EPSILON); + assert_eq!( + plugin.last_save_note.as_deref(), + Some("dark level captured: 0.004835 V") + ); + } + + #[test] + fn calibration_voltage_inputs_accept_microvolt_steps() { + let schema = StageAPhotodiodePlugin::default().settings_schema(); + + for key in ["reference_volts", "dark_volts"] { + let item = schema + .sections + .iter() + .flat_map(|section| section.items.iter()) + .find(|item| item.key == key) + .unwrap_or_else(|| panic!("missing {key} setting")); + let SettingKind::F64Drag { speed, .. } = &item.kind else { + panic!("{key} must remain an F64Drag setting"); + }; + assert_eq!(*speed, VOLTAGE_INPUT_STEP_VOLTS); + } + } + #[test] fn the_ui_mirror_keeps_the_operators_connect_intent() { // The mirror runs `apply_execution_context` every control tick. If it From 764f4f22737af88af04e1a9966c28c8c954db578 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 3 Aug 2026 14:01:05 +0200 Subject: [PATCH 35/46] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20drive=20A1?= =?UTF-8?q?=20surveys=20from=20declarative=20protocols?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks the Stage-A bench across all four crates so a whole `q_p(a, f)` survey runs unattended and every recording carries its own provenance. A1 (ADR 020-023, 027, 028) - `a` comes from the photodiode (measured) or the commanded calibrated drive (open loop), and the source is recorded in every artefact - `Find a₀` and the lock table apply only to a measured depth; a commanded depth has nothing to search for - the frequency ladder becomes an outer loop over the amplitude sweep, one `a₀` point or a whole depth sweep per `f` - surveys run from a `.csv` or `.toml` protocol naming `ū`, `f` and `a` per recording; examples ship in `protocols/` and install with the plugin - die temperature, pixel dead time and illumination land in every sidecar, absent rather than `0` when the host cannot report them - host sensor telemetry is compacted column-wise into the measurement folder under the run's own name Photodiode and calibration (ADR 017, 019, 024) - the detector learns `I_tot` from the brightest reading it takes; the four anchor settings are gone and dark cancels in the complement - rail detection is span-relative, and a withheld `a` names its gate across the plugin boundary - the transfer sweep judges residual and hysteresis against its own measured noise, over the window it swept Modulation (ADR 025, 026) - `ū` and `a` clamp into the achievable range instead of reverting, so every mode stays selectable - the applied lobe crosses the UI-mirror/live-worker boundary through a process-global generation Gating fixes (ADR 018): A1 requires only the output folder; the measurement and `I_k` ids are provenance, not preconditions. --- ...a-rail-detection-and-withheld-a-reasons.md | 115 + ...-stage-a-a1-required-vs-optional-inputs.md | 197 + ...e-a-calibration-measures-its-own-window.md | 145 + docs/adr/020-stage-a-a1-depth-source.md | 139 + ...ge-a-a1-no-search-for-a-commanded-depth.md | 121 + ...age-a-a1-sensor-conditions-on-every-run.md | 83 + ...stage-a-a1-nested-depth-frequency-sweep.md | 118 + ...tage-a-photodiode-learns-its-own-anchor.md | 126 + ...stage-a-drive-settings-clamp-not-refuse.md | 92 + ...lobe-crosses-the-mirror-worker-boundary.md | 82 + .../027-stage-a-a1-declarative-protocols.md | 146 + ...or-readout-travels-with-the-measurement.md | 92 + docs/features/README.md | 10 +- docs/features/stage-a-a1-event-count.md | 79 +- docs/features/stage-a-a1.md | 330 +- docs/features/stage-a-modulation.md | 53 +- docs/features/stage-a-photodiode.md | 83 +- docs/features/stage-a-pockels-calibration.md | 85 +- plugins/stage-a-a1/README.md | 169 +- plugins/stage-a-a1/protocols/example.csv | 71 + plugins/stage-a-a1/protocols/example.toml | 92 + plugins/stage-a-a1/src/csv.rs | 62 + plugins/stage-a-a1/src/lib.rs | 3 + plugins/stage-a-a1/src/protocol.rs | 970 +++ plugins/stage-a-a1/src/runtime.rs | 5756 +++++++++++++---- plugins/stage-a-a1/src/sensor.rs | 417 ++ plugins/stage-a-modulation/README.md | 22 +- plugins/stage-a-modulation/src/calibration.rs | 358 +- plugins/stage-a-modulation/src/lib.rs | 1320 ++-- plugins/stage-a-modulation/src/waveform.rs | 258 + .../testdata/pockels-20260730-083123.json | 705 ++ plugins/stage-a-photodiode/README.md | 28 +- plugins/stage-a-photodiode/src/lib.rs | 950 +-- scripts/install-built-plugins.sh | 7 + stage-a-io/src/estimator.rs | 132 +- stage-a-io/src/lib.rs | 3 +- stage-a-plugin-contract/src/lib.rs | 54 +- 37 files changed, 10849 insertions(+), 2624 deletions(-) create mode 100644 docs/adr/017-stage-a-rail-detection-and-withheld-a-reasons.md create mode 100644 docs/adr/018-stage-a-a1-required-vs-optional-inputs.md create mode 100644 docs/adr/019-stage-a-calibration-measures-its-own-window.md create mode 100644 docs/adr/020-stage-a-a1-depth-source.md create mode 100644 docs/adr/021-stage-a-a1-no-search-for-a-commanded-depth.md create mode 100644 docs/adr/022-stage-a-a1-sensor-conditions-on-every-run.md create mode 100644 docs/adr/023-stage-a-a1-nested-depth-frequency-sweep.md create mode 100644 docs/adr/024-stage-a-photodiode-learns-its-own-anchor.md create mode 100644 docs/adr/025-stage-a-drive-settings-clamp-not-refuse.md create mode 100644 docs/adr/026-stage-a-applied-lobe-crosses-the-mirror-worker-boundary.md create mode 100644 docs/adr/027-stage-a-a1-declarative-protocols.md create mode 100644 docs/adr/028-stage-a-sensor-readout-travels-with-the-measurement.md create mode 100644 plugins/stage-a-a1/protocols/example.csv create mode 100644 plugins/stage-a-a1/protocols/example.toml create mode 100644 plugins/stage-a-a1/src/csv.rs create mode 100644 plugins/stage-a-a1/src/protocol.rs create mode 100644 plugins/stage-a-a1/src/sensor.rs create mode 100644 plugins/stage-a-modulation/testdata/pockels-20260730-083123.json diff --git a/docs/adr/017-stage-a-rail-detection-and-withheld-a-reasons.md b/docs/adr/017-stage-a-rail-detection-and-withheld-a-reasons.md new file mode 100644 index 0000000..3e5b60e --- /dev/null +++ b/docs/adr/017-stage-a-rail-detection-and-withheld-a-reasons.md @@ -0,0 +1,115 @@ +# ADR 017 — Rail detection is span-relative, and a withheld `a` names its gate across the plugin boundary + +- **Status:** Accepted +- **Date:** 2026-07-30 +- **Relates to:** ADR 011 (Pockels transfer calibration), ADR 012 (contrast + geometry is bench, not display), ADR 013 (event-count depth lock), ADR 014 + (frequency ladder), + [Stage-A Photodiode](../features/stage-a-photodiode.md), + [Stage-A A1 Analysis](../features/stage-a-a1.md), + [Stage-A A1 Event-Count Depth](../features/stage-a-a1-event-count.md) + +## Context + +Two independent defects met on the bench and produced the same symptom: every +`a₀` action refused, and the panel could not say why. + +### The clip guard was calibrated for a volt-scale detector + +`estimate_contrast` is fail-closed on ADC clipping (ADR 012 §3): codes within +`CLIP_MARGIN_CODES = 4` of either rail counted as clipped, and more than 1 ‰ of +such samples refused the window. The margin was an **absolute** code count. + +The Stage-A reject-port detector operates around **0.5–15 mV** — the range the +µV-granularity calibration inputs and the span-relative Pockels fit were +introduced for. At 3.3 V over 4095 codes (0.806 mV per code) that whole waveform +lives inside the bottom ~20 codes, so a 4-code margin covers 3.2 mV of a 14.5 mV +signal. A perfectly clean millivolt-scale sine put **30.7 %** of its samples +inside the "near the rail" band, 300× over the 1 ‰ limit, and was refused as +clipped. None of those codes was the rail; they were the signal. + +The refusal was therefore unconditional at bench gain: `optical_summary` was +never published, and `a` was never available. + +### A withheld `a` did not survive the plugin boundary + +ADR 012 §3 established that a refusal is stated, not silent — but only in the +photodiode plugin's own status readout. `PhotodiodeSummaryV1` carried +`optical_summary: Option<…>` and nothing else, so a consumer saw absence with no +cause. + +A1 gates the `a₀` lock, the amplitude sweep and the frequency ladder on that +value, and refused all three with one fixed sentence: + +> No photodiode-measured a — connect the photodiode and anchor I_tot first + +which named the two most common causes whatever the real one was. With a railed +window, too few trigger markers, or a stale snapshot, that message sent the +operator to re-check an anchor that was already correct. The resting status line +was no better: `a = — (photodiode: connected)`. + +The same shape of problem sat on A1's event ingestion. With **Live analysis** +off, nothing is ingested at all, and the panel reported `0 events, …; +free-running (no EXT_TRIGGER)` — a description of a toggle, phrased as a +description of the bench. The frequency ladder refuses without phase-0 markers, +so an operator with Live analysis off was sent to check trigger wiring. + +## Decision + +### 1. The rail margin is capped against the window's own span + +The near-rail margin exists to catch a waveform that is *about to* truncate, +which is only meaningful while the margin is small compared to the signal. It is +now `min(CLIP_MARGIN_CODES, floor(span · CLIP_MARGIN_SPAN_FRACTION))` with +`CLIP_MARGIN_SPAN_FRACTION = 0.05`, where `span` is the window's observed +peak-to-peak code range. + +- Volt-scale windows (span ≥ 80 codes) keep the previous 4-code margin exactly. +- Millivolt-scale windows collapse the margin to 0, which leaves **precisely the + rails** — code 0 and `full_scale_code` — classified as clipped. + +Genuine saturation is still refused at every gain: a waveform driven below zero +pins samples *at* code 0, and the 1 ‰ limit still catches it. `MAX_CLIP_FRACTION` +is unchanged; this ADR narrows what counts as a rail, not how much clipping is +tolerated. + +This follows the same reasoning as the span-relative `NoModulation` threshold in +the Pockels fit (ADR 011): a fixed absolute voltage cut cannot serve a detector +whose gain is a bench property. + +### 2. The refusal reason is published on the contract + +`PhotodiodeSummaryV1` gains `optical_unavailable: Option` — additive in +V1, `#[serde(default)]`, skipped when absent, so older owners and consumers are +unaffected. It carries the owner's `EstimateError` rendering, set exactly when +`optical_summary` is `None` **and** a window existed to judge. + +A1 consumes it through one `measured_a_blocker()` helper that returns the +operator action, checked in the order the data flows: no status snapshot → not +connected → stale snapshot → the owner's reason → no samples yet. Every gate that +needs `a` quotes it, and the resting status line renders it without the operator +pressing anything. + +### 3. A1 distinguishes "Live analysis is off" from "no trigger" + +The status line and the frequency-ladder refusal name whichever it is. Marker +count alone cannot tell them apart, and only one of them is fixed with a +screwdriver. + +## Consequences + +- Millivolt-scale windows publish `a`. They are **quantisation-limited**: with a + ~19-code span the complement's excitation minimum is a fraction of one code, + so `a` is sensitive to single-code noise. The estimator's 1st/99th-percentile + extrema absorb spikes, but a bench wanting precise `a` at high contrast should + still raise the detector gain. This ADR makes such windows *estimable*, not + *precise* — the clip guard was never the right place to enforce resolution. +- The `Clipped` refusal now means the signal reached a rail, not that it sat near + one. A window previously refused for being small is now accepted, so an + operator who read that refusal as "gain too low" loses that (misleading) cue. +- Every A1 gate on `a` reports one of a bounded set of causes traceable to the + owner. New `EstimateError` variants surface in A1 with no A1 change. +- `optical_unavailable` is a human-readable string, not a typed error. The + contract crate is serde-only and does not depend on `stage-a-io`, and the + consumer renders rather than branches on it. A consumer that needs to *act* + per-variant will need the typed error on the contract instead. diff --git a/docs/adr/018-stage-a-a1-required-vs-optional-inputs.md b/docs/adr/018-stage-a-a1-required-vs-optional-inputs.md new file mode 100644 index 0000000..2b0a0a2 --- /dev/null +++ b/docs/adr/018-stage-a-a1-required-vs-optional-inputs.md @@ -0,0 +1,197 @@ +# ADR 018 — A1 gates on what it needs, not on what it would like to know + +- **Status:** Accepted +- **Date:** 2026-07-30 +- **Relates to:** ADR 010 (amplitude sweep), ADR 013 (event-count depth lock), + ADR 014 (frequency ladder), ADR 015 (recording robustness), ADR 017 (rail + detection and withheld-`a` reasons), + [Stage-A A1 Analysis](../features/stage-a-a1.md), + [Stage-A A1 Event-Count Depth](../features/stage-a-a1-event-count.md) + +## Context + +Every A1 recording path was unusable on the bench, and the panel described the +symptom rather than the cause. + +### Provenance metadata was enforced as a precondition + +`begin_recording` refused unless three fields were non-empty: the output folder, +the measurement id, and the physical `I_k` flux point id. Only the first is +something the plugin actually needs — it is where the files go. The measurement +id names a folder and a file stem, and the plugin has always shipped a generated +default for it. The flux point id is pure provenance: it records *which* +illumination calibration point a row belongs to, and nothing in the recording, +the sweep or the lock reads it. + +Enforcing them anyway meant a blank field could not be distinguished from a +misconfigured bench, and the refusals were spread unevenly across the entry +points: + +| entry point | folder | measurement id | flux point id | +| --- | --- | --- | --- | +| `begin_recording` | refused | refused | **refused** | +| `begin_sweep` | refused | refused | **refused** | +| `begin_leased_sweep` | refused | refused | — | +| `begin_freq_sweep` | refused | refused | **not checked** | + +The last row is the defect. The frequency ladder validated its whole plan up +front — deliberately, so that "a plan that cannot work should say so in a +message, not two hours into a block" — but did not ask the question its own +recordings would ask. It therefore took the modulation lease, retargeted the +drive, confirmed the frequency against the trigger, and ran a closed-loop `a₀` +lock, and only then handed off to `begin_recording`, which refused on the blank +flux point id. The recording coordinator stayed idle, the sweep saw +`point_started == false`, and the point was skipped — for every point. The panel +read `Frequency sweep 1/7 … — recording` next to `Recording: idle`, which is an +accurate description of two components and an explanation of neither. + +The same held for the amplitude sweep and, transitively, for the single +event-count point. + +### The test suite could not see any of it + +Every fixture in `runtime.rs` was built from `plugin_with_markers()`, which sets +`flux_point_id: "flux-test"`, or set the field explicitly. Fifty-one tests +passed, including one that drove the entire frequency ladder to completion, +because none of them ever exercised the state an operator actually starts in: a +fresh panel with nothing typed in. + +### A converged lock disarmed itself + +`armed_lock()` required `(lock.target_a - a0_target).abs() <= 1e-6`. `a0_target` +is an `F64Drag` with a 0.01 step that round-trips through JSON on every settings +sync. One stray pixel of drag after a successful `Find a₀` silently disarmed the +lock, and `begin_a0_point` then reported + +> No converged a₀ lock for 10.000 Hz — press Find a₀ at this frequency first + +which is the one instruction that does not help, addressed to an operator who +had just done it. That sentence also covered two other causes — no lock at this +frequency at all, and a lock that ran out of trials — without distinguishing +them. + +### The panel was written for the person who wrote it + +Section descriptions ran to full paragraphs of bench physics +(`q_p`, `S_p(t)`, `I_exc = I_tot − I_pd`, "marker-bounded window", +"refractory condition 2·f·a₀/C ≪ 1/τ_refr"). The prose was accurate and +unreadable, and it competed for attention with the one line that mattered — the +status message saying why the button had just refused. + +## Decision + +### 1. A gate exists only for an input the action cannot proceed without + +The output folder stays required: there is no defensible default destination for +measurement data, and the buttons are already disabled without one, which is +discoverable before the click rather than after. + +The measurement id is filled in on use. `ensure_measurement_id()` generates one +when the field is blank and **writes it back to the field**, so the run is filed +under a name the operator can see. It is tested on the raw field, not on +`sanitize_stem`'s output — the sanitizer substitutes `A1` for anything that +reduces to nothing, so asking it whether the id was blank always answers no, and +every unnamed run would have quietly shared one folder called `A1`. + +The flux point id is recorded, never enforced. A blank one is written as the +explicit sentinel `unspecified`, which keeps "not stated" distinguishable from a +real id downstream. A missing provenance field makes a recording *less +traceable*; it never makes it *wrong*, and that is not a reason to withhold the +operator's data. + +### 2. Every gate a run will eventually hit is asked before the drive moves + +`begin_sweep` and `begin_freq_sweep` now call `photodiode_blocker()` up front, +alongside the checks they already made. The principle ADR 015 established for +the recording coordinator — check before the camera starts, not after — extends +to the supervisors: a ladder must not take a lease and move the drive to +discover, at point 1, something it could have known at point 0. + +`begin_sweep` also drops its own hand-written photodiode sentence in favour of +`measured_a_blocker()` (ADR 017 §2). It was the last gate on `a` still naming +the anchor and the cable whatever the real cause was. + +### 3. A lock arms within the operator's own tolerance + +`armed_lock()` compares `lock.target_a` to `a0_target` against +`a0_tolerance`, not against `1e-6`. The tolerance is already the operator's +statement of how close to `a₀` counts as `a₀`; applying a stricter rule to the +*same* quantity one line later was never coherent. + +`armed_lock_blocker()` returns the actual cause — no lock at this frequency, a +lock that stopped short (and where), or a lock aimed at a different `a₀` (naming +both values) — and both `begin_a0_point` and the resting status line render it. +This is ADR 017 §2's shape applied to the second gate. + +### 4. The panel speaks to the operator + +Operator-visible strings — section descriptions, tooltips, status entries, +refusal messages, and the `EstimateError` renderings A1 quotes across the plugin +boundary — state what to do, in the words of someone standing at the bench. +Quantities keep their symbols (`a`, `a₀`, `f`) because those are on the +whiteboard too; the machinery behind them (fold windows, marker-bounded +estimation, reject-port complement algebra) belongs in these documents, which is +where a reader who wants it will look. + +Refusals name an action. `"ADC clipping: 307‰ low / 0‰ high"` became +`"the signal is hitting the ends of the detector's range (307‰ at the bottom, +0‰ at the top) — lower the drive amplitude or the detector gain"`. + +### 5. A missing frequency is not a disconnected plugin, and is stated once + +A frequency reaches A1 from two independent places: the phase-0 trigger markers, +or the drive the modulation plugin has *acknowledged*. Neither is the connection +state, which `modulation_connected()` checks separately. The resting line +nonetheless read + +> Frequency: unknown — connect the modulation plugin, or the trigger cable + +on a bench whose modulation plugin was connected. The usual cause is simply that +no periodic drive has been applied yet, and telling an operator to plug in +hardware that is already plugged in is worse than saying nothing. + +`frequency_blocker()` distinguishes the cases in the order the data flows — no +owner snapshot, not connected, connected with nothing applied, a waveform that is +not periodic, a periodic waveform at 0 Hz — and `begin_a0_lock`, +`armed_lock_blocker()` and the status line all render it. This is the third +application of the `measured_a_blocker()` shape from ADR 017 §2; the pattern is +now the house style for any gate an operator can see. + +Each fact also appears on exactly one line. A missing frequency previously +occupied three — the transient message, the `Frequency:` line, and the a₀ +readiness line, each with its own phrasing of the same cause — which reads as +three problems. The a₀ line now defers to the frequency line rather than +restating it, and the response-curve line is omitted entirely when it has neither +points nor windows to report, instead of stating the absence of the two facts +above it. + +## Consequences + +- The three recording workflows run with an output folder and nothing else + typed in. Regression tests cover exactly that state, and the frequency-ladder + test now exists in both variants — ids set and ids blank. +- Sidecars from unnamed rows carry `flux_point_id = "unspecified"`. Offline + analysis that joins on the flux point must treat that value as absent; it is a + sentinel, not an id. Analysis written against the old contract never saw a + blank field, because a blank field never produced a recording. +- Generated measurement ids are timestamp-derived (`A1--`), so two + unnamed runs started in the same millisecond would collide. They cannot be: + the id is generated inside `begin_recording`, which refuses re-entry while a + recording is active. +- A lock now survives an `a₀` nudge inside the tolerance. Widening + `a0_tolerance` therefore also widens what counts as "the same target", which + is the intended reading — but an operator who widens it to 0.5 to force a + stubborn lock through will find older locks arming for targets they did not + mean. The status line always names the lock's own target. +- The panel no longer states the estimator's gates in the estimator's terms. + Someone debugging the fold or the contrast geometry reads ADR 011, 012 and 017 + rather than a tooltip. +- The resting panel is shorter, and lines disappear when they have nothing to + say. An operator scanning for "did it change?" now has fewer stable lines to + scan, but cannot rely on a fixed line count or line order — anything parsing + `status_entries()` positionally would break. Nothing does; the host renders + them as a list. +- `frequency_blocker()` reports on the *acknowledged* drive, so a frequency the + operator has typed into the modulation plugin but not applied still reads as + "not applied yet". That is the intended reading — A1 measures against what the + bench is doing, never against what a field says. diff --git a/docs/adr/019-stage-a-calibration-measures-its-own-window.md b/docs/adr/019-stage-a-calibration-measures-its-own-window.md new file mode 100644 index 0000000..4f221a8 --- /dev/null +++ b/docs/adr/019-stage-a-calibration-measures-its-own-window.md @@ -0,0 +1,145 @@ +# ADR 019 — The calibration owns its measurement window, and judges itself against its own noise + +- **Status:** Accepted +- **Date:** 2026-07-30 +- **Relates to:** ADR 011 (measured Pockels transfer calibration), ADR 016 (the + lobe is two observed codes), ADR 017 (rail detection and withheld `a`), + [Stage-A Pockels Transfer Calibration](../features/stage-a-pockels-calibration.md), + [Stage-A Photodiode](../features/stage-a-photodiode.md) + +## Context + +On 2026-07-30 the operator ran a transfer-curve sweep on the orange bench. The +curve on the plot was clean and unmistakably a Pockels lobe. The plugin reported +it as bad on three counts at once: + +- residual **22.3 %** of the detector span, +- hysteresis **25.7 %** — "the cell is drifting or the settle time is too short", +- **34 of 98** points "clipped the ADC … add attenuation and re-measure". + +The record is kept verbatim at +`plugins/stage-a-modulation/testdata/pockels-20260730-083123.json`. All three +numbers were artifacts, and the fit underneath them was exactly right. + +### Every point was four ADC samples + +Each archived `volts` is an exact multiple of a quarter code — +`0.002619 V = 13/4`, `0.04412 V = 219/4`, `0.05581 V = 277/4`. The stream ran at +500 kSa/s, so each "settled" point was **8 µs** of signal, captured *after* the +sweep had already waited 4 ms for the cell to arrive. + +`PhotodiodeLevelV1` was computed over `avg_window_samples` — the photodiode +plugin's **chart smoothing** setting, default four samples. A display preference +was setting the precision of a physical calibration, and nothing named that +coupling anywhere. At 20 kSa/s it had been 200 µs and merely mediocre; the move +to 500 kSa/s made it 8 µs without changing a line of code. + +The consequences were all downstream of that one number. Per-point scatter came +out at σ = 11.3 mV against a 50.8 mV lobe, which *is* the reported 22.3 % +residual. + +### The hysteresis was the same noise, counted twice + +Both passes measure one curve, so at a matched code they differ by two +independent errors of scale σ, and `E|Δ| = σ√2·√(2/π) = 1.128 σ`. For this sweep +that predicts 12.8 mV; the measured mean |up − down| was 13.0 mV. The metric was +reporting its own point noise as cell drift, and a fixed 5 % threshold cannot +tell the two apart on any bench whose points are not far quieter than that. + +### The clipping flag was ADR 017's bug, one layer up + +`current_level` still marked a window clipped when its minimum fell within a +fixed 4 codes of the rail. The reject-port detector's dark end genuinely sits at +~3 codes (2.6 mV), so a third of every sweep was flagged. The span-relative +margin that ADR 017 introduced in the contrast estimator had never been carried +into the published level. + +### And a gate that would have got worse + +`fit_transfer` refused a sweep whose between-code span did not exceed the median +`peak_to_peak_volts` of the settled windows. That compares a span of *means* +against a *raw within-window excursion* — wrong by √N, and wrong in a way that +tightens as the averaging window grows. This sweep cleared it by a factor of 1.9. +Lengthening the window without touching this gate would have refused the very +sweeps the longer window was meant to rescue. + +## Decision + +### 1. The published level is a measurement, not a view of the chart + +`PhotodiodeStreamV1.level` is averaged over a **fixed duration owned by the +photodiode plugin** (`LEVEL_WINDOW_SECONDS = 20 ms`), independent of +`avg_samples` and `avg_sync_freq_hz`. `sample_count` reports what it actually +was. The chart's own averaging is untouched — it remains an operator preference, +and it no longer reaches anything downstream. + +A duration rather than a sample count, because what averages noise down is +time × bandwidth, not samples. 20 ms specifically because a boxcar of exactly one +mains period has a null at 50 Hz and every harmonic of it. At 500 kSa/s that is +10 000 samples in place of 4. + +### 2. Rail detection is shared, not re-derived + +`stage_a_io::near_rail_margin` is the single span-relative margin, used by both +`estimate_contrast` and the published level. A detector running a few codes above +zero is not truncating; the rails themselves stay guarded at every gain. + +### 3. Nothing judges the sweep by `peak_to_peak_volts` + +Both surviving gates use the fit's own RMS residual, which is the scatter of the +*averaged* points about the curve — the same quantity the lobe amplitude is +measured in, so the comparison is dimensionally honest and independent of +whatever window the owner publishes. + +**Is a lobe resolved?** Refuse when `rms ≥ 0.5·|span|`. The threshold needs +margin on both sides because a free period search over pure noise does not return +zero amplitude: with `n` points the quadrature pair has scale `σ√(2/n)`, and the +best of a 600-step scan inflates it by about `√(2 ln 600)`. Measured, that puts +noise-only quality at 0.7–1.0 (0.97 in the regression fixture) while the noisiest +real record on file reads 0.22. Half-way between is a plain statement — the lobe +must be at least twice its own scatter — with better than 2× margin either way. + +**Is the up/down difference drift?** Compare `hysteresis` against +`1.128 · rms / |span|`, the value it takes under noise alone. The ratio has two +derivable endpoints: **1.0** for pure noise, and **1.77** for pure drift, because +a systematic offset inflates the residual too (the fit splits the difference +between the passes, carrying `√(Δ²/4 + σ²)` while the metric carries +`√(Δ² + (1.128σ)²)`). The range is narrow and it is not optional to know that: a +generous multiple of the floor — 2×, the obvious first guess — sits above *both* +endpoints and never fires at all. The cut is at **1.33**, which detects a +systematic offset around 1.5× the point noise. Both endpoints are asserted in +`the_hysteresis_ratio_sits_between_its_two_derived_endpoints`. + +### 4. Settling is a duration + +`SETTLE_SECONDS = 0.1`, converted through the photodiode's published sample rate, +replacing a bare `SETTLE_SAMPLES = 2_000` that was written for 20 kSa/s and had +silently become 4 ms. `SETTLE_SAMPLES` remains only as the fallback for a stream +that has not published a rate. Settling is a property of the HV amplifier and the +crystal; nothing about it follows the acquisition rate. + +### 5. Clipping says what it costs + +Rail-touching points truncate the reported detector extrema, and with them the +`I_tot` lower bound. They do **not** move `V_null` or `Vπ`, which come from the +shape. The warning says so, and no longer advises attenuation — for a +reject-port detector it is the *dark* end that reaches the bottom rail, so the +fix is more gain, not less light. + +## Consequences + +- The chart's averaging setting no longer has any downstream effect. This is a + behaviour change for anyone who had turned it up expecting quieter sweep + points; they now get a quiet sweep without asking. +- A sweep costs ~120 ms per point (100 ms settle + 20 ms window), so ~12 s for + the full 98-point pass — still inside the documented ~20 s and far inside + `POINT_TIMEOUT`. +- The real bench record is a regression fixture. Synthetic sweeps could not have + caught any of this: they carry uniform noise, while a real detector's noise is + signal-proportional, and the metrics that broke were all compared against zero. +- The synthetic fixture's own "noise" was itself wrong for this question — a + wobble alternating with the point index is perfectly anti-correlated between + the two passes, i.e. systematic. `calibration::scatter` replaces it with + deterministic per-`(code, direction)` scatter. +- Nothing in `augur-rs` changes; this is entirely inside the Stage-A plugins and + their shared I/O crate. diff --git a/docs/adr/020-stage-a-a1-depth-source.md b/docs/adr/020-stage-a-a1-depth-source.md new file mode 100644 index 0000000..65af431 --- /dev/null +++ b/docs/adr/020-stage-a-a1-depth-source.md @@ -0,0 +1,139 @@ +# ADR 020 — A1 chooses where `a` comes from, and records the choice + +- **Status:** Accepted +- **Date:** 2026-07-31 +- **Relates to:** ADR 010 (amplitude sweep), ADR 011 (Pockels transfer + calibration), ADR 012 (contrast geometry is bench, not display), ADR 013 + (event-count depth lock), ADR 014 (frequency ladder), ADR 017 (rail detection + and withheld-`a` reasons), ADR 018 (A1 gates on what it needs), + [Stage-A A1 Analysis](../features/stage-a-a1.md), + [Stage-A A1 Event-Count Depth](../features/stage-a-a1-event-count.md), + [Stage-A Pockels Calibration](../features/stage-a-pockels-calibration.md) + +## Context + +`a = ln(I_exc,max / I_exc,min)` is a property of the excitation *light*. ADR 011 +and the estimator's module docs are emphatic about it: the Pockels V→T response +is non-linear, so the commanded DAC excursion is not a modulation depth and the +photodiode trace is the only valid source of `a`. Every A1 path that needs a +depth — the amplitude sweep, the `a₀` lock, the frequency ladder, the live +response curve — therefore read exactly one number: the photodiode owner's +`measured_log_contrast`. + +That number is fail-closed by design. The photodiode refuses to publish `a` +unless it can prove the window it estimated over covers whole modulation cycles, +which it does by bounding the window between firmware **phase-0 marker frames** +on its own stream port. With fewer than three retained markers it returns +`IncompleteModulationCycles` and publishes no `a` at all. + +On the bench this turned out to be reachable with the markers simply *absent*: + +``` +Measured depth a: not available. No stretch of samples covers two whole +modulation cycles between triggers (0 trigger(s) in the last 3446784 samples) +— lower the frequency, or raise the photodiode cache length +``` + +Three and a half million samples and zero markers is not a window that is too +short. It is a marker stream that is not arriving — no `MARKER` frames on the +stream port at all. The advice the refusal gives (lower the frequency, raise the +cache) cannot fix that, and no combination of settings can: without markers the +photodiode can never publish an `a`, so `Find a₀` refuses, the amplitude sweep +refuses, and the frequency ladder refuses. The entire A1 workflow is unreachable +on a bench whose Pockels cell is calibrated and whose drive is running +correctly. + +The workflow does not actually need a *measured* `a` to run. It needs **a +depth it can name, aim at, and record**. The modulation owner already has one: +it inverts the measured `V_null` / `Vπ` transfer curve to command a depth, and +publishes it as `OpticalDriveStateV1::depth_a_milli`. That is a calibrated +number — it comes from the same measurement ADR 011 exists to make — it is +simply not verified against the light afterwards. + +## Decision + +A1 gets one operator setting, **`depth_source`**, naming where its depth `a` +comes from: + +- `DepthSource::Photodiode` (**default**) — the photodiode's measured + excitation log-contrast. Unchanged behaviour, and the source of record. +- `DepthSource::Commanded` — the depth the modulation owner's calibrated + optical drive is commanding, read back from its published + `optical_drive.depth_a_milli`. + +One accessor, `depth_a()`, resolves the setting, and *every* consumer reads it: +the sweep's settle check, the `a₀` lock's readings, all three refusal gates, the +status panel, the live response curve, and the sidecar. The source is chosen in +exactly one place, so no path can be left reading the wrong one. + +`DepthSource::Commanded` is admissible only under the same conditions that make +a commanded depth mean anything at all. `optical_drive` is published solely for +`OPTICAL_LOG_SINE` / `OPTICAL_LINEAR_SINE` under an identified transfer +calibration, so a manual DAC band or a constant level yields no depth and the +gates refuse with that reason. A DAC number is never dressed up as an `a`. + +### Consequences for the closed loop + +The `a₀` lock is a feedback loop: command a depth, measure what the light did, +correct. Open loop the measurement *is* the command, so the loop converges on +trial 1 and the correction is a no-op. This is the honest degenerate case, not a +bug — there is nothing on the bench that could contradict the command — and it +is what makes the downstream machinery (the armed lock row, the event-count +point, the ladder) work unchanged. Two rules that exist for the estimator are +therefore scoped to the photodiode source: + +- the **stale-window rule** (only count summaries published after the depth was + commanded) — a commanded depth is not read out of a window, so enforcing it + would only couple the trial to the modulation owner's device-poll cadence; +- the **window-covers-a-cycle** and **clipping** checks — both are statements + about a detector window, and neither bounds a commanded depth. + +The operator's settle dwell still applies in both modes, so the drive gets its +physical time to move either way. + +### Provenance is not optional + +Everything that records an `a` records which source produced it: + +| artefact | field | +| --- | --- | +| A1 config sidecar (`.toml`) | `depth_a_source`, `depth_a` | +| A1 sidecar, `[a0_lock]` | `depth_source` | +| camera / PDQ recorder metadata | `depth_a_source`, `depth_a`, `a0_lock_depth_source` | +| `a0_locks.json` | `depth_source` (defaults to photodiode on older tables) | +| a₀ lock host view | `a from` column | + +`measured_a` keeps its historical meaning — a number the photodiode actually +measured — so an open-loop run simply carries no `measured_a`, rather than +carrying a commanded value under that name. A `q_p(a, f)` fit that pools the two +sources without looking at `depth_a_source` would be pooling two different error +budgets; the field is there so that cannot happen silently. + +The panel follows the same rule in prose: it says *"Commanded depth a (open +loop, not measured)"*, and an open-loop lock reports that the drive *"is +commanded as"* a value rather than that it *"measures"* one. + +## Alternatives considered + +**Fall back automatically when the photodiode withholds `a`.** Rejected. The +difference between a measured and a commanded depth is the difference between +two error budgets, and a silent switch would put both into one dataset with no +way to separate them afterwards. It also hides a real bench fault (a missing +trigger cable) behind a workflow that keeps running. + +**Loosen the photodiode's marker requirement instead** — estimate over a fixed +window when no markers exist. Rejected: a sub-cycle window *under*-reports `a`, +and the `a₀` lock divides by it, so it would drive the depth up until it rails. +ADR 017's fail-closed refusal is right; what was missing was a way past it that +does not lie about what was measured. + +**Enter `a` by hand.** Rejected — that is the datasheet number ADR 011 exists to +eliminate. The commanded depth comes from a measurement of *this* cell. + +## Status on the bench + +The commanded source is a way to keep working while the phase-0 marker stream is +diagnosed, not a replacement for measuring the light. A run recorded this way +carries the Pockels calibration's error plus any drift since it was taken, and +nothing checks it. Runs that go into the final `q_p(a, f)` fit should be +photodiode-measured. diff --git a/docs/adr/021-stage-a-a1-no-search-for-a-commanded-depth.md b/docs/adr/021-stage-a-a1-no-search-for-a-commanded-depth.md new file mode 100644 index 0000000..8516808 --- /dev/null +++ b/docs/adr/021-stage-a-a1-no-search-for-a-commanded-depth.md @@ -0,0 +1,121 @@ +# ADR 021 — There is nothing to search for in a depth you are commanding + +- **Status:** Accepted +- **Date:** 2026-07-31 +- **Relates to:** ADR 013 (event-count depth lock — the search this scopes), + ADR 014 (frequency ladder), ADR 020 (depth source), + [Stage-A A1 Exact Event Count](../features/stage-a-a1-event-count.md), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +ADR 013 built a closed-loop search, `Find a₀`, for one reason. The Pockels +transfer curve is measured once, so the inversion A1 commands through is +**static**, while the depth the cell actually delivers **rolls off with +frequency**. Holding one *measured* `a₀` across a frequency ladder therefore +means re-finding, per frequency, the commanded depth that produces it: + +``` +a_cmd ← a_cmd · a₀ / a_measured (≤ 8 trials, 3 readings each) +``` + +The result is stored per frequency in `a0_locks.json`, and every downstream +action — `Record a₀ point`, each rung of the ladder — replays a stored, +converged row. That is real work and it is the scientific core of the +exact-event-count workflow. + +ADR 020 then added a second depth source: when the photodiode cannot publish an +`a` at all (no phase-0 marker stream), A1 can take `a` from the modulation +owner's *commanded* calibrated drive. That made the workflow reachable again — +and immediately made the search meaningless, because in that mode the quantity +the loop measures *is* the quantity it commands: + +| step | measured source | commanded source | +| --- | --- | --- | +| command `a₀` | drive moves | drive moves | +| read back | photodiode reports what the light did | the owner reports the number just sent | +| correct | `a_cmd · a₀/a_measured`, repeat | ratio is exactly 1 — no-op | +| result | a per-frequency commanded depth | `commanded_a = a₀`, at every frequency | + +Eight ladder points produced eight identical rows carrying no information, each +behind a lease acquisition, a settle dwell and three "readings". The operator +was required to press `Find a₀` before `Record a₀ point` would arm, for a search +whose answer was already on screen. + +It was also actively harmful. `begin_a0_lock` warm-starts from any stored row at +the current frequency, so a row left over from a *measured* session (say +`commanded_a = 0.83` for `a₀ = 0.5`) would seed an open-loop run with a +closed-loop number, fail to converge on trial 1, and spend a second trial +correcting itself back to `0.5`. A no-op that can still be wrong is worse than +no operation at all. + +## Decision + +Whether the search is needed is a property of the depth source, expressed once +as `DepthSource::needs_a0_lock()`, and three things follow from it. + +**1. The lock table stops being the way to ask "what depth is armed here?"** +That question moves to `armed_a0()`, which returns a lock row from the table +under a measured source and *synthesises* one under a commanded source +(`commanded_a = target_a = a₀` at the current frequency, `trials: 0` recording +honestly that no search happened). `begin_a0_point` and the ladder both ask +this, and neither knows which regime it is in. Nothing is written to +`a0_locks.json` open loop, because nothing was found. + +**2. The ladder skips the `Locking` phase entirely.** Per rung it becomes +lease → set frequency → confirm → record. The `FreqSweepPhase::Locking` arm and +the direct path share one `start_freq_sweep_recording`, so there is a single +place where a point becomes a recording. + +**3. `Find a₀` refuses instead of pretending.** It states that `a₀` is commanded +directly and points at `Record a₀ point` / `Record all frequencies`. The button +is disabled rather than hidden — it is what the whole a₀ workflow is documented +around, so it has to stay visible and explain itself. + +### Frequency confirmation follows the same logic + +The ladder confirmed each commanded frequency against the **camera's** phase-0 +markers, on the correct principle that an ACK says the table was accepted, not +that the light is modulating at that rate. But a bench with no marker stream — +the exact bench the commanded source exists for — can never satisfy it, so +removing the search alone would have left the ladder refusing at the next gate. + +Under a commanded source the ladder therefore confirms against the modulation +owner's **acknowledged waveform**. This is not a new trust relationship: it is +the same owner, and the same acknowledged state, that mode already trusts to +state `a`. The cost is explicit — the live `q_p` fold goes free-running without +markers — and it is confined to the live quicklook. The recorded RAW and PDQ, +which are what the offline fit actually reads, are unaffected. + +Under a measured source nothing changes: markers confirm the frequency, and +`begin_freq_sweep` still refuses up front when they are absent. + +## Consequences + +- The commanded ladder runs on a bench with **no photodiode `a` and no camera + trigger**, with `Live analysis` off, and records every planned point. +- Open loop, the panel stops reporting a saved-depth count that is structurally + always zero, and says what will happen instead: *"the drive is commanded to + a = 0.500 at 1.000 kHz — no search needed"*. +- The a₀ section's description text switches with the source, and states the + trade in the operator's own terms: nothing verifies the light reached `a₀`, + and the static inversion does deliver less depth as `f` rises. +- **The measured workflow is untouched.** `Find a₀`, the per-frequency trim, the + lock table and its disk mirror all behave exactly as ADR 013 specifies the + moment the depth source is the photodiode again. + +## Alternatives considered + +**Delete the lock outright.** Simplest possible plugin, and wrong: it would +permanently give up ADR 013's guarantee that the same *measured* depth was held +across frequencies. The roll-off it corrects is real; only its applicability to +an open-loop depth is not. + +**Keep the search but make it one trial.** Still a lease, a dwell and a table +row per frequency, to reproduce a number the operator typed. The ceremony was +the complaint, not its duration. + +**Write the synthesised rows to `a0_locks.json` anyway**, for uniformity. They +would be eight identical restatements of the `a₀` setting, and a reader of that +file could no longer tell a found depth from an assumed one. Provenance already +lives in the sidecar (`depth_a_source`, `[a0_lock].depth_source`). diff --git a/docs/adr/022-stage-a-a1-sensor-conditions-on-every-run.md b/docs/adr/022-stage-a-a1-sensor-conditions-on-every-run.md new file mode 100644 index 0000000..7802c24 --- /dev/null +++ b/docs/adr/022-stage-a-a1-sensor-conditions-on-every-run.md @@ -0,0 +1,83 @@ +# ADR 022 — Every A1 run records the bench conditions the sensor measured + +- **Status:** Accepted +- **Date:** 2026-07-31 +- **Relates to:** ADR 009 (recording coordinator), ADR 015 (recording + robustness), ADR 020 (depth source provenance), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +The A1 sidecar already reproduces everything about the *drive* — frequency, +depth, calibration, anchor, ROI, trigger — but nothing about the physical state +of the sensor while a run was taken. Three quantities the camera measures for +itself are now available on the host's per-frame context bus +(`CTX_SENSOR_MONITORING` → `SensorMonitoringV1`): + +| quantity | field | why it matters to `q_p(a, f)` | +| --- | --- | --- | +| pixel dead time (refractory period), µs | `pixel_dead_time_us` | caps how many events a pixel can emit per half-cycle; at high `f` it *is* the ceiling the response saturates against | +| scene illumination, lux | `illumination_lux` | the physical `I_k` axis the whole experiment is stratified on | +| die temperature, °C | `temperature_c` | moves the biases, so two rows at nominally identical settings are not comparable across a large drift | + +None of these is derivable from the recording afterwards, and all three drift +over a session. A row that cannot be compared to another has to be identifiable +as such at analysis time, which means the numbers belong in the artefact, not in +a lab notebook. + +## Decision + +A1 mirrors `SensorMonitoringV1` every frame and writes it into **every** +recording, in every mode — normal, pilot, background, amplitude-sweep point and +event-count point alike. This needs no per-mode work: both write paths are +already shared, so the values go into `recording_metadata()` (which both the +camera and PDQ recorders embed) and into a new `[sensor]` section of the A1 +config sidecar. + +Four properties are load-bearing. + +**Mirrored above the `live` gate.** `process_frame` returns early when Live +analysis is off, and recordings are made that way at least as often as not. The +mirror therefore sits with the ROI mirror, before the gate. + +**Frozen at recording start.** These quantities drift; the sidecar is written at +finalize, seconds to minutes later. The number that belongs to a run is the one +that held when it began, so `begin_recording` snapshots `sensor_at_start` before +any of the start handshake runs, and the writers prefer it over the live value +(falling back to the live one only if the run began before any frame carried a +reading). + +**Absent, never zero.** Every field is optional at three levels: the host +publishes nothing at all during replay, decoded imports and offline re-runs +(there is no device to ask), a sensor without a monitoring block publishes +nothing, and an individual quantity can be `None` on a sensor that has one. A +`0 °C` die or a `0 lx` scene reaching an analysis script as a measurement is the +failure mode this exists to avoid, so a missing quantity omits its key entirely +rather than defaulting. + +**Provenance only, never an input.** No result A1 computes may depend on these +values. The API's own docs are explicit about why: a plugin whose answers vary +with them would disagree between a live run and a deterministic offline re-run +of the same data. `age_s` is recorded alongside (`sensor_reading_age_s` / +`reading_age_s`) because the host polls at a few hertz — a reading is never +simultaneous with the run it is attached to, and the sidecar says how stale it +was rather than implying it was not. + +The absolute bias codes that arrive with the same struct are written too +(`bias_diff_on`, `bias_diff_off`, `bias_fo`, `bias_hpf`, `bias_refr`). The host +camera config expresses biases as *relative* offsets around a per-unit factory +trim, so these are the only absolute record of what the sensor was actually +programmed to. + +## Consequences + +- The A1 sidecar gains an optional `[sensor]` section; both recorders' metadata + gains `sensor_temperature_c`, `sensor_pixel_dead_time_us`, + `sensor_illumination_lux` and `sensor_reading_age_s`. +- The status panel shows the live reading on one line when the host reports one, + and stays silent when it does not. +- Sidecars written from replay or from a camera without a monitoring block have + no `[sensor]` section at all — an analysis script must treat it as optional, + exactly as it must treat `optical.measured_a` under ADR 020. +- Only `refr` among the biases has a vendor-documented physical unit, which is + why the other four are recorded as codes and not converted. diff --git a/docs/adr/023-stage-a-a1-nested-depth-frequency-sweep.md b/docs/adr/023-stage-a-a1-nested-depth-frequency-sweep.md new file mode 100644 index 0000000..2497030 --- /dev/null +++ b/docs/adr/023-stage-a-a1-nested-depth-frequency-sweep.md @@ -0,0 +1,118 @@ +# ADR 023 — The frequency ladder is an outer loop, not one experiment + +- **Status:** Accepted +- **Date:** 2026-07-31 +- **Relates to:** ADR 010 (amplitude sweep — the inner run), ADR 013 (`a₀` + lock), ADR 014 (frequency ladder), ADR 021 (no search for a commanded depth), + [Stage-A A1 Analysis](../features/stage-a-a1.md), + [Stage-A A1 Exact Event Count](../features/stage-a-a1-event-count.md) + +## Context + +A1 had two multi-recording runs, and they were built as if they were unrelated: + +- **The amplitude sweep** (ADR 010) — leases the drive, walks `[min_a, max_a]`, + records one measurement per depth. One `q_p(a)` curve, at whatever frequency + the operator happened to have armed. +- **The frequency ladder** (ADR 014) — leases the drive, walks a log-spaced set + of frequencies, and records *one* point at each. + +The experiment the bench actually exists to produce is `q_p(a, f)` — a response +curve per frequency, from which `a50(f)` is fitted offline. Getting it meant +driving the amplitude sweep by hand once per frequency: set `f` in the +modulation plugin, press *Record depth sweep*, wait, come back, repeat. Seven +frequencies of that is seven manual interventions, seven opportunities for the +drive to be left somewhere unintended between blocks, and — because each sweep +takes and releases its own lease — seven windows in which the operator's own +settings are re-applied on top of the run. + +Meanwhile the ladder already had every part of that missing outer loop: log +spacing, visit order (ascending / descending / alternating / seeded random), +interleaved low-frequency reference repeats, per-frequency confirmation, one +lease held across the whole block, skip-and-report on a frequency it cannot +reach, and a summary. All of it was hard-wired to record exactly one thing at +each rung. + +## Decision + +The ladder becomes an outer loop with a **mode** naming what each rung records: + +```rust +enum FreqSweepMode { + A0Point, // one event-count point at the frozen depth a₀ (ADR 013/014) + DepthSweep, // the whole [min_a, max_a] sweep — the q_p(a, f) surface +} +``` + +`DepthSweep` reaches the inner run through `begin_leased_sweep` with +`SweepKind::Amplitude` and the ladder's own `lease_id` — the *unchanged* +amplitude sweep, inheriting the lease rather than taking one. So the operator's +drive settings stay locked out from the first frequency to the last, not merely +between the points of one curve, and are handed back once at the end. + +Everything else about the ladder is shared as it stands: ordering, the reference +repeats, the frequency confirmation of ADR 021, skip-and-report, the summary. +Adding the second experiment added one enum, one dispatch and one button. + +### A depth sweep never needs a search + +`FreqSweepMode::needs_armed_depth()` is false for `DepthSweep`, so the +`Locking` phase is skipped **in both depth sources** — not only the commanded +one of ADR 021. The reasoning is different from ADR 021's and worth stating: an +`a₀` rung replays a single depth that something must have chosen, whereas a +depth sweep commands every `a` in its range itself and settles on each against +the measured value. There is nothing for a lock to contribute at any frequency. +A measured-source nested sweep is therefore still fully closed-loop — each point +waits for the photodiode to reach its own target — it simply has no `a₀`. + +### Two things that had to change underneath + +**The ladder needed the inner run's verdict, not the last recording's.** It +advanced on `recording_completed_ok`, which describes one recording. A depth +sweep that gives up on point 4 of 5 leaves that flag `true` from point 3, and +the rung would have counted as finished with a half-recorded curve. `Sweep` now +carries `completed_ok`, set only on the branch that runs out of points with +every one recorded, and `finish_sweep` publishes it as +`last_sweep_completed_ok`. The `a₀` path reads the same flag — it is also a +`Sweep` — so this replaced the weaker check rather than adding a second one. + +**File names had to carry both axes.** Amplitude-sweep points are tagged `_pNN`, +which repeats at every rung and would collide inside one measurement id. A +nested point is now `…_fHz_pNN`, so the surface sorts by frequency and then +by depth. + +**The lease TTL is sized per mode.** A depth-sweep rung costs a whole inner +sweep; a TTL computed for one `a₀` point would expire mid-curve and hand the +drive back to the operator's settings while the block was still running. + +## Consequences + +- One button — *Record depth sweep at every frequency* — produces the whole + `q_p(a, f)` block: `frequency points × depth points` recordings on a single + lease, unattended. +- It introduces **no new settings**. The depth axis is the existing Recording + section (`min_a`, `max_a`, `sweep_count`, settle, duration); the frequency + axis is the existing ladder (`min_f`, `max_f`, `freq_count`, order, seed, + reference repeats). Its own section says so explicitly, because those two + groups live under headings named after other experiments. +- The run is *large* by construction. The button's tooltip states the + multiplication rather than discovering it at runtime, and the start message + reports `N × M recordings`. +- `a0_locks.json`, `Find a₀` and the `a₀` ladder are untouched. + +## Alternatives considered + +**A separate second ladder.** A copy of the outer loop specialised to depth +sweeps. Rejected: ordering, reference repeats, confirmation, skip-and-report and +the lease discipline would then exist twice and drift apart — ADR 014's +machinery is the valuable part, and it is entirely mode-agnostic. + +**Make the inner run a list of `(f, a)` pairs in one flat sweep.** Simpler +state, but it loses the ladder's per-frequency semantics: the reference repeats +are defined per frequency, the frequency confirmation happens per frequency, and +a failure has to skip a *frequency* rather than a point. Flattening would have +made the skip granularity wrong. + +**Reuse `Record all frequencies` with a mode setting instead of a second +button.** Rejected: a control whose meaning depends on a nearby dropdown is how +an operator records the wrong experiment overnight. Two buttons, two names. diff --git a/docs/adr/024-stage-a-photodiode-learns-its-own-anchor.md b/docs/adr/024-stage-a-photodiode-learns-its-own-anchor.md new file mode 100644 index 0000000..d46bcaa --- /dev/null +++ b/docs/adr/024-stage-a-photodiode-learns-its-own-anchor.md @@ -0,0 +1,126 @@ +# ADR 024 — The photodiode learns its own total-power anchor, and dark cancels + +- **Status:** Accepted +- **Date:** 2026-08-01 +- **Relates to:** ADR 006 (two-plugin split), ADR 011 (Pockels transfer + calibration), ADR 012 (contrast geometry is bench, not display), + ADR 019 (calibration measures its own window), + [Stage-A Photodiode](../features/stage-a-photodiode.md) + +## Context + +The Stage-A detector sits behind the PBS reject port and reads the complement +of the excitation, `I_pd = I_tot − I_exc`. Recovering the excitation contrast +`a = ln(I_exc,max / I_exc,min)` therefore needs `I_tot`, and the plugin asked +the operator for it through four settings: + +| setting | what it wanted | +| --- | --- | +| `reference_volts` | the detector reading with the whole beam sent into it | +| `reference_anchor_id` | a name for that reading, for provenance | +| `reference_confirmed` | a tick-box asserting it was measured *for this setup* | +| `dark_volts` + `capture_dark` | the reading with the beam blocked | + +Nothing downstream would publish `a` until the tick-box was ticked, and editing +either the value or the id un-ticked it. That gate is why the A1 plugin's +photodiode depth source appeared not to work at all: the default configuration +withholds `a` permanently, and the reason surfaces only as one line of status +text on a different plugin. + +Three things were wrong with this: + +1. **`I_tot` is measurable, not typeable.** Getting it by hand means blocking + the sample arm, reading a number off the chart, and typing it back — a + procedure that is re-done, or silently not re-done, every time the optics + are touched. The tick-box exists precisely because nobody can tell from the + number whether it is current. + +2. **The bench already measures it.** The Pockels transfer sweep (ADR 011) + walks the DAC across the whole lobe, which drives the excitation through its + null by construction. At the null all the light goes to the reject port, so + the detector reading *there* is `I_tot`. The calibration archive has + recorded `detector_volts_at_null` all along, described as "a lower bound on + the total-power anchor". + +3. **The dark level cancels.** With a DC dark offset `D`, the corrected + excitation is `(I_tot,obs − D) − (v − D) = I_tot,obs − v`. The `D` terms + cancel *exactly*, because both sides are readings from the same DC-coupled + detector. A dark setting can therefore only do harm: entered on one side + only, it biases `a`; entered on both, it does nothing. + +## Decision + +The photodiode learns `I_tot` from its own stream and the four settings are +removed. + +`SharedState` latches `observed_peak_code`: the maximum of the completed +64-sample summary-cell means since the port was opened. On the reject port the +detector is brightest exactly where the excitation is extinguished, so this is +`I_tot` by construction. The latch is over cell *means*, not raw samples, so a +single noise spike cannot pin the anchor high for every later `a`. + +Nothing has to be entered, and nothing has to be confirmed: the transfer sweep +the operator already runs before any measurement lands on the excitation null +and teaches the anchor as a side effect. The latch survives segment restarts — +a rate change, a dropped sample or an acquisition handover does not move the +optics, and the sweep that teaches the anchor is followed by exactly such a +handover. + +Dark correction is removed entirely. `AdcCalibration::dark_volts` is fixed at +zero and the published `dark_id` says `dark-cancels` rather than implying an +unmeasured zero. + +`PhotodiodeCalibrationV1` keeps its shape; `anchor_id` becomes +`observed-peak@` — provenance for a number nobody typed. + +The removed setting keys are still accepted by `set_setting` and ignored, so a +configuration written before this ADR still loads. + +## Consequences + +- The photodiode depth source works out of the box. `a` is withheld only for + reasons that are actually about the measurement — too few cycles in the + window, clipping, or an excitation that never dims below the brightest the + detector has been. +- That last case is a new refusal, and an honest one: if the modulation has not + yet been anywhere dimmer than the running peak, there is no complement to + take a contrast of. The message says to run the transfer sweep. +- `a` is now invariant to any DC offset in the front end, provably — there is a + unit test asserting that shifting the whole detector trace and the anchor + together leaves `a` unchanged to 1e-9, and a companion test asserting that + correcting one side alone *does* change it, so the first test cannot pass + vacuously. +- **Before any sweep has run, the estimator fails closed by construction.** The + obvious worry is that with only a modulated trace observed, the "total power" + sits barely above the signal and `a` explodes. It cannot: the anchor is the + maximum of 64-sample cell *means*, which for a modulated trace is always below + the robust high percentile the estimator compares against, so + `TotalPowerBelowSignal` fires instead. There is a test asserting exactly that + refusal — by name, not merely "some error" — at two very different + cycle-to-cell ratios, because a refusal for want of whole cycles would + otherwise make it pass without exercising the anchor at all. +- The anchor is a running maximum, so it never decreases within a session. + Reducing the laser power mid-session leaves it too high until the port is + reopened. This is the conservative direction — `a` comes out low rather than + high — and reconnecting resets it (`connect()` clears the ring). +- **The trustworthy path is still the transfer sweep.** The anchor is only as + good as the dimmest excitation the detector has seen; a bench that has never + been driven through the null has no anchor worth the name, and the estimator + says so. If that ever needs to be stronger, the modulation owner's fit already + holds a settled `detector_volts_at_null` and could push it over as a scoped + command — the same direction A1 already commands the photodiode in, so no + dependency cycle. +- The modulation plugin's calibration folder gains a stated purpose: its + archived `detector_volts_at_null` is the anchor a past run's `a` was measured + against. + +## Alternatives considered + +- **Publish the anchor from the modulation plugin's fit.** It has the number + already. Rejected: the photodiode is the upstream owner in the existing + dependency direction, and pushing the anchor back down it creates a cycle + between the two device owners for a value the detector can observe itself. +- **Keep `I_tot` as an optional override.** Rejected on the operator's own + reading of it: an escape hatch that is almost never the right path is still a + setting to understand, and its presence is what made the happy path feel + conditional. diff --git a/docs/adr/025-stage-a-drive-settings-clamp-not-refuse.md b/docs/adr/025-stage-a-drive-settings-clamp-not-refuse.md new file mode 100644 index 0000000..89eb512 --- /dev/null +++ b/docs/adr/025-stage-a-drive-settings-clamp-not-refuse.md @@ -0,0 +1,92 @@ +# ADR 025 — Drive settings clamp into the achievable range, they never refuse + +- **Status:** Accepted +- **Date:** 2026-08-01 +- **Relates to:** ADR 008 (optical waveform inversion), ADR 016 (lobe endpoints, + not a distance), [Stage-A Modulation](../features/stage-a-modulation.md), + [Stage-A Optical Waveform](../features/stage-a-optical-waveform.md) + +## Context + +The calibrated drive has two coupled controls — the cycle-mean lobe point `ū` +and the optical depth `a` — bounded by one shared constraint: the peak of the +swing must stay under the top of the Pockels lobe, and under the operator's DAC +max limit. + +Every setting that feeds that constraint was validated transactionally: + +```rust +let previous = self.mode; +self.mode = mode; +if let Err(error) = self.validate_drive() { + self.mode = previous; // snap back + return Err(error); +} +``` + +Which produces this, from the bench: + +> when using one mode, setting some of the I and a parameters it is blocking +> choosing other modes sometimes (which is a bug btw?) and then the user is +> asking himself why it isn't working + +It is a bug, and the mechanism is exactly the revert. With an `a` left over +from a different lobe, selecting `OPTICAL_LOG_SINE` builds a warp table that +saturates, so the *mode* is rejected and the dropdown snaps back — reporting an +error about `a`, a control the operator was not touching. There is no +indication of which value is in the way or how far it would have to move, and +the two controls can each block the other, so the way out is guesswork. + +## Decision + +Nothing in the drive settings reverts. Two changes: + +**1. One place that knows the constraint.** `waveform::PeakLaw` names how the +peak intensity follows from `ū` and `a`, one variant per calibrated mode: + +| variant | peak | used by | +| --- | --- | --- | +| `Constant` | `ū` | `CONST` | +| `LogSwing` | `ū·e^{a/2}` | `DAC_SINE`, `SQUARE` | +| `LogSine` | `ū·e^{a/2}/I₀(a/2)` | `OPTICAL_LOG_SINE` | +| `LinearSine` | `ū·(1 + tanh(a/2))` | `OPTICAL_LINEAR_SINE` | + +Solving each relation for one variable at a time gives `max_depth_for_mean` and +`max_mean_for_depth` — the achievable range. `LobeInversion::peak_intensity_ceiling` +turns the DAC max limit into the `u_max` they are solved against. + +These are asserted to agree with `warp_table()` to within 1e-4: a range that +disagreed with what the drive builder accepts would either offer a refused +setting or hide a working one. + +**2. Edits clamp, and only the edited control moves.** `reconcile_drive` takes +a `DriveKnob` naming what the operator just touched: + +- `Depth` — dragging `a` up means "more depth", so `a` is what gets limited and + `ū` stays put. +- `Mean` — and symmetrically. +- `Lobe` — a new calibration, a new ceiling or a new mode has no such + preference, so brightness settles first and the depth that fits under it + second. + +Modes and drive methods are always accepted. The achievable range is on the +status line and in the two control labels, so the boundary is visible before +the drag reaches it rather than reported after. + +An un-sendable drive is reported (`drive not sent: …`) instead of blocking the +edit, and the report is refreshed on every reconcile — a stale rejection from +an earlier combination no longer outlives the edit that fixed it. + +## Consequences + +- Every mode is selectable from every state. There is a test that walks all + five modes from a deliberately unbuildable `(ū = 1.0, a = 6.0)` and asserts + each one leaves a drive that builds. +- The labels carry the live bound: `Optical depth a (0..1.37 at ū=0.50)`. +- A clamp is a silent change to a value the operator asked for. That is right + for a drag, and wrong for automation: `SetOperatingPoint`, `SetOpticalDepth` + and `SetDriveFrequency` still refuse out-of-range requests rather than + clamping, because a protocol asked for a specific point and quietly recording + a different one would put the wrong parameters in every sidecar of a block. +- `validate_drive` is gone; `drive_command()` is consulted directly where its + verdict is wanted. diff --git a/docs/adr/026-stage-a-applied-lobe-crosses-the-mirror-worker-boundary.md b/docs/adr/026-stage-a-applied-lobe-crosses-the-mirror-worker-boundary.md new file mode 100644 index 0000000..a063c98 --- /dev/null +++ b/docs/adr/026-stage-a-applied-lobe-crosses-the-mirror-worker-boundary.md @@ -0,0 +1,82 @@ +# ADR 026 — The applied Pockels lobe crosses the mirror/worker boundary + +- **Status:** Accepted +- **Date:** 2026-08-01 +- **Relates to:** ADR 010 (button presses cross mirror → worker), ADR 011 + (Pockels transfer calibration), ADR 016 (lobe endpoints), + [Stage-A Pockels Calibration](../features/stage-a-pockels-calibration.md) + +## Context + +"Apply to V_null / V_peak" did nothing. Pressing it after a good sweep left the +two settings showing their old values and the drive on the old lobe. + +The host runs **two instances** of every plugin: a UI mirror that renders the +settings panel, and a live worker that owns the device link. Settings travel in +one direction only. Every live-analysis pass calls +`collect_live_plugin_state_snapshot`, which reads `get_setting` from the +mirror, and `apply_live_plugin_snapshot`, which writes each value onto the +worker. + +The measured fit lives on the worker — it is the instance with the photodiode +and the DAC. So the button failed twice over: + +1. The mirror ran `apply_calibration_fit` with `self.fit == None` and reported + "nothing to apply". +2. The worker applied the fit correctly, and the next settings sync overwrote + `v_null_dac` / `v_peak_dac` with the mirror's stale pair — within one frame. + +ADR 010 solved the *press* crossing this boundary (a monotonic counter through +`get_setting`). This is the opposite direction: a **result** produced on the +worker has to reach the mirror, and no channel carried one. + +## Decision + +Both instances live in the same process — the live worker is a thread, and the +plugin is one loaded `cdylib`. The applied lobe is published through a +process-global slot with a monotonic generation: + +```rust +static APPLIED_LOBE: Mutex> = Mutex::new(None); +static APPLIED_LOBE_GENERATION: AtomicU64 = AtomicU64::new(0); +``` + +Scoped by runtime role, which is what makes it a channel rather than shared +mutable state: + +- **only the live worker publishes** — it is the only instance that can have a + fit; +- **only the UI mirror adopts** — so no instance ever reads back its own + publication. + +The mirror consults it in two places. `get_setting` and `settings_schema` read +through `effective_lobe()`, so a freshly applied lobe reaches the panel *and* +the outgoing snapshot on the next repaint. `set_setting` calls +`adopt_applied_lobe()` first, so an incoming echo of the old codes cannot land +on top of a newer applied one. + +The generation makes adoption one-way and terminal: once the mirror is at +generation *n* it accepts ordinary edits again, so applying a calibration does +not freeze the two controls. + +A fresh instance starts at generation 0, not at the current value. A mirror +built after a calibration — a plugin reload — has to pick the applied lobe up, +not assume it is already current. + +## Consequences + +- The button works, and there is a regression test: after a sweep and an apply + on a worker, a newly constructed mirror's `get_setting("v_null_dac")` returns + the applied code, and a subsequent edit on the mirror sticks. +- Applying no longer refuses on the *drive*. A lobe that cannot express the + currently armed `a`/`ū` is still a valid measurement of the bench; the two + controls clamp to the new lobe instead (ADR 025). Applying still refuses two + codes that name no monotonic lobe at all. +- One global means one bench per process, which is what the hardware is. It + does mean unit tests that fabricate several plugins share it — the role + scoping keeps that harmless, since test plugins are live workers and never + adopt. +- This is a general shape, not a one-off. Any worker-produced value that has to + survive the settings snapshot needs the same treatment; the alternative — + making the host merge worker state back into the mirror — is an `augur-rs` + API change and is not warranted by one field. diff --git a/docs/adr/027-stage-a-a1-declarative-protocols.md b/docs/adr/027-stage-a-a1-declarative-protocols.md new file mode 100644 index 0000000..b645d3b --- /dev/null +++ b/docs/adr/027-stage-a-a1-declarative-protocols.md @@ -0,0 +1,146 @@ +# ADR 027 — A1 records surveys from a declarative protocol, including the `I_k` axis + +- **Status:** Accepted +- **Date:** 2026-08-01 +- **Relates to:** ADR 009 (recording coordinator), ADR 010 (amplitude sweep), + ADR 014 (frequency ladder), ADR 023 (nested depth/frequency sweep), + ADR 025 (clamping vs. refusing), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +A1 could sweep two of the three axes the experiment has: + +| axis | what moves it | how it is swept | +| --- | --- | --- | +| `a` — optical depth | `SetOpticalDepth` | amplitude sweep (ADR 010) | +| `f` — frequency | `SetDriveFrequency` | frequency ladder (ADR 014) | +| `I_k` — mean illumination | *nothing* | by hand, in the modulation plugin | + +Each sweep button moves its own axis and leaves the others wherever the +operator last put them. For exploring, that is the right shape. For a survey it +is not: + +- the `I_k` axis could not be swept at all, so a brightness series was one + manual edit per point with a lease released in between; +- what a block actually recorded lived in the UI at the time it ran, not in + anything that travels with the results; +- reproducing a survey six months later means reconstructing the panel state + from the sidecars it produced. + +The modulation plugin *did* have a protocol runner — a TOML list of timed `MOD` +steps. It was undocumented, unreferenced by any feature brief, drove raw DAC +codes rather than calibrated optical parameters, explicitly refused the optical +warp modes, and recorded nothing. It was removed. + +## Decision + +A1 gains a declarative protocol: a file naming every axis for every recording, +run by a supervisor built like the frequency ladder. Two front-ends produce the +same flat list of points, chosen by file extension, so nothing downstream knows +which was used. + +**CSV — one row per recording, and the one to reach for.** One line is one +recording, every parameter is a column, and the file opens in a spreadsheet or +comes straight out of a script: + +```csv +label,mean_u,frequency_hz,depth_a,duration_s,settle_s,role +floor,0.50,10,0.02,20,3,background +windows,0.50,10,2.00,20,3,pilot +ladder,0.40,1,0.80,40,4, +ladder,0.40,200,0.80,10,2, +``` + +Columns are located **by header name**, so their order does not matter and one +can be omitted entirely; blank lines and `#` comments are skipped; a blank cell +falls back to the default. Errors carry the **file line number**, because that +is what an editor and a spreadsheet both show. + +Two capabilities fall out of the row form that the block form cannot express +without one block per value: + +- **Per-recording duration and settle.** A 1 Hz point needs 40 s to cover + enough cycles and a 200 Hz point does not. This was the concrete ask. +- **A `role` column** (`normal` / `pilot` / `background`), so a file can carry + its own references — background floor first, pilot to freeze the ON/OFF + windows, then the points scored against them. A survey becomes a complete + measurement rather than something that needs two button presses first. + +**TOML — blocks and ranges.** Kept because it expresses a dense regular sweep +compactly, which a 96-row CSV does not: + +```toml +[defaults] +duration_s = 10 +settle_s = 2.0 + +[[block]] +name = "frequency-ladder" +mean_u = [0.3, 0.6] +frequency_hz = { min = 1.0, max = 200.0, points = 6, spacing = "log" } +depth_a = 0.8 +duration_s = 20 +``` + +Each axis takes a single value, an explicit list, or a `{ min, max, points }` +range with `linear` (default) or `log` spacing. A block records the full +product of its three axes. + +**`I_k` is `ū`.** The third axis is the normalized cycle-mean lobe point the +modulation plugin already exposes — dimensionless, not physical flux, but the +one control that moves the mean illumination without touching the depth. It is +driven by a new contract command, `ModulationCommandV1::SetOperatingPoint`, +scoped exactly like its two siblings: leased only, calibrated method only, and +the owner parks the operator's own value on the first retarget so `end_lease` +hands it back. + +**In the block form, points run `ū` outermost, then `f`, then `a`.** That is the +order of how expensive each change is to settle — the operating point makes the sensor +re-adapt, a frequency has to be confirmed against the phase-0 trigger, and the +depth is the cheap innermost step. Any other nesting spends the run settling. + +**All three axes are commanded at every point.** Not just the ones that +changed: a protocol states a whole operating condition, and a point that +inherited an axis from its predecessor would be recorded under parameters the +file does not name. A point waits for all three retargets to be acknowledged +before recording — recording after two of them would file the run under a +condition the bench was not at. + +**The file's `duration_s` wins over the panel's.** A survey whose recording +lengths silently came from the UI would not be reproducible from the protocol +alone. + +**Validation is up front.** Ranges, spacing, bounds and the total point count +are checked on the button press, before the drive moves, along with the same +whole-cycle window check the frequency ladder makes against its lowest +frequency. `MAX_POINTS = 4096` catches a three-axis product with one zero too +many *before* the bench spends a night on it. The status line reports the point +count and the expected bench time before the first recording starts. + +**A refused point is skipped, not fatal.** A `ū`/`a` pair that runs off the top +of the lobe is the ordinary failure in a long survey. The point is skipped +carrying the modulation owner's own wording, the run continues, and — because +the per-point message is overwritten within the same tick — the reason is kept +on the run and surfaced both in the status pane and in the closing summary. + +One lease covers the whole file. + +## Consequences + +- A survey is a file. It can be reviewed, diffed, version-controlled and + archived next to the data it produced. +- `plugins/stage-a-a1/protocols/example.csv` and `example.toml` ship as + commented starting points, installed alongside the plugin, and tests parse + both — a stale example is worse than none. The CSV test additionally asserts + that the shipped file really does use several different durations and both + reference roles, so it demonstrates what it claims to. +- The CSV reader shares its field splitter with the sensor readout compactor + (`src/csv.rs`); both locate columns by name for the same reason. +- The parse/expand core is a pure module with its own tests, so the axis + algebra is verified without a bench. +- `ū` is a normalized lobe coordinate, not a calibrated physical flux. Sweeping + it walks the brightness axis reproducibly; converting a point to photons + still needs the illumination calibration, exactly as before. +- The four sweep buttons remain. They are the right tool for exploring, and the + protocol is the right tool for the run that follows. diff --git a/docs/adr/028-stage-a-sensor-readout-travels-with-the-measurement.md b/docs/adr/028-stage-a-sensor-readout-travels-with-the-measurement.md new file mode 100644 index 0000000..57fbe5d --- /dev/null +++ b/docs/adr/028-stage-a-sensor-readout-travels-with-the-measurement.md @@ -0,0 +1,92 @@ +# ADR 028 — The sensor readout travels with the measurement, column-wise + +- **Status:** Accepted +- **Date:** 2026-08-01 +- **Relates to:** ADR 009 (recording coordinator), ADR 022 (sensor conditions on + every run), [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +ADR 022 put the sensor's own measurements — die temperature, pixel dead time, +scene illumination — into every A1 sidecar as provenance. That is *one* reading, +frozen at the moment the run started. + +The host separately polls the camera's monitoring block for the whole recording +and writes `.sensor-monitoring.csv` beside the RAW. Two problems: + +**1. It stayed behind.** A1 gathers the camera RAW, its bias sidecar, the +photodiode PDQ and its sidecar into one measurement folder under one name. +The telemetry was not in that list, so the record of how the bench actually +drifted during a run was separated from the run at the first move — left in the +host's capture folder under the host's own stem, alongside every other +recording's. + +**2. The layout is padding by construction.** The channels are polled on +different schedules: the die temperature drifts over minutes, the pixel dead +time is read far more often. A row-per-poll table with a column per channel is +therefore mostly empty cells. On top of that, five of its columns are bias +codes — already recorded in the camera's own bias sidecar, which *does* travel +with the RAW. + +## Decision + +`gather_into_measurement_folder` picks the telemetry up, rewrites it, and +removes the original. + +The rewrite is column-wise: one `{ t_us, value }` pair of arrays per channel, +carrying only the polls where that channel was actually read. + +```json +{ + "schema": "stage-a.a1.sensor.v1", + "measurement_id": "A1-20260801-1a2b", + "recording": "A1-20260801-1a2b_20260801-120000", + "polls": 412, + "channels": { + "pixel_dead_time_us": { "t_us": [1100,2100,…], "value": [12.7,12.8,…] }, + "temperature_c": { "t_us": [1100,61100,…], "value": [41.5,41.9,…] } + }, + "faults": [] +} +``` + +It lands in the measurement folder as `.sensor.json` and is +named in the sidecar's `[files]` block as `sensor_readout`, so it shares the +measurement's name and id like everything else in there. + +Decisions inside the rewrite: + +- **Nothing is resampled, interpolated or aligned.** The channels genuinely + have different rates; a reading exists at the instant it was taken or not at + all. Padding them onto a common grid would invent data. +- **A sample is timestamped at the midpoint of its poll.** A monitoring read + takes a few hundred microseconds; attributing it to the start would date + every reading systematically early. +- **Bias codes are dropped.** The camera's bias sidecar already carries them + and it travels with the RAW. +- **Failed polls are kept as `faults`,** so a gap in a channel is + distinguishable from a channel that was never polled — but an ordinary + "nothing due yet" row is not a fault. +- **Columns are located by name.** A host that inserts a column must not shift + every reading by one. +- **Rows that cannot be parsed are skipped, not fatal.** A truncated last line + is normal when a recording is cut short, and losing the other few thousand + samples over it would be the wrong trade. +- **The JSON is hand-rendered** so each channel's arrays stay on one line. + These files are read by eye as often as by script, and a pretty printer puts + one number per line. + +The whole path is best-effort: a missing or unreadable telemetry file is normal +(replay, a source with no monitoring block, a host that did not poll) and never +costs the operator the recording that just finished. + +## Consequences + +- A measurement folder is now self-contained for the bench conditions too: the + frozen start-of-run reading in the sidecar (ADR 022) *and* the full drift + across the run in the readout file. +- The host's capture folder is no longer littered with orphaned telemetry. +- The parse/compact core is a pure module with its own tests, including the + fault, truncation and column-reordering cases. +- The format is ours, versioned by the `schema` field. If the host ever emits + something richer, the reader changes and the schema tag moves with it. diff --git a/docs/features/README.md b/docs/features/README.md index d7a8ac9..7e97865 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -5,13 +5,13 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. -- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. +- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. The coupled `ū`/`a` controls **clamp into the achievable range instead of refusing**, so a leftover depth can no longer make an optical mode unselectable, and both live bounds are shown in the control labels (ADR 025). `V_peak` is the one operator-facing name for the lobe maximum; the half-wave span is derived and never entered. The undocumented TOML `MOD`-step protocol runner was removed — declarative recording protocols belong to A1 (ADR 027). - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC from measured `V_null`/`V_peak` endpoints, with target-specific headroom, Bessel-normalized cycle mean `ū`, and an explicit separation from physical flux `I_k`. -- [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry only after a named `I_tot` anchor is explicitly confirmed. -- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with physical `flux_point_id`, transfer/anchor provenance, and live response quicklooks. +- [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. Each point is a 20 ms measurement after a 0.1 s settle, and every verdict on the sweep — lobe resolved, cell drifting — is made against the fit's own residual rather than against zero (ADR 019). Applying the fit now actually reaches the panel: the measurement lives on the live worker while the settings snapshot is collected from the UI mirror, so the applied lobe used to be overwritten within one frame (ADR 026). +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. +- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. -- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. +- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-a1-event-count.md b/docs/features/stage-a-a1-event-count.md index b10ac64..6fc82aa 100644 --- a/docs/features/stage-a-a1-event-count.md +++ b/docs/features/stage-a-a1-event-count.md @@ -9,7 +9,17 @@ `SetOpticalDepth`), [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) (the RAW + PDQ + sidecar coordinator) and [ADR 012](../adr/012-stage-a-contrast-geometry-is-bench-not-display.md) (the - geometry the measured `a` is defined in) + geometry the measured `a` is defined in) and + [ADR 017](../adr/017-stage-a-rail-detection-and-withheld-a-reasons.md) (why a + gate refused, and millivolt-scale rail detection) and + [ADR 018](../adr/018-stage-a-a1-required-vs-optional-inputs.md) (a lock arms + within the operator's own `a₀` tolerance, and a lock that cannot arm names + which of the three causes it is) and + [ADR 020](../adr/020-stage-a-a1-depth-source.md) (`a` comes from the + photodiode or from the commanded drive) and + [ADR 021](../adr/021-stage-a-a1-no-search-for-a-commanded-depth.md) (with a + commanded depth there is nothing to search for: no `Find a₀`, no lock table, + and the ladder confirms each frequency against the modulation owner) - **Relates to:** [Stage-A A1 Analysis](./stage-a-a1.md), [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md), [Stage-A Photodiode](./stage-a-photodiode.md) @@ -91,6 +101,47 @@ across frequencies is not automated, i.e. off by default). Then: Steps 1–4 are what **Start frequency sweep** automates; see below. +### When a button refuses + +Every step from 2 on needs a photodiode-measured `a`, and each of these is +**fail-closed**: nothing touches the drive until the whole precondition set +passes. The refusal quotes the photodiode owner's own reason — a missing or +unconfirmed `I_tot` anchor, too few phase-0 markers in its ring, a window shorter +than one cycle, a railed window, a stale snapshot — instead of naming the two most +common causes regardless of the real one (ADR 017). The same reason is on the +resting status line as `Measured depth a: not available — `, so it can be +read without pressing anything, and the reason itself names an action rather than +an estimator gate (ADR 018). + +Some benches cannot produce a measured `a` at all — with no phase-0 markers on +the photodiode's stream port the estimator refuses whatever the settings say. +Every one of these refusals therefore also names the way past it: switching +**Depth `a` source** to the commanded drive (ADR 020). + +**Everything on this page below here describes the *measured* workflow.** With a +commanded depth there is nothing to search for, so the search does not run at +all (ADR 021): `Find a₀` is disabled and says so, `Record a₀ point` commands `a₀` +directly, the ladder goes lease → set `f` → confirm → record with no `Locking` +phase, and `a0_locks.json` stays untouched because nothing was found. The +ladder also confirms each frequency against the modulation owner's acknowledged +waveform rather than the camera trigger, so it needs neither EXT_TRIGGER markers +nor Live analysis. The whole workflow reduces to: set `a₀`, press **Record all +frequencies**. + +The trade is exactly the one the lock exists to remove — nothing verifies the +light reached `a₀`, and the static inversion delivers less depth as `f` rises — +so switch back to the photodiode once its markers work. + +The ladder additionally needs phase-0 markers on the **camera** side to confirm a +commanded frequency, which means **Live analysis** must be on. Its refusal says +which of the two is missing — the toggle or the trigger wiring. + +Preconditions a run will hit *later* are checked before the drive moves. The +ladder and the amplitude sweep both ask the recording's own photodiode question +up front: taking the lease, retargeting the drive and locking `a₀` only to be +refused by `begin_recording` at point 1 is what produced a panel reading +`Frequency sweep 1/7 … — recording` next to `Recording: idle` (ADR 018). + ## The frequency ladder (unattended) **Start frequency sweep (find a₀ + record per f)** runs the whole ladder on @@ -187,13 +238,27 @@ a_\text{cmd} \leftarrow a_\text{cmd}\cdot\frac{a_0}{a_\text{measured}} ## The lock table One row per frequency (a re-lock within 1 % of a stored frequency replaces it): -frequency, target `a₀`, commanded `a`, measured `a`, trials, state, locked-at. +frequency, target `a₀`, commanded `a`, observed `a`, **which source that `a` +came from**, trials, state, locked-at. Visible as the **A1 a₀ locks** host view and mirrored to `/a0_locks.json`, so the found depths survive a restart and can be cited offline. A non-converged row is kept for the record but **never** arms a recording; a stored lock only arms an event-count point when both its frequency **and** its `a₀` still match the current settings. +"Still matches" is judged against the operator's own **a₀ tolerance**, not exact +equality. `a₀` is a drag control with a 0.01 step, and the earlier `1e-6` +comparison meant one stray pixel of drag silently disarmed a lock that had just +converged — after which the panel asked for the `Find a₀` that had already been +done. The tolerance is already the statement of how close to `a₀` counts as +`a₀`; applying a stricter rule to the same quantity was never coherent (ADR 018). + +When no lock arms, the panel and the *Record a₀ point* refusal name **which** of +the three causes it is — no lock at this frequency, a lock that stopped short +(and the measured `a` it stopped at), or a lock aimed at a different `a₀` (naming +both values) — rather than telling the operator to press `Find a₀` in all three +cases. + ## Why recording re-applies the depth *Record a₀ point* does not simply record at whatever the drive currently is. It @@ -225,10 +290,12 @@ frequency sweep at the single frozen depth: Sub-hertz frequencies keep the decimal as `p` (`f0p5Hz`). The A1 sidecar adds `sweep.commanded_a` and an `[a0_lock]` section (`target_a`, `commanded_a`, -`measured_a_at_lock`, `frequency_hz_at_lock`, `trials`, `converged`, -`locked_at_utc`); both recorders' own sidecars carry `a0_target`, -`a0_commanded_a`, `a0_lock_measured_a` and `a0_lock_frequency_hz` as metadata. -The measured `a` of the recording itself stays in `[optical]` as for every run. +`measured_a_at_lock`, `depth_source`, `frequency_hz_at_lock`, `trials`, +`converged`, `locked_at_utc`); both recorders' own sidecars carry `a0_target`, +`a0_commanded_a`, `a0_lock_measured_a`, `a0_lock_depth_source` and +`a0_lock_frequency_hz` as metadata. The measured `a` of the recording itself +stays in `[optical]` as for every run, and its provenance in the top-level +`depth_a_source` (ADR 020). ## Choosing `a₀` (still an operator decision) diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 1d1edee..38e8800 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -10,7 +10,25 @@ duration, named failures), [ADR 014](../adr/014-stage-a-a1-frequency-ladder.md) (the unattended ladder), [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (exact-event-count - `a₀` lock) + `a₀` lock), + [ADR 017](../adr/017-stage-a-rail-detection-and-withheld-a-reasons.md) (a + withheld `a` names its gate; Live analysis vs. trigger), + [ADR 018](../adr/018-stage-a-a1-required-vs-optional-inputs.md) (the output + folder is the only required input; every gate is asked before the drive moves; + the panel speaks to the operator), + [ADR 020](../adr/020-stage-a-a1-depth-source.md) (`a` comes from the + photodiode or from the commanded drive, and every artefact says which), + [ADR 021](../adr/021-stage-a-a1-no-search-for-a-commanded-depth.md) (no `a₀` + search when `a` is the command; the ladder skips it), + [ADR 022](../adr/022-stage-a-a1-sensor-conditions-on-every-run.md) (die + temperature, pixel dead time and scene illumination on every run), + [ADR 023](../adr/023-stage-a-a1-nested-depth-frequency-sweep.md) (the + frequency ladder is an outer loop: a whole depth sweep per frequency gives the + `q_p(a, f)` surface in one press), + [ADR 027](../adr/027-stage-a-a1-declarative-protocols.md) (surveys are run + from a file, and `I_k` becomes a sweepable axis), + [ADR 028](../adr/028-stage-a-sensor-readout-travels-with-the-measurement.md) + (the sensor readout travels with the measurement, column-wise) - **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) - **Second workflow:** [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — hold one *measured* depth `a₀` across the frequency sweep @@ -40,23 +58,245 @@ folder. A1 makes each recording one button press: | Control | Meaning | |---|---| -| Output folder | where the A1 config sidecar is written (recommended shared experiment root) | -| Measurement id | one per `(I_k, f)` row; auto-generated default, editable, or press **New id** | -| Physical `I_k` flux point id | required canonical id of the cycle-mean local flux calibration/map point; never inferred from the modulator's normalized mean `ū` | -| Sweep min a / max a | the `a`-range for this row; the **Start sweep** button records it, and it is stored in every sidecar | -| Sweep points (count) | how many amplitudes Start sweep records, spaced evenly over `[min a, max a]` | -| Sweep settle (s) | dwell the fresh photodiode-measured `a` must hold the target (±10 %, ≥±0.05) before each sweep recording; timeout aborts the sweep | -| Duration (s) | each recording auto-stops and finalizes after this | -| Start recording (sweep point) | start camera RAW → connect and lease photodiode → start PDQ → auto-stop and save both → sidecar | -| Start sweep (record all points) | per point: lease the modulation owner → retarget the calibrated drive to `a_i` → settle → one recording (`…_pNN`) → next point | +| **Depth `a` source** | where every depth-dependent path reads `a` from: the **photodiode** (measured, default) or the **modulation drive** (commanded, open loop) — see below (ADR 020) | +| Output folder | **the only required field**: where the A1 config sidecar is written (recommended shared experiment root) | +| Measurement id | one per `(I_k, f)` row; auto-generated default, editable, or press **New id**. Optional — a blank field is filled in on the first recording and written back, so the panel shows the id that was used (ADR 018) | +| Duration (s) | each recording auto-stops and finalizes after this; applies to every button | +| Settle time (s) | dwell the depth or frequency must hold after being retargeted, before the recording starts; ignored by **Record once** | +| Depth axis: min a / max a / points | the `a`-range **Sweep a** walks, stored in every sidecar | +| Frequency axis: min f / max f / points / order / seed / repeat-lowest | the `f`-ladder **Sweep f** walks: log-spaced, visit order and interleaved reference repeats | +| **Record once** | one recording with the light exactly as armed: start camera RAW → connect and lease photodiode → start PDQ → auto-stop and save both → sidecar. Nothing is retargeted | +| **Sweep a** | per point: lease the modulation owner → retarget the calibrated drive to `a_i` → settle → one recording (`…_pNN`) → next | +| **Sweep f** | one recording per frequency at the same depth — the exact-event-count workflow, see [its brief](./stage-a-a1-event-count.md) | +| **Sweep a × f** | the **`q_p(a, f)` surface**: the whole depth sweep at every frequency, on one lease — see [below](#the-q_pa-f-surface-in-one-press-adr-023) | | Record pilot | records a bright reference (`…_pilot`) **and** freezes the ON/OFF windows for the row from the live signal | | Record background | records an unmodulated reference (`…_background`) **and** captures the false-response floor `q0` | -| Stop (abort recording / sweep) | finalize the current recording early; during a sweep also aborts the remaining points | -| a₀ / Find a₀ / Record a₀ point / Start frequency sweep | the **exact-event-count** workflow: hold one *measured* depth `a₀` across the frequency sweep, by hand or as an unattended ladder — see [its brief](./stage-a-a1-event-count.md) | +| **Stop** | stops whatever is running — a recording, a sweep, a ladder or a protocol — at its next safe point, so the file in flight is still finished and saved | + +All of it lives in **one Record section**. It used to be spread over three +(`Recording`, `Depth sweep at every frequency`, `Same depth at every +frequency`), each carrying part of the settings the others needed — so the +frequency axis was configured in the a₀ section and read by a button two +sections above it. **Live analysis** moved to the top of the panel for the same +reason: almost everything reads it. The record and sweep buttons stay **disabled until an output folder is selected**. +### Where `a` comes from (ADR 020) + +`a = ln(I_max/I_min)` is a property of the light, so the photodiode measurement +is the default and the source of record. It is also **fail-closed**: the +photodiode publishes no `a` unless it can prove its estimator window covers +whole modulation cycles, which it does from the firmware phase-0 **marker +frames** on its own stream port. If those markers never arrive — no trigger, or +a firmware build that does not stamp them — it refuses forever, with a reason +that reads like a settings problem: + +``` +No stretch of samples covers two whole modulation cycles between triggers +(0 trigger(s) in the last 3446784 samples) — lower the frequency, or raise the +photodiode cache length +``` + +Zero markers in millions of samples is a missing marker stream, not a short +window, and no setting fixes it. **Depth `a` source** is the way past: + +| Setting | `a` is | Needs | Verified against the light | +|---|---|---|---| +| `photodiode (measured)` — default | the photodiode's measured excitation log-contrast | phase-0 markers, a confirmed `I_tot` anchor, an unclipped window | yes | +| `modulation drive (commanded, open loop)` | the depth the modulation owner's calibrated drive is commanding (`optical_drive.depth_a_milli`) | an applied Pockels calibration and `OPTICAL_LOG_SINE` armed | **no** | + +The commanded depth is still a *calibrated* number — the modulation plugin +inverts the measured `V_null` / `V_peak` curve to produce it — it is simply not +checked afterwards, so it carries the calibration's error plus any drift since. +It is not a datasheet value and it is not a DAC excursion: a manual DAC band or +a constant level publishes no optical drive, and the gates refuse rather than +inventing a depth. + +Open loop there is **nothing to search for**, so `Find a₀` is not used at all and +the frequency ladder skips it — see [below](#a-and-the-a-ladder-adr-021). The +photodiode's window-length and clipping checks are skipped in this mode too, +because neither bounds a commanded depth; the operator's settle dwell still +applies. + +**Every artefact says which source it used**: the sidecar's `depth_a_source` / +`depth_a`, `[a0_lock].depth_source`, the recorders' `depth_a_source` metadata, +the `depth_source` field in `a0_locks.json`, and the *a from* column of the a₀ +lock view. `measured_a` keeps its narrow meaning — a number the photodiode +actually measured — so an open-loop run carries none, rather than carrying a +commanded value under that name. Runs destined for the final `q_p(a, f)` fit +should be photodiode-measured. + +### `a₀` and the a₀ ladder (ADR 021) + +`Find a₀` exists for one reason: the Pockels inversion is measured once and is +therefore **static**, while the depth the cell delivers **rolls off with +frequency**. Holding one *measured* `a₀` across a ladder means re-finding the +commanded depth that produces it at each frequency +(`a_cmd ← a_cmd · a₀/a_measured`). That is real work — and it only exists for a +measured depth. + +With the **commanded** source the loop measures the number it commands, so the +correction ratio is exactly 1. A search would command `a₀`, read back `a₀`, stop, +and store one identical row per frequency. So it is not run: + +| | photodiode (measured) | modulation drive (commanded) | +|---|---|---| +| `Find a₀` | trims the depth per frequency, stores a lock | **not needed** — disabled, and says so | +| `Record a₀ point` | replays the stored converged lock | commands `a₀` directly | +| Ladder per rung | lease → set `f` → **confirm via camera markers** → search → record | lease → set `f` → **confirm via the modulation owner's ACK** → record | +| `a0_locks.json` | one row per frequency | untouched — nothing was found | +| Needs camera EXT_TRIGGER + Live analysis | **yes** | no | + +The ladder's frequency check follows the same logic. Measured mode holds out for +the camera's phase-0 markers, because they define the period *and* anchor the +fold the point is scored in. Commanded mode asks the modulation owner instead — +the same owner, and the same acknowledged state, it already trusts for `a`. The +cost is confined to the live quicklook (the `q_p` fold goes free-running without +markers); the recorded RAW and PDQ that the offline fit reads are unaffected. + +**Net effect:** with the commanded source the ladder runs on a bench with no +photodiode `a`, no camera trigger and Live analysis off — set `a₀`, press +*Record all frequencies*. The trade is that nothing verifies the light reached +`a₀` at each frequency, and the roll-off the search corrects is real, so switch +back to the photodiode once its markers work. + +### The `q_p(a, f)` surface in one press (ADR 023) + +The frequency ladder is an **outer loop**, and what it records at each rung is a +mode: + +| button | per frequency | produces | +|---|---|---| +| *Record all frequencies* | one event-count point at `a₀` | `q_p(a₀, f)` — the same depth everywhere | +| **Record depth sweep at every frequency** | the **whole** `[min_a, max_a]` sweep | `q_p(a, f)` — one response curve per `f` | + +The second is the experiment `a50(f)` is fitted from, and it was previously a +manual loop: set `f`, press *Record depth sweep*, wait, repeat. It now runs +unattended as `frequency points × depth points` recordings **on a single lease**, +so the operator's drive settings stay locked out from the first frequency to the +last instead of being re-applied between blocks. + +It adds **no new settings**. The depth axis is the Recording section +(`Sweep min a` / `max a` / `points` / settle / duration); the frequency axis is +the ladder in the a₀ section (`Sweep min f` / `max f` / `points` / order / seed / +reference repeats). Ordering, the interleaved low-frequency reference, +per-frequency confirmation, skip-and-report and the summary are all the +unchanged ADR 014 machinery. + +**No `a₀` and no `Find a₀` are involved at any point**, in either depth source — +a depth sweep commands and settles every `a` in its range itself, so there is +nothing for a lock to contribute. With the photodiode source each point is still +fully closed-loop against the measured `a`; it simply has no `a₀`. + +Points are named `…_fHz_pNN`, so the surface sorts by frequency and then by +depth. A frequency whose curve cannot be recorded is skipped and named in the +summary rather than stopping the block — and a rung counts as done only when its +inner sweep recorded *every* point, not when its last recording happened to +succeed. + +### Protocol — a survey from a file (ADR 027) + +The four sweep buttons each move one axis and leave the others wherever they +are. That is right for exploring and wrong for a survey: `I_k` could not be +swept at all, and what a block recorded lived in the panel rather than in +anything that travels with the results. + +A **protocol** is a file naming every axis for every recording. The reader is +chosen by extension, and both produce the same flat list of points. + +**CSV — one row per recording**, and the one to reach for: it opens in a +spreadsheet, comes straight out of a script, and each row carries its own +duration. + +```csv +label,mean_u,frequency_hz,depth_a,duration_s,settle_s,role +floor,0.50,10,0.02,20,3,background +windows,0.50,10,2.00,20,3,pilot +ladder,0.40,1,0.80,40,4, +ladder,0.40,200,0.80,10,2, +``` + +Required: `mean_u`, `frequency_hz`, `depth_a`. Optional: `duration_s` +(default 10), `settle_s` (default 2), `role` (`normal`/`pilot`/`background`), +`label`. Columns are located by header name, `#` comments and blank lines are +skipped, a blank cell falls back to the default, and an error names the file +line number. + +Two capabilities follow from the row form: + +- **A different duration per recording** — a 1 Hz point needs 40 s of cycles + and a 200 Hz point does not. +- **A `role` column**, so a file carries its own background floor and pilot and + then the points scored against them: a complete measurement rather than one + that needs two button presses first. + +**TOML — blocks and ranges**, kept for a dense regular sweep: + +```toml +[defaults] +duration_s = 10 +settle_s = 2.0 + +[[block]] +name = "frequency-ladder" +mean_u = [0.3, 0.6] +frequency_hz = { min = 1.0, max = 200.0, points = 6, spacing = "log" } +depth_a = 0.8 +duration_s = 20 +``` + +Each axis takes a single value, an explicit list, or a `{ min, max, points }` +range with `linear` (default) or `log` spacing; a block records the product of +its three. + +- **`mean_u` is the `I_k` axis** — the normalized cycle-mean lobe point, driven + by the new `ModulationCommandV1::SetOperatingPoint`. Dimensionless, not + physical flux, but the one control that moves the mean illumination without + touching the depth. +- **Points run `ū` outermost, then `f`, then `a`** — the order of how expensive + each change is to settle. Any other nesting spends the run settling. +- **All three axes are commanded at every point,** and the point waits for all + three acknowledgements before recording. A point that inherited an axis from + its predecessor would be recorded under parameters the file does not name. +- **The file's `duration_s` wins** over the panel's, or the survey would not be + reproducible from the protocol alone. +- **Validated up front**: ranges, bounds, the `MAX_POINTS = 4096` product limit + and the same whole-cycle window check the ladder makes against its lowest + frequency — all on the button press, before the drive moves. The point count + and expected bench time are reported first. +- **A refused point is skipped, not fatal**, carrying the modulation owner's own + wording. Because the per-point message is overwritten within the same tick, + the reasons are kept on the run and shown in the status pane and the closing + summary. + +One lease covers the whole file. `plugins/stage-a-a1/protocols/example.toml` is +a commented file to copy. + +### Bench conditions on every run (ADR 022) + +Every recording — normal, pilot, background, sweep point, a₀ point — also +records what the camera measures about itself, from the host's +`CTX_SENSOR_MONITORING`: + +| quantity | sidecar `[sensor]` | recorder metadata | +|---|---|---| +| die temperature, °C | `temperature_c` | `sensor_temperature_c` | +| pixel dead time (refractory period), µs | `pixel_dead_time_us` | `sensor_pixel_dead_time_us` | +| scene illumination, lux | `illumination_lux` | `sensor_illumination_lux` | +| staleness of the reading, s | `reading_age_s` | `sensor_reading_age_s` | +| absolute bias codes | `bias_diff_on/_off/_fo/_hpf/_refr` | — | + +All three bear directly on `q_p(a, f)`: the dead time caps events per pixel per +half-cycle, the lux *is* the physical `I_k` axis, and temperature moves the +biases. The values are **frozen when the recording starts** (they drift, and the +sidecar is written at finalize), mirrored even with Live analysis off, and are +**provenance only** — no A1 result depends on them, or a live run would disagree +with an offline re-run of the same data. A quantity the sensor cannot report is +**omitted**, never written as `0`; replay and cameras without a monitoring block +produce no `[sensor]` section at all. + For manual recordings A1 never drives the Teensy: set the drive (high `a` for the pilot, `a≈0` for the background) in the modulation plugin, then press the matching button — the recording captures whatever `a` is currently set. @@ -102,19 +342,35 @@ the photodiode Data directory no longer have to be kept aligned by hand: A rename on one volume, a size-verified copy across volumes. A file that cannot be moved stays where it is and the sidecar points at it there. -**A1 config sidecar** captures: `measurement_id`, physical `flux_point_id`, file +- **The host's sensor telemetry is compacted in on the way.** The host writes a + wide `.sensor-monitoring.csv` beside the RAW; A1 rewrites it + column-wise as `.sensor.json` in the measurement folder and removes the + original. One `{ t_us, value }` pair of arrays per channel, carrying only the + polls where that channel was actually read — the channels sample on different + schedules, so a row-per-poll table is padding by construction. Bias codes are + dropped: the camera's own bias sidecar already carries them. Nothing is + resampled or aligned, failed polls are kept as `faults`, and the whole path is + best-effort — a source with no monitoring block simply produces no file + (ADR 028). + +**A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the acknowledged snapshot (frequency, center/amplitude DAC, waveform, transfer -`calibration_id`, optical target, requested and resolved normalized mean `ū`, -internal `u_g`/`u_c`, requested `a`, `V_null` and `Vπ`); the +`calibration_id`, optical target, requested and resolved normalized mean `ū`, +internal `u_g`/`u_c`, requested `a`, `V_null` and `V_peak`); the depth this run +was driven and judged by with its provenance (`depth_a`, `depth_a_source`); the photodiode-measured `a`, extrema, geometric pedestal, -headroom, clip fractions, dark/ADC ids, and named dark-corrected `I_tot` anchor; ROI + -masked-pixel count + `N_valid`; trigger info (marker-anchored, marker count, -measured period); and the resolved paths of the RAW (+ its camera-config sidecar) -and the PDQ (+ its sidecar). The **pilot** run additionally records the frozen -ON/OFF windows and the **background** run the floor `q0`, so returning to a -measurement (folder + id) auto-reloads them for the `q_p` plot. +headroom, clip fractions, ADC id, and the learned `I_tot` anchor with its +provenance (ADR 024); ROI + +masked-pixel count + `N_valid`; the `[sensor]` bench conditions (die temperature, +pixel dead time, illumination — ADR 022); +trigger info (marker-anchored, marker count, +measured period); and the resolved paths of the RAW (+ its camera-config +sidecar), the PDQ (+ its sidecar) and the `sensor_readout`. The **pilot** run +additionally records the frozen ON/OFF windows and the **background** run the +floor `q0`, so returning to a measurement (folder + id) auto-reloads them for +the `q_p` plot. **Mechanism.** A small control-plane state machine in `process_control` starts the host camera recorder first and waits for its receipt. Only after the host @@ -164,6 +420,12 @@ Both fold the camera event stream on `T` (from the firmware phase-0 `EXT_TRIGGER marker spacing, which *defines* the frequency; the modulation acknowledged waveform is the only fallback). Enable **Live analysis** to keep them updating. +With **Live analysis** off nothing is ingested at all — no events *and* no +phase-0 markers — so the status line says so by name rather than reporting +`0 events; free-running (no EXT_TRIGGER)`, which reads as a wiring fault. The +frequency ladder refuses on the marker count and distinguishes the two cases in +its message (ADR 017). + Marker hygiene: preview windows overlap, so the same trigger edge arrives on several consecutive frames — the marker buffer is sorted and deduplicated on every merge (duplicates used to fail marker validation and blank the plots). @@ -241,8 +503,9 @@ boundary is A1's own pipeline restart and only the event fold resets (ADR 015). | camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()`, trimmed to the same window | | phase-0 markers | rising `frame.external_triggers()` — the host **banks trigger edges from dropped preview frames** into the next processed frame (drain-to-newest and the preview throttle drop whole frames; at low modulation frequencies the survivors alone rarely held 2 markers inside the analysis window) | | modulation period `T` | measured from the `EXT_TRIGGER` marker spacing; else the modulation plugin's acknowledged waveform — which, since the board-echo fallback, includes the **operator-armed UI drive**, not only service-path (leased) targets | -| optical modulation depth `a` | fresh photodiode optical summary (`measured_log_contrast`) from complete marker-bounded cycles and a confirmed `I_tot` anchor — always the *excitation* contrast, independent of display mode (ADR 012) | +| optical modulation depth `a` | per the **Depth `a` source** setting (ADR 020). *Photodiode* (default): fresh optical summary (`measured_log_contrast`) from complete marker-bounded cycles and a confirmed `I_tot` anchor — always the *excitation* contrast, independent of display mode (ADR 012); when absent, `optical_unavailable` from the same snapshot carries the owner's refusal reason (ADR 017), and A1 appends the way past it. *Commanded*: the modulation owner's `optical_drive.depth_a_milli`, published only for a calibrated optical drive — open loop, tagged as such everywhere it is recorded | | ROI, masked pixels | augur-rs camera config (`CTX_GLOBAL_SETTINGS`) | +| die temperature, pixel dead time, illumination, bias codes | host `CTX_SENSOR_MONITORING` (`SensorMonitoringV1`), mirrored every frame regardless of Live analysis and frozen at recording start. Provenance only — absent on replay, imports and cameras without a monitoring block (ADR 022) | ## Tests @@ -262,3 +525,26 @@ mid-run keeps the camera recording for the full duration, names the cause in the closing message, and still gathers the RAW and its bias sidecar into the measurement folder; and a self-inflicted `SourceChanged` during a recording keeps the row's response points and pilot windows while still resetting the event fold. + +Four more cover the depth source (ADR 020): a withheld photodiode `a` keeps the +owner's own reason *and* names the setting that gets past it; the commanded +source reports a depth with no photodiode present at all, and refuses a drive +that is not a calibrated optical one; and both the recorder metadata and the +config sidecar carry `depth_a_source` on every run, with `measured_a` present +only when something actually measured it. + +Three cover the simplified ladder (ADR 021): `Find a₀` refuses to search for a +depth it is commanding and takes no lease doing so; an a₀ point is armed with no +stored lock and `trials: 0`; and the whole ladder runs to `3/3 points recorded` +with no photodiode `a`, **no camera trigger markers** and an empty lock table, +panicking if it ever enters the search phase. + +Two cover the bench conditions (ADR 022): the start-of-run snapshot wins over a +drifted live reading and reaches both the metadata and the sidecar's `[sensor]` +section; and a quantity the sensor cannot report is omitted rather than written +as a zero. + +Two cover the nested sweep (ADR 023): the whole 2 × 3 block records every depth +at every frequency in depth order, on exactly **one** lease acquisition, never +entering the search phase and finishing with `2/2 frequencies × 3 depths`; and a +nested point's file stem carries both axes (`…_f50Hz_p03`). diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index aed1ff0..d605647 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -10,7 +10,8 @@ Laser-modulation control for the Stage-A bench with two orthogonal axes: - **Drive method** defines the DAC operating band. `MANUAL` uses Power + Min threshold; - `CALIBRATED` derives it from `V_null`, `Vπ`, normalized cycle mean `ū`, and optical depth `a`. + `CALIBRATED` derives it from the lobe endpoints `V_null`/`V_peak`, the normalized cycle + mean `ū`, and the optical depth `a`. - **Mode** defines the shape that fills the band: `CONST`, `DAC_SINE`, `SQUARE`, `OPTICAL_LOG_SINE`, or `OPTICAL_LINEAR_SINE`. All five remain available under both methods. @@ -18,7 +19,7 @@ The always-visible **max limit** is the hard DAC ceiling for every manual and ca The settings schema shows only the selected method's parameter block and refreshes when Method changes; Manual is the default. -| Mode | Manual band `[min, power]` | Calibrated band from `ū`, `a`, `V_null`, `Vπ` | +| Mode | Manual band `[min, power]` | Calibrated band from `ū`, `a`, `V_null`, `V_peak` | |---|---|---| | `CONST` | hold `power` | hold the DAC code for `ū` | | `DAC_SINE` | DAC sine across the band | DAC sine across the band | @@ -26,14 +27,34 @@ changes; Manual is the default. | `OPTICAL_LOG_SINE` | intensity log-sine across the band | mean `ū`, converted to `u_g=ū/I_0(a/2)` | | `OPTICAL_LINEAR_SINE` | intensity linear-sine across the band | centre/mean `u_c=ū` | -Manual optical modes reuse the persisted `V_null`/`Vπ` lobe parameters and derive effective +Manual optical modes reuse the persisted `V_null`/`V_peak` lobe parameters and derive effective `(u, a)` from the manual DAC band through the forward `sin²` transfer. Both optical modes then use the same inversion path described in [Optical waveform drive](./stage-a-optical-waveform.md). `ū` is dimensionless and must not be confused with physical cycle-mean A1 flux `I_k`. -`V_null`/`Vπ` are measured, not typed: the Calibration section sweeps settled `CONST` codes +`V_null`/`V_peak` are measured, not typed: the Calibration section sweeps settled `CONST` codes against the photodiode and fits the lobe — see -[Pockels transfer calibration](./stage-a-pockels-calibration.md). +[Pockels transfer calibration](./stage-a-pockels-calibration.md). Both are **absolute DAC codes** +an operator can point at on the transfer curve; the half-wave span between them is derived and +never entered, and `Vπ` no longer appears anywhere the operator sets something (ADR 025). + +## Achievable ranges — settings clamp, they never refuse + +`ū` and `a` are coupled through one constraint: the peak of the swing has to stay under the top of +the lobe and under the max limit. `waveform::PeakLaw` names how the peak follows from the two, one +variant per mode (`Constant`, `LogSwing` for the DAC sine/square, `LogSine`, `LinearSine`), and +solving it for one variable at a time gives the achievable range. + +Edits **clamp into that range**; nothing reverts. Only the control the operator just touched is +limited — dragging `a` up means "more depth", so `a` is what stops and `ū` stays put — and a lobe, +ceiling or mode change settles brightness first, depth second. Modes and methods are always +accepted. + +Both bounds are live in the control labels (`Optical depth a (0..1.37 at ū=0.50)`) and on the +status line, along with where the current drive actually peaks. An un-sendable drive is reported as +`drive not sent: …` rather than blocking the edit. Previously a leftover `a` made an optical mode +simply unselectable, with an error naming a control the operator was not editing — see +[ADR 025](../adr/025-stage-a-drive-settings-clamp-not-refuse.md). Every accepted setting change is transferred to the Teensy **immediately** as one `MOD` command — no Apply button, no experiment state machine. The panel shows the modulation and live DAC code the @@ -56,8 +77,10 @@ board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded val - Status and commanded summaries include Method and the resolved `(lo, hi, hold)` DAC band. - `ModulationStateV1.optical_drive` publishes the exact resolved optical target, requested and resolved normalized mean `ū`, internal `u_g`/`u_c`, - requested `a`, `V_null`, and `Vπ` as an additive V1 field; A1 sidecars no - longer have to infer these from DAC endpoints. + requested `a`, `V_null` and `V_peak` as an additive V1 field; A1 sidecars no + longer have to infer these from DAC endpoints. `v_peak_dac` replaced the + earlier `v_pi_dac`, and carries the absolute peak code rather than the span + (ADR 016, ADR 025). - `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. - The workflow-owner service and `WaveformV1` automation path remain exact-waveform contracts and do not use the UI Drive method. @@ -71,9 +94,19 @@ board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded val auto-reconnects with a 2 s backoff while `connect` stays requested. Previously a wedged or dead link silently swallowed every queued command — the UI kept accepting mode changes while the board held the old waveform. -- **`protocol_run` forwarding**: the UI mirror records the request and the settings snapshot - starts/stops the protocol on the live worker (which owns the device link); only value - *transitions* act, so re-applied snapshots cannot restart a finished protocol. +- **`SetOperatingPoint`** (ADR 027): the leased counterpart for the *operating point* `ū` — the + third axis, alongside depth and frequency, and the one that moves the mean illumination without + touching the depth. Calibrated method only; the owner parks the operator's own `ū` on the first + retarget and restores it when the lease ends. Used by the A1 protocol runner's `I_k` axis. + Unlike an interactive edit it **refuses** rather than clamping: a protocol asked for a specific + brightness, and quietly recording a different one would put the wrong `ū` in every sidecar. +- **The applied lobe crosses to the UI mirror** (ADR 026): "Apply to V_null / V_peak" used to do + nothing, because the fit lives on the live worker while the settings snapshot is collected from + the mirror — so the mirror's stale codes overwrote the applied ones on the next sync. The applied + lobe is now published through a process-global generation the mirror adopts. +- **No protocol section.** The undocumented TOML `MOD`-step runner was removed; declarative + recording protocols belong to the A1 plugin, which can also record what they produce + ([ADR 027](../adr/027-stage-a-a1-declarative-protocols.md)). - **Board-echo `acknowledged` fallback**: the published `ModulationStateV1.acknowledged` now falls back to a revision-0 target built from the board's `MOD`/`STATUS` echo (`mod_wave`, `mod_level`, `mod_min`, `mod_freq_mhz`) when no service-path acknowledgement exists. UI-driven drives never diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index a576df8..14eca52 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -3,6 +3,15 @@ - **Crate:** `plugins/stage-a-photodiode` (`augur-plugin-stage-a-photodiode`) - **Firmware:** `stage-a-controller` 0.4.0+ (`PDSTREAM_PDA1`), Teensy **stream port** (second CDC port) - **Status:** Active (2026-07-16) — replaces the readout half of `stage-a-monitor` +- **Design:** [ADR 006](../adr/006-stage-a-two-plugin-split.md) (the split), + [ADR 012](../adr/012-stage-a-contrast-geometry-is-bench-not-display.md) (the + contrast geometry), + [ADR 024](../adr/024-stage-a-photodiode-learns-its-own-anchor.md) (the + learned total-power anchor; dark cancels), + [ADR 017](../adr/017-stage-a-rail-detection-and-withheld-a-reasons.md) + (span-relative rail detection; the published refusal reason), + [ADR 019](../adr/019-stage-a-calibration-measures-its-own-window.md) (the + published level owns its window) ## What it is @@ -33,38 +42,74 @@ never changes a published quantity (ADR 012). - **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). - **EXCITATION** — the diode sits at the PBS reject port and measures the light - removed from the sample beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set - reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. + removed from the sample beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the learned + total-power anchor: `I_exc = I_tot − I_pd`. Nothing to enter — see below. ## Optical log-contrast `a` `measured_log_contrast` in the published `PhotodiodeOpticalSummaryV1` is **always** the excitation contrast `a = ln(I_exc,max / I_exc,min)`, in **both** display modes. The detector sits behind the PBS reject port and measures the complement — that is a property of the bench, not of the display — -so the estimator always runs the `RejectedComplement` geometry against `reference_volts`. A1's +so the estimator always runs the `RejectedComplement` geometry against the learned anchor. A1's amplitude sweep settles on this value, so a display toggle must not be able to move it (ADR 012). -- **Reference I_tot** (`reference_volts`) is the total-power anchor: the PD reading with the full - beam diverted into the diode. The input accepts 1 µV steps (six decimal places in volts), which - covers the usual 0.0005–0.015 V detector range. A non-empty **anchor id** and explicit - **measured and current** confirmation are required. Changing the value or id - clears confirmation; until all three agree, `a` is withheld. -- **Dark level** (`dark_volts`) + the **Capture dark** button: block the beam and press; the mean of - every sample currently retained in the monitor cache becomes the dark level. This is not a new - fixed-duration acquisition and it does not measure or modify `I_tot`: after blocking the beam, - wait at least one configured cache duration so earlier illuminated samples have aged out. The - dark input also accepts 1 µV steps for a manual correction. The value is applied to the detector - samples *and* to the `I_tot` anchor, so it cancels out of the complement rather than biasing `a` - — its job is to keep the two sides consistent and to record the calibration the reading was - taken under. `dark_id` in the sidecar reads `dark-measured` or `dark-none` accordingly. +- **`I_tot` is learned, not entered** (ADR 024). The plugin latches the highest + smoothed detector level it has seen since the port was opened. On the reject + port the detector is brightest exactly where the excitation is extinguished, + so that reading *is* `I_tot` — and the Pockels transfer sweep, which walks the + DAC across the whole lobe, lands on the excitation null by construction. Run + the sweep once and the anchor is right. The latch is over completed 64-sample + summary-cell means, so one noise spike cannot pin it high, and it survives + segment restarts (a rate change or an acquisition handover does not move the + optics). Reconnecting the port relearns it. Provenance is published as + `anchor_id = "observed-peak@"`. +- **There is no dark level, and that is exact, not an approximation.** With a DC + dark offset `D`, the excitation is `(I_tot,obs − D) − (v − D) = I_tot,obs − v` + — the offset cancels, because both sides are readings from the same + DC-coupled detector. `dark_volts` is fixed at 0 and `dark_id` reads + `dark-cancels`. Two unit tests hold this down: one asserts that shifting the + whole trace *and* the anchor leaves `a` unchanged to 1e-9, and a companion + asserts that correcting only one side *does* move it, so the first cannot pass + vacuously. - The estimator uses only marker-bounded windows containing at least **two complete modulation cycles**, ending on phase 0. It no longer estimates extrema from an arbitrary trailing sample count; a low-frequency trace that does not fit the bounded window is withheld rather than phase biased. -- The estimator is **fail-closed**: it refuses on a missing/unconfirmed anchor, - incomplete cycles, ADC clipping, no headroom above dark, and when the anchor - is not above the measured signal. A refusal is shown as `a unavailable: ` +- The estimator is **fail-closed**: it refuses when no anchor has been observed + yet, on incomplete cycles, on ADC clipping, and when the excitation never dims + below the brightest the detector has been — where there is no complement left + to take a contrast of, and the fix is to run the transfer sweep. A refusal is shown as `a unavailable: ` rather than a missing row — a wrong `a` is worse than no `a`. +- The refusal reason is also **published** on the contract as + `PhotodiodeSummaryV1::optical_unavailable`, so a consumer that gates on `a` + (A1's a₀ lock, amplitude sweep and frequency ladder) can name the gate rather + than report absence. Set exactly when `optical_summary` is absent and a window + existed to judge (ADR 017). +- Clip detection is **span-relative**: the near-rail margin is capped at 5 % of + the window's own peak-to-peak code range. At this detector's 0.5–15 mV + operating range the whole waveform sits inside the bottom ~20 of 4095 codes, + where the former absolute 4-code margin classified 30 % of a clean sine as + clipped and withheld `a` unconditionally. The rails themselves (code 0, full + scale) stay guarded at every gain, so a waveform driven below zero is still + refused (ADR 017). +- The same span-relative margin decides `PhotodiodeLevelV1::clipped`, so the + Pockels sweep is not told that a detector running a few codes above zero is + truncating. + +## The published level owns its window + +`PhotodiodeStreamV1.level` is the settled detector reading other plugins consume +— today, the modulation plugin's Pockels transfer sweep, which reads one per +commanded DAC code. It is averaged over a **fixed 20 ms**, set here and +independent of every display setting; `sample_count` reports what it was. + +It used to be averaged over the chart's moving-average window below. That made a +display preference set the precision of a physical calibration: at the bench's +500 kSa/s the default of four samples published **8 µs** of signal per settled +code, and a clean Pockels curve came back reported as a 22 % residual with 26 % +"hysteresis" (ADR 019). 20 ms is one mains period, so the boxcar has a null at +50 Hz and every harmonic of it — and the chart's averaging is once again nothing +but a chart setting. ## Chart diff --git a/docs/features/stage-a-pockels-calibration.md b/docs/features/stage-a-pockels-calibration.md index 39d2408..b3f2509 100644 --- a/docs/features/stage-a-pockels-calibration.md +++ b/docs/features/stage-a-pockels-calibration.md @@ -5,7 +5,9 @@ - **Status:** built - **ADR:** [ADR 011](../adr/011-stage-a-pockels-transfer-calibration.md), [ADR 016](../adr/016-stage-a-lobe-endpoints-not-a-distance.md) (what the two - settings ask for) + settings ask for), + [ADR 019](../adr/019-stage-a-calibration-measures-its-own-window.md) (the + measurement window, and judging the sweep against its own noise) - **Knowledge base:** `methodology/pockels-waveform-linearisation.md` §4, `setup/optical-path.md` @@ -81,12 +83,27 @@ Several nulls are valid when a sweep spans multiple lobes; the fit reports the **lowest** one whose `[V_null, V_null + Vπ]` fits inside the max limit — least voltage across the crystal, most headroom, and a rule the operator can predict. -## Settling is proven, not timed +## Each point is a measurement the sweep controls -Every published level carries `end_sample_index` and `sample_count` on the -device sample clock. A point is accepted only from a window that *began* at -least `SETTLE_SAMPLES` (2 000 ≈ 100 ms at 20 kSa/s) after its code was -commanded. No shared wall clock, no sleeps, immune to control-tick jitter. +Two quantities, both owned deliberately and neither borrowed from a display +setting (ADR 019): + +**Settling is proven, not timed.** Every published level carries +`end_sample_index` and `sample_count` on the device sample clock. A point is +accepted only from a window that *began* at least `SETTLE_SECONDS` (0.1 s, +converted through the photodiode's published sample rate) after its code was +commanded. No shared wall clock, no sleeps, immune to control-tick jitter. A +duration and not a sample count, because settling is a property of the HV +amplifier and the crystal: the former fixed 2 000 samples was written for +20 kSa/s and silently became 4 ms when the bench moved to 500 kSa/s. + +**The level is averaged over 20 ms**, fixed by the photodiode plugin and +independent of its chart-averaging setting. That setting used to decide it, at a +default of four samples — 8 µs at 500 kSa/s — which is how a clean bench lobe +came back with a 22 % residual. 20 ms is one mains period, so the boxcar nulls +50 Hz and its harmonics. + +A point therefore costs ~120 ms, and the full 98-point sweep ~12 s. ## The sweep owns the DAC while it runs @@ -142,14 +159,32 @@ bad reason. A residual that stays high after rejection, with a visibly poor overlay, is the real signal — and as the last row shows, it comes with a `Vπ` that is wrong in a way the plot makes obvious. -There is deliberately no absolute minimum voltage. The earlier implementation -rejected every detector span below **10 mV**, while the real Stage-A -photodiode commonly reads only about **0.5–15 mV**. The fit now compares the -between-code sweep span with the median `peak_to_peak_volts` measured inside -the settled CONST windows. A repeatable millivolt-scale lobe is accepted; a -putative lobe no larger than the detector's own typical within-window -excursion is rejected as unresolved. The regression suite includes a 4 mV -transfer that the old threshold always refused. +One real bench sweep is kept as a fixture at +`plugins/stage-a-modulation/testdata/pockels-20260730-083123.json` and asserted +against directly. Synthetic sweeps carry uniform noise; a real detector's is +signal-proportional, and every metric that broke on that record was one compared +against zero (ADR 019). It is worth keeping for the same reason the table above +is: it is what the failure actually looked like. + +There is deliberately no absolute minimum voltage. An early implementation +rejected every detector span below **10 mV**, while the real Stage-A photodiode +commonly reads only about **0.5–15 mV**. Its replacement — the between-code span +against the median `peak_to_peak_volts` — was scale-free but still wrong: that +compares a span of *means* to a *raw within-window excursion*, so it tightens as +the averaging window grows, and it cleared a real bench sweep by only a factor of +1.9. + +Both gates are now measured against the fit's **own RMS residual**, the scatter +of the averaged points about the curve — the same quantity the lobe amplitude is +in, so the comparison is dimensionally honest and cannot be moved by how the +photodiode owner happens to average (ADR 019). + +A lobe counts as resolved when it stands at **twice its own scatter** +(`rms < 0.5·|span|`). The margin is not decoration: a free period search over +pure noise returns an apparent lobe, not zero, landing noise-only quality at +0.7–1.0 — while the noisiest real record on file reads 0.22. The regression suite +pins both ends, and includes a 4 mV transfer that the old absolute threshold +always refused. The fit is **never** applied automatically, and applying re-validates the resulting drive: a calibration that cannot be armed is rolled back rather than @@ -159,8 +194,26 @@ stored. Warnings surface as `Check:` lines in the status: |---|---| | residual > 5 % of the span | compare fit and points in the plot before trusting `Vπ` | | points dropped | a couple is ordinary; a large share means the sweep is the problem | -| hysteresis > 5 % | the cell is drifting, or the settle time is too short | -| clipped points | the extremum they sit on is not where the fit thinks it is | +| hysteresis past its noise floor | the cell is drifting, or the settle time is too short | +| points at an end of the detector's range | the reported extrema are truncated; `V_null`/`Vπ` are not | + +Two of those are stated carefully, because the obvious versions are wrong. + +**Hysteresis is compared against noise, not against zero.** Two independently +noisy passes over one curve already differ by `1.128 σ` on average, so a bare 5 % +cut fires on any bench whose points are not far quieter than that — and it did, +on a drift-free cell. The metric is judged against `1.128 · rms / |span|`, the +value it takes under noise alone. That ratio runs between two derivable ends: +**1.0** for pure noise and **1.77** for pure drift, because a systematic offset +inflates the residual as well. The range is narrow and worth knowing — the +obvious "warn at 2× the floor" sits above both ends and never fires. The cut is +at 1.33. + +**Clipping is a caveat on the extrema, not a verdict on the lobe.** Rail-touching +points truncate `detector_volts_at_null`/`_at_peak` and the `I_tot` lower bound; +`V_null` and `Vπ` come from the shape and barely move. The advice is to change +the detector **gain** — for a reject-port detector it is the dark end that +reaches the bottom rail, so adding attenuation is backwards. There is no separate "lobe coverage" gate: `fit_transfer` already refuses a sweep in which no full lobe fits inside the commandable range, so `Vπ` is always diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 4f5932a..193dbd9 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -10,12 +10,35 @@ per amplitude: it leases the modulation owner, retargets the armed calibrated dr point (`…_pNN`). Outside the leased sweep A1 owns no hardware and never drives the Teensy — arm the optical drive in the modulation plugin; A1 only reads its published settings. +## Where `a` comes from + +**Depth a source** picks what every depth-dependent path — the sweep, **Find a₀**, the frequency +ladder, the response curve — reads as `a`: + +- **photodiode (measured)** — the default and the source of record. Fail-closed: the photodiode + publishes no `a` unless firmware phase-0 markers on its stream port prove the estimator window + covers whole modulation cycles. If those markers never arrive (no trigger, or firmware that does + not stamp them) it refuses forever — *"0 trigger(s) in the last 3446784 samples"* is a missing + marker stream, not a short window, and no setting fixes it. +- **modulation drive (commanded, open loop)** — the depth the modulation owner's calibrated drive is + commanding (`optical_drive.depth_a_milli`). Still a calibrated number, inverted from the measured + `V_null`/`V_peak` curve, but **not checked against the light**: it carries the calibration's error plus + any drift since. Needs an applied calibration and `OPTICAL_LOG_SINE` armed; a manual DAC band + publishes no optical drive and the gates refuse rather than inventing a depth. + +Open loop there is **nothing to search for**, so `Find a₀` is not used and the frequency ladder skips +it (see below), and the photodiode's window-length and clipping checks are skipped because neither +bounds a commanded depth. Every artefact records the source: `depth_a_source`/`depth_a` in the sidecar, +`depth_source` in `[a0_lock]` and in `a0_locks.json`, and the *a from* column of the a₀ lock view. +`measured_a` stays reserved for a number the photodiode actually measured. See +[ADR 020](../../docs/adr/020-stage-a-a1-depth-source.md). + ## Recording - **Output folder** — where the A1 config sidecar is written (recommended shared experiment root). + The only field that has to be filled in before recording. - **Measurement id** — one per `(I_k, f)` pair; auto-generated default, editable, or press **New id**. -- **Physical I_k flux point id** — required canonical id of the cycle-mean local flux - calibration/map point; separate from normalized modulation mean `ū`. + Optional: a blank field is filled in on the first recording and written back (ADR 018). - **Duration (s)** — each recording auto-stops and finalizes after this. - **Start recording** — starts camera RAW, then connects/leases the photodiode and starts PDQ; the timer begins after both acknowledge. It auto-finalizes PDQ first, camera second, then writes @@ -45,17 +68,150 @@ measurement, not calculated. - **Record a₀ point (event-count)** — re-applies the locked depth under the lease (so the amplitude cannot change during the recorded interval), waits for the measured `a` to hold `a₀`, and records one atomic frequency point named `…_ec_fHz` with an `[a0_lock]` sidecar section. -- **Clear a₀ lock table** — after changing the flux point, the calibration or `a₀` itself. +- **Clear a₀ lock table** — after changing the illumination, the calibration or `a₀` itself. Frequency order, the interleaved low-frequency reference and the repeated blocks stay yours — every point is one button press. +### With the commanded depth source there is no search + +Everything above describes the *measured* workflow. `Find a₀` exists only because a **measured** `a₀` +has to be re-found per frequency against the static inversion's roll-off. A **commanded** depth is +the number being commanded, so the correction ratio is exactly 1 and a search would command `a₀`, +read back `a₀` and stop. It is therefore not run at all (ADR 021): + +| | photodiode (measured) | modulation drive (commanded) | +|---|---|---| +| `Find a₀` | trims per frequency, stores a lock | **disabled** — says why | +| `Record a₀ point` | replays the stored lock | commands `a₀` directly | +| ladder per rung | set `f` → confirm via **camera markers** → search → record | set `f` → confirm via the **modulation owner's ACK** → record | +| `a0_locks.json` | one row per frequency | untouched | +| needs EXT_TRIGGER + Live analysis | **yes** | no | + +So open loop the whole workflow is: set `a₀`, press **Record all frequencies**. The trade is the one +the lock removes — nothing verifies the light reached `a₀`, and the roll-off is real — so switch back +to the photodiode once its markers work. + +## Depth sweep at every frequency (the `q_p(a, f)` surface) + +The frequency ladder is an **outer loop**; what it records per rung is a mode: + +| button | per frequency | produces | +|---|---|---| +| **Record all frequencies** | one event-count point at `a₀` | `q_p(a₀, f)` | +| **Record depth sweep at every frequency** | the whole `[min a, max a]` sweep | `q_p(a, f)` — a curve per `f` | + +The second runs the block `a50(f)` is fitted from, unattended: `frequency points × +depth points` recordings on **one lease**, so the drive cannot move between rungs. +It adds no new settings — the depth axis is `Sweep min a`/`max a`/`points` from +**Recording**, the frequency axis is `Sweep min f`/`max f`/`points`/order/seed from +the a₀ section — and reuses the ladder's ordering, reference repeats, +per-frequency confirmation and skip-and-report unchanged. + +No `a₀` and no **Find a₀** are involved in either depth source: a depth sweep +commands and settles every `a` itself. Points are named `…_fHz_pNN`. A rung +counts as done only when its inner sweep recorded every point. See +[ADR 023](../../docs/adr/023-stage-a-a1-nested-depth-frequency-sweep.md). + +## Bench conditions on every run + +Every recording, in every mode, also records what the camera measures about itself (host +`CTX_SENSOR_MONITORING`): die **temperature** (°C), pixel **dead time / refractory period** (µs), +scene **illumination** (lux), the reading's age, and the absolute bias codes. They land in the +sidecar's `[sensor]` section and in both recorders' metadata as `sensor_*`. + +Frozen when the recording starts (they drift), mirrored even with Live analysis off, and provenance +only — no result depends on them. A quantity the sensor cannot report is **omitted, never `0`**; +replay and cameras without a monitoring block produce no `[sensor]` section at all. See +[ADR 022](../../docs/adr/022-stage-a-a1-sensor-conditions-on-every-run.md). + +The host also polls those quantities for the *whole* recording and writes a wide +`.sensor-monitoring.csv` beside the RAW. A1 gathers it into the measurement folder as +`.sensor.json`, rewritten column-wise — one `{ t_us, value }` pair of arrays per channel, +carrying only the polls where that channel was read. The channels are sampled on different +schedules, so a row-per-poll table is padding by construction; the bias columns are dropped because +the camera's own bias sidecar already carries them. Named in the sidecar's `[files]` block as +`sensor_readout`. See +[ADR 028](../../docs/adr/028-stage-a-sensor-readout-travels-with-the-measurement.md). + Files share an `_` stem: `/_.raw` (camera, under the host output root), -`/__pd.pdq` + `.json` (photodiode, under its data root), and +`/__pd.pdq` + `.json` (photodiode, under its data root), +`/_.sensor.json` (sensor readout), and `/__config.toml` (A1, under the chosen folder). Point all three roots at the same experiment directory to co-locate everything. The host also writes its own `.toml` next to the RAW with the camera biases/ROI; the A1 sidecar cross-references it. +## Protocol — run a survey from a file + +The four sweep buttons each move one axis and leave the others wherever they are. A **protocol** +names every axis for every recording instead, in a file that travels with the results. The reader +is chosen by extension. + +### CSV — one row per recording (the one to reach for) + +```csv +label,mean_u,frequency_hz,depth_a,duration_s,settle_s,role +floor,0.50,10,0.02,20,3,background +windows,0.50,10,2.00,20,3,pilot +ladder,0.40,1,0.80,40,4, +ladder,0.40,200,0.80,10,2, +``` + +| column | | | +|---|---|---| +| `mean_u` | required | normalized cycle-mean lobe point `ū` — the brightness (`I_k`) axis, 0.01–1.0 | +| `frequency_hz` | required | 0.01–2000 | +| `depth_a` | required | `a = ln(I_max/I_min)`, 0.01–6 | +| `duration_s` | optional, default 10 | seconds for **this** row, 1–3600 | +| `settle_s` | optional, default 2 | dwell after retargeting, 0–60 | +| `role` | optional, default `normal` | `normal`, `pilot` or `background` | +| `label` | optional | free text for the status line and sidecar; quote it if it contains a comma | + +Columns are found **by name**, so their order does not matter and one can be left out entirely. +Blank lines and `#` comments are skipped, and a blank cell falls back to the default. Errors carry +the **file line number**, which is what your editor and spreadsheet both show. + +Two things the row form gives you that blocks cannot without one block per value: **a different +duration per row** (1 Hz needs 40 s of cycles, 200 Hz does not), and **a `role` column**, so a file +can open with its own background floor and pilot and then record the points scored against them — +a complete measurement, not one that needs two button presses first. + +### TOML — blocks and ranges + +Kept for a dense regular sweep, which a 96-row CSV states badly: + +```toml +[defaults] +duration_s = 10 +settle_s = 2.0 + +[[block]] +name = "frequency-ladder" +mean_u = [0.3, 0.6] +frequency_hz = { min = 1.0, max = 200.0, points = 6, spacing = "log" } +depth_a = 0.8 +duration_s = 20 +``` + +Each axis takes a single value, a list, or a `{ min, max, points }` range (`linear` default, `log` +for per-decade ladders); a block records the product of its three, `ū` outermost then `f` then `a`, +which settles the slow axis least often. `duration_s`/`settle_s` are per block. + +### Either way + +**`mean_u` is the `I_k` axis** — the normalized cycle-mean lobe point, driven by the new +`SetOperatingPoint` command, and the axis no button could sweep. All three axes are commanded at +every point and the point waits for all three acknowledgements before recording, so nothing is +filed under parameters the file does not state. The file's `duration_s` wins over the panel's. One +lease covers the whole run; a point whose drive the modulation owner refuses is skipped carrying +its wording, and the reasons are kept on the status pane and in the closing summary. The whole file +is validated on the button press, before the drive moves, and the point count and expected bench +time are reported first. Use **Stop** in the Record section to end a run early. + +`protocols/example.csv` and `example.toml` are commented files to copy, installed to +`~/.augur/plugins/stage-a-a1/protocols/`. See +[ADR 027](../../docs/adr/027-stage-a-a1-declarative-protocols.md). + ## Live quicklooks - **Rolling half-period response** `S_p(t) = N_p(t−T/2, t] / N_valid` — events per valid pixel in the @@ -80,6 +236,9 @@ See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the ful [ADR 014](../../docs/adr/014-stage-a-a1-frequency-ladder.md) for the unattended frequency ladder, [ADR 015](../../docs/adr/015-stage-a-a1-recording-robustness.md) for the -recording coordinator's one-folder/full-duration guarantees, and +recording coordinator's one-folder/full-duration guarantees, +[ADR 017](../../docs/adr/017-stage-a-rail-detection-and-withheld-a-reasons.md) +for why an `a₀` gate refused and how Live analysis is distinguished from a +missing trigger, and [docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the planned amplitude-sweep automation on top of this. diff --git a/plugins/stage-a-a1/protocols/example.csv b/plugins/stage-a-a1/protocols/example.csv new file mode 100644 index 0000000..11899f5 --- /dev/null +++ b/plugins/stage-a-a1/protocols/example.csv @@ -0,0 +1,71 @@ +# Stage-A A1 recording protocol — one row per recording. +# +# Point the A1 plugin's "Protocol file" at a file like this and press "Run the +# protocol". Every row is recorded exactly as written, in the order written, so +# the survey is reproducible from this file alone. +# +# Columns are found BY NAME, so their order does not matter and you can drag +# them around in a spreadsheet. Blank lines and lines starting with # are +# skipped. Errors report the line number you see in your editor. +# +# Required columns +# mean_u normalized cycle-mean lobe point ū — the brightness (I_k) +# axis. 0.01..=1.0. 1.0 is the top of the Pockels lobe; the +# reachable maximum shrinks as depth_a grows, and the +# modulation plugin shows the current limit next to each +# control. A row the drive cannot reach is skipped and named, +# and the rest of the file still runs. +# frequency_hz modulation frequency, 0.01..=2000. +# depth_a optical depth a = ln(I_max / I_min), 0.01..=6. +# +# Optional columns (leave the cell blank on any row to take the default) +# duration_s seconds of camera + photodiode for THIS row. 1..=3600, +# default 10. This is the point of the row-per-recording form: +# low frequencies need several cycles, high ones do not. +# settle_s dwell after retargeting, before recording. 0..=60, default 2. +# role normal (default), pilot, or background. +# pilot — bright reference; freezes the ON/OFF windows the +# rest of the measurement is scored in. Record it +# early. +# background— unmodulated reference; gives the false-response +# floor. Put it first, at a very shallow depth. +# label free text, used in the status line and the sidecar so you can +# tell later which part of the survey a file came from. Quote it +# if it contains a comma. +# +# All three axes are commanded for every row, and the row waits for all three to +# be acknowledged before it starts recording — so a file never records under +# parameters it does not state. One modulation lease covers the whole run and +# your own drive settings are handed back at the end. + +label,mean_u,frequency_hz,depth_a,duration_s,settle_s,role + +# --- references first: the whole measurement is scored against these --------- +floor,0.50,10,0.02,20,3,background +windows,0.50,10,2.00,20,3,pilot + +# --- q_p(a) curve at 10 Hz --------------------------------------------------- +curve-10Hz,0.50,10,0.20,10,2, +curve-10Hz,0.50,10,0.45,10,2, +curve-10Hz,0.50,10,0.70,10,2, +curve-10Hz,0.50,10,1.10,10,2, +curve-10Hz,0.50,10,1.60,10,2, + +# --- frequency ladder at one depth ------------------------------------------- +# Longer at the bottom: 1 Hz needs 30 s to cover enough cycles, 200 Hz does not. +ladder,0.40,1,0.80,40,4, +ladder,0.40,5,0.80,25,3, +ladder,0.40,20,0.80,15,2, +ladder,0.40,80,0.80,10,2, +ladder,0.40,200,0.80,10,2, + +# --- brightness series at fixed (f, a) — the I_k axis ------------------------ +# Ascending, so the sensor adapts in one direction only. Longer settle: a change +# in mean illumination is the slowest thing on the bench to settle. +brightness,0.15,50,0.60,15,6, +brightness,0.30,50,0.60,15,6, +brightness,0.45,50,0.60,15,6, +brightness,0.60,50,0.60,15,6, + +# --- a repeat of the first curve point, to expose drift across the run ------- +curve-10Hz-repeat,0.50,10,0.45,10,2, diff --git a/plugins/stage-a-a1/protocols/example.toml b/plugins/stage-a-a1/protocols/example.toml new file mode 100644 index 0000000..1718b5d --- /dev/null +++ b/plugins/stage-a-a1/protocols/example.toml @@ -0,0 +1,92 @@ +# Stage-A A1 recording protocol — a worked example to copy. +# +# Point the A1 plugin's "Protocol file" setting at a file like this and press +# "Run the protocol". Every point is recorded with the parameters written here, +# so the survey is reproducible from this file alone: nothing is taken from +# whatever the modulation plugin happens to have armed. +# +# Three axes, all named for every point: +# +# mean_u the normalized cycle-mean lobe point ū — the brightness (I_k) +# axis. 1.0 is the top of the Pockels lobe; the achievable +# maximum shrinks as the depth grows, and the modulation plugin +# shows the current limit next to each control. +# frequency_hz the modulation frequency. +# depth_a the optical depth a = ln(I_max / I_min). +# +# Each axis takes either +# an explicit list depth_a = [0.5, 1.0, 2.0] +# a single value mean_u = 0.5 +# or a generated range frequency_hz = { min = 1.0, max = 100.0, points = 5 } +# frequency_hz = { min = 1.0, max = 100.0, points = 5, spacing = "log" } +# +# "linear" is the default spacing; use "log" for anything read per decade — +# a Bode ladder is not read per hertz. +# +# A block records the full product of its three axes. Points run mean_u +# outermost, then frequency_hz, then depth_a, because that settles the +# expensive axis least often: moving the brightness makes the sensor re-adapt, +# a new frequency has to be confirmed against the phase-0 trigger, and changing +# the depth is the cheap innermost step. +# +# One modulation lease is held for the whole file, so nothing can move the +# drive underneath the run, and the operator's own settings are handed back at +# the end. A point whose drive the modulation plugin refuses — usually a +# mean_u/depth_a pair that would run off the top of the lobe — is skipped and +# named in the status line rather than stopping the survey. + +name = "a1-example-survey" + +# Applied to every block that does not override them. +[defaults] +duration_s = 10 # seconds of camera + photodiode per recording (1..=3600) +settle_s = 2.0 # dwell after retargeting, before recording starts (0..=60) + + +# --------------------------------------------------------------------------- +# 1. One q_p(a) curve at a single frequency and brightness. +# The everyday depth sweep, written down. +# --------------------------------------------------------------------------- +[[block]] +name = "depth-curve-10Hz" +mean_u = 0.5 +frequency_hz = 10.0 +depth_a = { min = 0.2, max = 2.0, points = 7 } + + +# --------------------------------------------------------------------------- +# 2. The same depth held across a frequency ladder, at two brightnesses. +# Log-spaced, so the ladder is read per decade. +# Longer recordings: the low frequencies need several cycles. +# --------------------------------------------------------------------------- +[[block]] +name = "frequency-ladder" +mean_u = [0.3, 0.6] +frequency_hz = { min = 1.0, max = 200.0, points = 6, spacing = "log" } +depth_a = 0.8 +duration_s = 20 +settle_s = 3.0 + + +# --------------------------------------------------------------------------- +# 3. A small q_p(a, f) surface at one brightness — 3 × 4 = 12 recordings. +# Watch the totals here: the product grows fast, and the plugin reports how +# many recordings and roughly how long the whole file will take before the +# first one starts. +# --------------------------------------------------------------------------- +[[block]] +name = "surface" +mean_u = 0.5 +frequency_hz = { min = 5.0, max = 500.0, points = 3, spacing = "log" } +depth_a = { min = 0.4, max = 1.6, points = 4 } + + +# --------------------------------------------------------------------------- +# 4. Sweeping the brightness itself at a fixed (f, a) — the I_k axis. +# Ascending, so the sensor adapts in one direction only. +# --------------------------------------------------------------------------- +[[block]] +name = "brightness-series" +mean_u = { min = 0.1, max = 0.7, points = 4 } +frequency_hz = 50.0 +depth_a = 0.6 diff --git a/plugins/stage-a-a1/src/csv.rs b/plugins/stage-a-a1/src/csv.rs new file mode 100644 index 0000000..471bdd0 --- /dev/null +++ b/plugins/stage-a-a1/src/csv.rs @@ -0,0 +1,62 @@ +//! Minimal CSV record splitting, shared by the protocol reader and the sensor +//! readout compactor. +//! +//! Deliberately not a CSV *library*: both callers read small files this plugin +//! or the host wrote, and both locate their columns by header name rather than +//! by position. What is actually needed is one correct field splitter — the +//! doubled-quote escaping the host emits, and a quoted `label` in a +//! hand-written protocol, are the only cases that are not `split(',')`. + +/// Splits one CSV record, honouring `"…"` quoting and `""` as an escaped quote. +pub fn split_line(line: &str) -> Vec { + let mut fields = Vec::new(); + let mut current = String::new(); + let mut quoted = false; + let mut chars = line.chars().peekable(); + while let Some(character) = chars.next() { + match character { + '"' if quoted => { + if chars.peek() == Some(&'"') { + current.push('"'); + chars.next(); + } else { + quoted = false; + } + } + '"' => quoted = true, + ',' if !quoted => fields.push(std::mem::take(&mut current)), + other => current.push(other), + } + } + fields.push(current); + fields +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plain_fields_split_on_commas() { + assert_eq!(split_line("a,b,c"), vec!["a", "b", "c"]); + assert_eq!(split_line("a,,c"), vec!["a", "", "c"]); + assert_eq!(split_line(""), vec![""]); + } + + #[test] + fn quoted_fields_keep_their_commas() { + assert_eq!(split_line("a,\"b,c\",d"), vec!["a", "b,c", "d"]); + } + + #[test] + fn doubled_quotes_are_one_literal_quote() { + assert_eq!(split_line("\"a\"\"b\",c"), vec!["a\"b", "c"]); + } + + #[test] + fn an_unterminated_quote_takes_the_rest_of_the_line() { + // Better than dropping the row: the caller validates the fields it + // needs, and a truncated line is reported there with its line number. + assert_eq!(split_line("a,\"b,c"), vec!["a", "b,c"]); + } +} diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs index b67952b..14acc6a 100644 --- a/plugins/stage-a-a1/src/lib.rs +++ b/plugins/stage-a-a1/src/lib.rs @@ -4,10 +4,13 @@ //! PDQ writer. Hardware ownership remains with the Stage-A modulation and //! photodiode plugins; this code only validates and analyses immutable inputs. +mod csv; pub mod phase; +pub mod protocol; pub mod rates; pub mod response_curve; mod runtime; +pub mod sensor; pub mod types; pub use runtime::StageAA1Plugin; diff --git a/plugins/stage-a-a1/src/protocol.rs b/plugins/stage-a-a1/src/protocol.rs new file mode 100644 index 0000000..9434726 --- /dev/null +++ b/plugins/stage-a-a1/src/protocol.rs @@ -0,0 +1,970 @@ +//! Declarative recording protocols: a TOML file naming the points to record, +//! expanded into a flat list the runner walks. +//! +//! The buttons in the Record section each sweep exactly one axis (or two, for +//! the `a × f` surface) with whatever is currently armed on the other axes. +//! That is the right shape for exploring, and the wrong shape for a survey +//! that has to run overnight and be reproducible six months later. A protocol +//! is the survey form: it names every axis explicitly — the operating point +//! `ū` (the `I_k` axis), the frequency `f`, and the depth `a` — plus the dwell +//! and duration each point is recorded with, in a file that travels with the +//! results. +//! +//! ## Format +//! +//! ```toml +//! name = "a1-survey" +//! +//! [defaults] +//! duration_s = 10 +//! settle_s = 2.0 +//! +//! [[block]] +//! name = "depth-sweep-at-10Hz" +//! mean_u = [0.5] +//! frequency_hz = [10.0] +//! depth_a = { min = 0.2, max = 2.0, points = 7 } +//! +//! [[block]] +//! name = "frequency-ladder" +//! mean_u = [0.3, 0.5] +//! frequency_hz = { min = 1.0, max = 200.0, points = 5, spacing = "log" } +//! depth_a = [0.8] +//! duration_s = 20 +//! ``` +//! +//! Every axis takes either an explicit list or a `{ min, max, points }` range +//! (`spacing = "linear"` by default, `"log"` for anything read per decade). +//! A block expands to the full product of its three axes. +//! +//! ## Ordering +//! +//! Points come out `ū` outermost, then `f`, then `a`. That is the order of how +//! expensive each change is to settle: the operating point moves the mean +//! illumination the sensor has to re-adapt to, the frequency has to be +//! confirmed against the trigger, and the depth is the cheap innermost step. +//! Any other nesting would spend the whole run settling. + +use std::collections::BTreeMap; +use std::fmt; + +use serde::Deserialize; + +/// Hard ceiling on the points one protocol may expand to. A three-axis product +/// grows fast, and an operator who typed one zero too many should be told +/// before the bench spends a night on it, not after. +pub const MAX_POINTS: usize = 4_096; + +/// Bounds mirrored from the settings so a protocol cannot ask for a point the +/// plugin would refuse anyway — checked at parse time, where the operator can +/// still see which line was wrong. +const DEPTH_A_RANGE: (f64, f64) = (0.01, 6.0); +const MEAN_U_RANGE: (f64, f64) = (0.01, 1.0); +const FREQUENCY_RANGE: (f64, f64) = (0.01, 2_000.0); +const DURATION_RANGE: (i64, i64) = (1, 3_600); +const SETTLE_RANGE: (f64, f64) = (0.0, 60.0); + +/// What one protocol row records. The same three roles the Record section's +/// buttons offer, so a protocol can carry a complete measurement — its own +/// background reference and pilot, then the points scored against them — +/// instead of needing two button presses before it can be started. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum PointRole { + #[default] + Normal, + /// Bright reference; freezes the ON/OFF windows for the measurement. + Pilot, + /// Unmodulated reference; captures the false-response floor. + Background, +} + +impl PointRole { + fn parse(text: &str) -> Option { + match text.trim().to_ascii_lowercase().as_str() { + "" | "normal" | "point" => Some(Self::Normal), + "pilot" => Some(Self::Pilot), + "background" => Some(Self::Background), + _ => None, + } + } +} + +/// One recording the protocol asks for, with every parameter already resolved. +#[derive(Debug, Clone, PartialEq)] +pub struct ProtocolPoint { + /// Where this row came from — the `[[block]]` name, or a CSV `label` — for + /// the status line and the sidecar. + pub block: String, + /// Normalized cycle-mean lobe point `ū` — the `I_k` axis. + pub mean_u: f64, + pub frequency_hz: f64, + pub depth_a: f64, + pub duration_s: i64, + pub settle_s: f64, + pub role: PointRole, +} + +impl ProtocolPoint { + /// Filename fragment identifying this point inside the measurement folder. + pub fn tag(&self) -> String { + format!( + "u{:.0}m_f{}_a{:.0}m", + self.mean_u * 1_000.0, + frequency_tag(self.frequency_hz), + self.depth_a * 1_000.0, + ) + } +} + +/// A parsed protocol: what to record, in order. +#[derive(Debug, Clone, PartialEq)] +pub struct Protocol { + pub name: String, + pub points: Vec, +} + +impl Protocol { + /// Distinct values on each axis, for the summary shown before starting. + pub fn axis_counts(&self) -> (usize, usize, usize) { + let count = |values: Vec| { + let mut keys: Vec = values.into_iter().map(|value| value.to_bits()).collect(); + keys.sort_unstable(); + keys.dedup(); + keys.len() + }; + ( + count(self.points.iter().map(|point| point.mean_u).collect()), + count(self.points.iter().map(|point| point.frequency_hz).collect()), + count(self.points.iter().map(|point| point.depth_a).collect()), + ) + } + + /// Total bench time the protocol asks for, settling included. + pub fn total_seconds(&self) -> f64 { + self.points + .iter() + .map(|point| point.duration_s as f64 + point.settle_s) + .sum() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ProtocolError { + Toml(String), + /// A named axis, block or default is unusable, with the reason. + Invalid { + what: String, + detail: String, + }, + Empty, + TooManyPoints(usize), +} + +impl fmt::Display for ProtocolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Toml(detail) => write!(f, "the protocol file is not valid TOML: {detail}"), + Self::Invalid { what, detail } => write!(f, "{what}: {detail}"), + Self::Empty => f.write_str( + "the protocol has no points to record — add at least one [[block]] with a \ + mean_u, a frequency_hz and a depth_a", + ), + Self::TooManyPoints(count) => write!( + f, + "the protocol expands to {count} recordings, past the {MAX_POINTS} limit — \ + narrow one of the axes or split it into several files" + ), + } + } +} + +impl std::error::Error for ProtocolError {} + +// ---- wire form ------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +struct ProtocolDoc { + #[serde(default)] + name: Option, + #[serde(default)] + defaults: Defaults, + #[serde(default, rename = "block")] + blocks: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct Defaults { + #[serde(default)] + duration_s: Option, + #[serde(default)] + settle_s: Option, +} + +#[derive(Debug, Deserialize)] +struct BlockDoc { + #[serde(default)] + name: Option, + mean_u: Axis, + frequency_hz: Axis, + depth_a: Axis, + #[serde(default)] + duration_s: Option, + #[serde(default)] + settle_s: Option, +} + +/// One axis: an explicit list, a single value, or a generated range. +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum Axis { + One(f64), + List(Vec), + Range { + min: f64, + max: f64, + points: usize, + #[serde(default)] + spacing: Spacing, + }, +} + +#[derive(Debug, Default, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +enum Spacing { + #[default] + Linear, + Log, +} + +impl Axis { + fn values(&self, what: &str, range: (f64, f64)) -> Result, ProtocolError> { + let invalid = |detail: String| ProtocolError::Invalid { + what: what.to_owned(), + detail, + }; + let values = match self { + Self::One(value) => vec![*value], + Self::List(values) => { + if values.is_empty() { + return Err(invalid("the list is empty".into())); + } + values.clone() + } + Self::Range { + min, + max, + points, + spacing, + } => { + if *points == 0 { + return Err(invalid("points must be at least 1".into())); + } + if !min.is_finite() || !max.is_finite() { + return Err(invalid("min and max must be numbers".into())); + } + if max < min { + return Err(invalid(format!("max {max} is below min {min}"))); + } + if *spacing == Spacing::Log && *min <= 0.0 { + return Err(invalid( + "log spacing needs a min above 0 — a decade ladder has no zero".into(), + )); + } + if *points == 1 { + vec![*min] + } else { + let last = *points - 1; + (0..*points) + .map(|index| { + let t = index as f64 / last as f64; + match spacing { + Spacing::Linear => min + t * (max - min), + Spacing::Log => (min.ln() + t * (max.ln() - min.ln())).exp(), + } + }) + .collect() + } + } + }; + for value in &values { + if !value.is_finite() || *value < range.0 || *value > range.1 { + return Err(invalid(format!( + "{value} is outside the supported {}..={}", + range.0, range.1 + ))); + } + } + Ok(values) + } +} + +/// Parses a protocol and expands it into the points to record. +pub fn parse(text: &str) -> Result { + let doc: ProtocolDoc = + toml::from_str(text).map_err(|error| ProtocolError::Toml(error.to_string()))?; + + let default_duration = doc.defaults.duration_s.unwrap_or(10); + let default_settle = doc.defaults.settle_s.unwrap_or(2.0); + + let mut points = Vec::new(); + // Blocks may be named or not; unnamed ones get a stable positional name so + // every recording can still say which part of the protocol it belongs to. + let mut seen_names: BTreeMap = BTreeMap::new(); + for (index, block) in doc.blocks.iter().enumerate() { + let base = block + .name + .clone() + .filter(|name| !name.trim().is_empty()) + .unwrap_or_else(|| format!("block{}", index + 1)); + // Two blocks sharing a name would put two different sets of points in + // one namespace; keep them distinguishable rather than refusing. + let occurrence = seen_names.entry(base.clone()).or_insert(0); + *occurrence += 1; + let name = if *occurrence == 1 { + base + } else { + format!("{base}-{occurrence}") + }; + + let duration_s = block.duration_s.unwrap_or(default_duration); + if duration_s < DURATION_RANGE.0 || duration_s > DURATION_RANGE.1 { + return Err(ProtocolError::Invalid { + what: format!("block '{name}' duration_s"), + detail: format!( + "{duration_s} is outside the supported {}..={}", + DURATION_RANGE.0, DURATION_RANGE.1 + ), + }); + } + let settle_s = block.settle_s.unwrap_or(default_settle); + if !settle_s.is_finite() || settle_s < SETTLE_RANGE.0 || settle_s > SETTLE_RANGE.1 { + return Err(ProtocolError::Invalid { + what: format!("block '{name}' settle_s"), + detail: format!( + "{settle_s} is outside the supported {}..={}", + SETTLE_RANGE.0, SETTLE_RANGE.1 + ), + }); + } + + let mean_u = block + .mean_u + .values(&format!("block '{name}' mean_u"), MEAN_U_RANGE)?; + let frequency_hz = block + .frequency_hz + .values(&format!("block '{name}' frequency_hz"), FREQUENCY_RANGE)?; + let depth_a = block + .depth_a + .values(&format!("block '{name}' depth_a"), DEPTH_A_RANGE)?; + + // `ū` outermost, `a` innermost — see the module docs. + for mean_u in &mean_u { + for frequency_hz in &frequency_hz { + for depth_a in &depth_a { + points.push(ProtocolPoint { + block: name.clone(), + mean_u: *mean_u, + frequency_hz: *frequency_hz, + depth_a: *depth_a, + duration_s, + settle_s, + role: PointRole::Normal, + }); + if points.len() > MAX_POINTS { + return Err(ProtocolError::TooManyPoints(points.len())); + } + } + } + } + } + + if points.is_empty() { + return Err(ProtocolError::Empty); + } + Ok(Protocol { + name: doc + .name + .filter(|name| !name.trim().is_empty()) + .unwrap_or_else(|| "protocol".to_owned()), + points, + }) +} + +/// Compact frequency fragment for a filename: `10Hz`, `1500mHz`, `2k5Hz`. +fn frequency_tag(hz: f64) -> String { + if hz < 1.0 { + format!("{:.0}mHz", hz * 1_000.0) + } else if hz < 1_000.0 { + let rounded = (hz * 10.0).round() / 10.0; + if (rounded - rounded.round()).abs() < f64::EPSILON { + format!("{rounded:.0}Hz") + } else { + format!("{rounded:.1}Hz").replace('.', "p") + } + } else { + format!("{:.0}Hz", hz.round()) + } +} + +/// Reads a protocol from a file, choosing the reader by extension. +/// +/// `.csv` is the row-per-recording form and the one to reach for: one line is +/// one recording, every parameter is a column, and it opens in a spreadsheet +/// or comes straight out of a script. `.toml` is the block/range form — more +/// compact for a dense regular sweep, and kept because it expresses one. +/// +/// Both produce the same flat list, so nothing downstream knows which was used. +pub fn parse_file(path: &str, text: &str) -> Result { + let is_csv = std::path::Path::new(path) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("csv")); + if is_csv { + parse_csv(text) + } else { + parse(text) + } +} + +/// Columns a protocol CSV may carry. `mean_u`, `frequency_hz` and `depth_a` are +/// required; the rest fall back to their defaults. +const CSV_REQUIRED: [&str; 3] = ["mean_u", "frequency_hz", "depth_a"]; +const CSV_OPTIONAL: [&str; 4] = ["duration_s", "settle_s", "label", "role"]; + +/// Parses the row-per-recording CSV form. +/// +/// Columns are located **by header name**, so their order does not matter and a +/// column can be left out entirely — the same rule the sensor readout follows, +/// and the reason a file edited in a spreadsheet keeps working after someone +/// drags a column. +/// +/// Blank lines and `#` comments are skipped, so a file can explain itself. +/// Errors carry the **file line number**, not the row index, because that is +/// what an editor and a spreadsheet both show. +pub fn parse_csv(text: &str) -> Result { + let mut header: Option> = None; + let mut points = Vec::new(); + + for (offset, raw) in text.lines().enumerate() { + let line_no = offset + 1; + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let fields = crate::csv::split_line(raw); + + let Some(columns) = header.as_ref() else { + let columns: Vec = fields + .iter() + .map(|field| field.trim().to_ascii_lowercase()) + .collect(); + for required in CSV_REQUIRED { + if !columns.iter().any(|column| column == required) { + return Err(ProtocolError::Invalid { + what: format!("line {line_no}: the header"), + detail: format!( + "has no '{required}' column. Required: {}. Optional: {}", + CSV_REQUIRED.join(", "), + CSV_OPTIONAL.join(", ") + ), + }); + } + } + header = Some(columns); + continue; + }; + + let cell = |name: &str| -> Option<&str> { + let index = columns.iter().position(|column| column == name)?; + fields.get(index).map(|field| field.trim()) + }; + let number = |name: &str, range: (f64, f64)| -> Result { + let raw = cell(name).unwrap_or(""); + let invalid = |detail: String| ProtocolError::Invalid { + what: format!("line {line_no}: {name}"), + detail, + }; + if raw.is_empty() { + return Err(invalid("is empty".into())); + } + let value: f64 = raw + .parse() + .map_err(|_| invalid(format!("'{raw}' is not a number")))?; + if !value.is_finite() || value < range.0 || value > range.1 { + return Err(invalid(format!( + "{value} is outside the supported {}..={}", + range.0, range.1 + ))); + } + Ok(value) + }; + + let mean_u = number("mean_u", MEAN_U_RANGE)?; + let frequency_hz = number("frequency_hz", FREQUENCY_RANGE)?; + let depth_a = number("depth_a", DEPTH_A_RANGE)?; + + // Absent column *or* empty cell falls back, so a file can carry a + // duration column that only some rows fill in. + let duration_s = match cell("duration_s").unwrap_or("") { + "" => 10, + raw => { + let value: i64 = raw.parse().map_err(|_| ProtocolError::Invalid { + what: format!("line {line_no}: duration_s"), + detail: format!("'{raw}' is not a whole number of seconds"), + })?; + if value < DURATION_RANGE.0 || value > DURATION_RANGE.1 { + return Err(ProtocolError::Invalid { + what: format!("line {line_no}: duration_s"), + detail: format!( + "{value} is outside the supported {}..={}", + DURATION_RANGE.0, DURATION_RANGE.1 + ), + }); + } + value + } + }; + let settle_s = match cell("settle_s").unwrap_or("") { + "" => 2.0, + _ => number("settle_s", SETTLE_RANGE)?, + }; + let role = + PointRole::parse(cell("role").unwrap_or("")).ok_or_else(|| ProtocolError::Invalid { + what: format!("line {line_no}: role"), + detail: format!( + "'{}' is not one of normal, pilot, background", + cell("role").unwrap_or("") + ), + })?; + let label = cell("label").unwrap_or("").trim().to_owned(); + + points.push(ProtocolPoint { + block: if label.is_empty() { + format!("row{}", points.len() + 1) + } else { + label + }, + mean_u, + frequency_hz, + depth_a, + duration_s, + settle_s, + role, + }); + if points.len() > MAX_POINTS { + return Err(ProtocolError::TooManyPoints(points.len())); + } + } + + if header.is_none() { + return Err(ProtocolError::Invalid { + what: "the protocol file".into(), + detail: format!( + "has no header line. The first line that is not blank or a # comment must name \ + the columns: {}", + CSV_REQUIRED.join(", ") + ), + }); + } + if points.is_empty() { + return Err(ProtocolError::Empty); + } + Ok(Protocol { + name: "protocol".to_owned(), + points, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE: &str = r#" +name = "sample" + +[defaults] +duration_s = 5 +settle_s = 1.5 + +[[block]] +name = "flat" +mean_u = 0.5 +frequency_hz = [10.0, 20.0] +depth_a = { min = 0.5, max = 1.5, points = 3 } + +[[block]] +name = "ladder" +mean_u = [0.3, 0.6] +frequency_hz = { min = 1.0, max = 100.0, points = 3, spacing = "log" } +depth_a = 0.8 +duration_s = 30 +"#; + + #[test] + fn a_protocol_expands_to_the_product_of_its_axes() { + let protocol = parse(SAMPLE).expect("valid protocol"); + assert_eq!(protocol.name, "sample"); + // 1×2×3 + 2×3×1 + assert_eq!(protocol.points.len(), 6 + 6); + assert_eq!(protocol.axis_counts(), (3, 5, 4)); + } + + #[test] + fn points_run_mean_u_outermost_and_depth_innermost() { + // The nesting is the whole reason the protocol is worth having over + // three nested button presses: it settles the expensive axis least + // often. Assert the actual emitted order, not just the count. + let protocol = parse(SAMPLE).expect("valid protocol"); + let flat: Vec<(f64, f64, f64)> = protocol + .points + .iter() + .filter(|point| point.block == "flat") + .map(|point| (point.mean_u, point.frequency_hz, point.depth_a)) + .collect(); + assert_eq!( + flat, + vec![ + (0.5, 10.0, 0.5), + (0.5, 10.0, 1.0), + (0.5, 10.0, 1.5), + (0.5, 20.0, 0.5), + (0.5, 20.0, 1.0), + (0.5, 20.0, 1.5), + ] + ); + } + + #[test] + fn defaults_apply_unless_the_block_overrides_them() { + let protocol = parse(SAMPLE).expect("valid protocol"); + let flat = protocol + .points + .iter() + .find(|point| point.block == "flat") + .expect("flat block"); + assert_eq!(flat.duration_s, 5); + assert!((flat.settle_s - 1.5).abs() < f64::EPSILON); + + let ladder = protocol + .points + .iter() + .find(|point| point.block == "ladder") + .expect("ladder block"); + assert_eq!(ladder.duration_s, 30); + // settle_s was not overridden, so the default still applies. + assert!((ladder.settle_s - 1.5).abs() < f64::EPSILON); + } + + #[test] + fn log_spacing_is_geometric() { + let protocol = parse(SAMPLE).expect("valid protocol"); + let ladder: Vec = protocol + .points + .iter() + .filter(|point| point.block == "ladder" && point.mean_u == 0.3) + .map(|point| point.frequency_hz) + .collect(); + assert_eq!(ladder.len(), 3); + assert!((ladder[0] - 1.0).abs() < 1e-9); + assert!((ladder[1] - 10.0).abs() < 1e-9); + assert!((ladder[2] - 100.0).abs() < 1e-9); + } + + #[test] + fn total_seconds_counts_settling_too() { + let protocol = parse(SAMPLE).expect("valid protocol"); + // 6 × (5 + 1.5) + 6 × (30 + 1.5) + assert!((protocol.total_seconds() - (6.0 * 6.5 + 6.0 * 31.5)).abs() < 1e-9); + } + + #[test] + fn out_of_range_values_name_the_axis_that_is_wrong() { + let error = parse( + r#" +[[block]] +name = "too-deep" +mean_u = 0.5 +frequency_hz = 10.0 +depth_a = 99.0 +"#, + ) + .expect_err("a depth of 99 is not drivable"); + let text = error.to_string(); + assert!(text.contains("too-deep"), "{text}"); + assert!(text.contains("depth_a"), "{text}"); + } + + #[test] + fn log_spacing_from_zero_is_refused_rather_than_producing_infinities() { + let error = parse( + r#" +[[block]] +mean_u = 0.5 +depth_a = 1.0 +frequency_hz = { min = 0.0, max = 100.0, points = 3, spacing = "log" } +"#, + ) + .expect_err("log from zero"); + assert!(error.to_string().contains("no zero"), "{error}"); + } + + #[test] + fn an_empty_protocol_says_so_instead_of_running_nothing() { + assert_eq!( + parse("name = \"nothing\"").unwrap_err(), + ProtocolError::Empty + ); + } + + #[test] + fn a_runaway_product_is_refused_before_the_bench_spends_a_night_on_it() { + let error = parse( + r#" +[[block]] +mean_u = { min = 0.1, max = 1.0, points = 20 } +frequency_hz = { min = 1.0, max = 100.0, points = 20 } +depth_a = { min = 0.1, max = 2.0, points = 20 } +"#, + ) + .expect_err("8000 points"); + assert!( + matches!(error, ProtocolError::TooManyPoints(_)), + "{error:?}" + ); + } + + #[test] + fn unnamed_and_repeated_blocks_stay_distinguishable() { + let protocol = parse( + r#" +[[block]] +mean_u = 0.5 +frequency_hz = 10.0 +depth_a = 1.0 + +[[block]] +name = "dup" +mean_u = 0.5 +frequency_hz = 10.0 +depth_a = 1.0 + +[[block]] +name = "dup" +mean_u = 0.5 +frequency_hz = 20.0 +depth_a = 1.0 +"#, + ) + .expect("valid"); + let names: Vec<&str> = protocol + .points + .iter() + .map(|point| point.block.as_str()) + .collect(); + assert_eq!(names, vec!["block1", "dup", "dup-2"]); + } + + #[test] + fn a_point_tag_is_stable_and_filename_safe() { + let point = ProtocolPoint { + block: "b".into(), + mean_u: 0.5, + frequency_hz: 12.5, + depth_a: 0.75, + duration_s: 5, + settle_s: 1.0, + role: PointRole::Normal, + }; + assert_eq!(point.tag(), "u500m_f12p5Hz_a750m"); + assert!(!point.tag().contains('.')); + } +} + +#[cfg(test)] +mod csv_tests { + use super::*; + + const SAMPLE: &str = "\ +label,mean_u,frequency_hz,depth_a,duration_s,settle_s,role +floor,0.5,10,0.02,20,3,background +curve,0.5,10,0.5,,, +slow,0.4,1,0.8,40,4, +"; + + #[test] + fn one_row_is_one_recording_in_file_order() { + let protocol = parse_csv(SAMPLE).expect("valid CSV"); + assert_eq!(protocol.points.len(), 3); + let order: Vec<(f64, f64)> = protocol + .points + .iter() + .map(|point| (point.frequency_hz, point.depth_a)) + .collect(); + assert_eq!(order, vec![(10.0, 0.02), (10.0, 0.5), (1.0, 0.8)]); + } + + /// The reason for the row-per-recording form: a low frequency needs longer + /// than a high one, with no block gymnastics to express it. + #[test] + fn each_row_carries_its_own_duration_and_settle() { + let protocol = parse_csv(SAMPLE).expect("valid CSV"); + assert_eq!(protocol.points[0].duration_s, 20); + assert_eq!(protocol.points[2].duration_s, 40); + assert!((protocol.points[2].settle_s - 4.0).abs() < f64::EPSILON); + // Blank cells fall back rather than failing the row. + assert_eq!(protocol.points[1].duration_s, 10); + assert!((protocol.points[1].settle_s - 2.0).abs() < f64::EPSILON); + } + + #[test] + fn a_row_can_name_its_role_so_a_file_carries_its_own_references() { + let protocol = parse_csv(SAMPLE).expect("valid CSV"); + assert_eq!(protocol.points[0].role, PointRole::Background); + assert_eq!(protocol.points[1].role, PointRole::Normal); + } + + #[test] + fn columns_are_found_by_name_not_by_position() { + // Someone drags a column in a spreadsheet; the file must still mean the + // same thing. + let reordered = "\ +depth_a,role,frequency_hz,label,mean_u +0.02,background,10,floor,0.5 +"; + let protocol = parse_csv(reordered).expect("valid CSV"); + assert_eq!(protocol.points[0].depth_a, 0.02); + assert_eq!(protocol.points[0].mean_u, 0.5); + assert_eq!(protocol.points[0].role, PointRole::Background); + assert_eq!(protocol.points[0].block, "floor"); + } + + #[test] + fn comments_and_blank_lines_are_skipped_so_a_file_can_explain_itself() { + let commented = "\ +# a survey +mean_u,frequency_hz,depth_a + +# the only point +0.5,10,0.5 +"; + assert_eq!(parse_csv(commented).expect("valid").points.len(), 1); + } + + #[test] + fn errors_name_the_line_number_the_editor_shows() { + // Not a row index: the operator is looking at a spreadsheet. + let bad = "\ +# comment +mean_u,frequency_hz,depth_a +0.5,10,0.5 +0.5,10,99 +"; + let error = parse_csv(bad).expect_err("a depth of 99 is not drivable"); + let text = error.to_string(); + // Line 4 counting the comment and the header, which is what an editor + // and a spreadsheet both show. + assert!(text.contains("line 4"), "{text}"); + assert!(text.contains("depth_a"), "{text}"); + } + + #[test] + fn a_missing_required_column_says_which_one_and_lists_the_rest() { + let error = parse_csv("mean_u,frequency_hz\n0.5,10\n").expect_err("no depth_a"); + let text = error.to_string(); + assert!(text.contains("depth_a"), "{text}"); + assert!( + text.contains("duration_s"), + "optional columns unlisted: {text}" + ); + } + + #[test] + fn a_header_only_or_empty_file_is_refused_rather_than_running_nothing() { + assert_eq!( + parse_csv("mean_u,frequency_hz,depth_a\n").unwrap_err(), + ProtocolError::Empty + ); + assert!(matches!( + parse_csv("# nothing but a comment\n").unwrap_err(), + ProtocolError::Invalid { .. } + )); + } + + #[test] + fn a_quoted_label_may_contain_a_comma() { + let quoted = "label,mean_u,frequency_hz,depth_a\n\"ladder, low end\",0.5,1,0.8\n"; + let protocol = parse_csv(quoted).expect("valid"); + assert_eq!(protocol.points[0].block, "ladder, low end"); + } + + #[test] + fn an_unlabelled_row_still_gets_a_stable_name() { + let protocol = + parse_csv("mean_u,frequency_hz,depth_a\n0.5,10,0.5\n0.5,20,0.5\n").expect("valid"); + assert_eq!(protocol.points[0].block, "row1"); + assert_eq!(protocol.points[1].block, "row2"); + } + + #[test] + fn an_unknown_role_is_refused_rather_than_silently_recorded_as_normal() { + let error = parse_csv("mean_u,frequency_hz,depth_a,role\n0.5,10,0.5,piolt\n") + .expect_err("typo in role"); + assert!(error.to_string().contains("pilot"), "{error}"); + } + + #[test] + fn the_reader_is_chosen_by_extension() { + let csv = "mean_u,frequency_hz,depth_a\n0.5,10,0.5\n"; + assert_eq!(parse_file("survey.csv", csv).expect("csv").points.len(), 1); + assert_eq!(parse_file("SURVEY.CSV", csv).expect("csv").points.len(), 1); + // A .toml path goes to the block reader, and the CSV text is not TOML. + assert!(parse_file("survey.toml", csv).is_err()); + } +} + +#[cfg(test)] +mod example_file_tests { + use super::*; + + /// The shipped example is documentation the operator copies, so it has to + /// stay valid as the format moves — a stale example is worse than none. + /// The shipped CSV is what an operator copies, so it has to stay valid as + /// the format moves — a stale example is worse than none. + #[test] + fn the_shipped_example_csv_parses_and_exercises_every_column() { + let text = include_str!("../protocols/example.csv"); + let protocol = parse_csv(text).expect("the shipped CSV example must parse"); + assert!(protocol.points.len() > 10); + assert!(protocol + .points + .iter() + .any(|point| point.role == PointRole::Background)); + assert!(protocol + .points + .iter() + .any(|point| point.role == PointRole::Pilot)); + // The whole reason for the row form: durations genuinely differ. + let durations: std::collections::BTreeSet = protocol + .points + .iter() + .map(|point| point.duration_s) + .collect(); + assert!(durations.len() > 2, "{durations:?}"); + // And every axis is exercised. + let (means, frequencies, depths) = protocol.axis_counts(); + assert!(means > 1 && frequencies > 1 && depths > 1); + } + + #[test] + fn the_shipped_example_protocol_parses() { + let text = include_str!("../protocols/example.toml"); + let protocol = parse(text).expect("the shipped example must parse"); + assert_eq!(protocol.name, "a1-example-survey"); + // 1×1×7 + 2×6×1 + 1×3×4 + 4×1×1 + assert_eq!(protocol.points.len(), 7 + 12 + 12 + 4); + // Every axis is genuinely exercised, so the example demonstrates what + // it claims to. + let (means, frequencies, depths) = protocol.axis_counts(); + assert!(means > 1 && frequencies > 1 && depths > 1); + assert!(protocol.total_seconds() > 0.0); + } +} diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 457a40b..829abcf 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -21,6 +21,13 @@ //! trimmed depth under the same lease so one atomic frequency point is recorded at //! exactly `a₀`. //! +//! Where that `a` comes from is one operator setting, [`DepthSource`] (ADR 020). The +//! photodiode's measurement is the default and the source of record; it is also +//! fail-closed on firmware phase-0 markers, so a bench that never receives them can +//! fall back to the modulation owner's *commanded* calibrated depth and run the same +//! workflow open loop. Every artefact that carries an `a` carries which source +//! produced it. +//! //! 2. **Live sanity quicklooks.** Folding the camera event stream on the modulation //! period `T` (defined by the firmware phase-0 `EXT_TRIGGER`), it renders the //! **rolling half-period response** `S_p(t)` (a live "are events appearing, is the @@ -41,10 +48,10 @@ use augur_plugin_api::{ HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, PathDialogKind, Plugin, PluginCapabilities, PluginControlContext, PluginControlInbox, PluginDiscontinuity, PluginFrame, PluginInput, PluginRuntimeRole, PluginServiceOutcome, - PluginServiceReply, PluginServiceRequest, RoiV1, Series1dLine, Series1dPoint, Series1dV1, - SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, - TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, TableValueType, - CTX_GLOBAL_SETTINGS, + PluginServiceReply, PluginServiceRequest, RoiV1, SensorMonitoringV1, Series1dLine, + Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, + StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, + TableValueType, CTX_GLOBAL_SETTINGS, CTX_SENSOR_MONITORING, }; use serde::Serialize; use serde_json::{json, Value}; @@ -58,8 +65,10 @@ use stage_a_plugin_contract::{ }; use crate::phase::{fold_events, fold_events_free_running, MarkerValidationConfig, PhaseFold}; +use crate::protocol; use crate::rates::{rolling_half_period_response, RollingResponsePoint}; use crate::response_curve::{auto_windows, response_probability, PhaseWindow, ResponsePoint, Roi}; +use crate::sensor; use crate::types::{CameraEvent, Polarity}; const MODULATION_PLUGIN_ID: &str = "stage-a.modulation"; @@ -127,7 +136,6 @@ const COMMANDED_A_MAX: f64 = 6.0; const FREQUENCY_MATCH_FRACTION: f64 = 0.01; /// Lock table persisted in the output folder, so found depths survive a restart. const A0_LOCK_FILE: &str = "a0_locks.json"; - /// Frequency points a single run may visit, before the interleaved references. const FREQ_SWEEP_MAX_POINTS: usize = 64; /// How long the frequency sweep waits for the phase-0 trigger to report the @@ -171,6 +179,16 @@ fn frequency_label(hz: f64) -> String { format!("{hz:.3} Hz") } +/// Upper-cases the first character, so a blocker written as a sentence fragment +/// ("the total power …") can also stand as its own sentence in the status panel. +fn capitalize_first(text: &str) -> String { + let mut chars = text.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } +} + /// Compact file-safe frequency tag for an event-count point's stem: /// `50 Hz → f50Hz`, `0.5 Hz → f0p5Hz`. fn frequency_tag(hz: f64) -> String { @@ -328,6 +346,9 @@ struct Recording { cam_rejected: bool, pd_pdq_path: Option, pd_sidecar_path: Option, + /// Compacted sensor readout written into the measurement folder, if the + /// host produced any telemetry for this run. + sensor_readout_path: Option, pd_finalized: bool, pd_valid: bool, /// The photodiode rejected BeginRecording — skip the finalize and don't @@ -410,6 +431,13 @@ struct Sweep { /// Whether the current point's recording actually started (vs. was /// refused by validation before it began). point_started: bool, + /// Set only on the branch that runs out of points with every one recorded. + /// + /// An enclosing frequency ladder has to know whether the inner run it + /// handed a rung to *finished* or gave up, and it cannot tell from the + /// recording coordinator: a sweep that aborts on point 4 of 5 leaves + /// `recording_completed_ok` true from point 3. + completed_ok: bool, last_activity_ms: u64, stop_requested: bool, } @@ -488,6 +516,91 @@ struct A0Lock { stop_requested: bool, } +/// Where the modulation depth `a` that A1 works from comes from. +/// +/// `a = ln(I_max / I_min)` is a property of the *light*, so the photodiode is +/// the only source that can state it (ADR 011, and the estimator's own module +/// docs). That is the default and stays the source of record. +/// +/// The bench cannot always deliver it, though. The photodiode withholds `a` +/// whenever its estimator window cannot be proven to cover whole modulation +/// cycles, which needs firmware phase-0 markers on the stream port; without +/// them — no trigger cable, a firmware build that does not stamp them, a +/// frequency low enough that two cycles do not fit in the ring — every gate +/// that needs `a` refuses, and the whole a₀/sweep workflow is unreachable even +/// though the drive is calibrated and running. +/// +/// [`DepthSource::Commanded`] is the pragmatic way through: the modulation +/// owner already inverts a *measured* Pockels transfer curve (`V_null`, `Vπ`) +/// to command a depth, and publishes that depth as +/// `OpticalDriveStateV1::depth_a_milli`. Taking `a` from there is open loop — +/// it is what the drive asked the cell for, not what the light did, so it +/// carries the calibration's error and any drift since — but it is a +/// calibrated number, not a datasheet one, and it lets the workflow run. Every +/// artefact that records an `a` records which source produced it, so a run +/// taken this way is never mistaken for a measured one. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +enum DepthSource { + /// The photodiode's measured excitation log-contrast. + #[default] + Photodiode, + /// The depth the modulation owner's calibrated optical drive is commanding. + Commanded, +} + +impl DepthSource { + fn from_index(index: u64) -> Self { + match index { + 1 => Self::Commanded, + _ => Self::Photodiode, + } + } + + fn index(self) -> u64 { + match self { + Self::Photodiode => 0, + Self::Commanded => 1, + } + } + + /// Machine-readable tag written into sidecars and the lock table. + fn label(self) -> &'static str { + match self { + Self::Photodiode => "photodiode_measured", + Self::Commanded => "modulation_commanded", + } + } + + /// The verb the panel uses for a depth from this source: "measured a" is a + /// claim about the light and must not be printed for a commanded one. + fn verb(self) -> &'static str { + match self { + Self::Photodiode => "measured", + Self::Commanded => "commanded", + } + } + + /// Whether holding one `a₀` across frequencies requires the closed-loop + /// search (ADR 013), or whether commanding it is enough (ADR 021). + /// + /// The lock exists for one reason: the Pockels inversion is measured once + /// and is therefore *static*, so the depth it actually delivers rolls off + /// as `f` rises. Holding a **measured** `a₀` across a frequency ladder + /// means re-finding the commanded depth that produces it at every point — + /// `a_cmd ← a_cmd · a₀/a_measured`, a couple of trials per frequency. + /// + /// None of that applies to a **commanded** depth, because it *is* the + /// number being commanded. A search would command `a₀`, read back `a₀`, + /// converge on trial one, and store one identical row per frequency: pure + /// ceremony between the operator and a recording, and worse than nothing + /// once a stale row from a measured run warm-starts it (`begin_a0_lock`) + /// and drags a closed-loop number into an open-loop point. + fn needs_a0_lock(self) -> bool { + matches!(self, Self::Photodiode) + } +} + /// The result of one lock: the commanded depth that produced the frozen `a₀` at /// one frequency. Persisted in `a0_locks.json` and replayed by event-count points. #[derive(Debug, Clone, Serialize, serde::Deserialize)] @@ -497,7 +610,7 @@ struct A0LockPoint { target_a: f64, /// What the drive must be commanded to in order to *measure* `target_a`. commanded_a: f64, - /// The photodiode-measured `a` averaged over the final trial. + /// The `a` observed over the final trial, from `depth_source`. measured_a: f64, trials: u32, /// False when the lock ran out of trials or hit a drive limit; such a row is @@ -506,6 +619,10 @@ struct A0LockPoint { locked_at_unix_ms: u64, low_clip_fraction: Option, high_clip_fraction: Option, + /// Which source produced `measured_a`. Lock tables written before the + /// setting existed were all photodiode-measured, which is the default. + #[serde(default)] + depth_source: DepthSource, } /// Order the planned frequencies are actually visited in. @@ -556,6 +673,44 @@ impl FreqOrder { } } +/// What the frequency ladder records at each of its frequencies. +/// +/// The ladder is an outer loop over `f` that leases the drive once and hands +/// each confirmed frequency to an inner run. What that inner run *is* is the +/// only thing separating the bench's two multi-frequency experiments, so it is +/// one enum rather than two copies of the ladder. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum FreqSweepMode { + /// One event-count point at the frozen depth `a₀` (ADR 013 / ADR 014): + /// the same depth everywhere, so a change in the event count is a + /// frequency effect. + #[default] + A0Point, + /// The whole depth sweep over `[min_a, max_a]` at every frequency + /// (ADR 023) — one `q_p(a)` curve per `f`, i.e. the `q_p(a, f)` surface + /// the offline `a50(f)` fit is read from. + DepthSweep, +} + +impl FreqSweepMode { + fn label(self) -> &'static str { + match self { + Self::A0Point => "a₀ point", + Self::DepthSweep => "depth sweep", + } + } + + /// Whether a rung has to find a depth before it can record one. + /// + /// Only the `a₀` experiment does: it replays a single depth that something + /// has to have chosen. A depth sweep commands every `a` in its range + /// itself and settles on each, so there is nothing for a lock to add — in + /// either depth source. + fn needs_armed_depth(self) -> bool { + matches!(self, Self::A0Point) + } +} + /// One stop of the frequency sweep. #[derive(Debug, Clone, Copy, PartialEq)] struct FreqSweepPoint { @@ -589,6 +744,8 @@ enum FreqSweepPhase { /// stay locked out from the first frequency to the last. struct FreqSweep { phase: FreqSweepPhase, + /// What each rung records — see [`FreqSweepMode`]. + mode: FreqSweepMode, points: Vec, index: usize, lease_id: LeaseId, @@ -623,6 +780,63 @@ impl FreqSweep { } } +/// Where a protocol run is within its per-point cycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProtocolPhase { + /// AcquireLease sent to the modulation owner; waiting for the grant. + AcquiringLease, + /// The three retargets for this point are in flight; waiting for all of + /// them to come back Applied. + Retargeting, + /// Dwelling for the point's own settle time before the recording starts. + Settling, + /// The recording coordinator owns this phase. + Recording, +} + +/// One protocol run: walk the parsed points, retargeting all three axes at +/// each, on a single lease held for the whole file. +/// +/// It is a supervisor like [`FreqSweep`], not a fourth copy of the recording +/// machinery: it moves the drive and then hands off to the same +/// `begin_recording` every button uses. The difference from the sweep buttons +/// is that a protocol names *every* axis for *every* point, so nothing is left +/// implicitly at whatever the operator last armed. +struct ProtocolRun { + plan: protocol::Protocol, + phase: ProtocolPhase, + index: usize, + lease_id: LeaseId, + lease_granted: bool, + lease_req: u64, + /// Request ids of the retargets in flight for the current point. A point + /// only proceeds once this is empty: the three axes are applied + /// independently, and recording after two of them would file the run under + /// parameters the bench was not actually at. + pending_reqs: Vec, + /// Wall-clock instant the dwell ends. + settle_until_ms: u64, + /// Points whose retarget or recording failed, with the owner's own reason. + /// + /// Kept rather than aborting — the rest of the survey is still worth + /// having — and kept *with the reason*, because the per-point message is + /// overwritten by the next point within the same tick. Without this the + /// only thing an unattended run could report at the end was a count. + failed: Vec<(usize, String)>, + recorded: usize, + last_activity_ms: u64, + stop_requested: bool, + /// Why the current point is being given up, when that was decided in a + /// service reply rather than in the tick. + skip_reason: Option, +} + +impl ProtocolRun { + fn point(&self) -> Option<&protocol::ProtocolPoint> { + self.plan.points.get(self.index) + } +} + /// On-disk form of the per-frequency lock table. #[derive(Debug, Clone, Default, Serialize, serde::Deserialize)] struct A0LockTable { @@ -653,6 +867,21 @@ pub struct StageAA1Plugin { // -- host camera ROI/mask, mirrored from CTX_GLOBAL_SETTINGS -- host_roi: Option, masked_pixels: HashSet<(u16, u16)>, + /// Latest sensor-measured die temperature, pixel dead time and scene + /// illumination, mirrored from `CTX_SENSOR_MONITORING` every frame. + /// + /// Provenance only — never an input to any result. The host publishes it + /// solely while streaming from a camera with a monitoring block, so replay + /// and offline re-runs of the same data carry `None`, and a plugin whose + /// *answers* depended on it would disagree with itself between the two. + sensor: Option, + /// [`Self::sensor`] frozen when the current recording started. + /// + /// Written into the sidecar in preference to the live value: these drift + /// (the die warms, the room lights change), so the number that belongs to a + /// run is the one that held when it began, not the one that happens to be + /// current when the file is finalized seconds later. + sensor_at_start: Option, // -- response curve (auto-windowed Bernoulli q_p) -- /// Window floor as a fraction of the ON/OFF histogram peak (see `auto_windows`). window_floor: f64, @@ -666,9 +895,9 @@ pub struct StageAA1Plugin { // -- recording coordinator -- output_folder: String, measurement_id: String, - /// Canonical identifier of the physical cycle-mean local flux point - /// `I_k` (for example a row in the illumination calibration/map). - flux_point_id: String, + /// Where the depth `a` comes from — the photodiode's measurement, or the + /// modulation owner's commanded depth. See [`DepthSource`]. + depth_source: DepthSource, /// Sweep range `[min_a, max_a]` for this `(I_k, f)` row (automation template). min_a: f64, max_a: f64, @@ -695,6 +924,10 @@ pub struct StageAA1Plugin { /// Latched by the Start sweep button, consumed next control tick. sweep_pending: bool, sweep: Option, + /// Whether the most recent inner sweep ran out of points with all of them + /// recorded, as opposed to giving up. Read by the frequency ladder to + /// decide between advancing and skipping the rung. See [`Sweep::completed_ok`]. + last_sweep_completed_ok: bool, // -- exact event-count depth a₀ (ADR 013) -- /// The one photodiode-measured log contrast held across the frequency sweep. a0_target: f64, @@ -717,9 +950,25 @@ pub struct StageAA1Plugin { /// Insert the lowest planned frequency again after every N points, so drift /// across the block shows up as a disagreement between its repeats. 0 = off. freq_reference_every: u32, - /// Latched by the Start frequency sweep button, consumed next control tick. - freq_sweep_pending: bool, + /// Latched by whichever frequency-ladder button was pressed, carrying what + /// that button asked for. Consumed next control tick. + freq_sweep_pending: Option, freq_sweep: Option, + // -- declarative protocol runs -- + /// Path of the TOML protocol file to run. + protocol_path: String, + /// Latched by the Run protocol button, consumed next control tick. + protocol_pending: bool, + /// Recording length for the *next* run, overriding the panel's setting. + /// + /// The protocol's own `duration_s` has to win, or a survey's lengths would + /// silently come from the UI and the file would not describe what it + /// produced. It cannot be written into `duration_s` itself: the host + /// re-applies the whole settings snapshot from the UI mirror on every pass, + /// so an operator setting assigned on the worker is reverted within the + /// frame — the same trap that made the modulation Apply button look dead. + pending_duration_s: Option, + protocol: Option, /// One converged (or attempted) lock per frequency, newest per frequency /// wins; mirrored to `a0_locks.json` in the output folder. a0_locks: Vec, @@ -737,6 +986,8 @@ pub struct StageAA1Plugin { press_clear_curve: PressLatch, press_find_a0: PressLatch, press_freq_sweep: PressLatch, + press_freq_depth_sweep: PressLatch, + press_run_protocol: PressLatch, press_record_a0: PressLatch, press_clear_a0: PressLatch, } @@ -758,13 +1009,15 @@ impl Default for StageAA1Plugin { frame_height: 0, host_roi: None, masked_pixels: HashSet::new(), + sensor: None, + sensor_at_start: None, window_floor: DEFAULT_WINDOW_FLOOR, response_points: Vec::new(), pilot_windows: None, background_floor: None, output_folder: String::new(), measurement_id: generate_measurement_id(), - flux_point_id: String::new(), + depth_source: DepthSource::Photodiode, min_a: 0.0, max_a: 2.0, duration_s: 10, @@ -780,6 +1033,7 @@ impl Default for StageAA1Plugin { settle_s: 2.0, sweep_pending: false, sweep: None, + last_sweep_completed_ok: false, // No numerical a₀ is frozen in the repository: this default is a // placeholder the operator replaces with the scout result. a0_target: 0.5, @@ -793,8 +1047,12 @@ impl Default for StageAA1Plugin { freq_order: FreqOrder::Alternating, freq_seed: 1, freq_reference_every: 0, - freq_sweep_pending: false, + freq_sweep_pending: None, freq_sweep: None, + protocol_path: String::new(), + protocol_pending: false, + pending_duration_s: None, + protocol: None, a0_locks: Vec::new(), loaded_locks_folder: None, press_start: PressLatch::default(), @@ -807,6 +1065,8 @@ impl Default for StageAA1Plugin { press_clear_curve: PressLatch::default(), press_find_a0: PressLatch::default(), press_freq_sweep: PressLatch::default(), + press_freq_depth_sweep: PressLatch::default(), + press_run_protocol: PressLatch::default(), press_record_a0: PressLatch::default(), press_clear_a0: PressLatch::default(), } @@ -836,6 +1096,7 @@ impl Recording { lease_granted: false, cam_raw_path: None, cam_finalized_path: None, + sensor_readout_path: None, cam_complete: false, cam_rejected: false, pd_pdq_path: None, @@ -861,14 +1122,14 @@ impl Recording { fn state_label(&self) -> &'static str { match self.phase { - RecPhase::Idle => "idle", - RecPhase::StartingCamera => "starting camera", - RecPhase::ConnectingPhotodiode => "connecting photodiode", - RecPhase::AcquiringLease => "acquiring lease", - RecPhase::StartingPhotodiode => "starting photodiode", + RecPhase::Idle => "not recording", + RecPhase::StartingCamera => "starting the camera", + RecPhase::ConnectingPhotodiode => "connecting the photodiode", + RecPhase::AcquiringLease => "reserving the photodiode", + RecPhase::StartingPhotodiode => "starting the photodiode", RecPhase::Running => "recording", - RecPhase::StoppingPhotodiode => "finalizing photodiode", - RecPhase::StoppingCamera => "finalizing camera", + RecPhase::StoppingPhotodiode => "saving the photodiode data", + RecPhase::StoppingCamera => "saving the camera data", } } @@ -904,6 +1165,62 @@ impl StageAA1Plugin { self.dataset_generation = self.dataset_generation.wrapping_add(1); } + /// Releases the live analysis buffers and the fold memoised from them. + /// Returns whether anything was actually held. + /// + /// Assigning fresh `Vec`s rather than calling `clear()` is deliberate: the + /// event buffer reaches millions of entries, and `clear()` keeps every byte + /// of that capacity reserved. + /// + /// This is what switching Live analysis off has to do. Merely stopping the + /// *filling* left the last window's events live for every later + /// `current_fold()`, and the control tick re-folds them — so turning the + /// toggle off left the plugin folding millions of stale events on every + /// tick, forever. That is the lag that outlived the switch. + fn drop_live_buffers(&mut self) -> bool { + let held = !self.camera_events.is_empty() || !self.camera_markers_us.is_empty(); + self.camera_events = Vec::new(); + self.event_scratch = Vec::new(); + self.camera_markers_us = Vec::new(); + self.fold_cache.replace(None); + held + } + + /// One Stop for everything the Record section can start. + /// + /// Whatever is in flight — a single recording, a depth sweep, an `a₀` + /// lock, a frequency ladder or a protocol — asks it to wind down at its + /// next safe point, and any latched-but-not-yet-started press is dropped so + /// the stop is not immediately undone by a queued start. + fn request_stop(&mut self) { + if self.recording.is_active() { + self.recording.stop_requested = true; + } + if let Some(sweep) = self.sweep.as_mut() { + sweep.stop_requested = true; + self.message = "Sweep stop requested".into(); + } + if let Some(lock) = self.a0_lock.as_mut() { + lock.stop_requested = true; + self.message = "a₀ lock stop requested".into(); + } + // Before the protocol, so the outer runner's wording wins over the + // child it happens to be driving. + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.stop_requested = true; + self.message = "Frequency sweep stop requested".into(); + } + if let Some(protocol) = self.protocol.as_mut() { + protocol.stop_requested = true; + self.message = "Protocol stop requested".into(); + } + self.sweep_pending = false; + self.a0_lock_pending = false; + self.a0_point_pending = false; + self.freq_sweep_pending = None; + self.protocol_pending = false; + } + /// Sets the concise operator-facing recording result. fn note(&mut self, message: impl Into) { self.message = message.into(); @@ -965,6 +1282,59 @@ impl StageAA1Plugin { self.camera_markers_us.len() >= 2 } + /// Why there is no modulation frequency to work from, phrased as the + /// operator action that fixes it. `None` means the frequency is known. + /// + /// A frequency comes from two independent places — the phase-0 trigger + /// markers, or the drive the modulation plugin has *acknowledged*. Neither + /// has anything to do with whether that plugin is connected, which is a + /// separate check ([`Self::modulation_connected`]). Reporting "connect the + /// modulation plugin" for a missing frequency told operators their bench + /// was unplugged when it was not: the usual cause is simply that no + /// periodic drive has been armed yet. + fn frequency_blocker(&self) -> Option { + if self.frequency_hz().is_some() { + return None; + } + let Some(state) = self.modulation.as_ref() else { + return Some( + "the modulation plugin is not reporting status — enable it in the plugin list" + .into(), + ); + }; + if !matches!(state.connection, ConnectionStateV1::Connected { .. }) { + return Some(format!( + "the modulation plugin is {} — connect it", + connection_label(&state.connection) + )); + } + // Connected, so the question is what it is being asked to drive. + let Some(target) = state.acknowledged.as_ref() else { + return Some( + "the modulation plugin has not applied a drive yet — set one up there and apply it" + .into(), + ); + }; + match target.waveform.as_ref() { + Some(WaveformV1::Periodic { + frequency_millihz, .. + }) if *frequency_millihz == 0 => { + Some("the modulation drive frequency is set to 0 — raise it".into()) + } + Some(WaveformV1::Periodic { .. }) => None, + Some(_) => Some( + "the modulation drive is not a repeating waveform, so it has no frequency — \ + choose a periodic one" + .into(), + ), + None => Some( + "the modulation plugin has not applied a waveform yet — set one up there and \ + apply it" + .into(), + ), + } + } + fn frequency_source(&self) -> &'static str { if self.measured_period_us().is_some() { "trigger" @@ -1077,11 +1447,124 @@ impl StageAA1Plugin { } /// Fresh optical modulation depth `a` published by the photodiode plugin. - fn measured_a(&self) -> Option { + fn photodiode_a(&self) -> Option { self.fresh_optical_summary() .map(|summary| summary.measured_log_contrast) } + /// The depth `a` the modulation owner's calibrated optical drive is + /// currently commanding, from its published inversion provenance. + /// + /// `None` unless the owner is connected and has a calibrated optical drive + /// armed: `optical_drive` is published only for `OPTICAL_LOG_SINE` / + /// `OPTICAL_LINEAR_SINE` under an identified transfer calibration, which is + /// exactly the case in which a commanded `a` means anything at all. A + /// manual DAC band or a constant level publishes nothing here, and must not + /// be turned into a depth. + /// + /// Deliberately not freshness-gated. The published depth is what this + /// plugin's own `SetOpticalDepth` wrote into the owner a moment ago, not a + /// reading off the bench, so it does not go stale the way a measurement + /// does — and gating it on the device poll would make the sweep's settle + /// check flap between "settled" and "no a" on a slow reply. + fn commanded_a(&self) -> Option { + let state = self.modulation.as_ref()?; + if !matches!(state.connection, ConnectionStateV1::Connected { .. }) { + return None; + } + let depth = f64::from(state.optical_drive.as_ref()?.depth_a_milli) / 1_000.0; + (depth > 0.0).then_some(depth) + } + + /// The modulation depth `a` A1 works from, per the operator's chosen + /// [`DepthSource`]. Every gate, sweep settle check, plot and sidecar reads + /// this one accessor, so the source is chosen in exactly one place. + fn depth_a(&self) -> Option { + match self.depth_source { + DepthSource::Photodiode => self.photodiode_a(), + DepthSource::Commanded => self.commanded_a(), + } + } + + /// Why there is no `a` to work from, phrased as the operator action that + /// fixes it. + /// + /// Every gate that needs `a` — the a₀ lock, the amplitude sweep, the + /// frequency ladder — used to refuse with one fixed sentence naming the two + /// most common causes. When the real cause was a third thing (a railed + /// window, too few trigger markers, a stale snapshot) that sentence sent the + /// operator to re-check an anchor that was already fine. Ask the owner + /// instead, and only fall back to the local view of the snapshot. + /// + /// `None` means `a` is available. + fn depth_a_blocker(&self) -> Option { + match self.depth_source { + DepthSource::Photodiode => self.photodiode_a_blocker(), + DepthSource::Commanded => self.commanded_a_blocker(), + } + } + + fn photodiode_a_blocker(&self) -> Option { + if self.photodiode_a().is_some() { + return None; + } + // Every branch names the photodiode gate to fix *and* the way past it, + // because a bench that cannot produce a measured `a` at all — no + // trigger markers, say — otherwise leaves the operator with a correct + // diagnosis and no next step. + let fallback = " (or switch \"Depth a source\" to the commanded drive to work open loop)"; + let Some(state) = self.photodiode.as_ref() else { + return Some(format!( + "the photodiode plugin is not reporting status — enable it and connect the \ + detector{fallback}" + )); + }; + if !matches!(state.connection, ConnectionStateV1::Connected { .. }) { + return Some(format!( + "the photodiode is {} — connect it{fallback}", + connection_label(&state.connection) + )); + } + if state.freshness.is_stale_at(now_unix_ms()) { + return Some(format!( + "the photodiode status snapshot is stale — check that the stream is \ + running{fallback}" + )); + } + // The owner's own words: it is the only side that knows which estimator + // gate rejected the window. + if let Some(reason) = state.optical_unavailable.as_deref() { + return Some(format!("{reason}{fallback}")); + } + Some(format!( + "the photodiode is streaming no samples yet — start the stream{fallback}" + )) + } + + fn commanded_a_blocker(&self) -> Option { + if self.commanded_a().is_some() { + return None; + } + let Some(state) = self.modulation.as_ref() else { + return Some( + "the modulation plugin is not reporting status — enable it to read the commanded \ + depth" + .into(), + ); + }; + if !matches!(state.connection, ConnectionStateV1::Connected { .. }) { + return Some(format!( + "the modulation plugin is {} — connect it", + connection_label(&state.connection) + )); + } + Some( + "the modulation plugin is not running a calibrated optical drive — apply a Pockels \ + calibration and arm OPTICAL_LOG_SINE, or a commanded depth means nothing" + .into(), + ) + } + /// Current ROI from the host camera config, clamped to the frame. fn roi(&self) -> Option { if self.frame_width == 0 || self.frame_height == 0 { @@ -1169,11 +1652,15 @@ impl StageAA1Plugin { } } - /// Records one response-curve point at the current photodiode-measured `a`. + /// Records one response-curve point at the current depth `a`. fn record_response_point(&mut self) -> Result<(), String> { - let measured_a = self - .measured_a() - .ok_or("no photodiode-measured a available (connect the photodiode)")?; + let measured_a = self.depth_a().ok_or_else(|| { + format!( + "no modulation depth a available: {}", + self.depth_a_blocker() + .unwrap_or_else(|| "no depth source is reporting".into()) + ) + })?; let (q_on, q_off, cycles, valid_pixels) = self .current_response() .ok_or("no valid response window yet (need trigger-anchored events and a valid ROI)")?; @@ -1201,7 +1688,10 @@ impl StageAA1Plugin { points }; Series1dV1 { - x_label: "Measured modulation depth a = ln(I_max / I_min)".into(), + x_label: format!( + "Modulation depth a = ln(I_max / I_min) ({})", + self.depth_source.verb() + ), y_label: "Response probability q_p = fraction of pixel-cycles that fired".into(), lines: vec![ Series1dLine { @@ -1319,8 +1809,10 @@ impl StageAA1Plugin { ), cell( "a", - self.measured_a() - .map_or_else(|| "—".into(), |a| format!("{a:.3}")), + self.depth_a().map_or_else( + || "—".into(), + |a| format!("{a:.3} ({})", self.depth_source.verb()), + ), ), cell( "s_on", @@ -1422,7 +1914,6 @@ impl StageAA1Plugin { fn recording_metadata(&self) -> BTreeMap { let mut meta = BTreeMap::new(); meta.insert("a1_measurement_id".into(), self.recording.id.clone()); - meta.insert("a1_flux_point_id".into(), self.flux_point_id.clone()); meta.insert("a1_stem".into(), self.recording.stem.clone()); meta.insert("a1_role".into(), self.recording.role.label().into()); meta.insert( @@ -1454,12 +1945,24 @@ impl StageAA1Plugin { "a0_lock_measured_a".into(), format!("{:.6}", lock.measured_a), ); + meta.insert( + "a0_lock_depth_source".into(), + lock.depth_source.label().into(), + ); meta.insert( "a0_lock_frequency_hz".into(), format!("{:.6}", lock.frequency_hz), ); } - if let Some(a) = self.measured_a() { + // The depth this run was driven and judged by, always tagged with where + // it came from. `measured_a` keeps its historical meaning — a number the + // photodiode actually measured — so an open-loop run simply does not + // carry one, rather than carrying a commanded value under that name. + meta.insert("depth_a_source".into(), self.depth_source.label().into()); + if let Some(a) = self.depth_a() { + meta.insert("depth_a".into(), format!("{a:.6}")); + } + if let Some(a) = self.photodiode_a() { meta.insert("measured_a".into(), format!("{a:.6}")); } if let Some(hz) = self.period_us().map(|t| 1_000_000.0 / t) { @@ -1477,9 +1980,57 @@ impl StageAA1Plugin { if let Some(n) = self.valid_pixel_count() { meta.insert("n_valid".into(), n.to_string()); } + // Bench conditions, on every run and every role. Each key appears only + // when the sensor actually reported that quantity — an absent reading + // must not arrive downstream as 0 °C or 0 lux. + if let Some(sensor) = self.recorded_sensor() { + if let Some(celsius) = sensor.temperature_c { + meta.insert("sensor_temperature_c".into(), format!("{celsius:.2}")); + } + if let Some(dead_time_us) = sensor.pixel_dead_time_us { + meta.insert( + "sensor_pixel_dead_time_us".into(), + format!("{dead_time_us:.3}"), + ); + } + if let Some(lux) = sensor.illumination_lux { + meta.insert("sensor_illumination_lux".into(), format!("{lux:.3}")); + } + meta.insert( + "sensor_reading_age_s".into(), + format!("{:.3}", sensor.age_s), + ); + } meta } + /// The sensor reading that belongs to the run being written: the one frozen + /// when it started, falling back to the latest if the recording began + /// before any frame carried one. + fn recorded_sensor(&self) -> Option { + self.sensor_at_start.or(self.sensor) + } + + /// The measurement id to file this run under, generating one when the + /// operator has not typed anything. + /// + /// A blank id used to refuse the recording. It never had to: the id only + /// names a folder and a file stem, and the plugin already ships a generated + /// default for exactly that reason. Filling it in here (and writing it back, + /// so the panel shows what was used) means an operator who wants their data + /// grouped can say so, and one who just wants to record can press record. + /// Tested on the raw field, not on `sanitize_stem`'s output: the sanitizer + /// substitutes `A1` for anything that reduces to nothing, so asking it + /// whether the id was blank always answers no — and every unnamed run would + /// silently share one folder called `A1`. + fn ensure_measurement_id(&mut self) -> String { + if self.measurement_id.trim().is_empty() { + self.measurement_id = generate_measurement_id(); + self.bump(); + } + sanitize_stem(self.measurement_id.trim()) + } + /// Why the photodiode cannot record right now, phrased as the operator /// action that fixes it. `None` means the PDQ leg is expected to succeed. fn photodiode_blocker(&self) -> Option { @@ -1517,15 +2068,7 @@ impl StageAA1Plugin { return; } if self.output_folder.trim().is_empty() { - self.note("Set an output folder before recording"); - return; - } - if self.measurement_id.trim().is_empty() { - self.note("Set a measurement id before recording"); - return; - } - if self.flux_point_id.trim().is_empty() { - self.note("Set the physical I_k flux point id before recording"); + self.note("Pick an output folder first — that is where the files go"); return; } // Checked before the camera starts: every one of these used to surface @@ -1536,18 +2079,33 @@ impl StageAA1Plugin { return; } let now_ms = now_unix_ms(); - let id = sanitize_stem(self.measurement_id.trim()); + // Freeze the bench conditions this run begins under, before any of the + // start handshake has had time to move them. + self.sensor_at_start = self.sensor; + let id = self.ensure_measurement_id(); // Sweep points get a stable per-point tag so the row's files sort by // sweep order as well as by timestamp. Event-count points instead carry // their frequency, because one measurement id spans the whole frequency // sweep at the single frozen depth a₀. let live_hz = self.frequency_hz(); + // A depth sweep nested inside the frequency ladder repeats its point + // indices at every rung, so `_p03` alone would collide across + // frequencies within one measurement id. Prefix the ladder's frequency + // so the whole q_p(a, f) surface sorts by f, then by depth. + let nested_freq_tag = self + .freq_sweep + .as_ref() + .filter(|sweep| sweep.mode == FreqSweepMode::DepthSweep) + .map(|sweep| format!("_{}", frequency_tag(sweep.frequency_hz()))) + .unwrap_or_default(); let sweep_tag = self .sweep .as_ref() .filter(|sweep| sweep.phase == SweepPhase::Recording) .map(|sweep| match sweep.kind { - SweepKind::Amplitude => format!("_p{:02}", sweep.index + 1), + SweepKind::Amplitude => { + format!("{nested_freq_tag}_p{:02}", sweep.index + 1) + } SweepKind::EventCount => { let hz = sweep .lock @@ -1571,7 +2129,11 @@ impl StageAA1Plugin { recording.id = id; recording.stem = stem; recording.folder = self.output_folder.trim().to_string(); - recording.duration_s = self.duration_s.max(1) as u64; + recording.duration_s = self + .pending_duration_s + .take() + .unwrap_or(self.duration_s) + .max(1) as u64; // The measurement clock starts only after both recorders acknowledge // that they are running. recording.start_unix_ms = 0; @@ -1822,6 +2384,11 @@ impl StageAA1Plugin { if let Some(bias) = sibling_toml(&raw) { move_into(&dir, &bias); } + // The host's sensor telemetry is written as another sibling of the + // RAW and used to be left behind entirely, which separated a run + // from the bench conditions it was taken under at the first move. + // It is rewritten column-wise on the way in — see `sensor`. + self.recording.sensor_readout_path = self.gather_sensor_readout(&dir, &raw); } // PDQ receipts report the *label* A1 asked for, which is relative to the // photodiode's data directory — resolve it before touching the file, and @@ -1836,6 +2403,39 @@ impl StageAA1Plugin { } } + /// Compacts the host's sensor-telemetry CSV into the measurement folder, + /// under the recording's own stem, and removes the original. + /// + /// Best-effort throughout: a missing or unreadable telemetry file is normal + /// (replay, a camera with no monitoring block, a host that did not poll) + /// and must not cost the operator the recording that has just finished. + fn gather_sensor_readout(&self, dir: &Path, raw: &str) -> Option { + let source = Path::new(raw) + .file_stem() + .map(|stem| { + Path::new(raw) + .parent() + .unwrap_or(Path::new(".")) + .join(format!("{}.sensor-monitoring.csv", stem.to_string_lossy())) + }) + .filter(|path| path.exists())?; + let text = std::fs::read_to_string(&source).ok()?; + let readout = sensor::parse_csv(&text); + if readout.is_empty() { + // Nothing worth keeping, but the wide original is still clutter in + // the host's capture folder. + let _ = std::fs::remove_file(&source); + return None; + } + let destination = dir.join(format!("{}.sensor.json", self.recording.stem)); + let json = readout.to_json(&self.recording.id, &self.recording.stem); + if std::fs::write(&destination, json).is_err() { + return None; + } + let _ = std::fs::remove_file(&source); + Some(destination.display().to_string()) + } + /// Absolute location of a photodiode-reported recording path. Receipts name /// the *label* A1 asked for, which is relative to whichever root the owner /// used: the folder A1 named in the start spec, or — for an owner too old to @@ -1941,19 +2541,13 @@ impl StageAA1Plugin { return; } if self.output_folder.trim().is_empty() { - self.message = "Set an output folder before sweeping".into(); - return; - } - if self.measurement_id.trim().is_empty() { - self.message = "Set a measurement id before sweeping".into(); - return; - } - if self.flux_point_id.trim().is_empty() { - self.message = "Set the physical I_k flux point id before sweeping".into(); + self.message = "Pick an output folder first — that is where the files go".into(); return; } if !self.modulation_connected() { - self.message = "Modulation owner is not connected — cannot sweep".into(); + self.message = "The modulation plugin is not connected — connect it to drive the \ + depth" + .into(); return; } if self @@ -1962,23 +2556,35 @@ impl StageAA1Plugin { .and_then(|state| state.calibration_id.as_deref()) .is_none() { - self.message = - "Apply a measured Pockels transfer calibration before starting an A1 sweep".into(); + self.message = "Run the Pockels calibration in the modulation plugin first — without \ + it a commanded depth means nothing" + .into(); return; } - if self.fresh_optical_summary().is_none() { - self.message = "Connect the photodiode and obtain a fresh, marker-bounded optical \ - summary from a confirmed I_tot anchor before sweeping" - .into(); + // Same wording every other gate uses, from the same helper: the owner + // knows which estimator gate withheld `a`, and a fixed sentence here + // used to send the operator after the wrong thing. + if let Some(reason) = self.depth_a_blocker() { + self.message = format!( + "The sweep needs a {} depth a, but {reason}", + self.depth_source.verb() + ); + return; + } + // The sweep ends in a recording, so ask the recording's own question now + // rather than after the drive has already moved to point 1. + if let Some(blocker) = self.photodiode_blocker() { + self.message = blocker; return; } if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { - self.message = - "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); + self.message = "Set Sweep min a above 0 — a = 0 is the background reference, which \ + has its own button" + .into(); return; } if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { - self.message = "Sweep needs max a > min a".into(); + self.message = "Sweep max a must be larger than Sweep min a".into(); return; } let points = self.sweep_points(); @@ -2006,15 +2612,12 @@ impl StageAA1Plugin { return; } if self.output_folder.trim().is_empty() { - self.message = "Set an output folder before recording".into(); - return; - } - if self.measurement_id.trim().is_empty() { - self.message = "Set a measurement id before recording".into(); + self.message = "Pick an output folder first — that is where the files go".into(); return; } if !self.modulation_connected() { - self.message = "Modulation owner is not connected — cannot drive the depth".into(); + self.message = + "The modulation plugin is not connected — connect it to drive the depth".into(); return; } if points.is_empty() { @@ -2022,6 +2625,10 @@ impl StageAA1Plugin { return; } let now_ms = now_unix_ms(); + // A verdict belongs to the run that produced it. Clearing it here means + // an enclosing ladder can never read the previous rung's outcome if + // this one ends without reaching `finish_sweep`. + self.last_sweep_completed_ok = false; let owns_lease = inherited_lease.is_none(); let lease_id = inherited_lease.unwrap_or_else(|| { LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))) @@ -2051,6 +2658,7 @@ impl StageAA1Plugin { settled_since_ms: None, settle_deadline_ms: 0, point_started: false, + completed_ok: false, last_activity_ms: now_ms, stop_requested: false, }); @@ -2068,14 +2676,11 @@ impl StageAA1Plugin { context: &mut impl RecordingControl, inherited_lease: Option, ) { - let Some(hz) = self.frequency_hz() else { - self.message = "No modulation frequency yet — arm the drive first".into(); - return; - }; - let Some(lock) = self.armed_lock().cloned() else { + let Some(lock) = self.armed_a0() else { self.message = format!( - "No converged a₀ lock for {} — press Find a₀ at this frequency first", - frequency_label(hz) + "Cannot record the a₀ point: {}", + self.armed_a0_blocker() + .unwrap_or_else(|| "no depth is armed".into()) ); return; }; @@ -2102,6 +2707,7 @@ impl StageAA1Plugin { /// Release the modulation lease (if held) and clear the sweep. fn finish_sweep(&mut self, context: &mut impl RecordingControl, message: String) { if let Some(sweep) = self.sweep.take() { + self.last_sweep_completed_ok = sweep.completed_ok; if sweep.owns_lease && sweep.lease_granted { let request = self.modulation_request( ModulationCommandV1::ReleaseLease { @@ -2157,7 +2763,8 @@ impl StageAA1Plugin { ) } else { format!( - "Event-count point: commanding a = {commanded_a:.3} for a measured a₀ = {target_a:.3}…" + "Event-count point: commanding a = {commanded_a:.3} for a {} a₀ = {target_a:.3}…", + self.depth_source.verb() ) }; } @@ -2251,8 +2858,14 @@ impl StageAA1Plugin { SweepKind::Amplitude => sweep_tolerance(target), SweepKind::EventCount => self.a0_tolerance.max(1e-3), }; + // With `DepthSource::Commanded` this compares the commanded + // depth against itself and settles as soon as the owner has + // applied it — which is the honest answer for an open-loop + // sweep: nothing on the bench can contradict the command. The + // operator's settle dwell below still applies, so the drive + // gets its physical time to move either way. let settled = self - .measured_a() + .depth_a() .is_some_and(|measured| (measured - target).abs() <= tolerance); let dwell_ms = (self.settle_s.max(0.0) * 1_000.0) as u64; let mut start_recording = false; @@ -2315,6 +2928,9 @@ impl StageAA1Plugin { SweepKind::Amplitude => format!("Sweep complete: {total} points recorded"), SweepKind::EventCount => self.message.clone(), }; + if let Some(sweep) = self.sweep.as_mut() { + sweep.completed_ok = true; + } self.finish_sweep(context, message); } else { if let Some(sweep) = self.sweep.as_mut() { @@ -2390,33 +3006,133 @@ impl StageAA1Plugin { /// The lock that applies to the drive right now: same frequency, converged, /// and aimed at the `a₀` currently entered. + /// + /// "Aimed at the same `a₀`" is judged against the operator's own convergence + /// tolerance, not on exact equality. The a₀ field is a drag control with a + /// 0.01 step, so a strict comparison disarmed a lock the operator had just + /// found the moment they nudged the slider — and then asked them to press + /// Find a₀ again, which is what they had done. fn armed_lock(&self) -> Option<&A0LockPoint> { let hz = self.frequency_hz()?; + let tolerance = self.a0_tolerance.max(1e-3); self.lock_for_frequency(hz) - .filter(|lock| lock.converged && (lock.target_a - self.a0_target).abs() <= 1e-6) + .filter(|lock| lock.converged && (lock.target_a - self.a0_target).abs() <= tolerance) } - fn a0_locks_path(&self) -> Option { - let folder = self.output_folder.trim(); - (!folder.is_empty()).then(|| Path::new(folder).join(A0_LOCK_FILE)) + /// The depth an `a₀` recording would be made at right now — the one + /// question every consumer of the lock table actually asks. + /// + /// With a measured depth source this is a stored, converged lock: the + /// commanded depth that was *found* to produce `a₀` at this frequency, and + /// there is no answer until [`Self::begin_a0_lock`] has found one. + /// + /// With a commanded depth source there is nothing to look up. `a₀` is + /// commanded directly, at every frequency, so the answer is always + /// available and is synthesised here rather than round-tripped through a + /// table of identical rows ([`DepthSource::needs_a0_lock`]). `trials: 0` + /// records honestly that no search happened. + fn armed_a0(&self) -> Option { + if self.depth_source.needs_a0_lock() { + return self.armed_lock().cloned(); + } + let hz = self.frequency_hz()?; + let target = self.a0_target; + (COMMANDED_A_MIN..=COMMANDED_A_MAX) + .contains(&target) + .then(|| A0LockPoint { + frequency_hz: hz, + target_a: target, + commanded_a: clamp_commanded_a(target), + measured_a: target, + trials: 0, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: None, + high_clip_fraction: None, + depth_source: self.depth_source, + }) } - /// Store a finished lock, replacing any earlier one at the same frequency, - /// and mirror the table to disk. Returns a save failure for the caller to - /// append to its own message. - fn store_lock(&mut self, lock: A0LockPoint) -> Result<(), String> { - self.a0_locks - .retain(|existing| !same_frequency(existing.frequency_hz, lock.frequency_hz)); - self.a0_locks.push(lock); - self.a0_locks - .sort_by(|left, right| left.frequency_hz.total_cmp(&right.frequency_hz)); - self.save_a0_locks() + /// Why no `a₀` recording can be made right now, phrased as the operator + /// action that fixes it. `None` means [`Self::armed_a0`] has an answer. + fn armed_a0_blocker(&self) -> Option { + if self.depth_source.needs_a0_lock() { + return self.armed_lock_blocker(); + } + if self.armed_a0().is_some() { + return None; + } + if self.frequency_hz().is_none() { + return Some(format!( + "there is no modulation frequency yet: {}", + self.frequency_blocker() + .unwrap_or_else(|| "no drive is armed".into()) + )); + } + Some(format!( + "a₀ = {:.3} is outside the drivable {COMMANDED_A_MIN}..={COMMANDED_A_MAX}", + self.a0_target + )) } - /// Persist the lock table next to the recordings, so the found depths survive - /// a restart and can be cited offline. + /// Why the stored locks do not arm a recording at the current frequency, + /// phrased as the operator action that fixes it. /// - /// Returns the failure so the caller can append it to its own message: a + /// The three causes — no lock at this frequency, a lock that did not + /// converge, a lock aimed at a different a₀ — used to share one sentence + /// telling the operator to press Find a₀, which only helps for the first. + fn armed_lock_blocker(&self) -> Option { + if self.armed_lock().is_some() { + return None; + } + let Some(hz) = self.frequency_hz() else { + return Some(format!( + "there is no modulation frequency yet: {}", + self.frequency_blocker() + .unwrap_or_else(|| "no drive is armed".into()) + )); + }; + let label = frequency_label(hz); + let Some(lock) = self.lock_for_frequency(hz) else { + return Some(format!( + "no depth has been found for {label} yet — press Find a₀ at this frequency" + )); + }; + if !lock.converged { + return Some(format!( + "the last Find a₀ at {label} did not reach a₀ (it stopped at a measured {:.3}) — \ + press Find a₀ again, or widen the a₀ tolerance", + lock.measured_a + )); + } + Some(format!( + "the depth found for {label} was aimed at a₀ = {:.3}, and a₀ is now {:.3} — press \ + Find a₀ again at the new a₀", + lock.target_a, self.a0_target + )) + } + + fn a0_locks_path(&self) -> Option { + let folder = self.output_folder.trim(); + (!folder.is_empty()).then(|| Path::new(folder).join(A0_LOCK_FILE)) + } + + /// Store a finished lock, replacing any earlier one at the same frequency, + /// and mirror the table to disk. Returns a save failure for the caller to + /// append to its own message. + fn store_lock(&mut self, lock: A0LockPoint) -> Result<(), String> { + self.a0_locks + .retain(|existing| !same_frequency(existing.frequency_hz, lock.frequency_hz)); + self.a0_locks.push(lock); + self.a0_locks + .sort_by(|left, right| left.frequency_hz.total_cmp(&right.frequency_hz)); + self.save_a0_locks() + } + + /// Persist the lock table next to the recordings, so the found depths survive + /// a restart and can be cited offline. + /// + /// Returns the failure so the caller can append it to its own message: a /// lock the operator can see on screen but that never reached disk is a /// lock they will not have after a restart. fn save_a0_locks(&mut self) -> Result<(), String> { @@ -2474,6 +3190,18 @@ impl StageAA1Plugin { self.message = "A recording, sweep or a₀ lock is already running".into(); return; } + // There is nothing to search for when `a` *is* the command: the search + // would command a₀, read back a₀ and stop. Say so instead of spending + // a lease and a trial to arrive back where the operator already is. + if !self.depth_source.needs_a0_lock() { + self.message = format!( + "No search needed: with the depth coming from the commanded drive, a₀ = {:.3} is \ + simply commanded at every frequency. Press \"Record a₀ point\", or \"Record all \ + frequencies\" for the whole ladder.", + self.a0_target + ); + return; + } if !self.modulation_connected() { self.message = "Modulation owner is not connected — cannot find a₀".into(); return; @@ -2486,12 +3214,18 @@ impl StageAA1Plugin { return; } let Some(hz) = self.frequency_hz() else { - self.message = "No modulation frequency yet — arm the drive before finding a₀".into(); + self.message = format!( + "Cannot find a₀ without a modulation frequency: {}", + self.frequency_blocker() + .unwrap_or_else(|| "no drive is armed".into()) + ); return; }; - if self.measured_a().is_none() { - self.message = - "No photodiode-measured a — connect the photodiode and anchor I_tot first".into(); + if let Some(reason) = self.depth_a_blocker() { + self.message = format!( + "Cannot find a₀ without a {} a: {reason}", + self.depth_source.verb() + ); return; } // Refuse before touching the drive, not after eight trials of chasing a @@ -2592,8 +3326,9 @@ impl StageAA1Plugin { lock.last_activity_ms = now_ms; } self.message = format!( - "a₀ lock trial {trial}/{A0_LOCK_MAX_TRIALS}: commanding a = {commanded:.3} for a \ - measured a₀ = {target:.3}…" + "a₀ lock trial {trial}/{A0_LOCK_MAX_TRIALS}: commanding a = {commanded:.3} for a {} \ + a₀ = {target:.3}…", + self.depth_source.verb() ); } @@ -2623,6 +3358,11 @@ impl StageAA1Plugin { /// one inside the estimate. Owners that predate the field do not publish /// it; then only the operator's settle dwell is available. fn optical_window_seconds(&self) -> Option { + // A window only bounds a depth that is read out of it. A commanded + // depth is not, so there is nothing here to wait for or to check. + if self.depth_source != DepthSource::Photodiode { + return None; + } self.photodiode .as_ref()? .optical_summary @@ -2674,19 +3414,23 @@ impl StageAA1Plugin { /// otherwise they share nearly all their samples and three of them say no /// more than one. fn sample_a0_measurement(&mut self, now_ms: u64) { - let Some((revision, measured)) = self.photodiode.as_ref().and_then(|summary| { - summary - .optical_summary - .as_ref() - .map(|optical| (summary.service_revision, optical.measured_log_contrast)) - }) else { + let Some((revision, measured)) = self.depth_reading() else { return; }; let spacing_ms = self.a0_sample_spacing_ms(); + // The stale-window rule exists because the photodiode's estimate mixes + // samples from before and after the depth changed. A commanded depth is + // not read out of a window at all — it is the value that was just + // applied — so holding it to the same rule would only make the trial + // depend on the modulation owner's device-poll cadence, and time out + // whenever that owner had nothing new to say. + let requires_new_revision = self.depth_source == DepthSource::Photodiode; let Some(lock) = self.a0_lock.as_mut() else { return; }; - if now_ms < lock.measure_from_ms || lock.sampled_revision == Some(revision) { + if now_ms < lock.measure_from_ms + || (requires_new_revision && lock.sampled_revision == Some(revision)) + { return; } lock.sampled_revision = Some(revision); @@ -2694,6 +3438,30 @@ impl StageAA1Plugin { lock.measure_from_ms = now_ms.saturating_add(spacing_ms); } + /// One depth reading from the active [`DepthSource`], tagged with the + /// publishing owner's service revision. + /// + /// The revision is what makes a reading *independent*: the lock only counts + /// values published after it commanded the depth, so a trial never averages + /// in the previous one. Both owners bump their revision on every state + /// change, so the same rule works for either source. + fn depth_reading(&self) -> Option<(u64, f64)> { + match self.depth_source { + DepthSource::Photodiode => self.photodiode.as_ref().and_then(|summary| { + summary + .optical_summary + .as_ref() + .map(|optical| (summary.service_revision, optical.measured_log_contrast)) + }), + DepthSource::Commanded => self.commanded_a().map(|a| { + ( + self.modulation.as_ref().map_or(0, |s| s.service_revision), + a, + ) + }), + } + } + /// Minimum gap between two readings of one trial. fn a0_sample_spacing_ms(&self) -> u64 { let window_ms = self @@ -2706,6 +3474,11 @@ impl StageAA1Plugin { /// Photodiode clipping note for a lock message, empty when the windows are clean. fn clip_warning(&self) -> String { + // A clipped detector window says nothing about a commanded depth, and + // appending it to that lock's message would suggest it did. + if self.depth_source != DepthSource::Photodiode { + return String::new(); + } let Some(optical) = self .photodiode .as_ref() @@ -2740,15 +3513,15 @@ impl StageAA1Plugin { let mut readings = lock.samples.clone(); if readings.is_empty() { // The owner withholds `a` for a stated reason (clipping, no - // headroom, a bad `I_tot` anchor, a sub-cycle window). It does not - // publish the reason on the contract, so name the likely ones - // rather than leave the operator with "nothing happened". + // headroom, a bad `I_tot` anchor, a sub-cycle window). Ask the + // blocker for it rather than leaving the operator with "nothing + // happened" — and it answers for whichever source is selected. + let reason = self + .depth_a_blocker() + .unwrap_or_else(|| "it published nothing while the lock was measuring".into()); self.finish_a0_lock( context, - "a₀ lock aborted: the photodiode published no a while measuring — it withholds \ - one when the window clips, has no headroom above dark, the I_tot anchor is \ - below the signal, or the window is shorter than one modulation cycle" - .into(), + format!("a₀ lock aborted: no depth a arrived while measuring — {reason}"), ); return; } @@ -2759,8 +3532,9 @@ impl StageAA1Plugin { self.finish_a0_lock( context, format!( - "a₀ lock aborted: the photodiode measured a = {measured:.3} — check the I_tot \ - anchor and that the drive is modulating" + "a₀ lock aborted: the {} a = {measured:.3} — check the I_tot anchor and that \ + the drive is modulating", + self.depth_source.verb() ), ); return; @@ -2771,7 +3545,7 @@ impl StageAA1Plugin { self.finish_a0_lock( context, format!( - "a₀ lock aborted at {}: the measured a is not settled — {} readings spread \ + "a₀ lock aborted at {}: the observed a is not settled — {} readings spread \ {spread:.3} across {}× the ±{tolerance:.3} tolerance (median {measured:.3}). \ Increase Sweep settle (s) or check the drive and the I_tot anchor", frequency_label(hz), @@ -2797,8 +3571,9 @@ impl StageAA1Plugin { lock.trial += 1; } self.message = format!( - "a₀ lock trial {trial}: measured a = {measured:.3} vs a₀ = {target:.3} — \ - correcting the commanded depth to {next:.3}" + "a₀ lock trial {trial}: {} a = {measured:.3} vs a₀ = {target:.3} — correcting the \ + commanded depth to {next:.3}", + self.depth_source.verb() ); self.send_a0_depth(context); return; @@ -2818,22 +3593,29 @@ impl StageAA1Plugin { locked_at_unix_ms: now_unix_ms(), low_clip_fraction: optical.map(|optical| optical.low_clip_fraction), high_clip_fraction: optical.map(|optical| optical.high_clip_fraction), + depth_source: self.depth_source, }); let label = frequency_label(hz); + // "measures" is a claim about the light. Open loop the lock has only + // confirmed that the drive accepted the depth, so say that instead. + let verb = match self.depth_source { + DepthSource::Photodiode => "measures", + DepthSource::Commanded => "is commanded as", + }; let message = if converged { format!( - "a₀ locked at {label}: commanded a = {commanded:.3} measures a = {measured:.3} \ + "a₀ locked at {label}: commanded a = {commanded:.3} {verb} a = {measured:.3} \ (a₀ = {target:.3}, {trial} trial(s)){}", self.clip_warning() ) } else if railed { format!( "a₀ lock stopped at {label}: commanded a = {commanded:.3} is at the drivable limit \ - and only measures a = {measured:.3} — lower a₀ or the operating point I_k" + and only {verb} a = {measured:.3} — lower a₀ or the operating point I_k" ) } else { format!( - "a₀ lock did not converge at {label}: best commanded a = {commanded:.3} measures \ + "a₀ lock did not converge at {label}: best commanded a = {commanded:.3} {verb} \ a = {measured:.3} after {trial} trials — widen the tolerance or check the drive" ) }; @@ -2899,10 +3681,7 @@ impl StageAA1Plugin { let dwell_ms = ((self.settle_s.max(0.0) * 1_000.0) as u64).max(window_ms); // Only summaries published *after* this depth was commanded // count, so the trial never averages the previous depth. - let published = self - .photodiode - .as_ref() - .map(|summary| summary.service_revision); + let published = self.depth_reading().map(|(revision, _)| revision); if let Some(lock) = self.a0_lock.as_mut() { lock.phase = A0LockPhase::Measuring; lock.window_ms = window_ms; @@ -3067,12 +3846,20 @@ impl StageAA1Plugin { points } - /// Lease TTL for the whole ladder: every point pays a lock and a recording. - fn freq_sweep_lease_ttl_ms(&self, remaining_points: usize) -> u64 { - let per_point_ms = self - .a0_lock_lease_ttl_ms() - .saturating_add(self.sweep_lease_ttl_ms(1)) - .saturating_add(FREQ_CONFIRM_BASE_MS); + /// Lease TTL for the whole ladder: what one rung costs, times the rungs + /// left. A depth-sweep rung is a whole inner sweep, so it is the expensive + /// one by a factor of the point count — a TTL sized for an `a₀` point would + /// expire mid-curve and hand the drive back to the operator's settings. + fn freq_sweep_lease_ttl_ms(&self, remaining_points: usize, mode: FreqSweepMode) -> u64 { + let inner_ms = match mode { + FreqSweepMode::A0Point => self + .a0_lock_lease_ttl_ms() + .saturating_add(self.sweep_lease_ttl_ms(1)), + FreqSweepMode::DepthSweep => { + self.sweep_lease_ttl_ms(self.sweep_count.clamp(2, 64) as usize) + } + }; + let per_point_ms = inner_ms.saturating_add(FREQ_CONFIRM_BASE_MS); (remaining_points as u64) .saturating_mul(per_point_ms) .saturating_add(60_000) @@ -3083,7 +3870,7 @@ impl StageAA1Plugin { /// Everything checkable is checked *here*, before the drive moves: a plan /// that cannot work at its lowest frequency should say so in a message, not /// two hours into a block. - fn begin_freq_sweep(&mut self, context: &mut impl RecordingControl) { + fn begin_freq_sweep(&mut self, context: &mut impl RecordingControl, mode: FreqSweepMode) { if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() @@ -3093,15 +3880,21 @@ impl StageAA1Plugin { return; } if self.output_folder.trim().is_empty() { - self.message = "Set an output folder before sweeping the frequency".into(); + self.message = "Pick an output folder first — that is where the files go".into(); return; } - if self.measurement_id.trim().is_empty() { - self.message = "Set a measurement id before sweeping the frequency".into(); + if !self.modulation_connected() { + self.message = "The modulation plugin is not connected — connect it to drive the \ + frequency" + .into(); return; } - if !self.modulation_connected() { - self.message = "Modulation owner is not connected — cannot drive the frequency".into(); + // Every point of the ladder ends in a recording, so ask the recording's + // own question here. It used to be asked for the first time three stages + // in, at point 1, after the drive had already been retargeted — which is + // how the panel came to read "Recording: idle" mid-ladder. + if let Some(blocker) = self.photodiode_blocker() { + self.message = blocker; return; } // Written through `partial_cmp` so a NaN from the settings drag is @@ -3112,21 +3905,45 @@ impl StageAA1Plugin { Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) ); if !range_ok { - self.message = "Frequency sweep needs 0 < min f ≤ max f".into(); + self.message = "Set Sweep min f above 0 and Sweep max f at or above it".into(); return; } - if self.measured_a().is_none() { - self.message = - "No photodiode-measured a — connect the photodiode and anchor I_tot first".into(); - return; - } - let target = self.a0_target; - if !(COMMANDED_A_MIN..=COMMANDED_A_MAX).contains(&target) { + if let Some(reason) = self.depth_a_blocker() { self.message = format!( - "a₀ = {target:.3} is outside the drivable {COMMANDED_A_MIN}..={COMMANDED_A_MAX}" + "The frequency sweep needs a {} depth a, but {reason}", + self.depth_source.verb() ); return; } + // Each mode reads a different depth setting, so each validates its own. + match mode { + FreqSweepMode::A0Point => { + let target = self.a0_target; + if !(COMMANDED_A_MIN..=COMMANDED_A_MAX).contains(&target) { + self.message = format!( + "a₀ = {target:.3} is outside the drivable \ + {COMMANDED_A_MIN}..={COMMANDED_A_MAX}" + ); + return; + } + } + FreqSweepMode::DepthSweep => { + // Same two questions `begin_sweep` asks of the depth range, + // asked here before the drive moves rather than at the first + // rung — a ladder that cannot record its inner sweep should say + // so on the button press. + if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { + self.message = "Set Sweep min a above 0 — a = 0 is the background reference, \ + which has its own button" + .into(); + return; + } + if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { + self.message = "Sweep max a must be larger than Sweep min a".into(); + return; + } + } + } // The photodiode estimates `a` over one window for all frequencies, so // the *lowest* planned frequency decides whether the ladder is // measurable at all. Refuse the plan, not its 9th point. @@ -3134,13 +3951,25 @@ impl StageAA1Plugin { self.message = format!("Frequency sweep refused at its lowest point: {reason}"); return; } - if !self.is_marker_anchored() { + // Only the measured source needs the camera trigger: it is what confirms + // each commanded frequency and what anchors the fold the point is scored + // in. Commanded mode confirms against the modulation owner instead + // (ADR 021), so requiring markers here would refuse a ladder that can + // run perfectly well. + if self.depth_source.needs_a0_lock() && !self.is_marker_anchored() { // Without the phase-0 trigger there is nothing that can confirm the // drive actually reached a commanded frequency, and the fold has no - // anchor either. - self.message = "No phase-0 trigger markers — the sweep cannot confirm a commanded \ - frequency. Enable Live analysis and check EXT_TRIGGER" - .into(); + // anchor either. Live analysis off and a missing trigger cable look + // identical from the marker count, so name whichever one it is. + self.message = if self.live { + "No phase-0 trigger markers — the sweep cannot confirm a commanded frequency. \ + Check the EXT_TRIGGER wiring from the Teensy to the camera" + .into() + } else { + "Live analysis is off, so no phase-0 markers are ingested and the sweep cannot \ + confirm a commanded frequency. Enable Live analysis" + .into() + }; return; } let points = self.freq_sweep_points(); @@ -3150,7 +3979,7 @@ impl StageAA1Plugin { } let now_ms = now_unix_ms(); let lease_id = LeaseId::new(format!("a1-fsweep-{}", format_compact_utc(now_ms / 1_000))); - let ttl_ms = self.freq_sweep_lease_ttl_ms(points.len()); + let ttl_ms = self.freq_sweep_lease_ttl_ms(points.len(), mode); let request = self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); let lease_req = request.request_id; @@ -3158,6 +3987,7 @@ impl StageAA1Plugin { let total = points.len(); self.freq_sweep = Some(FreqSweep { phase: FreqSweepPhase::AcquiringLease, + mode, points, index: 0, lease_id, @@ -3174,10 +4004,18 @@ impl StageAA1Plugin { last_activity_ms: now_ms, stop_requested: false, }); - self.message = format!( - "Frequency sweep: acquiring the modulation lease for {total} points ({} order)…", - self.freq_order.label() - ); + self.message = match mode { + FreqSweepMode::A0Point => format!( + "Frequency sweep: acquiring the modulation lease for {total} points ({} order)…", + self.freq_order.label() + ), + FreqSweepMode::DepthSweep => format!( + "Depth sweep at every frequency: acquiring the modulation lease for {total} × {} \ + recordings ({} order)…", + self.sweep_count.clamp(2, 64), + self.freq_order.label() + ), + }; } /// Release the ladder's lease (if this run holds it) and clear the sweep. @@ -3212,8 +4050,9 @@ impl StageAA1Plugin { let hz = sweep.frequency_hz(); let (index, total) = (sweep.index, sweep.points.len()); let is_reference = sweep.point().is_some_and(|point| point.is_reference); + let mode = sweep.mode; - let ttl_ms = self.freq_sweep_lease_ttl_ms(remaining); + let ttl_ms = self.freq_sweep_lease_ttl_ms(remaining, mode); let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); context.request_service(&renew); let request = self.modulation_request( @@ -3252,6 +4091,74 @@ impl StageAA1Plugin { ); } + /// Hand the current ladder point to the one-point event-count recording. + /// + /// Reached either from the finished `a₀` search (measured depth source) or + /// straight from the confirmed frequency (commanded source, where there is + /// no search). Both need the same question answered — *what depth is armed + /// at this frequency?* — so both ask [`Self::armed_a0`] and neither knows + /// which regime it is in. + fn start_freq_sweep_recording(&mut self, context: &mut impl RecordingControl) { + let Some((mode, hz, index, total)) = self.freq_sweep.as_ref().map(|sweep| { + ( + sweep.mode, + sweep.frequency_hz(), + sweep.index, + sweep.points.len(), + ) + }) else { + return; + }; + // A non-converged lock is stored but never arms a recording, so this is + // the single question worth asking for an `a₀` rung. + if mode.needs_armed_depth() && self.armed_a0().is_none() { + let reason = self + .armed_a0_blocker() + .unwrap_or_else(|| self.message.clone()); + self.fail_freq_sweep_point(context, reason); + return; + } + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::Recording; + } + let lease = self.freq_sweep.as_ref().map(|sweep| sweep.lease_id.clone()); + let label = frequency_label(hz); + match mode { + FreqSweepMode::A0Point => self.begin_a0_point(context, lease), + // The inner run is the *unchanged* amplitude sweep, handed this + // ladder's lease so the operator's drive settings stay locked out + // from the first frequency to the last rather than being handed + // back between rungs. + FreqSweepMode::DepthSweep => { + let points = self.sweep_points(); + let message = format!( + "Depth sweep {}/{total} at {label}: {} depths…", + index + 1, + points.len() + ); + self.begin_leased_sweep( + context, + SweepKind::Amplitude, + points, + None, + lease, + message, + ); + } + } + if self.sweep.is_none() { + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + return; + } + if mode == FreqSweepMode::A0Point { + self.message = format!( + "Frequency sweep {}/{total}: recording the a₀ point at {label}…", + index + 1 + ); + } + } + /// Give up on the current point and move to the next one. /// /// A frequency that cannot be locked or recorded does not end the ladder: @@ -3286,11 +4193,10 @@ impl StageAA1Plugin { self.send_freq_sweep_frequency(context); return; } - let (recorded, failed, total, order, seed) = self - .freq_sweep - .as_ref() - .map(|sweep| { + let Some((mode, recorded, failed, total, order, seed)) = + self.freq_sweep.as_ref().map(|sweep| { ( + sweep.mode, sweep.recorded, sweep.failed.clone(), sweep.points.len(), @@ -3298,11 +4204,22 @@ impl StageAA1Plugin { sweep.seed, ) }) - .unwrap_or_default(); - let mut message = format!( - "Frequency sweep complete: {recorded}/{total} points recorded ({} order, seed {seed})", - order.label() - ); + else { + return; + }; + let mut message = match mode { + FreqSweepMode::A0Point => format!( + "Frequency sweep complete: {recorded}/{total} points recorded ({} order, seed \ + {seed})", + order.label() + ), + FreqSweepMode::DepthSweep => format!( + "Depth sweep at every frequency complete: {recorded}/{total} frequencies × {} \ + depths recorded ({} order, seed {seed})", + self.sweep_count.clamp(2, 64), + order.label() + ), + }; if !failed.is_empty() { let list = failed .iter() @@ -3310,85 +4227,541 @@ impl StageAA1Plugin { .collect::>() .join(", "); message.push_str(&format!( - " — {} skipped: {list}. See the a₀ lock table", - failed.len() + " — {} skipped: {list}{}", + failed.len(), + if mode.needs_armed_depth() && self.depth_source.needs_a0_lock() { + ". See the a₀ lock table" + } else { + "" + } )); } self.finish_freq_sweep(context, message); } - /// Advance the multi-frequency run one control tick. Runs before the lock - /// and the point sweep, so a child it starts runs on the same tick. - fn drive_freq_sweep(&mut self, context: &mut impl RecordingControl) { - if self.freq_sweep.is_none() { - if std::mem::take(&mut self.freq_sweep_pending) { - self.begin_freq_sweep(context); - } + // ---- declarative protocol runs ------------------------------------- + + /// Lease TTL covering the whole protocol, plus a minute of slack. + /// + /// One lease for the whole file, like the frequency ladder: a TTL that + /// expired between points would hand the drive back to the operator's + /// armed settings mid-survey, and the remaining points would record + /// against them without saying so. + fn protocol_lease_ttl_ms(plan: &protocol::Protocol, from: usize) -> u64 { + let remaining: f64 = plan.points[from.min(plan.points.len())..] + .iter() + .map(|point| point.duration_s as f64 + point.settle_s) + .sum(); + // Doubled: every point also spends time on the start/finalize + // handshake, which is not in the protocol's own numbers. + ((remaining * 2_000.0) as u64).saturating_add(60_000) + } + + /// Load, validate and start the protocol named in the settings. + /// + /// Everything checkable is checked here, before the drive moves — the + /// whole point of a protocol is that it runs unattended, so a file that + /// cannot work should say so on the button press rather than at 3 a.m. + fn begin_protocol(&mut self, context: &mut impl RecordingControl) { + if self.recording.is_active() + || self.sweep.is_some() + || self.a0_lock.is_some() + || self.freq_sweep.is_some() + || self.protocol.is_some() + { + self.message = "A recording, sweep or protocol is already running".into(); return; } - self.freq_sweep_pending = false; - let now_ms = now_unix_ms(); - let (phase, stop_requested, lease_granted, freq_applied, last_activity_ms, index, total) = { - let sweep = self.freq_sweep.as_ref().expect("sweep checked above"); - ( - sweep.phase, - sweep.stop_requested, - sweep.lease_granted, - sweep.freq_applied, - sweep.last_activity_ms, - sweep.index, - sweep.points.len(), - ) + if self.output_folder.trim().is_empty() { + self.message = "Pick an output folder first — that is where the files go".into(); + return; + } + let path = self.protocol_path.trim().to_owned(); + if path.is_empty() { + self.message = "Choose a protocol file first".into(); + return; + } + if !self.modulation_connected() { + self.message = + "The modulation plugin is not connected — connect it to drive the protocol".into(); + return; + } + if let Some(blocker) = self.photodiode_blocker() { + self.message = blocker; + return; + } + let text = match std::fs::read_to_string(&path) { + Ok(text) => text, + Err(error) => { + self.message = format!("Cannot read {path}: {error}"); + return; + } }; - // A stop propagates into whichever child is running; the ladder ends - // once that child has let go. - if stop_requested { - if let Some(lock) = self.a0_lock.as_mut() { - lock.stop_requested = true; + let plan = match protocol::parse_file(&path, &text) { + Ok(plan) => plan, + Err(error) => { + self.message = format!("Protocol rejected — {error}"); return; } - if let Some(sweep) = self.sweep.as_mut() { - sweep.stop_requested = true; + }; + // The photodiode measures `a` over one window for every frequency, so + // the lowest frequency in the file decides whether the survey is + // measurable at all. Refuse the plan, not its 40th point. + let lowest = plan + .points + .iter() + .map(|point| point.frequency_hz) + .fold(f64::INFINITY, f64::min); + if lowest.is_finite() { + if let Err(reason) = self.optical_window_covers_a_cycle(lowest) { + self.message = format!( + "Protocol refused at its lowest frequency ({}): {reason}", + frequency_label(lowest) + ); return; } - let message = if self.message.is_empty() { - "Frequency sweep stopped".into() - } else { - self.message.clone() - }; - self.finish_freq_sweep(context, message); - return; } - match phase { - FreqSweepPhase::AcquiringLease => { - if lease_granted { - self.send_freq_sweep_frequency(context); - } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { - self.finish_freq_sweep( - context, - "Frequency sweep aborted: timed out acquiring the modulation lease".into(), - ); - } + + let now_ms = now_unix_ms(); + let lease_id = LeaseId::new(format!( + "a1-protocol-{}", + format_compact_utc(now_ms / 1_000) + )); + let ttl_ms = Self::protocol_lease_ttl_ms(&plan, 0); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + let lease_req = request.request_id; + context.request_service(&request); + + let (means, frequencies, depths) = plan.axis_counts(); + let total = plan.points.len(); + let minutes = plan.total_seconds() / 60.0; + self.message = format!( + "Protocol '{}': {total} recordings ({means} × ū, {frequencies} × f, {depths} × a), \ + about {minutes:.0} min of bench time — acquiring the modulation lease…", + plan.name + ); + self.protocol = Some(ProtocolRun { + plan, + phase: ProtocolPhase::AcquiringLease, + index: 0, + lease_id, + lease_granted: false, + lease_req, + pending_reqs: Vec::new(), + settle_until_ms: 0, + failed: Vec::new(), + recorded: 0, + last_activity_ms: now_ms, + stop_requested: false, + skip_reason: None, + }); + } + + /// Release the protocol's lease (if this run holds it) and clear it. + fn finish_protocol(&mut self, context: &mut impl RecordingControl, message: String) { + if let Some(run) = self.protocol.take() { + if run.lease_granted { + let request = self.modulation_request( + ModulationCommandV1::ReleaseLease { + safe_off: false, + reason: "a1 protocol finished".into(), + }, + &run.lease_id, + ); + context.request_service(&request); } - FreqSweepPhase::SettingFrequency => { - // A refused frequency is a property of this point, not of the - // ladder; the owner already said why. - if let Some(reason) = self - .freq_sweep - .as_mut() - .and_then(|sweep| sweep.skip_reason.take()) - { - self.fail_freq_sweep_point(context, reason); - } else if freq_applied { - let hz = self - .freq_sweep - .as_ref() - .map(FreqSweep::frequency_hz) - .unwrap_or_default(); - // Confirming needs whole cycles at the *new* period, so the - // budget has to scale with it: 4 cycles at 0.1 Hz is 40 s. - let cycles_ms = if hz > 0.0 { - (FREQ_CONFIRM_CYCLES / hz * 1_000.0).ceil() as u64 + } + self.message = message; + } + + /// Renew the lease and retarget all three axes at the current point. + fn send_protocol_point(&mut self, context: &mut impl RecordingControl) { + let Some(run) = self.protocol.as_ref() else { + return; + }; + let Some(point) = run.point().cloned() else { + return; + }; + let lease_id = run.lease_id.clone(); + let (index, total) = (run.index, run.plan.points.len()); + let ttl_ms = Self::protocol_lease_ttl_ms(&run.plan, index); + + let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + context.request_service(&renew); + + // All three axes, every point. A protocol states the whole operating + // condition, so nothing is left at whatever the previous point or the + // operator happened to leave behind. + let mut pending = Vec::with_capacity(3); + for command in [ + ModulationCommandV1::SetOperatingPoint { + mean_u_milli: (point.mean_u * 1_000.0).round().clamp(0.0, 1_000.0) as u32, + }, + ModulationCommandV1::SetDriveFrequency { + frequency_millihz: (point.frequency_hz * 1_000.0).round().max(0.0) as u64, + }, + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: depth_a_milli(point.depth_a), + }, + ] { + let request = self.modulation_request(command, &lease_id); + pending.push(request.request_id); + context.request_service(&request); + } + + // The retained markers and events belong to the previous point's + // frequency; the measured period is their mean spacing, so leaving + // them would confirm this point against a mixture of the two. Pilot + // windows are frozen at a phase of the old period and do not transfer. + self.camera_markers_us.clear(); + self.camera_events.clear(); + self.fold_cache.replace(None); + self.pilot_windows = None; + + let now_ms = now_unix_ms(); + if let Some(run) = self.protocol.as_mut() { + run.phase = ProtocolPhase::Retargeting; + run.pending_reqs = pending; + run.skip_reason = None; + run.last_activity_ms = now_ms; + } + self.message = format!( + "Protocol {}/{total} [{}]: ū={:.2}, f={}, a={:.2}…", + index + 1, + point.block, + point.mean_u, + frequency_label(point.frequency_hz), + point.depth_a, + ); + } + + /// Give up on the current point and move to the next. + fn fail_protocol_point(&mut self, context: &mut impl RecordingControl, reason: String) { + let Some(run) = self.protocol.as_mut() else { + return; + }; + let index = run.index; + run.failed.push((index, reason.clone())); + let point = run.plan.points[index].clone(); + self.message = format!( + "Protocol point {} (ū={:.2}, f={}, a={:.2}) skipped: {reason}", + index + 1, + point.mean_u, + frequency_label(point.frequency_hz), + point.depth_a, + ); + self.advance_protocol(context); + } + + /// Step to the next point, or finish. + fn advance_protocol(&mut self, context: &mut impl RecordingControl) { + let Some(run) = self.protocol.as_mut() else { + return; + }; + run.index += 1; + run.last_activity_ms = now_unix_ms(); + if run.index < run.plan.points.len() && !run.stop_requested { + self.send_protocol_point(context); + return; + } + let (name, recorded, failed, total) = ( + run.plan.name.clone(), + run.recorded, + run.failed.clone(), + run.plan.points.len(), + ); + let stopped = run.stop_requested; + let mut message = format!( + "Protocol '{name}' {}: {recorded}/{total} recorded", + if stopped { "stopped" } else { "finished" } + ); + if !failed.is_empty() { + // Name the reasons, not just the count: an unattended run's whole + // report is this one line. + let mut reasons: Vec = failed + .iter() + .map(|(_, reason)| reason.clone()) + .collect::>() + .into_iter() + .collect(); + reasons.truncate(3); + message.push_str(&format!( + " — {} skipped ({})", + failed.len(), + reasons.join("; ") + )); + } + self.finish_protocol(context, message); + } + + /// Advance a protocol run one control tick. Runs outermost: a point it + /// starts hands off to the recording coordinator on the same tick. + fn drive_protocol(&mut self, context: &mut impl RecordingControl) { + if self.protocol.is_none() { + if self.protocol_pending { + self.protocol_pending = false; + self.begin_protocol(context); + } + return; + } + if self.protocol_pending { + // Say so rather than swallowing the press: Stop is a different + // button, and a silently ignored one reads as a dead control. + self.protocol_pending = false; + self.message = "A protocol is already running — press Stop to end it".into(); + } + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, retargets_left, settle_until_ms, last_activity) = { + let run = self.protocol.as_ref().expect("run checked above"); + ( + run.phase, + run.stop_requested, + run.lease_granted, + run.pending_reqs.len(), + run.settle_until_ms, + run.last_activity_ms, + ) + }; + + // A stop waits for the recording in flight to wind down, then ends the + // run — a protocol that abandoned a half-written file would leave a + // truncated RAW behind. + if stop_requested { + if self.recording.is_active() { + self.recording.stop_requested = true; + return; + } + self.advance_protocol(context); + return; + } + + match phase { + ProtocolPhase::AcquiringLease => { + if !lease_granted { + if now_ms.saturating_sub(last_activity) > REPLY_TIMEOUT_MS { + self.finish_protocol( + context, + "Protocol aborted: the modulation plugin did not grant the lease" + .into(), + ); + } + return; + } + self.send_protocol_point(context); + } + ProtocolPhase::Retargeting => { + if let Some(reason) = self + .protocol + .as_mut() + .and_then(|run| run.skip_reason.take()) + { + self.fail_protocol_point(context, reason); + return; + } + if retargets_left > 0 { + if now_ms.saturating_sub(last_activity) > REPLY_TIMEOUT_MS { + self.fail_protocol_point( + context, + "the modulation plugin did not apply the requested drive".into(), + ); + } + return; + } + let settle_ms = self + .protocol + .as_ref() + .and_then(|run| run.point()) + .map(|point| (point.settle_s * 1_000.0).round().max(0.0) as u64) + .unwrap_or(0); + if let Some(run) = self.protocol.as_mut() { + run.phase = ProtocolPhase::Settling; + run.settle_until_ms = now_ms.saturating_add(settle_ms); + run.last_activity_ms = now_ms; + } + } + ProtocolPhase::Settling => { + if now_ms < settle_until_ms { + return; + } + let duration_s = self + .protocol + .as_ref() + .and_then(|run| run.point()) + .map(|point| point.duration_s) + .unwrap_or(self.duration_s); + // The point's own duration wins over the panel's: the file says + // how long each point runs, and a survey whose lengths silently + // came from the UI would not be reproducible from the protocol + // alone. Through the override, not `duration_s` — see + // `pending_duration_s`. + self.pending_duration_s = Some(duration_s); + if let Some(run) = self.protocol.as_mut() { + run.phase = ProtocolPhase::Recording; + run.last_activity_ms = now_ms; + } + // The row says what it is: a protocol can carry its own + // background and pilot, so a survey does not need two button + // presses before it can be started. + let role = self + .protocol + .as_ref() + .and_then(|run| run.point()) + .map(|point| match point.role { + protocol::PointRole::Normal => RecRole::Normal, + protocol::PointRole::Pilot => RecRole::Pilot, + protocol::PointRole::Background => RecRole::Background, + }) + .unwrap_or(RecRole::Normal); + self.begin_recording(context, role); + // `begin_recording` refuses through `message` rather than a + // result, so a point that never started has to be caught here + // or the run would wait on a recording that does not exist. + if !self.recording.is_active() { + let reason = self.message.clone(); + self.fail_protocol_point(context, reason); + } + } + ProtocolPhase::Recording => { + if self.recording.is_active() { + return; + } + if self.recording_completed_ok { + if let Some(run) = self.protocol.as_mut() { + run.recorded += 1; + } + self.advance_protocol(context); + } else { + let reason = self.message.clone(); + self.fail_protocol_point(context, reason); + } + } + } + } + + /// Routes modulation-service replies belonging to the protocol run. + fn on_protocol_reply(&mut self, reply: &PluginServiceReply) -> bool { + let Some(run) = self.protocol.as_ref() else { + return false; + }; + let lease_req = run.lease_req; + let is_retarget = run.pending_reqs.contains(&reply.request_id); + if reply.request_id == lease_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(run) = self.protocol.as_mut() { + run.lease_granted = true; + run.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => { + self.message = + format!("Protocol aborted: modulation lease rejected: {message}"); + if let Some(run) = self.protocol.as_mut() { + run.stop_requested = true; + } + } + } + return true; + } + if !is_retarget { + return false; + } + let now_ms = now_unix_ms(); + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(run) = self.protocol.as_mut() { + run.pending_reqs.retain(|id| *id != reply.request_id); + run.last_activity_ms = now_ms; + } + } + PluginServiceOutcome::Rejected { message, .. } => { + // Carry the owner's own wording through to the skip message: + // "ū=0.90 rejected: peak exceeds the lobe ceiling" tells the + // operator which line of the file to fix, "retarget failed" + // does not. + if let Some(run) = self.protocol.as_mut() { + run.pending_reqs.clear(); + run.skip_reason = Some(message.clone()); + run.last_activity_ms = now_ms; + } + } + } + true + } + + /// Advance the multi-frequency run one control tick. Runs before the lock + /// and the point sweep, so a child it starts runs on the same tick. + fn drive_freq_sweep(&mut self, context: &mut impl RecordingControl) { + if self.freq_sweep.is_none() { + if let Some(mode) = self.freq_sweep_pending.take() { + self.begin_freq_sweep(context, mode); + } + return; + } + self.freq_sweep_pending = None; + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, freq_applied, last_activity_ms, index, total) = { + let sweep = self.freq_sweep.as_ref().expect("sweep checked above"); + ( + sweep.phase, + sweep.stop_requested, + sweep.lease_granted, + sweep.freq_applied, + sweep.last_activity_ms, + sweep.index, + sweep.points.len(), + ) + }; + // A stop propagates into whichever child is running; the ladder ends + // once that child has let go. + if stop_requested { + if let Some(lock) = self.a0_lock.as_mut() { + lock.stop_requested = true; + return; + } + if let Some(sweep) = self.sweep.as_mut() { + sweep.stop_requested = true; + return; + } + let message = if self.message.is_empty() { + "Frequency sweep stopped".into() + } else { + self.message.clone() + }; + self.finish_freq_sweep(context, message); + return; + } + match phase { + FreqSweepPhase::AcquiringLease => { + if lease_granted { + self.send_freq_sweep_frequency(context); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_freq_sweep( + context, + "Frequency sweep aborted: timed out acquiring the modulation lease".into(), + ); + } + } + FreqSweepPhase::SettingFrequency => { + // A refused frequency is a property of this point, not of the + // ladder; the owner already said why. + if let Some(reason) = self + .freq_sweep + .as_mut() + .and_then(|sweep| sweep.skip_reason.take()) + { + self.fail_freq_sweep_point(context, reason); + } else if freq_applied { + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + // Confirming needs whole cycles at the *new* period, so the + // budget has to scale with it: 4 cycles at 0.1 Hz is 40 s. + let cycles_ms = if hz > 0.0 { + (FREQ_CONFIRM_CYCLES / hz * 1_000.0).ceil() as u64 } else { 0 }; @@ -3415,16 +4788,33 @@ impl StageAA1Plugin { .as_ref() .map(FreqSweep::frequency_hz) .unwrap_or_default(); - // The trigger *defines* the frequency, so the point only starts - // once the markers say the drive is really there — an ACK from - // the firmware says the table was accepted, not that the light - // is modulating at that rate. Enough markers must have arrived - // at the new period for their mean spacing to mean anything. - let enough_markers = self.camera_markers_us.len() as f64 >= FREQ_CONFIRM_CYCLES; - let confirmed = enough_markers - && self - .frequency_hz() - .is_some_and(|measured| same_frequency(measured, hz)); + // Which side is entitled to say the drive really reached the new + // frequency. + // + // Measured mode holds out for the camera's phase-0 markers: they + // *define* the period, the fold that scores the point is anchored + // on them, and an ACK from the firmware only says the table was + // accepted, not that the light is modulating at that rate. Enough + // markers must have arrived at the new period for their mean + // spacing to mean anything. + // + // Commanded mode asks the modulation owner instead. That is the + // same contract it already relies on for the depth — if the + // owner's acknowledged waveform is trusted to state `a`, it is + // trusted to state `f` — and it does not strand a bench whose + // camera trigger is not wired, which is the whole reason the + // commanded source exists (ADR 021). The live fold goes + // free-running without markers; the recorded RAW and PDQ, which + // are what the offline fit reads, are unaffected. + let confirmed = if self.depth_source.needs_a0_lock() { + self.camera_markers_us.len() as f64 >= FREQ_CONFIRM_CYCLES + && self + .frequency_hz() + .is_some_and(|measured| same_frequency(measured, hz)) + } else { + self.acknowledged_frequency_hz() + .is_some_and(|acknowledged| same_frequency(acknowledged, hz)) + }; let deadline = self .freq_sweep .as_ref() @@ -3435,6 +4825,20 @@ impl StageAA1Plugin { self.fail_freq_sweep_point(context, reason); return; } + // The search stands between the frequency and the recording + // in exactly one case: an `a₀` rung whose depth is measured. + // A depth sweep commands and settles every `a` in its range + // itself, so there is nothing for a lock to contribute at + // any frequency, in either depth source. + let needs_search = self + .freq_sweep + .as_ref() + .is_some_and(|sweep| sweep.mode.needs_armed_depth()) + && self.depth_source.needs_a0_lock(); + if !needs_search { + self.start_freq_sweep_recording(context); + return; + } if let Some(sweep) = self.freq_sweep.as_mut() { sweep.phase = FreqSweepPhase::Locking; } @@ -3446,55 +4850,41 @@ impl StageAA1Plugin { self.fail_freq_sweep_point(context, reason); } } else if now_ms >= deadline { - let measured = self - .frequency_hz() - .map_or_else(|| "—".into(), frequency_label); - self.fail_freq_sweep_point( - context, + let reason = if self.depth_source.needs_a0_lock() { + let measured = self + .frequency_hz() + .map_or_else(|| "—".into(), frequency_label); format!( "the trigger never reported it (measured {measured} from {} markers)", self.camera_markers_us.len() - ), - ); + ) + } else { + let acknowledged = self + .acknowledged_frequency_hz() + .map_or_else(|| "—".into(), frequency_label); + format!( + "the modulation plugin never acknowledged it (its armed drive still \ + reads {acknowledged})" + ) + }; + self.fail_freq_sweep_point(context, reason); } } FreqSweepPhase::Locking => { if self.a0_lock.is_some() { return; } - let hz = self - .freq_sweep - .as_ref() - .map(FreqSweep::frequency_hz) - .unwrap_or_default(); - // A non-converged lock is stored but never arms a recording, so - // `armed_lock` is the single question worth asking here. - if self.armed_lock().is_none() { - let reason = self.message.clone(); - self.fail_freq_sweep_point(context, reason); - return; - } - if let Some(sweep) = self.freq_sweep.as_mut() { - sweep.phase = FreqSweepPhase::Recording; - } - let lease = self.freq_sweep.as_ref().map(|sweep| sweep.lease_id.clone()); - self.begin_a0_point(context, lease); - if self.sweep.is_none() { - let reason = self.message.clone(); - self.fail_freq_sweep_point(context, reason); - return; - } - self.message = format!( - "Frequency sweep {}/{total}: recording the a₀ point at {}…", - index + 1, - frequency_label(hz), - ); + self.start_freq_sweep_recording(context); } FreqSweepPhase::Recording => { if self.sweep.is_some() || self.recording.is_active() { return; } - if self.recording_completed_ok { + // The inner run's own verdict, not the last recording's. A + // depth sweep that gives up on point 4 of 5 leaves + // `recording_completed_ok` true from point 3, which would have + // counted a half-recorded curve as a finished rung. + if self.last_sweep_completed_ok { if let Some(sweep) = self.freq_sweep.as_mut() { sweep.recorded += 1; } @@ -3578,6 +4968,7 @@ impl StageAA1Plugin { map(|lock| format!("{:.3}", lock.commanded_a)), ), column("measured_a", map(|lock| format!("{:.3}", lock.measured_a))), + column("depth_source", map(|lock| lock.depth_source.verb().into())), column("trials", map(|lock| lock.trials.to_string())), column( "state", @@ -3644,6 +5035,7 @@ impl StageAA1Plugin { if self.on_sweep_reply(reply) || self.on_a0_lock_reply(reply) || self.on_freq_sweep_reply(reply) + || self.on_protocol_reply(reply) { return; } @@ -3843,7 +5235,6 @@ impl StageAA1Plugin { let doc = SidecarDoc { measurement_id: self.recording.id.clone(), - flux_point_id: self.flux_point_id.trim().to_owned(), file_stem: self.recording.stem.clone(), role: self.recording.role.label().into(), recorded_at_utc: format_iso_utc( @@ -3855,6 +5246,8 @@ impl StageAA1Plugin { ), finalized_at_utc: format_iso_utc(now_ms / 1_000), duration_s: self.recording.duration_s, + depth_a_source: self.depth_source.label().into(), + depth_a: self.depth_a(), sweep: { let point = self .sweep @@ -3877,6 +5270,7 @@ impl StageAA1Plugin { target_a: lock.target_a, commanded_a: lock.commanded_a, measured_a_at_lock: lock.measured_a, + depth_source: lock.depth_source.label().into(), frequency_hz_at_lock: lock.frequency_hz, trials: lock.trials, converged: lock.converged, @@ -3926,7 +5320,7 @@ impl StageAA1Plugin { internal_u: mod_optical.map(|drive| f64::from(drive.internal_u_milli) / 1_000.0), requested_a: mod_optical.map(|drive| f64::from(drive.depth_a_milli) / 1_000.0), v_null_dac: mod_optical.map(|drive| drive.v_null_dac), - v_pi_dac: mod_optical.map(|drive| drive.v_pi_dac), + v_peak_dac: mod_optical.map(|drive| drive.v_peak_dac), center_dac: a1_config.map(|c| c.center_dac), amplitude_dac: a1_config.map(|c| c.amplitude_dac), waveform: modulation @@ -3957,6 +5351,20 @@ impl StageAA1Plugin { masked_pixels: self.masked_pixels.len(), n_valid: self.valid_pixel_count(), }, + sensor: self.recorded_sensor().map(|sensor| { + let codes = sensor.bias_codes.map(|readback| readback.current); + SensorSidecar { + temperature_c: sensor.temperature_c, + pixel_dead_time_us: sensor.pixel_dead_time_us, + illumination_lux: sensor.illumination_lux, + reading_age_s: sensor.age_s, + bias_diff_on: codes.map(|c| c.diff_on), + bias_diff_off: codes.map(|c| c.diff_off), + bias_fo: codes.map(|c| c.fo), + bias_hpf: codes.map(|c| c.hpf), + bias_refr: codes.map(|c| c.refr), + } + }), trigger: TriggerSidecar { marker_anchored: self.is_marker_anchored(), marker_count: self.camera_markers_us.len(), @@ -3967,6 +5375,7 @@ impl StageAA1Plugin { camera_config_sidecar: camera_bias_sidecar, photodiode_pdq: self.recording.pd_pdq_path.clone(), photodiode_sidecar: self.recording.pd_sidecar_path.clone(), + sensor_readout: self.recording.sensor_readout_path.clone(), }, }; @@ -3984,12 +5393,22 @@ impl StageAA1Plugin { #[derive(Serialize)] struct SidecarDoc { measurement_id: String, - flux_point_id: String, file_stem: String, role: String, recorded_at_utc: String, finalized_at_utc: String, duration_s: u64, + /// The modulation depth this run was driven and judged by, and which source + /// produced it (`photodiode_measured` / `modulation_commanded`). + /// + /// Written on every run, so offline analysis never has to infer the depth's + /// provenance from which of `optical.measured_a` and `modulation.requested_a` + /// happens to be present. A commanded depth is an open-loop number carrying + /// the Pockels calibration's error; a fit that mixes the two sources without + /// looking here would silently mix two error budgets. + depth_a_source: String, + #[serde(skip_serializing_if = "Option::is_none")] + depth_a: Option, sweep: SweepSidecar, /// Present on **event-count** points: the `a₀` lock this point replayed. #[serde(skip_serializing_if = "Option::is_none")] @@ -4004,6 +5423,10 @@ struct SidecarDoc { modulation: ModulationSidecar, optical: OpticalSidecar, camera: CameraSidecar, + /// Absent when the host had no camera able to measure these (replay, + /// imports, a sensor without a monitoring block). + #[serde(skip_serializing_if = "Option::is_none")] + sensor: Option, trigger: TriggerSidecar, files: FilesSidecar, } @@ -4035,6 +5458,8 @@ struct A0LockSidecar { target_a: f64, commanded_a: f64, measured_a_at_lock: f64, + /// Which source `measured_a_at_lock` came from — see `depth_a_source`. + depth_source: String, frequency_hz_at_lock: f64, trials: u32, converged: bool, @@ -4111,7 +5536,7 @@ struct ModulationSidecar { #[serde(skip_serializing_if = "Option::is_none")] v_null_dac: Option, #[serde(skip_serializing_if = "Option::is_none")] - v_pi_dac: Option, + v_peak_dac: Option, #[serde(skip_serializing_if = "Option::is_none")] center_dac: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -4161,6 +5586,42 @@ struct CameraSidecar { n_valid: Option, } +/// Bench conditions the sensor measured for itself at the start of the run. +/// +/// Provenance, never an input: the `q_p(a, f)` response depends on the pixel +/// dead time and on the scene illumination, and the die temperature moves the +/// biases, so a row that cannot be compared to another has to be identifiable +/// as such afterwards. Present only when the host was streaming from a camera +/// with a monitoring block — an offline re-run of the same RAW has no sensor to +/// ask, and every field stays absent rather than becoming zero. +#[derive(Serialize)] +struct SensorSidecar { + /// Sensor die temperature, °C. + #[serde(skip_serializing_if = "Option::is_none")] + temperature_c: Option, + /// Measured pixel dead time (refractory period), µs. + #[serde(skip_serializing_if = "Option::is_none")] + pixel_dead_time_us: Option, + /// Scene illumination integrated by the sensor, lux. + #[serde(skip_serializing_if = "Option::is_none")] + illumination_lux: Option, + /// Seconds between the host's last read of these values and the moment the + /// recording started — the host polls at a few hertz, so this is never 0. + reading_age_s: f64, + /// Absolute programmed bias codes, and the per-unit factory trim the + /// host's relative offsets are expressed against. + #[serde(skip_serializing_if = "Option::is_none")] + bias_diff_on: Option, + #[serde(skip_serializing_if = "Option::is_none")] + bias_diff_off: Option, + #[serde(skip_serializing_if = "Option::is_none")] + bias_fo: Option, + #[serde(skip_serializing_if = "Option::is_none")] + bias_hpf: Option, + #[serde(skip_serializing_if = "Option::is_none")] + bias_refr: Option, +} + #[derive(Serialize)] struct TriggerSidecar { marker_anchored: bool, @@ -4179,6 +5640,11 @@ struct FilesSidecar { photodiode_pdq: Option, #[serde(skip_serializing_if = "Option::is_none")] photodiode_sidecar: Option, + /// Compacted per-channel sensor readout for this run — the die + /// temperature, pixel dead time and illumination the host polled while it + /// was recording. Absent when the source had no monitoring block. + #[serde(skip_serializing_if = "Option::is_none")] + sensor_readout: Option, } // ---- free functions -------------------------------------------------------- @@ -4464,7 +5930,24 @@ impl Plugin for StageAA1Plugin { self.host_roi = Some(settings.roi); self.masked_pixels = settings.masked_pixels.into_iter().collect(); } + // Mirrored above the `live` gate on purpose: these are recorded with + // every run, and recordings are made with Live analysis off just as + // often as with it on. Absent whenever the host has no camera that can + // measure them (replay, imports, a sensor without a monitoring block), + // which stays `None` rather than becoming a zero. + if let Some(monitoring) = context + .get::(CTX_SENSOR_MONITORING) + .ok() + .flatten() + { + self.sensor = Some(monitoring); + } if !self.live { + // Drop the analysis buffers on the way out, not just stop filling + // them — see `drop_live_buffers`. + if self.drop_live_buffers() { + self.bump(); + } return; } @@ -4546,10 +6029,11 @@ impl Plugin for StageAA1Plugin { self.scan_measurement_folder(); self.load_a0_locks(); } - // Outermost first: the frequency sweep starts the lock or the point it - // supervises, and each of those starts its own next stage, so one tick - // carries a hand-off all the way down. They are mutually exclusive at - // the top, guarded where they begin. + // Outermost first: the protocol and the frequency sweep each start the + // stage below them, and each of those starts its own next stage, so one + // tick carries a hand-off all the way down. They are mutually exclusive + // at the top, guarded where they begin. + self.drive_protocol(context); self.drive_freq_sweep(context); self.drive_a0_lock(context); self.drive_sweep(context); @@ -4559,139 +6043,199 @@ impl Plugin for StageAA1Plugin { } fn settings_schema(&self) -> SettingsSchema { - // The record/sweep buttons stay disabled until the recording has a + // The record buttons stay disabled until the recording has a // destination, instead of failing with a status message after a click. let can_record = !self.output_folder.trim().is_empty(); + // Deliberately *not* gated on "is something running": `settings_schema` + // is rendered by the UI mirror, and every run — the recording, the + // sweeps, the ladder, the protocol — lives on the live worker, which is + // the only instance the host calls `process_control` on. A mirror + // reading its own always-idle state would disable nothing and mislead + // the next reader into thinking it did. The authoritative interlocks + // stay worker-side, where each `begin_*` refuses with a message that + // names what is already running (ADR 010, same reason as the modulation + // plugin's `calibration_offered`). SettingsSchema { sections: vec![ SettingsSection { - label: "Recording".into(), + label: "Live analysis".into(), description: Some( - "Records the camera RAW stream and the photodiode PDQ stream together for \ - a fixed duration and writes an A1 config sidecar (.toml) linking them. \ - Everything lands under // and shares an \ - _ stem: the RAW and PDQ are gathered here once both are \ - finalized, wherever their own recorders wrote them. Arm the optical \ - drive in the modulation plugin first; A1 only reads its settings — it \ - never drives the Teensy. The photodiode must be connected and have a \ - data directory set, otherwise the recording is refused before it starts." + "A live look at the camera events folded against the modulation cycle. \ + Nothing is saved from here — but almost everything else reads it, so \ + it belongs at the top rather than buried below the recording controls.\n\n\ + Leave it ON. The frequency sweep and the protocol both need it to see \ + the phase-0 trigger, and the response curve below scores its points \ + out of the same buffer." .into(), ), default_open: true, items: vec![ SettingItem { - key: "output_folder".into(), - label: "Output folder".into(), - tooltip: Some( - "Experiment directory for this measurement. The config sidecar is \ - written here, and the camera RAW and photodiode PDQ are moved \ - here once finalized, so one measurement is one folder." - .into(), - ), - kind: SettingKind::Path { - dialog: PathDialogKind::Directory, - default: self.output_folder.clone(), - }, - }, - SettingItem { - key: "measurement_id".into(), - label: "Measurement id (one per I_k, f pair)".into(), + key: "live".into(), + label: "Live analysis".into(), tooltip: Some( - "Groups every repeat of one illumination/frequency pair. Included \ - in every file name. Edit it freely or press New id." + "On: read incoming events and triggers and update the plots. \ + Off: the buffer is dropped and nothing is read at all — the \ + frequency sweep and the protocol will refuse to start." .into(), ), - kind: SettingKind::Text { - default: self.measurement_id.clone(), - }, + kind: SettingKind::Bool { default: self.live }, }, SettingItem { - key: "flux_point_id".into(), - label: "Physical I_k flux point id".into(), + key: "analysis_window_ms".into(), + label: "Analysis window (ms)".into(), tooltip: Some( - "Canonical id of the cycle-mean local flux calibration/map point. \ - This is physical photons/pixel/s provenance, not the modulation \ - plugin's dimensionless lobe coordinate u." + "How far back the live plots look. Longer covers more cycles but \ + folds more events on every update — if the plots feel heavy, \ + shorten this first." .into(), ), - kind: SettingKind::Text { - default: self.flux_point_id.clone(), + kind: SettingKind::I64Drag { + min: 1, + max: 120_000, + default: self.analysis_window_ms, }, }, SettingItem { - key: "new_id".into(), - label: "New id".into(), - tooltip: Some("Generate a fresh default measurement id.".into()), + key: "clear".into(), + label: "Clear captured events".into(), + tooltip: Some("Empties the buffer and resets the live plots.".into()), kind: SettingKind::Button { enabled: true }, }, SettingItem { - key: "min_a".into(), - label: "Sweep min a".into(), + key: "window_floor".into(), + label: "Window width threshold".into(), tooltip: Some( - "Low end of the modulation-depth sweep for this (I_k, f) row. \ - Stored in every sidecar as the automation template; A1 does not \ - drive it — you set the drive in the modulation plugin." + "How the bright and dark windows are found automatically: each \ + one widens out from its busiest moment until the event rate \ + drops below this fraction of the peak. 0.10 = stop at 10 %." .into(), ), kind: SettingKind::F64Drag { - min: 0.0, - max: 10.0, + min: 0.02, + max: 0.5, speed: 0.01, - default: self.min_a, + default: self.window_floor, }, }, SettingItem { - key: "max_a".into(), - label: "Sweep max a".into(), + key: "record_point".into(), + label: "Add response-curve point (at the current depth)".into(), tooltip: Some( - "High end of the modulation-depth sweep for this (I_k, f) row \ - (also the natural amplitude for the pilot). Stored in every \ - sidecar; A1 does not drive it." + "Adds one dot to the response curve, using the events in the \ + live buffer and the depth measured right now. A preview to \ + check the shape looks sensible — saves no files, and the real \ + fit is done afterwards from the recorded ones." .into(), ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 10.0, - speed: 0.01, - default: self.max_a, - }, + kind: SettingKind::Button { enabled: true }, }, SettingItem { - key: "sweep_count".into(), - label: "Sweep points (count)".into(), + key: "clear_curve".into(), + label: "Clear the response curve".into(), + tooltip: Some("Removes all the dots from the curve.".into()), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + SettingsSection { + label: "Modulation depth a".into(), + description: Some( + "Where the depth a comes from. Everything that needs a depth — the \ + sweeps, the protocol, the response curve — reads this one setting.\n\n\ + The photodiode is the honest source: it watches the light itself. But \ + it only reports a depth when it can prove its window covers whole \ + modulation cycles, which needs the phase-0 trigger markers on the \ + photodiode's own stream. Without them (no trigger, or a frequency so \ + low that two cycles do not fit in the photodiode's cache) it reports \ + nothing and every button refuses.\n\n\ + The commanded drive gets you running in that case: the modulation \ + plugin already inverts your measured Pockels curve to command a depth, \ + so the number is calibrated — it just is not checked against the light. \ + Runs recorded this way are tagged as such in their description file." + .into(), + ), + default_open: true, + items: vec![SettingItem { + key: "depth_source".into(), + label: "Depth a source".into(), + tooltip: Some( + "Photodiode: use the depth the photodiode measures (accurate, needs \ + the trigger markers). Modulation drive: use the depth the modulation \ + plugin is commanding (works without the photodiode, but open loop — \ + it is not verified against the light)." + .into(), + ), + kind: SettingKind::Enum { + variants: vec![ + "photodiode (measured)".into(), + "modulation drive (commanded, open loop)".into(), + ], + default: self.depth_source.index() as usize, + }, + }], + }, + SettingsSection { + label: "Record".into(), + description: Some( + "Everything that writes files, in one place. Each measurement gets its \ + own folder holding the camera file, the photodiode file, the sensor \ + readout and a description file, all sharing one name.\n\n\ + Four ways to run, all using the settings below and whatever the \ + modulation plugin currently has armed for the axes they do not sweep:\n\ + • Record once — one recording, exactly as the bench stands now.\n\ + • Sweep a — a range of depths at the armed frequency.\n\ + • Sweep f — a range of frequencies at the armed depth.\n\ + • Sweep a × f — every depth at every frequency (the q_p(a, f) surface).\n\n\ + You need an output folder, a connected photodiode, and the modulation \ + plugin running the light. The measurement id is filled in for you if \ + you leave it blank." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "output_folder".into(), + label: "Output folder".into(), tooltip: Some( - "How many amplitudes the Start sweep button records, spaced \ - evenly from Sweep min a to Sweep max a (inclusive)." + "Where the recordings go. Each measurement gets its own subfolder \ + in here. Required." .into(), ), - kind: SettingKind::I64Drag { - min: 2, - max: 64, - default: self.sweep_count, + kind: SettingKind::Path { + dialog: PathDialogKind::Directory, + default: self.output_folder.clone(), }, }, SettingItem { - key: "settle_s".into(), - label: "Sweep settle (s)".into(), + key: "measurement_id".into(), + label: "Measurement id".into(), tooltip: Some( - "After retargeting the drive, the sweep waits until the \ - photodiode-measured a holds the target (±10 %, at least ±0.05) \ - for this long before recording. Gives up after 30 s and records \ - anyway — the sidecar stores the measured a." + "Names the subfolder and every file in it. Use one id for all the \ + repeats that belong together. Optional — leave it blank and one \ + is generated when you press record." .into(), ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 60.0, - speed: 0.1, - default: self.settle_s, + kind: SettingKind::Text { + default: self.measurement_id.clone(), }, }, + SettingItem { + key: "new_id".into(), + label: "New id".into(), + tooltip: Some( + "Put a fresh generated id in the field above, so the next \ + recording starts a new measurement folder." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, SettingItem { key: "duration_s".into(), label: "Duration (s)".into(), tooltip: Some( - "How long each recording runs before it auto-stops and finalizes." + "How many seconds each recording lasts before it stops and saves \ + itself. Applies to every button here." .into(), ), kind: SettingKind::I64Drag { @@ -4701,106 +6245,73 @@ impl Plugin for StageAA1Plugin { }, }, SettingItem { - key: "start_recording".into(), - label: "Start recording (sweep point)".into(), + key: "settle_s".into(), + label: "Settle time (s)".into(), tooltip: Some( - "Acquire the photodiode lease, start the camera RAW + photodiode \ - PDQ recording, auto-stop after the duration, and write the \ - sidecar. Disabled until an output folder is selected." + "After changing the depth or the frequency, wait this long with \ + the reading holding steady before recording. Longer is safer if \ + your signal drifts. Ignored by Record once, which records what \ + is already there." .into(), ), - kind: SettingKind::Button { - enabled: can_record, + kind: SettingKind::F64Drag { + min: 0.0, + max: 60.0, + speed: 0.1, + default: self.settle_s, }, }, SettingItem { - key: "start_sweep".into(), - label: "Start sweep (record all points)".into(), + key: "min_a".into(), + label: "Depth axis: min a".into(), tooltip: Some( - "Sweeps the modulation depth over [Sweep min a, Sweep max a] in \ - the configured number of points: per point A1 leases the \ - modulation owner, retargets the armed calibrated drive, waits \ - for the photodiode-measured a to settle, and records one sweep \ - point (…_pNN) like the Start recording button. Requires the \ - modulation plugin to have a calibrated periodic/optical drive \ - armed and Sweep min a > 0. Disabled until an output folder is \ - selected." + "Shallowest depth the a sweep records. Must be above 0 — a = 0 \ + is the background reference, which has its own button." .into(), ), - kind: SettingKind::Button { - enabled: can_record, + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: self.min_a, }, }, SettingItem { - key: "record_pilot".into(), - label: "Record pilot (freeze ON/OFF windows)".into(), - tooltip: Some( - "Records a bright reference for this row into the same folder \ - (…_pilot) and freezes the ON/OFF windows from the current live \ - signal. Set a high, non-saturating a in the modulation plugin \ - first. The frozen windows are reused for the whole row's q_p. \ - Disabled until an output folder is selected." - .into(), - ), - kind: SettingKind::Button { - enabled: can_record, + key: "max_a".into(), + label: "Depth axis: max a".into(), + tooltip: Some("Deepest depth the a sweep records.".into()), + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: self.max_a, }, }, SettingItem { - key: "record_background".into(), - label: "Record background (a≈0 floor)".into(), + key: "sweep_count".into(), + label: "Depth axis: points".into(), tooltip: Some( - "Records an unmodulated reference (…_background) and captures the \ - false-response floor q0 in the current windows. Set a≈0 in the \ - modulation plugin first. Disabled until an output folder is \ - selected." + "How many depths the a sweep records, spread evenly from min a \ + to max a." .into(), ), - kind: SettingKind::Button { - enabled: can_record, + kind: SettingKind::I64Drag { + min: 2, + max: 64, + default: self.sweep_count, }, }, - SettingItem { - key: "stop_recording".into(), - label: "Stop (abort recording / sweep)".into(), - tooltip: Some( - "Stop and finalize the current recording before the duration \ - ends; during a sweep this also aborts the remaining points." - .into(), - ), - kind: SettingKind::Button { enabled: true }, - }, - ], - }, - SettingsSection { - label: "Exact event-count depth a₀".into(), - description: Some( - "Second Stage-A workflow, on top of the minimum-depth sweep above: hold \ - ONE photodiode-measured depth a₀ = ln(I_exc,max / I_exc,min) constant \ - across the frequency sweep. Freeze the flux point, camera configuration \ - and references first (pilot and background are recorded above), then per \ - frequency: set f in the modulation plugin, press Find a₀ — A1 leases the \ - drive and trims the *commanded* depth until the photodiode *measures* a₀ \ - — and then press Record a₀ point, which re-applies that depth under the \ - same lease (so the amplitude cannot change during the recorded interval) \ - and records one atomic RAW + PDQ + sidecar point named …_ec_fHz. The \ - found depths are kept per frequency, listed in the a₀ lock table view and \ - mirrored to a0_locks.json in the output folder. Randomising the frequency \ - order, interleaving the low-frequency reference and repeating blocks stay \ - yours — every point is one button press." - .into(), - ), - default_open: false, - items: vec![ SettingItem { key: "a0_target".into(), - label: "a₀ (measured log contrast)".into(), + label: "Frequency axis: the depth a to hold".into(), tooltip: Some( - "The one photodiode-measured depth held across the whole frequency \ - sweep — never a DAC excursion. Pick it from the low-frequency \ - scout: high enough for several events per pixel per half-cycle, \ - still proportional (not saturated), and refractory-safe at the \ - highest frequency." + "Sweep f records every frequency at this one depth, so a change \ + in the event count comes from the frequency and not from the \ + depth. Not used by Sweep a or Sweep a × f, which command each \ + depth themselves.\n\n\ + With the photodiode depth source the drive does not hold a depth \ + by itself as f changes, so it is re-found by measurement at \ + every frequency — see the a₀ section below." .into(), ), kind: SettingKind::F64Drag { @@ -4810,61 +6321,14 @@ impl Plugin for StageAA1Plugin { default: self.a0_target, }, }, - SettingItem { - key: "a0_tolerance".into(), - label: "a₀ tolerance (absolute)".into(), - tooltip: Some( - "Convergence band on |measured a − a₀| for the lock, and the \ - settle band an event-count point must hold before it records." - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.002, - max: 0.5, - speed: 0.002, - default: self.a0_tolerance, - }, - }, - SettingItem { - key: "find_a0".into(), - label: "Find a₀ (lock the drive depth)".into(), - tooltip: Some( - "Leases the modulation owner and iterates commanded a ← commanded \ - a · a₀/measured a until the photodiode-measured depth is a₀ at the \ - current frequency (up to 8 trials, waiting Sweep settle (s) per \ - trial). Records nothing, leaves the drive at the depth it found, \ - and stores it for this frequency. Requires a calibrated \ - periodic/optical drive armed in the modulation plugin and a \ - photodiode-measured a. Disabled until an output folder is selected." - .into(), - ), - kind: SettingKind::Button { - enabled: can_record, - }, - }, - SettingItem { - key: "record_a0_point".into(), - label: "Record a₀ point (event-count)".into(), - tooltip: Some( - "Records one atomic frequency point at the locked depth: re-applies \ - the found commanded a under a modulation lease, waits for the \ - measured a to hold a₀, then records camera RAW + photodiode PDQ + \ - sidecar under one run id (…_ec_fHz). Needs a converged lock for \ - the current frequency and an output folder." - .into(), - ), - kind: SettingKind::Button { - enabled: can_record, - }, - }, SettingItem { key: "min_f".into(), - label: "Sweep min f (Hz)".into(), + label: "Frequency axis: min f (Hz)".into(), tooltip: Some( - "Lowest frequency of the automatic ladder. It decides whether the \ - ladder is measurable at all: the photodiode needs a contrast \ - window of at least one cycle at this frequency, so raise its \ - Cache length if the sweep refuses to start." + "Lowest frequency the f sweep records. It also decides whether \ + the run is measurable at all: the photodiode needs whole \ + modulation cycles inside its cache, so a very low value is \ + refused up front rather than mid-run." .into(), ), kind: SettingKind::F64Drag { @@ -4876,10 +6340,10 @@ impl Plugin for StageAA1Plugin { }, SettingItem { key: "max_f".into(), - label: "Sweep max f (Hz)".into(), + label: "Frequency axis: max f (Hz)".into(), tooltip: Some( - "Highest frequency of the automatic ladder. Check the refractory \ - condition 2·f·a₀/C ≪ 1/τ_refr here — the plugin does not." + "Highest frequency to record. Check yourself that the sensor can \ + follow it." .into(), ), kind: SettingKind::F64Drag { @@ -4891,85 +6355,100 @@ impl Plugin for StageAA1Plugin { }, SettingItem { key: "freq_count".into(), - label: "Frequency points".into(), + label: "Frequency axis: points".into(), tooltip: Some( - "Points on the ladder, log-spaced and inclusive of both ends: \ - |H(f)| is read per decade, so a linear ladder would spend most of \ - its points on the flat part." + "How many frequencies to record, spaced by decade rather than by \ + hertz — a Bode ladder is read per decade." .into(), ), - kind: SettingKind::I64Slider { + kind: SettingKind::I64Drag { min: 1, max: FREQ_SWEEP_MAX_POINTS as i64, default: i64::from(self.freq_count), - suffix: None, }, }, SettingItem { key: "freq_order".into(), - label: "Frequency order".into(), + label: "Frequency axis: order".into(), tooltip: Some( - "Order the ladder is visited in. Low-to-high confounds frequency \ - with anything that drifts through the block (bleaching, thermal \ - bias drift), so prefer alternating or a seeded random order — \ - both are recorded in the sidecar." + "The order the frequencies are visited in. Anything but ascending \ + separates a real frequency effect from slow drift across the \ + block, because neighbouring points are no longer neighbours in \ + time." .into(), ), kind: SettingKind::Enum { variants: vec![ "ascending".into(), - "descending".into(), - "alternating".into(), - "random (seeded)".into(), + "alternating (low, high, low…)".into(), + "shuffled".into(), ], default: self.freq_order.index() as usize, }, }, SettingItem { key: "freq_seed".into(), - label: "Random order seed".into(), + label: "Frequency axis: shuffle seed".into(), tooltip: Some( - "Seed for the random order, so the executed schedule is \ - reproducible and can be frozen in the session plan. Recorded in \ - every point's sidecar." + "Makes the shuffled order repeatable: the same seed always gives \ + the same order. Ignored unless the order is shuffled." .into(), ), kind: SettingKind::I64Drag { min: 1, - max: 9_999, + max: 1_000_000, default: self.freq_seed as i64, }, }, SettingItem { key: "freq_reference_every".into(), - label: "Low-f reference every N points".into(), + label: "Frequency axis: repeat the lowest every N".into(), tooltip: Some( - "Re-visit the lowest planned frequency after every N points, so \ - drift across the block shows up as a disagreement between its \ - repeats. 0 disables it." + "Re-records the lowest frequency after every N points, so drift \ + across the block shows up as a disagreement between its \ + repeats. 0 = off." .into(), ), - kind: SettingKind::I64Slider { + kind: SettingKind::I64Drag { min: 0, max: 10, default: i64::from(self.freq_reference_every), - suffix: None, + }, + }, + SettingItem { + key: "start_recording".into(), + label: "Record once".into(), + tooltip: Some( + "One recording with the light exactly as the modulation plugin \ + has it armed right now. Nothing is retargeted and nothing \ + settles first." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "start_sweep".into(), + label: "Sweep a".into(), + tooltip: Some( + "Records one file at each depth from min a to max a, at the \ + armed frequency. Settles on each depth before recording it." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, }, }, SettingItem { key: "start_freq_sweep".into(), - label: "Start frequency sweep (find a₀ + record per f)".into(), + label: "Sweep f".into(), tooltip: Some( - "Runs the whole ladder unattended on one modulation lease: per \ - frequency it retargets the drive, waits for the phase-0 trigger \ - to confirm the new period, locks a₀ closed-loop, and records one \ - atomic RAW + PDQ + sidecar point. A frequency whose a₀ cannot be \ - reached is skipped and named in the summary rather than ending \ - the ladder. The operator's own frequency and depth come back when \ - the lease is released. References (pilot, background, I_tot \ - anchor) and the flux point stay yours — and pilot windows are \ - dropped at every frequency change, because windows frozen at one \ - period do not transfer to another." + "Records one file at each frequency from min f to max f, all at \ + the depth set above (\"the depth a to hold\"). With the \ + photodiode depth source that depth is re-found by measurement at \ + every frequency, because the drive does not hold it by itself as \ + f changes; with the commanded source it is simply commanded." .into(), ), kind: SettingKind::Button { @@ -4977,12 +6456,52 @@ impl Plugin for StageAA1Plugin { }, }, SettingItem { - key: "clear_a0_locks".into(), - label: "Clear a₀ lock table".into(), + key: "start_freq_depth_sweep".into(), + label: "Sweep a × f".into(), + tooltip: Some( + "The whole surface: every depth in the a range, at every \ + frequency in the f range. That is (frequency points × depth \ + points) recordings — check both counts and the duration before \ + starting. Files are named …_fHz_pNN so the surface sorts by \ + frequency and then by depth." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_pilot".into(), + label: "Record pilot (freezes the ON/OFF windows)".into(), + tooltip: Some( + "A bright reference recording whose ON/OFF windows are reused by \ + every later recording in the same measurement, so the whole row \ + is scored consistently." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_background".into(), + label: "Record background (a ≈ 0 reference)".into(), + tooltip: Some( + "An unmodulated reference giving the false-response floor the \ + later points are measured above." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "stop_recording".into(), + label: "Stop".into(), tooltip: Some( - "Drops every stored per-frequency lock and rewrites \ - a0_locks.json. Use it after changing the flux point, the \ - calibration or a₀ itself." + "Stops whatever is running — a recording, a sweep, a ladder or a \ + protocol — at its next safe point, so the file in flight is \ + still finished and saved." .into(), ), kind: SettingKind::Button { enabled: true }, @@ -4990,102 +6509,130 @@ impl Plugin for StageAA1Plugin { ], }, SettingsSection { - label: "Live analysis".into(), + label: "Protocol (run a survey from a file)".into(), description: Some( - "Live sanity quicklook. Folds the camera event stream on the modulation \ - period T (defined by the firmware phase-0 EXT_TRIGGER) and renders the \ - rolling half-period response S_p(t): events per valid pixel in the \ - trailing T/2, ON and OFF separately. Use it to confirm events are \ - appearing and the ON/OFF timing looks sane before recording. Nothing is \ - recorded here." + "The buttons above sweep one axis with the others left wherever they \ + happen to be. A protocol names every axis for every recording instead, \ + in a file that travels with the results.\n\n\ + **CSV — one row per recording.** Columns: mean_u (the brightness / I_k \ + axis), frequency_hz, depth_a, plus optional duration_s, settle_s, role \ + (normal / pilot / background) and label. Columns are found by name so \ + their order does not matter; blank lines and # comments are skipped, \ + and a blank cell falls back to the default. Because each row carries \ + its own duration, a 1 Hz point can record for 40 s and a 200 Hz point \ + for 10 — and a file can start with its own background and pilot.\n\n\ + **TOML — blocks and ranges.** [[block]] with a list or \ + { min, max, points } range on each axis, expanded to their product. \ + More compact for a dense regular sweep. Points run ū outermost, then f, \ + then a, which settles the slow axis least often.\n\n\ + Either way: all three axes are commanded at every point and the point \ + waits for all three to be acknowledged before recording, so nothing is \ + ever filed under parameters the file does not state. One modulation \ + lease covers the whole run and your armed settings are handed back at \ + the end. A point the drive cannot reach is skipped and named rather \ + than stopping the survey.\n\n\ + Commented examples ship with the plugin, at \ + ~/.augur/plugins/stage-a-a1/protocols/ — example.csv and example.toml." .into(), ), - default_open: true, + default_open: false, items: vec![ SettingItem { - key: "live".into(), - label: "Live analysis".into(), + key: "protocol_path".into(), + label: "Protocol file".into(), tooltip: Some( - "Fold incoming events into the live plots. Off freezes the plots \ - at their current values. This does not record anything." + ".csv (one row per recording) or .toml (blocks and ranges) — the \ + reader is chosen by the extension. Read and fully validated \ + when you press Run, so a bad value is reported with its line \ + number before the drive moves." .into(), ), - kind: SettingKind::Bool { default: self.live }, + kind: SettingKind::Path { + dialog: PathDialogKind::OpenFile, + default: self.protocol_path.clone(), + }, }, SettingItem { - key: "analysis_window_ms".into(), - label: "Analysis window (ms)".into(), + key: "run_protocol".into(), + label: "Run the protocol".into(), tooltip: Some( - "Trailing window pulled exactly from the retained EventStore. \ - Longer windows cover more cycles; bounded by the host event-store \ - memory budget." + "Reads and validates the file, then records every point in it. \ + The status line reports how many recordings and roughly how long \ + it will take before the first one starts, and tracks progress \ + after that.\n\n\ + Use Stop in the Record section to end it early — the recording \ + in flight is still finished and saved." .into(), ), - kind: SettingKind::I64Drag { - min: 1, - max: 120_000, - default: self.analysis_window_ms, + kind: SettingKind::Button { + enabled: can_record, }, }, - SettingItem { - key: "clear".into(), - label: "Clear captured events".into(), - tooltip: Some( - "Empties the fold buffer and resets the live plots.".into(), - ), - kind: SettingKind::Button { enabled: true }, - }, ], }, SettingsSection { - label: "Response probability q_p (live quicklook)".into(), + label: "Advanced: hold one depth across frequencies (a₀)".into(), description: Some( - "Live view of the response-curve metric q_p: the fraction of valid \ - pixel-cycles that fire at least once in the ON/OFF phase window (unlike \ - S_p, each pixel-cycle counts at most once). The ON/OFF windows come from \ - the row's pilot when one has been recorded (frozen, in the Recording \ - section) and otherwise from the trigger-anchored fold automatically — each \ - window grows out from its histogram peak until events drop below the \ - window floor or the opposite polarity takes over. Press Record point at \ - each amplitude to append a q_p(a) dot at the photodiode-measured a. The \ - ROI and masked pixels come from the camera config. The authoritative fit \ - is computed offline from the recordings; this is a quicklook." + "Only needed with the photodiode depth source. The drive does not \ + deliver the same depth at every frequency by itself, so before \ + recording a frequency point the plugin adjusts it until the photodiode \ + measures a₀. That search is what Find a₀ does, and Sweep f runs it \ + automatically at every frequency.\n\n\ + With the commanded depth source there is nothing to search for and \ + none of this is used. The depths found are saved per frequency in the \ + output folder and survive a restart." .into(), ), default_open: false, items: vec![ SettingItem { - key: "window_floor".into(), - label: "Window floor (fraction of peak)".into(), + key: "a0_tolerance".into(), + label: "a₀ tolerance".into(), tooltip: Some( - "Each ON/OFF window grows out from its histogram peak until events \ - fall below this fraction of the peak (or the opposite polarity \ - takes over). 0.10 = stop at 10 % of the peak." + "How close the measured depth has to get to a₀ before the search \ + calls it done. Tighter takes longer and can fail on a noisy \ + reading." .into(), ), kind: SettingKind::F64Drag { - min: 0.02, + min: 0.002, max: 0.5, - speed: 0.01, - default: self.window_floor, + speed: 0.002, + default: self.a0_tolerance, }, }, SettingItem { - key: "record_point".into(), - label: "Record point (at current a)".into(), + key: "find_a0".into(), + label: "Find a₀ (at the armed frequency)".into(), tooltip: Some( - "Computes q_on/q_off for the current buffer against the \ - auto-detected windows and appends a point at the \ - photodiode-measured a." + "Searches for the drive depth that makes the photodiode measure \ + a₀ at the frequency currently armed, and remembers it." .into(), ), kind: SettingKind::Button { enabled: true }, }, SettingItem { - key: "clear_curve".into(), - label: "Clear response curve".into(), - tooltip: Some("Drops the recorded response-curve points.".into()), - kind: SettingKind::Button { enabled: true }, + key: "record_a0_point".into(), + label: "Record a₀ point".into(), + tooltip: Some( + "Records one file at the depth Find a₀ found for the armed \ + frequency." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "clear_a0_locks".into(), + label: "Forget saved depths".into(), + tooltip: Some( + "Throws away every depth Find a₀ has found. Do this after \ + changing the illumination, the calibration, or a₀ itself — the \ + old depths no longer apply." + .into(), + ), + kind: SettingKind::Button { enabled: true }, }, ], }, @@ -5097,7 +6644,7 @@ impl Plugin for StageAA1Plugin { match key { "output_folder" => Some(json!(self.output_folder)), "measurement_id" => Some(json!(self.measurement_id)), - "flux_point_id" => Some(json!(self.flux_point_id)), + "depth_source" => Some(json!(self.depth_source.index())), "min_a" => Some(json!(self.min_a)), "max_a" => Some(json!(self.max_a)), "sweep_count" => Some(json!(self.sweep_count)), @@ -5129,6 +6676,9 @@ impl Plugin for StageAA1Plugin { "freq_seed" => Some(json!(self.freq_seed)), "freq_reference_every" => Some(json!(self.freq_reference_every)), "start_freq_sweep" => Some(self.press_freq_sweep.value()), + "start_freq_depth_sweep" => Some(self.press_freq_depth_sweep.value()), + "protocol_path" => Some(json!(self.protocol_path)), + "run_protocol" => Some(self.press_run_protocol.value()), // New id regenerates the measurement id locally; the id itself is // what synchronizes, so the press must not be forwarded (both // instances would generate different ids). @@ -5151,15 +6701,13 @@ impl Plugin for StageAA1Plugin { .ok_or("measurement_id must be a string")? .to_string(); } - "flux_point_id" => { - self.flux_point_id = value - .as_str() - .ok_or("flux_point_id must be a string")? - .to_string(); - } "new_id" if value.as_bool() == Some(true) => { self.measurement_id = generate_measurement_id(); } + "depth_source" => { + self.depth_source = + DepthSource::from_index(value.as_u64().ok_or("depth_source must be an index")?); + } "min_a" => { self.min_a = value .as_f64() @@ -5210,33 +6758,30 @@ impl Plugin for StageAA1Plugin { self.sweep_pending = true; } } + "start_freq_sweep" => { + if self.press_freq_sweep.accept(&value) { + self.freq_sweep_pending = Some(FreqSweepMode::A0Point); + } + } + "start_freq_depth_sweep" => { + if self.press_freq_depth_sweep.accept(&value) { + self.freq_sweep_pending = Some(FreqSweepMode::DepthSweep); + } + } "stop_recording" => { if self.press_stop.accept(&value) { - if self.recording.is_active() { - self.recording.stop_requested = true; - } - if let Some(sweep) = self.sweep.as_mut() { - sweep.stop_requested = true; - self.message = "Sweep stop requested".into(); - } - if let Some(lock) = self.a0_lock.as_mut() { - lock.stop_requested = true; - self.message = "a₀ lock stop requested".into(); - } - // Last, so its wording wins: a stop during a ladder is a - // stop of the ladder, whatever child was mid-flight. - if let Some(sweep) = self.freq_sweep.as_mut() { - sweep.stop_requested = true; - self.message = "Frequency sweep stop requested".into(); - } - self.sweep_pending = false; - self.a0_lock_pending = false; - self.a0_point_pending = false; - self.freq_sweep_pending = false; + self.request_stop(); } } "live" => { self.live = value.as_bool().ok_or("live must be a boolean")?; + if !self.live { + // Immediately, not on the next frame: with no camera + // running there is no next frame, and the operator who just + // switched this off is the one waiting for the plots to + // stop being slow. + self.drop_live_buffers(); + } } "analysis_window_ms" => { self.analysis_window_ms = value @@ -5246,9 +6791,7 @@ impl Plugin for StageAA1Plugin { } "clear" => { if self.press_clear.accept(&value) { - self.camera_events.clear(); - self.event_scratch.clear(); - self.camera_markers_us.clear(); + self.drop_live_buffers(); } } "window_floor" => { @@ -5320,9 +6863,15 @@ impl Plugin for StageAA1Plugin { .ok_or("freq_reference_every must be an integer")? .min(10) as u32; } - "start_freq_sweep" => { - if self.press_freq_sweep.accept(&value) { - self.freq_sweep_pending = true; + "protocol_path" => { + self.protocol_path = value + .as_str() + .ok_or("protocol_path must be a string")? + .to_string(); + } + "run_protocol" => { + if self.press_run_protocol.accept(&value) { + self.protocol_pending = true; } } "clear_a0_locks" => { @@ -5347,6 +6896,25 @@ impl Plugin for StageAA1Plugin { value: self.recording.state_label().into(), color: None, }]; + if let Some(run) = self.protocol.as_ref() { + entries.push(StatusEntry::Text(format!( + "Protocol '{}': point {}/{} — {} recorded, {} skipped", + run.plan.name, + (run.index + 1).min(run.plan.points.len()), + run.plan.points.len(), + run.recorded, + run.failed.len(), + ))); + // The per-point message is overwritten within the tick that skips a + // point, so the most recent reason lives here instead of scrolling + // past unread. + if let Some((index, reason)) = run.failed.last() { + entries.push(StatusEntry::Text(format!( + "Last skipped point {}: {reason}", + index + 1 + ))); + } + } if self.recording.is_active() { if let Some(remaining) = self.recording.remaining_s(now_unix_ms()) { entries.push(StatusEntry::Text(format!( @@ -5357,15 +6925,22 @@ impl Plugin for StageAA1Plugin { } if let Some(sweep) = &self.freq_sweep { let phase = match sweep.phase { - FreqSweepPhase::AcquiringLease => "leasing modulation", - FreqSweepPhase::SettingFrequency => "retargeting frequency", - FreqSweepPhase::ConfirmingFrequency => "confirming from the trigger", - FreqSweepPhase::Locking => "locking a₀", - FreqSweepPhase::Recording => "recording", + FreqSweepPhase::AcquiringLease => "taking control of the drive", + FreqSweepPhase::SettingFrequency => "changing the frequency", + FreqSweepPhase::ConfirmingFrequency => "checking the frequency really changed", + FreqSweepPhase::Locking => "finding the depth a₀", + FreqSweepPhase::Recording => match sweep.mode { + // The inner sweep prints its own point-by-point line below, + // so this one only has to say which stage of the *ladder* + // the run is in. + FreqSweepMode::A0Point => "recording", + FreqSweepMode::DepthSweep => "recording the depth curve", + }, }; let point = sweep.point(); entries.push(StatusEntry::Text(format!( - "Frequency sweep {}/{} at {}{} — {phase} ({} recorded, {} skipped)", + "Frequency ladder ({}) {}/{} at {}{} — {phase} ({} done, {} skipped)", + sweep.mode.label(), sweep.index + 1, sweep.points.len(), frequency_label(sweep.frequency_hz()), @@ -5380,17 +6955,17 @@ impl Plugin for StageAA1Plugin { } if let Some(sweep) = &self.sweep { let phase = match sweep.phase { - SweepPhase::AcquiringLease => "leasing modulation", - SweepPhase::SettingDepth => "retargeting drive", - SweepPhase::Settling => "settling", + SweepPhase::AcquiringLease => "taking control of the drive", + SweepPhase::SettingDepth => "changing the depth", + SweepPhase::Settling => "waiting for the depth to settle", SweepPhase::Recording => "recording", }; let label = match sweep.kind { - SweepKind::Amplitude => "Sweep", - SweepKind::EventCount => "Event-count point", + SweepKind::Amplitude => "Depth sweep", + SweepKind::EventCount => "a₀ point", }; entries.push(StatusEntry::Text(format!( - "{label}: point {}/{} commanding a = {:.3} for a measured {:.3} ({phase})", + "{label}: point {}/{}, asking for a = {:.3} to measure {:.3} ({phase})", sweep.index + 1, sweep.total(), sweep.commanded_a(), @@ -5399,13 +6974,13 @@ impl Plugin for StageAA1Plugin { } if let Some(lock) = &self.a0_lock { let phase = match lock.phase { - A0LockPhase::AcquiringLease => "leasing modulation", - A0LockPhase::SettingDepth => "commanding depth", + A0LockPhase::AcquiringLease => "taking control of the drive", + A0LockPhase::SettingDepth => "setting the depth", A0LockPhase::Measuring => "measuring", }; entries.push(StatusEntry::Text(format!( - "a₀ lock at {}: trial {}/{A0_LOCK_MAX_TRIALS} commanding a = {:.3} for a₀ = {:.3} \ - ({phase}, {} sample(s))", + "Find a₀ at {}: try {}/{A0_LOCK_MAX_TRIALS}, asking for a = {:.3} to measure a₀ = \ + {:.3} ({phase}, {} reading(s))", frequency_label(lock.frequency_hz), lock.trial, lock.commanded_a, @@ -5416,87 +6991,150 @@ impl Plugin for StageAA1Plugin { if !self.message.is_empty() { entries.push(StatusEntry::Text(self.message.clone())); } + // Each fact appears once. The panel used to state a missing frequency on + // three separate lines — the transient message, this line, and the a₀ + // readiness line — which reads as three problems instead of one. + let no_frequency = self.frequency_hz().is_none(); match self.period_us() { Some(period_us) => { - let source = self.frequency_source(); + let source = match self.frequency_source() { + "trigger" => "measured from the trigger", + _ => "as set in the modulation plugin", + }; entries.push(StatusEntry::Text(format!( - "T = {:.3} ms ({:.3} Hz, {source})", - period_us / 1_000.0, + "Frequency: {:.3} Hz ({source}) — one cycle is {:.3} ms", 1_000_000.0 / period_us, + period_us / 1_000.0, ))); } - None => entries.push(StatusEntry::Text( - "No modulation period (connect modulation or the EXT_TRIGGER)".into(), - )), + None => entries.push(StatusEntry::Text(format!( + "Frequency: unknown. {}", + capitalize_first( + &self + .frequency_blocker() + .unwrap_or_else(|| "no drive is armed".into()) + ) + ))), } - let anchor = if self.is_marker_anchored() { + // With Live analysis off nothing is ingested at all, so an event count + // and a pixel count describe the switch rather than the bench. Say only + // what is true. + entries.push(StatusEntry::Text(if !self.live { + "Camera: Live analysis is OFF — no events or triggers are being read".into() + } else { + let anchor = if self.is_marker_anchored() { + format!("{} triggers seen", self.camera_markers_us.len()) + } else { + "no trigger signal — check the cable from the Teensy to the camera".into() + }; format!( - "{} phase-0 markers (trigger-anchored)", - self.camera_markers_us.len() + "Camera: {} events, {} usable pixels; {anchor}", + self.camera_events.len(), + self.valid_pixel_count().unwrap_or(0) ) - } else { - "free-running (no EXT_TRIGGER)".into() + })); + let depth_label = match self.depth_source { + DepthSource::Photodiode => "Measured depth a", + DepthSource::Commanded => "Commanded depth a (open loop, not measured)", }; - entries.push(StatusEntry::Text(format!( - "{} events, {} valid pixels; {anchor}", - self.camera_events.len(), - self.valid_pixel_count().unwrap_or(0) - ))); - entries.push(StatusEntry::Text(match self.measured_a() { - Some(a) => format!("a = {a:.3} (photodiode)"), - None => { - let detail = self - .photodiode - .as_ref() - .map(|summary| connection_label(&summary.connection)) - .unwrap_or("no snapshot"); - format!("a = — (photodiode: {detail})") - } + entries.push(StatusEntry::Text(match self.depth_a() { + Some(a) => format!("{depth_label} = {a:.3}"), + // Name the gate, not just its effect: every a₀ and sweep button + // refuses on this value, so the panel has to say what to fix. A + // colon, not a dash: the reason carries a dash of its own. + None => format!( + "{depth_label}: not available. {}", + capitalize_first( + &self + .depth_a_blocker() + .unwrap_or_else(|| "no depth source has sent anything yet".into()) + ) + ), })); + // Silent when the host reports nothing (replay, or a camera without a + // monitoring block) rather than printing three dashes. + if let Some(sensor) = self.sensor { + let mut parts = Vec::new(); + if let Some(celsius) = sensor.temperature_c { + parts.push(format!("{celsius:.1} °C")); + } + if let Some(dead_time_us) = sensor.pixel_dead_time_us { + parts.push(format!("{dead_time_us:.1} µs dead time")); + } + if let Some(lux) = sensor.illumination_lux { + parts.push(format!("{lux:.0} lx")); + } + if !parts.is_empty() { + entries.push(StatusEntry::Text(format!( + "Sensor: {} (read {:.1} s ago; recorded with every run)", + parts.join(", "), + sensor.age_s + ))); + } + } if let Some((on, off)) = self.latest_rolling() { entries.push(StatusEntry::Text(format!( - "S_on = {on:.4}, S_off = {off:.4} (events/pixel per T/2)" + "Events per pixel per half-cycle: {on:.4} bright, {off:.4} dark" + ))); + } + // Silent until there is something to report: at rest this line said + // "0 point(s), no bright/dark windows yet", which is just the absence of + // the two facts above it. + let windows = self.current_windows(); + if !self.response_points.is_empty() || windows.is_some() { + let source = if self.windows_are_frozen() { + "from the pilot" + } else { + "found automatically" + }; + let windows = windows.map_or_else( + || "no bright/dark windows yet".into(), + |(on, off)| { + format!( + "bright/dark windows {source} (bright {:.2}–{:.2}, dark {:.2}–{:.2} of a \ + cycle)", + on.start, on.end, off.start, off.end + ) + }, + ); + entries.push(StatusEntry::Text(format!( + "Response curve: {} point(s), {windows}", + self.response_points.len() ))); } - let source = if self.windows_are_frozen() { - "pilot-frozen" - } else { - "auto" - }; - let windows = self.current_windows().map_or_else( - || "windows —".into(), - |(on, off)| { - format!( - "windows ({source}) ON [{:.2},{:.2}) OFF [{:.2},{:.2})", - on.start, on.end, off.start, off.end - ) - }, - ); - let valid = self - .valid_pixel_count() - .map_or_else(|| "—".into(), |n| n.to_string()); - entries.push(StatusEntry::Text(format!( - "Response curve: {windows}, N_valid = {valid}, {} point(s)", - self.response_points.len() - ))); if let Some((q0_on, q0_off)) = self.background_floor { entries.push(StatusEntry::Text(format!( - "Background floor: q0_on = {q0_on:.3}, q0_off = {q0_off:.3}" + "Background floor recorded: {q0_on:.3} bright, {q0_off:.3} dark" ))); } - entries.push(StatusEntry::Text(match self.armed_lock() { - Some(lock) => format!( - "a₀ = {:.3} armed at {}: commanded a = {:.3} (measured {:.3}); {} lock(s) stored", + // Open loop there is no lock table and nothing was searched for, so the + // line says what will happen rather than reporting a saved-depth count + // that is structurally always zero. + entries.push(StatusEntry::Text(match (self.armed_a0(), no_frequency) { + (Some(lock), _) if !self.depth_source.needs_a0_lock() => format!( + "Ready to record at a₀ = {:.3}: the drive is commanded to a = {:.3} at {} — no \ + search needed, press \"Record a₀ point\" or \"Record all frequencies\".", + lock.target_a, + lock.commanded_a, + frequency_label(lock.frequency_hz), + ), + (Some(lock), _) => format!( + "Ready to record at a₀ = {:.3}: at {} the drive is set to a = {:.3} and the \ + photodiode measures {:.3}. {} depth(s) saved.", lock.target_a, frequency_label(lock.frequency_hz), lock.commanded_a, lock.measured_a, self.a0_locks.len() ), - None => format!( - "a₀ = {:.3}: no lock for this frequency — press Find a₀; {} lock(s) stored", - self.a0_target, - self.a0_locks.len() + // Without a frequency nothing about a₀ can be judged yet, and the + // Frequency line above already says what to fix. Point at it rather + // than repeating it. + (None, true) => "a₀ points: waiting for a frequency (above).".into(), + (None, false) => format!( + "Not ready to record an a₀ point — {}.", + self.armed_a0_blocker() + .unwrap_or_else(|| "no depth is armed".into()), ), })); entries @@ -5521,7 +7159,7 @@ impl Plugin for StageAA1Plugin { column("measurement_id", "Measurement id"), column("remaining", "Remaining"), column("frequency", "Frequency"), - column("a", "a (photodiode)"), + column("a", "a (depth)"), column("s_on", "S_on"), column("s_off", "S_off"), column("events", "Events"), @@ -5557,7 +7195,8 @@ impl Plugin for StageAA1Plugin { column("frequency", "Frequency"), column("target_a", "a₀ (target)"), column("commanded_a", "Commanded a"), - column("measured_a", "Measured a"), + column("measured_a", "Observed a"), + column("depth_source", "a from"), column("trials", "Trials"), column("state", "State"), column("locked_at", "Locked at (UTC)"), @@ -5674,6 +7313,7 @@ mod tests { } // Same order as `process_control`: outermost supervisor first, so one // tick can carry a hand-off from the ladder down into a recording. + plugin.drive_protocol(sink); plugin.drive_freq_sweep(sink); plugin.drive_a0_lock(sink); plugin.drive_sweep(sink); @@ -5694,6 +7334,21 @@ mod tests { } } + /// A control inbox carrying just these service replies. + fn inbox_with(service_replies: Vec) -> PluginControlInbox { + PluginControlInbox { + service_replies, + ..PluginControlInbox::default() + } + } + + /// The modulation command inside a routed service request, if it is one. + fn modulation_command(request: &PluginServiceRequest) -> Option { + serde_json::from_value::(request.payload.clone()) + .ok() + .map(|envelope| envelope.command) + } + fn rejected(request_id: u64, message: &str) -> PluginServiceReply { PluginServiceReply { request_id, @@ -5736,6 +7391,24 @@ mod tests { } } + /// A connected modulation owner running a calibrated optical drive at + /// `depth_a`, published at `revision`. + fn commanded_modulation(revision: u64, depth_a: f64) -> ModulationStateV1 { + ModulationStateV1 { + service_revision: revision, + optical_drive: Some(stage_a_plugin_contract::OpticalDriveStateV1 { + target: OpticalTargetV1::LogSine, + requested_mean_u_milli: 500, + resolved_mean_u_milli: 500, + internal_u_milli: 500, + depth_a_milli: (depth_a * 1_000.0).round() as u32, + v_null_dac: 100, + v_peak_dac: 800, + }), + ..connected_modulation() + } + } + /// A photodiode snapshot reporting `measured_a`, published at `revision`. fn photodiode_measuring(revision: u64, measured_a: f64) -> PhotodiodeSummaryV1 { PhotodiodeSummaryV1 { @@ -5785,6 +7458,7 @@ mod tests { window_seconds: Some(0.001), covered_cycles: Some(8.0), }), + optical_unavailable: None, synchronization: stage_a_plugin_contract::SynchronizationV1::Unsynced { reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, detail: None, @@ -5881,348 +7555,937 @@ mod tests { max_ticks } - fn pd_reply(request_id: u64, receipt: Option) -> PluginServiceReply { - let response = PhotodiodeResponseV1 { - common: ResponseCommonV1 { - contract_version: CONTRACT_VERSION_V1, - request_id: RequestId(request_id), - owner_instance: OwnerInstanceId::new("pd-test"), - run_id: None, - requested_revision: None, - acknowledged_revision: None, - outcome: RequestOutcomeV1::Applied, - completed_at_unix_ms: Some(now_unix_ms()), - error: None, - }, - receipt, - }; - PluginServiceReply { - request_id, - source_plugin_id: A1_PLUGIN_ID.into(), - target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), - service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), - outcome: PluginServiceOutcome::Accepted { - payload: serde_json::to_value(response).expect("response"), - }, - } - } - - /// A photodiode summary that passes the pre-flight: connected, unleased, - /// and with somewhere to put the PDQ. - fn ready_photodiode() -> PhotodiodeSummaryV1 { + /// The bench that motivated the commanded source: the photodiode streams + /// fine but withholds `a` because no phase-0 markers ever arrive, so no + /// window can be proven to cover whole modulation cycles. + fn photodiode_without_triggers() -> PhotodiodeSummaryV1 { PhotodiodeSummaryV1 { - contract_version: CONTRACT_VERSION_V1, - owner_instance: OwnerInstanceId::new("pd-test"), - service_revision: 1, - connection: ConnectionStateV1::Connected { - port_label: "mock".into(), - firmware_version: None, - }, - lease: None, - active_run_id: None, - requested_revision: None, - acknowledged_revision: None, - stream: PhotodiodeStreamV1 { - stream_epoch: 1, - sample_range: None, - sample_rate_hz: Some(20_000), - latest_adc_code: Some(1_000), - integrity: StreamIntegrityV1::default(), - level: None, - }, - data_dir: Some("/pd".into()), - active_recording: None, - last_finalized_recording: None, optical_summary: None, - synchronization: SynchronizationV1::Unsynced { - reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, - detail: None, - }, - last_response: None, - freshness: FreshnessV1 { - observed_at_unix_ms: now_unix_ms(), - valid_for_ms: 60_000, - }, + optical_unavailable: Some( + "no stretch of samples covers two whole modulation cycles between triggers \ + (0 trigger(s) in the last 3446784 samples) — lower the frequency, or raise the \ + photodiode cache length" + .into(), + ), + ..photodiode_measuring(1, 0.5) } } - fn on(timestamp_us: u64) -> CameraEvent { - CameraEvent { - timestamp_us, - x: 0, - y: 0, - polarity: Polarity::On, - } + #[test] + fn a_withheld_photodiode_depth_names_the_commanded_fallback() { + // The photodiode's own reason has to survive verbatim — it is the only + // side that knows which gate refused — but a bench with no trigger + // markers at all cannot act on it, so the way past must be on the same + // line as the diagnosis. + let mut plugin = plugin_with_markers(); + plugin.photodiode = Some(photodiode_without_triggers()); + + let blocker = plugin.depth_a_blocker().expect("a withheld a has a reason"); + assert!( + blocker.contains("two whole modulation cycles"), + "the owner's own words must survive: {blocker}" + ); + assert!( + blocker.contains("Depth a source"), + "and must name the setting that gets past it: {blocker}" + ); } - fn fresh_photodiode_summary() -> PhotodiodeSummaryV1 { - PhotodiodeSummaryV1 { - contract_version: CONTRACT_VERSION_V1, - owner_instance: OwnerInstanceId::new("pd-test"), - service_revision: 1, - connection: ConnectionStateV1::Connected { - port_label: "mock".into(), - firmware_version: Some("test".into()), - }, - lease: None, - active_run_id: None, - requested_revision: None, - acknowledged_revision: None, - stream: PhotodiodeStreamV1 { - stream_epoch: 1, - sample_range: None, - sample_rate_hz: Some(20_000), - latest_adc_code: Some(1_000), - integrity: StreamIntegrityV1::default(), - level: None, - }, - active_recording: None, - last_finalized_recording: None, - data_dir: Some(std::env::temp_dir().display().to_string()), - optical_summary: Some(PhotodiodeOpticalSummaryV1 { - run_id: RunId::from("test-run"), - calibration: PhotodiodeCalibrationV1 { - adc_calibration_id: "adc-test".into(), - dark_id: "dark-test".into(), - anchor_id: "itot-test".into(), - dark_volts: 0.05, - total_power_volts: 3.0, - }, - measured_log_contrast: 1.0, - log_contrast_stddev: None, - excitation_min_volts: 0.8, - excitation_max_volts: 0.8 * std::f64::consts::E, - excitation_headroom_volts: 0.8, - low_clip_fraction: 0.0, - high_clip_fraction: 0.0, - measured_frequency_hz: Some(1_000.0), - fundamental_phase_rad: None, - total_harmonic_distortion: None, - window_seconds: Some(0.008), - covered_cycles: Some(8.0), + #[test] + fn the_commanded_source_reports_a_depth_with_no_photodiode_at_all() { + let mut plugin = plugin_with_markers(); + plugin.photodiode = None; + plugin.modulation = Some(commanded_modulation(1, 0.75)); + plugin.depth_source = DepthSource::Commanded; + + assert_eq!(plugin.depth_a(), Some(0.75)); + assert!( + plugin.depth_a_blocker().is_none(), + "the commanded drive is a depth source in its own right" + ); + } + + #[test] + fn the_commanded_source_refuses_a_drive_that_is_not_calibrated() { + // Without a calibrated optical drive the owner publishes no inversion, + // and a "commanded a" would be a DAC number wearing a physical name. + let mut plugin = plugin_with_markers(); + plugin.photodiode = None; + plugin.modulation = Some(connected_modulation()); + plugin.depth_source = DepthSource::Commanded; + + assert!(plugin.depth_a().is_none()); + let blocker = plugin.depth_a_blocker().expect("a reason"); + assert!( + blocker.contains("calibration") && blocker.contains("OPTICAL_LOG_SINE"), + "{blocker}" + ); + } + + /// An acknowledged periodic drive at `hz`, i.e. what the modulation owner + /// publishes once it has really applied a commanded frequency. + fn acknowledged_sine(hz: f64) -> stage_a_plugin_contract::ModulationTargetV1 { + stage_a_plugin_contract::ModulationTargetV1 { + revision: SemanticRevision(1), + waveform: Some(WaveformV1::Periodic { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Sine, + min_dac: 100, + max_dac: 900, + frequency_millihz: (hz * 1_000.0).round() as u64, }), - synchronization: SynchronizationV1::Unsynced { - reason: UnsyncedReasonV1::NoLease, - detail: None, - }, - last_response: None, - freshness: FreshnessV1 { - observed_at_unix_ms: now_unix_ms(), - valid_for_ms: 60_000, - }, + a1_configuration: None, + acquisition_running: true, + board_dac_code: None, + firmware_configuration_revision: None, } } - /// A plugin whose period comes from marker spacing (no fallback frequency). - fn plugin_with_markers() -> StageAA1Plugin { + /// A plugin ready to record a₀ points open loop: calibrated commanded + /// drive, no photodiode, and — deliberately — no camera trigger markers. + fn plugin_commanded_a0(folder: &Path) -> StageAA1Plugin { StageAA1Plugin { - // 10 x 1 sensor, no host ROI => valid_pixel_count() == 10. frame_width: 10, frame_height: 1, - camera_markers_us: vec![0, 1_000, 2_000, 3_000], - flux_point_id: "flux-test".into(), + depth_source: DepthSource::Commanded, photodiode: Some(fresh_photodiode_summary()), - ..StageAA1Plugin::default() - } - } - - #[test] - fn period_comes_from_the_trigger_marker_spacing() { - let plugin = plugin_with_markers(); - let period = plugin.period_us().expect("measured period"); - assert!((period - 1_000.0).abs() < 1e-6, "period={period}"); - assert_eq!(plugin.frequency_source(), "trigger"); + modulation: Some(commanded_modulation(1, 0.5)), + output_folder: folder.display().to_string(), + measurement_id: "A1-cmd".into(), + settle_s: 0.0, + a0_target: 0.5, + a0_tolerance: 0.02, + ..StageAA1Plugin::default() + } } #[test] - fn no_markers_and_no_modulation_yields_no_period() { - let plugin = StageAA1Plugin::default(); - assert!(plugin.period_us().is_none()); - assert!(plugin.rolling_dataset().lines[0].points.is_empty()); - } + fn find_a0_refuses_to_search_for_a_depth_it_is_commanding() { + // The search commands a₀, reads back a₀ and stops — one identical row + // per frequency and nothing learned. Refuse and say so rather than + // spending a lease and a trial to arrive where the operator already is. + let dir = temp_folder("commanded-find"); + let mut plugin = plugin_commanded_a0(&dir); + let mut sink = ControlSink::default(); - #[test] - fn external_triggers_anchor_the_fold() { - let mut plugin = plugin_with_markers(); - for cycle in 0..3 { - plugin.camera_events.push(on(cycle * 1_000 + 200)); - } - assert!(plugin.is_marker_anchored()); - let fold = plugin.current_fold().expect("marker fold"); - assert_eq!(fold.validation.cycle_count, 3); - assert!((fold.events[0].phase - 0.2).abs() < 1e-9); - } + plugin.set_setting("find_a0", json!(true)).expect("press"); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); - #[test] - fn rolling_dataset_keeps_on_and_off_separate() { - let mut plugin = plugin_with_markers(); - for cycle in 0..3 { - let base = cycle * 1_000; - plugin.camera_events.push(on(base + 100)); - plugin.camera_events.push(CameraEvent { - polarity: Polarity::Off, - ..on(base + 600) - }); - } - let rolling = plugin.rolling_dataset(); - assert_eq!(rolling.lines.len(), 2); - assert_eq!(rolling.lines[0].name, "ON"); - assert!(rolling.lines[0].points.len() >= 2); + assert!(plugin.a0_lock.is_none(), "no search may start"); + assert!( + sink.services.is_empty(), + "and no lease may be taken for it: {:?}", + sink.services.len() + ); + assert!( + plugin.message.contains("No search needed"), + "{}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&dir); } #[test] - fn response_curve_auto_windows_without_a_pilot_and_refuses_without_a() { - let mut plugin = plugin_with_markers(); - plugin.frame_width = 8; - plugin.frame_height = 1; - for cycle in 0..20 { - let base = cycle * 1_000; - for x in 0..4 { - plugin.camera_events.push(CameraEvent { - timestamp_us: base + 200, - x, - y: 0, - polarity: Polarity::On, - }); - plugin.camera_events.push(CameraEvent { - timestamp_us: base + 700, - x, - y: 0, - polarity: Polarity::Off, - }); - } - } - plugin.camera_markers_us = (0..=20).map(|c| c * 1_000).collect(); - plugin.host_roi = Some(RoiV1 { - x: 0, - y: 0, - width: 4, - height: 1, + fn a_commanded_a0_point_is_armed_without_any_lock() { + // `Record a₀ point` and the ladder both ask one question — what depth + // is armed here — and open loop the answer needs no stored table. + let dir = temp_folder("commanded-armed"); + let mut plugin = plugin_commanded_a0(&dir); + // 1 kHz from the modulation owner's acknowledged drive; no markers. + plugin.modulation = Some(ModulationStateV1 { + acknowledged: Some(acknowledged_sine(1_000.0)), + ..commanded_modulation(1, 0.5) }); - // Windows come straight from the fold — no pilot capture needed. - let (q_on, q_off, _, valid) = plugin.current_response().expect("response"); - assert_eq!(valid, 4); - assert!(q_on > 0.9 && q_off > 0.9, "q_on={q_on} q_off={q_off}"); - // Recording a point is still refused without a photodiode-measured a. - plugin.photodiode = None; - assert!(plugin.measured_a().is_none()); - assert!(plugin.record_response_point().is_err()); + assert!( + !plugin.is_marker_anchored(), + "no camera trigger in this test" + ); + assert!( + plugin.armed_a0_blocker().is_none(), + "{:?}", + plugin.armed_a0_blocker() + ); + let armed = plugin.armed_a0().expect("an armed depth"); + assert!((armed.commanded_a - 0.5).abs() < 1e-9); + assert!((armed.frequency_hz - 1_000.0).abs() < 1e-6); + assert_eq!(armed.trials, 0, "no search happened, and it must say so"); + assert_eq!(armed.depth_source, DepthSource::Commanded); + assert!( + plugin.a0_locks.is_empty(), + "and nothing was written to the lock table" + ); + let _ = std::fs::remove_dir_all(&dir); } #[test] - fn the_rolling_response_is_normalised_over_the_roi_not_the_sensor() { - // `q_p` counts ROI-minus-masked pixels; the rolling half-period rate is - // plotted next to it and must agree. Normalising by the whole sensor - // under-reported S_p by the ROI/frame ratio *and* counted events from - // outside the ROI. - let mut plugin = StageAA1Plugin { - frame_width: 10, - frame_height: 10, - camera_markers_us: vec![0, 1_000, 2_000, 3_000], - ..StageAA1Plugin::default() - }; - plugin.host_roi = Some(RoiV1 { - x: 0, - y: 0, - width: 2, - height: 2, - }); - let event = |x: u16, y: u16, timestamp_us: u64| CameraEvent { - timestamp_us, - x, - y, - polarity: Polarity::On, - }; - // Two ON events inside the 2x2 ROI, five well outside it, all inside - // the trailing half period the status readout samples. - plugin.camera_events.push(event(0, 0, 2_800)); - plugin.camera_events.push(event(1, 1, 2_850)); - for x in 5..10_u16 { - plugin.camera_events.push(event(x, 9, 2_900)); - } + fn the_commanded_ladder_records_every_point_without_a_lock_or_a_trigger() { + // The whole simplification in one test: no photodiode `a`, no camera + // markers, no Find a₀ — the ladder still leases once, walks the + // frequencies confirming each against the modulation owner, and records + // a point at a₀ at every one of them. + let dir = temp_folder("commanded-ladder"); + let mut plugin = plugin_commanded_a0(&dir); + plugin.min_f = 10.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 3; + plugin.freq_order = FreqOrder::Ascending; + plugin.duration_s = 1; - let (on_rate, _) = plugin.latest_rolling().expect("rolling value"); + let mut sink = ControlSink::default(); + plugin + .set_setting("start_freq_sweep", json!(true)) + .expect("press"); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); assert!( - (on_rate - 0.5).abs() < 1e-9, - "expected 2 ROI events over 4 valid pixels, got {on_rate}" + plugin.freq_sweep.is_some(), + "the ladder must start with no trigger: {}", + plugin.message ); - } - #[test] - fn the_fold_cache_tracks_its_inputs() { - let mut plugin = plugin_with_markers(); - for cycle in 0..8 { - plugin.camera_events.push(on(cycle * 1_000 + 200)); + let mut recorded = Vec::new(); + for _ in 0..400 { + let Some((phase, target_hz, lease_req, freq_req, granted, applied)) = + plugin.freq_sweep.as_ref().map(|sweep| { + ( + sweep.phase, + sweep.frequency_hz(), + sweep.lease_req, + sweep.freq_req, + sweep.lease_granted, + sweep.freq_applied, + ) + }) + else { + break; + }; + let mut replies = Vec::new(); + match phase { + FreqSweepPhase::AcquiringLease if !granted => replies.push(accepted(lease_req)), + FreqSweepPhase::SettingFrequency if !applied && freq_req != 0 => { + // The owner acknowledges the new frequency. In commanded + // mode that ack — not the camera trigger — is what confirms + // the point, so nothing here ever writes a marker. + plugin.modulation = Some(ModulationStateV1 { + acknowledged: Some(acknowledged_sine(target_hz)), + ..commanded_modulation(2, 0.5) + }); + replies.push(accepted(freq_req)); + } + FreqSweepPhase::Locking => { + panic!("the commanded ladder must never enter the search phase") + } + FreqSweepPhase::Recording => { + if let Some(sweep) = plugin.sweep.as_ref() { + if !sweep.depth_applied && sweep.depth_req != 0 { + replies.push(accepted(sweep.depth_req)); + } + } + // Short-circuit the recording coordinator once the point has + // started: this test is about the ladder, and the + // coordinator has tests of its own. + if plugin.recording.is_active() + && plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.point_started) + { + plugin.recording = Recording::idle(); + plugin.recording_completed_ok = true; + recorded.push(target_hz); + } + } + _ => {} + } + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + &mut sink, + ); } - let first = plugin.current_fold().expect("fold"); - // Repeated calls within a repaint must be identical, not merely equal - // to a fresh recomputation. - assert_eq!(plugin.current_fold().as_ref(), Some(&first)); - assert_eq!(plugin.compute_fold().as_ref(), Some(&first)); - - // ...and adding an event inside the marker span must invalidate it. - plugin.camera_events.push(on(2_500)); - plugin.camera_events.sort_by_key(|event| event.timestamp_us); - let second = plugin.current_fold().expect("fold"); - assert_eq!(second.events.len(), first.events.len() + 1); - // A changed ROI also invalidates, even at identical event counts. - plugin.host_roi = Some(RoiV1 { - x: 0, - y: 0, - width: 1, - height: 1, - }); - let third = plugin.current_fold().expect("fold"); - assert_eq!(third.events.len(), second.events.len()); - plugin.host_roi = Some(RoiV1 { - x: 5, - y: 0, - width: 1, - height: 1, - }); - let fourth = plugin.current_fold().expect("fold"); assert!( - fourth.events.is_empty(), - "ROI moved off the events but the cache served a stale fold" + plugin.freq_sweep.is_none(), + "the ladder must finish: {}", + plugin.message + ); + assert_eq!( + recorded.len(), + 3, + "every planned point records: {recorded:?}" ); - } - - #[test] - fn a_failed_pilot_freeze_clears_stale_windows() { - // `scan_measurement_folder` may have loaded windows from an earlier - // pilot for this measurement. If the freeze then fails, the sidecar - // must not record those as if they had come from this run. - let mut plugin = plugin_with_markers(); - plugin.pilot_windows = Some(( - PhaseWindow { - start: 0.0, - end: 0.2, - }, - PhaseWindow { - start: 0.5, - end: 0.7, - }, - )); - // No events => the fold carries no signal => the freeze cannot pick - // windows and must not leave the loaded ones in place. - assert!(plugin.camera_events.is_empty()); - plugin.freeze_pilot_windows(); assert!( - plugin.pilot_windows.is_none(), - "stale pilot windows survived a failed freeze" + plugin.message.contains("3/3 points recorded"), + "{}", + plugin.message ); - assert!(!plugin.windows_are_frozen()); + assert!( + plugin.a0_locks.is_empty(), + "and the lock table stays empty — nothing was searched for" + ); + let _ = std::fs::remove_dir_all(&dir); } #[test] - fn press_latch_distinguishes_clicks_baselines_and_advances() { - let mut latch = PressLatch::default(); - // Direct click on this instance: an edge, and the counter advances. - assert!(latch.accept(&json!(true))); - assert_eq!(latch.value(), json!(1)); - // `false` writes (legacy snapshots) are never edges. + fn the_nested_sweep_records_every_depth_at_every_frequency_on_one_lease() { + // The q_p(a, f) surface in one press: the outer ladder walks the + // frequencies, and each rung runs the *whole* inner depth sweep. No a₀ + // and no search are involved at any point. + let dir = temp_folder("nested-sweep"); + let mut plugin = plugin_commanded_a0(&dir); + plugin.min_f = 10.0; + plugin.max_f = 100.0; + plugin.freq_count = 2; + plugin.freq_order = FreqOrder::Ascending; + plugin.min_a = 0.5; + plugin.max_a = 1.5; + plugin.sweep_count = 3; + plugin.duration_s = 1; + + let mut sink = ControlSink::default(); + plugin + .set_setting("start_freq_depth_sweep", json!(true)) + .expect("press"); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert!( + plugin.freq_sweep.is_some(), + "the nested sweep must start: {}", + plugin.message + ); + + // (frequency, depth index) of every recording that actually started. + let mut recorded: Vec<(f64, usize)> = Vec::new(); + for _ in 0..600 { + let Some((phase, mode, target_hz, lease_req, freq_req, granted, applied)) = + plugin.freq_sweep.as_ref().map(|sweep| { + ( + sweep.phase, + sweep.mode, + sweep.frequency_hz(), + sweep.lease_req, + sweep.freq_req, + sweep.lease_granted, + sweep.freq_applied, + ) + }) + else { + break; + }; + assert_eq!(mode, FreqSweepMode::DepthSweep); + assert_ne!( + phase, + FreqSweepPhase::Locking, + "a depth sweep never needs an a₀ search" + ); + let mut replies = Vec::new(); + match phase { + FreqSweepPhase::AcquiringLease if !granted => replies.push(accepted(lease_req)), + FreqSweepPhase::SettingFrequency if !applied && freq_req != 0 => { + plugin.modulation = Some(ModulationStateV1 { + acknowledged: Some(acknowledged_sine(target_hz)), + ..commanded_modulation(2, 0.5) + }); + replies.push(accepted(freq_req)); + } + FreqSweepPhase::Recording => { + if let Some(sweep) = plugin.sweep.as_ref() { + let (depth_req, depth_applied, index, commanded) = ( + sweep.depth_req, + sweep.depth_applied, + sweep.index, + sweep.commanded_a(), + ); + if !depth_applied && depth_req != 0 { + // The owner applies the depth this point asked for, + // which is what the settle check then reads back. + plugin.modulation = Some(ModulationStateV1 { + acknowledged: Some(acknowledged_sine(target_hz)), + ..commanded_modulation(3, commanded) + }); + replies.push(accepted(depth_req)); + } + if plugin.recording.is_active() && sweep.point_started { + plugin.recording = Recording::idle(); + plugin.recording_completed_ok = true; + recorded.push((target_hz, index)); + } + } + } + _ => {} + } + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + &mut sink, + ); + } + + assert!( + plugin.freq_sweep.is_none(), + "the nested sweep must finish: {}", + plugin.message + ); + assert_eq!( + recorded.len(), + 6, + "2 frequencies × 3 depths: {recorded:?} — {}", + plugin.message + ); + // Every frequency saw its whole curve, in depth order. + assert_eq!( + recorded.iter().map(|(_, index)| *index).collect::>(), + vec![0, 1, 2, 0, 1, 2] + ); + assert!((recorded[0].0 - 10.0).abs() < 1e-6, "{recorded:?}"); + assert!((recorded[3].0 - 100.0).abs() < 1e-6, "{recorded:?}"); + assert!( + plugin + .message + .contains("2/2 frequencies × 3 depths recorded"), + "{}", + plugin.message + ); + + // One lease for the whole block: the inner sweeps inherit it, so the + // operator's drive cannot move between rungs. + let leases = sink + .services + .iter() + .filter_map(|request| { + serde_json::from_value::(request.payload.clone()).ok() + }) + .filter(|envelope| matches!(envelope.command, ModulationCommandV1::AcquireLease { .. })) + .count(); + assert_eq!( + leases, 1, + "exactly one lease acquisition for the whole block" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_nested_sweep_point_is_named_by_frequency_and_depth() { + // `_p03` alone repeats at every rung, so the surface would collide + // inside one measurement id. + let dir = temp_folder("nested-name"); + let mut plugin = plugin_commanded_a0(&dir); + plugin.freq_sweep = Some(FreqSweep { + phase: FreqSweepPhase::Recording, + mode: FreqSweepMode::DepthSweep, + points: vec![FreqSweepPoint { + frequency_hz: 50.0, + is_reference: false, + }], + index: 0, + lease_id: LeaseId::new("nested"), + lease_granted: true, + lease_req: 0, + freq_req: 0, + freq_applied: true, + confirm_deadline_ms: 0, + skip_reason: None, + failed: Vec::new(), + recorded: 0, + order: FreqOrder::Ascending, + seed: 1, + last_activity_ms: 0, + stop_requested: false, + }); + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + kind: SweepKind::Amplitude, + points: vec![ + SweepPoint { + commanded_a: 1.0, + expected_a: 1.0, + }; + 3 + ], + lock: None, + index: 2, + lease_id: LeaseId::new("nested"), + lease_granted: true, + lease_req: 0, + owns_lease: false, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: false, + completed_ok: false, + last_activity_ms: 0, + stop_requested: false, + }); + + let mut sink = ControlSink::default(); + plugin.begin_recording(&mut sink, RecRole::Normal); + let stem = plugin.recording.stem.clone(); + assert!(stem.ends_with("_f50Hz_p03"), "stem: {stem}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn every_run_records_the_bench_conditions_the_sensor_measured() { + let mut plugin = plugin_with_markers(); + plugin.sensor = Some(SensorMonitoringV1 { + pixel_dead_time_us: Some(102.5), + illumination_lux: Some(742.0), + temperature_c: Some(41.25), + bias_codes: Some(augur_plugin_api::SensorBiasReadbackV1 { + current: augur_plugin_api::SensorBiasCodesV1 { + diff_on: 115, + diff_off: 52, + fo: 55, + hpf: 0, + refr: 20, + }, + factory_default: augur_plugin_api::SensorBiasCodesV1::default(), + }), + age_s: 0.25, + }); + // Frozen at start: the die warms and the room lights move, so what + // belongs to a run is what held when it began. + plugin.sensor_at_start = plugin.sensor; + plugin.sensor = Some(SensorMonitoringV1 { + temperature_c: Some(99.0), + ..plugin.sensor.expect("set above") + }); + + let meta = plugin.recording_metadata(); + assert_eq!( + meta.get("sensor_temperature_c").map(String::as_str), + Some("41.25"), + "the start snapshot wins over the drifted live one" + ); + assert_eq!( + meta.get("sensor_pixel_dead_time_us").map(String::as_str), + Some("102.500") + ); + assert_eq!( + meta.get("sensor_illumination_lux").map(String::as_str), + Some("742.000") + ); + assert_eq!( + meta.get("sensor_reading_age_s").map(String::as_str), + Some("0.250") + ); + + plugin.recording.id = "A1-sensor".into(); + plugin.recording.stem = "A1-sensor_20260731-000000".into(); + plugin.recording.folder = std::env::temp_dir().display().to_string(); + plugin.recording.duration_s = 5; + let doc = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&doc).expect("read sidecar"); + assert!(text.contains("[sensor]"), "{text}"); + assert!(text.contains("temperature_c = 41.25"), "{text}"); + assert!(text.contains("pixel_dead_time_us = 102.5"), "{text}"); + assert!(text.contains("illumination_lux = 742.0"), "{text}"); + assert!(text.contains("bias_refr = 20"), "{text}"); + let _ = std::fs::remove_file(&doc); + } + + #[test] + fn a_quantity_the_sensor_cannot_report_is_absent_rather_than_zero() { + // Replay, decoded imports and cameras without a monitoring block have + // no sensor to ask. A 0 °C die or 0 lx scene would be read downstream + // as a measurement. + let mut plugin = plugin_with_markers(); + assert!(plugin.recorded_sensor().is_none()); + let meta = plugin.recording_metadata(); + assert!(!meta.contains_key("sensor_temperature_c")); + assert!(!meta.contains_key("sensor_illumination_lux")); + + // A sensor that reports only some of the three is equally honest. + plugin.sensor_at_start = Some(SensorMonitoringV1 { + temperature_c: Some(38.0), + age_s: 0.1, + ..SensorMonitoringV1::default() + }); + let meta = plugin.recording_metadata(); + assert_eq!( + meta.get("sensor_temperature_c").map(String::as_str), + Some("38.00") + ); + assert!(!meta.contains_key("sensor_illumination_lux")); + assert!(!meta.contains_key("sensor_pixel_dead_time_us")); + } + + #[test] + fn a_run_records_which_source_its_depth_came_from() { + let mut plugin = plugin_with_markers(); + plugin.photodiode = None; + plugin.modulation = Some(commanded_modulation(1, 0.75)); + plugin.depth_source = DepthSource::Commanded; + + let meta = plugin.recording_metadata(); + assert_eq!( + meta.get("depth_a_source").map(String::as_str), + Some("modulation_commanded") + ); + assert_eq!(meta.get("depth_a").map(String::as_str), Some("0.750000")); + assert!( + !meta.contains_key("measured_a"), + "`measured_a` names a measurement, and there was none" + ); + + plugin.depth_source = DepthSource::Photodiode; + plugin.photodiode = Some(photodiode_measuring(1, 0.42)); + let meta = plugin.recording_metadata(); + assert_eq!( + meta.get("depth_a_source").map(String::as_str), + Some("photodiode_measured") + ); + assert_eq!(meta.get("measured_a").map(String::as_str), Some("0.420000")); + } + + fn pd_reply(request_id: u64, receipt: Option) -> PluginServiceReply { + let response = PhotodiodeResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: RequestId(request_id), + owner_instance: OwnerInstanceId::new("pd-test"), + run_id: None, + requested_revision: None, + acknowledged_revision: None, + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(now_unix_ms()), + error: None, + }, + receipt, + }; + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Accepted { + payload: serde_json::to_value(response).expect("response"), + }, + } + } + + /// A photodiode summary that passes the pre-flight: connected, unleased, + /// and with somewhere to put the PDQ. + fn ready_photodiode() -> PhotodiodeSummaryV1 { + PhotodiodeSummaryV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("pd-test"), + service_revision: 1, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: None, + }, + lease: None, + active_run_id: None, + requested_revision: None, + acknowledged_revision: None, + stream: PhotodiodeStreamV1 { + stream_epoch: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + latest_adc_code: Some(1_000), + integrity: StreamIntegrityV1::default(), + level: None, + }, + data_dir: Some("/pd".into()), + active_recording: None, + last_finalized_recording: None, + optical_summary: None, + optical_unavailable: None, + synchronization: SynchronizationV1::Unsynced { + reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 60_000, + }, + } + } + + fn on(timestamp_us: u64) -> CameraEvent { + CameraEvent { + timestamp_us, + x: 0, + y: 0, + polarity: Polarity::On, + } + } + + fn fresh_photodiode_summary() -> PhotodiodeSummaryV1 { + PhotodiodeSummaryV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("pd-test"), + service_revision: 1, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: Some("test".into()), + }, + lease: None, + active_run_id: None, + requested_revision: None, + acknowledged_revision: None, + stream: PhotodiodeStreamV1 { + stream_epoch: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + latest_adc_code: Some(1_000), + integrity: StreamIntegrityV1::default(), + level: None, + }, + active_recording: None, + last_finalized_recording: None, + data_dir: Some(std::env::temp_dir().display().to_string()), + optical_summary: Some(PhotodiodeOpticalSummaryV1 { + run_id: RunId::from("test-run"), + calibration: PhotodiodeCalibrationV1 { + adc_calibration_id: "adc-test".into(), + dark_id: "dark-test".into(), + anchor_id: "itot-test".into(), + dark_volts: 0.05, + total_power_volts: 3.0, + }, + measured_log_contrast: 1.0, + log_contrast_stddev: None, + excitation_min_volts: 0.8, + excitation_max_volts: 0.8 * std::f64::consts::E, + excitation_headroom_volts: 0.8, + low_clip_fraction: 0.0, + high_clip_fraction: 0.0, + measured_frequency_hz: Some(1_000.0), + fundamental_phase_rad: None, + total_harmonic_distortion: None, + window_seconds: Some(0.008), + covered_cycles: Some(8.0), + }), + optical_unavailable: None, + synchronization: SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 60_000, + }, + } + } + + /// A plugin whose period comes from marker spacing (no fallback frequency). + fn plugin_with_markers() -> StageAA1Plugin { + StageAA1Plugin { + // 10 x 1 sensor, no host ROI => valid_pixel_count() == 10. + frame_width: 10, + frame_height: 1, + camera_markers_us: vec![0, 1_000, 2_000, 3_000], + photodiode: Some(fresh_photodiode_summary()), + ..StageAA1Plugin::default() + } + } + + #[test] + fn period_comes_from_the_trigger_marker_spacing() { + let plugin = plugin_with_markers(); + let period = plugin.period_us().expect("measured period"); + assert!((period - 1_000.0).abs() < 1e-6, "period={period}"); + assert_eq!(plugin.frequency_source(), "trigger"); + } + + #[test] + fn no_markers_and_no_modulation_yields_no_period() { + let plugin = StageAA1Plugin::default(); + assert!(plugin.period_us().is_none()); + assert!(plugin.rolling_dataset().lines[0].points.is_empty()); + } + + #[test] + fn external_triggers_anchor_the_fold() { + let mut plugin = plugin_with_markers(); + for cycle in 0..3 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + assert!(plugin.is_marker_anchored()); + let fold = plugin.current_fold().expect("marker fold"); + assert_eq!(fold.validation.cycle_count, 3); + assert!((fold.events[0].phase - 0.2).abs() < 1e-9); + } + + #[test] + fn rolling_dataset_keeps_on_and_off_separate() { + let mut plugin = plugin_with_markers(); + for cycle in 0..3 { + let base = cycle * 1_000; + plugin.camera_events.push(on(base + 100)); + plugin.camera_events.push(CameraEvent { + polarity: Polarity::Off, + ..on(base + 600) + }); + } + let rolling = plugin.rolling_dataset(); + assert_eq!(rolling.lines.len(), 2); + assert_eq!(rolling.lines[0].name, "ON"); + assert!(rolling.lines[0].points.len() >= 2); + } + + #[test] + fn response_curve_auto_windows_without_a_pilot_and_refuses_without_a() { + let mut plugin = plugin_with_markers(); + plugin.frame_width = 8; + plugin.frame_height = 1; + for cycle in 0..20 { + let base = cycle * 1_000; + for x in 0..4 { + plugin.camera_events.push(CameraEvent { + timestamp_us: base + 200, + x, + y: 0, + polarity: Polarity::On, + }); + plugin.camera_events.push(CameraEvent { + timestamp_us: base + 700, + x, + y: 0, + polarity: Polarity::Off, + }); + } + } + plugin.camera_markers_us = (0..=20).map(|c| c * 1_000).collect(); + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 4, + height: 1, + }); + + // Windows come straight from the fold — no pilot capture needed. + let (q_on, q_off, _, valid) = plugin.current_response().expect("response"); + assert_eq!(valid, 4); + assert!(q_on > 0.9 && q_off > 0.9, "q_on={q_on} q_off={q_off}"); + // Recording a point is still refused without a photodiode-measured a. + plugin.photodiode = None; + assert!(plugin.depth_a().is_none()); + assert!(plugin.record_response_point().is_err()); + } + + #[test] + fn the_rolling_response_is_normalised_over_the_roi_not_the_sensor() { + // `q_p` counts ROI-minus-masked pixels; the rolling half-period rate is + // plotted next to it and must agree. Normalising by the whole sensor + // under-reported S_p by the ROI/frame ratio *and* counted events from + // outside the ROI. + let mut plugin = StageAA1Plugin { + frame_width: 10, + frame_height: 10, + camera_markers_us: vec![0, 1_000, 2_000, 3_000], + ..StageAA1Plugin::default() + }; + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 2, + height: 2, + }); + let event = |x: u16, y: u16, timestamp_us: u64| CameraEvent { + timestamp_us, + x, + y, + polarity: Polarity::On, + }; + // Two ON events inside the 2x2 ROI, five well outside it, all inside + // the trailing half period the status readout samples. + plugin.camera_events.push(event(0, 0, 2_800)); + plugin.camera_events.push(event(1, 1, 2_850)); + for x in 5..10_u16 { + plugin.camera_events.push(event(x, 9, 2_900)); + } + + let (on_rate, _) = plugin.latest_rolling().expect("rolling value"); + assert!( + (on_rate - 0.5).abs() < 1e-9, + "expected 2 ROI events over 4 valid pixels, got {on_rate}" + ); + } + + #[test] + fn the_fold_cache_tracks_its_inputs() { + let mut plugin = plugin_with_markers(); + for cycle in 0..8 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + let first = plugin.current_fold().expect("fold"); + // Repeated calls within a repaint must be identical, not merely equal + // to a fresh recomputation. + assert_eq!(plugin.current_fold().as_ref(), Some(&first)); + assert_eq!(plugin.compute_fold().as_ref(), Some(&first)); + + // ...and adding an event inside the marker span must invalidate it. + plugin.camera_events.push(on(2_500)); + plugin.camera_events.sort_by_key(|event| event.timestamp_us); + let second = plugin.current_fold().expect("fold"); + assert_eq!(second.events.len(), first.events.len() + 1); + + // A changed ROI also invalidates, even at identical event counts. + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 1, + height: 1, + }); + let third = plugin.current_fold().expect("fold"); + assert_eq!(third.events.len(), second.events.len()); + plugin.host_roi = Some(RoiV1 { + x: 5, + y: 0, + width: 1, + height: 1, + }); + let fourth = plugin.current_fold().expect("fold"); + assert!( + fourth.events.is_empty(), + "ROI moved off the events but the cache served a stale fold" + ); + } + + #[test] + fn a_failed_pilot_freeze_clears_stale_windows() { + // `scan_measurement_folder` may have loaded windows from an earlier + // pilot for this measurement. If the freeze then fails, the sidecar + // must not record those as if they had come from this run. + let mut plugin = plugin_with_markers(); + plugin.pilot_windows = Some(( + PhaseWindow { + start: 0.0, + end: 0.2, + }, + PhaseWindow { + start: 0.5, + end: 0.7, + }, + )); + // No events => the fold carries no signal => the freeze cannot pick + // windows and must not leave the loaded ones in place. + assert!(plugin.camera_events.is_empty()); + plugin.freeze_pilot_windows(); + assert!( + plugin.pilot_windows.is_none(), + "stale pilot windows survived a failed freeze" + ); + assert!(!plugin.windows_are_frozen()); + } + + #[test] + fn press_latch_distinguishes_clicks_baselines_and_advances() { + let mut latch = PressLatch::default(); + // Direct click on this instance: an edge, and the counter advances. + assert!(latch.accept(&json!(true))); + assert_eq!(latch.value(), json!(1)); + // `false` writes (legacy snapshots) are never edges. assert!(!latch.accept(&json!(false))); // A fresh instance adopts the first forwarded counter silently… @@ -6262,7 +8525,6 @@ mod tests { let mut plugin = StageAA1Plugin { output_folder: folder.display().to_string(), measurement_id: "A1-row".into(), - flux_point_id: "flux-row-1".into(), photodiode: Some(fresh_photodiode_summary()), duration_s: 1, pending_role: Some(RecRole::Normal), @@ -6494,6 +8756,7 @@ mod tests { settled_since_ms: None, settle_deadline_ms: 0, point_started: true, + completed_ok: false, last_activity_ms: 0, stop_requested: false, }); @@ -6753,7 +9016,7 @@ mod tests { plugin.freq_count = 2; plugin.freq_order = FreqOrder::Ascending; photodiode_window(&mut plugin, 0.02); - plugin.freq_sweep_pending = true; + plugin.freq_sweep_pending = Some(FreqSweepMode::A0Point); let mut sink = ControlSink::default(); let recorded = run_freq_sweep_to_completion(&mut plugin, &mut sink, 0.6, 4_000); @@ -6819,7 +9082,7 @@ mod tests { plugin.freq_count = 2; plugin.freq_order = FreqOrder::Ascending; photodiode_window(&mut plugin, 0.02); - plugin.freq_sweep_pending = true; + plugin.freq_sweep_pending = Some(FreqSweepMode::A0Point); let mut sink = ControlSink::default(); control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); @@ -6929,7 +9192,7 @@ mod tests { optical.window_seconds = Some(1.0); // 0.1 cycles at 0.1 Hz } } - plugin.freq_sweep_pending = true; + plugin.freq_sweep_pending = Some(FreqSweepMode::A0Point); let mut sink = ControlSink::default(); control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); @@ -6944,394 +9207,1012 @@ mod tests { } #[test] - fn changing_frequency_drops_the_previous_period_s_markers_and_windows() { - // The measured period is the mean marker spacing, so markers from the - // old drive would confirm the new frequency against a mixture. Pilot - // windows are frozen at a phase of the old period and do not transfer. - let folder = temp_folder("fflush"); - let mut plugin = plugin_locking(1.0, &folder); - plugin.pilot_windows = Some(( - PhaseWindow { - start: 0.0, - end: 0.2, - }, - PhaseWindow { - start: 0.5, - end: 0.7, + fn changing_frequency_drops_the_previous_period_s_markers_and_windows() { + // The measured period is the mean marker spacing, so markers from the + // old drive would confirm the new frequency against a mixture. Pilot + // windows are frozen at a phase of the old period and do not transfer. + let folder = temp_folder("fflush"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.pilot_windows = Some(( + PhaseWindow { + start: 0.0, + end: 0.2, + }, + PhaseWindow { + start: 0.5, + end: 0.7, + }, + )); + plugin.freq_sweep = Some(FreqSweep { + phase: FreqSweepPhase::AcquiringLease, + mode: FreqSweepMode::A0Point, + points: vec![FreqSweepPoint { + frequency_hz: 50.0, + is_reference: false, + }], + index: 0, + lease_id: LeaseId::new("a1-fsweep-test"), + lease_granted: true, + lease_req: 0, + freq_req: 0, + freq_applied: false, + confirm_deadline_ms: 0, + skip_reason: None, + failed: Vec::new(), + recorded: 0, + order: FreqOrder::Ascending, + seed: 1, + last_activity_ms: now_unix_ms(), + stop_requested: false, + }); + let mut sink = ControlSink::default(); + plugin.send_freq_sweep_frequency(&mut sink); + + assert!(plugin.camera_markers_us.is_empty()); + assert!(plugin.camera_events.is_empty()); + assert!( + plugin.pilot_windows.is_none(), + "windows frozen at another period must not carry over" + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_refuses_a_photodiode_window_shorter_than_one_cycle() { + // `a` is peak-to-peak. Under one cycle the photodiode under-reports it, + // and the lock divides by it — so it would inflate the drive until it + // railed. Refuse before touching the drive, and say what to change. + let folder = temp_folder("subcycle"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + // 1 kHz markers give the plugin its frequency; make the estimator + // window 0.4 ms, i.e. 0.4 of a cycle. + if let Some(summary) = plugin.photodiode.as_mut() { + if let Some(optical) = summary.optical_summary.as_mut() { + optical.window_seconds = Some(0.000_4); + } + } + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert!(plugin.a0_lock.is_none(), "the lock must not start"); + assert!(sink.services.is_empty(), "no lease may be requested"); + assert!( + plugin.message.contains("0.40 cycles") && plugin.message.contains("cache length"), + "message: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_gates_quote_the_owners_reason_for_withholding_a() { + // The old refusal named the anchor and the cable whatever the real cause + // was, which sent the operator to re-check a calibration that was + // already fine. Whatever gate the owner closed has to reach the panel. + let folder = temp_folder("blocker"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + if let Some(summary) = plugin.photodiode.as_mut() { + summary.optical_summary = None; + summary.optical_unavailable = Some("ADC clipping: 307‰ low / 0‰ high".into()); + } + + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert!(plugin.a0_lock.is_none(), "the lock must not start"); + assert!( + plugin.message.contains("307‰ low"), + "the a₀ refusal must quote the owner: {}", + plugin.message + ); + // And without pressing anything: the resting panel says the same thing. + let status = plugin + .status_entries() + .into_iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) => Some(text), + _ => None, + }) + .collect::>() + .join("\n"); + assert!( + status.contains("307‰ low"), + "the status panel must name the gate: {status}" + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn the_status_panel_names_live_analysis_when_it_is_off() { + // "0 events, free-running" describes the toggle, not the bench, and the + // frequency sweep refuses on the marker count it produces. + let folder = temp_folder("liveoff"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.live = false; + plugin.camera_markers_us.clear(); + // Clear the gates that sit *before* the marker check, so the refusal + // under test is the marker one and not the optical-window one. + plugin.min_f = 1.0; + plugin.max_f = 1.0; + photodiode_window(&mut plugin, 4.0); + + let status = plugin + .status_entries() + .into_iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) => Some(text), + _ => None, + }) + .collect::>() + .join("\n"); + assert!(status.contains("Live analysis is OFF"), "status: {status}"); + + plugin.freq_sweep_pending = Some(FreqSweepMode::A0Point); + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert!(plugin.freq_sweep.is_none(), "the sweep must not start"); + assert!( + plugin.message.contains("Live analysis"), + "the sweep refusal must name the toggle: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_refuses_to_lock_onto_an_unsettled_operating_point() { + // Readings that walk across the target are not a lock: the next action + // would record at wherever the drive drifted to, not at a₀. + let folder = temp_folder("unsettled"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + let mut revision = 1; + let mut drift = 0.30; + for _ in 0..64 { + if plugin.a0_lock.is_none() { + break; + } + let (lease_req, depth_req, granted, applied) = { + let lock = plugin.a0_lock.as_ref().expect("lock"); + ( + lock.lease_req, + lock.depth_req, + lock.lease_granted, + lock.depth_applied, + ) + }; + let mut replies = Vec::new(); + if !granted { + replies.push(accepted(lease_req)); + } else if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + drift += 0.20; // 0.50, 0.70, 0.90 — straddling a₀ = 0.50 + plugin.photodiode = Some(photodiode_measuring(revision, drift)); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + &mut sink, + ); + } + + assert!(plugin.a0_lock.is_none(), "the lock must end"); + assert!( + plugin.message.contains("not settled"), + "message: {}", + plugin.message + ); + // Nothing is stored, so nothing can arm a recording. + assert!(plugin.a0_locks.is_empty()); + assert!(plugin.armed_lock().is_none()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_reports_an_unreachable_depth_instead_of_arming_a_recording() { + // The bench delivers 5 % of the commanded depth: a₀ = 0.5 would need a + // commanded depth far beyond what the owner accepts. + let folder = temp_folder("unreachable"); + let mut plugin = plugin_locking(0.05, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + + assert!(run_lock_to_completion(&mut plugin, &mut sink, 0.05, 256) < 256); + let lock = plugin.a0_locks.first().expect("the attempt is recorded"); + assert!(!lock.converged); + assert!((lock.commanded_a - COMMANDED_A_MAX).abs() < 1e-9); + assert!( + plugin.message.contains("drivable limit") + || plugin.message.contains("did not converge"), + "message: {}", + plugin.message + ); + // A non-converged lock must never arm an event-count recording. + assert!(plugin.armed_lock().is_none()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_surfaces_a_drive_rejection_verbatim() { + let folder = temp_folder("reject"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + // Tick 1: begin and lease. + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = plugin.a0_lock.as_ref().expect("lock").lease_req; + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![accepted(lease_req)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let depth_req = plugin.a0_lock.as_ref().expect("lock").depth_req; + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![rejected( + depth_req, + "calibrated optical peak u = 1.2 exceeds the lobe ceiling", + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert!(plugin.a0_lock.is_none(), "the lock must not keep trying"); + assert!( + plugin.message.contains("lobe ceiling"), + "message: {}", + plugin.message + ); + assert!(plugin.a0_locks.is_empty(), "a rejected lock stores nothing"); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn event_count_point_commands_the_locked_depth_not_a0() { + let folder = temp_folder("ecpoint"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.a0_target = 0.5; + plugin.a0_locks.push(A0LockPoint { + frequency_hz: 1_000.0, + target_a: 0.5, + commanded_a: 0.8333, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: Some(0.0), + high_clip_fraction: Some(0.0), + depth_source: DepthSource::Photodiode, + }); + let mut sink = ControlSink::default(); + + plugin + .set_setting("record_a0_point", json!(true)) + .expect("press"); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let sweep = plugin.sweep.as_ref().expect("event-count sweep"); + assert_eq!(sweep.kind, SweepKind::EventCount); + assert_eq!(sweep.total(), 1); + let lease_req = sweep.lease_req; + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![accepted(lease_req)], + ..PluginControlInbox::default() }, - )); - plugin.freq_sweep = Some(FreqSweep { - phase: FreqSweepPhase::AcquiringLease, - points: vec![FreqSweepPoint { - frequency_hz: 50.0, - is_reference: false, + &mut sink, + ); + // The drive is commanded to the locked depth, *not* to a₀ itself. + let commanded = last_commanded_depth(&sink).expect("commanded depth"); + assert!((commanded - 0.833).abs() < 0.002, "commanded {commanded}"); + assert!((plugin.sweep.as_ref().expect("sweep").target_a() - 0.5).abs() < 1e-9); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn event_count_stems_and_sidecars_carry_the_frequency_and_the_lock() { + let folder = temp_folder("ecstem"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.measurement_id = "A1-ecrow".into(); + plugin.frame_width = 4; + plugin.frame_height = 1; + let lock = A0LockPoint { + frequency_hz: 1_000.0, + target_a: 0.5, + commanded_a: 0.8333, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: 1_784_764_800_000, + low_clip_fraction: Some(0.0), + high_clip_fraction: Some(0.0), + depth_source: DepthSource::Photodiode, + }; + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + kind: SweepKind::EventCount, + points: vec![SweepPoint { + commanded_a: 0.8333, + expected_a: 0.5, }], + lock: Some(lock), index: 0, - lease_id: LeaseId::new("a1-fsweep-test"), + lease_id: LeaseId::new("a1-sweep-test"), lease_granted: true, lease_req: 0, - freq_req: 0, - freq_applied: false, - confirm_deadline_ms: 0, - skip_reason: None, - failed: Vec::new(), - recorded: 0, - order: FreqOrder::Ascending, - seed: 1, - last_activity_ms: now_unix_ms(), + owns_lease: true, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: true, + completed_ok: false, + last_activity_ms: 0, stop_requested: false, }); + + // The stem carries the frequency instead of a sweep-point index. let mut sink = ControlSink::default(); - plugin.send_freq_sweep_frequency(&mut sink); + plugin.begin_recording(&mut sink, RecRole::EventCount); + let stem = plugin.recording.stem.clone(); + assert!(stem.ends_with("_ec_f1000Hz"), "stem: {stem}"); + + plugin.recording.duration_s = 5; + plugin.recording.start_unix_ms = 1_784_764_800_000; + let path = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&path).expect("read sidecar"); + assert!(text.contains("role = \"event-count point\""), "{text}"); + assert!(text.contains("[a0_lock]"), "{text}"); + assert!(text.contains("target_a = 0.5"), "{text}"); + assert!(text.contains("commanded_a = 0.8333"), "{text}"); + assert!(text.contains("converged = true"), "{text}"); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn frequency_tags_are_file_safe() { + assert_eq!(frequency_tag(50.0), "f50Hz"); + assert_eq!(frequency_tag(0.5), "f0p5Hz"); + assert_eq!(frequency_tag(1_200.0), "f1200Hz"); + assert_eq!(frequency_tag(12.345), "f12p345Hz"); + assert_eq!(sanitize_stem(&frequency_tag(0.5)), frequency_tag(0.5)); + } + + #[test] + fn locks_are_one_per_frequency_and_round_trip_through_the_folder() { + let dir = std::env::temp_dir().join(format!("a1-a0-{}", now_unix_ms())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let folder = dir.display().to_string(); + + let mut plugin = StageAA1Plugin { + output_folder: folder.clone(), + ..StageAA1Plugin::default() + }; + let point = |hz: f64, commanded_a: f64| A0LockPoint { + frequency_hz: hz, + target_a: 0.5, + commanded_a, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: None, + high_clip_fraction: None, + depth_source: DepthSource::Photodiode, + }; + plugin.store_lock(point(1_000.0, 0.83)).expect("saved"); + plugin.store_lock(point(50.0, 0.52)).expect("saved"); + // Re-locking the same frequency replaces the row rather than appending. + plugin.store_lock(point(1_000.5, 0.86)).expect("saved"); + assert_eq!(plugin.a0_locks.len(), 2); + assert!( + (plugin.a0_locks[0].frequency_hz - 50.0).abs() < 1e-9, + "sorted by frequency" + ); + + let mut other = StageAA1Plugin { + output_folder: folder.clone(), + ..StageAA1Plugin::default() + }; + other.load_a0_locks(); + assert_eq!(other.a0_locks.len(), 2); + let reloaded = other.lock_for_frequency(1_000.0).expect("reloaded lock"); + assert!((reloaded.commanded_a - 0.86).abs() < 1e-9); + assert_eq!(other.a0_locks_dataset().columns.len(), 8); + // A table written before the depth-source setting existed loads as the + // photodiode-measured rows it was: the field defaults, it is not lost. + assert_eq!(reloaded.depth_source, DepthSource::Photodiode); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn settings_discontinuities_keep_the_response_curve() { + let mut plugin = StageAA1Plugin::default(); + plugin.response_points.push(ResponsePoint { + measured_a: 1.0, + q_on: 0.5, + q_off: 0.1, + cycles: 10, + valid_pixels: 4, + }); + plugin.on_discontinuity(PluginDiscontinuity::SettingsChanged); + assert_eq!(plugin.response_points.len(), 1); + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + assert!(plugin.response_points.is_empty()); + } + + #[test] + fn measurement_id_generation_is_file_safe_and_prefixed() { + let id = generate_measurement_id(); + assert!(id.starts_with("A1-")); + assert_eq!(sanitize_stem(&id), id); + assert_eq!(sanitize_stem("I_k 3 / f=10Hz"), "I_k_3_f_10Hz"); + } + + #[test] + fn compact_utc_formats_a_known_epoch() { + // 2026-07-23T00:00:00Z = 1_784_764_800 s; +3661 s = 01:01:01. + assert_eq!(format_compact_date(1_784_764_800), "20260723"); + assert_eq!(format_iso_utc(1_784_764_800), "2026-07-23T00:00:00Z"); + assert_eq!(format_compact_utc(1_784_764_800), "20260723-000000"); + assert_eq!( + format_iso_utc(1_784_764_800 + 3_661), + "2026-07-23T01:01:01Z" + ); + } + + /// Turning Live analysis off used to leave the last analysis window in + /// place — up to millions of events — and the control tick keeps folding + /// whatever is in the buffer. So the plots stayed slow after the switch was + /// off again, which is exactly what the operator reported. + #[test] + fn turning_live_analysis_off_releases_the_event_buffer() { + let mut plugin = plugin_with_markers(); + plugin.live = true; + plugin.camera_events = (0..50_000) + .map(|index| CameraEvent { + x: 0, + y: 0, + timestamp_us: index, + polarity: Polarity::On, + }) + .collect(); + plugin.camera_markers_us = (0..100).map(|index| index * 1_000).collect(); + // Prime the memoised fold so the stale one cannot survive either. + let _ = plugin.current_fold(); + + plugin.set_setting("live", json!(false)).expect("live off"); - assert!(plugin.camera_markers_us.is_empty()); assert!(plugin.camera_events.is_empty()); + assert!(plugin.camera_markers_us.is_empty()); assert!( - plugin.pilot_windows.is_none(), - "windows frozen at another period must not carry over" + plugin.camera_events.capacity() == 0, + "the buffer kept {} events' worth of capacity reserved", + plugin.camera_events.capacity() + ); + assert!( + plugin.fold_cache.borrow().is_none(), + "a stale fold survived" ); - let _ = std::fs::remove_dir_all(&folder); } + /// The Clear button and the off switch must leave the plugin in the same + /// state — they are the same operation. #[test] - fn a0_lock_refuses_a_photodiode_window_shorter_than_one_cycle() { - // `a` is peak-to-peak. Under one cycle the photodiode under-reports it, - // and the lock divides by it — so it would inflate the drive until it - // railed. Refuse before touching the drive, and say what to change. - let folder = temp_folder("subcycle"); - let mut plugin = plugin_locking(1.0, &folder); - plugin.a0_target = 0.5; - // 1 kHz markers give the plugin its frequency; make the estimator - // window 0.4 ms, i.e. 0.4 of a cycle. - if let Some(summary) = plugin.photodiode.as_mut() { - if let Some(optical) = summary.optical_summary.as_mut() { - optical.window_seconds = Some(0.000_4); - } + fn clearing_captured_events_releases_the_same_buffers() { + let mut plugin = plugin_with_markers(); + plugin.live = true; + plugin.camera_events = vec![CameraEvent { + x: 0, + y: 0, + timestamp_us: 1, + polarity: Polarity::On, + }]; + plugin.camera_markers_us = vec![0, 1_000]; + + plugin.set_setting("clear", json!(true)).expect("clear"); + + assert!(plugin.camera_events.is_empty()); + assert!(plugin.camera_markers_us.is_empty()); + assert!(plugin.fold_cache.borrow().is_none()); + } + + /// `settings_schema` is rendered by the UI mirror, which never runs + /// `process_control` — so every run lives on an instance the panel cannot + /// see. Gating a button on "is something running" therefore disables + /// nothing and lies to the next reader; the interlocks belong worker-side. + #[test] + fn buttons_are_not_gated_on_state_the_ui_mirror_cannot_see() { + let mut mirror = StageAA1Plugin { + output_folder: "/tmp/a1-mirror".into(), + ..StageAA1Plugin::default() + }; + mirror.set_runtime_role(PluginRuntimeRole::UiMirror); + + let enabled_of = |plugin: &StageAA1Plugin, key: &str| { + plugin + .settings_schema() + .sections + .iter() + .flat_map(|section| section.items.iter()) + .find(|item| item.key == key) + .and_then(|item| match item.kind { + SettingKind::Button { enabled } => Some(enabled), + _ => None, + }) + .unwrap_or_else(|| panic!("missing button {key}")) + }; + + let buttons = [ + "start_recording", + "start_sweep", + "start_freq_sweep", + "start_freq_depth_sweep", + "run_protocol", + ]; + let before: Vec = buttons.iter().map(|key| enabled_of(&mirror, key)).collect(); + assert!(before.iter().all(|enabled| *enabled), "{before:?}"); + + // Now make the *worker-side* state look busy. The mirror renders the + // same either way, because it never sees any of this. + mirror.recording.phase = RecPhase::StartingCamera; + let after: Vec = buttons.iter().map(|key| enabled_of(&mirror, key)).collect(); + assert_eq!(before, after, "a button was gated on worker-only state"); + + // Without an output folder they *are* disabled — that is mirrored + // state, so it is a legitimate gate. + mirror.output_folder = String::new(); + for key in buttons { + assert!(!enabled_of(&mirror, key), "{key} ignored the output folder"); } - plugin.a0_lock_pending = true; - let mut sink = ControlSink::default(); - control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + } - assert!(plugin.a0_lock.is_none(), "the lock must not start"); - assert!(sink.services.is_empty(), "no lease may be requested"); - assert!( - plugin.message.contains("0.40 cycles") && plugin.message.contains("cache length"), - "message: {}", - plugin.message - ); - let _ = std::fs::remove_dir_all(&folder); + /// The Sweep f button and the depth it holds have to be in the same place: + /// the button used to be in Record while `a₀` sat in a collapsed section. + #[test] + fn the_depth_sweep_f_holds_sits_beside_the_frequency_axis() { + let schema = StageAA1Plugin::default().settings_schema(); + let record = schema + .sections + .iter() + .find(|section| section.label == "Record") + .expect("a single Record section"); + for key in ["a0_target", "min_f", "max_f", "start_freq_sweep"] { + assert!( + record.items.iter().any(|item| item.key == key), + "{key} is not in the Record section" + ); + } } + /// The protocol's own duration must not be written into the panel setting: + /// the host re-applies the mirror's snapshot every pass, so it would be + /// reverted within the frame and the operator's value would flicker. #[test] - fn a0_lock_refuses_to_lock_onto_an_unsettled_operating_point() { - // Readings that walk across the target are not a lock: the next action - // would record at wherever the drive drifted to, not at a₀. - let folder = temp_folder("unsettled"); - let mut plugin = plugin_locking(1.0, &folder); - plugin.a0_target = 0.5; - plugin.a0_lock_pending = true; + fn a_protocol_duration_overrides_without_touching_the_panel_setting() { + let folder = temp_folder("protocol-duration-override"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + plugin.duration_s = 999; let mut sink = ControlSink::default(); - control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); - let mut revision = 1; - let mut drift = 0.30; - for _ in 0..64 { - if plugin.a0_lock.is_none() { - break; - } - let (lease_req, depth_req, granted, applied) = { - let lock = plugin.a0_lock.as_ref().expect("lock"); - ( - lock.lease_req, - lock.depth_req, - lock.lease_granted, - lock.depth_applied, + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = sink.services[0].request_id; + sink.services.clear(); + control_tick( + &mut plugin, + inbox_with(vec![accepted(lease_req)]), + &mut sink, + ); + let retargets: Vec = sink + .services + .iter() + .filter(|request| { + matches!( + modulation_command(request), + Some( + ModulationCommandV1::SetOperatingPoint { .. } + | ModulationCommandV1::SetDriveFrequency { .. } + | ModulationCommandV1::SetOpticalDepth { .. } + ) ) - }; - let mut replies = Vec::new(); - if !granted { - replies.push(accepted(lease_req)); - } else if !applied && depth_req != 0 { - replies.push(accepted(depth_req)); - } else { - revision += 1; - drift += 0.20; // 0.50, 0.70, 0.90 — straddling a₀ = 0.50 - plugin.photodiode = Some(photodiode_measuring(revision, drift)); - std::thread::sleep(std::time::Duration::from_millis(1)); - } - control_tick( - &mut plugin, - PluginControlInbox { - service_replies: replies, - ..PluginControlInbox::default() - }, - &mut sink, - ); - } + }) + .map(|request| request.request_id) + .collect(); + control_tick( + &mut plugin, + inbox_with(retargets.into_iter().map(accepted).collect()), + &mut sink, + ); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); - assert!(plugin.a0_lock.is_none(), "the lock must end"); - assert!( - plugin.message.contains("not settled"), - "message: {}", - plugin.message + assert_eq!( + plugin.recording.duration_s, 3, + "the protocol's duration lost" ); - // Nothing is stored, so nothing can arm a recording. - assert!(plugin.a0_locks.is_empty()); - assert!(plugin.armed_lock().is_none()); + assert_eq!( + plugin.duration_s, 999, + "the protocol overwrote the operator's own duration setting" + ); + // And it is consumed, so the next hand-driven recording is the panel's. + assert!(plugin.pending_duration_s.is_none()); + let _ = std::fs::remove_dir_all(&folder); } + /// A protocol file on disk, and a plugin ready to run it. + fn protocol_plugin(folder: &Path, body: &str) -> (StageAA1Plugin, PathBuf) { + std::fs::create_dir_all(folder).expect("protocol folder"); + let path = folder.join("protocol.toml"); + std::fs::write(&path, body).expect("write protocol"); + let mut plugin = plugin_with_markers(); + plugin.modulation = Some(connected_modulation()); + plugin.photodiode = Some(ready_photodiode()); + plugin.output_folder = folder.display().to_string(); + plugin.measurement_id = "A1-proto".into(); + plugin.protocol_path = path.display().to_string(); + plugin.protocol_pending = true; + (plugin, path) + } + + const TWO_POINT_PROTOCOL: &str = r#" +name = "two-point" + +[defaults] +duration_s = 3 +settle_s = 0.0 + +[[block]] +name = "pair" +mean_u = [0.4, 0.6] +frequency_hz = 25.0 +depth_a = 0.7 +"#; + + /// The reason a protocol exists rather than three nested button presses: + /// every point states its whole operating condition, so all three axes are + /// commanded at every point instead of being left wherever the last one + /// happened to leave them. `I_k` (ū) is the axis the buttons could not + /// sweep at all. #[test] - fn a0_lock_reports_an_unreachable_depth_instead_of_arming_a_recording() { - // The bench delivers 5 % of the commanded depth: a₀ = 0.5 would need a - // commanded depth far beyond what the owner accepts. - let folder = temp_folder("unreachable"); - let mut plugin = plugin_locking(0.05, &folder); - plugin.a0_target = 0.5; - plugin.a0_lock_pending = true; + fn a_protocol_commands_all_three_axes_at_every_point() { + let folder = temp_folder("protocol-axes"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); let mut sink = ControlSink::default(); - assert!(run_lock_to_completion(&mut plugin, &mut sink, 0.05, 256) < 256); - let lock = plugin.a0_locks.first().expect("the attempt is recorded"); - assert!(!lock.converged); - assert!((lock.commanded_a - COMMANDED_A_MAX).abs() < 1e-9); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = sink + .services + .iter() + .find_map(|request| match modulation_command(request) { + Some(ModulationCommandV1::AcquireLease { .. }) => Some(request.request_id), + _ => None, + }) + .expect("the protocol takes a modulation lease"); + + sink.services.clear(); + control_tick( + &mut plugin, + inbox_with(vec![accepted(lease_req)]), + &mut sink, + ); + + let commands: Vec = sink + .services + .iter() + .filter_map(modulation_command) + .collect(); assert!( - plugin.message.contains("drivable limit") - || plugin.message.contains("did not converge"), - "message: {}", - plugin.message + commands.iter().any(|command| matches!( + command, + ModulationCommandV1::SetOperatingPoint { mean_u_milli: 400 } + )), + "the I_k axis was not commanded: {commands:?}" ); - // A non-converged lock must never arm an event-count recording. - assert!(plugin.armed_lock().is_none()); + assert!( + commands.iter().any(|command| matches!( + command, + ModulationCommandV1::SetDriveFrequency { + frequency_millihz: 25_000 + } + )), + "the frequency axis was not commanded: {commands:?}" + ); + assert!( + commands.iter().any(|command| matches!( + command, + ModulationCommandV1::SetOpticalDepth { depth_a_milli: 700 } + )), + "the depth axis was not commanded: {commands:?}" + ); + let _ = std::fs::remove_dir_all(&folder); } + /// Recording before every axis has been acknowledged would file the run + /// under parameters the bench was not actually at. #[test] - fn a0_lock_surfaces_a_drive_rejection_verbatim() { - let folder = temp_folder("reject"); - let mut plugin = plugin_locking(1.0, &folder); - plugin.a0_lock_pending = true; + fn a_protocol_point_waits_for_all_three_retargets_before_recording() { + let folder = temp_folder("protocol-wait"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); let mut sink = ControlSink::default(); - // Tick 1: begin and lease. + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); - let lease_req = plugin.a0_lock.as_ref().expect("lock").lease_req; + let lease_req = sink.services[0].request_id; + sink.services.clear(); control_tick( &mut plugin, - PluginControlInbox { - service_replies: vec![accepted(lease_req)], - ..PluginControlInbox::default() - }, + inbox_with(vec![accepted(lease_req)]), &mut sink, ); - let depth_req = plugin.a0_lock.as_ref().expect("lock").depth_req; + + let retargets: Vec = sink + .services + .iter() + .filter(|request| { + matches!( + modulation_command(request), + Some( + ModulationCommandV1::SetOperatingPoint { .. } + | ModulationCommandV1::SetDriveFrequency { .. } + | ModulationCommandV1::SetOpticalDepth { .. } + ) + ) + }) + .map(|request| request.request_id) + .collect(); + assert_eq!(retargets.len(), 3); + + // Two of three applied: still not recording. + sink.services.clear(); control_tick( &mut plugin, - PluginControlInbox { - service_replies: vec![rejected( - depth_req, - "calibrated optical peak u = 1.2 exceeds the lobe ceiling", - )], - ..PluginControlInbox::default() - }, + inbox_with(vec![accepted(retargets[0]), accepted(retargets[1])]), &mut sink, ); - assert!(plugin.a0_lock.is_none(), "the lock must not keep trying"); assert!( - plugin.message.contains("lobe ceiling"), - "message: {}", + !plugin.recording.is_active(), + "recording started with a retarget still outstanding" + ); + + control_tick( + &mut plugin, + inbox_with(vec![accepted(retargets[2])]), + &mut sink, + ); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert!( + plugin.recording.is_active(), + "the point never started recording; message: {}", plugin.message ); - assert!(plugin.a0_locks.is_empty(), "a rejected lock stores nothing"); + let _ = std::fs::remove_dir_all(&folder); } + /// A refused point is the common case in a long survey — a `ū`/`a` pair + /// that runs off the top of the lobe. It must cost that point and carry the + /// owner's own wording, not abandon the rest of the night's work. #[test] - fn event_count_point_commands_the_locked_depth_not_a0() { - let folder = temp_folder("ecpoint"); - let mut plugin = plugin_locking(0.6, &folder); - plugin.a0_target = 0.5; - plugin.a0_locks.push(A0LockPoint { - frequency_hz: 1_000.0, - target_a: 0.5, - commanded_a: 0.8333, - measured_a: 0.5, - trials: 2, - converged: true, - locked_at_unix_ms: now_unix_ms(), - low_clip_fraction: Some(0.0), - high_clip_fraction: Some(0.0), - }); + fn a_refused_point_is_skipped_with_the_owners_reason_and_the_run_continues() { + let folder = temp_folder("protocol-skip"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); let mut sink = ControlSink::default(); - plugin - .set_setting("record_a0_point", json!(true)) - .expect("press"); control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); - let sweep = plugin.sweep.as_ref().expect("event-count sweep"); - assert_eq!(sweep.kind, SweepKind::EventCount); - assert_eq!(sweep.total(), 1); - let lease_req = sweep.lease_req; - + let lease_req = sink.services[0].request_id; + sink.services.clear(); control_tick( &mut plugin, - PluginControlInbox { - service_replies: vec![accepted(lease_req)], - ..PluginControlInbox::default() - }, + inbox_with(vec![accepted(lease_req)]), &mut sink, ); - // The drive is commanded to the locked depth, *not* to a₀ itself. - let commanded = last_commanded_depth(&sink).expect("commanded depth"); - assert!((commanded - 0.833).abs() < 0.002, "commanded {commanded}"); - assert!((plugin.sweep.as_ref().expect("sweep").target_a() - 0.5).abs() < 1e-9); - let _ = std::fs::remove_dir_all(&folder); - } + let first_retarget = sink + .services + .iter() + .find(|request| { + matches!( + modulation_command(request), + Some(ModulationCommandV1::SetOperatingPoint { .. }) + ) + }) + .expect("operating point retarget") + .request_id; - #[test] - fn event_count_stems_and_sidecars_carry_the_frequency_and_the_lock() { - let folder = temp_folder("ecstem"); - let mut plugin = plugin_locking(0.6, &folder); - plugin.measurement_id = "A1-ecrow".into(); - plugin.frame_width = 4; - plugin.frame_height = 1; - let lock = A0LockPoint { - frequency_hz: 1_000.0, - target_a: 0.5, - commanded_a: 0.8333, - measured_a: 0.5, - trials: 2, - converged: true, - locked_at_unix_ms: 1_784_764_800_000, - low_clip_fraction: Some(0.0), - high_clip_fraction: Some(0.0), - }; - plugin.sweep = Some(Sweep { - phase: SweepPhase::Recording, - kind: SweepKind::EventCount, - points: vec![SweepPoint { - commanded_a: 0.8333, - expected_a: 0.5, - }], - lock: Some(lock), - index: 0, - lease_id: LeaseId::new("a1-sweep-test"), - lease_granted: true, - lease_req: 0, - owns_lease: true, - depth_req: 0, - depth_applied: true, - settled_since_ms: None, - settle_deadline_ms: 0, - point_started: true, - last_activity_ms: 0, - stop_requested: false, - }); + sink.services.clear(); + control_tick( + &mut plugin, + inbox_with(vec![rejected( + first_retarget, + "operating point ū=0.400 rejected: peak exceeds the lobe ceiling", + )]), + &mut sink, + ); - // The stem carries the frequency instead of a sweep-point index. - let mut sink = ControlSink::default(); - plugin.begin_recording(&mut sink, RecRole::EventCount); - let stem = plugin.recording.stem.clone(); - assert!(stem.ends_with("_ec_f1000Hz"), "stem: {stem}"); + let run = plugin + .protocol + .as_ref() + .expect("the run abandoned the survey"); + assert_eq!(run.index, 1, "the run did not move on to the second point"); + let (index, reason) = run.failed.last().expect("the skip was recorded"); + assert_eq!(*index, 0); + assert!( + reason.contains("lobe ceiling"), + "the owner's reason was replaced: {reason}" + ); + // And it is on the status pane, because the per-point message has + // already been overwritten by the next point. + let status = plugin + .status_entries() + .iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) => Some(text.clone()), + _ => None, + }) + .collect::>() + .join(" | "); + assert!(status.contains("lobe ceiling"), "{status}"); - plugin.recording.duration_s = 5; - plugin.recording.start_unix_ms = 1_784_764_800_000; - let path = plugin.write_sidecar().expect("sidecar path"); - let text = std::fs::read_to_string(&path).expect("read sidecar"); - assert!(text.contains("role = \"event-count point\""), "{text}"); - assert!(text.contains("[a0_lock]"), "{text}"); - assert!(text.contains("target_a = 0.5"), "{text}"); - assert!(text.contains("commanded_a = 0.8333"), "{text}"); - assert!(text.contains("converged = true"), "{text}"); let _ = std::fs::remove_dir_all(&folder); } + /// A protocol whose file is wrong must say so on the button press, before + /// the drive has moved — the whole point is that it runs unattended. #[test] - fn frequency_tags_are_file_safe() { - assert_eq!(frequency_tag(50.0), "f50Hz"); - assert_eq!(frequency_tag(0.5), "f0p5Hz"); - assert_eq!(frequency_tag(1_200.0), "f1200Hz"); - assert_eq!(frequency_tag(12.345), "f12p345Hz"); - assert_eq!(sanitize_stem(&frequency_tag(0.5)), frequency_tag(0.5)); - } + fn an_invalid_protocol_is_refused_before_the_drive_moves() { + let folder = temp_folder("protocol-invalid"); + let (mut plugin, _) = protocol_plugin( + &folder, + r#" +[[block]] +mean_u = 0.5 +frequency_hz = 10.0 +depth_a = 99.0 +"#, + ); + let mut sink = ControlSink::default(); - #[test] - fn locks_are_one_per_frequency_and_round_trip_through_the_folder() { - let dir = std::env::temp_dir().join(format!("a1-a0-{}", now_unix_ms())); - std::fs::create_dir_all(&dir).expect("temp dir"); - let folder = dir.display().to_string(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); - let mut plugin = StageAA1Plugin { - output_folder: folder.clone(), - ..StageAA1Plugin::default() - }; - let point = |hz: f64, commanded_a: f64| A0LockPoint { - frequency_hz: hz, - target_a: 0.5, - commanded_a, - measured_a: 0.5, - trials: 2, - converged: true, - locked_at_unix_ms: now_unix_ms(), - low_clip_fraction: None, - high_clip_fraction: None, - }; - plugin.store_lock(point(1_000.0, 0.83)).expect("saved"); - plugin.store_lock(point(50.0, 0.52)).expect("saved"); - // Re-locking the same frequency replaces the row rather than appending. - plugin.store_lock(point(1_000.5, 0.86)).expect("saved"); - assert_eq!(plugin.a0_locks.len(), 2); + assert!(plugin.protocol.is_none(), "a bad protocol started anyway"); assert!( - (plugin.a0_locks[0].frequency_hz - 50.0).abs() < 1e-9, - "sorted by frequency" + sink.services.is_empty(), + "the drive was touched before the file was validated: {:?}", + sink.services + ); + assert!( + plugin.message.contains("depth_a"), + "the message does not name the offending axis: {}", + plugin.message ); - let mut other = StageAA1Plugin { - output_folder: folder.clone(), - ..StageAA1Plugin::default() - }; - other.load_a0_locks(); - assert_eq!(other.a0_locks.len(), 2); - let reloaded = other.lock_for_frequency(1_000.0).expect("reloaded lock"); - assert!((reloaded.commanded_a - 0.86).abs() < 1e-9); - assert_eq!(other.a0_locks_dataset().columns.len(), 7); - - let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&folder); } + /// Each point's own duration governs the recording, not the panel's — a + /// survey whose lengths silently came from the UI would not be + /// reproducible from the protocol alone. #[test] - fn settings_discontinuities_keep_the_response_curve() { - let mut plugin = StageAA1Plugin::default(); - plugin.response_points.push(ResponsePoint { - measured_a: 1.0, - q_on: 0.5, - q_off: 0.1, - cycles: 10, - valid_pixels: 4, - }); - plugin.on_discontinuity(PluginDiscontinuity::SettingsChanged); - assert_eq!(plugin.response_points.len(), 1); - plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); - assert!(plugin.response_points.is_empty()); - } + fn a_point_records_for_the_duration_the_file_asks_for() { + let folder = temp_folder("protocol-duration"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + plugin.duration_s = 999; + let mut sink = ControlSink::default(); - #[test] - fn measurement_id_generation_is_file_safe_and_prefixed() { - let id = generate_measurement_id(); - assert!(id.starts_with("A1-")); - assert_eq!(sanitize_stem(&id), id); - assert_eq!(sanitize_stem("I_k 3 / f=10Hz"), "I_k_3_f_10Hz"); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = sink.services[0].request_id; + sink.services.clear(); + control_tick( + &mut plugin, + inbox_with(vec![accepted(lease_req)]), + &mut sink, + ); + let retargets: Vec = sink + .services + .iter() + .filter(|request| { + matches!( + modulation_command(request), + Some( + ModulationCommandV1::SetOperatingPoint { .. } + | ModulationCommandV1::SetDriveFrequency { .. } + | ModulationCommandV1::SetOpticalDepth { .. } + ) + ) + }) + .map(|request| request.request_id) + .collect(); + control_tick( + &mut plugin, + inbox_with(retargets.into_iter().map(accepted).collect()), + &mut sink, + ); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert_eq!( + plugin.recording.duration_s, 3, + "the panel's duration overrode the protocol's" + ); + + let _ = std::fs::remove_dir_all(&folder); } + /// The host writes its sensor telemetry beside the RAW and A1 moves the + /// recording somewhere else, so the conditions a run was taken under used + /// to be separated from the run itself at the first gather. It has to + /// arrive in the measurement folder under the measurement's own name. #[test] - fn compact_utc_formats_a_known_epoch() { - // 2026-07-23T00:00:00Z = 1_784_764_800 s; +3661 s = 01:01:01. - assert_eq!(format_compact_date(1_784_764_800), "20260723"); - assert_eq!(format_iso_utc(1_784_764_800), "2026-07-23T00:00:00Z"); - assert_eq!(format_compact_utc(1_784_764_800), "20260723-000000"); - assert_eq!( - format_iso_utc(1_784_764_800 + 3_661), - "2026-07-23T01:01:01Z" + fn the_sensor_readout_lands_in_the_measurement_folder_under_the_run_name() { + let capture = temp_folder("sensor-capture"); + std::fs::create_dir_all(&capture).expect("capture dir"); + let raw = capture.join("host-capture.raw"); + std::fs::write(&raw, b"raw").expect("raw"); + std::fs::write( + capture.join("host-capture.sensor-monitoring.csv"), + "schema_version,sample_id,poll_kind,host_elapsed_start_us,host_elapsed_end_us,\ +raw_data_offset_before_bytes,raw_data_offset_after_bytes,illumination_lux,temperature_c,\ +pixel_dead_time_us,bias_diff_on_code,bias_diff_off_code,bias_fo_code,bias_hpf_code,\ +bias_refr_code,status,error\n\ +1,1,full,1000,1200,0,0,140.0,41.5,12.7,10,20,30,40,50,ok,\n\ +1,2,fast,2000,2200,0,0,,,12.8,,,,,,ok,\n", + ) + .expect("telemetry"); + + let output = temp_folder("sensor-output"); + let mut plugin = StageAA1Plugin { + output_folder: output.display().to_string(), + ..StageAA1Plugin::default() + }; + plugin.recording.folder = output.display().to_string(); + plugin.recording.id = "A1-sensor".into(); + plugin.recording.stem = "A1-sensor_20260731-120000".into(); + plugin.recording.cam_finalized_path = Some(raw.display().to_string()); + + plugin.gather_into_measurement_folder(); + + let written = plugin + .recording + .sensor_readout_path + .as_deref() + .expect("a sensor readout was written"); + assert!( + written.ends_with("A1-sensor_20260731-120000.sensor.json"), + "{written}" + ); + assert!( + Path::new(written).starts_with(output.join("A1-sensor")), + "{written} is outside the measurement folder" ); + let parsed: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(written).expect("read")).expect("JSON"); + assert_eq!(parsed["measurement_id"], "A1-sensor"); + assert_eq!(parsed["channels"]["pixel_dead_time_us"]["value"][1], 12.8); + assert_eq!(parsed["channels"]["temperature_c"]["value"][0], 41.5); + // The wide original does not stay behind in the capture folder. + assert!(!capture.join("host-capture.sensor-monitoring.csv").exists()); + + let _ = std::fs::remove_dir_all(&capture); + let _ = std::fs::remove_dir_all(&output); } #[test] @@ -7349,7 +10230,9 @@ mod tests { let doc = plugin.write_sidecar().expect("sidecar path"); let text = std::fs::read_to_string(&doc).expect("read sidecar"); assert!(text.contains("measurement_id = \"A1-test\"")); - assert!(text.contains("flux_point_id = \"flux-test\"")); + // Provenance of `a` is unconditional: offline analysis must never have + // to guess whether a run's depth was measured or merely commanded. + assert!(text.contains("depth_a_source = \"photodiode_measured\"")); assert!(text.contains("[modulation]")); assert!(text.contains("[camera]")); assert!(text.contains("[files]")); @@ -7419,7 +10302,6 @@ mod tests { let mut plugin = StageAA1Plugin { output_folder: "/tmp/a1-preflight".into(), measurement_id: "A1-row".into(), - flux_point_id: "flux-row-1".into(), duration_s: 10, pending_role: Some(RecRole::Normal), photodiode: Some(photodiode), @@ -7441,6 +10323,244 @@ mod tests { ); } + /// A connected modulation plugin with no drive applied yet must never be + /// reported as disconnected. + /// + /// The frequency comes from the phase-0 trigger markers or from the + /// *acknowledged* drive; neither is the connection state, which is a + /// separate check. Conflating them told operators to plug in a bench that + /// was already plugged in. + #[test] + fn a_missing_frequency_is_not_reported_as_a_disconnected_plugin() { + let mut plugin = StageAA1Plugin { + // Connected, but nothing applied yet, and no markers (Live off). + modulation: Some(connected_modulation()), + photodiode: Some(fresh_photodiode_summary()), + frame_width: 10, + frame_height: 1, + ..StageAA1Plugin::default() + }; + assert!(plugin.modulation_connected()); + assert!(plugin.frequency_hz().is_none()); + + let blocker = plugin.frequency_blocker().expect("a reason"); + assert!( + blocker.contains("has not applied a drive yet"), + "an armed-nothing bench must be named as such: {blocker}" + ); + assert!( + !blocker.contains("connect"), + "a connected plugin must not be reported as needing connecting: {blocker}" + ); + + // A genuinely absent owner still says so. + plugin.modulation = None; + let absent = plugin.frequency_blocker().expect("a reason"); + assert!(absent.contains("not reporting status"), "{absent}"); + } + + /// One fact, one line. A missing frequency used to be stated three times. + #[test] + fn the_status_panel_states_a_missing_frequency_exactly_once() { + let plugin = StageAA1Plugin { + modulation: Some(connected_modulation()), + photodiode: Some(fresh_photodiode_summary()), + frame_width: 10, + frame_height: 1, + ..StageAA1Plugin::default() + }; + let lines: Vec = plugin + .status_entries() + .into_iter() + .map(|entry| match entry { + StatusEntry::Text(text) => text, + StatusEntry::LabeledValue { label, value, .. } => format!("{label}: {value}"), + _ => String::new(), + }) + .collect(); + + let explaining = lines + .iter() + .filter(|line| line.contains("has not applied a drive yet")) + .count(); + assert_eq!( + explaining, 1, + "the cause belongs on one line, not three: {lines:#?}" + ); + // The a₀ line points at that line instead of restating it. + assert!( + lines + .iter() + .any(|line| line.contains("waiting for a frequency")), + "{lines:#?}" + ); + // Nothing to say about the response curve at rest. + assert!( + !lines.iter().any(|line| line.starts_with("Response curve")), + "an empty response curve must not take a line: {lines:#?}" + ); + } + + /// An output folder is the only thing an operator must type before they can + /// record. The measurement id names a folder and the flux point id is + /// provenance; neither has ever been a reason to refuse the run, and every + /// fixture in this file used to pre-fill both, which is how the refusal + /// survived. Nothing here sets them. + #[test] + fn recording_needs_only_an_output_folder_not_the_optional_ids() { + let mut plugin = StageAA1Plugin { + output_folder: "/tmp/a1-no-ids".into(), + measurement_id: String::new(), + duration_s: 10, + pending_role: Some(RecRole::Normal), + photodiode: Some(ready_photodiode()), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert_eq!( + plugin.recording.phase, + RecPhase::StartingCamera, + "blank ids must not refuse the recording; message={}", + plugin.message + ); + // The generated id is written back, so the panel shows what was used + // rather than filing the run under a name the operator cannot see. + assert!( + !plugin.measurement_id.trim().is_empty(), + "a generated id must land in the field the operator reads" + ); + assert_eq!(plugin.recording.id, sanitize_stem(&plugin.measurement_id)); + assert!(plugin.recording.stem.starts_with(&plugin.recording.id)); + } + + /// An operator id that is present is kept exactly as it was. + #[test] + fn a_typed_measurement_id_is_never_replaced_by_a_generated_one() { + let mut plugin = StageAA1Plugin { + output_folder: "/tmp/a1-typed-id".into(), + measurement_id: "row-7".into(), + pending_role: Some(RecRole::Normal), + photodiode: Some(ready_photodiode()), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert_eq!(plugin.measurement_id, "row-7"); + assert_eq!(plugin.recording.id, "row-7"); + } + + /// The whole unattended ladder, with nothing typed in but the folder. + /// + /// Each point ends in a recording, and `begin_recording` used to refuse a + /// blank id — three stages after the ladder had already taken the lease and + /// moved the drive. The panel showed the ladder running and the recording + /// idle, and every point was skipped. + #[test] + fn the_frequency_sweep_runs_with_no_ids_typed_in() { + let folder = temp_folder("fsweep-no-ids"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.measurement_id = String::new(); + plugin.a0_target = 0.5; + plugin.min_f = 100.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 2; + plugin.freq_order = FreqOrder::Ascending; + photodiode_window(&mut plugin, 0.02); + plugin.freq_sweep_pending = Some(FreqSweepMode::A0Point); + let mut sink = ControlSink::default(); + + let recorded = run_freq_sweep_to_completion(&mut plugin, &mut sink, 0.6, 4_000); + + assert_eq!( + recorded.len(), + 2, + "every ladder point must record; message: {}", + plugin.message + ); + assert!(plugin.message.contains("2/2 points recorded")); + assert!(!plugin.measurement_id.trim().is_empty()); + let _ = std::fs::remove_dir_all(&folder); + } + + /// The a₀ field is a drag control with a 0.01 step. Comparing it to the + /// lock's target on exact equality meant one stray pixel of drag disarmed a + /// lock that had just converged, and the panel then asked for the very thing + /// the operator had done. The operator's own tolerance is the right band. + #[test] + fn nudging_a0_inside_its_tolerance_keeps_the_lock_armed() { + let mut plugin = plugin_with_markers(); + plugin.a0_target = 0.5; + plugin.a0_tolerance = 0.02; + plugin.a0_locks.push(A0LockPoint { + frequency_hz: plugin.frequency_hz().expect("frequency"), + target_a: 0.5, + commanded_a: 0.83, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: None, + high_clip_fraction: None, + depth_source: DepthSource::Photodiode, + }); + assert!(plugin.armed_lock().is_some(), "the fresh lock must arm"); + + plugin.a0_target = 0.51; + assert!( + plugin.armed_lock().is_some(), + "a nudge inside the tolerance must not disarm the lock" + ); + + // Beyond the tolerance it really is a different target, and the refusal + // says so instead of asking for a Find a₀ that was already done. + plugin.a0_target = 0.70; + assert!(plugin.armed_lock().is_none()); + let blocker = plugin.armed_lock_blocker().expect("a reason"); + assert!( + blocker.contains("0.500") && blocker.contains("0.700"), + "the refusal must name both targets: {blocker}" + ); + } + + /// Three different causes used to share one sentence telling the operator to + /// press Find a₀ — which only fixes the first of them. + #[test] + fn a_lock_that_cannot_arm_names_which_of_the_three_causes_it_is() { + let mut plugin = plugin_with_markers(); + plugin.a0_target = 0.5; + plugin.a0_tolerance = 0.02; + let hz = plugin.frequency_hz().expect("frequency"); + + let no_lock = plugin.armed_lock_blocker().expect("a reason"); + assert!( + no_lock.contains("press Find a₀"), + "with no lock at all, pressing Find a₀ is the fix: {no_lock}" + ); + + plugin.a0_locks.push(A0LockPoint { + frequency_hz: hz, + target_a: 0.5, + commanded_a: 6.0, + measured_a: 0.31, + trials: 8, + converged: false, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: None, + high_clip_fraction: None, + depth_source: DepthSource::Photodiode, + }); + let not_converged = plugin.armed_lock_blocker().expect("a reason"); + assert!( + not_converged.contains("0.310") && not_converged.contains("tolerance"), + "a lock that stopped short must report where it stopped: {not_converged}" + ); + } + /// The photodiode's own Data directory is irrelevant to a recording started /// from A1: A1 names the destination root, so the run proceeds and the PDQ /// is written into A1's measurement folder. @@ -7451,7 +10571,6 @@ mod tests { let mut plugin = StageAA1Plugin { output_folder: "/tmp/a1-destination".into(), measurement_id: "A1-row".into(), - flux_point_id: "flux-row-1".into(), duration_s: 10, pending_role: Some(RecRole::Normal), photodiode: Some(photodiode), @@ -7528,7 +10647,6 @@ mod tests { let mut plugin = StageAA1Plugin { output_folder: folder.display().to_string(), measurement_id: "A1-row".into(), - flux_point_id: "flux-row-1".into(), duration_s: 10, pending_role: Some(RecRole::Normal), photodiode: Some(fresh_photodiode_summary()), diff --git a/plugins/stage-a-a1/src/sensor.rs b/plugins/stage-a-a1/src/sensor.rs new file mode 100644 index 0000000..d426e42 --- /dev/null +++ b/plugins/stage-a-a1/src/sensor.rs @@ -0,0 +1,417 @@ +//! Compacts the host's sensor-telemetry CSV into the per-measurement readout +//! file that travels with a recording. +//! +//! The host polls the camera's monitoring block while recording and writes +//! `.sensor-monitoring.csv` next to the RAW. Two things are wrong +//! with keeping that file as it is: +//! +//! 1. **It stays behind.** A1 gathers the camera RAW, its bias sidecar, the +//! photodiode PDQ and the description file into one measurement folder +//! under one name; the telemetry did not travel with them, so the bench +//! conditions of a run were separated from the run at the first `mv`. +//! +//! 2. **It is a wide table of mostly-empty cells.** The channels are polled on +//! different schedules — the die temperature drifts over minutes, the pixel +//! dead time is read far more often — so a row-per-poll layout with a column +//! per channel is padding by construction. The bias columns are pure +//! duplication on top of that: the same codes are already in the camera's +//! own bias sidecar, which travels with the RAW. +//! +//! So this rewrites it column-wise: one timestamp/value pair list per channel, +//! carrying only the samples where that channel was actually read. Nothing is +//! resampled, interpolated or aligned — a reading exists at the instant it was +//! taken or not at all. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +/// Schema tag written into every readout file. +pub const SCHEMA: &str = "stage-a.a1.sensor.v1"; + +/// One channel's samples, in acquisition order. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Channel { + /// Microseconds since the recording's host clock anchor — the midpoint of + /// the poll, because a monitoring read is not instantaneous and the + /// midpoint is the least wrong single instant to attribute it to. + pub t_us: Vec, + pub value: Vec, +} + +impl Channel { + fn push(&mut self, t_us: i64, value: f64) { + self.t_us.push(t_us); + self.value.push(value); + } + + pub fn len(&self) -> usize { + self.t_us.len() + } + + pub fn is_empty(&self) -> bool { + self.t_us.is_empty() + } +} + +/// A poll that returned nothing usable, kept so a gap in a channel is +/// distinguishable from a channel that was never polled. +#[derive(Debug, Clone, PartialEq)] +pub struct PollFault { + pub t_us: i64, + pub status: String, + pub message: String, +} + +/// The compacted readout for one recording. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct SensorReadout { + /// Channel name → samples. Empty channels are dropped entirely. + pub channels: BTreeMap, + pub faults: Vec, + /// Polls read out of the source file, including the ones that failed. + pub polls: usize, +} + +impl SensorReadout { + pub fn is_empty(&self) -> bool { + self.channels.is_empty() && self.faults.is_empty() + } + + /// Renders the readout as JSON. + /// + /// Hand-written rather than via `serde_json` so the arrays stay on one line + /// each: these files are read by eye as often as by script, and a pretty + /// printer puts one number per line — thousands of lines for what is + /// conceptually one row. + pub fn to_json(&self, measurement_id: &str, recording_stem: &str) -> String { + let mut out = String::with_capacity(1_024 + self.polls * 24); + out.push_str("{\n"); + let _ = writeln!(out, " \"schema\": \"{SCHEMA}\","); + let _ = writeln!( + out, + " \"measurement_id\": {},", + json_string(measurement_id) + ); + let _ = writeln!(out, " \"recording\": {},", json_string(recording_stem)); + out.push_str( + " \"time_base\": \"t_us is the midpoint of each poll, in microseconds on the \ + host clock anchored at the start of this recording\",\n", + ); + out.push_str( + " \"note\": \"Channels are sampled independently and are not aligned; bias codes \ + are omitted because the camera's own bias sidecar already carries them.\",\n", + ); + let _ = writeln!(out, " \"polls\": {},", self.polls); + out.push_str(" \"channels\": {\n"); + let mut first = true; + for (name, channel) in &self.channels { + if !first { + out.push_str(",\n"); + } + first = false; + let _ = write!( + out, + " {}: {{ \"t_us\": [{}], \"value\": [{}] }}", + json_string(name), + join_i64(&channel.t_us), + join_f64(&channel.value), + ); + } + out.push_str("\n },\n"); + out.push_str(" \"faults\": ["); + for (index, fault) in self.faults.iter().enumerate() { + if index > 0 { + out.push(','); + } + let _ = write!( + out, + "\n {{ \"t_us\": {}, \"status\": {}, \"message\": {} }}", + fault.t_us, + json_string(&fault.status), + json_string(&fault.message), + ); + } + if self.faults.is_empty() { + out.push_str("]\n"); + } else { + out.push_str("\n ]\n"); + } + out.push_str("}\n"); + out + } +} + +/// Parses the host's `.sensor-monitoring.csv` into a compact readout. +/// +/// Unknown or reordered columns are handled by name, so a host that adds a +/// column does not shift every value by one. Rows that cannot be read are +/// skipped rather than failing the whole file: a truncated last line is normal +/// if the recording was cut short, and losing the other 4 000 samples over it +/// would be the wrong trade. +pub fn parse_csv(text: &str) -> SensorReadout { + let mut lines = text.lines(); + let Some(header) = lines.next() else { + return SensorReadout::default(); + }; + let columns: Vec<&str> = header.split(',').map(str::trim).collect(); + let index_of = |name: &str| columns.iter().position(|column| *column == name); + + let start = index_of("host_elapsed_start_us"); + let end = index_of("host_elapsed_end_us"); + let status = index_of("status"); + let error = index_of("error"); + // Bias columns are deliberately absent from this list. + let measured: Vec<(&str, usize)> = ["illumination_lux", "temperature_c", "pixel_dead_time_us"] + .into_iter() + .filter_map(|name| index_of(name).map(|index| (name, index))) + .collect(); + + let mut readout = SensorReadout::default(); + for line in lines { + if line.trim().is_empty() { + continue; + } + let fields = crate::csv::split_line(line); + let at = |index: Option| { + index + .and_then(|index| fields.get(index)) + .map(String::as_str) + }; + let midpoint = match ( + at(start).and_then(|value| value.parse::().ok()), + at(end).and_then(|value| value.parse::().ok()), + ) { + (Some(start), Some(end)) => start + (end - start) / 2, + (Some(start), None) => start, + _ => continue, + }; + readout.polls += 1; + + let mut any = false; + for (name, index) in &measured { + let Some(raw) = fields.get(*index) else { + continue; + }; + if raw.is_empty() { + continue; + } + let Ok(value) = raw.parse::() else { + continue; + }; + if !value.is_finite() { + continue; + } + readout + .channels + .entry((*name).to_owned()) + .or_default() + .push(midpoint, value); + any = true; + } + // A poll that produced no reading is only worth recording when the host + // said why; an ordinary "nothing due yet" row is not a fault. + let status_text = at(status).unwrap_or("").to_owned(); + let error_text = at(error).unwrap_or("").to_owned(); + if !any && (!error_text.is_empty() || (!status_text.is_empty() && status_text != "ok")) { + readout.faults.push(PollFault { + t_us: midpoint, + status: status_text, + message: error_text, + }); + } + } + readout +} + +fn json_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for character in value.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + other if (other as u32) < 0x20 => { + let _ = write!(out, "\\u{:04x}", other as u32); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +fn join_i64(values: &[i64]) -> String { + let mut out = String::new(); + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.push(','); + } + let _ = write!(out, "{value}"); + } + out +} + +fn join_f64(values: &[f64]) -> String { + let mut out = String::new(); + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.push(','); + } + // Shortest round-trip form: these are f32 readings widened to f64, so + // the default Display is both exact and compact. + let _ = write!(out, "{value}"); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const HEADER: &str = "schema_version,sample_id,poll_kind,host_elapsed_start_us,\ +host_elapsed_end_us,raw_data_offset_before_bytes,raw_data_offset_after_bytes,illumination_lux,\ +temperature_c,pixel_dead_time_us,bias_diff_on_code,bias_diff_off_code,bias_fo_code,bias_hpf_code,\ +bias_refr_code,status,error"; + + fn csv(rows: &[&str]) -> String { + let mut text = String::from(HEADER); + for row in rows { + text.push('\n'); + text.push_str(row); + } + text.push('\n'); + text + } + + #[test] + fn channels_keep_only_the_polls_that_actually_read_them() { + // The whole point: the die temperature is polled far less often than + // the dead time, and a row-per-poll table pads the difference with + // empty cells. Each channel carries its own samples and nothing else. + let text = csv(&[ + "1,1,fast,1000,1200,0,0,,,12.5,,,,,,ok,", + "1,2,fast,2000,2200,0,0,,,12.6,,,,,,ok,", + "1,3,full,3000,3400,0,0,140.0,41.5,12.7,10,20,30,40,50,ok,", + "1,4,fast,4000,4200,0,0,,,12.8,,,,,,ok,", + ]); + let readout = parse_csv(&text); + + assert_eq!(readout.polls, 4); + assert_eq!(readout.channels["pixel_dead_time_us"].len(), 4); + assert_eq!(readout.channels["temperature_c"].len(), 1); + assert_eq!(readout.channels["illumination_lux"].len(), 1); + assert_eq!(readout.channels["temperature_c"].value, vec![41.5]); + } + + #[test] + fn bias_codes_are_dropped_because_the_bias_sidecar_already_has_them() { + let text = csv(&["1,1,full,1000,1200,0,0,140.0,41.5,12.7,10,20,30,40,50,ok,"]); + let readout = parse_csv(&text); + assert_eq!(readout.channels.len(), 3); + for name in readout.channels.keys() { + assert!(!name.starts_with("bias"), "bias channel survived: {name}"); + } + let parsed: serde_json::Value = + serde_json::from_str(&readout.to_json("m", "s")).expect("valid JSON"); + let channels = parsed["channels"].as_object().expect("channels object"); + assert!( + channels.keys().all(|name| !name.starts_with("bias")), + "{channels:?}" + ); + } + + #[test] + fn a_sample_is_timestamped_at_the_midpoint_of_its_poll() { + // A monitoring read takes a few hundred microseconds; attributing it to + // its start would systematically date every reading early. + let text = csv(&["1,1,full,1000,1400,0,0,140.0,41.5,12.7,,,,,,ok,"]); + let readout = parse_csv(&text); + assert_eq!(readout.channels["temperature_c"].t_us, vec![1200]); + } + + #[test] + fn a_failed_poll_is_kept_as_a_fault_so_a_gap_is_explainable() { + let text = csv(&[ + "1,1,full,1000,1200,0,0,140.0,41.5,12.7,,,,,,ok,", + "1,2,full,2000,2200,0,0,,,,,,,,,error,\"i2c timeout, retrying\"", + ]); + let readout = parse_csv(&text); + assert_eq!(readout.polls, 2); + assert_eq!(readout.faults.len(), 1); + assert_eq!(readout.faults[0].t_us, 2100); + assert_eq!(readout.faults[0].status, "error"); + assert_eq!(readout.faults[0].message, "i2c timeout, retrying"); + } + + #[test] + fn an_ordinary_empty_poll_is_not_a_fault() { + let text = csv(&["1,1,fast,1000,1200,0,0,,,,,,,,,ok,"]); + let readout = parse_csv(&text); + assert_eq!(readout.polls, 1); + assert!(readout.faults.is_empty()); + assert!(readout.channels.is_empty()); + } + + #[test] + fn a_truncated_final_row_does_not_cost_the_rest_of_the_file() { + // Cutting a recording short leaves a partial last line. Losing 4 000 + // good samples over it would be the wrong trade. + let mut text = csv(&["1,1,full,1000,1200,0,0,140.0,41.5,12.7,,,,,,ok,"]); + text.push_str("1,2,full,20"); + let readout = parse_csv(&text); + assert_eq!(readout.channels["temperature_c"].len(), 1); + } + + #[test] + fn columns_are_found_by_name_not_by_position() { + // A host that inserts a column must not shift every reading by one. + let text = "host_elapsed_start_us,host_elapsed_end_us,new_column,temperature_c,status\n\ + 1000,1200,x,41.5,ok\n"; + let readout = parse_csv(text); + assert_eq!(readout.channels["temperature_c"].value, vec![41.5]); + } + + #[test] + fn an_empty_or_header_only_file_produces_an_empty_readout() { + assert!(parse_csv("").is_empty()); + assert!(parse_csv(HEADER).is_empty()); + } + + #[test] + fn the_json_is_one_line_per_channel_and_parses_back() { + let text = csv(&[ + "1,1,full,1000,1200,0,0,140.0,41.5,12.7,,,,,,ok,", + "1,2,fast,2000,2200,0,0,,,12.8,,,,,,ok,", + ]); + let json = parse_csv(&text).to_json("meas-1", "meas-1_20260731T120000Z"); + + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + assert_eq!(parsed["schema"], SCHEMA); + assert_eq!(parsed["measurement_id"], "meas-1"); + assert_eq!(parsed["polls"], 2); + assert_eq!(parsed["channels"]["pixel_dead_time_us"]["value"][1], 12.8); + assert_eq!(parsed["channels"]["temperature_c"]["t_us"][0], 1100); + + // Compactness is the point of hand-rendering it: a pretty printer would + // put one number per line. + for line in json.lines() { + assert!( + !line.trim_start().starts_with("12.8"), + "an array was expanded one value per line:\n{json}" + ); + } + } + + #[test] + fn quoted_error_text_with_commas_survives_the_round_trip() { + let text = csv(&["1,1,full,1000,1200,0,0,,,,,,,,,error,\"a, b, \"\"c\"\"\""]); + let readout = parse_csv(&text); + assert_eq!(readout.faults[0].message, "a, b, \"c\""); + let parsed: serde_json::Value = + serde_json::from_str(&readout.to_json("m", "s")).expect("valid JSON"); + assert_eq!(parsed["faults"][0]["message"], "a, b, \"c\""); + } +} diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index 1c9cb62..a3ae241 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -11,7 +11,7 @@ KD*P, 3×3 mm, 400–850 nm, 5 W. - `MANUAL`: **Power** is the peak/operating code and **Min threshold** is the lower endpoint. - `CALIBRATED`: `V_null`, `V_peak`, normalized cycle mean `ū`, and optical depth `a` determine the endpoints. Both lobe fields are **absolute DAC codes** — where the light is dimmest and where it is - brightest — and the quarter wave `Vπ = |V_peak − V_null|` is derived, never typed + brightest — and the half-wave span `|V_peak − V_null|` is derived, never typed ([ADR 016](../../docs/adr/016-stage-a-lobe-endpoints-not-a-distance.md)). Measure them with the built-in transfer sweep — see [Calibration](#calibration--measuring-v_null-and-v_peak). - **Mode** independently selects the waveform that fills that band. All five modes are available @@ -34,7 +34,7 @@ Manual optical modes reuse the stored `V_null`/`V_peak` lobe and derive their ef from the slider band through the forward optical transfer. In calibrated `CONST`, `a` is irrelevant: the hold is -`V_null + (2Vπ/π)·asin(sqrt(u))`. With `V_null=1630` and `Vπ=860`, this is +`V_null + (2·span/π)·asin(sqrt(u))`. With `V_null=1630` and a span of `860`, this is 2490 at `ū=1` and 1685 at `ū=0.01`. This dimensionless `ū` is not the physical A1 flux point `I_k`. Periodic modes still need optical headroom and reject impossible `ū`/`a` combinations without changing the @@ -57,18 +57,30 @@ gain, temperature, and the actual electrical load all enter the realised map, so 4. Read the result in the **Pockels transfer curve** view and the status line, then press **Apply to V_null / V_peak**, which writes both endpoint codes. Anything questionable — a high residual, dropped points, hysteresis, clipping — appears as a `Check:` line but does not block the apply: the plot is - the arbiter, and a single stray sample can inflate the residual fivefold while leaving `Vπ` + the arbiter, and a single stray sample can inflate the residual fivefold while leaving the fit accurate to a few codes. Wild points are dropped from the fit automatically. +Each point is a real measurement, not a sample: the sweep waits 0.1 s for the cell to settle and +then takes the photodiode's 20 ms averaged level. Both are durations the sweep and the photodiode +plugin own, deliberately not sample counts and not the chart's averaging setting — those made the +precision of the calibration follow the acquisition rate and a display knob +([ADR 019](../../docs/adr/019-stage-a-calibration-measures-its-own-window.md)). + +The `Check:` lines are measured against the fit's own noise, never against zero. Hysteresis is +compared with what independent point scatter alone would produce, so a noisy bench is not reported +as a drifting cell. Points that reach an end of the detector's range truncate the reported detector +extrema but not `V_null`/`V_peak`, and the fix for them is detector **gain** — on the reject port it is +the dark end that hits the bottom rail, so attenuation is backwards. + The view also works *before* any measurement: it draws the lobe your current `V_null`/`V_peak` claim, on a normalised axis, with markers at `V_null` and `V_peak` — the two settings themselves, so the plot reads straight back into the two fields. The status pane states the same thing in -codes: `Lobe: Vπ = 860 codes — u 0 → 1630 (min light), 0.5 → 2060, 1 → 2490 (max light)`. +codes: `Lobe: half-wave span 860 codes — u 0 → 1630 (min light), 0.5 → 2060, 1 → 2490 (max light)`. Two properties worth knowing: - `V_null`/`V_peak` need **no** dark measurement and **no** total-power anchor — the fitted offset and - amplitude absorb the dark level and the front-end gain. + amplitude absorb any DC offset and the front-end gain. - The detector level at the null is reported as a **lower bound** on the total-power anchor `I_tot`, *not* as the anchor. On the reject port the residual transmitted floor is not separable from it; freezing a real anchor needs a transmitted-port power measurement. diff --git a/plugins/stage-a-modulation/src/calibration.rs b/plugins/stage-a-modulation/src/calibration.rs index f4ef7ea..be4d658 100644 --- a/plugins/stage-a-modulation/src/calibration.rs +++ b/plugins/stage-a-modulation/src/calibration.rs @@ -49,6 +49,15 @@ //! — the obvious approach — breaks on exactly the sweeps that matter: with a //! real `Vπ` near 860 the DAC range holds ~2.4 lobes, so the global minimum //! and maximum can sit whole periods apart and the seed is meaningless. +//! +//! # Noise is measured, not assumed +//! +//! Every judgement about whether a sweep is good — was a lobe resolved at all, +//! is the up/down difference real drift — is made against the **fit's own RMS +//! residual**, which is the scatter of the averaged points about the curve. +//! Nothing here reads `peak_to_peak_volts`, which measures the detector *before* +//! averaging and therefore says more about the photodiode owner's window length +//! than about the precision of a point (ADR 019). use std::f64::consts::PI; @@ -78,7 +87,9 @@ pub struct SweepPoint { pub direction: Direction, /// Raw detector level in volts, as published by the photodiode owner. pub volts: f64, - /// Spread over the averaged window; a settle-quality witness. + /// Spread over the averaged window. Archived as a settle-quality witness the + /// operator can read next to the plot; the fit deliberately does not use it + /// (see the module docs). pub peak_to_peak_volts: f64, pub clipped: bool, } @@ -128,7 +139,8 @@ pub struct TransferFit { pub quality: f64, pub geometry: DetectorGeometry, /// Mean |ascending − descending| at matched codes, as a fraction of the - /// span. `None` when the sweep ran in one direction only. + /// span. `None` when the sweep ran in one direction only. Judge it against + /// [`TransferFit::hysteresis_noise_floor`], never against zero. pub hysteresis: Option, /// Fraction of one full lobe (`Vπ` codes) the sweep actually covered. /// Below ~1 the half-wave-voltage span is extrapolated, not measured. @@ -167,17 +179,73 @@ impl TransferFit { pub fn detector_volts_at_peak(&self) -> f64 { self.offset_volts + self.span_volts } + + /// The value [`Self::hysteresis`] takes when the two passes differ by + /// nothing but independent point noise. + /// + /// Both passes measure the same curve, so their difference at a matched code + /// is the difference of two independent errors of scale `σ` — and for those, + /// `E|Δ| = σ√2 · √(2/π) = 1.128 σ`. The fit already measures `σ` as its RMS + /// residual, so the floor comes out of numbers that are on the table. + /// + /// Without it the metric reports noise as drift: on a real bench sweep whose + /// points carried 11.3 mV of scatter against a 50.8 mV lobe, the "hysteresis" + /// read 25.7 % against a floor of 25.1 % — a clean, drift-free cell flagged + /// as drifting (ADR 019). + pub fn hysteresis_noise_floor(&self) -> f64 { + let span = self.span_volts.abs(); + if span <= f64::EPSILON { + return f64::INFINITY; + } + 1.128 * self.rms_residual_volts / span + } + + /// How far the up/down disagreement stands above what the point noise alone + /// explains: [`Self::hysteresis`] over [`Self::hysteresis_noise_floor`]. + /// + /// The ratio lives between two derivable endpoints, which is what makes it + /// usable as a test. Write `Δ` for a systematic offset between the passes and + /// `σ` for the per-point noise. The metric itself behaves as + /// `√(Δ² + (1.128σ)²)`, while the fit — which splits the difference between + /// the two passes — carries a residual of `√(Δ²/4 + σ²)`. So: + /// + /// - **pure noise** (`Δ = 0`) → **1.0**, by construction; + /// - **pure drift** (`Δ ≫ σ`) → `Δ / (1.128 · Δ/2)` = **1.77**. + /// + /// A systematic offset therefore inflates the residual too, and the ratio + /// saturates rather than growing without bound — which is exactly why a + /// generous multiple of the floor (2×, say) never fires at all. The + /// discriminating range is narrow and known, so the threshold belongs inside + /// it: [`Self::hysteresis_is_systematic`]. + /// + /// `None` when the sweep ran in one direction only. + pub fn hysteresis_above_noise(&self) -> Option { + self.hysteresis + .map(|value| value / self.hysteresis_noise_floor()) + } + + /// Whether the up/down disagreement is drift rather than scatter. + /// + /// The cut sits between the two endpoints derived in + /// [`Self::hysteresis_above_noise`], at the point where the systematic part + /// is about 1.5× the point noise — sensitive enough to catch a real lag, + /// blind to a bench that is merely noisy. + pub fn hysteresis_is_systematic(&self) -> bool { + const SYSTEMATIC_ABOVE: f64 = 1.33; + self.hysteresis_above_noise() + .is_some_and(|ratio| ratio > SYSTEMATIC_ABOVE) + } } #[derive(Debug, Clone, PartialEq)] pub enum FitError { /// Fewer points than parameters can be resolved from. TooFewPoints { count: usize, minimum: usize }, - /// The between-code signal span is not larger than the detector's typical - /// within-window excursion, so the sweep does not resolve a lobe. + /// The fitted lobe does not stand above the scatter of the points about it, + /// so the sweep does not resolve a lobe. NoModulation { - signal_span_volts: f64, - noise_span_volts: f64, + span_volts: f64, + residual_volts: f64, }, /// A fitted lobe exists but no `[V_null, V_null+Vπ]` fits inside the /// commandable range, so no monotonic branch is usable. @@ -191,13 +259,13 @@ impl std::fmt::Display for FitError { write!(f, "only {count} sweep points (minimum {minimum})") } Self::NoModulation { - signal_span_volts, - noise_span_volts, + span_volts, + residual_volts, } => write!( f, - "detector sweep span {signal_span_volts:.6} V does not exceed the typical \ - within-window excursion {noise_span_volts:.6} V; check the light path and HV \ - amplifier, or reduce detector noise / increase averaging" + "the fitted lobe spans {span_volts:.6} V but the points scatter {residual_volts:.6} \ + V about it, so no lobe is resolved; check the light path and HV amplifier, or \ + reduce detector noise" ), Self::NoLobeInRange { v_pi_dac } => write!( f, @@ -213,23 +281,10 @@ impl std::error::Error for FitError {} /// Smallest usable sweep: four points per fitted parameter. pub const MIN_POINTS: usize = 16; -/// Median raw peak-to-peak excursion inside one settled CONST window. -/// -/// This is the scale a between-code transfer curve has to beat. Unlike the -/// former absolute 10 mV cut, it follows the detector gain and acquisition -/// noise, so millivolt-scale but repeatable Pockels sweeps remain usable. -fn typical_window_noise(points: &[SweepPoint]) -> f64 { - let mut spans: Vec = points - .iter() - .map(|point| point.peak_to_peak_volts) - .filter(|span| span.is_finite() && *span >= 0.0) - .collect(); - if spans.is_empty() { - return 0.0; - } - spans.sort_by(f64::total_cmp); - spans[spans.len() / 2] -} +/// Largest `rms_residual / |span|` that still counts as a resolved lobe. See the +/// gate in [`fit_transfer`] for where the number comes from; above it the sweep +/// is refused outright, below it the residual only warns. +const MAX_RESOLVED_QUALITY: f64 = 0.5; /// Least-squares solution for one candidate half-wave-voltage span `w`. struct Harmonic { @@ -419,11 +474,20 @@ fn hysteresis_fraction(points: &[SweepPoint], span: f64) -> Option { /// Scans the half-wave-voltage span over every period the sweep could resolve, then /// refines. Returns the best `(Vπ, harmonic)`. -fn fit_period(points: &[SweepPoint], swept_span: f64) -> Option<(f64, Harmonic)> { +/// +/// `code_count` is the number of **distinct** codes visited, not the number of +/// points: a sweep that runs up and back visits each code twice, and counting +/// the repeats halves the apparent code step and pushes the scan floor below +/// what the sweep can resolve — straight into aliasing. +fn fit_period( + points: &[SweepPoint], + swept_span: f64, + code_count: usize, +) -> Option<(f64, Harmonic)> { // From four samples per lobe (below that the lobe is aliased) out to a // lobe twice the swept span (a barely-curved arc). Log-spaced, because a // fixed step wastes resolution at long periods and misses short ones. - let point_spacing = swept_span / points.len().max(2) as f64; + let point_spacing = swept_span / code_count.max(2) as f64; let w_min = (2.0 * point_spacing).max(1.0); let w_max = (2.0 * swept_span).max(w_min * 1.5); const SCAN_STEPS: usize = 600; @@ -492,20 +556,20 @@ pub fn fit_transfer( let min_volts = profile.iter().map(|(_, v)| *v).fold(f64::MAX, f64::min); let max_volts = profile.iter().map(|(_, v)| *v).fold(f64::MIN, f64::max); let observed_span = max_volts - min_volts; - let noise_span = typical_window_noise(points); - if !observed_span.is_finite() - || observed_span <= f64::EPSILON - || (noise_span > 0.0 && observed_span <= noise_span) - { + // Only the degenerate case is refused before fitting — a flat or non-finite + // sweep has no curve to measure anything against. Whether a real lobe was + // resolved is decided *after* the fit, from the fit's own residual. + if !observed_span.is_finite() || observed_span <= f64::EPSILON { return Err(FitError::NoModulation { - signal_span_volts: observed_span.max(0.0), - noise_span_volts: noise_span, + span_volts: observed_span.max(0.0), + residual_volts: 0.0, }); } let swept_lo = profile.first().map(|(code, _)| *code).unwrap_or(0.0); let swept_hi = profile.last().map(|(code, _)| *code).unwrap_or(max_code); let swept_span = (swept_hi - swept_lo).max(1.0); + let code_count = profile.len(); // A single stray point — one window caught mid-settle, one stream hiccup — // barely moves the fitted period but inflates the RMS residual several @@ -513,13 +577,13 @@ pub fn fit_transfer( // what is left, so the reported residual describes the curve rather than // the worst sample. let (w, harmonic, rejected_points) = { - let first = fit_period(points, swept_span).ok_or(FitError::NoModulation { - signal_span_volts: observed_span, - noise_span_volts: noise_span, + let first = fit_period(points, swept_span, code_count).ok_or(FitError::NoModulation { + span_volts: observed_span, + residual_volts: 0.0, })?; let kept = without_outliers(points, first.0, &first.1); if kept.len() < points.len() && kept.len() >= MIN_POINTS { - match fit_period(&kept, swept_span) { + match fit_period(&kept, swept_span, code_count) { Some((w, harmonic)) => (w, harmonic, points.len() - kept.len()), None => (first.0, first.1, 0), } @@ -544,18 +608,38 @@ pub fn fit_transfer( 2.0 * radius, ), }; - if !p1.is_finite() || p1.abs() <= f64::EPSILON || (noise_span > 0.0 && p1.abs() <= noise_span) { + + // Over the points the fit actually used: dividing the kept residual by the + // full count would flatter the number. + let rms = (harmonic.sse / (points.len() - rejected_points).max(1) as f64).sqrt(); + // A lobe is resolved when its amplitude stands above the scatter of the + // points about it. `p1` and the residual are spans of the *same* averaged + // points, so they are directly comparable — which the previous test, against + // the median raw within-window excursion, was not: that measures the detector + // *before* averaging, so it tracks whatever window the photodiode owner + // happens to publish rather than the precision of a point. It came within a + // factor of two of refusing a real, clean bench sweep, and would have got + // stricter as the owner's window grew (ADR 019). + // + // The threshold has to leave room on both sides, because a free period + // search over pure noise does *not* return an amplitude of zero: with `n` + // points the quadrature pair has scale `σ√(2/n)`, and taking the best of a + // 600-step scan inflates it by about `√(2 ln 600)`. For the sweeps this + // module actually sees (n = 49 and n = 98) that lands the noise-only quality + // at 0.7–1.0 — measured at 0.97 in `refuses_a_lobe_that_does_not_stand_above + // _the_point_scatter`. A resolved lobe sits far below: the noisiest real + // bench record on file reads 0.22. Half-way between, at 0.5, is a plain + // statement — the lobe must be at least twice its own scatter — with better + // than 2× margin either way. + if !p1.is_finite() || p1.abs() <= f64::EPSILON || rms >= MAX_RESOLVED_QUALITY * p1.abs() { return Err(FitError::NoModulation { - signal_span_volts: p1.abs(), - noise_span_volts: noise_span, + span_volts: p1.abs(), + residual_volts: rms, }); } let v_null = select_lobe(v, w, max_code).ok_or(FitError::NoLobeInRange { v_pi_dac: w })?; - // Over the points the fit actually used: dividing the kept residual by the - // full count would flatter the number. - let rms = (harmonic.sse / (points.len() - rejected_points).max(1) as f64).sqrt(); Ok(TransferFit { v_null_dac: v_null, v_pi_dac: w, @@ -601,12 +685,140 @@ pub fn sweep_codes( codes } +/// Deterministic per-point scatter in `[-1, 1]`, shared by the fit tests and the +/// plugin's warning tests. +/// +/// Not an RNG — failures reproduce — but genuinely *uncorrelated between the two +/// passes*, which a wobble alternating with the point index is not: with an odd +/// number of points per pass, matched codes always land on opposite signs, so +/// what looks like noise is a systematic offset between the passes. That is the +/// exact thing the hysteresis test has to tell apart, so the fixture must not +/// quietly be the wrong one. +#[cfg(test)] +pub(crate) fn scatter(code: u16, direction: Direction) -> f64 { + let mut x = u64::from(code).wrapping_mul(0x9E37_79B9_7F4A_7C15) + ^ match direction { + Direction::Ascending => 0, + Direction::Descending => 0xD1B5_4A32_D192_ED03, + }; + x ^= x >> 33; + x = x.wrapping_mul(0xFF51_AFD7_ED55_8CCD); + x ^= x >> 33; + ((x >> 11) as f64 / (1u64 << 53) as f64) * 2.0 - 1.0 +} + #[cfg(test)] mod tests { use super::*; + /// The sweep the operator recorded on 2026-07-30, verbatim. + /// + /// A clean 625-code lobe that the plugin then reported as bad: 22 % residual, + /// 26 % hysteresis, 34 "clipped" points. Every one of those was an artifact + /// of publishing four ADC samples per settled code (ADR 019). Kept as a + /// fixture because synthetic sweeps cannot reproduce what a real detector's + /// signal-proportional noise does to metrics that are compared against zero. + const REAL_SWEEP: &str = include_str!("../testdata/pockels-20260730-083123.json"); + + fn real_sweep_points() -> Vec { + let record: serde_json::Value = + serde_json::from_str(REAL_SWEEP).expect("the archived record parses"); + record["points"] + .as_array() + .expect("points array") + .iter() + .map(|point| SweepPoint { + code: point["code"].as_u64().expect("code") as u16, + direction: match point["direction"].as_str().expect("direction") { + "up" => Direction::Ascending, + "down" => Direction::Descending, + other => panic!("unknown direction {other}"), + }, + volts: point["volts"].as_f64().expect("volts"), + peak_to_peak_volts: point["peak_to_peak_volts"].as_f64().expect("p2p"), + clipped: point["clipped"].as_bool().expect("clipped"), + }) + .collect() + } + + #[test] + fn the_recorded_bench_sweep_resolves_its_lobe() { + let points = real_sweep_points(); + assert_eq!(points.len(), 98); + let fit = + fit_transfer(&points, 3_000.0, DetectorGeometry::RejectedComplement).expect("fits"); + + assert!((fit.v_pi_dac - 625.4).abs() < 1.0, "Vπ = {}", fit.v_pi_dac); + assert!( + (fit.v_null_dac - 711.9).abs() < 1.0, + "V_null = {}", + fit.v_null_dac + ); + assert!(fit.span_volts < 0.0, "reject port darkens with excitation"); + assert!(fit.lobe_coverage > 4.0, "coverage = {}", fit.lobe_coverage); + } + + #[test] + fn the_recorded_sweeps_hysteresis_is_exactly_its_point_noise() { + // The load-bearing claim behind the hysteresis noise floor, and the + // reason the operator's clean cell was reported as drifting. + // + // Both passes measure one curve, so at a matched code they differ by two + // independent errors of scale σ, for which E|Δ| = 1.128 σ. The fit + // measures σ as its RMS residual. If the observed 25.7 % lands on that + // prediction, the passes disagree by nothing but noise — there is no + // drift to warn about, at any threshold that ignores the noise. + let fit = fit_transfer( + &real_sweep_points(), + 3_000.0, + DetectorGeometry::RejectedComplement, + ) + .expect("fits"); + + let hysteresis = fit.hysteresis.expect("both directions were swept"); + let floor = fit.hysteresis_noise_floor(); + assert!( + (hysteresis / floor - 1.0).abs() < 0.05, + "hysteresis {hysteresis:.4} vs. noise floor {floor:.4}: not explained by noise alone" + ); + assert!( + !fit.hysteresis_is_systematic(), + "ratio = {:?}", + fit.hysteresis_above_noise() + ); + } + + #[test] + fn the_hysteresis_ratio_sits_between_its_two_derived_endpoints() { + // The threshold in `hysteresis_is_systematic` is only meaningful if the + // ratio really does run from 1.0 (pure noise) to 1.77 (pure drift). Both + // ends are asserted here, because the cut sits between them and nowhere + // else would work. + let scattered = fit_transfer( + &synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.050, true), + 4_095.0, + DetectorGeometry::RejectedComplement, + ) + .expect("fits"); + let noise_end = scattered.hysteresis_above_noise().expect("both directions"); + assert!((noise_end - 1.0).abs() < 0.15, "noise end = {noise_end}"); + + // Same curve, no scatter, one pass offset wholesale: pure drift. + let mut points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, true); + for point in &mut points { + if point.direction == Direction::Descending { + point.volts -= 0.2; + } + } + let drifting = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + let drift_end = drifting.hysteresis_above_noise().expect("both directions"); + assert!((drift_end - 1.772).abs() < 0.15, "drift end = {drift_end}"); + assert!(drifting.hysteresis_is_systematic()); + } + /// Synthesizes a sweep of a known lobe as seen through a given port. - /// `noise` is a deterministic zig-zag, not an RNG, so failures reproduce. + /// `noise` is the amplitude of the deterministic per-point [`scatter`]. fn synthetic_sweep( v_null: f64, v_pi: f64, @@ -622,14 +834,12 @@ mod tests { }; sweep_codes(max_code, 49, both_directions) .into_iter() - .enumerate() - .map(|(index, (code, direction))| { + .map(|(code, direction)| { let u = lobe.u_for_dac(f64::from(code)); - let wobble = if index % 2 == 0 { noise } else { -noise }; SweepPoint { code, direction, - volts: offset + span * u + wobble, + volts: offset + span * u + noise * scatter(code, direction), peak_to_peak_volts: 0.002, clipped: false, } @@ -769,8 +979,8 @@ mod tests { #[test] fn accepts_a_repeatable_sub_10mv_transfer() { // The real detector commonly operates between roughly 0.5 and 15 mV. - // A repeatable 4 mV lobe was rejected by the former absolute 10 mV - // threshold even though it is twice the measured window excursion. + // A repeatable 4 mV lobe was rejected by an absolute 10 mV threshold + // even though the points sit tightly on it. let points = synthetic_sweep(300.0, 1_600.0, 0.010, -0.004, 4_095, 0.000_05, true); let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement) .expect("a resolved millivolt-scale lobe must fit"); @@ -785,15 +995,39 @@ mod tests { } #[test] - fn refuses_apparent_modulation_below_the_window_noise() { + fn refuses_a_lobe_that_does_not_stand_above_the_point_scatter() { + // No lobe at all (span 0), only scatter. A free period search over noise + // does not return zero amplitude — it returns the best of 600 tries — + // which is exactly why the gate cannot sit at `residual >= span`. let points = synthetic_sweep(300.0, 1_600.0, 0.008, 0.0, 4_095, 0.000_4, false); - assert!(matches!( - fit_transfer(&points, 4_095.0, DetectorGeometry::Direct), - Err(FitError::NoModulation { - noise_span_volts, - .. - }) if (noise_span_volts - 0.002).abs() < 1e-12 - )); + let error = fit_transfer(&points, 4_095.0, DetectorGeometry::Direct) + .expect_err("noise alone must not pass as a lobe"); + let FitError::NoModulation { + span_volts, + residual_volts, + } = error + else { + panic!("{error:?}"); + }; + // Pins the noise-only quality the threshold was chosen against: this + // fixture reads ~0.97, and the cut at 0.5 keeps a factor of two clear. + let noise_quality = residual_volts / span_volts; + assert!( + (0.6..1.2).contains(&noise_quality), + "noise-only quality = {noise_quality}" + ); + } + + #[test] + fn a_noisy_but_real_lobe_still_resolves() { + // The gate is about resolution, not tidiness: a lobe carrying a fifth of + // its own span in scatter — the state the bench was actually in — must + // still fit. Only the warnings are allowed to comment on it. + let points = synthetic_sweep(700.0, 625.0, 0.058, -0.051, 3_000, 0.011, true); + let fit = fit_transfer(&points, 3_000.0, DetectorGeometry::RejectedComplement) + .expect("a noisy but resolved lobe must fit"); + assert!((fit.v_pi_dac - 625.0).abs() < 20.0, "Vπ = {}", fit.v_pi_dac); + assert!(fit.quality > 0.1, "quality = {}", fit.quality); } #[test] diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 3a92171..8dbf1b7 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -59,10 +59,17 @@ const CURVE_VIEW_ID: &str = "stage-a-modulation.transfer-curve.view"; /// Codes measured per sweep pass. 49 points over the full range put a sample /// every ~85 codes, ~19 per lobe at a typical Vπ of 860. const SWEEP_POINTS_PER_PASS: usize = 49; -/// Samples the detector must have taken *after* a code was commanded before its -/// window counts as settled. At the firmware's 20 kSa/s that is 100 ms — enough -/// for the HV amplifier and the cell to arrive, proven from the sample clock -/// rather than assumed from a timer. +/// How long the detector must have run *after* a code was commanded before its +/// window counts as settled — enough for the HV amplifier and the cell to +/// arrive, proven from the sample clock rather than assumed from a timer. +/// +/// A duration, not a sample count: settling is a property of the amplifier and +/// the crystal, not of the acquisition rate. The former fixed 2 000 samples was +/// written for 20 kSa/s (100 ms) and silently became 4 ms when the bench moved to +/// 500 kSa/s. +const SETTLE_SECONDS: f64 = 0.1; +/// Settle window when the photodiode has not published a sample rate. Matches +/// [`SETTLE_SECONDS`] at the firmware's original 20 kSa/s. const SETTLE_SAMPLES: u64 = 2_000; /// Give up on a point if no settled level arrives within this long. A stalled /// photodiode stream must abort the sweep, not hang it. @@ -89,6 +96,60 @@ const REQUEST_CACHE_LIMIT: usize = 256; const MIN_LEASE_TTL_MS: u64 = 250; const MAX_LEASE_TTL_MS: u64 = 60_000; +/// The lobe a calibration was last applied to, published process-wide. +/// +/// The host runs two instances of this plugin — a UI mirror that renders the +/// settings and a live worker that owns the device link — and settings only +/// ever travel *mirror → worker*: every live-analysis pass collects +/// `get_setting` from the mirror and writes it onto the worker. +/// +/// The measured fit lives on the worker (it is the instance with the +/// photodiode and the DAC), so "Apply to V_null / V_peak" wrote the new lobe +/// into the worker's fields and the very next sync overwrote it with the +/// mirror's stale codes. The button appeared to do nothing, twice over: the +/// mirror had no fit to apply, and the worker's result did not survive a tick. +/// +/// Both instances live in the same process, so the applied lobe is published +/// here with a monotonic generation and adopted by whichever instance is +/// behind. The generation is what makes adoption one-way: a mirror that has +/// already seen generation *n* keeps accepting ordinary edits. +static APPLIED_LOBE: Mutex> = Mutex::new(None); +static APPLIED_LOBE_GENERATION: AtomicU64 = AtomicU64::new(0); + +#[derive(Debug, Clone)] +struct AppliedLobe { + generation: u64, + v_null_dac: i64, + v_peak_dac: i64, + calibration_id: String, +} + +/// Publishes a freshly applied lobe to every instance in this process. +fn publish_applied_lobe(v_null_dac: i64, v_peak_dac: i64, calibration_id: String) -> u64 { + let generation = APPLIED_LOBE_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; + if let Ok(mut slot) = APPLIED_LOBE.lock() { + *slot = Some(AppliedLobe { + generation, + v_null_dac, + v_peak_dac, + calibration_id, + }); + } + generation +} + +/// The applied lobe an instance at `seen` generation has not adopted yet. +fn applied_lobe_after(seen: u64) -> Option { + if APPLIED_LOBE_GENERATION.load(Ordering::SeqCst) <= seen { + return None; + } + APPLIED_LOBE + .lock() + .ok()? + .clone() + .filter(|applied| applied.generation > seen) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Mode { Const, @@ -153,6 +214,18 @@ impl Mode { _ => None, } } + + /// How this mode's peak intensity follows from `ū` and `a` on a calibrated + /// band. This is what bounds both controls — see [`waveform::PeakLaw`]. + fn peak_law(self) -> waveform::PeakLaw { + match self.optical_target() { + Some(target) => waveform::PeakLaw::of(target), + None => match self { + Self::Const => waveform::PeakLaw::Constant, + _ => waveform::PeakLaw::LogSwing, + }, + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -610,201 +683,6 @@ fn now_unix_ms() -> u64 { .unwrap_or(0) } -/// One validated protocol step: the exact MOD command plus how long to hold -/// it before advancing. -#[derive(Debug, Clone, PartialEq)] -struct ProtocolStep { - duration: Duration, - command: Command, - summary: String, -} - -#[derive(Debug, Clone, Default)] -struct ProtocolProgress { - loops: usize, - total_steps: usize, - /// 1-based while running. - loop_index: usize, - step_index: usize, - summary: String, - finished: bool, - stopped: bool, -} - -/// Running protocol executor; dropping it stops the thread. Commands go -/// through the same coalescing pending slot the device thread drains, so the -/// executor never touches the serial port itself. -struct ProtocolRun { - stop: Arc, - join: Option>, - progress: Arc>, -} - -impl Drop for ProtocolRun { - fn drop(&mut self) { - self.stop.store(true, Ordering::Relaxed); - if let Some(join) = self.join.take() { - let _ = join.join(); - } - } -} - -/// Parses the TOML protocol format: -/// -/// ```toml -/// loops = 2 # optional, default 1 -/// [[steps]] -/// duration_s = 5.0 -/// wave = "SINE" # OFF | CONST | SINE | SQUARE -/// level = 2000 # required unless OFF -/// min = 0 # optional, periodic only -/// frequency_hz = 100.0 # required for SINE/SQUARE (0.01–2000) -/// ``` -fn parse_protocol(text: &str) -> Result<(Vec, usize), String> { - let table: toml::Table = text - .parse() - .map_err(|err| format!("protocol is not valid TOML: {err}"))?; - let loops = match table.get("loops") { - None => 1, - Some(value) => { - let loops = value.as_integer().ok_or("loops must be an integer")?; - if !(1..=10_000).contains(&loops) { - return Err("loops must be between 1 and 10000".into()); - } - loops as usize - } - }; - let raw_steps = table - .get("steps") - .and_then(|value| value.as_array()) - .ok_or("protocol needs at least one [[steps]] entry")?; - if raw_steps.is_empty() { - return Err("protocol needs at least one [[steps]] entry".into()); - } - - let mut steps = Vec::with_capacity(raw_steps.len()); - for (index, raw) in raw_steps.iter().enumerate() { - let step = raw - .as_table() - .ok_or_else(|| format!("step {} must be a table", index + 1))?; - let context = |msg: &str| format!("step {}: {msg}", index + 1); - - let duration_s = step - .get("duration_s") - .and_then(|value| value.as_float().or(value.as_integer().map(|v| v as f64))) - .ok_or_else(|| context("duration_s is required"))?; - if !(0.001..=3_600.0).contains(&duration_s) { - return Err(context("duration_s must be between 0.001 and 3600")); - } - let wave = step - .get("wave") - .and_then(|value| value.as_str()) - .ok_or_else(|| context("wave is required (OFF/CONST/SINE/SQUARE)"))? - .to_uppercase(); - - let (command, summary) = if wave == "OFF" { - ( - Command::new("MOD").field("wave", "OFF"), - format!("OFF for {duration_s} s"), - ) - } else { - let mode = Mode::from_name(&wave) - .ok_or_else(|| context("wave must be OFF, CONST, SINE, or SQUARE"))?; - if mode.optical_target().is_some() { - return Err(context( - "optical warp modes are not available in TOML protocol steps; drive them from the modulation UI", - )); - } - let level = step - .get("level") - .and_then(|value| value.as_integer()) - .ok_or_else(|| context("level is required"))?; - if !(0..=MAX_DAC_CODE).contains(&level) { - return Err(context("level must be between 0 and 4095")); - } - let mut command = Command::new("MOD") - .field("wave", mode.wire_wave()) - .field("level", level); - let summary; - if mode.is_periodic() { - let frequency_hz = step - .get("frequency_hz") - .and_then(|value| value.as_float().or(value.as_integer().map(|v| v as f64))) - .ok_or_else(|| context("frequency_hz is required for SINE/SQUARE"))?; - if !(0.01..=2_000.0).contains(&frequency_hz) { - return Err(context("frequency_hz must be between 0.01 and 2000")); - } - let min = step - .get("min") - .and_then(|value| value.as_integer()) - .unwrap_or(0); - if !(0..=level).contains(&min) { - return Err(context("min must be between 0 and level")); - } - command = command - .field("min", min) - .field("freq_mhz", (frequency_hz * 1_000.0).round() as i64); - summary = format!( - "{} {min}..{level} @ {frequency_hz} Hz for {duration_s} s", - mode.name() - ); - } else { - summary = format!("CONST level={level} for {duration_s} s"); - } - (command, summary) - }; - steps.push(ProtocolStep { - duration: Duration::from_secs_f64(duration_s), - command, - summary, - }); - } - Ok((steps, loops)) -} - -/// Walks the steps on an absolute schedule (no drift accumulation); the last -/// commanded step holds after completion — set-and-hold, like the firmware. -fn run_protocol( - steps: Vec, - loops: usize, - shared: Arc, - stop: Arc, - progress: Arc>, -) { - let mut next_deadline = Instant::now(); - 'run: for loop_index in 1..=loops { - for (step_index, step) in steps.iter().enumerate() { - if stop.load(Ordering::Relaxed) { - break 'run; - } - if let Ok(mut progress) = progress.lock() { - progress.loop_index = loop_index; - progress.step_index = step_index + 1; - progress.summary = step.summary.clone(); - } - *shared.pending.lock().expect("pending lock") = Some(PendingOperation { - commands: vec![step.command.clone()], - purpose: "PROTOCOL", - meta: None, - }); - shared.bump(); - next_deadline += step.duration; - while Instant::now() < next_deadline { - if stop.load(Ordering::Relaxed) { - break 'run; - } - let remaining = next_deadline.saturating_duration_since(Instant::now()); - std::thread::sleep(remaining.min(Duration::from_millis(10))); - } - } - } - if let Ok(mut progress) = progress.lock() { - progress.finished = true; - progress.stopped = stop.load(Ordering::Relaxed); - } - shared.bump(); -} - /// A transfer-curve sweep in flight. One point at a time: command a settled /// `CONST` code, wait for a photodiode window that *starts* after the command, /// record it, move on. @@ -893,7 +771,6 @@ pub struct StageAModulationPlugin { request_cache: VecDeque<(PluginServiceRequest, PluginServiceReply)>, link: Option, shared: Arc, - protocol: Option, // -- settings (every accepted change is sent immediately) -- connect_requested: bool, port_hint: String, @@ -912,13 +789,16 @@ pub struct StageAModulationPlugin { /// The operator's armed `frequency_hz`, parked while a lease drives the /// frequency (A1's frequency sweep) and restored by [`Self::end_lease`]. armed_frequency_hz: Option, + /// The operator's armed `operating_point`, parked while a lease drives it + /// (A1's `I_k` sweep) and restored by [`Self::end_lease`]. + armed_operating_point: Option, /// Dimensionless, floor-subtracted **cycle-mean** lobe coordinate /// `ū ∈ (0,1]`; not the physical A1 flux point `I_k`. operating_point: f64, /// DAC code at the excitation minimum of one monotonic Pockels lobe. v_null_dac: i64, /// DAC code at the excitation maximum of that lobe. An absolute code like - /// `v_null_dac`, not a distance: the quarter wave `Vπ` is derived from the + /// `v_null_dac`, not a distance: the half-wave span is derived from the /// pair (see [`waveform::LobeInversion::resolve`]). v_peak_dac: i64, // -- measured transfer calibration -- @@ -932,6 +812,10 @@ pub struct StageAModulationPlugin { /// Set once a fit has been applied to `v_null_dac`/`v_peak_dac`; published on /// the contract so a consumer's sidecar can cite the inversion in use. calibration_id: Option, + /// Highest [`APPLIED_LOBE`] generation this instance has taken on. Below + /// the published one, its `v_null_dac`/`v_peak_dac` are stale and must not + /// be exported into the settings snapshot. + applied_lobe_generation: u64, /// Directory for the archived calibration record; empty means "apply the /// fit but do not archive it". calibration_dir: String, @@ -940,16 +824,34 @@ pub struct StageAModulationPlugin { /// Momentary calibration buttons, forwarded mirror → worker (ADR 010). press_measure: PressLatch, press_apply: PressLatch, - protocol_path: String, last_error: Option, /// Last automatic reconnect attempt after the device thread died, for the /// watchdog backoff in `apply_execution_context`. last_reconnect_ms: u64, - /// Last `protocol_run` value this instance saw. On the UI mirror this is - /// the operator's request (exported through `get_setting`); everywhere it - /// gates actions to value transitions, because the host re-applies the - /// full settings snapshot on every sync. - protocol_requested: bool, +} + +/// What the configured lobe and DAC ceiling can express right now. Shown to +/// the operator and used to clamp edits instead of refusing them. +#[derive(Debug, Clone, Copy)] +struct Achievable { + /// Highest normalised intensity the DAC ceiling leaves reachable. + u_max: f64, + /// Where the current drive actually peaks, against that ceiling. + peak_u: f64, + /// Deepest `a` at the current operating point. + max_depth_a: f64, + /// Brightest operating point at the current `a`. + max_mean_u: f64, +} + +/// Which of the two coupled optical controls the operator just moved. The one +/// they touched is the one [`StageAModulationPlugin::reconcile_drive`] limits. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DriveKnob { + Depth, + Mean, + /// The lobe, the ceiling or the mode moved, so neither control has priority. + Lobe, } #[derive(Clone)] @@ -977,7 +879,6 @@ impl Default for StageAModulationPlugin { request_cache: VecDeque::new(), link: None, shared: Arc::new(SharedLink::new()), - protocol: None, connect_requested: false, port_hint: "auto".into(), max_level: MAX_DAC_CODE, @@ -989,6 +890,7 @@ impl Default for StageAModulationPlugin { depth_a: 0.5, armed_depth_a: None, armed_frequency_hz: None, + armed_operating_point: None, operating_point: 0.5, v_null_dac: 0, v_peak_dac: 2_048, @@ -996,14 +898,16 @@ impl Default for StageAModulationPlugin { sweep: None, fit: None, calibration_id: None, + // Deliberately zero rather than the current generation: a mirror + // built after a calibration — a plugin reload, say — has to pick + // the applied lobe up, not assume it is already current. + applied_lobe_generation: 0, calibration_dir: String::new(), calibration_status: String::new(), press_measure: PressLatch::default(), press_apply: PressLatch::default(), - protocol_path: String::new(), last_error: None, last_reconnect_ms: 0, - protocol_requested: false, } } } @@ -1069,59 +973,10 @@ impl StageAModulationPlugin { } fn disconnect(&mut self) { - // A protocol without a device to drain its commands is meaningless. - self.protocol = None; self.link = None; // Drop stops and joins the device thread. self.shared.bump(); } - fn protocol_active(&self) -> bool { - self.protocol - .as_ref() - .is_some_and(|run| !run.progress.lock().map(|p| p.finished).unwrap_or(true)) - } - - fn start_protocol(&mut self) -> Result<(), String> { - if self.protocol_active() { - return Ok(()); - } - if self.link.is_none() { - return Err("connect to the controller before running a protocol".into()); - } - if self.protocol_path.trim().is_empty() { - return Err("choose a protocol file first".into()); - } - let text = std::fs::read_to_string(self.protocol_path.trim()) - .map_err(|err| format!("reading {} failed: {err}", self.protocol_path.trim()))?; - let (steps, loops) = parse_protocol(&text)?; - let stop = Arc::new(AtomicBool::new(false)); - let progress = Arc::new(Mutex::new(ProtocolProgress { - loops, - total_steps: steps.len(), - ..ProtocolProgress::default() - })); - let join = std::thread::Builder::new() - .name("stage-a-modulation-protocol".into()) - .spawn({ - let shared = Arc::clone(&self.shared); - let stop = Arc::clone(&stop); - let progress = Arc::clone(&progress); - move || run_protocol(steps, loops, shared, stop, progress) - }) - .expect("spawning the protocol thread must succeed"); - self.protocol = Some(ProtocolRun { - stop, - join: Some(join), - progress, - }); - Ok(()) - } - - fn stop_protocol(&mut self) { - self.protocol = None; // Drop stops and joins; last command holds. - self.shared.bump(); - } - /// The lobe the two configured codes name, or why they name none. /// /// Resolved against the **DAC's** range, not the operator's `max_level` @@ -1143,6 +998,123 @@ impl StageAModulationPlugin { self.resolved_lobe().map(|lobe| lobe.inversion) } + /// Takes on a lobe the live worker applied. Idempotent and cheap: an atomic + /// load unless there is something new to adopt. + /// + /// Only the UI mirror adopts, and only the live worker publishes. That is + /// the direction the problem actually has — the worker is the instance with + /// the photodiode and the fit, the mirror is the one whose stale codes were + /// overwriting it — and it keeps an instance from ever reading back its own + /// publication. + fn adopt_applied_lobe(&mut self) -> bool { + if self.runtime_role != PluginRuntimeRole::UiMirror { + return false; + } + let Some(applied) = applied_lobe_after(self.applied_lobe_generation) else { + return false; + }; + self.v_null_dac = applied.v_null_dac; + self.v_peak_dac = applied.v_peak_dac; + self.calibration_id = Some(applied.calibration_id); + self.applied_lobe_generation = applied.generation; + true + } + + /// The lobe codes to export, without needing `&mut self`. + /// + /// `get_setting` and `settings_schema` are `&self`, and the UI mirror only + /// reaches [`Self::adopt_applied_lobe`] when the operator next edits + /// something. Reading through here means a freshly applied lobe shows up in + /// the panel — and in the snapshot pushed to the worker — on the very next + /// repaint instead of being overwritten by the stale pair. + fn effective_lobe(&self) -> (i64, i64) { + if self.runtime_role != PluginRuntimeRole::UiMirror { + return (self.v_null_dac, self.v_peak_dac); + } + match applied_lobe_after(self.applied_lobe_generation) { + Some(applied) => (applied.v_null_dac, applied.v_peak_dac), + None => (self.v_null_dac, self.v_peak_dac), + } + } + + /// What the current lobe and DAC ceiling can actually produce for the + /// current mode. `None` for the manual method (where the band is entered + /// directly) or when the two endpoint codes name no drivable lobe. + fn achievable(&self) -> Option { + if self.method != DriveMethod::Calibrated { + return None; + } + let law = self.mode.peak_law(); + let inversion = self.lobe_inversion().ok()?; + let u_max = inversion.peak_intensity_ceiling(self.max_level.clamp(0, MAX_DAC_CODE) as f64); + Some(Achievable { + u_max, + peak_u: law.peak(self.operating_point, self.depth_a), + max_depth_a: law.max_depth_for_mean(self.operating_point, u_max), + max_mean_u: law.max_mean_for_depth(self.depth_a, u_max), + }) + } + + /// Brings `a` and `ū` back inside what the bench can actually produce, then + /// re-arms the drive. + /// + /// Nothing here rejects the operator's edit. Refusing a setting and + /// snapping the control back is what made the mode dropdown feel broken: an + /// `a` left over from a different lobe made an optical mode simply + /// unselectable, with no indication of *which* value was in the way. + /// Clamping moves the drive to the nearest thing the lobe can do, and the + /// achievable range is on the status line either way. + /// + /// Only the knob the operator just touched is clamped — dragging `a` up + /// means "more depth", so `a` is what gets limited, and the operating point + /// stays where it was put. A lobe change has no such preference, so it + /// settles the brightness first and then the depth that fits under it. + fn reconcile_drive(&mut self, edited: DriveKnob) { + if self.method == DriveMethod::Calibrated { + if let Ok(inversion) = self.lobe_inversion() { + let law = self.mode.peak_law(); + let u_max = + inversion.peak_intensity_ceiling(self.max_level.clamp(0, MAX_DAC_CODE) as f64); + let clamp_mean = |plugin: &mut Self| { + let max = law + .max_mean_for_depth(plugin.depth_a, u_max) + .max(waveform::MEAN_U_MIN); + plugin.operating_point = + plugin.operating_point.clamp(waveform::MEAN_U_MIN, max); + }; + let clamp_depth = |plugin: &mut Self| { + let max = law + .max_depth_for_mean(plugin.operating_point, u_max) + .max(waveform::DEPTH_A_MIN); + plugin.depth_a = plugin.depth_a.clamp(waveform::DEPTH_A_MIN, max); + }; + match edited { + DriveKnob::Depth => clamp_depth(self), + DriveKnob::Mean => clamp_mean(self), + DriveKnob::Lobe => { + clamp_mean(self); + clamp_depth(self); + } + } + } + } + // An edit made while a lease drives the depth is withheld from the + // board (`send_modulation` is guarded), so it has to land in the parked + // value or it would be lost when the lease ends. + if self.armed_depth_a.is_some() { + self.armed_depth_a = Some(self.depth_a); + } + // Re-judge the drive whether or not it can be sent right now. A stale + // "drive rejected" left over from an earlier combination would + // otherwise outlive the edit that fixed it — and with no link attached + // `send_modulation` never reaches the point where it clears one. + self.last_error = match self.drive_command() { + Ok(_) => None, + Err(error) => Some(format!("drive not sent: {error}")), + }; + self.send_modulation(); + } + /// Resolves the calibrated UI setting (cycle-mean normalized lobe /// coordinate) to the target law's internal pedestal/centre. fn periodic_lobe_point(&self) -> f64 { @@ -1275,7 +1247,7 @@ impl StageAModulationPlugin { internal_u_milli: (drive.operating_point * 1_000.0).round() as u32, depth_a_milli: (self.depth_a * 1_000.0).round() as u32, v_null_dac: u16::try_from(self.v_null_dac).ok()?, - v_pi_dac: u16::try_from(drive.inversion.v_pi_dac.round() as i64).ok()?, + v_peak_dac: u16::try_from(drive.inversion.v_peak_dac().round() as i64).ok()?, }) } @@ -1328,19 +1300,14 @@ impl StageAModulationPlugin { /// newer changes overwrite queued ones (drag coalescing). /// /// Silent while another owner holds the DAC: an automation lease, a - /// calibration sweep, or a running protocol. The host re-applies the + /// calibration sweep. The host re-applies the /// *whole* settings snapshot on every sync, and most handlers here call /// this unconditionally, so without the guard every sync would re-arm the /// operator's drive on top of the code the current owner just commanded — /// the sweep would measure the armed waveform instead of its own - /// staircase, and a protocol step would be overwritten mid-step and held - /// until the next step boundary. + /// staircase. fn send_modulation(&mut self) { - if self.link.is_none() - || self.lease.is_some() - || self.sweep.is_some() - || self.protocol_active() - { + if self.link.is_none() || self.lease.is_some() || self.sweep.is_some() { return; } let command = match self.drive_command() { @@ -1358,21 +1325,6 @@ impl StageAModulationPlugin { }); } - /// Reject a settings update before it can leave the UI showing a drive - /// that was never sent to the board. - fn validate_drive(&mut self) -> Result<(), String> { - match self.drive_command() { - Ok(_) => { - self.last_error = None; - Ok(()) - } - Err(error) => { - self.last_error = Some(format!("drive rejected: {error}")); - Err(error) - } - } - } - /// Builds the DAC warp table for the current optical drive settings. The /// max limit is the hard ceiling; absolute lobe codes cannot be rescaled /// without distorting the target, so an over-limit drive is refused. @@ -1385,7 +1337,7 @@ impl StageAModulationPlugin { if i64::from(peak) > self.max_level { return Err(format!( "optical peak {peak} exceeds the max limit {}; raise the max limit or lower the \ - operating band / a / u / Vπ", + operating band, a or ū", self.max_level )); } @@ -1421,9 +1373,6 @@ impl StageAModulationPlugin { // would interleave silently. return Some("the drive is leased by an automation client".into()); } - if self.protocol_active() { - return Some("a protocol is running".into()); - } None } @@ -1488,8 +1437,18 @@ impl StageAModulationPlugin { }); } - /// One tick of the sweep. `level` is the newest photodiode reading, if any. - fn drive_calibration(&mut self, level: Option) { + /// Settle window in samples for a stream running at `sample_rate_hz`. + /// [`SETTLE_SECONDS`] is the physical quantity; the rate only converts it. + fn settle_samples(sample_rate_hz: Option) -> u64 { + match sample_rate_hz { + Some(rate) if rate > 0 => (f64::from(rate) * SETTLE_SECONDS).round() as u64, + _ => SETTLE_SAMPLES, + } + } + + /// One tick of the sweep. `level` is the newest photodiode reading, if any, + /// and `sample_rate_hz` the stream it was measured on. + fn drive_calibration(&mut self, level: Option, sample_rate_hz: Option) { if self.sweep.is_none() { return; } @@ -1534,7 +1493,7 @@ impl StageAModulationPlugin { }; let window_start = level.end_sample_index.saturating_sub(level.sample_count); - if window_start < commanded_at + SETTLE_SAMPLES { + if window_start < commanded_at + Self::settle_samples(sample_rate_hz) { if self.sweep.as_ref().expect("sweep").point_started.elapsed() > POINT_TIMEOUT { self.finish_calibration_sweep( "sweep aborted: the photodiode stream stalled".into(), @@ -1569,7 +1528,7 @@ impl StageAModulationPlugin { match calibration::fit_transfer(&points, max_code, self.detector_geometry) { Ok(fit) => { let status = format!( - "V_null {:.0} Vπ {:.0} span {:.3} V residual {:.1}%{}{} ({:.1} lobes)", + "V_null {:.0} V_peak {:.0} span {:.3} V residual {:.1}%{}{} ({:.1} lobes)", fit.v_null_dac, fit.v_pi_dac, fit.span_volts.abs(), @@ -1612,7 +1571,7 @@ impl StageAModulationPlugin { if fit.quality > WARN_QUALITY { warnings.push(format!( "residual is {:.1}% of the detector span — check the fit against the points \ - in the transfer-curve plot before trusting Vπ", + in the transfer-curve plot before trusting the fit", fit.quality * 100.0 )); } @@ -1623,18 +1582,35 @@ impl StageAModulationPlugin { fit.points.len() )); } - if let Some(hysteresis) = fit.hysteresis.filter(|value| *value > WARN_HYSTERESIS) { + // Two independently noisy passes over the same curve already differ by + // `1.128 σ` on average, so a raw 5 % cut reports point scatter as cell + // drift on any bench whose points are not far quieter than that. Both + // tests have to pass: the disagreement must be large enough to matter + // *and* systematic rather than scatter. + let hysteresis = fit + .hysteresis + .filter(|value| *value > WARN_HYSTERESIS && fit.hysteresis_is_systematic()); + if let Some(hysteresis) = hysteresis { warnings.push(format!( - "up and down passes differ by {:.1}% of the span — the cell is drifting or \ - the settle time is too short", - hysteresis * 100.0 + "up and down passes differ by {:.1}% of the span, past the {:.1}% the point \ + scatter alone explains — the cell is drifting or the settle time is too short", + hysteresis * 100.0, + fit.hysteresis_noise_floor() * 100.0 )); } let clipped = fit.points.iter().filter(|point| point.clipped).count(); if clipped > 0 { + // Naming what it actually costs. Clipping truncates the reported + // detector extrema (and with them the I_tot lower bound); V_null and + // Vπ come from the *shape*, which a truncated extremum barely moves. + // The old text advised attenuation, which is backwards for the + // reject-port detector — there it is the dark end that reaches the + // bottom rail, and the fix is more gain, not less light. warnings.push(format!( - "{clipped} points clipped the ADC; the extremum they sit on is not where the \ - fit thinks it is — add attenuation and re-measure" + "{clipped} of {} points reached an end of the detector's range — V_null and V_peak \ + are unaffected, but the reported detector extrema (and the I_tot lower bound) \ + are truncated there; change the detector gain if you need them", + fit.points.len() )); } warnings @@ -1649,12 +1625,15 @@ impl StageAModulationPlugin { }; let previous = (self.v_null_dac, self.v_peak_dac); self.v_null_dac = fit.v_null_dac.round().clamp(0.0, MAX_DAC_CODE as f64) as i64; - // The fit reports the quarter wave; the settings hold the peak code the - // operator can see on the plot. + // The fit reports the half-wave span; the settings hold the peak code + // the operator can see on the plot. self.v_peak_dac = fit.v_peak_dac().round().clamp(0.0, MAX_DAC_CODE as f64) as i64; - // The applied lobe must still produce a legal drive; a calibration that - // cannot be armed is not an improvement. - if let Err(error) = self.validate_drive() { + // The lobe itself has to be resolvable — two codes that name no + // monotonic branch are not a calibration. Whether the *drive* fits + // under the current a/ū is not this button's business: those clamp + // themselves to the new lobe rather than blocking the measurement the + // operator just took. + if let Err(error) = self.lobe_inversion() { self.v_null_dac = previous.0; self.v_peak_dac = previous.1; self.calibration_status = format!("not applied: {error}"); @@ -1666,14 +1645,21 @@ impl StageAModulationPlugin { Ok(None) => ", not archived (no calibration folder set)".into(), Err(error) => format!(", archive failed: {error}"), }; - self.calibration_id = Some(calibration_id); + self.calibration_id = Some(calibration_id.clone()); + // Hand the lobe to the UI mirror before anything else can overwrite it + // — see [`APPLIED_LOBE`]. Only the live worker ever gets this far: it + // is the instance the sweep and the fit live on. + if self.runtime_role == PluginRuntimeRole::LiveWorker { + self.applied_lobe_generation = + publish_applied_lobe(self.v_null_dac, self.v_peak_dac, calibration_id); + } self.calibration_status = format!( - "applied V_null {} / V_peak {} (Vπ {} codes){archived}", + "applied V_null {} / V_peak {} (span {} codes){archived}", self.v_null_dac, self.v_peak_dac, self.v_peak_dac - self.v_null_dac ); - self.send_modulation(); + self.reconcile_drive(DriveKnob::Lobe); self.shared.bump(); } @@ -1697,7 +1683,8 @@ impl StageAModulationPlugin { "max_level": self.max_level, "detector_geometry": fit.geometry.name(), "v_null_dac": fit.v_null_dac, - "v_pi_dac": fit.v_pi_dac, + "v_peak_dac": fit.v_peak_dac(), + "half_wave_span_dac": fit.v_pi_dac, "detector_volts_at_null": fit.detector_volts_at_null(), "detector_volts_at_peak": fit.detector_volts_at_peak(), "span_volts": fit.span_volts, @@ -1934,7 +1921,6 @@ impl StageAModulationPlugin { )); } } - self.protocol = None; self.lease = Some(ControlLease { lease_id, holder: request.requester.clone(), @@ -2006,7 +1992,6 @@ impl StageAModulationPlugin { let mut target = self.base_target(revision); target.waveform = Some(WaveformV1::Off); target.acquisition_running = false; - self.protocol = None; self.shared .fail_closed_on_stop .store(true, Ordering::Relaxed); @@ -2147,6 +2132,66 @@ impl StageAModulationPlugin { self.shared.bump(); self.immediate_response(request, RequestOutcomeV1::Applied, None) } + ModulationCommandV1::SetOperatingPoint { mean_u_milli } => { + self.require_lease(request)?; + if self.link.is_none() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "the modulation owner is not connected to the device", + false, + )); + } + let mean_u = f64::from(*mean_u_milli) / 1_000.0; + if !(waveform::MEAN_U_MIN..=1.0).contains(&mean_u) { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + format!( + "operating point ū={mean_u:.3} outside the supported {:.2}..=1.0", + waveform::MEAN_U_MIN + ), + false, + )); + } + // `ū` only means anything against a measured lobe: on the + // manual method the band is the operator's two DAC codes and + // there is no normalized coordinate to retarget. + if self.method != DriveMethod::Calibrated { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "arm the calibrated drive method in the modulation plugin before \ + sweeping the operating point", + false, + )); + } + let previous = self.operating_point; + self.operating_point = mean_u; + let command = match self.drive_command() { + Ok(command) => command, + Err(error) => { + // Unlike an interactive edit this is *not* clamped: a + // protocol asked for a specific brightness, and quietly + // recording a different one would put the wrong `ū` in + // every sidecar of that block. + self.operating_point = previous; + return Err(service_error( + ServiceErrorCodeV1::DeviceRejected, + format!("operating point ū={mean_u:.3} rejected: {error}"), + false, + )); + } + }; + // As for depth and frequency: park the operator's own value on + // the first retarget only, so `end_lease` hands back what they + // armed rather than the sweep's last point. + self.armed_operating_point.get_or_insert(previous); + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); + self.shared.bump(); + self.immediate_response(request, RequestOutcomeV1::Applied, None) + } ModulationCommandV1::PrepareA1 { configuration } => { self.require_lease(request)?; let revision = self.requested_revision(request)?; @@ -2306,13 +2351,17 @@ impl StageAModulationPlugin { self.lease = None; let depth = self.armed_depth_a.take(); let frequency = self.armed_frequency_hz.take(); + let operating_point = self.armed_operating_point.take(); if let Some(depth) = depth { self.depth_a = depth; } if let Some(frequency) = frequency { self.frequency_hz = frequency; } - if depth.is_some() || frequency.is_some() { + if let Some(operating_point) = operating_point { + self.operating_point = operating_point; + } + if depth.is_some() || frequency.is_some() || operating_point.is_some() { // Re-arm the board only if nobody else now owns the DAC; // `send_modulation` is itself guarded. self.send_modulation(); @@ -2327,7 +2376,6 @@ impl StageAModulationPlugin { if !expired { return; } - self.protocol = None; self.shared .fail_closed_on_stop .store(true, Ordering::Relaxed); @@ -2894,8 +2942,9 @@ impl Plugin for StageAModulationPlugin { .and_then(|snapshot| { serde_json::from_value::(snapshot.payload.clone()).ok() }) - .and_then(|summary| summary.stream.level); - self.drive_calibration(level); + .map(|summary| (summary.stream.level, summary.stream.sample_rate_hz)); + let (level, sample_rate_hz) = level.unwrap_or((None, None)); + self.drive_calibration(level, sample_rate_hz); } fn handle_service_request( @@ -3082,7 +3131,7 @@ impl Plugin for StageAModulationPlugin { label: "Drive method".into(), tooltip: Some( "MANUAL defines the DAC band with Power and Min threshold. CALIBRATED \ - derives it from V_null, Vπ, normalized u, and optical depth a." + derives it from V_null, V_peak, the mean lobe point ū and the depth a." .into(), ), kind: SettingKind::Enum { @@ -3148,47 +3197,60 @@ impl Plugin for StageAModulationPlugin { }); } DriveMethod::Calibrated => { + let (v_null, v_peak) = self.effective_lobe(); + let range = self.achievable(); modulation_items.push(SettingItem { key: "v_null_dac".into(), label: "V_null (DAC code at MIN light)".into(), tooltip: Some( - "DAC code where excitation light bottoms out (sin² = 0) on one \ - monotonic Pockels lobe. Measure it; do not trust nominal Vπ." + "DAC code where the excitation light bottoms out (sin² = 0) on one \ + monotonic Pockels lobe. Read it off a sweep — Measure transfer curve \ + below fills both codes in for you." .into(), ), kind: SettingKind::I64Drag { min: 0, max: MAX_DAC_CODE, - default: self.v_null_dac, + default: v_null, }, }); modulation_items.push(SettingItem { key: "v_peak_dac".into(), label: "V_peak (DAC code at MAX light)".into(), tooltip: Some( - "DAC code where excitation light is brightest, on the same lobe as \ - V_null. Both fields are codes you read off a sweep — the quarter \ - wave Vπ = |V_peak − V_null| is derived, never typed. u = 1 holds \ + "DAC code where the excitation light is brightest, on the same lobe as \ + V_null. Both fields are codes you can point at on the transfer curve; \ + the half-wave span between them is derived, never typed. u = 1 holds \ exactly here and u = 0 at V_null." .into(), ), kind: SettingKind::I64Drag { min: 0, max: MAX_DAC_CODE, - default: self.v_peak_dac, + default: v_peak, }, }); modulation_items.push(SettingItem { key: "operating_point".into(), - label: "Normalized mean lobe point ū (0..1)".into(), + label: match range { + Some(range) => { + format!( + "Mean lobe point ū (0..{:.2} at a={:.2})", + range.max_mean_u, self.depth_a + ) + } + None => "Mean lobe point ū (0..1)".into(), + }, tooltip: Some( - "Dimensionless floor-subtracted cycle mean, not physical A1 flux I_k. \ - OPTICAL_LOG_SINE converts it to u_g = ū/I₀(a/2), so sweeping a keeps \ - the normalized mean fixed; OPTICAL_LINEAR_SINE uses it as its centre." + "How bright the light sits on average, as a fraction of the lobe's \ + maximum. Dimensionless — not the physical flux I_k.\n\n\ + The label shows how far up you can go at the depth a currently set; \ + past that the peak of the swing would run off the top of the lobe. \ + Dragging beyond it stops at the limit instead of refusing the edit." .into(), ), kind: SettingKind::F64Drag { - min: 0.01, + min: waveform::MEAN_U_MIN, max: 1.0, speed: 0.01, default: self.operating_point, @@ -3196,15 +3258,23 @@ impl Plugin for StageAModulationPlugin { }); modulation_items.push(SettingItem { key: "depth_a".into(), - label: "Optical depth a".into(), + label: match range { + Some(range) => format!( + "Optical depth a (0..{:.2} at ū={:.2})", + range.max_depth_a, self.operating_point + ), + None => "Optical depth a".into(), + }, tooltip: Some( - "Peak-to-trough natural-log contrast a = ln(I_max/I_min). Together with \ - the normalized lobe point u it defines the requested optical band." + "Peak-to-trough log contrast a = ln(I_max/I_min) of the light.\n\n\ + The label shows the deepest a the lobe can reach at the mean point \ + currently set — lower ū to get more depth. Dragging beyond it stops at \ + the limit instead of refusing the edit." .into(), ), kind: SettingKind::F64Drag { - min: 0.01, - max: 6.0, + min: waveform::DEPTH_A_MIN, + max: waveform::DEPTH_A_MAX, speed: 0.01, default: self.depth_a, }, @@ -3237,7 +3307,7 @@ impl Plugin for StageAModulationPlugin { label: "Calibration".into(), description: Some( "Measures the Pockels/PBS transfer curve: steps settled CONST DAC codes \ - across the range while reading the photodiode, then fits V_null and Vπ. \ + across the range while reading the photodiode, then fits the lobe. \ Needs the photodiode plugin connected. The sweep restores your armed \ drive when it finishes, and the fit is never applied without your \ confirmation. Watch the transfer-curve view." @@ -3282,10 +3352,11 @@ impl Plugin for StageAModulationPlugin { key: "calibrate_apply".into(), label: "Apply to V_null / V_peak".into(), tooltip: Some( - "Writes the reviewed fit into the two observed endpoint settings. \ - Residual, dropped points, hysteresis, and clipping remain visible \ - as warnings; applying only refuses a lobe that cannot produce a \ - legal drive." + "Writes the measured lobe into V_null and V_peak above. Residual, \ + dropped points, hysteresis and clipping stay visible as warnings \ + — applying only refuses a pair that names no monotonic lobe at \ + all. If the depth or mean point no longer fit the new lobe they \ + move to its nearest reachable value rather than blocking this." .into(), ), kind: SettingKind::Button { @@ -3296,8 +3367,14 @@ impl Plugin for StageAModulationPlugin { key: "calibration_dir".into(), label: "Calibration folder (optional)".into(), tooltip: Some( - "Where the applied calibration record is archived, with its \ - points and fit. Leave empty to apply without archiving." + "Where each applied calibration is archived, as one named JSON \ + file per apply. It holds every swept point (code, direction, \ + volts, whether it clipped), the fitted V_null and V_peak, the \ + residual and hysteresis, and the detector reading at the \ + excitation null — which is the total power I_tot the photodiode \ + plugin measures its depth against. That makes an old run's \ + inversion reproducible after the bench has been touched.\n\n\ + Leave it empty to apply the fit without keeping a record." .into(), ), kind: SettingKind::Path { @@ -3307,46 +3384,6 @@ impl Plugin for StageAModulationPlugin { }, ], }, - SettingsSection { - label: "Protocol".into(), - description: Some( - "Timed sequence of MOD steps from a TOML file: `loops = N` plus \ - [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, \ - min, frequency_hz. Steps run on an absolute schedule; the last \ - step holds after completion (set-and-hold). Stopping never \ - switches the output off by itself." - .into(), - ), - default_open: false, - items: vec![ - SettingItem { - key: "protocol_path".into(), - label: "Protocol file".into(), - tooltip: Some("TOML protocol file (validated on start).".into()), - kind: SettingKind::Path { - dialog: PathDialogKind::OpenFile, - default: self.protocol_path.clone(), - }, - }, - SettingItem { - key: "protocol_run".into(), - label: "Run protocol".into(), - tooltip: Some( - "Start/stop the loaded protocol. Requires an open connection; \ - manual drive controls stay live and override the current step \ - until the next one begins." - .into(), - ), - kind: SettingKind::Bool { - default: if self.runtime_role == PluginRuntimeRole::LiveWorker { - self.protocol_active() - } else { - self.protocol_requested - }, - }, - }, - ], - }, ], } } @@ -3383,8 +3420,11 @@ impl Plugin for StageAModulationPlugin { "min_level" => Some(json!(self.min_level)), "depth_a" => Some(json!(self.depth_a)), "operating_point" => Some(json!(self.operating_point)), - "v_null_dac" => Some(json!(self.v_null_dac)), - "v_peak_dac" => Some(json!(self.v_peak_dac)), + // Through `effective_lobe`, so a lobe another instance just + // applied is what goes into the settings snapshot — not the stale + // pair that would overwrite it (see APPLIED_LOBE). + "v_null_dac" => Some(json!(self.effective_lobe().0)), + "v_peak_dac" => Some(json!(self.effective_lobe().1)), "detector_geometry" => { let index = calibration::DetectorGeometry::VARIANTS .iter() @@ -3398,22 +3438,19 @@ impl Plugin for StageAModulationPlugin { // snapshot (ADR 010). "calibrate" => Some(self.press_measure.value()), "calibrate_apply" => Some(self.press_apply.value()), - "protocol_path" => Some(json!(self.protocol_path)), // The live worker reports the actual run state; the UI mirror // reports the operator's request so the settings snapshot can // transport the start to the worker (which owns the device link). - "protocol_run" => Some(json!( - if self.runtime_role == PluginRuntimeRole::LiveWorker { - self.protocol_active() - } else { - self.protocol_requested - } - )), _ => None, } } fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + // Catch up on a lobe applied elsewhere in this process before reading + // any of it, so an incoming echo of the old codes cannot land on top. + if self.adopt_applied_lobe() { + self.reconcile_drive(DriveKnob::Lobe); + } if self.lease.is_some() { return Err(format!( "setting '{key}' is locked while automation holds the modulation lease" @@ -3459,7 +3496,9 @@ impl Plugin for StageAModulationPlugin { if self.min_level > self.max_level { self.min_level = self.max_level; } - self.send_modulation(); + // The ceiling caps the reachable intensity, so it moves the + // optical range too. + self.reconcile_drive(DriveKnob::Lobe); Ok(()) } "method" => { @@ -3468,30 +3507,22 @@ impl Plugin for StageAModulationPlugin { .map(|method| method.name().to_owned()) .collect(); let name = enum_choice(&value, &method_names)?; - let method = DriveMethod::from_name(&name) + self.method = DriveMethod::from_name(&name) .ok_or_else(|| format!("unknown drive method: {name}"))?; - let previous = self.method; - self.method = method; - if let Err(error) = self.validate_drive() { - self.method = previous; - return Err(error); - } - self.last_error = None; - self.send_modulation(); + self.reconcile_drive(DriveKnob::Lobe); Ok(()) } "mode" => { let mode_names: Vec = Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); let name = enum_choice(&value, &mode_names)?; - let mode = Mode::from_name(&name).ok_or_else(|| format!("unknown mode: {name}"))?; - let previous = self.mode; - self.mode = mode; - if let Err(error) = self.validate_drive() { - self.mode = previous; - return Err(error); - } - self.send_modulation(); + // Always accepted. Selecting a mode whose parameters do not + // fit yet used to snap the dropdown back with an error about a + // value the operator was not editing; now the mode takes and + // the parameters follow it. + self.mode = + Mode::from_name(&name).ok_or_else(|| format!("unknown mode: {name}"))?; + self.reconcile_drive(DriveKnob::Lobe); Ok(()) } "frequency_hz" => { @@ -3516,55 +3547,25 @@ impl Plugin for StageAModulationPlugin { let depth_a = value .as_f64() .ok_or("depth_a must be a number")? - .clamp(0.01, 6.0); - let previous = self.depth_a; + .clamp(waveform::DEPTH_A_MIN, waveform::DEPTH_A_MAX); self.depth_a = depth_a; - if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { - if let Err(error) = self.validate_drive() { - self.depth_a = previous; - return Err(error); - } - self.send_modulation(); - } - // An edit made while a lease drives the depth is withheld from - // the board (`send_modulation` is guarded), so it has to land - // in the parked value or it would be lost when the lease ends - // — same rule the calibration sweep follows. - if self.armed_depth_a.is_some() { - self.armed_depth_a = Some(self.depth_a); - } + self.reconcile_drive(DriveKnob::Depth); Ok(()) } "operating_point" => { - let operating_point = value + self.operating_point = value .as_f64() .ok_or("operating_point must be a number")? - .clamp(0.01, 1.0); - let previous = self.operating_point; - self.operating_point = operating_point; - if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { - if let Err(error) = self.validate_drive() { - self.operating_point = previous; - return Err(error); - } - self.send_modulation(); - } + .clamp(waveform::MEAN_U_MIN, 1.0); + self.reconcile_drive(DriveKnob::Mean); Ok(()) } "v_null_dac" => { - let v_null_dac = value + self.v_null_dac = value .as_i64() .ok_or("v_null_dac must be an integer")? .clamp(0, MAX_DAC_CODE); - let previous = self.v_null_dac; - self.v_null_dac = v_null_dac; - if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { - if let Err(error) = self.validate_drive() { - self.v_null_dac = previous; - return Err(error); - } - self.send_modulation(); - } + self.reconcile_drive(DriveKnob::Lobe); Ok(()) } // `v_pi_dac` is the pre-endpoint key: a *distance* from V_null. Kept @@ -3581,15 +3582,8 @@ impl Plugin for StageAModulationPlugin { } else { entered }; - let previous = self.v_peak_dac; self.v_peak_dac = v_peak_dac; - if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { - if let Err(error) = self.validate_drive() { - self.v_peak_dac = previous; - return Err(error); - } - self.send_modulation(); - } + self.reconcile_drive(DriveKnob::Lobe); Ok(()) } "detector_geometry" => { @@ -3640,39 +3634,6 @@ impl Plugin for StageAModulationPlugin { self.apply_calibration_fit(); Ok(()) } - "protocol_path" => { - self.protocol_path = value - .as_str() - .ok_or("protocol_path must be a string")? - .to_owned(); - Ok(()) - } - "protocol_run" => { - let requested = value.as_bool().ok_or("protocol_run must be a boolean")?; - // The host re-applies the full settings snapshot on every - // sync, so only value *transitions* are actions — otherwise a - // finished protocol would silently restart on the next sync. - if requested == self.protocol_requested { - return Ok(()); - } - self.protocol_requested = requested; - if requested { - // Only the live worker owns the device link; the UI mirror - // records the request and the settings snapshot starts the - // protocol on the worker. Failures surface through status - // entries (like `connect`). - if self.runtime_role == PluginRuntimeRole::LiveWorker { - match self.start_protocol() { - Ok(()) => self.last_error = None, - Err(err) => self.last_error = Some(err), - } - } - } else { - self.stop_protocol(); - } - self.shared.bump(); - Ok(()) - } _ => Err(format!("unknown setting: {key}")), } } @@ -3707,7 +3668,7 @@ impl Plugin for StageAModulationPlugin { Ok(lobe) => { let inversion = lobe.inversion; format!( - "Lobe: Vπ = {:.0} codes — u 0 → {:.0} (min light), 0.5 → {:.0}, \ + "Lobe: half-wave span {:.0} codes — u 0 → {:.0} (min light), 0.5 → {:.0}, \ 1 → {:.0} (max light){}", inversion.v_pi_dac, inversion.dac_for_u(0.0), @@ -3751,29 +3712,25 @@ impl Plugin for StageAModulationPlugin { self.frequency_hz, ))); } - (Err(error), _) | (_, Err(error)) => { - entries.push(StatusEntry::Text(format!("Optical drive invalid: {error}"))) - } + (Err(error), _) | (_, Err(error)) => entries.push(StatusEntry::Text(format!( + "Optical drive not sent: {error}" + ))), } - } - if let Some(run) = &self.protocol { - if let Ok(progress) = run.progress.lock() { - entries.push(StatusEntry::Text(if progress.finished { - if progress.stopped { - "Protocol: stopped (last step holds)".into() - } else { - "Protocol: finished (last step holds)".into() - } - } else { - format!( - "Protocol: loop {}/{} step {}/{} — {}", - progress.loop_index, - progress.loops, - progress.step_index, - progress.total_steps, - progress.summary - ) - })); + // The two optical controls are coupled through one ceiling, so the + // useful readout is not "that value is invalid" but where the + // boundary actually is. Edits clamp against exactly these numbers. + if let Some(range) = self.achievable() { + entries.push(StatusEntry::Text(format!( + "Achievable now: a ≤ {:.2} at ū={:.2} · ū ≤ {:.2} at a={:.2} \ + (peaks at u={:.2} of {:.2}; max limit {})", + range.max_depth_a, + self.operating_point, + range.max_mean_u, + self.depth_a, + range.peak_u, + range.u_max, + self.max_level, + ))); } } if let Some(sweep) = self.sweep.as_ref() { @@ -4021,123 +3978,6 @@ mod tests { plugin.set_setting("connect", json!(false)).unwrap(); } - const TEST_PROTOCOL: &str = r#" -loops = 2 - -[[steps]] -duration_s = 0.03 -wave = "SINE" -level = 2000 -min = 100 -frequency_hz = 100.0 - -[[steps]] -duration_s = 0.03 -wave = "CONST" -level = 750 -"#; - - #[test] - fn protocol_parsing_validates_steps() { - let (steps, loops) = parse_protocol(TEST_PROTOCOL).expect("valid protocol"); - assert_eq!(loops, 2); - assert_eq!(steps.len(), 2); - let encoded = |command: &Command, seq: u32| { - String::from_utf8(command.encode(seq).expect("encodes")).expect("utf8") - }; - assert_eq!( - encoded(&steps[0].command, 1), - "@1 MOD wave=SINE level=2000 min=100 freq_mhz=100000\n" - ); - assert_eq!( - encoded(&steps[1].command, 2), - "@2 MOD wave=CONST level=750\n" - ); - assert!((steps[0].duration.as_secs_f64() - 0.03).abs() < 1e-9); - - assert!(parse_protocol("loops = 1").is_err(), "steps required"); - assert!( - parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100").is_err(), - "periodic steps need a frequency" - ); - assert!( - parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"CONST\"\nlevel = 9999").is_err(), - "level range enforced" - ); - assert!( - parse_protocol( - "[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100\nmin = 200\nfrequency_hz = 10.0" - ) - .is_err(), - "min above level rejected" - ); - let (off, _) = parse_protocol("[[steps]]\nduration_s = 0.5\nwave = \"OFF\"") - .expect("OFF needs no level"); - assert_eq!(encoded(&off[0].command, 1), "@1 MOD wave=OFF\n"); - } - - /// A protocol against the mock walks every step, holds the last one, and - /// reports finished. - #[test] - fn protocol_runs_to_completion_on_the_mock() { - let dir = std::env::temp_dir().join(format!( - "stage-a-modulation-protocol-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join("protocol.toml"); - std::fs::write(&path, TEST_PROTOCOL).unwrap(); - - let mut plugin = live_plugin(); - plugin.set_setting("port", json!("mock")).unwrap(); - plugin.set_setting("connect", json!(true)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); - - plugin - .set_setting("protocol_path", json!(path.display().to_string())) - .unwrap(); - plugin.set_setting("protocol_run", json!(true)).unwrap(); - assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); - assert_eq!(plugin.get_setting("protocol_run"), Some(json!(true))); - - // 2 loops × 2 steps × 30 ms ≈ 120 ms; wait for the final CONST 750. - wait_until(&plugin, Duration::from_secs(3), |p| { - !p.protocol_active() && board_code(p) == Some(750) - }); - assert!(!plugin.protocol_active()); - assert_eq!(board_code(&plugin), Some(750), "last step holds"); - let progress = plugin - .protocol - .as_ref() - .unwrap() - .progress - .lock() - .unwrap() - .clone(); - assert!(progress.finished && !progress.stopped); - assert_eq!((progress.loop_index, progress.step_index), (2, 2)); - - plugin.set_setting("connect", json!(false)).unwrap(); - std::fs::remove_dir_all(dir).unwrap(); - } - - #[test] - fn protocol_requires_a_connection() { - let mut plugin = live_plugin(); - plugin - .set_setting("protocol_path", json!("/tmp/x.toml")) - .unwrap(); - plugin.set_setting("protocol_run", json!(true)).unwrap(); - assert!(plugin - .last_error - .as_deref() - .is_some_and(|err| err.contains("connect"))); - assert_eq!(plugin.get_setting("protocol_run"), Some(json!(false))); - } - /// The host settings UI exchanges enum values as indices into the /// schema's variant list (radio buttons send `json!(index)`). #[test] @@ -4435,14 +4275,17 @@ level = 750 .sin() .powi(2); sample_index += SETTLE_SAMPLES; - plugin.drive_calibration(Some(PhotodiodeLevelV1 { - // Reject port: brightest at the excitation null. - mean_volts: 2.4 - 2.2 * u, - peak_to_peak_volts: 0.001, - sample_count: SETTLE_SAMPLES, - end_sample_index: sample_index, - clipped: false, - })); + plugin.drive_calibration( + Some(PhotodiodeLevelV1 { + // Reject port: brightest at the excitation null. + mean_volts: 2.4 - 2.2 * u, + peak_to_peak_volts: 0.001, + sample_count: SETTLE_SAMPLES, + end_sample_index: sample_index, + clipped: false, + }), + None, + ); } } @@ -4489,6 +4332,26 @@ level = 750 assert!((plugin.v_peak_dac - plugin.v_null_dac - 1_600).abs() <= 10); assert!(plugin.calibration_id.is_some()); assert!(plugin.control_state().calibration_id.is_some()); + + // And the UI mirror — the instance the host actually collects the + // settings snapshot from — reports the applied lobe rather than its own + // stale codes. Without this the next sync wrote the old pair straight + // back onto the worker and the button looked like it did nothing. + let mut mirror = StageAModulationPlugin::default(); + mirror.set_runtime_role(PluginRuntimeRole::UiMirror); + assert_eq!( + mirror.get_setting("v_null_dac"), + Some(json!(plugin.v_null_dac)), + "the mirror kept exporting a pre-calibration V_null" + ); + assert_eq!( + mirror.get_setting("v_peak_dac"), + Some(json!(plugin.v_peak_dac)) + ); + // Adoption is one-way and settles: once taken on, the mirror is free to + // be edited again without the applied lobe snapping back. + mirror.set_setting("v_null_dac", json!(111)).unwrap(); + assert_eq!(mirror.get_setting("v_null_dac"), Some(json!(111))); } /// The host re-applies the **whole** settings snapshot on every sync, and @@ -4563,13 +4426,16 @@ level = 750 .sin() .powi(2); sample_index += SETTLE_SAMPLES; - plugin.drive_calibration(Some(PhotodiodeLevelV1 { - mean_volts: 2.4 - 2.2 * u, - peak_to_peak_volts: 0.001, - sample_count: SETTLE_SAMPLES, - end_sample_index: sample_index, - clipped: false, - })); + plugin.drive_calibration( + Some(PhotodiodeLevelV1 { + mean_volts: 2.4 - 2.2 * u, + peak_to_peak_volts: 0.001, + sample_count: SETTLE_SAMPLES, + end_sample_index: sample_index, + clipped: false, + }), + None, + ); } let fit = plugin @@ -4605,16 +4471,16 @@ level = 750 clipped: false, }; // First tick commands the point and adopts the sample index. - plugin.drive_calibration(Some(stale(10_000))); + plugin.drive_calibration(Some(stale(10_000)), None); assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 0); // A window that began before the command must not be accepted, however // many times it arrives — this is what makes settling provable. for _ in 0..5 { - plugin.drive_calibration(Some(stale(10_050))); + plugin.drive_calibration(Some(stale(10_050)), None); } assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 0); // Once the window starts past the settle margin the point is taken. - plugin.drive_calibration(Some(stale(10_000 + SETTLE_SAMPLES + 100))); + plugin.drive_calibration(Some(stale(10_000 + SETTLE_SAMPLES + 100)), None); assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 1); } @@ -4720,7 +4586,18 @@ level = 750 })); let warnings = plugin.fit_warnings().join(" | "); - assert!(warnings.contains("clipped"), "{warnings}"); + assert!( + warnings.contains("end of the detector's range"), + "{warnings}" + ); + // Rail-touching points are a caveat on the reported extrema, not a + // verdict on the lobe, and the advice must not send the operator to add + // attenuation when the detector is against its *bottom* rail. + assert!( + warnings.contains("V_null and V_peak are unaffected"), + "{warnings}" + ); + assert!(!warnings.contains("attenuation"), "{warnings}"); plugin.set_setting("calibrate_apply", json!(true)).unwrap(); assert!( @@ -4735,6 +4612,71 @@ level = 750 ); } + /// Noisy points must not be reported as a drifting cell. + /// + /// Two independent passes over one curve already differ by `1.128 σ` on + /// average, so a bare 5 % cut fires on any bench whose points are not far + /// quieter than that — and it did, on a real recorded sweep whose cell was + /// not drifting at all. + #[test] + fn hysteresis_at_the_noise_floor_is_not_reported_as_drift() { + // A two-pass sweep of one 90 mV lobe, scattered by an alternating ±4 mV + // that is noise and nothing else. The raw metric reads well past + // WARN_HYSTERESIS; the passes still disagree by nothing but their own + // noise, so there is no drift to report. + let two_pass = |offset_descending: f64| -> calibration::TransferFit { + let points: Vec = calibration::sweep_codes(4_095, 49, true) + .into_iter() + .map(|(code, direction)| { + let u = (std::f64::consts::PI * (f64::from(code) - 300.0) / 1_720.0) + .sin() + .powi(2); + let scatter = 0.012 * calibration::scatter(code, direction); + let drift = if direction == calibration::Direction::Descending { + offset_descending + } else { + 0.0 + }; + calibration::SweepPoint { + code, + direction, + volts: 0.098 - 0.090 * u + scatter + drift, + peak_to_peak_volts: 0.001, + clipped: false, + } + }) + .collect(); + calibration::fit_transfer( + &points, + 4_095.0, + calibration::DetectorGeometry::RejectedComplement, + ) + .expect("fits") + }; + + let mut plugin = live_plugin(); + let scattered = two_pass(0.0); + assert!( + scattered.hysteresis.expect("both directions") > WARN_HYSTERESIS, + "the raw metric must be past the bare threshold for this test to mean anything" + ); + plugin.fit = Some(scattered); + assert!( + !plugin.fit_warnings().join(" | ").contains("drifting"), + "{}", + plugin.fit_warnings().join(" | ") + ); + + // A systematic offset between the passes is real drift and must still + // be called out. + plugin.fit = Some(two_pass(-0.030)); + assert!( + plugin.fit_warnings().join(" | ").contains("drifting"), + "{}", + plugin.fit_warnings().join(" | ") + ); + } + fn calibration_button(plugin: &StageAModulationPlugin, key: &str) -> SettingKind { plugin .settings_schema() @@ -4916,7 +4858,7 @@ level = 750 }) .collect::>() .join(" | "); - assert!(status.contains("Vπ = 860 codes"), "{status}"); + assert!(status.contains("half-wave span 860 codes"), "{status}"); assert!(status.contains("0 → 1630"), "{status}"); assert!(status.contains("1 → 2490"), "{status}"); @@ -5012,12 +4954,17 @@ level = 750 ); assert_eq!(drive.depth_a_milli, 1_000); assert_eq!(drive.v_null_dac, 400); - assert_eq!(drive.v_pi_dac, 900); + assert_eq!(drive.v_peak_dac, 1_300); assert_eq!(state.calibration_id.as_deref(), Some("cal-test")); } #[test] - fn rejected_operating_point_does_not_diverge_from_the_board_target() { + fn an_out_of_range_operating_point_is_clamped_not_rejected() { + // Refusing the edit and snapping the control back is what made these + // two coupled controls feel broken: the operator was told a value they + // were not editing was wrong, with no indication of where the boundary + // is. The edit lands on the boundary instead, and the drive stays + // arm-able the whole time. let mut plugin = live_plugin(); plugin.method = DriveMethod::Calibrated; plugin.mode = Mode::Sine; @@ -5026,12 +4973,21 @@ level = 750 plugin.depth_a = 0.5; plugin.operating_point = 0.5; - let error = plugin + plugin .set_setting("operating_point", json!(1.0)) - .expect_err("periodic u=1 has no modulation headroom"); - assert!(error.contains("lobe ceiling")); - assert_eq!(plugin.operating_point, 0.5); + .expect("an out-of-range operating point is still accepted"); + let range = plugin.achievable().expect("calibrated range"); + assert!( + (plugin.operating_point - range.max_mean_u).abs() < 1e-9, + "ū landed on {} rather than the boundary {}", + plugin.operating_point, + range.max_mean_u + ); + // Clamped, so the drive is buildable — no stale rejection left behind. + assert!(plugin.drive_command().is_ok()); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + // CONST modulates nothing, so the full lobe is reachable again. plugin.set_setting("mode", json!(0)).expect("CONST"); plugin .set_setting("operating_point", json!(1.0)) @@ -5039,6 +4995,33 @@ level = 750 assert_eq!(plugin.dac_band().unwrap(), (2_490, 2_490, 2_490)); } + #[test] + fn every_mode_stays_selectable_whatever_the_parameters_are() { + // The reported bug: with a leftover `a` from another lobe, selecting an + // optical mode snapped the dropdown back. A mode is a statement of + // intent — it always takes, and the parameters follow it. + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.v_null_dac = 1_630; + plugin.v_peak_dac = 2_490; + plugin.operating_point = 1.0; + plugin.depth_a = 6.0; + + for index in 0..Mode::VARIANTS.len() { + plugin + .set_setting("mode", json!(index)) + .unwrap_or_else(|error| panic!("mode {index} refused: {error}")); + assert_eq!(plugin.mode, Mode::VARIANTS[index]); + assert!( + plugin.drive_command().is_ok(), + "{} left an unbuildable drive: a={} ū={}", + plugin.mode.name(), + plugin.depth_a, + plugin.operating_point + ); + } + } + #[test] fn calibrated_const_sends_the_expected_codes_to_the_board() { let mut plugin = live_plugin(); @@ -5160,43 +5143,6 @@ level = 750 plugin.disconnect(); } - #[test] - fn a_running_protocol_owns_the_pending_slot() { - // The host re-applies the whole settings snapshot on every sync. An - // unguarded `send_modulation` would drop the operator's armed drive - // into the slot the protocol step is queued in, and the board would - // hold it until the next step boundary. - let mut plugin = live_plugin(); - plugin.port_hint = "mock".into(); - plugin.connect_requested = true; - plugin.connect(); - wait_until(&plugin, Duration::from_secs(2), |owner| { - owner.device_connected() - }); - - let progress = Arc::new(Mutex::new(ProtocolProgress::default())); - plugin.protocol = Some(ProtocolRun { - progress: Arc::clone(&progress), - stop: Arc::new(AtomicBool::new(false)), - join: None, - }); - assert!(plugin.protocol_active()); - - *plugin.shared.pending.lock().unwrap() = None; - plugin.set_setting("max_level", json!(3_000)).expect("set"); - assert!( - plugin.shared.pending.lock().unwrap().is_none(), - "a settings sync overwrote the protocol's pending slot" - ); - - // Once the protocol finishes, the operator's drive gets through again. - progress.lock().unwrap().finished = true; - assert!(!plugin.protocol_active()); - plugin.set_setting("max_level", json!(3_100)).expect("set"); - assert!(plugin.shared.pending.lock().unwrap().is_some()); - plugin.disconnect(); - } - #[test] fn a_decimal_frequency_echo_still_yields_millihertz() { // Firmware echoing "10000.0" used to parse as u64 -> None, which diff --git a/plugins/stage-a-modulation/src/waveform.rs b/plugins/stage-a-modulation/src/waveform.rs index ca6decc..06d9b7e 100644 --- a/plugins/stage-a-modulation/src/waveform.rs +++ b/plugins/stage-a-modulation/src/waveform.rs @@ -32,6 +32,14 @@ pub const WARP_TABLE_LEN: usize = 256; /// Full-scale DAC code (12-bit). pub const DAC_FULL_SCALE: u16 = 4_095; +/// Shallowest optical depth the UI offers. Below this the warp table is +/// indistinguishable from a constant drive. +pub const DEPTH_A_MIN: f64 = 0.01; +/// Deepest optical depth the UI offers, before the lobe is consulted. +pub const DEPTH_A_MAX: f64 = 6.0; +/// Dimmest cycle-mean lobe point the UI offers. +pub const MEAN_U_MIN: f64 = 0.01; + /// Modified Bessel function `I₀(x)` for the Stage-A depth range (`|x| ≤ 3`). /// /// The positive power series converges rapidly here and avoids adding a @@ -204,6 +212,122 @@ impl LobeInversion { pub fn dac_for_u(&self, u: f64) -> f64 { self.v_null_dac + (2.0 * self.v_pi_dac / PI) * u.clamp(0.0, 1.0).sqrt().asin() } + + /// Highest normalised intensity a drive may peak at without exceeding the + /// operator's DAC ceiling `max_code`. + /// + /// `u = 1` sits at `v_peak`; a ceiling below that clips the lobe short, and + /// the drive has to stay under whatever `u` the ceiling code produces. + pub fn peak_intensity_ceiling(&self, max_code: f64) -> f64 { + if max_code >= self.v_peak_dac() { + return 1.0; + } + if max_code <= self.v_null_dac { + return 0.0; + } + self.u_for_dac(max_code) + } +} + +/// How the peak normalised intensity of a drive follows from its requested +/// cycle mean `ū` and depth `a`. +/// +/// Every calibrated mode has one of these, and they are the *only* thing that +/// limits `a` and `ū`: the swing has to stay under the top of the lobe (and +/// under the operator's DAC ceiling, expressed as the same `u_max`). Solving +/// one relation for each variable in turn gives the achievable ranges the UI +/// shows — and clamps against, instead of refusing the edit and snapping the +/// control back, which told the operator nothing about where the boundary was. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PeakLaw { + /// A constant hold modulates nothing, so the peak *is* the mean and `a` + /// does not enter. + Constant, + /// Bare DAC sine/square on a calibrated band: `ū · e^{a/2}`. + LogSwing, + /// [`OpticalTarget::LogSine`], whose pedestal preserves the cycle mean: + /// `ū · e^{a/2} / I₀(a/2)`. + LogSine, + /// [`OpticalTarget::LinearSine`]: `ū · (1 + tanh(a/2))`. + LinearSine, +} + +impl PeakLaw { + pub fn of(target: OpticalTarget) -> Self { + match target { + OpticalTarget::LogSine => Self::LogSine, + OpticalTarget::LinearSine => Self::LinearSine, + } + } + + /// Peak normalised intensity of the drive, in lobe coordinate. + pub fn peak(self, mean_u: f64, depth_a: f64) -> f64 { + let depth_a = depth_a.max(0.0); + mean_u * self.swing(depth_a) + } + + /// Factor the peak sits above the requested cycle mean. Monotonically + /// non-decreasing in `a` in every variant, which is what makes the + /// inversions below well defined. + fn swing(self, depth_a: f64) -> f64 { + match self { + Self::Constant => 1.0, + Self::LogSwing => (0.5 * depth_a).exp(), + Self::LogSine => (0.5 * depth_a).exp() / modified_bessel_i0(0.5 * depth_a), + Self::LinearSine => 1.0 + (0.5 * depth_a).tanh(), + } + } + + /// Deepest `a` expressible at this cycle mean under the ceiling `u_max`. + pub fn max_depth_for_mean(self, mean_u: f64, u_max: f64) -> f64 { + // Written through `partial_cmp` so a NaN is rejected rather than + // silently passing a negated comparison. + let usable = |value: f64| value.partial_cmp(&0.0) == Some(std::cmp::Ordering::Greater); + if !usable(mean_u) || !usable(u_max) || mean_u > u_max { + return 0.0; + } + let headroom = u_max / mean_u; + match self { + // Nothing swings, so the UI limit is the only bound. + Self::Constant => DEPTH_A_MAX, + Self::LogSwing => (2.0 * headroom.ln()).clamp(0.0, DEPTH_A_MAX), + // Below twice the mean the swing never reaches the ceiling. + Self::LinearSine => { + let m = headroom - 1.0; + if m >= 1.0 { + DEPTH_A_MAX + } else { + (2.0 * m.atanh()).clamp(0.0, DEPTH_A_MAX) + } + } + // No closed form (I₀ grows like e^x/√(2πx)), but `swing` is + // monotonic, so bisect it. + Self::LogSine => { + if self.swing(DEPTH_A_MAX) <= headroom { + return DEPTH_A_MAX; + } + let (mut lo, mut hi) = (0.0_f64, DEPTH_A_MAX); + for _ in 0..64 { + let mid = 0.5 * (lo + hi); + if self.swing(mid) <= headroom { + lo = mid; + } else { + hi = mid; + } + } + lo + } + } + } + + /// Brightest cycle mean the requested depth leaves room for, under the same + /// ceiling. The counterpart of [`PeakLaw::max_depth_for_mean`]. + pub fn max_mean_for_depth(self, depth_a: f64, u_max: f64) -> f64 { + if u_max.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { + return 0.0; + } + (u_max / self.swing(depth_a.max(0.0))).clamp(0.0, 1.0) + } } #[derive(Debug, Clone, Copy, PartialEq)] @@ -629,3 +753,137 @@ mod tests { } } } + +#[cfg(test)] +mod range_tests { + use super::*; + + /// The whole point of the range helpers: what they report as the boundary + /// has to be exactly where `warp_table` stops accepting the drive. If they + /// disagree, the UI either offers a drive that is refused or hides one that + /// would work. + fn table_is_buildable(target: OpticalTarget, mean_u: f64, depth_a: f64) -> bool { + let inversion = LobeInversion { + v_null_dac: 200.0, + v_pi_dac: 1_600.0, + }; + let operating_point = match target { + OpticalTarget::LogSine => log_sine_geometric_pedestal(mean_u, depth_a), + OpticalTarget::LinearSine => mean_u, + }; + OpticalDrive { + target, + depth_a, + operating_point, + inversion, + } + .warp_table() + .is_ok() + } + + #[test] + fn the_reported_max_depth_is_exactly_where_the_table_stops_building() { + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + for mean_u in [0.2, 0.5, 0.8, 0.95] { + let max_a = PeakLaw::of(target).max_depth_for_mean(mean_u, 1.0); + if max_a >= DEPTH_A_MAX { + continue; + } + assert!( + table_is_buildable(target, mean_u, max_a - 1e-4), + "{target:?} mean_u={mean_u} refused a just inside the reported max {max_a}" + ); + assert!( + !table_is_buildable(target, mean_u, max_a + 1e-2), + "{target:?} mean_u={mean_u} accepted a past the reported max {max_a}" + ); + } + } + } + + #[test] + fn the_reported_max_mean_is_exactly_where_the_table_stops_building() { + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + for depth_a in [0.1, 0.5, 1.5, 3.0] { + let max_u = PeakLaw::of(target).max_mean_for_depth(depth_a, 1.0); + assert!( + table_is_buildable(target, max_u - 1e-4, depth_a), + "{target:?} a={depth_a} refused a mean just inside the reported max {max_u}" + ); + if max_u < 1.0 - 1e-3 { + assert!( + !table_is_buildable(target, max_u + 1e-2, depth_a), + "{target:?} a={depth_a} accepted a mean past the reported max {max_u}" + ); + } + } + } + } + + #[test] + fn the_two_helpers_are_inverses_of_each_other() { + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + for mean_u in [0.3, 0.6, 0.9] { + let max_a = PeakLaw::of(target).max_depth_for_mean(mean_u, 1.0); + if max_a >= DEPTH_A_MAX { + continue; + } + let back = PeakLaw::of(target).max_mean_for_depth(max_a, 1.0); + assert!( + (back - mean_u).abs() < 1e-4, + "{target:?}: mean {mean_u} → a {max_a} → mean {back}" + ); + } + } + } + + #[test] + fn a_dac_ceiling_below_v_peak_lowers_the_reachable_intensity() { + let inversion = LobeInversion { + v_null_dac: 200.0, + v_pi_dac: 1_600.0, + }; + // The ceiling at the peak code imposes no limit at all. + assert_eq!(inversion.peak_intensity_ceiling(1_800.0), 1.0); + assert_eq!(inversion.peak_intensity_ceiling(4_095.0), 1.0); + // Halfway up the lobe in code is sin²(π/4) = 0.5 in intensity. + let half = inversion.peak_intensity_ceiling(1_000.0); + assert!( + (half - 0.5).abs() < 1e-9, + "u at the half-span code = {half}" + ); + // A ceiling at or below the null leaves nothing drivable. + assert_eq!(inversion.peak_intensity_ceiling(200.0), 0.0); + } + + #[test] + fn the_log_swing_law_matches_the_calibrated_dac_sine_band() { + // A calibrated DAC_SINE/SQUARE spans u in [ū·e^{-a/2}, ū·e^{+a/2}], so + // its ceiling is reached at exactly a = 2 ln(u_max/ū). + let law = PeakLaw::LogSwing; + let max_a = law.max_depth_for_mean(0.5, 1.0); + assert!((max_a - 2.0 * 2.0_f64.ln()).abs() < 1e-9, "max a = {max_a}"); + assert!((law.peak(0.5, max_a) - 1.0).abs() < 1e-9); + assert!((law.max_mean_for_depth(max_a, 1.0) - 0.5).abs() < 1e-9); + } + + #[test] + fn a_constant_hold_is_limited_only_by_its_own_brightness() { + // CONST modulates nothing, so `a` must not restrict it — requiring the + // modulated band here is what used to freeze a calibrated constant + // drive at its last accepted code. + let law = PeakLaw::Constant; + assert_eq!(law.max_depth_for_mean(1.0, 1.0), DEPTH_A_MAX); + assert_eq!(law.max_mean_for_depth(5.0, 1.0), 1.0); + assert_eq!(law.peak(0.8, 3.0), 0.8); + } + + #[test] + fn a_mean_above_the_ceiling_reports_no_usable_depth() { + // Not a panic and not a silently huge number: the operator has to see + // that this operating point is simply out of reach. + assert_eq!(PeakLaw::LogSine.max_depth_for_mean(0.9, 0.5), 0.0); + assert_eq!(PeakLaw::LinearSine.max_depth_for_mean(0.9, 0.5), 0.0); + assert_eq!(PeakLaw::LogSwing.max_depth_for_mean(0.9, 0.5), 0.0); + } +} diff --git a/plugins/stage-a-modulation/testdata/pockels-20260730-083123.json b/plugins/stage-a-modulation/testdata/pockels-20260730-083123.json new file mode 100644 index 0000000..aa4d030 --- /dev/null +++ b/plugins/stage-a-modulation/testdata/pockels-20260730-083123.json @@ -0,0 +1,705 @@ +{ + "anchor_note": "detector_volts_at_null is a lower bound on the total-power anchor I_tot, not the anchor: on the reject port the residual transmitted floor is not separable from it", + "calibration_id": "pockels-20260730-083123", + "detector_geometry": "REJECT PORT (PD falls as light rises)", + "detector_volts_at_null": 0.05832880721662921, + "detector_volts_at_peak": 0.0075745061683017215, + "hysteresis": 0.25679725274851173, + "lobe_coverage": 4.796857893386575, + "max_level": 3000, + "points": [ + { + "clipped": true, + "code": 0, + "direction": "up", + "peak_to_peak_volts": 0.004835164835164834, + "volts": 0.002619047619047619 + }, + { + "clipped": true, + "code": 63, + "direction": "up", + "peak_to_peak_volts": 0.0016117216117216115, + "volts": 0.002216117216117216 + }, + { + "clipped": true, + "code": 125, + "direction": "up", + "peak_to_peak_volts": 0.004835164835164834, + "volts": 0.002619047619047619 + }, + { + "clipped": true, + "code": 188, + "direction": "up", + "peak_to_peak_volts": 0.006446886446886447, + "volts": 0.004230769230769231 + }, + { + "clipped": true, + "code": 250, + "direction": "up", + "peak_to_peak_volts": 0.002417582417582417, + "volts": 0.0034249084249084244 + }, + { + "clipped": true, + "code": 313, + "direction": "up", + "peak_to_peak_volts": 0.01128205128205128, + "volts": 0.006849816849816849 + }, + { + "clipped": false, + "code": 375, + "direction": "up", + "peak_to_peak_volts": 0.05399267399267399, + "volts": 0.044120879120879114 + }, + { + "clipped": false, + "code": 438, + "direction": "up", + "peak_to_peak_volts": 0.05399267399267399, + "volts": 0.04230769230769231 + }, + { + "clipped": false, + "code": 500, + "direction": "up", + "peak_to_peak_volts": 0.025787545787545788, + "volts": 0.024175824175824177 + }, + { + "clipped": false, + "code": 563, + "direction": "up", + "peak_to_peak_volts": 0.029816849816849816, + "volts": 0.0558058608058608 + }, + { + "clipped": false, + "code": 625, + "direction": "up", + "peak_to_peak_volts": 0.004835164835164834, + "volts": 0.06366300366300366 + }, + { + "clipped": false, + "code": 688, + "direction": "up", + "peak_to_peak_volts": 0.02498168498168498, + "volts": 0.0543956043956044 + }, + { + "clipped": false, + "code": 750, + "direction": "up", + "peak_to_peak_volts": 0.012087912087912088, + "volts": 0.06043956043956044 + }, + { + "clipped": false, + "code": 813, + "direction": "up", + "peak_to_peak_volts": 0.020146520146520148, + "volts": 0.055 + }, + { + "clipped": false, + "code": 875, + "direction": "up", + "peak_to_peak_volts": 0.03465201465201465, + "volts": 0.039285714285714285 + }, + { + "clipped": false, + "code": 938, + "direction": "up", + "peak_to_peak_volts": 0.02256410256410256, + "volts": 0.05983516483516483 + }, + { + "clipped": false, + "code": 1000, + "direction": "up", + "peak_to_peak_volts": 0.05479853479853479, + "volts": 0.05177655677655677 + }, + { + "clipped": true, + "code": 1063, + "direction": "up", + "peak_to_peak_volts": 0.010476190476190476, + "volts": 0.007051282051282051 + }, + { + "clipped": true, + "code": 1125, + "direction": "up", + "peak_to_peak_volts": 0.041098901098901096, + "volts": 0.021153846153846155 + }, + { + "clipped": true, + "code": 1188, + "direction": "up", + "peak_to_peak_volts": 0.008864468864468863, + "volts": 0.00402930402930403 + }, + { + "clipped": true, + "code": 1250, + "direction": "up", + "peak_to_peak_volts": 0.03304029304029304, + "volts": 0.01672161172161172 + }, + { + "clipped": true, + "code": 1313, + "direction": "up", + "peak_to_peak_volts": 0.03223443223443224, + "volts": 0.0139010989010989 + }, + { + "clipped": true, + "code": 1375, + "direction": "up", + "peak_to_peak_volts": 0.01531135531135531, + "volts": 0.009267399267399268 + }, + { + "clipped": false, + "code": 1438, + "direction": "up", + "peak_to_peak_volts": 0.017728937728937726, + "volts": 0.01652014652014652 + }, + { + "clipped": false, + "code": 1500, + "direction": "up", + "peak_to_peak_volts": 0.008864468864468863, + "volts": 0.029816849816849816 + }, + { + "clipped": true, + "code": 1563, + "direction": "up", + "peak_to_peak_volts": 0.021758241758241755, + "volts": 0.008864468864468863 + }, + { + "clipped": true, + "code": 1625, + "direction": "up", + "peak_to_peak_volts": 0.04996336996336996, + "volts": 0.01631868131868132 + }, + { + "clipped": false, + "code": 1688, + "direction": "up", + "peak_to_peak_volts": 0.03948717948717948, + "volts": 0.03485347985347985 + }, + { + "clipped": false, + "code": 1750, + "direction": "up", + "peak_to_peak_volts": 0.014505494505494505, + "volts": 0.06064102564102564 + }, + { + "clipped": false, + "code": 1813, + "direction": "up", + "peak_to_peak_volts": 0.029816849816849816, + "volts": 0.058424908424908426 + }, + { + "clipped": false, + "code": 1875, + "direction": "up", + "peak_to_peak_volts": 0.041098901098901096, + "volts": 0.053791208791208786 + }, + { + "clipped": false, + "code": 1938, + "direction": "up", + "peak_to_peak_volts": 0.021758241758241755, + "volts": 0.04714285714285714 + }, + { + "clipped": false, + "code": 2000, + "direction": "up", + "peak_to_peak_volts": 0.02498168498168498, + "volts": 0.0554029304029304 + }, + { + "clipped": false, + "code": 2063, + "direction": "up", + "peak_to_peak_volts": 0.012893772893772894, + "volts": 0.06023809523809523 + }, + { + "clipped": false, + "code": 2125, + "direction": "up", + "peak_to_peak_volts": 0.02820512820512821, + "volts": 0.05036630036630037 + }, + { + "clipped": false, + "code": 2188, + "direction": "up", + "peak_to_peak_volts": 0.025787545787545788, + "volts": 0.05822344322344322 + }, + { + "clipped": false, + "code": 2250, + "direction": "up", + "peak_to_peak_volts": 0.038681318681318674, + "volts": 0.05076923076923076 + }, + { + "clipped": false, + "code": 2313, + "direction": "up", + "peak_to_peak_volts": 0.03223443223443224, + "volts": 0.02296703296703297 + }, + { + "clipped": false, + "code": 2375, + "direction": "up", + "peak_to_peak_volts": 0.014505494505494505, + "volts": 0.009670329670329669 + }, + { + "clipped": true, + "code": 2438, + "direction": "up", + "peak_to_peak_volts": 0.008864468864468863, + "volts": 0.004835164835164834 + }, + { + "clipped": false, + "code": 2500, + "direction": "up", + "peak_to_peak_volts": 0.03626373626373627, + "volts": 0.02478021978021978 + }, + { + "clipped": true, + "code": 2563, + "direction": "up", + "peak_to_peak_volts": 0.03465201465201465, + "volts": 0.02095238095238095 + }, + { + "clipped": false, + "code": 2625, + "direction": "up", + "peak_to_peak_volts": 0.02336996336996337, + "volts": 0.014505494505494505 + }, + { + "clipped": false, + "code": 2688, + "direction": "up", + "peak_to_peak_volts": 0.004835164835164834, + "volts": 0.01672161172161172 + }, + { + "clipped": true, + "code": 2750, + "direction": "up", + "peak_to_peak_volts": 0.007252747252747252, + "volts": 0.005641025641025641 + }, + { + "clipped": true, + "code": 2813, + "direction": "up", + "peak_to_peak_volts": 0.037875457875457874, + "volts": 0.02195970695970696 + }, + { + "clipped": true, + "code": 2875, + "direction": "up", + "peak_to_peak_volts": 0.03948717948717948, + "volts": 0.013095238095238096 + }, + { + "clipped": false, + "code": 2938, + "direction": "up", + "peak_to_peak_volts": 0.040293040293040296, + "volts": 0.037875457875457874 + }, + { + "clipped": false, + "code": 3000, + "direction": "up", + "peak_to_peak_volts": 0.031428571428571424, + "volts": 0.04976190476190476 + }, + { + "clipped": false, + "code": 3000, + "direction": "down", + "peak_to_peak_volts": 0.02901098901098901, + "volts": 0.05238095238095238 + }, + { + "clipped": false, + "code": 2938, + "direction": "down", + "peak_to_peak_volts": 0.008864468864468863, + "volts": 0.0552014652014652 + }, + { + "clipped": false, + "code": 2875, + "direction": "down", + "peak_to_peak_volts": 0.041098901098901096, + "volts": 0.02498168498168498 + }, + { + "clipped": false, + "code": 2813, + "direction": "down", + "peak_to_peak_volts": 0.03545787545787545, + "volts": 0.021758241758241755 + }, + { + "clipped": false, + "code": 2750, + "direction": "down", + "peak_to_peak_volts": 0.008864468864468863, + "volts": 0.02880952380952381 + }, + { + "clipped": false, + "code": 2688, + "direction": "down", + "peak_to_peak_volts": 0.012087912087912088, + "volts": 0.011886446886446888 + }, + { + "clipped": true, + "code": 2625, + "direction": "down", + "peak_to_peak_volts": 0.02095238095238095, + "volts": 0.014706959706959706 + }, + { + "clipped": true, + "code": 2563, + "direction": "down", + "peak_to_peak_volts": 0.01128205128205128, + "volts": 0.004835164835164834 + }, + { + "clipped": true, + "code": 2500, + "direction": "down", + "peak_to_peak_volts": 0.0427106227106227, + "volts": 0.017527472527472526 + }, + { + "clipped": true, + "code": 2438, + "direction": "down", + "peak_to_peak_volts": 0.02820512820512821, + "volts": 0.014706959706959706 + }, + { + "clipped": false, + "code": 2375, + "direction": "down", + "peak_to_peak_volts": 0.05479853479853479, + "volts": 0.0408974358974359 + }, + { + "clipped": false, + "code": 2313, + "direction": "down", + "peak_to_peak_volts": 0.01128205128205128, + "volts": 0.011483516483516485 + }, + { + "clipped": false, + "code": 2250, + "direction": "down", + "peak_to_peak_volts": 0.018534798534798533, + "volts": 0.05983516483516483 + }, + { + "clipped": false, + "code": 2188, + "direction": "down", + "peak_to_peak_volts": 0.038681318681318674, + "volts": 0.039285714285714285 + }, + { + "clipped": false, + "code": 2125, + "direction": "down", + "peak_to_peak_volts": 0.02336996336996337, + "volts": 0.052985347985347986 + }, + { + "clipped": false, + "code": 2063, + "direction": "down", + "peak_to_peak_volts": 0.02336996336996337, + "volts": 0.045128205128205125 + }, + { + "clipped": false, + "code": 2000, + "direction": "down", + "peak_to_peak_volts": 0.01531135531135531, + "volts": 0.058424908424908426 + }, + { + "clipped": false, + "code": 1938, + "direction": "down", + "peak_to_peak_volts": 0.006446886446886447, + "volts": 0.06124542124542124 + }, + { + "clipped": false, + "code": 1875, + "direction": "down", + "peak_to_peak_volts": 0.007252747252747252, + "volts": 0.06326007326007327 + }, + { + "clipped": false, + "code": 1813, + "direction": "down", + "peak_to_peak_volts": 0.013699633699633696, + "volts": 0.06386446886446887 + }, + { + "clipped": false, + "code": 1750, + "direction": "down", + "peak_to_peak_volts": 0.03948717948717948, + "volts": 0.04049450549450549 + }, + { + "clipped": false, + "code": 1688, + "direction": "down", + "peak_to_peak_volts": 0.038681318681318674, + "volts": 0.02860805860805861 + }, + { + "clipped": false, + "code": 1625, + "direction": "down", + "peak_to_peak_volts": 0.02659340659340659, + "volts": 0.04452380952380952 + }, + { + "clipped": true, + "code": 1563, + "direction": "down", + "peak_to_peak_volts": 0.045128205128205125, + "volts": 0.017124542124542126 + }, + { + "clipped": true, + "code": 1500, + "direction": "down", + "peak_to_peak_volts": 0.016923076923076923, + "volts": 0.007051282051282051 + }, + { + "clipped": false, + "code": 1438, + "direction": "down", + "peak_to_peak_volts": 0.012087912087912088, + "volts": 0.02195970695970696 + }, + { + "clipped": true, + "code": 1375, + "direction": "down", + "peak_to_peak_volts": 0.01128205128205128, + "volts": 0.006648351648351648 + }, + { + "clipped": false, + "code": 1313, + "direction": "down", + "peak_to_peak_volts": 0.005641025641025641, + "volts": 0.009670329670329669 + }, + { + "clipped": true, + "code": 1250, + "direction": "down", + "peak_to_peak_volts": 0.03465201465201465, + "volts": 0.01631868131868132 + }, + { + "clipped": true, + "code": 1188, + "direction": "down", + "peak_to_peak_volts": 0.027399267399267395, + "volts": 0.010677655677655676 + }, + { + "clipped": true, + "code": 1125, + "direction": "down", + "peak_to_peak_volts": 0.005641025641025641, + "volts": 0.003626373626373626 + }, + { + "clipped": false, + "code": 1063, + "direction": "down", + "peak_to_peak_volts": 0.04351648351648351, + "volts": 0.04230769230769231 + }, + { + "clipped": false, + "code": 1000, + "direction": "down", + "peak_to_peak_volts": 0.007252747252747252, + "volts": 0.011886446886446888 + }, + { + "clipped": false, + "code": 938, + "direction": "down", + "peak_to_peak_volts": 0.04351648351648351, + "volts": 0.02901098901098901 + }, + { + "clipped": false, + "code": 875, + "direction": "down", + "peak_to_peak_volts": 0.006446886446886447, + "volts": 0.06426739926739927 + }, + { + "clipped": false, + "code": 813, + "direction": "down", + "peak_to_peak_volts": 0.01611721611721612, + "volts": 0.042912087912087914 + }, + { + "clipped": false, + "code": 750, + "direction": "down", + "peak_to_peak_volts": 0.012087912087912088, + "volts": 0.06043956043956044 + }, + { + "clipped": false, + "code": 688, + "direction": "down", + "peak_to_peak_volts": 0.019340659340659337, + "volts": 0.05661172161172161 + }, + { + "clipped": false, + "code": 625, + "direction": "down", + "peak_to_peak_volts": 0.02659340659340659, + "volts": 0.0554029304029304 + }, + { + "clipped": false, + "code": 563, + "direction": "down", + "peak_to_peak_volts": 0.04593406593406593, + "volts": 0.03747252747252747 + }, + { + "clipped": false, + "code": 500, + "direction": "down", + "peak_to_peak_volts": 0.05963369963369963, + "volts": 0.04956043956043956 + }, + { + "clipped": true, + "code": 438, + "direction": "down", + "peak_to_peak_volts": 0.027399267399267395, + "volts": 0.010073260073260074 + }, + { + "clipped": false, + "code": 375, + "direction": "down", + "peak_to_peak_volts": 0.06285714285714285, + "volts": 0.031025641025641024 + }, + { + "clipped": false, + "code": 313, + "direction": "down", + "peak_to_peak_volts": 0.006446886446886447, + "volts": 0.062454212454212454 + }, + { + "clipped": true, + "code": 250, + "direction": "down", + "peak_to_peak_volts": 0.005641025641025641, + "volts": 0.004835164835164834 + }, + { + "clipped": false, + "code": 188, + "direction": "down", + "peak_to_peak_volts": 0.002417582417582417, + "volts": 0.005439560439560439 + }, + { + "clipped": true, + "code": 125, + "direction": "down", + "peak_to_peak_volts": 0.0016117216117216115, + "volts": 0.003223443223443223 + }, + { + "clipped": true, + "code": 63, + "direction": "down", + "peak_to_peak_volts": 0.002417582417582417, + "volts": 0.002417582417582417 + }, + { + "clipped": true, + "code": 0, + "direction": "down", + "peak_to_peak_volts": 0.0016117216117216115, + "volts": 0.002014652014652015 + } + ], + "port": "auto", + "quality": 0.2227805649633928, + "rejected_points": 0, + "rms_residual_volts": 0.011307071861868518, + "span_volts": -0.05075430104832749, + "v_null_dac": 711.9225837107243, + "v_pi_dac": 625.4093964584814 +} \ No newline at end of file diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md index 44d48d7..2406154 100644 --- a/plugins/stage-a-photodiode/README.md +++ b/plugins/stage-a-photodiode/README.md @@ -9,9 +9,12 @@ read-only by construction; the command port belongs to `stage-a-modulation`. - **RAW** — shows the ADC code and its voltage, `V = code · 3.3 / 4095`. - **EXCITATION** — the photodiode sits in the excitation path behind the PBS and measures the - light *removed* from the beam: `I_pd = I_tot − I_exc`. Given the user-set reference **I_tot** - (in photodiode volts — the reading with the full beam on the diode), the plugin shows - `I_exc = I_tot − I_pd`. + light *removed* from the beam: `I_pd = I_tot − I_exc`, so the plugin shows `I_exc = I_tot − I_pd`. + `I_tot` is **learned, not entered**: it is the brightest smoothed reading the detector has taken + since the port opened, which on the reject port is where the excitation is extinguished. The + Pockels transfer sweep drives through that null by construction, so running it once teaches the + anchor. There is no dark level either — a DC offset cancels exactly out of the complement. + See [ADR 024](../../docs/adr/024-stage-a-photodiode-learns-its-own-anchor.md). ## Views @@ -34,11 +37,24 @@ plugins control named recordings through the versioned receive raw sample arrays through the control plane; finalized PDQ files remain the replay and analysis source of truth. -The snapshot's `stream.level` block carries the settled detector level over the -moving-average window in **raw** detector volts — the ADC map only, never the -RAW/EXCITATION display transform and never the optical geometry transform. It +The snapshot's `stream.level` block carries the settled detector level in **raw** +detector volts — the ADC map only, never the RAW/EXCITATION display transform and +never the optical geometry transform. It is averaged over a fixed **20 ms** +owned here and independent of the chart's moving-average setting, because that +setting is a display preference and this is a measurement: deriving one from the +other let a default of four samples publish 8 µs per point at 500 kSa/s and +report a clean Pockels calibration as a 22 % residual +([ADR 019](../../docs/adr/019-stage-a-calibration-measures-its-own-window.md)). +It also reports the window's peak-to-peak spread and the sample index it ends at, so a consumer can prove a reading was taken *after* it changed something without a shared clock. Unlike `optical_summary` it never refuses: it stays present while the window clips (flagged), because the Pockels transfer sweep needs a reading exactly where the reject-port detector is brightest. + +When `optical_summary` *is* refused, `optical_unavailable` on the same snapshot +carries the reason, so a consumer that gates on `a` can name the gate instead of +reporting absence. Clip detection is span-relative — the near-rail margin is +capped at 5 % of the window's own peak-to-peak span, so this detector's 0.5–15 mV +operating range is not mistaken for a waveform truncating at code 0. See +[ADR 017](../../docs/adr/017-stage-a-rail-detection-and-withheld-a-reasons.md). diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index f6b24d9..4d4cf3e 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -41,8 +41,8 @@ use augur_plugin_api::{ }; use serde_json::{json, Value}; use stage_a_io::{ - estimate_contrast, AdcCalibration, ContrastGeometry, EstimateError, FrameParser, ParseEvent, - PdqWriter, StreamIntegrity, + estimate_contrast, near_rail_margin, AdcCalibration, ContrastGeometry, EstimateError, + FrameParser, ParseEvent, PdqWriter, StreamIntegrity, }; use stage_a_plugin_contract::{ ClientId, ConnectionStateV1, FreshnessV1, LeaseId, LeaseSnapshotV1, OwnerInstanceId, @@ -64,12 +64,6 @@ const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; -/// Drag increment for manually entered photodiode calibration voltages. -/// -/// The host derives the displayed decimal precision from this increment. The -/// Stage-A detector normally operates around 0.0005–0.015 V, so the former -/// 10 mV / 1 mV increments hid physically relevant values. -const VOLTAGE_INPUT_STEP_VOLTS: f64 = 0.000_001; /// Default monitor cache, in seconds of samples at the active stream rate /// (user-settable 1–130 s). const DEFAULT_CACHE_SECONDS: f64 = 20.0; @@ -109,9 +103,11 @@ const MOCK_BLOCK_SAMPLES: usize = 256; const MAX_MARKERS: usize = 4_096; /// Mock phase-0 marker period in samples (20 kSa/s / 40 = 500 Hz modulation). const MOCK_MARKER_PERIOD_SAMPLES: u64 = 40; -/// Codes within this margin of an ADC rail mark a level window as clipped; -/// mirrors the estimator's own clip margin. -const CLIP_MARGIN_CODES: u16 = 4; +/// Seconds of samples the **published** level averages over, independent of the +/// chart's averaging setting. One full mains period: a boxcar of exactly this +/// length nulls 50 Hz and every harmonic of it. See +/// [`StageAPhotodiodePlugin::level_window_samples`] and ADR 019. +const LEVEL_WINDOW_SECONDS: f64 = 0.020; const REQUEST_CACHE_LIMIT: usize = 256; const MIN_LEASE_TTL_MS: u64 = 1_000; const MAX_LEASE_TTL_MS: u64 = 60_000; @@ -208,6 +204,20 @@ struct SharedState { segments: u64, /// Monitor-cache length driving ring eviction (user setting). cache_seconds: f64, + /// Highest smoothed detector level observed since the port was opened, in + /// raw ADC codes. **This is the total-power anchor `I_tot`.** + /// + /// The detector sits on the PBS reject port and reads the complement + /// `I_pd = I_tot − I_exc`, so it is brightest exactly where the excitation + /// is fully extinguished — and there `I_pd = I_tot`. Nothing has to be + /// typed in: the Pockels transfer sweep walks the DAC across the whole + /// lobe, which lands on the excitation null by construction, so the anchor + /// is learned by the calibration the operator already runs (ADR 024). + /// + /// Latched over completed [`SUMMARY_CELL`]-sample cells, never over raw + /// samples: one noise spike must not become the anchor every later `a` is + /// divided against. + observed_peak_code: Option, error: Option, last_update_unix_ms: u64, } @@ -254,6 +264,7 @@ impl Default for SharedState { resync_bytes: 0, segments: 0, cache_seconds: DEFAULT_CACHE_SECONDS, + observed_peak_code: None, error: None, last_update_unix_ms: 0, } @@ -307,6 +318,14 @@ impl SharedState { cell.max = cell.max.max(code); cell.sum += u32::from(code); } + // Learn the total-power anchor as we go. The latch survives segment + // restarts on purpose: a rate change, a drop or an acquisition + // handover does not move the optics, and the calibration sweep that + // teaches the anchor is followed by exactly such a handover. + let cell_mean = f64::from(cell.sum) / SUMMARY_CELL as f64; + if self.observed_peak_code.is_none_or(|peak| cell_mean > peak) { + self.observed_peak_code = Some(cell_mean); + } self.cells.push_back(cell); } @@ -759,19 +778,6 @@ pub struct StageAPhotodiodePlugin { connect_requested: bool, port_hint: String, mode: Mode, - reference_volts: f64, - /// Stable provenance identifier for the measured full-extinction - /// total-power reading in `reference_volts`. - reference_anchor_id: String, - /// Explicit operator confirmation that `reference_volts` is a measured - /// full-extinction anchor for the current optical configuration. - reference_confirmed: bool, - /// Measured dark level in photodiode volts (beam blocked). Applied to both - /// the detector samples and the `reference_volts` anchor, so it cancels out - /// of the rejected-complement contrast rather than biasing it — its job is - /// to keep the two sides consistent and to record the calibration that the - /// reading was taken under. Captured via the "Capture dark" action. - dark_volts: f64, window_s: f64, avg_samples: usize, avg_sync_freq_hz: f64, @@ -783,7 +789,6 @@ pub struct StageAPhotodiodePlugin { press_save_snapshot: PressLatch, press_record_start: PressLatch, press_record_stop: PressLatch, - press_capture_dark: PressLatch, } /// Forwards momentary button presses across the host's UI-mirror → live-worker @@ -866,10 +871,6 @@ impl Default for StageAPhotodiodePlugin { connect_requested: false, port_hint: "auto".into(), mode: Mode::Raw, - reference_volts: 3.3, - reference_anchor_id: String::new(), - reference_confirmed: false, - dark_volts: 0.0, window_s: 10.0, avg_samples: 4, avg_sync_freq_hz: 0.0, @@ -879,7 +880,6 @@ impl Default for StageAPhotodiodePlugin { press_save_snapshot: PressLatch::default(), press_record_start: PressLatch::default(), press_record_stop: PressLatch::default(), - press_capture_dark: PressLatch::default(), } } } @@ -889,42 +889,35 @@ impl StageAPhotodiodePlugin { self.reader.is_some() } - /// The ADC calibration handed to the contrast estimator, including the - /// measured dark level. + /// The ADC calibration handed to the contrast estimator. + /// + /// `dark_volts` is deliberately zero. A DC dark offset `D` cancels + /// *exactly* out of the rejected-complement contrast once the total-power + /// anchor is read from the same detector: the excitation is + /// `(I_tot,obs − D) − (v − D) = I_tot,obs − v`, with no `D` left in it. + /// Subtracting a separately entered dark from only one of the two sides is + /// what would bias `a` — which is why there is no dark setting any more + /// (ADR 024). fn adc_calibration(&self) -> AdcCalibration { AdcCalibration { volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, offset_volts: 0.0, - dark_volts: self.dark_volts, + dark_volts: 0.0, full_scale_code: ADC_MAX_CODE as u16, } } - /// Captures the dark level as the mean of the current ring: the operator - /// blocks the beam, presses the button, and every later contrast is - /// dark-corrected against it. - fn capture_dark(&mut self) -> Result<(), String> { - let mean = { - let state = self - .shared - .lock() - .map_err(|_| "photodiode state lock poisoned".to_owned())?; - if state.samples.is_empty() { - return Err("no samples cached yet — connect and stream first".into()); - } - let sum: u64 = state.samples.iter().map(|&code| u64::from(code)).sum(); - code_to_volts(sum as f64 / state.samples.len() as f64) - }; - if mean >= self.reference_volts { - return Err(format!( - "dark level {mean:.6} V is not below the I_tot reference \ - {:.6} V — is the beam actually blocked?", - self.reference_volts - )); - } - self.dark_volts = mean; - self.last_save_note = Some(format!("dark level captured: {mean:.6} V")); - Ok(()) + /// The learned total-power anchor `I_tot` in volts, if the stream has run + /// long enough to complete one summary cell. + fn total_power_volts(&self, state: &SharedState) -> Option { + state.observed_peak_code.map(code_to_volts) + } + + /// [`Self::total_power_volts`] for callers that do not already hold the ring + /// lock (sidecars, status entries, the chart transform). + fn learned_anchor_volts(&self) -> Option { + let state = self.shared.lock().ok()?; + self.total_power_volts(&state) } fn connect(&mut self) { @@ -1339,9 +1332,8 @@ impl StageAPhotodiodePlugin { "termination": receipt.termination, "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), - "reference_volts": self.reference_volts, - "reference_anchor_id": self.reference_anchor_id, - "reference_confirmed": self.reference_confirmed, + "total_power_volts": self.learned_anchor_volts(), + "total_power_source": "observed-peak", "integrity": { "resync_bytes": summary.integrity.skipped_bytes, "crc_failures": summary.integrity.crc_failures, @@ -1610,30 +1602,25 @@ impl StageAPhotodiodePlugin { /// bench, settled by construction (knowledge base: /// `setup/optical-path.md`), not of what the operator chose to plot. So the /// geometry is always [`ContrastGeometry::RejectedComplement`] anchored on - /// `reference_volts`, and `measured_log_contrast` is always the *excitation* - /// contrast `a = ln(I_exc,max / I_exc,min)`. + /// [`SharedState::observed_peak_code`], and `measured_log_contrast` is + /// always the *excitation* contrast `a = ln(I_exc,max / I_exc,min)`. /// /// The display [`Mode`] is presentational only. It must never reach this /// function: A1's amplitude sweep settles on this value against a target /// `a`, so letting a display toggle change its meaning would silently /// retarget the sweep and write a wrong `measured_a` into every sidecar. /// - /// `None` when there is no valid whole-cycle window or no explicitly - /// confirmed total-power anchor. - fn optical_summary(&self, state: &SharedState) -> Option { - self.optical_summary_result(state).ok() - } - - /// [`Self::optical_summary`], keeping the rejection reason so the status - /// readout can explain *why* `a` is being withheld instead of silently - /// showing nothing. + /// `Err` — never a silent `None` — when there is no valid whole-cycle + /// window or no learned total-power anchor: the rejection reason is what + /// the status readout and the A1 panel render instead of `a`, so a withheld + /// `a` names the gate the operator has to fix. fn optical_summary_result( &self, state: &SharedState, ) -> Result { - if !self.reference_confirmed || self.reference_anchor_id.trim().is_empty() { - return Err(EstimateError::MissingTotalPowerAnchor); - } + let total_power_volts = self + .total_power_volts(state) + .ok_or(EstimateError::MissingTotalPowerAnchor)?; let ring_end = state.ring_first_index + state.samples.len() as u64; let markers: Vec = state @@ -1664,16 +1651,11 @@ impl StageAPhotodiodePlugin { let covered_cycles = Some((markers.len() - 1 - start_marker) as f64); let window_seconds = end_index.saturating_sub(start_index) as f64 / rate_hz; let calibration = self.adc_calibration(); - // `ContrastGeometry::RejectedComplement` wants the *dark-corrected* - // I_tot, and the estimator dark-corrects the detector samples. The - // reference is a reading from the same DC-coupled detector, so it - // carries the same dark offset and has to be corrected the same way. - // Correcting only one side is what would bias `a`; corrected on both, - // the dark term cancels out of the complement exactly (it is a - // difference of two readings), which is the physically right answer. - let geometry = ContrastGeometry::RejectedComplement { - total_power_volts: self.reference_volts - self.dark_volts, - }; + // The anchor and the samples come from the same DC-coupled detector, so + // any dark offset appears identically on both sides of the complement + // and cancels exactly. Nothing here is dark-corrected, and that is the + // physically right answer — see [`Self::adc_calibration`]. + let geometry = ContrastGeometry::RejectedComplement { total_power_volts }; let estimate = estimate_contrast(&window, &calibration, geometry)?; let run_id = self .lease @@ -1684,25 +1666,23 @@ impl StageAPhotodiodePlugin { run_id, calibration: PhotodiodeCalibrationV1 { adc_calibration_id: "adc-default".into(), - // Name the dark level honestly: consumers must be able to tell - // a measured dark from the un-measured zero default. - dark_id: if self.dark_volts > 0.0 { - "dark-measured".into() - } else { - "dark-none".into() - }, - anchor_id: self.reference_anchor_id.clone(), + // The complement is dark-invariant, so there is no dark level + // to name — say that rather than imply an unmeasured zero. + dark_id: "dark-cancels".into(), + // Provenance for an anchor nobody typed: the detector sample + // index the learned peak was still valid at. + anchor_id: format!("observed-peak@{ring_end}"), dark_volts: calibration.dark_volts, - total_power_volts: self.reference_volts - self.dark_volts, + total_power_volts, }, measured_log_contrast: estimate.a, log_contrast_stddev: None, excitation_min_volts: estimate.v_min_volts, excitation_max_volts: estimate.v_max_volts, - // Both geometries are dark-referenced (`reference_volts` is the - // dark-corrected `I_tot`), so the excitation minimum *is* the - // margin above dark. Same number as `excitation_min_volts` by - // construction; kept because the contract publishes both. + // The excitation minimum is measured from the same anchor the + // detector samples are, so it *is* the margin above the floor. + // Same number as `excitation_min_volts` by construction; kept + // because the contract publishes both. excitation_headroom_volts: estimate.v_min_volts, low_clip_fraction: estimate.low_clip_fraction, high_clip_fraction: estimate.high_clip_fraction, @@ -1718,6 +1698,13 @@ impl StageAPhotodiodePlugin { }) } + /// [`Self::optical_summary_result`] reduced to the accepted value, for tests + /// that assert on `a` itself rather than on which gate refused it. + #[cfg(test)] + fn optical_summary(&self, state: &SharedState) -> Option { + self.optical_summary_result(state).ok() + } + /// Locks the ring and returns the current optical log-contrast summary, /// keeping the rejection reason so the caller can explain a withheld `a`. fn latest_optical_result(&self) -> Option> { @@ -1726,66 +1713,80 @@ impl StageAPhotodiodePlugin { } fn control_summary(&self) -> PhotodiodeSummaryV1 { - let (stream, connection, observed_at, optical_summary) = match self.shared.lock() { - Ok(state) => { - let sample_range = (!state.samples.is_empty()).then_some(SampleRangeV1 { - first_sample_index: state.ring_first_index, - end_sample_index_exclusive: state.ring_first_index + state.samples.len() as u64, - sample_count: state.samples.len() as u64, - }); - let optical_summary = self.optical_summary(&state); - let level = self.current_level(&state); - let connection = if self.connected() { - ConnectionStateV1::Connected { - port_label: self.port_hint.clone(), - firmware_version: None, - } - } else if let Some(message) = - state.error.clone().or_else(|| self.last_error.clone()) - { - ConnectionStateV1::Faulted { message } - } else if self.connect_requested { - ConnectionStateV1::Connecting - } else { - ConnectionStateV1::Disconnected - }; - ( - PhotodiodeStreamV1 { - stream_epoch: state.segments, - sample_range, - sample_rate_hz: (state.rate_hz != 0).then_some(state.rate_hz), - latest_adc_code: state.latest, - integrity: StreamIntegrityV1 { - skipped_bytes: state.resync_bytes, - crc_failures: state.crc_failures, - sequence_gaps: state.segments, - dropped_samples: u64::from(state.device_dropped), - segment_restarts: state.segments, - truncated_bytes: 0, + let (stream, connection, observed_at, optical_summary, optical_unavailable) = + match self.shared.lock() { + Ok(state) => { + let sample_range = (!state.samples.is_empty()).then_some(SampleRangeV1 { + first_sample_index: state.ring_first_index, + end_sample_index_exclusive: state.ring_first_index + + state.samples.len() as u64, + sample_count: state.samples.len() as u64, + }); + // Publish the refusal reason alongside the absent summary: A1 + // gates the a₀ lock and the frequency sweep on `a`, and without + // this the operator only learns that `a` is missing, not which + // gate to fix. + let (optical_summary, optical_unavailable) = match (!state.samples.is_empty()) + .then(|| self.optical_summary_result(&state)) + { + Some(Ok(summary)) => (Some(summary), None), + Some(Err(error)) => (None, Some(error.to_string())), + None => (None, None), + }; + let level = self.current_level(&state); + let connection = if self.connected() { + ConnectionStateV1::Connected { + port_label: self.port_hint.clone(), + firmware_version: None, + } + } else if let Some(message) = + state.error.clone().or_else(|| self.last_error.clone()) + { + ConnectionStateV1::Faulted { message } + } else if self.connect_requested { + ConnectionStateV1::Connecting + } else { + ConnectionStateV1::Disconnected + }; + ( + PhotodiodeStreamV1 { + stream_epoch: state.segments, + sample_range, + sample_rate_hz: (state.rate_hz != 0).then_some(state.rate_hz), + latest_adc_code: state.latest, + integrity: StreamIntegrityV1 { + skipped_bytes: state.resync_bytes, + crc_failures: state.crc_failures, + sequence_gaps: state.segments, + dropped_samples: u64::from(state.device_dropped), + segment_restarts: state.segments, + truncated_bytes: 0, + }, + level, }, - level, + connection, + state.last_update_unix_ms, + optical_summary, + optical_unavailable, + ) + } + Err(_) => ( + PhotodiodeStreamV1 { + stream_epoch: 0, + sample_range: None, + sample_rate_hz: None, + latest_adc_code: None, + integrity: StreamIntegrityV1::default(), + level: None, }, - connection, - state.last_update_unix_ms, - optical_summary, - ) - } - Err(_) => ( - PhotodiodeStreamV1 { - stream_epoch: 0, - sample_range: None, - sample_rate_hz: None, - latest_adc_code: None, - integrity: StreamIntegrityV1::default(), - level: None, - }, - ConnectionStateV1::Faulted { - message: "photodiode state lock poisoned".into(), - }, - 0, - None, - ), - }; + ConnectionStateV1::Faulted { + message: "photodiode state lock poisoned".into(), + }, + 0, + None, + Some("photodiode state lock poisoned".to_owned()), + ), + }; let active_recording = self .recording .lock() @@ -1828,6 +1829,7 @@ impl StageAPhotodiodePlugin { active_recording, last_finalized_recording: self.last_finalized_recording.clone(), optical_summary, + optical_unavailable, synchronization, last_response: self.last_response.clone(), freshness: FreshnessV1 { @@ -1956,10 +1958,8 @@ impl StageAPhotodiodePlugin { "csv_path": csv_path, "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), - "reference_volts": self.reference_volts, - "reference_anchor_id": self.reference_anchor_id, - "reference_confirmed": self.reference_confirmed, - "dark_volts": self.dark_volts, + "total_power_volts": self.learned_anchor_volts(), + "total_power_source": "observed-peak", "time_base": "t_s = sample_index / sample_rate_hz, device clock, segment-relative", "integrity": integrity, }); @@ -1972,10 +1972,16 @@ impl StageAPhotodiodePlugin { } /// Value shown for one sample under the current mode, in volts. - fn display_volts(&self, code: f64) -> f64 { - match self.mode { - Mode::Raw => code_to_volts(code), - Mode::Excitation => self.reference_volts - code_to_volts(code), + /// + /// EXCITATION needs the learned total-power anchor to take the complement. + /// The anchor is latched from the first completed summary cell — a few + /// milliseconds after the stream opens — so the `None` arm is only ever the + /// very first repaint; it shows the raw reading rather than a trace + /// referenced to a number that does not exist yet. + fn display_volts(&self, code: f64, anchor_volts: Option) -> f64 { + match (self.mode, anchor_volts) { + (Mode::Raw, _) | (Mode::Excitation, None) => code_to_volts(code), + (Mode::Excitation, Some(anchor)) => anchor - code_to_volts(code), } } @@ -2004,12 +2010,34 @@ impl StageAPhotodiodePlugin { Some(state.range_summary(start, state.samples.len()).mean()) } - /// Settled detector level over the same window, published on the contract - /// in **raw** detector volts — never `display_volts`, so a consumer does - /// not have to know the display mode, and never the optical geometry + /// Samples the published level averages over: a fixed duration, deliberately + /// **not** [`Self::avg_window_samples`]. + /// + /// The chart's averaging is an operator preference; this window is a + /// measurement. Tying the two together made a display knob set the precision + /// of the Pockels calibration: at the bench's 500 kSa/s the default of four + /// samples published 8 µs of signal per settled `CONST` code, so every sweep + /// point carried ~12 mV of scatter against a 51 mV lobe and the fit reported + /// a perfect curve as 22 % residual (ADR 019). + /// + /// [`LEVEL_WINDOW_SECONDS`] is a duration rather than a sample count because + /// what averages noise down is time × bandwidth, not samples — and 20 ms in + /// particular is one full mains period, so a boxcar of that length has a null + /// at 50 Hz and every harmonic of it. + fn level_window_samples(&self, rate_hz: u32) -> usize { + if rate_hz == 0 { + return self.avg_window_samples(rate_hz); + } + ((f64::from(rate_hz) * LEVEL_WINDOW_SECONDS).round() as usize).max(1) + } + + /// Settled detector level over the measurement window, published on the + /// contract in **raw** detector volts — never `display_volts`, so a consumer + /// does not have to know the display mode, and never the optical geometry /// transform, which needs an anchor this reading must not depend on. /// - /// Deliberately fail-open where [`Self::optical_summary`] is fail-closed: + /// Deliberately fail-open where [`Self::optical_summary_result`] is + /// fail-closed: /// a transfer-curve sweep needs a level exactly at the excitation null, /// where the reject-port detector is brightest and may rail. Clipping is /// reported rather than refused. @@ -2018,7 +2046,7 @@ impl StageAPhotodiodePlugin { return None; } let window = self - .avg_window_samples(state.rate_hz) + .level_window_samples(state.rate_hz) .min(state.samples.len()); let start = state.samples.len() - window; let summary = state.range_summary(start, state.samples.len()); @@ -2026,6 +2054,10 @@ impl StageAPhotodiodePlugin { return None; } let full_scale = ADC_MAX_CODE as u16; + // Span-relative, like the contrast estimator: this detector runs a few + // codes above zero, and a fixed margin calls every one of its windows + // truncated. + let margin = near_rail_margin(summary.max.saturating_sub(summary.min)); Some(PhotodiodeLevelV1 { mean_volts: code_to_volts(summary.mean()), // `code_to_volts` is a pure scale, so it maps a code difference to @@ -2033,8 +2065,7 @@ impl StageAPhotodiodePlugin { peak_to_peak_volts: code_to_volts(f64::from(summary.max - summary.min)), sample_count: summary.count as u64, end_sample_index: state.ring_first_index + state.samples.len() as u64, - clipped: summary.min <= CLIP_MARGIN_CODES - || summary.max >= full_scale.saturating_sub(CLIP_MARGIN_CODES), + clipped: summary.min <= margin || summary.max >= full_scale.saturating_sub(margin), }) } @@ -2075,6 +2106,7 @@ impl StageAPhotodiodePlugin { let avg_window = self.avg_window_samples(state.rate_hz); let avg_enabled = avg_window > 1; + let anchor = self.total_power_volts(&state); let mut mean_points = Vec::with_capacity(MAX_PLOT_BUCKETS + 1); let mut min_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); @@ -2096,13 +2128,13 @@ impl StageAPhotodiodePlugin { }; mean_points.push(Series1dPoint { x, - y: self.display_volts(bucket.mean()), + y: self.display_volts(bucket.mean(), anchor), }); if decimating { // EXCITATION inverts the axis, so min/max swap roles. let (low, high) = ( - self.display_volts(f64::from(bucket.min)), - self.display_volts(f64::from(bucket.max)), + self.display_volts(f64::from(bucket.min), anchor), + self.display_volts(f64::from(bucket.max), anchor), ); min_points.push(Series1dPoint { x, @@ -2120,7 +2152,7 @@ impl StageAPhotodiodePlugin { let window = state.range_summary(window_start, last + 1); avg_points.push(Series1dPoint { x, - y: self.display_volts(window.mean()), + y: self.display_volts(window.mean(), anchor), }); } bucket_start = bucket_end; @@ -2274,7 +2306,7 @@ impl StageAPhotodiodePlugin { } fn status_dataset(&self) -> TableDatasetV1 { - let (latest, rate_hz, average, integrity, stream_error) = match self.shared.lock() { + let (latest, rate_hz, average, integrity, stream_error, anchor) = match self.shared.lock() { Ok(state) => ( state.latest, state.rate_hz, @@ -2284,8 +2316,9 @@ impl StageAPhotodiodePlugin { state.device_dropped, state.crc_failures, state.resync_bytes, state.segments ), state.error.clone(), + self.total_power_volts(&state), ), - Err(_) => (None, 0, None, String::new(), None), + Err(_) => (None, 0, None, String::new(), None, None), }; let state_text = if self.connected() { format!("reading ({})", self.port_hint) @@ -2300,7 +2333,7 @@ impl StageAPhotodiodePlugin { let (code_text, value_text) = match latest { Some(sample) => ( format!("{sample}"), - format!("{:.4} V", self.display_volts(f64::from(sample))), + format!("{:.4} V", self.display_volts(f64::from(sample), anchor)), ), None => ("—".into(), "—".into()), }; @@ -2309,7 +2342,7 @@ impl StageAPhotodiodePlugin { let window = self.avg_window_samples(rate_hz); format!( "{:.4} V ({} spl ≈ {:.2} ms)", - self.display_volts(code), + self.display_volts(code, anchor), window, if rate_hz > 0 { window as f64 * 1_000.0 / f64::from(rate_hz) @@ -2829,10 +2862,14 @@ impl Plugin for StageAPhotodiodePlugin { SettingsSection { label: "Photodiode readout".into(), description: Some( - "Reads the free-running PDA1 sample stream on the Teensy's SECOND serial \ - port (firmware 0.4.0+, 20 kSa/s default). EXCITATION shows \ - I_exc = I_tot − I_pd: the diode sits behind the PBS and sees the light \ - removed from the excitation beam." + "Reads the photodiode on the Teensy's SECOND serial port. This is what \ + measures the modulation depth the A1 plugin records against.\n\n\ + The detector sits behind the beamsplitter, so it sees the light taken \ + *out* of the excitation beam. The total power I_tot is the brightest \ + reading it has taken since the port opened — the excitation is fully \ + extinguished there, so that reading is I_tot by construction. Nothing \ + to enter: the Pockels transfer sweep walks the whole lobe and lands on \ + it. EXCITATION mode subtracts the live reading from it." .into(), ), default_open: true, @@ -2841,9 +2878,9 @@ impl Plugin for StageAPhotodiodePlugin { key: "port".into(), label: "Port".into(), tooltip: Some( - "auto (recommended) listens on the attached usbmodem ports and \ - picks the one streaming PDA1 sample frames — the Teensy stream \ - port; mock = synthetic data" + "auto (recommended) finds the Teensy port that is sending \ + photodiode samples. mock produces fake data for testing without \ + hardware." .into(), ), kind: SettingKind::Enum { @@ -2866,7 +2903,9 @@ impl Plugin for StageAPhotodiodePlugin { key: "mode".into(), label: "Mode".into(), tooltip: Some( - "RAW: ADC code and volts as measured. EXCITATION: I_tot − I_pd" + "RAW shows what the detector reads. EXCITATION shows the \ + excitation beam instead (total power minus the detector \ + reading) — use this one to measure the modulation depth." .into(), ), kind: SettingKind::Enum { @@ -2874,74 +2913,6 @@ impl Plugin for StageAPhotodiodePlugin { default: mode_default, }, }, - SettingItem { - key: "reference_volts".into(), - label: "Reference I_tot".into(), - tooltip: Some( - "Total power reference for EXCITATION mode, in photodiode volts: \ - the PD reading with the full beam diverted into the diode. The field \ - accepts 1 µV increments; Capture dark does not set this value." - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: ADC_FULL_SCALE_VOLTS, - speed: VOLTAGE_INPUT_STEP_VOLTS, - default: self.reference_volts, - }, - }, - SettingItem { - key: "reference_anchor_id".into(), - label: "I_tot anchor id".into(), - tooltip: Some( - "Stable identifier for the measured full-extinction reference \ - (for example the calibration/run id)." - .into(), - ), - kind: SettingKind::Text { - default: self.reference_anchor_id.clone(), - }, - }, - SettingItem { - key: "reference_confirmed".into(), - label: "I_tot measured and current".into(), - tooltip: Some( - "Confirm only after measuring I_tot for the current optical \ - configuration. Changing the value or anchor id clears this." - .into(), - ), - kind: SettingKind::Bool { - default: self.reference_confirmed, - }, - }, - SettingItem { - key: "dark_volts".into(), - label: "Dark level".into(), - tooltip: Some( - "Measured detector offset in photodiode volts with the beam \ - blocked. The field accepts 1 µV increments; Capture dark can fill \ - it from the current sample cache." - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: ADC_FULL_SCALE_VOLTS, - speed: VOLTAGE_INPUT_STEP_VOLTS, - default: self.dark_volts, - }, - }, - SettingItem { - key: "capture_dark".into(), - label: "Capture dark".into(), - tooltip: Some( - "Block the beam and wait until earlier illuminated samples have \ - left the cache, then press. Uses the mean of every sample currently \ - retained in the cache as Dark level; it does not measure I_tot or \ - start a separate acquisition." - .into(), - ), - kind: SettingKind::Button { enabled: true }, - }, SettingItem { key: "window_s".into(), label: "Chart window".into(), @@ -3025,11 +2996,12 @@ impl Plugin for StageAPhotodiodePlugin { SettingsSection { label: "Data".into(), description: Some( - "Monitor cache and disk recording. The cache always holds the last \ - N seconds; recording tees every incoming frame to a .pdq file \ - (+ JSON sidecar) so length is disk-bound. CSV/PDQ store raw codes \ - and raw volts on the device clock; mode and reference go into the \ - sidecar." + "Recording the photodiode on its own. The A1 plugin drives its own \ + recordings and does not need anything here.\n\n\ + The last few seconds are always kept in memory for the chart; recording \ + writes everything to disk instead, so it can run as long as you have \ + space. Files store the raw readings, with the mode and total power in a \ + companion file." .into(), ), default_open: false, @@ -3049,8 +3021,10 @@ impl Plugin for StageAPhotodiodePlugin { key: "cache_s".into(), label: "Cache length".into(), tooltip: Some( - "Seconds of raw samples kept in memory for the chart and \ - cache snapshots." + "How many seconds of samples to keep in memory. This is also the \ + stretch the modulation depth is measured over, so it must cover \ + at least one full cycle of your slowest frequency — raise it if \ + A1 says the window is too short." .into(), ), kind: SettingKind::F64Drag { @@ -3122,9 +3096,6 @@ impl Plugin for StageAPhotodiodePlugin { .unwrap_or(0); Some(json!(index)) } - "reference_volts" => Some(json!(self.reference_volts)), - "reference_anchor_id" => Some(json!(self.reference_anchor_id)), - "reference_confirmed" => Some(json!(self.reference_confirmed)), "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), "show_markers" => Some(json!(self.show_markers)), @@ -3150,8 +3121,6 @@ impl Plugin for StageAPhotodiodePlugin { // live worker (see PressLatch). "record_start" => Some(self.press_record_start.value()), "record_stop" => Some(self.press_record_stop.value()), - "dark_volts" => Some(json!(self.dark_volts)), - "capture_dark" => Some(self.press_capture_dark.value()), "save_snapshot" => Some(self.press_save_snapshot.value()), _ => None, } @@ -3186,40 +3155,6 @@ impl Plugin for StageAPhotodiodePlugin { .ok_or_else(|| format!("unknown mode: {name} (RAW/EXCITATION)"))?; Ok(()) } - "reference_volts" => { - let volts = value.as_f64().ok_or("reference_volts must be a number")?; - let volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); - if self.reference_volts != volts { - self.reference_volts = volts; - self.reference_confirmed = false; - } - Ok(()) - } - "reference_anchor_id" => { - let anchor_id = value - .as_str() - .ok_or("reference_anchor_id must be a string")? - .trim() - .to_owned(); - if self.reference_anchor_id != anchor_id { - self.reference_anchor_id = anchor_id; - self.reference_confirmed = false; - } - Ok(()) - } - "reference_confirmed" => { - let confirmed = value - .as_bool() - .ok_or("reference_confirmed must be a boolean")?; - if confirmed && self.reference_anchor_id.trim().is_empty() { - return Err("set a non-empty I_tot anchor id before confirming".into()); - } - if confirmed && self.reference_volts <= self.dark_volts { - return Err("I_tot must be above the measured dark level".into()); - } - self.reference_confirmed = confirmed; - Ok(()) - } "show_markers" => { self.show_markers = value.as_bool().ok_or("show_markers must be a boolean")?; Ok(()) @@ -3302,23 +3237,14 @@ impl Plugin for StageAPhotodiodePlugin { } Ok(()) } - "dark_volts" => { - let volts = value.as_f64().ok_or("dark_volts must be a number")?; - self.dark_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); - Ok(()) - } - "capture_dark" => { - // Edge-guarded like every other effectful arm: the host - // re-applies the whole settings snapshot on each sync. - if self.press_capture_dark.accept(&value) { - match self.capture_dark() { - Ok(()) => self.last_error = None, - Err(err) => self.last_error = Some(err), - } - self.generation.fetch_add(1, Ordering::Relaxed); - } - Ok(()) - } + // Accepted and ignored so a config saved before ADR 024 still + // loads. The anchor is learned from the stream and dark cancels out + // of the complement, so there is nothing left for these to set. + "reference_volts" + | "reference_anchor_id" + | "reference_confirmed" + | "dark_volts" + | "capture_dark" => Ok(()), "save_snapshot" => { // Edge-guarded: the host re-applies the full settings snapshot // on every sync, and an unguarded arm wrote one cache file per @@ -3338,14 +3264,15 @@ impl Plugin for StageAPhotodiodePlugin { fn status_entries(&self) -> Vec { let mut entries = Vec::new(); - let (latest, rate_hz, average, stream_error) = match self.shared.lock() { + let (latest, rate_hz, average, stream_error, anchor) = match self.shared.lock() { Ok(state) => ( state.latest, state.rate_hz, self.current_average_code(&state), state.error.clone(), + self.total_power_volts(&state), ), - Err(_) => (None, 0, None, None), + Err(_) => (None, 0, None, None, None), }; entries.push(StatusEntry::Text(if self.connected() { if rate_hz > 0 { @@ -3363,9 +3290,12 @@ impl Plugin for StageAPhotodiodePlugin { code_to_volts(f64::from(sample)) ))), Mode::Excitation => entries.push(StatusEntry::Text(format!( - "Excitation: {:.4} V (I_tot={:.3} V, PD={:.4} V)", - self.display_volts(f64::from(sample)), - self.reference_volts, + "Excitation: {:.4} V (I_tot={}, PD={:.4} V)", + self.display_volts(f64::from(sample), anchor), + match anchor { + Some(volts) => format!("{volts:.4} V"), + None => "learning…".into(), + }, code_to_volts(f64::from(sample)) ))), } @@ -3375,7 +3305,7 @@ impl Plugin for StageAPhotodiodePlugin { if window > 1 { entries.push(StatusEntry::Text(format!( "Avg ({window} spl): {:.4} V", - self.display_volts(average) + self.display_volts(average, anchor) ))); } } @@ -3389,11 +3319,6 @@ impl Plugin for StageAPhotodiodePlugin { optical.excitation_min_volts, optical.excitation_max_volts ))); - if self.dark_volts <= 0.0 { - entries.push(StatusEntry::Text( - "a is uncorrected for dark — capture a dark level".into(), - )); - } } // A withheld `a` is a fail-closed refusal, not an absence of data: // say which gate rejected the window so the operator can fix it. @@ -3532,11 +3457,15 @@ mod tests { let mut plugin = StageAPhotodiodePlugin::default(); plugin.set_runtime_role(PluginRuntimeRole::LiveWorker); plugin.effects_allowed = true; - plugin.reference_anchor_id = "test-itot".into(); - plugin.reference_confirmed = true; plugin } + /// Pins the learned total-power anchor to a known `I_tot`, standing in for + /// the excitation null the Pockels sweep drives the detector through. + fn anchor_at(state: &mut SharedState, volts: f64) { + state.observed_peak_code = Some(volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS); + } + /// A clean rejected-port sine: the detector swings around `center` while /// the excitation is its complement against `I_tot`. fn rejected_port_samples(center: f64, amplitude: f64, count: usize) -> Vec { @@ -3610,11 +3539,11 @@ mod tests { // `a` is peak-to-peak. Below one full cycle the robust extrema see an // arc of the sine, so `a` comes out low — and A1's a₀ lock divides by // it, inflating its drive against a bias it cannot see. Fail closed. - let mut plugin = live_plugin(); - plugin.reference_volts = 3.0; + let plugin = live_plugin(); // 0.5 Hz at 20 kSa/s = 40 000 samples per cycle; retain 0.6 of one. - let partial = slow_sine_state(40_000, 24_000, 200_000); + let mut partial = slow_sine_state(40_000, 24_000, 200_000); + anchor_at(&mut partial, 3.0); let error = plugin .optical_summary_result(&partial) .expect_err("a partial cycle must not publish an a"); @@ -3625,7 +3554,8 @@ mod tests { // Two whole cycles of the same drive: published, and the window is // reported so a consumer can wait it out before trusting a re-read. - let whole = slow_sine_state(40_000, 80_000, 200_000); + let mut whole = slow_sine_state(40_000, 80_000, 200_000); + anchor_at(&mut whole, 3.0); let summary = plugin .optical_summary(&whole) .expect("two whole cycles estimate"); @@ -3674,8 +3604,8 @@ mod tests { // is plotting, so a display toggle must not move a published // scientific quantity. A1's amplitude sweep settles on this value. let mut plugin = live_plugin(); - plugin.reference_volts = 3.0; - let state = rejected_port_state(1_600.0, 700.0, 4_096, true); + let mut state = rejected_port_state(1_600.0, 700.0, 4_096, true); + anchor_at(&mut state, 3.0); plugin.mode = Mode::Raw; let raw = plugin.optical_summary(&state).expect("raw display"); @@ -3683,8 +3613,8 @@ mod tests { let excitation = plugin.optical_summary(&state).expect("excitation display"); assert_eq!(raw.measured_log_contrast, excitation.measured_log_contrast); - assert_eq!(raw.calibration.anchor_id, "test-itot"); - assert_eq!(excitation.calibration.anchor_id, "test-itot"); + assert!(raw.calibration.anchor_id.starts_with("observed-peak@")); + assert_eq!(raw.calibration.anchor_id, excitation.calibration.anchor_id); assert!((raw.calibration.total_power_volts - 3.0).abs() < 1e-12); assert!((raw.measured_frequency_hz.expect("marker frequency") - 39.0625).abs() < 1e-12); @@ -3699,18 +3629,17 @@ mod tests { } #[test] - fn optical_contrast_requires_a_confirmed_anchor_and_complete_cycles() { - let mut plugin = live_plugin(); - plugin.reference_volts = 3.0; + fn optical_contrast_requires_a_learned_anchor_and_complete_cycles() { + let plugin = live_plugin(); let mut state = rejected_port_state(1_600.0, 700.0, 4_096, true); - plugin.reference_confirmed = false; + state.observed_peak_code = None; assert_eq!( plugin.optical_summary_result(&state), Err(EstimateError::MissingTotalPowerAnchor) ); - plugin.reference_confirmed = true; + anchor_at(&mut state, 3.0); state.markers = VecDeque::from([0, 512]); assert!(matches!( plugin.optical_summary_result(&state), @@ -3722,60 +3651,167 @@ mod tests { } #[test] - fn changing_the_total_power_anchor_invalidates_confirmation() { - let mut plugin = live_plugin(); - plugin - .set_setting("reference_volts", json!(2.9)) - .expect("reference"); - assert!(!plugin.reference_confirmed); + fn a_withheld_contrast_publishes_its_reason_on_the_contract() { + // A1 gates the a₀ lock and the frequency ladder on `a`. When `a` is + // refused, the reason has to travel with the absent summary or the only + // thing the operator can read is that it is missing. + let plugin = live_plugin(); + { + let mut state = plugin.shared.lock().expect("state"); + *state = rejected_port_state(1_600.0, 700.0, 4_096, true); + state.observed_peak_code = None; + } - plugin.reference_confirmed = true; - plugin - .set_setting("reference_anchor_id", json!("itot-next")) - .expect("anchor id"); - assert!(!plugin.reference_confirmed); + let summary = plugin.control_summary(); + assert!(summary.optical_summary.is_none()); + assert_eq!( + summary.optical_unavailable.as_deref(), + Some(EstimateError::MissingTotalPowerAnchor.to_string().as_str()) + ); + + { + let mut state = plugin.shared.lock().expect("state"); + anchor_at(&mut state, 3.0); + } + let summary = plugin.control_summary(); + assert!(summary.optical_summary.is_some()); + assert!( + summary.optical_unavailable.is_none(), + "an accepted a must not also carry a refusal: {:?}", + summary.optical_unavailable + ); } #[test] - fn captured_dark_level_reaches_the_estimator_and_is_named() { - let mut plugin = live_plugin(); - plugin.reference_volts = 3.0; - let state = rejected_port_state(1_600.0, 700.0, 4_096, true); - - let undarkened = plugin.optical_summary(&state).expect("no dark yet"); - assert_eq!(undarkened.calibration.dark_id, "dark-none"); - assert_eq!(undarkened.calibration.dark_volts, 0.0); - - plugin.dark_volts = 0.05; - let darkened = plugin.optical_summary(&state).expect("with dark"); - assert_eq!(darkened.calibration.dark_id, "dark-measured"); - assert_eq!(darkened.calibration.dark_volts, 0.05); - // A DC dark offset is common to the detector samples and to the - // reference reading, so it cancels out of the complement. Anything - // else means one of the two sides is being corrected without the - // other — which is what would actually bias `a`. + fn the_millivolt_scale_reject_port_detector_still_yields_a() { + // The bench detector operates around 0.5–15 mV, inside the bottom ~20 + // codes of the 12-bit range. That is not the bottom rail, so `a` must be + // published rather than refused as clipped. + let plugin = live_plugin(); + { + let mut state = plugin.shared.lock().expect("state"); + // Detector swinging between ~0.6 and ~18.6 codes = 0.5..15 mV. + *state = rejected_port_state(9.6, 9.0, 4_096, true); + anchor_at(&mut state, 0.015_5); + } + + let summary = plugin.control_summary(); + assert!( + summary.optical_unavailable.is_none(), + "a millivolt-scale window must not be refused: {:?}", + summary.optical_unavailable + ); + let optical = summary.optical_summary.expect("a is published"); + assert!( + optical.measured_log_contrast > 0.0 && optical.measured_log_contrast.is_finite(), + "a = {}", + optical.measured_log_contrast + ); + } + + #[test] + fn the_anchor_is_learned_from_the_brightest_reading_the_detector_takes() { + // The excitation null the Pockels sweep drives through is where the + // reject-port detector reads I_tot. Nothing is entered by hand. + let plugin = live_plugin(); + { + let mut state = plugin.shared.lock().expect("state"); + // A sweep peak, then ordinary modulation well below it. + state.ingest(0, 20_000, 0, &[3_000; SUMMARY_CELL]); + state.ingest(SUMMARY_CELL as u64, 20_000, 0, &[1_200; SUMMARY_CELL * 4]); + } + + let anchor = plugin.learned_anchor_volts().expect("anchor learned"); + assert!( + (anchor - code_to_volts(3_000.0)).abs() < 1e-9, + "anchor {anchor} did not latch on the sweep peak" + ); + } + + #[test] + fn a_single_spike_cannot_become_the_anchor() { + // The latch runs on completed 64-sample cell means, so one outlier + // sample cannot pin I_tot high for every later `a`. + let plugin = live_plugin(); + { + let mut state = plugin.shared.lock().expect("state"); + let mut codes = vec![1_000u16; SUMMARY_CELL]; + codes[7] = 4_095; + state.ingest(0, 20_000, 0, &codes); + } + + let anchor = plugin.learned_anchor_volts().expect("anchor learned"); + assert!( + anchor < code_to_volts(1_100.0), + "a single spike pulled the anchor to {anchor}" + ); + } + + /// The anchor is learned from the detector's own stream, so before any + /// Pockels sweep has driven the excitation to its null the only thing it + /// has seen is the modulation itself — and then the "total power" is barely + /// above the signal. That must refuse, not publish an enormous `a`: the + /// excitation minimum would be dominated by the anchor's own error rather + /// than by the light. + #[test] + fn a_modulation_only_anchor_refuses_instead_of_reporting_a_huge_contrast() { + let plugin = live_plugin(); + // 8 cycles in 4096 samples at 20 kSa/s = ~39 Hz, and a slow 1 Hz case. + for (count, label) in [(4_096usize, "39 Hz"), (160_000, "1 Hz")] { + let state = rejected_port_state(1_600.0, 700.0, count, true); + // No `anchor_at`: whatever `ingest` learned from this trace alone. + match plugin.optical_summary_result(&state) { + // Named, not just "some error": a refusal for want of cycles + // would make this test pass without exercising the anchor at + // all. + Err(EstimateError::TotalPowerBelowSignal { .. }) => {} + Err(other) => panic!("{label}: refused for the wrong reason: {other:?}"), + Ok(summary) => panic!( + "{label}: published a = {} from an anchor that never saw the excitation \ + null (headroom {} V)", + summary.measured_log_contrast, summary.excitation_headroom_volts + ), + } + } + } + + #[test] + fn a_dc_dark_offset_cancels_out_of_the_complement() { + // Both sides of `I_exc = I_tot - I_pd` are readings from the same + // DC-coupled detector, so a dark offset appears in both and cancels + // exactly. That is why there is no dark setting: there is nothing for + // it to correct, and correcting only one side is the actual bug. + let plugin = live_plugin(); + + let contrast_with_offset = |offset: f64| { + let mut state = rejected_port_state(1_600.0 + offset, 700.0, 4_096, true); + // I_tot is read by the same detector, so it carries the offset too. + anchor_at(&mut state, 3.0 + code_to_volts(offset)); + plugin + .optical_summary(&state) + .expect("estimate") + .measured_log_contrast + }; + + let baseline = contrast_with_offset(0.0); + let offset = contrast_with_offset(200.0); assert!( - (darkened.measured_log_contrast - undarkened.measured_log_contrast).abs() < 1e-9, - "dark did not cancel: {} vs {}", - darkened.measured_log_contrast, - undarkened.measured_log_contrast + (baseline - offset).abs() < 1e-9, + "dark did not cancel: {baseline} vs {offset}" ); } #[test] fn a_dark_offset_on_only_one_side_would_bias_the_contrast() { // Guards the invariance above against a regression that dark-corrects - // the detector but leaves the anchor raw (or vice versa): that is the - // asymmetry the estimator contract warns about. + // the detector but leaves the anchor raw (or vice versa). let calibration = AdcCalibration { volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, offset_volts: 0.0, dark_volts: 0.05, full_scale_code: ADC_MAX_CODE as u16, }; - let samples: Vec = rejected_port_samples(1_600.0, 700.0, 4_096) - .into_iter() - .collect(); + let samples: Vec = rejected_port_samples(1_600.0, 700.0, 4_096); let consistent = estimate_contrast( &samples, &calibration, @@ -3799,51 +3835,21 @@ mod tests { } #[test] - fn capture_dark_refuses_a_level_at_or_above_the_anchor() { + fn the_removed_anchor_settings_still_load_from_an_old_config() { + // A config written before ADR 024 must not fail to load; the keys are + // accepted and ignored. let mut plugin = live_plugin(); - plugin.reference_volts = 0.5; - if let Ok(mut state) = plugin.shared.lock() { - state.ingest(0, 20_000, 0, &[4_000; 256]); - } - let err = plugin.capture_dark().expect_err("beam clearly not blocked"); - assert!(err.contains("is not below the I_tot reference"), "{err}"); - assert_eq!(plugin.dark_volts, 0.0); - } - - #[test] - fn capture_dark_uses_the_mean_of_the_retained_cache() { - let mut plugin = live_plugin(); - plugin.reference_volts = 0.015; - if let Ok(mut state) = plugin.shared.lock() { - state.ingest(0, 20_000, 0, &[4, 6, 8]); - } - - plugin.capture_dark().expect("blocked-beam cache accepted"); - - let expected = code_to_volts(6.0); - assert!((plugin.dark_volts - expected).abs() < f64::EPSILON); - assert_eq!( - plugin.last_save_note.as_deref(), - Some("dark level captured: 0.004835 V") - ); - } - - #[test] - fn calibration_voltage_inputs_accept_microvolt_steps() { - let schema = StageAPhotodiodePlugin::default().settings_schema(); - - for key in ["reference_volts", "dark_volts"] { - let item = schema - .sections - .iter() - .flat_map(|section| section.items.iter()) - .find(|item| item.key == key) - .unwrap_or_else(|| panic!("missing {key} setting")); - let SettingKind::F64Drag { speed, .. } = &item.kind else { - panic!("{key} must remain an F64Drag setting"); - }; - assert_eq!(*speed, VOLTAGE_INPUT_STEP_VOLTS); + for (key, value) in [ + ("reference_volts", json!(2.9)), + ("reference_anchor_id", json!("itot-old")), + ("reference_confirmed", json!(true)), + ("dark_volts", json!(0.05)), + ] { + plugin + .set_setting(key, value) + .unwrap_or_else(|error| panic!("{key} rejected: {error}")); } + assert!(plugin.get_setting("reference_volts").is_none()); } #[test] @@ -4039,18 +4045,24 @@ mod tests { assert!((average - 250.0).abs() < 1e-9); } + /// Codes for a full measurement window at `rate_hz`, all at `code`. + fn level_window_codes(rate_hz: u32, code: u16) -> Vec { + vec![code; (f64::from(rate_hz) * LEVEL_WINDOW_SECONDS).round() as usize] + } + #[test] fn published_level_is_raw_volts_and_survives_clipping() { - let mut plugin = StageAPhotodiodePlugin::default(); // window = 4 samples + let mut plugin = StageAPhotodiodePlugin::default(); + let mut codes = level_window_codes(20_000, 200); + codes.extend([100, 200, 300, 400]); let mut state = SharedState::default(); - state.ingest(0, 20_000, 0, &[0, 0, 0, 0, 100, 200, 300, 400]); + state.ingest(0, 20_000, 0, &codes); let level = plugin.current_level(&state).expect("has samples"); - assert!((level.mean_volts - code_to_volts(250.0)).abs() < 1e-9); - assert!((level.peak_to_peak_volts - code_to_volts(300.0)).abs() < 1e-9); - assert_eq!(level.sample_count, 4); - // The window is the newest 4 of 8 ingested samples. - assert_eq!(level.end_sample_index, 8); + // 400 samples at 20 kSa/s = the full 20 ms window, ending on the newest + // sample — not the four the chart happens to be smoothing over. + assert_eq!(level.sample_count, 400); + assert_eq!(level.end_sample_index, codes.len() as u64); assert!(!level.clipped); // EXCITATION display must not leak into the published level: it stays @@ -4062,12 +4074,76 @@ mod tests { // At the rail the optical summary refuses; the level must not, because // that is exactly where a transfer sweep needs a reading. let mut railed = SharedState::default(); - railed.ingest(0, 20_000, 0, &[4_095; 8]); + railed.ingest(0, 20_000, 0, &level_window_codes(20_000, 4_095)); let clipped = plugin.current_level(&railed).expect("still reports"); assert!(clipped.clipped); assert!(plugin.optical_summary(&railed).is_none()); } + #[test] + fn the_published_level_window_ignores_the_chart_averaging_setting() { + // The sweep's precision is a measurement property. Deriving it from the + // chart's averaging made a display knob decide it: at the bench's + // 500 kSa/s the default of four samples published 8 µs per settled CONST + // code, and a clean Pockels curve came back as a 22 % residual (ADR 019). + let mut plugin = StageAPhotodiodePlugin::default(); + let mut state = SharedState::default(); + state.ingest(0, 500_000, 0, &level_window_codes(500_000, 300)); + + let level = plugin.current_level(&state).expect("has samples"); + assert_eq!(level.sample_count, 10_000, "20 ms at 500 kSa/s"); + + for avg in [1, 4, 4_096] { + plugin + .set_setting("avg_samples", json!(avg)) + .expect("valid"); + assert_eq!( + plugin + .current_level(&state) + .expect("has samples") + .sample_count, + level.sample_count, + "avg_samples = {avg} moved the published window" + ); + } + plugin + .set_setting("avg_sync_freq_hz", json!(10.0)) + .expect("valid"); + assert_eq!( + plugin + .current_level(&state) + .expect("has samples") + .sample_count, + level.sample_count, + "the sync-averaging setting moved the published window" + ); + } + + #[test] + fn a_millivolt_scale_level_is_not_reported_as_railed() { + // The reject-port detector's dark end sits a few codes above zero. A + // fixed rail margin called every one of those windows clipped, which the + // transfer fit then reported as "add attenuation and re-measure". + let plugin = StageAPhotodiodePlugin::default(); + let rate = 500_000; + let window = (f64::from(rate) * LEVEL_WINDOW_SECONDS).round() as usize; + // Swinging between codes 6 and 26 — clear of the rail at both ends. + let codes: Vec = (0..window) + .map(|index| if index % 2 == 0 { 6 } else { 26 }) + .collect(); + let mut state = SharedState::default(); + state.ingest(0, rate, 0, &codes); + assert!(!plugin.current_level(&state).expect("has samples").clipped); + + // Driven into the bottom rail, the refusal must survive. + let railed: Vec = (0..window) + .map(|index| if index % 2 == 0 { 0 } else { 26 }) + .collect(); + let mut state = SharedState::default(); + state.ingest(0, rate, 0, &railed); + assert!(plugin.current_level(&state).expect("has samples").clipped); + } + #[test] fn series_dataset_decimates_with_envelope_and_average() { let mut plugin = StageAPhotodiodePlugin::default(); @@ -4116,16 +4192,18 @@ mod tests { } #[test] - fn excitation_mode_inverts_against_the_reference() { + fn excitation_mode_inverts_against_the_learned_anchor() { let mut plugin = StageAPhotodiodePlugin::default(); plugin.set_setting("mode", json!("EXCITATION")).unwrap(); - plugin.set_setting("reference_volts", json!(2.0)).unwrap(); // I_pd = 0.5 V → I_exc = I_tot − I_pd = 1.5 V. let code = 0.5 * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS; - assert!((plugin.display_volts(code) - 1.5).abs() < 1e-9); + assert!((plugin.display_volts(code, Some(2.0)) - 1.5).abs() < 1e-9); + // Before the anchor is learned there is nothing to take a complement + // against, so the raw reading is shown rather than a wrong one. + assert!((plugin.display_volts(code, None) - 0.5).abs() < 1e-9); // RAW mode shows the measured voltage itself. plugin.set_setting("mode", json!("RAW")).unwrap(); - assert!((plugin.display_volts(code) - 0.5).abs() < 1e-9); + assert!((plugin.display_volts(code, Some(2.0)) - 0.5).abs() < 1e-9); } #[test] diff --git a/scripts/install-built-plugins.sh b/scripts/install-built-plugins.sh index d39a8be..4822be9 100755 --- a/scripts/install-built-plugins.sh +++ b/scripts/install-built-plugins.sh @@ -146,6 +146,13 @@ for plugin_dir in "${repo_root}"/plugins/*; do installed_library_path="${install_dir}/$(basename "${library_path}")" cp "${library_path}" "${installed_library_path}" rewrite_macos_install_name "${installed_library_path}" + # Operator-facing example files a plugin ships alongside its library (A1's + # recording protocols). The bench has the installed folder, not the repo, + # so an example the settings panel points at has to travel with the plugin. + if [[ -d "${plugin_dir}/protocols" ]]; then + rm -rf "${install_dir}/protocols" + cp -R "${plugin_dir}/protocols" "${install_dir}/protocols" + fi echo "Installed ${plugin_id} -> ${install_dir}" installed=$((installed + 1)) done diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs index dfb2eaf..78fab3b 100644 --- a/stage-a-io/src/estimator.rs +++ b/stage-a-io/src/estimator.rs @@ -26,6 +26,11 @@ //! - refuses to produce a value at all when the window clips (top/bottom of //! the ADC range), has no headroom above dark, or the total-power anchor is //! below the measured signal — a wrong `a` is worse than no `a`. +//! +//! Rail detection is *span-relative*: the near-rail margin is capped at a small +//! fraction of the window's own peak-to-peak span, so a detector operating a few +//! millivolts above zero is not mistaken for one truncating at the bottom rail. +//! The rails themselves stay guarded at every gain. use serde::{Deserialize, Serialize}; @@ -132,44 +137,55 @@ impl std::fmt::Display for EstimateError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::MissingTotalPowerAnchor => f.write_str( - "no confirmed, named total-power anchor; excitation contrast a is withheld", + "no total power I_tot has been observed yet — let the photodiode stream for a \ + moment; it learns I_tot from the brightest reading it sees, which the Pockels \ + transfer sweep produces exactly", ), Self::IncompleteModulationCycles { marker_count, max_samples, } => write!( f, - "no marker-bounded window with at least two complete cycles fits in \ - {max_samples} samples ({marker_count} usable markers)" + "no stretch of samples covers two whole modulation cycles between triggers \ + ({marker_count} trigger(s) in the last {max_samples} samples) — lower the \ + frequency, or raise the photodiode cache length" + ), + Self::TooFewSamples { count, minimum } => write!( + f, + "only {count} samples have arrived so far, and {minimum} are needed — wait a \ + moment, or check the photodiode stream is running" ), - Self::TooFewSamples { count, minimum } => { - write!(f, "only {count} samples (minimum {minimum})") - } Self::Clipped { low_fraction_permille, high_fraction_permille, } => write!( f, - "ADC clipping: {low_fraction_permille}‰ low / {high_fraction_permille}‰ high" + "the signal is hitting the ends of the detector's range \ + ({low_fraction_permille}‰ at the bottom, {high_fraction_permille}‰ at the top) \ + — lower the drive amplitude or the detector gain" + ), + Self::NoHeadroomAboveDark => f.write_str( + "the signal never rises above the dark level — check the dark level is right and \ + that light is reaching the detector", ), - Self::NoHeadroomAboveDark => { - f.write_str("dark-corrected minimum is not positive; a is undefined") - } Self::TotalPowerBelowSignal { total_power_volts, detector_max_volts, } => write!( f, - "total-power anchor {total_power_volts:.4} V is not above the detector \ - maximum {detector_max_volts:.4} V; a is undefined" + "the excitation never dims below the brightest the detector has been \ + (I_tot {total_power_volts:.4} V vs. {detector_max_volts:.4} V now), so there is \ + no complement left to take a contrast of — run the Pockels transfer sweep so the \ + detector sees the excitation null and learns the real I_tot" ), Self::WindowShorterThanCycle { covered_cycles, window_seconds, } => write!( f, - "the {window_seconds:.2} s window covers only {covered_cycles:.2} modulation \ - cycles; a needs at least one full cycle — raise the cache length" + "the photodiode only watches {window_seconds:.2} s at a time, which is \ + {covered_cycles:.2} of a modulation cycle — it needs at least one whole cycle, \ + so raise the photodiode cache length" ), } } @@ -178,14 +194,48 @@ impl std::fmt::Display for EstimateError { impl std::error::Error for EstimateError {} pub const MIN_SAMPLES: usize = 64; -/// Codes within this margin of the rails count as clipped. +/// Codes within this margin of the rails count as clipped — but never more +/// than [`CLIP_MARGIN_SPAN_FRACTION`] of the window's own span. pub const CLIP_MARGIN_CODES: u16 = 4; +/// Largest share of the observed peak-to-peak span the rail margin may claim. +/// +/// The margin exists to catch a waveform that is *about* to truncate at a rail, +/// which only makes sense while it is small compared to the signal. The Stage-A +/// reject-port detector operates around 0.5–15 mV, i.e. inside the bottom ~20 +/// codes of the 12-bit range, where a fixed 4-code margin covers a third of a +/// perfectly good sine and refused every millivolt-scale window as clipped. +/// Capping it against the span keeps the guard on volt-scale signals, and +/// leaves the true rails (code 0 and full scale) guarded at every gain. +const CLIP_MARGIN_SPAN_FRACTION: f64 = 0.05; /// Reject the window when more than 1‰ of samples clip. pub const MAX_CLIP_FRACTION: f64 = 0.001; /// Robust extrema: 1st / 99th percentile. const LOW_PERCENTILE: f64 = 0.01; const HIGH_PERCENTILE: f64 = 0.99; +/// Rail margin for a window whose observed excursion is `span_codes`: the fixed +/// code margin, shrunk so it can never swallow a signal that legitimately sits +/// close to a rail. Returns 0 for spans narrower than +/// `1 / CLIP_MARGIN_SPAN_FRACTION` codes, which leaves exactly the rails +/// themselves classified as clipped. +/// +/// Shared with the photodiode owner's published level, which faces the same +/// question one window at a time: the Stage-A reject-port detector runs a few +/// codes above zero, and a fixed margin calls every one of those windows +/// truncated. +pub fn near_rail_margin(span_codes: u16) -> u16 { + let allowed = (f64::from(span_codes) * CLIP_MARGIN_SPAN_FRACTION).floor(); + allowed.min(f64::from(CLIP_MARGIN_CODES)) as u16 +} + +/// [`near_rail_margin`] for a window still held as raw codes. +fn clip_margin_codes(codes: &[u16]) -> u16 { + let (min, max) = codes.iter().fold((u16::MAX, u16::MIN), |(lo, hi), &code| { + (lo.min(code), hi.max(code)) + }); + near_rail_margin(max.saturating_sub(min)) +} + /// Estimates the excitation log-contrast from one settled, phase-attributed /// ADC window. The window must span at least a few full modulation cycles; /// enforcing that is the caller's job (it knows the drive frequency). The @@ -203,10 +253,9 @@ pub fn estimate_contrast( }); } - let low_clip_threshold = CLIP_MARGIN_CODES; - let high_clip_threshold = calibration - .full_scale_code - .saturating_sub(CLIP_MARGIN_CODES); + let margin = clip_margin_codes(codes); + let low_clip_threshold = margin; + let high_clip_threshold = calibration.full_scale_code.saturating_sub(margin); let low_clipped = codes.iter().filter(|&&c| c <= low_clip_threshold).count(); let high_clipped = codes.iter().filter(|&&c| c >= high_clip_threshold).count(); let low_clip_fraction = low_clipped as f64 / codes.len() as f64; @@ -398,4 +447,49 @@ mod tests { .expect("spiked"); assert!((clean.a - spiked.a).abs() < 0.005); } + + #[test] + fn accepts_the_millivolt_scale_reject_port_window() { + // The Stage-A reject-port detector operates around 0.5–15 mV, i.e. the + // whole waveform lives inside the bottom ~20 codes of the 12-bit range + // (0.806 mV per code). None of those codes is the bottom rail, so the + // window must estimate rather than be refused as clipped. + let calibration = AdcCalibration::default(); + let per_code = calibration.volts_per_code; + let detector_low = 0.000_5; + let detector_high = 0.015; + let center = (detector_high + detector_low) / 2.0 / per_code; + let amplitude = (detector_high - detector_low) / 2.0 / per_code; + let codes = sine_codes(center, amplitude, 4_096); + let total_power_volts = 0.015_5; + + let estimate = estimate_contrast( + &codes, + &calibration, + ContrastGeometry::RejectedComplement { total_power_volts }, + ) + .expect("a millivolt-scale reject-port window must estimate"); + assert!( + estimate.a > 0.0 && estimate.a.is_finite(), + "a = {}", + estimate.a + ); + } + + #[test] + fn still_rejects_a_window_pinned_at_the_bottom_rail() { + // Same millivolt scale, but driven below zero: the waveform truncates + // at code 0 and `a` would be biased high, so the refusal must survive + // the span-relative margin. + let codes = sine_codes(4.0, 9.0, 4_096); + let err = estimate_contrast( + &codes, + &AdcCalibration::default(), + ContrastGeometry::RejectedComplement { + total_power_volts: 0.015_5, + }, + ) + .expect_err("a rail-pinned window must be refused"); + assert!(matches!(err, EstimateError::Clipped { .. }), "{err:?}"); + } } diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs index bf726e7..5e17e58 100644 --- a/stage-a-io/src/lib.rs +++ b/stage-a-io/src/lib.rs @@ -34,7 +34,8 @@ pub mod wire; pub use client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; pub use estimator::{ - estimate_contrast, AdcCalibration, ContrastEstimate, ContrastGeometry, EstimateError, + estimate_contrast, near_rail_margin, AdcCalibration, ContrastEstimate, ContrastGeometry, + EstimateError, }; pub use mock::{MockController, MockState, MockWave}; pub use pdq::{ diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index 9e975eb..d3bc593 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -308,6 +308,25 @@ pub enum ModulationCommandV1 { SetDriveFrequency { frequency_millihz: u64, }, + /// Retarget the armed drive's *operating point* — the normalized cycle-mean + /// lobe coordinate `ū`, in milli-units — leaving the waveform, depth, + /// frequency and calibration alone. The third axis alongside + /// [`ModulationCommandV1::SetOpticalDepth`] and + /// [`ModulationCommandV1::SetDriveFrequency`], and scoped the same way: + /// leased only, rejected when the link is closed or the armed drive has no + /// operating point to retarget (manual DAC method). + /// + /// This is what makes an `I_k` sweep possible. `ū` is a *normalized* lobe + /// coordinate, not physical flux — but it is the one knob that moves the + /// mean illumination without touching the depth, so a protocol that walks + /// it walks the bench's brightness axis. + /// + /// The owner parks the operator's armed `ū` on the first point and restores + /// it when the lease ends, so a finished sweep does not leave the bench on + /// its last one. + SetOperatingPoint { + mean_u_milli: u32, + }, PrepareA1 { configuration: A1AcquisitionConfigV1, }, @@ -373,9 +392,16 @@ pub struct OpticalDriveStateV1 { /// field (`u_g` for log-sine, `u_c` for linear-sine). pub internal_u_milli: u32, pub depth_a_milli: u32, + /// DAC code at the excitation minimum of the lobe in use. pub v_null_dac: u16, - /// Null-to-maximum half-wave-voltage span in DAC codes. - pub v_pi_dac: u16, + /// DAC code at the excitation maximum of the same lobe. + /// + /// An absolute code, like `v_null_dac` — not the half-wave *span* between + /// them, which the earlier `v_pi_dac` field carried. One lobe is named by + /// two codes an operator can point at on the transfer curve, and mixing an + /// absolute code with a distance is exactly the confusion this pair exists + /// to prevent (ADR 016). The span is `v_peak_dac − v_null_dac`. + pub v_peak_dac: u16, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -607,12 +633,17 @@ pub struct PhotodiodeOpticalSummaryV1 { pub covered_cycles: Option, } -/// Settled detector level over the newest averaging window, in **raw detector -/// volts**: the ADC affine map only, before dark subtraction and before any -/// [`PhotodiodeOpticalSummaryV1`] geometry transform. Unlike the optical +/// Settled detector level over the owner's **measurement** window, in **raw +/// detector volts**: the ADC affine map only, before dark subtraction and before +/// any [`PhotodiodeOpticalSummaryV1`] geometry transform. Unlike the optical /// summary this never refuses — it stays present while the window clips (see /// `clipped`), because a consumer sweeping a static transfer curve needs a /// level exactly where the detector is brightest. +/// +/// The window is fixed by the owner and **independent of any display setting**; +/// `sample_count` reports how long it actually was. Deriving it from the chart's +/// averaging preference instead let a display knob set the precision of the +/// Pockels transfer calibration downstream (ADR 019). #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct PhotodiodeLevelV1 { pub mean_volts: f64, @@ -660,6 +691,17 @@ pub struct PhotodiodeSummaryV1 { pub active_recording: Option, pub last_finalized_recording: Option, pub optical_summary: Option, + /// Why `optical_summary` is absent, in the owner's own words. + /// + /// A withheld `a` is a fail-closed refusal, not missing data, and every + /// automation client that gates on `a` has to be able to tell the operator + /// which gate rejected the window — otherwise the only readout is "no `a`" + /// and the fix is a guess. Set exactly when `optical_summary` is `None` and + /// a window was available to judge. + /// + /// Additive in V1: absent from older owners, and older consumers ignore it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optical_unavailable: Option, pub synchronization: SynchronizationV1, pub last_response: Option, pub freshness: FreshnessV1, @@ -820,7 +862,7 @@ mod tests { internal_u_milli: 355, depth_a_milli: 1_000, v_null_dac: 1_630, - v_pi_dac: 860, + v_peak_dac: 1_160, }), }; let encoded = serde_json::to_vec(&snapshot).expect("serializes"); From 938d76b7db14b90c015440e45e2a59d7baf59a8e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 3 Aug 2026 16:58:15 +0200 Subject: [PATCH 36/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20renew=20le?= =?UTF-8?q?ases=20against=20the=20deadline=20the=20owner=20granted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both device owners cap the automation lease TTL they hand out at 60 s — a dead-man switch, and correct — but the clamp is silent: the request comes back `Applied`, so A1 believed it held the drive for a whole survey when it held it for a minute. Renewing once per point was therefore only ever right for points shorter than the cap. The shipped example protocol has a 40 s row, every row also pays the camera and photodiode handshakes, and `acquire_photodiode` asks for `duration_s + 60 s`, so any longer recording outlived its own leases. Past the deadline the modulation owner does what an expired lease must do — STOP, output off — and that single event surfaced as three unrelated-looking faults at once: the owner refusing every later command for want of a lease, the sidecar refusing to write because the photodiode had no fresh optical summary, and a "no trigger signal" line that read exactly like an unplugged EXT_TRIGGER cable but was the drive being off. A1 now heartbeats both leases against `expires_at_unix_ms` from the owner's own snapshot — which both owners already published and A1 never read — renewing once less than 20 s of the granted window is left, no more often than every 2 s. The owners' cap is untouched: raising it to survey length would fix the symptom by deleting the safety property that motivated it. Also fixes `on_discontinuity` asking `recording.is_active() || sweep.is_some()` to decide whether a SourceChanged was self-inflicted. Starting and stopping the host recorder raises it twice per recording, and between two points of a protocol or a ladder neither is true — so the run's own boundary was treated as an idle-time reset and wiped the survey's pilot windows, background floor and response curve mid-run. It now asks `automation_active()`: the same set `request_stop` winds down. Refs ADR 029. Co-Authored-By: Claude Opus 5 --- ...re-renewed-against-the-granted-deadline.md | 106 ++++++ docs/features/README.md | 2 +- docs/features/stage-a-a1.md | 20 +- plugins/stage-a-a1/src/runtime.rs | 314 +++++++++++++++++- 4 files changed, 428 insertions(+), 14 deletions(-) create mode 100644 docs/adr/029-stage-a-leases-are-renewed-against-the-granted-deadline.md diff --git a/docs/adr/029-stage-a-leases-are-renewed-against-the-granted-deadline.md b/docs/adr/029-stage-a-leases-are-renewed-against-the-granted-deadline.md new file mode 100644 index 0000000..137878e --- /dev/null +++ b/docs/adr/029-stage-a-leases-are-renewed-against-the-granted-deadline.md @@ -0,0 +1,106 @@ +# ADR 029 — A leased run renews against the deadline the owner granted, not the one it asked for + +- **Status:** Accepted +- **Date:** 2026-08-03 +- **Relates to:** ADR 005 (device ownership), ADR 007 (owner orchestration), + ADR 009 (recording coordinator), ADR 027 (declarative protocols), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +Both Stage-A device owners hand out an automation lease with a TTL, and both +**cap** the TTL they grant: + +```rust +// modulation and photodiode, independently +const MAX_LEASE_TTL_MS: u64 = 60_000; +fn lease_deadline(ttl_ms: u64) -> u64 { + now_unix_ms().saturating_add(ttl_ms.clamp(MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS)) +} +``` + +The cap is a dead-man switch and it is right: an automation client that crashes +mid-run must not leave the laser driven indefinitely. A lease that lapses makes +the modulation owner queue `STOP` + `MOD wave=OFF`, and makes the photodiode +owner finalize its recording as `LeaseExpired`. + +A1 asked for a TTL covering its whole run — a frequency ladder, an amplitude +sweep, or a protocol file's remaining points — and renewed **once per point**, +in the same tick that retargeted the drive. The clamp is silent: the request is +answered `Applied`, so A1 believed it held the drive for forty minutes when the +owner had granted sixty seconds. + +That worked only while every point was shorter than the cap. It is not: + +- the shipped example protocol has a `duration_s = 40, settle_s = 4` row, and + every row also pays the camera start/stop and photodiode + connect/lease/start/finalize handshakes; +- `acquire_photodiode` asks for `duration_s + 60 s`, so **any recording longer + than the cap** outlived its own photodiode lease. + +Past the granted deadline, one root cause surfaced as three unrelated-looking +failures in the same status line: + +| symptom | actual cause | +| --- | --- | +| `the modulation owner requires an active automation lease` | the lease was reaped and the drive safe-offed | +| `cannot write a quantitative A1 sidecar without a fresh photodiode optical summary` | no drive → no modulated light, and the PDQ had been finalized as `LeaseExpired` | +| `Camera: … events, … no trigger signal` | no drive → the Teensy stopped emitting the phase-0 `EXT_TRIGGER` | + +The third is the one that reads as a hardware fault. It sent the operator after +a trigger cable that was never disconnected. + +## Decision + +**The owner's cap stays. The client renews against the deadline the owner +publishes.** + +Both owners already advertise the truth: `ModulationStateV1.lease` and +`PhotodiodeSummaryV1.lease` carry a `LeaseSnapshotV1 { lease_id, holder, +expires_at_unix_ms, .. }`. A1 never read it. + +A1 gains one heartbeat, `drive_lease_heartbeat`, running on every control tick +ahead of the runners: + +- it finds the modulation lease A1 currently holds — outermost runner first, + since a nested run inherits the enclosing lease id — and the photodiode lease + of a recording in flight; +- it renews only what the **owner's own snapshot** confirms A1 is holding, so a + lease the owner has already dropped is not chased; +- it renews once less than `LEASE_RENEW_MARGIN_MS` (20 s) of the granted window + is left, no more often than every `LEASE_RENEW_MIN_INTERVAL_MS` (2 s) — the + control plane ticks at 20 Hz and the owner's snapshot lags a renewal by a tick + or two. + +The per-point renewals stay. They are correct and they cost nothing; the +heartbeat covers the interval between them. + +The whole-run TTL helpers stay too, and keep asking for the run's real remaining +time. That is the honest statement of need, and it is the owner's job — not the +client's — to decide how much of it to grant. + +## Consequences + +- A point may now be arbitrarily long. The protocol's `duration_s` is bounded + by the protocol schema (1..=3600 s), not by an owner's lease cap. +- The dead-man switch is intact: if A1 stops ticking, the heartbeat stops with + it and the lease lapses within the cap, exactly as before. +- A lease A1 loses anyway (owner restart, an operator disconnect) is not + papered over. The heartbeat goes quiet because the owner's snapshot no longer + names A1 as the holder, and the runner's own retarget reports the real + failure in its own words. +- Renewal replies are not routed to any runner. An unmatched `request_id` + already falls through `on_service_reply` untouched, so a heartbeat cannot + be mistaken for a point's retarget outcome. +- The owners were left alone. Raising `MAX_LEASE_TTL_MS` to survey length would + have fixed the symptom by deleting the safety property that motivated it. + +## Also fixed here + +`on_discontinuity` asked `recording.is_active() || sweep.is_some()` to decide +whether a `SourceChanged` was self-inflicted. Starting and stopping the host +recorder raises it twice per recording, and between two points of a protocol or +a frequency ladder neither of those is true — so the run's own boundary was +treated as an idle-time reset and wiped the survey's pilot windows, background +floor and response curve mid-run. The question is now `automation_active()`: +the same set `request_stop` winds down. diff --git a/docs/features/README.md b/docs/features/README.md index 7e97865..f740496 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -9,7 +9,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC from measured `V_null`/`V_peak` endpoints, with target-specific headroom, Bessel-normalized cycle mean `ū`, and an explicit separation from physical flux `I_k`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. Each point is a 20 ms measurement after a 0.1 s settle, and every verdict on the sweep — lobe resolved, cell drifting — is made against the fit's own residual rather than against zero (ADR 019). Applying the fit now actually reaches the panel: the measurement lives on the live worker while the settings snapshot is collected from the UI mirror, so the applied lobe used to be overwritten within one frame (ADR 026). - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. -- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). +- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). Every leased run heartbeats its modulation and photodiode leases against the deadline the owner actually granted, so a recording longer than the owner's TTL cap no longer loses the drive — and with it the phase-0 trigger and the photodiode's optical summary — in the middle of a point (ADR 029). - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 38e8800..007ac0a 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -28,7 +28,10 @@ [ADR 027](../adr/027-stage-a-a1-declarative-protocols.md) (surveys are run from a file, and `I_k` becomes a sweepable axis), [ADR 028](../adr/028-stage-a-sensor-readout-travels-with-the-measurement.md) - (the sensor readout travels with the measurement, column-wise) + (the sensor readout travels with the measurement, column-wise), + [ADR 029](../adr/029-stage-a-leases-are-renewed-against-the-granted-deadline.md) + (a leased run heartbeats against the deadline the owner granted, so a point + longer than the owner's TTL cap no longer loses the drive mid-recording) - **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) - **Second workflow:** [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — hold one *measured* depth `a₀` across the frequency sweep @@ -308,7 +311,7 @@ drive the operator already armed (frequency, normalized cycle mean `ū`, and calibration stay untouched). The modulation owner accepts this command only with an applied measured calibration and `OPTICAL_LOG_SINE`; manual, constant, DAC-sine, square, and optical-linear modes are rejected. It renews the lease per -point, waits for a fresh, marker-bounded photodiode `a` from a confirmed `I_tot` +point *and* on a heartbeat between points, waits for a fresh, marker-bounded photodiode `a` from a confirmed `I_tot` anchor to settle, hands the point to the normal recording coordinator, and releases the lease at the end or on abort. Sweep points require `min a > 0` — record `a≈0` with the background button instead. Sidecars @@ -319,6 +322,19 @@ last sweep amplitude until the operator's own `depth a` setting is re-applied event-count point re-applies its locked depth under the lease instead of trusting the drive to still be where a previous action left it (ADR 013). +**Leases are kept alive against the deadline the owner granted, not the one A1 +asked for** (ADR 029). Both owners cap the TTL they hand out — a client that +dies must not hold the laser — so the whole-run TTL a sweep, a ladder or a +protocol asks for is *not* what it gets. A1 reads the real +`expires_at_unix_ms` off the owner's own snapshot and renews on a heartbeat once +less than 20 s of the granted window is left. Without it, any point longer than +the cap outlived its lease mid-recording and the owner did what an expired lease +must do — `STOP`, output off — which then read as three separate faults at once: +`the modulation owner requires an active automation lease`, `cannot write a +quantitative A1 sidecar without a fresh photodiode optical summary`, and a +`Camera: … no trigger signal` line that looked exactly like an unplugged +`EXT_TRIGGER` cable but was the drive being off. + **Naming.** Files share an `_[_role]` stem under an `/` subfolder (`_pilot` / `_background` tag the reference runs, `_ec_fHz` an event-count point): diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 829abcf..402fb63 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -56,7 +56,7 @@ use augur_plugin_api::{ use serde::Serialize; use serde_json::{json, Value}; use stage_a_plugin_contract::{ - ClientId, ConnectionStateV1, LeaseId, ModulationCommandV1, ModulationRequestV1, + ClientId, ConnectionStateV1, LeaseId, LeaseSnapshotV1, ModulationCommandV1, ModulationRequestV1, ModulationStateV1, OpticalTargetV1, PdqReceiptV1, PdqStartSpecV1, PhotodiodeCommandV1, PhotodiodeOpticalSummaryV1, PhotodiodeRequestV1, PhotodiodeResponseV1, PhotodiodeSummaryV1, RequestId, RunId, SemanticRevision, WaveformV1, CTX_STAGE_A_MODULATION_STATE_V1, @@ -149,6 +149,28 @@ const FREQ_CONFIRM_BASE_MS: u64 = 20_000; /// of the old and the new drive. const FREQ_CONFIRM_CYCLES: f64 = 4.0; +/// Renew a held lease once less than this much of the owner's *granted* window +/// is left. +/// +/// Both owners cap the TTL they hand out — a client that dies must not hold the +/// drive indefinitely, so the cap is a dead-man switch and is right. What that +/// means here is that the whole-run TTL a leased run asks for is emphatically +/// not what it gets: ask for forty minutes, be granted a minute. Renewing once +/// per point was therefore only ever correct for points shorter than the cap. +/// A longer one (the shipped example protocol has a 40 s row, and every row +/// also pays the start/stop handshake) ran past the granted deadline mid +/// recording, and the owner did what an expired lease must do — STOP, output +/// off. The run then lost the drive, the phase-0 trigger and the photodiode's +/// optical summary at once, and reported three unrelated-looking failures. +/// +/// So the run renews against the deadline the owner actually advertises, not +/// against the one it asked for. +const LEASE_RENEW_MARGIN_MS: u64 = 20_000; +/// Shortest gap between two heartbeat renewals of the same lease. The owner's +/// snapshot lags a renewal by a tick or two, so without this the margin test +/// re-fires on every control tick until the new deadline comes back. +const LEASE_RENEW_MIN_INTERVAL_MS: u64 = 2_000; + /// Absolute/relative tolerance for "the measured `a` reached the sweep target". fn sweep_tolerance(target_a: f64) -> f64 { (target_a * 0.10).max(0.05) @@ -975,6 +997,11 @@ pub struct StageAA1Plugin { /// Output folder the lock table was last read for, so it is re-read only /// when the experiment folder changes. loaded_locks_folder: Option, + // -- lease heartbeats (see LEASE_RENEW_MARGIN_MS) -- + /// When the modulation lease was last renewed by the heartbeat. + mod_renewed_ms: u64, + /// When the photodiode lease was last renewed by the heartbeat. + pd_renewed_ms: u64, // -- momentary-button press forwarding (see PressLatch) -- press_start: PressLatch, press_pilot: PressLatch, @@ -1055,6 +1082,8 @@ impl Default for StageAA1Plugin { protocol: None, a0_locks: Vec::new(), loaded_locks_folder: None, + mod_renewed_ms: 0, + pd_renewed_ms: 0, press_start: PressLatch::default(), press_pilot: PressLatch::default(), press_background: PressLatch::default(), @@ -1221,6 +1250,104 @@ impl StageAA1Plugin { self.protocol_pending = false; } + /// Whether anything the Record section started is still in flight. + /// + /// The same set [`Self::request_stop`] winds down. A discontinuity that + /// arrives between two points of a run is still *inside* that run, so it + /// must not be treated as an idle-time reset. + fn automation_active(&self) -> bool { + self.recording.is_active() + || self.sweep.is_some() + || self.a0_lock.is_some() + || self.freq_sweep.is_some() + || self.protocol.is_some() + } + + /// The modulation lease this run holds, and how much longer it still needs + /// it. Outermost runner first — a nested run inherits the enclosing lease + /// id, so the outermost one names the lease and owns the remaining time. + /// + /// `None` until the owner has granted it: renewing a lease that does not + /// exist yet is rejected, and the acquire is already in flight. + fn held_modulation_lease(&self) -> Option<(LeaseId, u64)> { + if let Some(run) = self.protocol.as_ref().filter(|run| run.lease_granted) { + return Some(( + run.lease_id.clone(), + Self::protocol_lease_ttl_ms(&run.plan, run.index), + )); + } + if let Some(sweep) = self.freq_sweep.as_ref().filter(|s| s.lease_granted) { + let remaining = sweep.points.len().saturating_sub(sweep.index); + return Some(( + sweep.lease_id.clone(), + self.freq_sweep_lease_ttl_ms(remaining, sweep.mode), + )); + } + if let Some(lock) = self.a0_lock.as_ref().filter(|l| l.lease_granted) { + return Some((lock.lease_id.clone(), self.a0_lock_lease_ttl_ms())); + } + if let Some(sweep) = self.sweep.as_ref().filter(|s| s.lease_granted) { + let remaining = sweep.total().saturating_sub(sweep.index); + return Some((sweep.lease_id.clone(), self.sweep_lease_ttl_ms(remaining))); + } + None + } + + /// Whether `lease` is the one A1 is holding right now, per the owner's own + /// snapshot. Renewing on our own bookkeeping alone would keep re-asking + /// after the owner had already dropped it. + fn owner_holds(lease: Option<&LeaseSnapshotV1>, held: &LeaseId) -> Option { + let lease = lease?; + (&lease.lease_id == held && lease.holder.as_str() == A1_PLUGIN_ID) + .then_some(lease.expires_at_unix_ms) + } + + /// Keep both leases alive against the deadline each owner advertises. + /// + /// Runs on every control tick, ahead of the runners: the owners cap the TTL + /// they grant well below the length of a survey (see + /// [`LEASE_RENEW_MARGIN_MS`]), so a run that renewed only when it moved to + /// its next point lost the drive in the middle of any point longer than the + /// cap. + fn drive_lease_heartbeat(&mut self, context: &mut impl RecordingControl) { + let now_ms = now_unix_ms(); + let due = |last_ms: u64, expires_at: u64| { + now_ms.saturating_sub(last_ms) >= LEASE_RENEW_MIN_INTERVAL_MS + && expires_at.saturating_sub(now_ms) <= LEASE_RENEW_MARGIN_MS + }; + + if let Some((lease_id, ttl_ms)) = self.held_modulation_lease() { + let expires_at = Self::owner_holds( + self.modulation.as_ref().and_then(|s| s.lease.as_ref()), + &lease_id, + ); + if expires_at.is_some_and(|at| due(self.mod_renewed_ms, at)) { + let request = + self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + context.request_service(&request); + self.mod_renewed_ms = now_ms; + } + } + + // The photodiode lease covers one recording, and A1 asks for its + // duration plus slack — which the owner caps just as hard, so any + // recording longer than the cap used to be finalized underneath itself + // as `LeaseExpired` and left no optical summary for the sidecar. + if self.recording.is_active() && self.recording.lease_granted { + let held = self.recording.lease_id.clone(); + let expires_at = Self::owner_holds( + self.photodiode.as_ref().and_then(|s| s.lease.as_ref()), + &held, + ); + if expires_at.is_some_and(|at| due(self.pd_renewed_ms, at)) { + let ttl_ms = self.photodiode_lease_ttl_ms(); + let request = self.photodiode_request(PhotodiodeCommandV1::RenewLease { ttl_ms }); + context.request_service(&request); + self.pd_renewed_ms = now_ms; + } + } + } + /// Sets the concise operator-facing recording result. fn note(&mut self, message: impl Into) { self.message = message.into(); @@ -2211,12 +2338,18 @@ impl StageAA1Plugin { )); } - fn acquire_photodiode(&mut self, context: &mut impl RecordingControl) { - let ttl_ms = self - .recording + /// How much longer the photodiode is needed: the recording's own length + /// plus the start/stop handshake. The owner caps what it grants, so the + /// heartbeat re-asks — see [`LEASE_RENEW_MARGIN_MS`]. + fn photodiode_lease_ttl_ms(&self) -> u64 { + self.recording .duration_s .saturating_mul(1_000) - .saturating_add(60_000); + .saturating_add(60_000) + } + + fn acquire_photodiode(&mut self, context: &mut impl RecordingControl) { + let ttl_ms = self.photodiode_lease_ttl_ms(); let request = self.photodiode_request(PhotodiodeCommandV1::AcquireLease { ttl_ms }); self.recording.lease_req = request.request_id; context.request_service(&request); @@ -4241,12 +4374,14 @@ impl StageAA1Plugin { // ---- declarative protocol runs ------------------------------------- - /// Lease TTL covering the whole protocol, plus a minute of slack. + /// How much longer the whole protocol still needs the drive, plus a minute + /// of slack. /// - /// One lease for the whole file, like the frequency ladder: a TTL that - /// expired between points would hand the drive back to the operator's - /// armed settings mid-survey, and the remaining points would record - /// against them without saying so. + /// One lease for the whole file, like the frequency ladder: handing the + /// drive back to the operator's armed settings mid-survey would let the + /// remaining points record against them without saying so. This is what + /// A1 *asks* for, not what it gets — the owner caps the TTL it grants, and + /// [`Self::drive_lease_heartbeat`] is what actually keeps the lease alive. fn protocol_lease_ttl_ms(plan: &protocol::Protocol, from: usize) -> u64 { let remaining: f64 = plan.points[from.min(plan.points.len())..] .iter() @@ -5887,7 +6022,14 @@ impl Plugin for StageAA1Plugin { // must not wipe the row's pilot windows, background floor, or // the response points collected across a sweep. The event fold // still resets: that timeline really did restart. - if self.recording.is_active() || self.sweep.is_some() { + // + // Every runner counts, not just a recording in flight: a + // protocol or ladder spends the gap between two points + // retargeting the drive, and the stop boundary of the point + // just finished lands squarely in it. Asking only about the + // recording wiped the survey's own pilot windows and + // background floor between every pair of points. + if self.automation_active() { self.camera_events.clear(); self.event_scratch.clear(); self.camera_markers_us.clear(); @@ -6029,6 +6171,9 @@ impl Plugin for StageAA1Plugin { self.scan_measurement_folder(); self.load_a0_locks(); } + // Before the runners: a lease that lapses is not the runners' problem + // to notice, and the owner safe-offs the drive the moment it does. + self.drive_lease_heartbeat(context); // Outermost first: the protocol and the frequency sweep each start the // stage below them, and each of those starts its own next stage, so one // tick carries a hand-off all the way down. They are mutually exclusive @@ -7313,6 +7458,7 @@ mod tests { } // Same order as `process_control`: outermost supervisor first, so one // tick can carry a hand-off from the ladder down into a recording. + plugin.drive_lease_heartbeat(sink); plugin.drive_protocol(sink); plugin.drive_freq_sweep(sink); plugin.drive_a0_lock(sink); @@ -10109,6 +10255,152 @@ depth_a = 99.0 let _ = std::fs::remove_dir_all(&folder); } + /// The owners cap the lease TTL they grant far below the length of a + /// survey, so a run that renewed only when it stepped to its next point + /// lost the drive in the middle of any point longer than that cap — the + /// owner STOPs and switches the output off, which took the phase-0 trigger + /// and the photodiode's optical summary with it. + #[test] + fn a_long_point_renews_the_modulation_lease_before_the_owner_drops_it() { + let folder = temp_folder("protocol-lease-heartbeat"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = sink.services[0].request_id; + let lease_id = plugin + .protocol + .as_ref() + .expect("the protocol is running") + .lease_id + .clone(); + sink.services.clear(); + control_tick( + &mut plugin, + inbox_with(vec![accepted(lease_req)]), + &mut sink, + ); + + // The owner granted far less than the whole-survey TTL that was asked + // for, and the point is still running. + let now_ms = now_unix_ms(); + if let Some(state) = plugin.modulation.as_mut() { + state.lease = Some(LeaseSnapshotV1 { + lease_id: lease_id.clone(), + holder: ClientId::new(A1_PLUGIN_ID), + expires_at_unix_ms: now_ms + LEASE_RENEW_MARGIN_MS / 2, + run_id: None, + }); + } + sink.services.clear(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + let renewed = sink + .services + .iter() + .filter_map(modulation_command) + .any(|command| matches!(command, ModulationCommandV1::RenewLease { .. })); + assert!( + renewed, + "the lease was left to expire underneath the point: {:?}", + sink.services + ); + + let _ = std::fs::remove_dir_all(&folder); + } + + /// A lease with plenty of time left must not be renewed on every tick: the + /// control plane runs at 20 Hz and each renewal is a device round trip. + #[test] + fn a_lease_with_time_left_is_not_renewed_every_tick() { + let folder = temp_folder("protocol-lease-quiet"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = sink.services[0].request_id; + let lease_id = plugin + .protocol + .as_ref() + .expect("the protocol is running") + .lease_id + .clone(); + control_tick( + &mut plugin, + inbox_with(vec![accepted(lease_req)]), + &mut sink, + ); + if let Some(state) = plugin.modulation.as_mut() { + state.lease = Some(LeaseSnapshotV1 { + lease_id, + holder: ClientId::new(A1_PLUGIN_ID), + expires_at_unix_ms: now_unix_ms() + LEASE_RENEW_MARGIN_MS * 4, + run_id: None, + }); + } + + sink.services.clear(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert!( + !sink + .services + .iter() + .filter_map(modulation_command) + .any(|command| matches!(command, ModulationCommandV1::RenewLease { .. })), + "a lease that is nowhere near expiry was renewed anyway: {:?}", + sink.services + ); + + let _ = std::fs::remove_dir_all(&folder); + } + + /// Starting and stopping the host recorder is reported as `SourceChanged`, + /// twice per recording. Between two points a protocol is not "recording", + /// so asking only about the recording treated its own self-inflicted + /// boundary as an idle-time reset and wiped the survey's pilot windows, + /// background floor and response curve mid-run. + #[test] + fn a_source_change_between_two_protocol_points_keeps_the_survey_state() { + let folder = temp_folder("protocol-discontinuity"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = sink.services[0].request_id; + control_tick( + &mut plugin, + inbox_with(vec![accepted(lease_req)]), + &mut sink, + ); + plugin.background_floor = Some((0.25, 0.25)); + plugin.pilot_windows = Some(( + PhaseWindow { + start: 0.1, + end: 0.2, + }, + PhaseWindow { + start: 0.6, + end: 0.7, + }, + )); + assert!(!plugin.recording.is_active(), "the point is between stages"); + + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + + assert_eq!( + plugin.background_floor, + Some((0.25, 0.25)), + "the survey's background reference was wiped between two points" + ); + assert!( + plugin.pilot_windows.is_some(), + "the survey's pilot windows were wiped between two points" + ); + + let _ = std::fs::remove_dir_all(&folder); + } + /// Each point's own duration governs the recording, not the panel's — a /// survey whose lengths silently came from the UI would not be /// reproducible from the protocol alone. From 30e677c5dff6912ec8af6cc7504e1981f7a5e2d3 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 3 Aug 2026 20:21:18 +0200 Subject: [PATCH 37/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20say=20when?= =?UTF-8?q?=20a=20run=20wrote=20no=20sensor=20readout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compacted `.sensor.json` depends on a companion CSV the host writes only when its own "Record sensor monitoring" switch is on. A1 cannot set that switch and cannot query it, so an absent readout was indistinguishable from a camera with no monitoring block — and the sidecar field's own doc comment said exactly that, which is what made the absence so hard to trace. A finished run that produced no readout now says so in the panel and names the switch, instead of leaving a survey to discover months later that it kept none of its bench conditions. The single-point readings in `[sensor]` ride the context bus and are unaffected either way. Co-Authored-By: Claude Opus 5 --- docs/features/stage-a-a1.md | 14 +++++- plugins/stage-a-a1/src/runtime.rs | 84 ++++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 007ac0a..8cf4b67 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -366,8 +366,18 @@ the photodiode Data directory no longer have to be kept aligned by hand: schedules, so a row-per-poll table is padding by construction. Bias codes are dropped: the camera's own bias sidecar already carries them. Nothing is resampled or aligned, failed polls are kept as `faults`, and the whole path is - best-effort — a source with no monitoring block simply produces no file - (ADR 028). + best-effort (ADR 028). + + **The companion CSV only exists if the host is asked for it.** It is governed + by the host's own **Record sensor monitoring** checkbox in the recording + panel, which A1 cannot set and cannot query — so no telemetry file means no + `.sensor.json`, whatever the camera supports. That switch used to reset to off + on every app start, which is how a survey could record forty runs and keep the + bench conditions of none of them; it is now persisted across restarts + (augur-rs). A1 reports it either way: when a finished run wrote no readout, + the panel names the switch rather than leaving the absence silent. The + single-point die temperature / dead time / illumination in `[sensor]` come + from the context bus and are recorded with every run regardless (ADR 022). **A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 402fb63..18e1ead 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -997,6 +997,10 @@ pub struct StageAA1Plugin { /// Output folder the lock table was last read for, so it is re-read only /// when the experiment folder changes. loaded_locks_folder: Option, + /// Whether the last finished recording produced no sensor readout, so the + /// panel can name the host switch that governs it. Observed rather than + /// asked: the host does not publish whether it is recording telemetry. + last_run_had_no_readout: bool, // -- lease heartbeats (see LEASE_RENEW_MARGIN_MS) -- /// When the modulation lease was last renewed by the heartbeat. mod_renewed_ms: u64, @@ -1082,6 +1086,7 @@ impl Default for StageAA1Plugin { protocol: None, a0_locks: Vec::new(), loaded_locks_folder: None, + last_run_had_no_readout: false, mod_renewed_ms: 0, pd_renewed_ms: 0, press_start: PressLatch::default(), @@ -2522,6 +2527,7 @@ impl StageAA1Plugin { // from the bench conditions it was taken under at the first move. // It is rewritten column-wise on the way in — see `sensor`. self.recording.sensor_readout_path = self.gather_sensor_readout(&dir, &raw); + self.last_run_had_no_readout = self.recording.sensor_readout_path.is_none(); } // PDQ receipts report the *label* A1 asked for, which is relative to the // photodiode's data directory — resolve it before touching the file, and @@ -5777,7 +5783,13 @@ struct FilesSidecar { photodiode_sidecar: Option, /// Compacted per-channel sensor readout for this run — the die /// temperature, pixel dead time and illumination the host polled while it - /// was recording. Absent when the source had no monitoring block. + /// was recording. + /// + /// Absent whenever the host wrote no telemetry companion. Usually that is + /// the host's own **Record sensor monitoring** switch being off, not a + /// camera without a monitoring block: the switch governs the whole file + /// and A1 cannot ask for it. The single-point readings in `[sensor]` come + /// from the context bus and are there either way. #[serde(skip_serializing_if = "Option::is_none")] sensor_readout: Option, } @@ -7216,6 +7228,21 @@ impl Plugin for StageAA1Plugin { sensor.age_s ))); } + // The point values above ride the context bus and are always + // there. The per-run *time series* is a separate host feature the + // operator switches on, and it is off by default — so a survey + // could record forty runs, keep the bench conditions of none of + // them, and say nothing until the analysis. A1 cannot ask the host + // whether it is on, but it can report that the last run produced + // no readout, which is the same fact one recording later. + if self.last_run_had_no_readout { + entries.push(StatusEntry::Text( + "Sensor readout: the last run wrote none — tick \"Record sensor monitoring\" \ + in the recording panel, or runs keep only the single reading above and not \ + the series" + .into(), + )); + } } if let Some((on, off)) = self.latest_rolling() { entries.push(StatusEntry::Text(format!( @@ -10502,6 +10529,61 @@ bias_refr_code,status,error\n\ assert_eq!(parsed["channels"]["temperature_c"]["value"][0], 41.5); // The wide original does not stay behind in the capture folder. assert!(!capture.join("host-capture.sensor-monitoring.csv").exists()); + assert!( + !plugin.last_run_had_no_readout, + "a run that did write a readout must not warn about one" + ); + + let _ = std::fs::remove_dir_all(&capture); + let _ = std::fs::remove_dir_all(&output); + } + + /// The host's telemetry companion is governed by its own **Record sensor + /// monitoring** switch, which is off by default and which A1 cannot ask + /// about. A survey that recorded forty runs and kept the bench conditions + /// of none of them used to say nothing at all — the absence surfaced in + /// the analysis, months later. + #[test] + fn a_run_with_no_host_telemetry_says_so_in_the_panel() { + let capture = temp_folder("sensor-off-capture"); + std::fs::create_dir_all(&capture).expect("capture dir"); + let raw = capture.join("host-capture.raw"); + std::fs::write(&raw, b"raw").expect("raw"); + // No `.sensor-monitoring.csv` beside it: the host switch was off. + + let output = temp_folder("sensor-off-output"); + let mut plugin = plugin_with_markers(); + plugin.sensor = Some(SensorMonitoringV1 { + temperature_c: Some(21.5), + pixel_dead_time_us: Some(18.1), + illumination_lux: Some(0.07), + ..SensorMonitoringV1::default() + }); + plugin.output_folder = output.display().to_string(); + plugin.recording.folder = output.display().to_string(); + plugin.recording.id = "A1-sensor-off".into(); + plugin.recording.stem = "A1-sensor-off_20260803-120000".into(); + plugin.recording.cam_finalized_path = Some(raw.display().to_string()); + + plugin.gather_into_measurement_folder(); + + assert!( + plugin.recording.sensor_readout_path.is_none(), + "there was no telemetry to compact" + ); + let panel = plugin + .status_entries() + .into_iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) => Some(text), + _ => None, + }) + .collect::>() + .join("\n"); + assert!( + panel.contains("Record sensor monitoring"), + "the panel does not name the switch that governs the readout:\n{panel}" + ); let _ = std::fs::remove_dir_all(&capture); let _ = std::fs::remove_dir_all(&output); From 5f118e57261922afc5ed8b0b68f1e6e3e34f9fb4 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 21:41:49 +0200 Subject: [PATCH 38/46] =?UTF-8?q?ci:=20=F0=9F=91=B7=20build=20installable?= =?UTF-8?q?=20plugin=20bundles=20for=20four=20platforms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installing a plugin required a Rust toolchain, a sibling augur-rs checkout and a working cargo, which made every measurement PC a development machine. Build all runtime plugins on each pull request and each push to main for macOS arm64/x86_64, Linux x86_64 and Windows x86_64, staged in the exact layout ~/.augur/plugins/ expects, so installing is a copy. main also publishes a rolling plugins-latest release, because workflow artifacts need a login and expire after 90 days while a bench should be able to curl a URL. The workspace depends on the host by path, so the job lays out two sibling checkouts. build-runtime-plugins.sh patches a git source whenever it finds a sibling augur-rs checkout; with path deps that patch matches nothing but still costs a fetch, so the checkout's .git is dropped right after cloning. CI calls the repository's own build and install scripts instead of restating the install layout in YAML — those scripts already own plugin discovery, library naming, A1's protocols folder and the macOS install-name rewrite. Plugins are dlopened into the host process, so pin rust-toolchain.toml to the same 1.95.0 augur-rs pins and read the channel out of that file rather than naming a version in the workflow. Every bundle carries a BUILD-INFO.txt with the augur-rs revision and rustc version behind it, which is what makes an ABI mismatch reported from the bench answerable. --- .github/workflows/build-plugins.yml | 184 ++++++++++++++++++ .gitignore | 1 + README.md | 26 +++ .../030-prebuilt-plugin-bundles-from-ci.md | 87 +++++++++ docs/features/README.md | 1 + docs/features/ci-prebuilt-plugin-bundles.md | 135 +++++++++++++ docs/installing-plugins.md | 35 ++++ rust-toolchain.toml | 8 + 8 files changed, 477 insertions(+) create mode 100644 .github/workflows/build-plugins.yml create mode 100644 docs/adr/030-prebuilt-plugin-bundles-from-ci.md create mode 100644 docs/features/ci-prebuilt-plugin-bundles.md create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/build-plugins.yml b/.github/workflows/build-plugins.yml new file mode 100644 index 0000000..303e653 --- /dev/null +++ b/.github/workflows/build-plugins.yml @@ -0,0 +1,184 @@ +name: Build Plugins + +# Produces drop-in plugin folders for a machine that has no Rust toolchain: the +# bench downloads a bundle, copies its folders into ~/.augur/plugins/, and hits +# "Scan for New Plugins". Pull requests get workflow artifacts; main also +# publishes a rolling release so the download needs no GitHub login. + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + inputs: + augur_rs_ref: + description: "augur-rs ref to build against (branch, tag or SHA)" + required: false + default: main + +concurrency: + group: build-plugins-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + AUGUR_RS_REF: ${{ inputs.augur_rs_ref || 'main' }} + +jobs: + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: macos-latest + name: macOS (arm64) + bundle: macos-arm64 + - os: macos-13 + name: macOS (x86_64) + bundle: macos-x86_64 + - os: ubuntu-latest + name: Linux (x86_64) + bundle: linux-x86_64 + - os: windows-latest + name: Windows (x86_64) + bundle: windows-x86_64 + + defaults: + run: + # The repo drives its builds through two bash scripts; use the same + # shell on Windows so there is exactly one code path to reason about. + shell: bash + + steps: + # This workspace depends on the host by path (../augur-rs/augur-core), so + # CI has to reproduce the two-sibling-checkout layout, not clone one repo. + - name: Check out augur-plugins + uses: actions/checkout@v5 + with: + path: augur-plugins + + - name: Check out augur-rs + uses: actions/checkout@v5 + with: + repository: muthmann/augur-rs + ref: ${{ env.AUGUR_RS_REF }} + path: augur-rs + + - name: Pin the host revision and disarm the source patch + run: | + echo "AUGUR_RS_SHA=$(git -C augur-rs rev-parse HEAD)" >> "$GITHUB_ENV" + # build-runtime-plugins.sh patches [patch."…/augur-rs.git"] whenever a + # sibling augur-rs *git checkout* exists. This workspace already + # depends on it by path, so that patch matches nothing in the crate + # graph — it only costs cargo a fetch of the checkout. Removing .git + # makes the script's detection fail and the path deps win outright. + rm -rf augur-rs/.git + + - name: Resolve the pinned Rust toolchain + run: | + channel="$(sed -n 's/^channel *= *"\(.*\)"$/\1/p' augur-plugins/rust-toolchain.toml | head -n 1)" + if [[ -z "${channel}" ]]; then + echo "No channel found in augur-plugins/rust-toolchain.toml" >&2 + exit 1 + fi + echo "RUST_CHANNEL=${channel}" >> "$GITHUB_ENV" + + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ env.RUST_CHANNEL }} + cache-workspaces: augur-plugins + + - name: Install Linux system dependencies + if: runner.os == 'Linux' + # Reuse the host's own dependency list rather than a second copy that can + # drift; it is a superset of what the plugins need (serialport/libudev). + run: bash augur-rs/.github/scripts/install-linux-deps.sh + + - name: Build runtime plugins + working-directory: augur-plugins + run: bash scripts/build-runtime-plugins.sh --profile release + + - name: Stage installable plugin folders + working-directory: augur-plugins + run: bash scripts/install-built-plugins.sh --profile release --dest "dist/${{ matrix.bundle }}" + + - name: Write build provenance + working-directory: augur-plugins + run: | + { + echo "bundle: ${{ matrix.bundle }}" + echo "built_at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "augur_plugins: $(git rev-parse HEAD)" + echo "augur_rs_ref: ${AUGUR_RS_REF}" + echo "augur_rs_sha: ${AUGUR_RS_SHA}" + echo "rustc: $(rustc --version)" + echo + echo "Copy the plugin folders next to this file into ~/.augur/plugins/," + echo "then use Plugins -> Scan for New Plugins in augur-gui." + } > "dist/${{ matrix.bundle }}/BUILD-INFO.txt" + + - name: Upload plugin bundle + uses: actions/upload-artifact@v4 + with: + name: augur-plugins-${{ matrix.bundle }} + path: augur-plugins/dist/${{ matrix.bundle }} + if-no-files-found: error + + release: + name: Publish rolling release + needs: build + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Download every plugin bundle + uses: actions/download-artifact@v4 + with: + path: bundles + pattern: augur-plugins-* + + - name: Package one archive per platform + run: | + set -euo pipefail + mkdir -p dist + for bundle_dir in bundles/augur-plugins-*/; do + bundle="$(basename "${bundle_dir%/}")" + (cd "${bundle_dir}" && zip -qr "${GITHUB_WORKSPACE}/dist/${bundle}.zip" .) + echo "Packaged ${bundle}.zip" + done + (cd dist && sha256sum ./*.zip > SHA256SUMS.txt) + ls -l dist + + - name: Publish rolling release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + tag="plugins-latest" + # Delete and recreate rather than upload --clobber: it retags at the + # new commit and guarantees no asset from an older build survives. + gh release delete "${tag}" --yes --cleanup-tag || true + gh release create "${tag}" dist/* \ + --title "Prebuilt plugins (latest main)" \ + --notes "$(printf '%s\n' \ + "Prebuilt AugurRS plugins, rebuilt on every push to \`main\`." \ + "" \ + "- augur-plugins: \`${GITHUB_SHA}\`" \ + "- built against augur-rs \`${AUGUR_RS_REF}\`" \ + "" \ + "Download the archive for your platform, unpack it, and copy the" \ + "plugin folders inside into \`~/.augur/plugins/\`. Then open augur-gui," \ + "go to **Plugins**, and click **Scan for New Plugins**." \ + "" \ + "\`BUILD-INFO.txt\` in each archive records the exact revisions and" \ + "compiler the libraries were built with. Verify downloads against" \ + "\`SHA256SUMS.txt\`.")" diff --git a/.gitignore b/.gitignore index 39875ee..f240a43 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +/dist Cargo.lock *.swp *.swo diff --git a/README.md b/README.md index 346be75..da10175 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,31 @@ The plugin crates under `plugins/` are under active development and not yet read ## Quick Start +### Download Prebuilt Plugins (no toolchain needed) + +Every push to `main` publishes freshly built plugins for macOS (arm64 and x86_64), +Linux and Windows to the rolling +[`plugins-latest`](https://github.com/muthmann/augur-plugins/releases/tag/plugins-latest) +release. This is the recommended route for a measurement machine. + +```bash +curl -LO https://github.com/muthmann/augur-plugins/releases/download/plugins-latest/augur-plugins-macos-arm64.zip +unzip augur-plugins-macos-arm64.zip -d augur-plugins-bundle +mkdir -p ~/.augur/plugins +cp -R augur-plugins-bundle/*/ ~/.augur/plugins/ +``` + +Pick the archive matching the machine: `macos-arm64`, `macos-x86_64`, +`linux-x86_64`, or `windows-x86_64`. Then open `augur-gui`, go to **Plugins**, and +click **Scan for New Plugins**. + +Each archive contains a `BUILD-INFO.txt` recording the `augur-rs` revision and the +`rustc` version the libraries were built against — quote it in any ABI-mismatch +report. Verify downloads against `SHA256SUMS.txt` from the same release. + +Pull requests build the same bundles as workflow artifacts. See +[CI Prebuilt Plugin Bundles](./docs/features/ci-prebuilt-plugin-bundles.md). + ### Build One Plugin ```bash @@ -145,6 +170,7 @@ augur-plugins/ - [Plugin API Notes](./docs/plugin-api.md) — repo-local summary of the current runtime contract - [Installing Plugins](./docs/installing-plugins.md) — build, copy, reload, and troubleshoot installed plugins +- [CI Prebuilt Plugin Bundles](./docs/features/ci-prebuilt-plugin-bundles.md) — how the downloadable per-platform bundles are built and published - [Architecture Notes](./docs/architecture.md) — repository role, execution model, host views, and shared settings - [augur-rs Plugin Authoring Guide](https://github.com/muthmann/augur-rs/blob/main/docs/features/plugin-authoring-guide.md) — canonical host/runtime authoring guide - [augur-rs Global Settings Guide](https://github.com/muthmann/augur-rs/blob/main/docs/features/global-settings-menu.md) — host-owned settings published to plugins diff --git a/docs/adr/030-prebuilt-plugin-bundles-from-ci.md b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md new file mode 100644 index 0000000..797edf3 --- /dev/null +++ b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md @@ -0,0 +1,87 @@ +# ADR 030 — Prebuilt plugin bundles are produced by CI, not by the bench + +**Status:** accepted +**Date:** 2026-08-04 +**Feature brief:** [CI Prebuilt Plugin Bundles](../features/ci-prebuilt-plugin-bundles.md) + +## Context + +A runtime plugin is a `cdylib` plus a `plugin.toml`. Getting one onto a machine +required a Rust toolchain, a sibling `augur-rs` checkout, and `cargo`, because +this workspace depends on the host by path. That made the measurement PC a +development machine by necessity: every plugin fix had to be compiled where it +was used. + +Two properties of the plugin model make "just compile it there" worse than it +looks. Plugins are dlopened into the host process, so the compiler that builds a +plugin and the compiler that builds `augur-gui` have to agree — and this +repository pinned no toolchain at all while `augur-rs` pinned `1.95.0`. And the +installed folder is not just a library: A1 ships operator-facing `protocols/` +examples, and macOS copies need their dylib id rewritten or Plugin Manager +reloads resolve back into Cargo's build tree. + +## Decision + +CI builds the runtime plugins on every pull request and every push to `main`, for +macOS arm64, macOS x86_64, Linux x86_64 and Windows x86_64, and publishes the +result as a folder that is copied verbatim into `~/.augur/plugins/`. + +Three things follow from that, and they are the actual decision: + +1. **This repository pins the host's toolchain.** `rust-toolchain.toml` carries + the same `1.95.0` as `augur-rs`, and the workflow reads the channel out of + that file instead of naming a version in YAML. A bundle built by a different + compiler than the host is not a bundle, it is a load failure waiting to + happen, and the pin is the only thing that makes that guarantee checkable. + +2. **CI runs the repository's own build and install scripts.** It does not + reimplement plugin discovery, library naming, the `protocols/` copy or the + macOS install-name rewrite in YAML. The scripts are the single definition of + what an installed plugin is; CI is one more caller of them, with + `--dest dist/` instead of `~/.augur/plugins`. + +3. **`main` publishes a rolling release, not just artifacts.** Workflow artifacts + need a GitHub login and expire after 90 days. The bench is the consumer, and + it should be able to `curl` a URL. The tag `plugins-latest` is deleted and + recreated on every push to `main`, so its assets can never be a mixture of two + builds. + +Every bundle carries a `BUILD-INFO.txt` recording the `augur-plugins` commit, the +`augur-rs` ref and SHA, and the exact `rustc` version. + +## Consequences + +- The measurement PC needs no toolchain, no checkout, and no `cargo`. +- An ABI-mismatch report from the bench is now answerable: the provenance file + says which host revision and compiler the installed library came from. +- Local builds in this repository move from whatever `rustc` is on `PATH` to the + pinned `1.95.0` — but only for people whose `cargo` is the rustup shim. A + Homebrew `cargo` earlier on `PATH` ignores `rust-toolchain.toml` entirely and + will keep producing plugins for a compiler the host does not use. +- The macOS bundles are per-architecture while `augur-gui` ships universal, so + the download page has one more choice on it than the host's does. +- `main` gains a permanent release tag. The repository had no releases before, so + `releases/latest` now resolves to `plugins-latest`; a future versioned release + scheme would have to account for that. + +## Alternatives considered + +**Publish only workflow artifacts.** Simplest, and rejected: it puts a GitHub +login between the bench and a fix, and the artifact disappears after 90 days. + +**Build against the newest `augur-rs` release tag instead of `main`.** Matches a +bench running a released host, but lags every unpublished API change — and the +Stage-A work in this repository routinely depends on unreleased host changes. +`main` is the default; `workflow_dispatch` takes an `augur_rs_ref` for the cases +where a specific host revision is wanted. + +**Reimplement the install layout in the workflow.** Would have avoided calling +shell scripts from YAML, at the cost of a second, silently divergent definition +of what an installed plugin contains. The `protocols/` folder and the macOS +install-name rewrite were both added to the script after the fact; a YAML copy +would have missed both. + +**`lipo` the two macOS builds into universal libraries.** Attractive, since the +host is universal, but `install-built-plugins.sh` reads `target/` only +and a cross-build lands in `target//`. Deferred rather than +special-cased in CI, since it belongs in the script if it is worth doing. diff --git a/docs/features/README.md b/docs/features/README.md index f740496..fb930c9 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -13,6 +13,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. +- [CI Prebuilt Plugin Bundles](./ci-prebuilt-plugin-bundles.md) — every pull request and every push to `main` builds all runtime plugins for macOS (arm64/x86_64), Linux and Windows, staged in the exact `~/.augur/plugins/` layout so a bench machine installs by copying instead of compiling. `main` publishes them as a rolling `plugins-latest` release that needs no GitHub login, each bundle carrying a `BUILD-INFO.txt` with the `augur-rs` revision and `rustc` version it was built against. CI calls the repo's own build/install scripts rather than restating the install layout in YAML, and `rust-toolchain.toml` now pins the host's `1.95.0` because plugins are dlopened into the host process (ADR 030). - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. - [Investigation Workspace Alignment](./investigation-workspace-alignment.md) — in-tree plugins updated for stable ids, linked 2D/3D/table datasets, and candidate-stage accepted/rejected event inspection. diff --git a/docs/features/ci-prebuilt-plugin-bundles.md b/docs/features/ci-prebuilt-plugin-bundles.md new file mode 100644 index 0000000..c597487 --- /dev/null +++ b/docs/features/ci-prebuilt-plugin-bundles.md @@ -0,0 +1,135 @@ +# CI Prebuilt Plugin Bundles + +**Status:** built +**Workflow:** [`.github/workflows/build-plugins.yml`](../../.github/workflows/build-plugins.yml) +**ADR:** [030 — Prebuilt plugin bundles are produced by CI](../adr/030-prebuilt-plugin-bundles-from-ci.md) + +## Problem + +Installing a plugin used to require a Rust toolchain, a sibling `augur-rs` +checkout, and a working `cargo`. That is a reasonable ask of a contributor and an +unreasonable ask of the bench machine that actually runs the experiment. A +measurement PC should not need a development environment just to pick up a fixed +plugin. + +## What it does + +Every pull request and every push to `main` builds all runtime plugins on four +platforms and stages them in the exact layout `~/.augur/plugins/` expects: + +```text +augur-plugins-macos-arm64/ + BUILD-INFO.txt + stage-a-a1/ + plugin.toml + libaugur_plugin_stage_a_a1.dylib + protocols/ + example.csv + example.toml + stage-a-modulation/ + stage-a-photodiode/ + localization/ + … +``` + +Installing is then a copy — no build step, no toolchain. + +| Bundle | Runner | Library | +|---|---|---| +| `macos-arm64` | `macos-latest` | `.dylib` | +| `macos-x86_64` | `macos-13` | `.dylib` | +| `linux-x86_64` | `ubuntu-latest` | `.so` | +| `windows-x86_64` | `windows-latest` | `.dll` | + +Pull requests publish the bundles as workflow artifacts. Pushes to `main` +additionally publish a rolling GitHub Release tagged `plugins-latest`, one zip per +platform plus `SHA256SUMS.txt`. The release exists because artifacts require a +GitHub login and expire; a release asset can be fetched from the bench with +`curl` and no account. + +`workflow_dispatch` takes an `augur_rs_ref` input for building a bundle against a +host branch or tag other than `main`. + +## Why it is shaped this way + +**Two sibling checkouts, not one.** The workspace depends on the host by path +(`augur-core = { path = "../augur-rs/augur-core" }`), so the job checks +`augur-plugins` and `augur-rs` out next to each other under the workspace root +and builds from the former. A single-repo checkout cannot resolve the dependency +at all. + +**`augur-rs/.git` is deleted right after checkout.** `build-runtime-plugins.sh` +adds `--config patch."…augur-rs.git"…` flags whenever it finds a sibling +`augur-rs` *git checkout*. With path dependencies that patch matches nothing — +cargo reports `Patch … was not used in the crate graph` and exits 0 — but it +still costs a git fetch of the checkout. Removing `.git` makes the script's +detection fail, and the path dependencies are used directly. + +**The toolchain is pinned and read from the file.** Plugins are `cdylib`s the +host `dlopen`s into its own process, so they must be built by the same compiler +as `augur-gui`. [`rust-toolchain.toml`](../../rust-toolchain.toml) pins the same +`1.95.0` as `augur-rs`, and the workflow parses the channel out of that file +rather than repeating the version — CI cannot drift from the pin. + +**Linux system dependencies come from the host's own script.** The job runs +`augur-rs/.github/scripts/install-linux-deps.sh` from the checkout it already +has, instead of keeping a second list that can go stale. `serialport` (used by +`stage-a-modulation` and `stage-a-photodiode`) needs `libudev`, and that script +is guaranteed to be a superset of what the plugins need. + +**The build goes through the repo's own two scripts.** `build-runtime-plugins.sh` +and `install-built-plugins.sh` already know which crates are runtime plugins, +which library name each `plugin.toml` declares, that A1's `protocols/` folder has +to travel with the plugin, and that macOS copies need their dylib id rewritten to +`@loader_path/`. Re-implementing any of that in YAML would be a second +source of truth. CI runs the same commands a developer runs, only with +`--dest dist/`. + +**Archiving happens once, in the release job.** The build matrix uploads raw +folders; the Ubuntu release job zips them. `zip` is not available in the Windows +runner's bash by default, so packaging on each runner would have needed a +per-platform branch for no benefit. + +## Provenance + +Each bundle carries `BUILD-INFO.txt`: + +```text +bundle: macos-arm64 +built_at: 2026-08-04T19:38:11Z +augur_plugins: 30e677c… +augur_rs_ref: main +augur_rs_sha: d43652a… +rustc: rustc 1.95.0 (…) +``` + +That is what turns an "ABI mismatch" report from the bench into an answerable +question: it records exactly which host revision and which compiler the installed +library was built against. + +## Installing a bundle + +1. Download the archive for the platform from the + [`plugins-latest` release](https://github.com/muthmann/augur-plugins/releases/tag/plugins-latest) +2. Unpack it +3. Copy the plugin folders inside into `~/.augur/plugins/` +4. In `augur-gui`: **Plugins** → **Scan for New Plugins** → enable + +See [Installing Runtime Plugins](../installing-plugins.md) for the full +installed layout and troubleshooting. + +## Limitations + +- The macOS bundles are single-architecture, not universal. `augur-gui` ships as + a universal binary, so an Intel Mac needs `macos-x86_64` and an Apple Silicon + Mac needs `macos-arm64`; picking the wrong one fails at load, not at copy. +- `macos-13` is GitHub's last x86_64 macOS runner image. When it is retired, the + Intel bundle needs a cross-build (`--target x86_64-apple-darwin`), which the + install script does not currently look for — it only reads `target/`. +- The bundles are unsigned. macOS Gatekeeper does not quarantine libraries loaded + by `dlopen` from a user directory, so this has not needed handling, but a + downloaded archive may still need `xattr -d com.apple.quarantine` if Safari + attached the flag. +- `Cargo.lock` is gitignored, so builds are not `--locked`. A dependency + publishing a broken semver-compatible release can turn CI red without a commit + in either repository. diff --git a/docs/installing-plugins.md b/docs/installing-plugins.md index 45a79e9..0b17073 100644 --- a/docs/installing-plugins.md +++ b/docs/installing-plugins.md @@ -21,6 +21,33 @@ On Linux the library ends in `.so`. On Windows it ends in `.dll`. Host-owned built-in tools are part of `augur-gui` and are not installed from this repository. +## Install Without A Toolchain (recommended for bench machines) + +CI builds every runtime plugin on each push to `main` and publishes them as a +rolling [`plugins-latest`](https://github.com/muthmann/augur-plugins/releases/tag/plugins-latest) +release, already in the layout above. Installing is then a copy: + +```bash +curl -LO https://github.com/muthmann/augur-plugins/releases/download/plugins-latest/augur-plugins-macos-arm64.zip +unzip augur-plugins-macos-arm64.zip -d bundle +mkdir -p ~/.augur/plugins +cp -R bundle/*/ ~/.augur/plugins/ +``` + +Archives exist for `macos-arm64`, `macos-x86_64`, `linux-x86_64` and +`windows-x86_64`. The macOS libraries are per-architecture, not universal, so an +Apple Silicon machine needs `macos-arm64` even though `augur-gui` itself ships +universal — the wrong one fails at load time, not at copy time. + +Every archive carries a `BUILD-INFO.txt` naming the `augur-rs` revision and the +`rustc` version it was built with. That is the first thing to check against a +[plugin ABI mismatch](#plugin-abi-mismatch). Verify downloads against +`SHA256SUMS.txt` from the same release. + +The sections below cover building from source, which contributors still need. +See [CI Prebuilt Plugin Bundles](./features/ci-prebuilt-plugin-bundles.md) for how +the bundles are produced. + ## Build One Plugin ```bash @@ -99,6 +126,14 @@ The library was built against an older plugin interface or does not export the r The installed runtime library is stale relative to the host ABI. +If the library came from a release bundle, compare its `BUILD-INFO.txt` against the +running host first — `augur_rs_sha` says which host revision it was built for, and +`rustc` says which compiler produced it. Plugins are loaded into the host process, +so a compiler mismatch is as much a cause as a stale revision; +[`rust-toolchain.toml`](../rust-toolchain.toml) pins the same version `augur-rs` +does, but only a rustup-managed `cargo` honours it. Check with +`cargo --version` — a Homebrew or distro `cargo` earlier on `PATH` ignores the pin. + 1. Rebuild the plugin against the current sibling `augur-rs` checkout. 2. Replace the installed runtime library in `~/.augur/plugins//`. 3. On macOS, prefer `./scripts/install-built-plugins.sh --profile release` or rewrite the copied dylib id with `install_name_tool -id "@loader_path/" ...`. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..544a2fa --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,8 @@ +[toolchain] +# Must match augur-rs/rust-toolchain.toml. Plugins are cdylibs that the host +# dlopens into its own process, so the compiler that builds them and the +# compiler that builds augur-gui have to agree — an unpinned "stable" here +# means CI can hand the bench a bundle built against a different std. +# Bump this together with augur-rs, in its own commit. +channel = "1.95.0" +components = ["clippy", "rustfmt"] From c61fa6c59e527063cadc5f0a01c25b7699c55b8b Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 21:49:25 +0200 Subject: [PATCH 39/46] =?UTF-8?q?ci:=20=F0=9F=90=9B=20stop=20the=20build?= =?UTF-8?q?=20failing=20on=20the=20host's=20own=20CI=20layout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects the first run exposed, both independent of the plugin sources. The Linux job borrowed augur-rs/.github/scripts/install-linux-deps.sh from the host checkout to avoid keeping a second dependency list. That script does not exist on every augur-rs revision the job can be pointed at, so the Linux build failed on the value of augur_rs_ref rather than on anything in this repository. It was also a superset: it installs the X11/Wayland/GL stack for the GUI, which no plugin crate links. Install what the plugins actually need instead — pkg-config and libudev-dev for serialport. setup-rust-toolchain injects RUSTFLAGS="-D warnings" by default. That is right for a lint job and wrong for one that ships artifacts: a dead-code warning in one plugin would have denied the bench a bundle for all of them. Lint gating belongs in its own job. Also record what the run proved about the repository itself: these plugins do not compile against augur-rs main, which lacks the TableSchema, host-view and dataset-descriptor API they use. --- .github/workflows/build-plugins.yml | 16 ++++++++-- docs/features/ci-prebuilt-plugin-bundles.md | 33 +++++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-plugins.yml b/.github/workflows/build-plugins.yml index 303e653..499e495 100644 --- a/.github/workflows/build-plugins.yml +++ b/.github/workflows/build-plugins.yml @@ -94,12 +94,22 @@ jobs: with: toolchain: ${{ env.RUST_CHANNEL }} cache-workspaces: augur-plugins + # The action injects RUSTFLAGS="-D warnings" by default. That is right + # for a lint job and wrong here: this job ships artifacts, and a dead- + # code warning in one plugin must not deny the bench a bundle for all + # of them. Lint gating belongs in its own job, not in the build. + rustflags: "" - name: Install Linux system dependencies if: runner.os == 'Linux' - # Reuse the host's own dependency list rather than a second copy that can - # drift; it is a superset of what the plugins need (serialport/libudev). - run: bash augur-rs/.github/scripts/install-linux-deps.sh + # Only what the plugin crates actually link. augur-gui's own dependency + # script is deliberately not reused: it is a superset (X11/Wayland/GL for + # the GUI, which no plugin links) and it does not exist on every augur-rs + # revision this job can be pointed at, so borrowing it made the Linux + # build fail on the value of augur_rs_ref. serialport needs libudev. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends pkg-config libudev-dev - name: Build runtime plugins working-directory: augur-plugins diff --git a/docs/features/ci-prebuilt-plugin-bundles.md b/docs/features/ci-prebuilt-plugin-bundles.md index c597487..d2a43ec 100644 --- a/docs/features/ci-prebuilt-plugin-bundles.md +++ b/docs/features/ci-prebuilt-plugin-bundles.md @@ -71,11 +71,18 @@ as `augur-gui`. [`rust-toolchain.toml`](../../rust-toolchain.toml) pins the same `1.95.0` as `augur-rs`, and the workflow parses the channel out of that file rather than repeating the version — CI cannot drift from the pin. -**Linux system dependencies come from the host's own script.** The job runs -`augur-rs/.github/scripts/install-linux-deps.sh` from the checkout it already -has, instead of keeping a second list that can go stale. `serialport` (used by -`stage-a-modulation` and `stage-a-photodiode`) needs `libudev`, and that script -is guaranteed to be a superset of what the plugins need. +**Linux system dependencies are the plugins' own, not the host's.** The job +installs `pkg-config` and `libudev-dev`, which is what `serialport` (used by +`stage-a-modulation` and `stage-a-photodiode`) needs. Reusing +`augur-rs/.github/scripts/install-linux-deps.sh` was tried first and reverted: it +pulls the whole GUI stack that no plugin links, and it does not exist on every +`augur-rs` revision this job can be pointed at, so the Linux build failed on the +value of `augur_rs_ref` rather than on anything in this repository. + +**Warnings are not errors here.** `actions-rust-lang/setup-rust-toolchain` +injects `RUSTFLAGS="-D warnings"` by default. This job ships artifacts, so it +sets `rustflags: ""` — a dead-code warning in one plugin must not deny the bench +a bundle for all of them. Lint gating belongs in its own job. **The build goes through the repo's own two scripts.** `build-runtime-plugins.sh` and `install-built-plugins.sh` already know which crates are runtime plugins, @@ -118,6 +125,22 @@ library was built against. See [Installing Runtime Plugins](../installing-plugins.md) for the full installed layout and troubleshooting. +## Prerequisite: the host API this repo targets must be on `augur-rs` `main` + +The workflow defaults to building against `augur-rs` `main`, and that only works +once `main` actually carries the host API these plugins use. At the time this +workflow was added it did not: the eveSMLM plugins reference `TableSchema` +fields (`layer_id`, `semantic_label`, `provenance`, `column_display`, +`row_id_column`, `time_column`, `coordinate_space_3d`), +`HostViewKind::Scatter3dFromTable`, `HostDatasetDescriptor.relations` / +`.display` and `HostViewRegistry.actions`, none of which exist on `augur-rs` +`main` — they live on an unmerged host branch. + +This is a real finding rather than a CI defect: it means the repository as +checked in cannot be built by anyone who does not already have that unmerged +host branch on disk. Until it lands, point `workflow_dispatch` at a pushed +`augur_rs_ref` that carries the API. + ## Limitations - The macOS bundles are single-architecture, not universal. `augur-gui` ships as From 671b25434074bb6654c5b2aececce1010072d530 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 21:51:57 +0200 Subject: [PATCH 40/46] =?UTF-8?q?ci:=20=F0=9F=94=A7=20build=20against=20th?= =?UTF-8?q?e=20host=20branch=20that=20has=20the=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit augur-rs main still has a two-field TableSchema and no Scatter3dFromTable, HostDatasetDescriptor.relations/display or HostViewRegistry.actions, all of which the plugins in this repository already use. Defaulting AUGUR_RS_REF to main is therefore a guaranteed red build that never hands the bench a bundle. Default to the open host branch that does carry the API instead, and record the coupling in the brief and the ADR. BUILD-INFO.txt already names the exact host ref and SHA behind every library, so this stays visible rather than becoming folklore. Move the default back to main in the same commit that the host API lands there. --- .github/workflows/build-plugins.yml | 9 +++-- .../030-prebuilt-plugin-bundles-from-ci.md | 13 +++++--- docs/features/ci-prebuilt-plugin-bundles.md | 33 +++++++++++-------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-plugins.yml b/.github/workflows/build-plugins.yml index 499e495..7f8278b 100644 --- a/.github/workflows/build-plugins.yml +++ b/.github/workflows/build-plugins.yml @@ -15,7 +15,7 @@ on: augur_rs_ref: description: "augur-rs ref to build against (branch, tag or SHA)" required: false - default: main + default: fix/gui-layout-and-alignment concurrency: group: build-plugins-${{ github.ref }} @@ -26,7 +26,12 @@ permissions: env: CARGO_TERM_COLOR: always - AUGUR_RS_REF: ${{ inputs.augur_rs_ref || 'main' }} + # The host branch this repository actually compiles against. augur-rs `main` + # does not carry the TableSchema, host-view and dataset-descriptor API these + # plugins use, so defaulting to `main` would be a guaranteed red build and + # would never hand the bench a bundle. Move this back to `main` in the same + # commit that the host API lands there. + AUGUR_RS_REF: ${{ inputs.augur_rs_ref || 'fix/gui-layout-and-alignment' }} jobs: build: diff --git a/docs/adr/030-prebuilt-plugin-bundles-from-ci.md b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md index 797edf3..d00a3dc 100644 --- a/docs/adr/030-prebuilt-plugin-bundles-from-ci.md +++ b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md @@ -69,11 +69,14 @@ Every bundle carries a `BUILD-INFO.txt` recording the `augur-plugins` commit, th **Publish only workflow artifacts.** Simplest, and rejected: it puts a GitHub login between the bench and a fix, and the artifact disappears after 90 days. -**Build against the newest `augur-rs` release tag instead of `main`.** Matches a -bench running a released host, but lags every unpublished API change — and the -Stage-A work in this repository routinely depends on unreleased host changes. -`main` is the default; `workflow_dispatch` takes an `augur_rs_ref` for the cases -where a specific host revision is wanted. +**Build against the newest `augur-rs` release tag, or against `main`.** Both were +rejected by fact rather than by preference: `augur-rs` `main` does not carry the +`TableSchema`, host-view or dataset-descriptor API these plugins already use, so +either choice is a guaranteed red build. The default host ref is therefore the +open host branch that does carry it, and `BUILD-INFO.txt` records the exact ref +and SHA behind every library so the coupling stays visible. This is temporary by +construction: the default moves to `main` in the same commit that the host API +lands there. **Reimplement the install layout in the workflow.** Would have avoided calling shell scripts from YAML, at the cost of a second, silently divergent definition diff --git a/docs/features/ci-prebuilt-plugin-bundles.md b/docs/features/ci-prebuilt-plugin-bundles.md index d2a43ec..bdacf01 100644 --- a/docs/features/ci-prebuilt-plugin-bundles.md +++ b/docs/features/ci-prebuilt-plugin-bundles.md @@ -48,7 +48,9 @@ GitHub login and expire; a release asset can be fetched from the bench with `curl` and no account. `workflow_dispatch` takes an `augur_rs_ref` input for building a bundle against a -host branch or tag other than `main`. +different host branch or tag — but note that GitHub only offers `workflow_dispatch` +for workflows present on the default branch, so until this lands on `main` the +`AUGUR_RS_REF` default below is the only way to retarget the host revision. ## Why it is shaped this way @@ -125,21 +127,26 @@ library was built against. See [Installing Runtime Plugins](../installing-plugins.md) for the full installed layout and troubleshooting. -## Prerequisite: the host API this repo targets must be on `augur-rs` `main` +## Which host revision the bundles are built against -The workflow defaults to building against `augur-rs` `main`, and that only works -once `main` actually carries the host API these plugins use. At the time this -workflow was added it did not: the eveSMLM plugins reference `TableSchema` -fields (`layer_id`, `semantic_label`, `provenance`, `column_display`, -`row_id_column`, `time_column`, `coordinate_space_3d`), +`AUGUR_RS_REF` currently defaults to the `augur-rs` branch +`fix/gui-layout-and-alignment` (open host PR #36), **not** to `main`. + +That is not a preference, it is the state of the two repositories. These plugins +reference `TableSchema` fields (`layer_id`, `semantic_label`, `provenance`, +`column_display`, `row_id_column`, `time_column`, `coordinate_space_3d`), `HostViewKind::Scatter3dFromTable`, `HostDatasetDescriptor.relations` / -`.display` and `HostViewRegistry.actions`, none of which exist on `augur-rs` -`main` — they live on an unmerged host branch. +`.display` and `HostViewRegistry.actions` — none of which exist on `augur-rs` +`main`, which still has a two-field `TableSchema`. Defaulting to `main` would be +a guaranteed red build that never hands the bench a bundle. + +The first CI run is what surfaced this: the repository as checked in cannot be +built by anyone who does not already have an unmerged host branch on disk. -This is a real finding rather than a CI defect: it means the repository as -checked in cannot be built by anyone who does not already have that unmerged -host branch on disk. Until it lands, point `workflow_dispatch` at a pushed -`augur_rs_ref` that carries the API. +**Move the default back to `main` in the same commit that the host API lands +there.** Until then, `BUILD-INFO.txt` is the thing that keeps this honest — it +records the exact host ref and SHA behind every library, so an installed plugin +can always be traced to the host revision it matches. ## Limitations From a24e135e0e6079808a31a3cca4fdae1eac9db05a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 22:04:37 +0200 Subject: [PATCH 41/46] =?UTF-8?q?refactor(evesmlm):=20=E2=99=BB=EF=B8=8F?= =?UTF-8?q?=20share=20types=20through=20a=20crate,=20not=20between=20plugi?= =?UTF-8?q?ns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eveSMLM chain expressed its stage dependencies directly: fitting depended on the candidates crate, post-processing on the fitting crate. Plugin crates are cdylibs that each export augur_plugin_vtable, so linking one plugin's rlib into another pulled that symbol in twice. Apple's linker tolerates the duplicate. rust-lld and MSVC's link.exe do not: rust-lld: error: duplicate symbol: augur_plugin_vtable LNK2005: augur_plugin_vtable already defined … fatal error LNK1169 That went unnoticed for as long as the only build machine was a Mac. The first CI run on four platforms found it: macOS produced a complete bundle while Linux and Windows failed to link, which also denied the bench a Windows bundle for the Stage-A plugins, since the build is all-or-nothing. Move everything that crosses a stage boundary into evesmlm-types, a plain library crate that exports no vtable — the wire contract plus the current-localization dataset and registry builders that both fitting and post-processing publish. Plugin-private state stays with its plugin: the candidate tracker's TrackedCluster moves back into the candidates crate. Each plugin still re-exports the names it used to own, so downstream use paths keep compiling. This generalizes what stage-a-plugin-contract already does for the Stage-A owners, and replaces the repo convention that shared types belong in the producing plugin's crate. --- Cargo.toml | 2 + ...031-evesmlm-plugins-share-a-types-crate.md | 86 +++ docs/features/README.md | 2 +- docs/features/evesmlm.md | 15 + evesmlm-types/Cargo.toml | 13 + .../src/candidates.rs | 31 +- evesmlm-types/src/datasets.rs | 488 ++++++++++++++++++ evesmlm-types/src/lib.rs | 35 ++ .../src/localization.rs | 0 plugins/evesmlm-candidates/Cargo.toml | 1 + plugins/evesmlm-candidates/src/lib.rs | 24 +- plugins/evesmlm-candidates/src/tracking.rs | 20 + plugins/evesmlm-fitting/Cargo.toml | 2 +- plugins/evesmlm-fitting/src/gaussian.rs | 2 +- plugins/evesmlm-fitting/src/lib.rs | 488 +----------------- plugins/evesmlm-fitting/src/log_gaussian.rs | 2 +- plugins/evesmlm-fitting/src/mean_xy.rs | 2 +- plugins/evesmlm-fitting/src/phasor.rs | 2 +- .../evesmlm-fitting/src/radial_symmetry.rs | 2 +- plugins/evesmlm-postproc/Cargo.toml | 2 +- .../evesmlm-postproc/src/drift_correction.rs | 2 +- plugins/evesmlm-postproc/src/evaluation.rs | 2 +- plugins/evesmlm-postproc/src/filtering.rs | 2 +- plugins/evesmlm-postproc/src/lib.rs | 8 +- 24 files changed, 704 insertions(+), 529 deletions(-) create mode 100644 docs/adr/031-evesmlm-plugins-share-a-types-crate.md create mode 100644 evesmlm-types/Cargo.toml rename plugins/evesmlm-candidates/src/types.rs => evesmlm-types/src/candidates.rs (87%) create mode 100644 evesmlm-types/src/datasets.rs create mode 100644 evesmlm-types/src/lib.rs rename plugins/evesmlm-fitting/src/types.rs => evesmlm-types/src/localization.rs (100%) create mode 100644 plugins/evesmlm-candidates/src/tracking.rs diff --git a/Cargo.toml b/Cargo.toml index abb6328..3d4a38f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "evesmlm-types", "stage-a-io", "stage-a-plugin-contract", "plugins/stage-a-a1", @@ -27,6 +28,7 @@ augur-core = { path = "../augur-rs/augur-core" } augur-plugin-api = { path = "../augur-rs/augur-plugin-api" } augur-plugin-types = { path = "../augur-rs/augur-plugin-types" } egui = "0.27" +evesmlm-types = { path = "evesmlm-types" } rustfft = "6" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/docs/adr/031-evesmlm-plugins-share-a-types-crate.md b/docs/adr/031-evesmlm-plugins-share-a-types-crate.md new file mode 100644 index 0000000..a75f624 --- /dev/null +++ b/docs/adr/031-evesmlm-plugins-share-a-types-crate.md @@ -0,0 +1,86 @@ +# ADR 031 — Plugins share a types crate, never each other + +**Status:** accepted +**Date:** 2026-08-04 +**Supersedes:** the "shared types are exported from the producing plugin's crate" convention +**Feature brief:** [eveSMLM Pipeline](../features/evesmlm.md) + +## Context + +The eveSMLM plugins form a chain: fitting consumes what candidates publishes, +post-processing consumes what fitting publishes. The repository convention was +that shared types are exported from the producing plugin's crate, so +`augur-plugin-evesmlm-fitting` depended on `augur-plugin-evesmlm-candidates`, and +`augur-plugin-evesmlm-postproc` depended on fitting. + +Plugin crates are built as `crate-type = ["cdylib", "rlib"]`, and each one +invokes `export_plugin!`, which emits `#[no_mangle] augur_plugin_vtable`. A +plugin that depends on another plugin therefore links that plugin's rlib — and +its vtable symbol — into its own `cdylib`. + +The Apple linker tolerates the duplicate. `rust-lld` and MSVC's `link.exe` do +not: + +``` +rust-lld: error: duplicate symbol: augur_plugin_vtable +LNK2005: augur_plugin_vtable already defined … fatal error LNK1169 +``` + +This was invisible for as long as the only build machine was a Mac. It surfaced +the first time CI built the repository on Linux and Windows (ADR 030): macOS +arm64 produced a complete bundle while both other platforms failed to link. +Every non-macOS user was locked out of the eveSMLM chain, and Stage-A users were +locked out of a Windows bundle entirely, because the build is all-or-nothing. + +## Decision + +**A plugin crate may not depend on another plugin crate.** Everything that +crosses a plugin boundary lives in a plain library crate that exports no vtable. + +For eveSMLM that crate is `evesmlm-types`, holding the wire contract +(`EveEvent`, `EveCluster`, `EveCandidates`, `EveLocalization`, +`EveLocalizationResults`, `FitMethod`, the `CTX_*` channel names) and the +current-localization dataset surface that both fitting and post-processing +publish (`current_localizations_registry_for_results`, +`current_localizations_dataset`, `localization_row_id`, +`to_localization_results`, the `CURRENT_LOCALIZATIONS_*` ids). + +Plugin-private types stay in their plugin: the candidate tracker's +`TrackedCluster` moved back out of the shared crate into +`plugins/evesmlm-candidates/src/tracking.rs`. The test is whether another plugin +names the type, not whether it happens to sit next to one that does. + +Each plugin keeps re-exporting the shared names it used to own, so downstream +`use augur_plugin_evesmlm_fitting::EveLocalization` keeps compiling. + +## Consequences + +- The eveSMLM chain links on Linux and Windows, so CI can produce bundles for all + four platforms rather than macOS only. +- Every plugin `cdylib` exports exactly one `augur_plugin_vtable`, which is what + the host's loader assumes in the first place. +- `stage-a-plugin-contract` was already built this way for the Stage-A owner + plugins (ADR 005/006). This generalizes that pattern instead of treating it as + a Stage-A peculiarity. +- The repository convention in `CLAUDE.md` and `CONTRIBUTING.md` — "shared types + between plugins should be exported from the producing plugin's crate" — is + wrong as stated and is replaced by this ADR. +- One more crate per plugin family. That is the cost of the rule, and it is + smaller than the cost of a platform-specific link failure that only shows up + on a machine nobody builds on. + +## Alternatives considered + +**Feature-gate `export_plugin!` and have dependents disable it.** Would keep the +plugin-to-plugin dependency. Rejected: cargo unifies features across a workspace +build, so the `cdylib` target and the same crate consumed as an rlib dependency +resolve to one feature set — the vtable would be on for both, or off for both. + +**Duplicate the shared type definitions in each plugin.** No new crate, and no +shared contract either: the two copies would drift, and the published JSON is +exactly what must not drift. + +**Build the eveSMLM plugins only on macOS.** Considered because the bench PC that +needed a Windows bundle runs Stage-A, not eveSMLM. Rejected: it encodes a +linker accident as a platform policy, and it leaves the bug in place for the +next plugin family that chains. diff --git a/docs/features/README.md b/docs/features/README.md index fb930c9..845eac3 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -23,4 +23,4 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Clickable 2D Overlays via Marker `source_row`](./clickable-overlays-source-row.md) — plugin-api ABI 4 `source_dataset_id`/`source_row_id` plumbing and failed-fit click-to-select loop. - [Action Requests And Single-Cluster Refit](./action-requests-and-refit.md) — plugin-declared host actions, eveSMLM refit/commit/discard flow on the `augur.evesmlm.refit_preview` dataset. - [Reconstruction Workflow](./reconstruction.md) — accumulated localization tables rendered and exported by the host. -- [eveSMLM Pipeline](./evesmlm.md) — candidate finding, fitting, and post-processing as three chainable plugins. +- [eveSMLM Pipeline](./evesmlm.md) — candidate finding, fitting, and post-processing as three chainable plugins, chained through the shared `evesmlm-types` contract crate rather than through each other: every plugin exports `augur_plugin_vtable`, so a plugin-to-plugin rlib dependency duplicated that symbol and failed to link on Linux and Windows while macOS accepted it (ADR 031). diff --git a/docs/features/evesmlm.md b/docs/features/evesmlm.md index 8ef6cb6..f65d73f 100644 --- a/docs/features/evesmlm.md +++ b/docs/features/evesmlm.md @@ -8,6 +8,21 @@ The eveSMLM pipeline is implemented as three focused plugins so each stage can b 2. **EVE Candidate Fitting** (`DerivedData`) converts each completed candidate into one or more sub-pixel localization estimates, republishes `EveLocalizationResults` and `LocalizationResults`, and exposes both the shared host-view dataset `augur.evesmlm.current_localizations` and the rejected-fit dataset `augur.evesmlm.rejected_fits`. 3. **EVE Post-Processing** (`DerivedData`) filters, drift-corrects, and evaluates the fitted localizations, then republishes the same host-view dataset id and view ids with the same schema and metadata. +## Shared Contract Crate + +The three plugins do **not** depend on each other. Everything that crosses a +stage boundary — `EveEvent`, `EveCluster`, `EveCandidates`, `EveLocalization`, +`EveLocalizationResults`, `FitMethod`, the `CTX_*` channel names, and the +`augur.evesmlm.current_localizations` dataset/registry builders that both +fitting and post-processing publish — lives in the `evesmlm-types` crate. + +That is not a stylistic choice. Each plugin `cdylib` exports +`augur_plugin_vtable`, so a plugin that linked another plugin's rlib pulled the +symbol in twice. macOS linked it anyway; `rust-lld` and MSVC's `link.exe` +refused, which meant the chain silently only worked on macOS until CI first +built the repository on Linux and Windows (ADR 031). Each plugin still +re-exports the names it used to own, so existing `use` paths keep working. + ## Why Three Plugins - Keeps raw-event grouping separate from numerical fitting, so candidate quality can be inspected directly. diff --git a/evesmlm-types/Cargo.toml b/evesmlm-types/Cargo.toml new file mode 100644 index 0000000..ffab69b --- /dev/null +++ b/evesmlm-types/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "evesmlm-types" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +description = "Shared eveSMLM contract types and current-localization dataset builders for the candidate/fitting/post-processing plugin chain" + +[dependencies] +augur-plugin-api.workspace = true +augur-plugin-types.workspace = true +serde.workspace = true diff --git a/plugins/evesmlm-candidates/src/types.rs b/evesmlm-types/src/candidates.rs similarity index 87% rename from plugins/evesmlm-candidates/src/types.rs rename to evesmlm-types/src/candidates.rs index 6bf8b1b..2320657 100644 --- a/plugins/evesmlm-candidates/src/types.rs +++ b/evesmlm-types/src/candidates.rs @@ -2,6 +2,7 @@ use augur_plugin_api::FfiCdEvent; use serde::{Deserialize, Serialize}; pub const CTX_EVE_CANDIDATES: &str = "augur.evesmlm.candidates"; +pub const ACCEPTED_CANDIDATE_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; fn default_cluster_complete() -> bool { true @@ -17,6 +18,22 @@ pub enum CandidateFindingMethod { } impl CandidateFindingMethod { + pub fn from_index(index: usize) -> Self { + match index { + 1 => Self::Eigenfeature, + 2 => Self::FrameBased, + _ => Self::Dbscan, + } + } + + pub fn index(self) -> usize { + match self { + Self::Dbscan => 0, + Self::Eigenfeature => 1, + Self::FrameBased => 2, + } + } + pub fn label(self) -> &'static str { match self { Self::Dbscan => "DBSCAN", @@ -124,17 +141,3 @@ pub struct EveCandidates { pub n_events_processed: usize, pub finding_method: CandidateFindingMethod, } - -#[derive(Debug, Clone)] -pub(crate) struct TrackedCluster { - pub id: u64, - pub centroid_x: f64, - pub centroid_y: f64, - pub event_count: usize, - pub last_seen_frame: u64, - pub last_grown_frame: u64, - pub frames_since_growth: usize, - pub complete: bool, - pub emitted: bool, - pub cluster: EveCluster, -} diff --git a/evesmlm-types/src/datasets.rs b/evesmlm-types/src/datasets.rs new file mode 100644 index 0000000..fe3d43f --- /dev/null +++ b/evesmlm-types/src/datasets.rs @@ -0,0 +1,488 @@ +//! The current-localization dataset, its schema and the host-view registry +//! built from it, plus the conversion to the standard `LocalizationResults`. +//! +//! These live here rather than in the fitting plugin because post-processing +//! republishes the same dataset — see the crate docs for why a plugin must not +//! link another plugin's rlib. + +use augur_plugin_api::{ + HostDatasetDescriptor, HostDatasetDisplayMetadata, HostDatasetKind, HostDatasetRelation, + HostMarkerShape, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, + TableColumn, TableColumnData, TableColumnDisplayEntry, TableColumnDisplayFormat, + TableColumnDisplayMetadata, TableColumnValues, TableCoordinateSpace2d, TableCoordinateSpace3d, + TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, +}; +use augur_plugin_types::{Localization, LocalizationResults}; + +use crate::candidates::ACCEPTED_CANDIDATE_EVENTS_DATASET_ID; +use crate::localization::{EveLocalization, EveLocalizationResults}; + +pub const CURRENT_LOCALIZATIONS_DATASET_ID: &str = "augur.evesmlm.current_localizations"; +pub const CURRENT_LOCALIZATIONS_LAYER_ID: &str = "augur.layer.evesmlm.current_localizations"; +pub const CURRENT_LOCALIZATIONS_VIEW_ID: &str = "augur.evesmlm.current_localizations.compact"; +pub const CURRENT_LOCALIZATIONS_3D_VIEW_ID: &str = "augur.evesmlm.current_localizations.scatter3d"; + +pub fn current_localizations_registry() -> HostViewRegistry { + current_localizations_registry_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_registry_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + title: "Current EVE localizations".into(), + kind: HostDatasetKind::TableV1(current_localizations_schema_for_results( + results, + sensor_dims, + )), + empty_message: "No EVE localizations in the current frame.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Current EVE localizations".into()), + default_visibility: Some(true), + default_color: Some([90, 170, 255, 255]), + default_marker_shape: Some(HostMarkerShape::Cross), + default_size: Some(6.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }], + views: vec![ + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), + title: "Current Localizations".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_3D_VIEW_ID.into(), + title: "Current Localizations 3D".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), + } +} + +pub fn current_localizations_schema() -> TableSchema { + current_localizations_schema_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_schema_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> TableSchema { + TableSchema { + columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_start_us".into(), + title: "Span Start (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_end_us".into(), + title: "Span End (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "x_px".into(), + title: "X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "y_px".into(), + title: "Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_x_px".into(), + title: "Sigma X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_y_px".into(), + title: "Sigma Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "n_events".into(), + title: "Events".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "polarity_balance".into(), + title: "Polarity balance".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_residual".into(), + title: "Fit residual".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_method".into(), + title: "Fit method".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: current_localizations_2d_space(results, sensor_dims), + coordinate_space_3d: current_localizations_3d_space(results, sensor_dims), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(CURRENT_LOCALIZATIONS_LAYER_ID.into()), + semantic_label: Some("localizations".into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("span_start_us".into()), + span_end_column: Some("span_end_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "row_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_start_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span start".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_end_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span end".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_residual".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_method".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + ..Default::default() + }, + }, + ], + } +} + +pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { + TableDatasetV1::new(vec![ + TableColumnData { + column_id: "row_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(localization_row_id) + .collect(), + ), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.cluster_id) + .collect(), + ), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.timestamp_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_start_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_start_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_end_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_end_us) + .collect(), + ), + }, + TableColumnData { + column_id: "x_px".into(), + values: TableColumnValues::F64( + results.localizations.iter().map(|value| value.x).collect(), + ), + }, + TableColumnData { + column_id: "y_px".into(), + values: TableColumnValues::F64( + results.localizations.iter().map(|value| value.y).collect(), + ), + }, + TableColumnData { + column_id: "sigma_x_px".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.sigma_x) + .collect(), + ), + }, + TableColumnData { + column_id: "sigma_y_px".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.sigma_y) + .collect(), + ), + }, + TableColumnData { + column_id: "n_events".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.n_events as u64) + .collect(), + ), + }, + TableColumnData { + column_id: "polarity_balance".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.polarity_balance) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_residual".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.fit_residual) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_method".into(), + values: TableColumnValues::String( + results + .localizations + .iter() + .map(|value| value.fit_method.label().to_owned()) + .collect(), + ), + }, + ]) + .expect("current localization columns should stay aligned") +} + +fn current_localizations_2d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results)) + .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min, + x_max, + y_min, + y_max, + }) +} + +fn current_localizations_3d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + let (x_min, x_max, y_min, y_max) = sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results))?; + let (z_min, z_max) = localization_time_bounds(results)?; + Some(TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min, + x_max, + y_min, + y_max, + z_min, + z_max, + }) +} +pub fn localization_xy_bounds(results: &EveLocalizationResults) -> Option<(f64, f64, f64, f64)> { + let mut localizations = results.localizations.iter(); + let first = localizations.next()?; + let mut x_min = first.x; + let mut x_max = first.x; + let mut y_min = first.y; + let mut y_max = first.y; + for localization in localizations { + x_min = x_min.min(localization.x); + x_max = x_max.max(localization.x); + y_min = y_min.min(localization.y); + y_max = y_max.max(localization.y); + } + Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) +} + +pub fn localization_time_bounds(results: &EveLocalizationResults) -> Option<(f64, f64)> { + if let Some(first) = results.localizations.first() { + let mut min_time = first.timestamp_us; + let mut max_time = first.timestamp_us; + for localization in &results.localizations { + min_time = min_time.min(localization.timestamp_us); + max_time = max_time.max(localization.timestamp_us); + } + return Some((min_time as f64, max_time.max(min_time) as f64)); + } + + if results.frame_window_end_us >= results.frame_window_start_us { + return Some(( + results.frame_window_start_us as f64, + results.frame_window_end_us as f64, + )); + } + + None +} + +pub fn localization_row_id(localization: &EveLocalization) -> u64 { + localization.cluster_id.rotate_left(3) + ^ localization.timestamp_us + ^ localization.x.to_bits().rotate_left(7) + ^ localization.y.to_bits().rotate_left(19) + ^ localization.sigma_x.to_bits().rotate_left(31) + ^ localization.sigma_y.to_bits().rotate_left(43) + ^ localization.fit_residual.to_bits().rotate_left(53) + ^ (localization.n_events as u64).rotate_left(11) + ^ (localization.fit_method.index() as u64).rotate_left(59) + ^ localization.span_start_us.rotate_left(17) + ^ localization.span_end_us.rotate_left(29) +} + +pub fn to_localization_results(results: &EveLocalizationResults) -> LocalizationResults { + LocalizationResults { + localizations: results + .localizations + .iter() + .map(|localization| Localization { + x: localization.x, + y: localization.y, + sigma_x: localization.sigma_x, + sigma_y: localization.sigma_y, + amplitude: 0.0, + background: 0.0, + timestamp_us: localization.timestamp_us, + fit_error: localization.fit_residual, + }) + .collect(), + frame_window_start_us: results.frame_window_start_us, + frame_window_end_us: results.frame_window_end_us, + } +} diff --git a/evesmlm-types/src/lib.rs b/evesmlm-types/src/lib.rs new file mode 100644 index 0000000..00007be --- /dev/null +++ b/evesmlm-types/src/lib.rs @@ -0,0 +1,35 @@ +//! Shared eveSMLM contract. +//! +//! The candidate, fitting and post-processing plugins form a chain: fitting +//! consumes what candidates publishes, and post-processing consumes what +//! fitting publishes. Expressing that by having one plugin crate depend on +//! another looks natural, but plugin crates are `cdylib`s that each export +//! `augur_plugin_vtable` — and a plugin that links another plugin's rlib pulls +//! that symbol in twice. The Apple linker tolerates the duplicate; `rust-lld` +//! and MSVC's `link.exe` do not, so the chain built on macOS and failed to link +//! on Linux and Windows. +//! +//! Everything that crosses a plugin boundary therefore lives here, in a plain +//! library crate that exports no vtable. Plugins depend on this crate, never on +//! each other. + +pub mod candidates; +pub mod datasets; +pub mod localization; + +pub use candidates::{ + CandidateFindingMethod, ClusterBoundary, EveCandidates, EveCluster, EveEvent, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID, CTX_EVE_CANDIDATES, +}; +pub use datasets::{ + current_localizations_dataset, current_localizations_registry, + current_localizations_registry_for_results, current_localizations_schema, + current_localizations_schema_for_results, localization_row_id, localization_time_bounds, + localization_xy_bounds, to_localization_results, CURRENT_LOCALIZATIONS_3D_VIEW_ID, + CURRENT_LOCALIZATIONS_DATASET_ID, CURRENT_LOCALIZATIONS_LAYER_ID, + CURRENT_LOCALIZATIONS_VIEW_ID, +}; +pub use localization::{ + EveLocalization, EveLocalizationResults, FitMethod, RejectedFitRow, RejectionReason, + CTX_EVE_LOCALIZATION_RESULTS, +}; diff --git a/plugins/evesmlm-fitting/src/types.rs b/evesmlm-types/src/localization.rs similarity index 100% rename from plugins/evesmlm-fitting/src/types.rs rename to evesmlm-types/src/localization.rs diff --git a/plugins/evesmlm-candidates/Cargo.toml b/plugins/evesmlm-candidates/Cargo.toml index f8e7d66..07c2377 100644 --- a/plugins/evesmlm-candidates/Cargo.toml +++ b/plugins/evesmlm-candidates/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true +evesmlm-types.workspace = true nalgebra = "0.33" serde.workspace = true serde_json.workspace = true diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index d51776b..4556b2e 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -6,7 +6,7 @@ pub mod dbscan; pub mod eigenfeature; -pub mod types; +mod tracking; use std::collections::{HashMap, HashSet}; @@ -23,11 +23,11 @@ use augur_plugin_api::{ }; use serde_json::{json, Value}; -use types::TrackedCluster; -pub use types::{ +pub use evesmlm_types::{ CandidateFindingMethod, ClusterBoundary, EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, }; +use tracking::TrackedCluster; const KERNEL_G1: [f64; 5] = [1.0 / 16.0, 0.25, 3.0 / 8.0, 0.25, 1.0 / 16.0]; const KERNEL_G2: [f64; 9] = [ @@ -134,24 +134,6 @@ impl PolarityMode { } } -impl CandidateFindingMethod { - fn from_index(index: usize) -> Self { - match index { - 1 => Self::Eigenfeature, - 2 => Self::FrameBased, - _ => Self::Dbscan, - } - } - - fn index(self) -> usize { - match self { - Self::Dbscan => 0, - Self::Eigenfeature => 1, - Self::FrameBased => 2, - } - } -} - #[derive(Debug, Clone)] pub struct CandidateSettings { pub finding_method: CandidateFindingMethod, diff --git a/plugins/evesmlm-candidates/src/tracking.rs b/plugins/evesmlm-candidates/src/tracking.rs new file mode 100644 index 0000000..04fd140 --- /dev/null +++ b/plugins/evesmlm-candidates/src/tracking.rs @@ -0,0 +1,20 @@ +//! Internal cluster-tracking bookkeeping. +//! +//! Not part of the cross-plugin contract — the shared eveSMLM types live in +//! the `evesmlm-types` crate. + +use evesmlm_types::EveCluster; + +#[derive(Debug, Clone)] +pub(crate) struct TrackedCluster { + pub id: u64, + pub centroid_x: f64, + pub centroid_y: f64, + pub event_count: usize, + pub last_seen_frame: u64, + pub last_grown_frame: u64, + pub frames_since_growth: usize, + pub complete: bool, + pub emitted: bool, + pub cluster: EveCluster, +} diff --git a/plugins/evesmlm-fitting/Cargo.toml b/plugins/evesmlm-fitting/Cargo.toml index 4e3e8c1..1e14aaa 100644 --- a/plugins/evesmlm-fitting/Cargo.toml +++ b/plugins/evesmlm-fitting/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true augur-plugin-types.workspace = true -augur-plugin-evesmlm-candidates = { path = "../evesmlm-candidates" } +evesmlm-types.workspace = true levenberg-marquardt = "0.14" nalgebra = "0.33" num-complex = "0.4" diff --git a/plugins/evesmlm-fitting/src/gaussian.rs b/plugins/evesmlm-fitting/src/gaussian.rs index 429e8dd..848113d 100644 --- a/plugins/evesmlm-fitting/src/gaussian.rs +++ b/plugins/evesmlm-fitting/src/gaussian.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use crate::{mean_xy, FitEstimate}; diff --git a/plugins/evesmlm-fitting/src/lib.rs b/plugins/evesmlm-fitting/src/lib.rs index 31b83b3..51103d9 100644 --- a/plugins/evesmlm-fitting/src/lib.rs +++ b/plugins/evesmlm-fitting/src/lib.rs @@ -10,7 +10,6 @@ pub mod log_gaussian; pub mod mean_xy; pub mod phasor; pub mod radial_symmetry; -pub mod types; use augur_plugin_api::{ export_plugin, AnalysisSeverity, EventStoreHandle, FfiColorRgba, FfiMarkerOverlayItem, @@ -26,22 +25,21 @@ use augur_plugin_api::{ TableColumnDisplayMetadata, TableColumnValues, TableCoordinateSpace2d, TableCoordinateSpace3d, TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, }; -pub use augur_plugin_evesmlm_candidates::{ - EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, -}; use augur_plugin_types::{Localization, LocalizationResults, CTX_LOCALIZATION_RESULTS}; -use serde_json::{json, Value}; -pub use types::{ +pub use evesmlm_types::{ + current_localizations_dataset, current_localizations_registry, + current_localizations_registry_for_results, current_localizations_schema, + current_localizations_schema_for_results, localization_row_id, localization_time_bounds, + localization_xy_bounds, to_localization_results, EveCandidates, EveCluster, EveEvent, EveLocalization, EveLocalizationResults, FitMethod, RejectedFitRow, RejectionReason, - CTX_EVE_LOCALIZATION_RESULTS, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID, CTX_EVE_CANDIDATES, CTX_EVE_LOCALIZATION_RESULTS, + CURRENT_LOCALIZATIONS_3D_VIEW_ID, CURRENT_LOCALIZATIONS_DATASET_ID, + CURRENT_LOCALIZATIONS_LAYER_ID, CURRENT_LOCALIZATIONS_VIEW_ID, }; +use serde_json::{json, Value}; const OVERLAY_COLOR: [u8; 4] = [60, 220, 140, 220]; const CANDIDATE_DEPENDENCY: [&str; 1] = ["EVE Candidate Finding"]; -pub const CURRENT_LOCALIZATIONS_DATASET_ID: &str = "augur.evesmlm.current_localizations"; -pub const CURRENT_LOCALIZATIONS_LAYER_ID: &str = "augur.layer.evesmlm.current_localizations"; -pub const CURRENT_LOCALIZATIONS_VIEW_ID: &str = "augur.evesmlm.current_localizations.compact"; -pub const CURRENT_LOCALIZATIONS_3D_VIEW_ID: &str = "augur.evesmlm.current_localizations.scatter3d"; pub const REJECTED_FITS_DATASET_ID: &str = "augur.evesmlm.rejected_fits"; pub const REJECTED_FITS_LAYER_ID: &str = "augur.layer.evesmlm.rejected_fits"; pub const REJECTED_FITS_COMPACT_VIEW_ID: &str = "augur.evesmlm.rejected_fits.compact"; @@ -52,406 +50,10 @@ pub const REFIT_PREVIEW_DATASET_ID: &str = "augur.evesmlm.refit_preview"; pub const REFIT_PREVIEW_LAYER_ID: &str = "augur.layer.evesmlm.refit_preview"; pub const REFIT_PREVIEW_VIEW_ID: &str = "augur.evesmlm.refit_preview.compact"; -pub const ACCEPTED_CANDIDATE_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; - pub const ACTION_REFIT_CLUSTER: &str = "augur.evesmlm.refit_cluster"; pub const ACTION_COMMIT_REFIT: &str = "augur.evesmlm.commit_refit"; pub const ACTION_DISCARD_REFIT: &str = "augur.evesmlm.discard_refit"; -pub fn current_localizations_registry() -> HostViewRegistry { - current_localizations_registry_for_results(&EveLocalizationResults::default(), None) -} - -pub fn current_localizations_registry_for_results( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![HostDatasetDescriptor { - id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - title: "Current EVE localizations".into(), - kind: HostDatasetKind::TableV1(current_localizations_schema_for_results( - results, - sensor_dims, - )), - empty_message: "No EVE localizations in the current frame.".into(), - display: Some(HostDatasetDisplayMetadata { - layer_title: Some("Current EVE localizations".into()), - default_visibility: Some(true), - default_color: Some([90, 170, 255, 255]), - default_marker_shape: Some(HostMarkerShape::Cross), - default_size: Some(6.0), - }), - relations: vec![HostDatasetRelation { - target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), - via_column: "cluster_id".into(), - target_column: "cluster_id".into(), - }], - }], - views: vec![ - HostViewDescriptor { - id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), - title: "Current Localizations".into(), - dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - HostViewDescriptor { - id: CURRENT_LOCALIZATIONS_3D_VIEW_ID.into(), - title: "Current Localizations 3D".into(), - dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::Scatter3dFromTable { - x_column: "x_px".into(), - y_column: "y_px".into(), - z_column: "timestamp_us".into(), - }, - }, - ], - actions: Vec::new(), - } -} - -pub fn current_localizations_schema() -> TableSchema { - current_localizations_schema_for_results(&EveLocalizationResults::default(), None) -} - -pub fn current_localizations_schema_for_results( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> TableSchema { - TableSchema { - columns: vec![ - TableColumn { - id: "row_id".into(), - title: "ID".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "cluster_id".into(), - title: "Cluster".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "timestamp_us".into(), - title: "Timestamp (us)".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "span_start_us".into(), - title: "Span Start (us)".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "span_end_us".into(), - title: "Span End (us)".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "x_px".into(), - title: "X (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "y_px".into(), - title: "Y (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "sigma_x_px".into(), - title: "Sigma X (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "sigma_y_px".into(), - title: "Sigma Y (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "n_events".into(), - title: "Events".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "polarity_balance".into(), - title: "Polarity balance".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "fit_residual".into(), - title: "Fit residual".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "fit_method".into(), - title: "Fit method".into(), - value_type: TableValueType::String, - }, - ], - coordinate_space_2d: current_localizations_2d_space(results, sensor_dims), - coordinate_space_3d: current_localizations_3d_space(results, sensor_dims), - row_id_column: Some("row_id".into()), - time_column: Some("timestamp_us".into()), - layer_id: Some(CURRENT_LOCALIZATIONS_LAYER_ID.into()), - semantic_label: Some("localizations".into()), - provenance: Some(TableRowProvenance { - anchor_time_column: Some("timestamp_us".into()), - span_start_column: Some("span_start_us".into()), - span_end_column: Some("span_end_us".into()), - anchor_frame_column: None, - }), - column_display: vec![ - TableColumnDisplayEntry { - column_id: "row_id".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::Identifier), - hide_in_compact: true, - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "cluster_id".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::Identifier), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "timestamp_us".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::TimestampMicros), - label: Some("Time".into()), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "span_start_us".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::TimestampMicros), - label: Some("Span start".into()), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "span_end_us".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::TimestampMicros), - label: Some("Span end".into()), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "x_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "y_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "sigma_x_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "sigma_y_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "fit_residual".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "fit_method".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::Category), - ..Default::default() - }, - }, - ], - } -} - -pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { - TableDatasetV1::new(vec![ - TableColumnData { - column_id: "row_id".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(localization_row_id) - .collect(), - ), - }, - TableColumnData { - column_id: "cluster_id".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.cluster_id) - .collect(), - ), - }, - TableColumnData { - column_id: "timestamp_us".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.timestamp_us) - .collect(), - ), - }, - TableColumnData { - column_id: "span_start_us".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.span_start_us) - .collect(), - ), - }, - TableColumnData { - column_id: "span_end_us".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.span_end_us) - .collect(), - ), - }, - TableColumnData { - column_id: "x_px".into(), - values: TableColumnValues::F64( - results.localizations.iter().map(|value| value.x).collect(), - ), - }, - TableColumnData { - column_id: "y_px".into(), - values: TableColumnValues::F64( - results.localizations.iter().map(|value| value.y).collect(), - ), - }, - TableColumnData { - column_id: "sigma_x_px".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.sigma_x) - .collect(), - ), - }, - TableColumnData { - column_id: "sigma_y_px".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.sigma_y) - .collect(), - ), - }, - TableColumnData { - column_id: "n_events".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.n_events as u64) - .collect(), - ), - }, - TableColumnData { - column_id: "polarity_balance".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.polarity_balance) - .collect(), - ), - }, - TableColumnData { - column_id: "fit_residual".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.fit_residual) - .collect(), - ), - }, - TableColumnData { - column_id: "fit_method".into(), - values: TableColumnValues::String( - results - .localizations - .iter() - .map(|value| value.fit_method.label().to_owned()) - .collect(), - ), - }, - ]) - .expect("current localization columns should stay aligned") -} - -fn current_localizations_2d_space( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> Option { - sensor_dims - .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) - .or_else(|| localization_xy_bounds(results)) - .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { - x_column: "x_px".into(), - y_column: "y_px".into(), - x_min, - x_max, - y_min, - y_max, - }) -} - -fn current_localizations_3d_space( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> Option { - let (x_min, x_max, y_min, y_max) = sensor_dims - .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) - .or_else(|| localization_xy_bounds(results))?; - let (z_min, z_max) = localization_time_bounds(results)?; - Some(TableCoordinateSpace3d { - x_column: "x_px".into(), - y_column: "y_px".into(), - z_column: "timestamp_us".into(), - x_min, - x_max, - y_min, - y_max, - z_min, - z_max, - }) -} - pub fn refit_preview_registry_for_results( results: &EveLocalizationResults, sensor_dims: Option<(u16, u16)>, @@ -563,57 +165,6 @@ fn refit_action_param_schema() -> SettingsSchema { } } -fn localization_xy_bounds(results: &EveLocalizationResults) -> Option<(f64, f64, f64, f64)> { - let mut localizations = results.localizations.iter(); - let first = localizations.next()?; - let mut x_min = first.x; - let mut x_max = first.x; - let mut y_min = first.y; - let mut y_max = first.y; - for localization in localizations { - x_min = x_min.min(localization.x); - x_max = x_max.max(localization.x); - y_min = y_min.min(localization.y); - y_max = y_max.max(localization.y); - } - Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) -} - -fn localization_time_bounds(results: &EveLocalizationResults) -> Option<(f64, f64)> { - if let Some(first) = results.localizations.first() { - let mut min_time = first.timestamp_us; - let mut max_time = first.timestamp_us; - for localization in &results.localizations { - min_time = min_time.min(localization.timestamp_us); - max_time = max_time.max(localization.timestamp_us); - } - return Some((min_time as f64, max_time.max(min_time) as f64)); - } - - if results.frame_window_end_us >= results.frame_window_start_us { - return Some(( - results.frame_window_start_us as f64, - results.frame_window_end_us as f64, - )); - } - - None -} - -pub fn localization_row_id(localization: &EveLocalization) -> u64 { - localization.cluster_id.rotate_left(3) - ^ localization.timestamp_us - ^ localization.x.to_bits().rotate_left(7) - ^ localization.y.to_bits().rotate_left(19) - ^ localization.sigma_x.to_bits().rotate_left(31) - ^ localization.sigma_y.to_bits().rotate_left(43) - ^ localization.fit_residual.to_bits().rotate_left(53) - ^ (localization.n_events as u64).rotate_left(11) - ^ (localization.fit_method.index() as u64).rotate_left(59) - ^ localization.span_start_us.rotate_left(17) - ^ localization.span_end_us.rotate_left(29) -} - pub fn rejected_fit_row_id(row: &RejectedFitRow) -> u64 { row.timestamp_us ^ row.cluster_id.rotate_left(7) @@ -2344,27 +1895,6 @@ fn estimate_timestamp_us(events: &[EveEvent], x: f64, y: f64, radius: f64) -> u6 } } -pub fn to_localization_results(results: &EveLocalizationResults) -> LocalizationResults { - LocalizationResults { - localizations: results - .localizations - .iter() - .map(|localization| Localization { - x: localization.x, - y: localization.y, - sigma_x: localization.sigma_x, - sigma_y: localization.sigma_y, - amplitude: 0.0, - background: 0.0, - timestamp_us: localization.timestamp_us, - fit_error: localization.fit_residual, - }) - .collect(), - frame_window_start_us: results.frame_window_start_us, - frame_window_end_us: results.frame_window_end_us, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/plugins/evesmlm-fitting/src/log_gaussian.rs b/plugins/evesmlm-fitting/src/log_gaussian.rs index ad59c90..241e052 100644 --- a/plugins/evesmlm-fitting/src/log_gaussian.rs +++ b/plugins/evesmlm-fitting/src/log_gaussian.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use nalgebra::{DMatrix, DVector}; use crate::FitEstimate; diff --git a/plugins/evesmlm-fitting/src/mean_xy.rs b/plugins/evesmlm-fitting/src/mean_xy.rs index 95462b8..1c95852 100644 --- a/plugins/evesmlm-fitting/src/mean_xy.rs +++ b/plugins/evesmlm-fitting/src/mean_xy.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use crate::FitEstimate; diff --git a/plugins/evesmlm-fitting/src/phasor.rs b/plugins/evesmlm-fitting/src/phasor.rs index dccdbb9..cba433b 100644 --- a/plugins/evesmlm-fitting/src/phasor.rs +++ b/plugins/evesmlm-fitting/src/phasor.rs @@ -1,6 +1,6 @@ use std::f64::consts::TAU; -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use num_complex::Complex64; use crate::{mean_xy, FitEstimate}; diff --git a/plugins/evesmlm-fitting/src/radial_symmetry.rs b/plugins/evesmlm-fitting/src/radial_symmetry.rs index 6c45441..0acee29 100644 --- a/plugins/evesmlm-fitting/src/radial_symmetry.rs +++ b/plugins/evesmlm-fitting/src/radial_symmetry.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use crate::FitEstimate; diff --git a/plugins/evesmlm-postproc/Cargo.toml b/plugins/evesmlm-postproc/Cargo.toml index ec547d4..b18013e 100644 --- a/plugins/evesmlm-postproc/Cargo.toml +++ b/plugins/evesmlm-postproc/Cargo.toml @@ -12,6 +12,6 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true augur-plugin-types.workspace = true -augur-plugin-evesmlm-fitting = { path = "../evesmlm-fitting" } +evesmlm-types.workspace = true nalgebra = "0.33" serde_json.workspace = true diff --git a/plugins/evesmlm-postproc/src/drift_correction.rs b/plugins/evesmlm-postproc/src/drift_correction.rs index 9f16e50..ce85aba 100644 --- a/plugins/evesmlm-postproc/src/drift_correction.rs +++ b/plugins/evesmlm-postproc/src/drift_correction.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_fitting::EveLocalizationResults; +use evesmlm_types::EveLocalizationResults; pub fn estimate_correction_shift( reference_points: &[(f64, f64)], diff --git a/plugins/evesmlm-postproc/src/evaluation.rs b/plugins/evesmlm-postproc/src/evaluation.rs index eac5e72..ee0bd25 100644 --- a/plugins/evesmlm-postproc/src/evaluation.rs +++ b/plugins/evesmlm-postproc/src/evaluation.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use augur_plugin_evesmlm_fitting::{EveLocalization, EveLocalizationResults}; +use evesmlm_types::{EveLocalization, EveLocalizationResults}; const DEFAULT_PSF_SIZE: usize = 9; const TRACK_LINK_RADIUS_PX: f64 = 1.5; diff --git a/plugins/evesmlm-postproc/src/filtering.rs b/plugins/evesmlm-postproc/src/filtering.rs index 752f125..a0b2de7 100644 --- a/plugins/evesmlm-postproc/src/filtering.rs +++ b/plugins/evesmlm-postproc/src/filtering.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_fitting::{EveLocalization, EveLocalizationResults}; +use evesmlm_types::{EveLocalization, EveLocalizationResults}; pub fn filter_results( results: &EveLocalizationResults, diff --git a/plugins/evesmlm-postproc/src/lib.rs b/plugins/evesmlm-postproc/src/lib.rs index 4413f9d..44b0739 100644 --- a/plugins/evesmlm-postproc/src/lib.rs +++ b/plugins/evesmlm-postproc/src/lib.rs @@ -15,13 +15,13 @@ use augur_plugin_api::{ PluginInput, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, CTX_GLOBAL_SETTINGS, }; -pub use augur_plugin_evesmlm_fitting::{ +use augur_plugin_types::CTX_LOCALIZATION_RESULTS; +use evaluation::EvaluationState; +pub use evesmlm_types::{ current_localizations_dataset, current_localizations_registry_for_results, localization_row_id, to_localization_results, EveLocalization, EveLocalizationResults, FitMethod, CTX_EVE_LOCALIZATION_RESULTS, CURRENT_LOCALIZATIONS_DATASET_ID, CURRENT_LOCALIZATIONS_LAYER_ID, }; -use augur_plugin_types::CTX_LOCALIZATION_RESULTS; -use evaluation::EvaluationState; use serde_json::{json, Value}; const OVERLAY_COLOR: [u8; 4] = [90, 170, 255, 220]; @@ -690,7 +690,7 @@ mod tests { #[test] fn current_localizations_descriptor_matches_fitting() { - use augur_plugin_evesmlm_fitting::current_localizations_registry_for_results as fitting_registry; + use evesmlm_types::current_localizations_registry_for_results as fitting_registry; let results = EveLocalizationResults::default(); let fitting = fitting_registry(&results, None); let postproc = current_localizations_registry_for_results(&results, None); From e5cd98b75010263a0338dd15e1e4dc2179c2363e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 23:16:13 +0200 Subject: [PATCH 42/46] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20find=20the?= =?UTF-8?q?=20Teensy=20on=20Windows'=20nameless=20COM=20ports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port discovery filtered candidates by the two Unix name patterns (`cu.usbmodem`, `ttyACM`) before probing. Windows names no device — every port is `COMn` — so an attached, correctly driven Teensy was filtered out before any probe could run, and both plugins reported "no USB serial device found (looked for usbmodem/ttyACM)": the two things Windows cannot produce. Move the filter into `stage-a-io::transport::candidate_ports()`, where it is platform-aware: the callout node on macOS, `ttyACM*` on Linux, and every USB-classified port on Windows, falling back to the whole list when the OS classifies nothing. What identifies the device is still the probe (HELLO on the command port, PDA1 frames on the stream port); the filter only keeps probes off unrelated ports. Also open every port with DTR asserted. macOS and Linux do this implicitly, Windows does not, so a sketch gating on `if (Serial)` would stay silent even once the right port was found. The filter existed in four places in two implementations; it is now one function with unit tests covering both platform branches, and the failure message names the ports the OS actually enumerated. Refs ADR 032 --- ...teensy-port-discovery-is-platform-aware.md | 114 ++++++++++++ docs/features/README.md | 4 +- docs/features/stage-a-modulation.md | 11 ++ docs/features/stage-a-photodiode.md | 15 +- plugins/stage-a-modulation/README.md | 12 +- plugins/stage-a-modulation/src/lib.rs | 23 +-- plugins/stage-a-photodiode/Cargo.toml | 4 +- plugins/stage-a-photodiode/README.md | 9 +- plugins/stage-a-photodiode/src/lib.rs | 45 ++--- stage-a-io/src/transport.rs | 169 +++++++++++++++--- 10 files changed, 329 insertions(+), 77 deletions(-) create mode 100644 docs/adr/032-teensy-port-discovery-is-platform-aware.md diff --git a/docs/adr/032-teensy-port-discovery-is-platform-aware.md b/docs/adr/032-teensy-port-discovery-is-platform-aware.md new file mode 100644 index 0000000..64e57ca --- /dev/null +++ b/docs/adr/032-teensy-port-discovery-is-platform-aware.md @@ -0,0 +1,114 @@ +# ADR 032 — Teensy port discovery is platform-aware and lives in `stage-a-io` + +**Status:** accepted +**Date:** 2026-08-04 +**Feature briefs:** [Stage-A Modulation](../features/stage-a-modulation.md), [Stage-A Photodiode](../features/stage-a-photodiode.md) + +## Context + +Both Stage-A owner plugins find the Teensy by enumerating serial ports and +probing the candidates: the modulation plugin opens each and keeps the one that +answers `HELLO` (the command port), the photodiode plugin listens on each and +keeps the one streaming CRC-clean PDA1 sample frames (the stream port). The +probe is what identifies the device; enumeration only decides what gets probed. + +That candidate filter was written on a Mac and hard-coded the two Unix name +patterns: + +```rust +name.contains("cu.usbmodem") || name.contains("ttyACM") +``` + +Windows names no device. Every serial port is `COMn`, so a correctly attached, +correctly driven Teensy matched neither pattern and was filtered out *before* +any probe could run. Both plugins then reported + +``` +no USB serial device found (looked for usbmodem/ttyACM) +``` + +which names the two things the machine cannot produce, so it reads as "nothing +is attached" when the device is in fact attached and enumerated. The first +Windows bundles from CI (ADR 030) made this reachable for the first time. + +The filter existed in four places — `serial_ports()` and `port_variants()` in +each plugin — and had drifted into two implementations: modulation went through +`stage-a-io`, photodiode called `serialport` directly with `stage-a-io`'s +`hardware` feature switched off. + +## Decision + +**One platform-aware candidate filter, in `stage-a-io::transport`.** + +`available_ports()` returns `PortInfo { name, label, is_usb }` — the OS path, +the USB manufacturer/product label where the OS reports one, and whether the OS +classified the port as USB at all. `candidate_ports()` narrows that list: + +- **macOS** — `cu.usbmodem*`. Every device is listed twice (`tty.*` and `cu.*`) + and only the callout node may be opened, so the name filter is also a dedupe. +- **Linux** — `ttyACM*`, the CDC-ACM class node a Teensy enumerates as. +- **Windows** — every USB-classified port. The name carries no device + information, so USB-ness is the only signal available. If the OS classified + *no* port as USB, the whole list is probed rather than none: a missing + SetupAPI classification must not be able to hide the device the way the name + filter did. + +Both plugins call `candidate_ports()` for probing and `PortInfo::variant()` for +the settings picker, so the probed set and the listed set cannot disagree. +The photodiode crate enables `stage-a-io`'s `hardware` feature to reach it; +`serialport` was already a direct dependency there, so nothing new enters the +build. + +**The filter is a probe-cost optimisation, not the identity check.** It exists +to keep the probes off unrelated ports — notably Windows' phantom Bluetooth +`COM` entries, which can block on open. Being too permissive costs a few +hundred milliseconds of probing; being too strict makes the hardware +unreachable. When in doubt, probe. + +`no_candidate_ports_message()` replaces the fixed string with what the OS +actually enumerated, distinguishing "no serial ports found" from "no serial +port looked like a Teensy (the OS offered COM1 (Bluetooth), …)". + +The platform branch is a `windows: bool` parameter to a private +`narrow_to_candidates`, not a `#[cfg]`, so the unit tests cover both branches +from any build host — including the Windows regression that motivated this ADR. + +**Every port is opened with `dtr_on_open(true)`.** macOS and Linux assert DTR +when a tty is opened; Windows does not — `serialport` sets +`DTR_CONTROL_DISABLE` in the DCB. A Teensyduino sketch that gates its output on +`if (Serial)` (which is `usb_configuration && usb_cdc_line_rtsdtr`) therefore +stays silent on Windows even once the right port is found, and the probes would +report "no port streamed PDA1 sample frames" on a working device. Asserting DTR +on all three platforms makes the port behave the same everywhere; on macOS and +Linux it is a no-op. + +## Consequences + +- The Stage-A plugins connect on Windows: the ports are found, and the opened + port has DTR asserted the way the Unix platforms already did implicitly. +- A port picker entry and a probe candidate come from one function, so a port + that appears in the dropdown is one `auto` would also have found. +- The failure message names the enumerated ports, which is the difference + between "check the cable" and "the filter dropped my device". +- `available_port_names()` and `available_ports_with_labels()` are replaced by + `available_ports()`. Both were internal to this repository. +- Windows probes any non-USB port when the OS classifies nothing at all, which + can add probe latency on a machine with legacy `COM` hardware. Accepted: an + unreachable device is worse than a slow scan. + +## Alternatives considered + +**Match the Teensy by USB VID/PID (0x16C0).** The most precise filter, and it +would work identically on all three platforms. Rejected for now: it hard-codes +the vendor of one board revision into the discovery path, and the probes +already establish identity positively — a VID match that skipped probing would +still have to tell the two ports of the dual-serial device apart. + +**Probe every enumerated port on every platform.** Simplest possible rule, and +correct. Rejected: on macOS it would open the `tty.*` twin of each device, +which blocks waiting for carrier detect, and on Windows it would sit on +phantom Bluetooth ports. + +**Keep the filter in the plugins and add a Windows arm to each.** Rejected: it +was already four copies in two implementations, and the copy that broke was +the one that had drifted. diff --git a/docs/features/README.md b/docs/features/README.md index 845eac3..8fbad49 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -5,10 +5,10 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. -- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. The coupled `ū`/`a` controls **clamp into the achievable range instead of refusing**, so a leftover depth can no longer make an optical mode unselectable, and both live bounds are shown in the control labels (ADR 025). `V_peak` is the one operator-facing name for the lobe maximum; the half-wave span is derived and never entered. The undocumented TOML `MOD`-step protocol runner was removed — declarative recording protocols belong to A1 (ADR 027). +- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. The coupled `ū`/`a` controls **clamp into the achievable range instead of refusing**, so a leftover depth can no longer make an optical mode unselectable, and both live bounds are shown in the control labels (ADR 025). `V_peak` is the one operator-facing name for the lobe maximum; the half-wave span is derived and never entered. The undocumented TOML `MOD`-step protocol runner was removed — declarative recording protocols belong to A1 (ADR 027). Port discovery is platform-aware and shared with the photodiode plugin, so `auto` finds the Teensy on Windows' nameless `COMn` ports too (ADR 032). - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC from measured `V_null`/`V_peak` endpoints, with target-specific headroom, Bessel-normalized cycle mean `ū`, and an explicit separation from physical flux `I_k`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. Each point is a 20 ms measurement after a 0.1 s settle, and every verdict on the sweep — lobe resolved, cell drifting — is made against the fit's own residual rather than against zero (ADR 019). Applying the fit now actually reaches the panel: the measurement lives on the live worker while the settings snapshot is collected from the UI mirror, so the applied lobe used to be overwritten within one frame (ADR 026). -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. Port discovery is platform-aware and shared with the modulation plugin (ADR 032). - [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). Every leased run heartbeats its modulation and photodiode leases against the deadline the owner actually granted, so a recording longer than the owner's TTL cap no longer loses the drive — and with it the phase-0 trigger and the photodiode's optical summary — in the middle of a point (ADR 029). - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index d605647..a6c8daf 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -60,6 +60,17 @@ Every accepted setting change is transferred to the Teensy **immediately** as on no Apply button, no experiment state machine. The panel shows the modulation and live DAC code the board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded values. +## Port discovery + +`auto` opens every candidate port and keeps the one that answers `HELLO` — the probe, not the port +name, tells the command port from the photodiode stream port of the same dual-serial device. Which +ports are candidates is platform-specific and shared with the photodiode plugin through +`stage-a-io::transport::candidate_ports()`: `cu.usbmodem*` on macOS (the callout node only, since +every device is listed twice), `ttyACM*` on Linux, and every USB-classified `COMn` on Windows, +where the name carries no device information at all (ADR 032). The settings picker lists exactly +the same set with each port's USB label. When nothing qualifies, the error names the ports the OS +did enumerate. + ## Contract - Owns the Teensy **command port** exclusively (one owner per port, ADR 006). The photodiode diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 14eca52..70d51a5 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -137,11 +137,22 @@ but a chart setting. always-false state to the worker, so it could never stay recording. The `record` boolean setting remains as a non-schema compatibility alias. +## Port discovery + +`auto` listens briefly on every candidate port and keeps the one streaming CRC-clean PDA1 sample +frames — the probe, not the port name, is what identifies the stream port. Which ports are +candidates is platform-specific and shared with the modulation plugin through +`stage-a-io::transport::candidate_ports()`: `cu.usbmodem*` on macOS (the callout node only, since +every device is listed twice), `ttyACM*` on Linux, and every USB-classified `COMn` on Windows, +where the name carries no device information at all (ADR 032). The settings picker lists exactly +the same set, so a port offered in the dropdown is one `auto` would also have probed. When nothing +qualifies, the error names the ports the OS did enumerate. + ## Contract - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the - plugin is read-only by construction. It reuses `stage-a-io` (`default-features = false`) only - for the PDA1 wire parser — no client, worker, or transport. + plugin is read-only by construction. It uses `stage-a-io` for the PDA1 wire parser and for port + discovery — no client, worker, or transport. - **Frame-independent**: connecting is a checkbox setting; the reader thread and all views work with no camera attached (the host only calls `process_frame()` while frames flow). - Garbage on the port resynchronises at the next CRC-clean frame; skipped bytes and CRC failures diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index a3ae241..6bac4a9 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -102,10 +102,14 @@ inversion they used. Full detail: [feature brief](../../docs/features/stage-a-po ## Ports -**Use `auto` (default recommendation):** it probes every attached usbmodem/ttyACM device and -connects to the one that answers `HELLO` — that is always the Teensy command port, never the -photodiode stream port. Explicit ports remain selectable; `mock` runs an in-process simulated -controller for hardware-free testing. +**Use `auto` (default recommendation):** it probes every attached USB serial port and connects to +the one that answers `HELLO` — that is always the Teensy command port, never the photodiode +stream port. Explicit ports remain selectable; `mock` runs an in-process simulated controller for +hardware-free testing. + +Which ports get probed is platform-specific: `cu.usbmodem*` on macOS, `ttyACM*` on Linux, and +every USB-classified `COMn` on Windows (ADR 032). The picker lists the same set with each port's +USB label, so the Teensy is recognisable by name. Replaying a recording disconnects the plugin defensively; live control itself needs no capture session. diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 8dbf1b7..634a762 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -2680,7 +2680,7 @@ fn open_serial(port_hint: &str) -> Result Result Vec { - stage_a_io::transport::available_port_names() + stage_a_io::transport::candidate_ports() .into_iter() - // macOS lists each device twice; use the callout (cu.*) node only. - .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) + .map(|port| port.name) .collect() } @@ -2829,15 +2828,11 @@ fn serial_ports() -> Vec { /// only the leading path is the value. fn port_variants() -> Vec { let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; - for (name, label) in stage_a_io::transport::available_ports_with_labels() { - if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { - continue; - } - variants.push(match label { - Some(label) => format!("{name} ({label})"), - None => name, - }); - } + variants.extend( + stage_a_io::transport::candidate_ports() + .iter() + .map(stage_a_io::transport::PortInfo::variant), + ); variants } @@ -3090,7 +3085,7 @@ impl Plugin for StageAModulationPlugin { key: "port".into(), label: "Port".into(), tooltip: Some( - "auto (recommended) probes the attached usbmodem ports and picks \ + "auto (recommended) probes the attached USB serial ports and picks \ the one that answers HELLO — the Teensy command port; \ mock = in-process simulated controller" .into(), diff --git a/plugins/stage-a-photodiode/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml index 3e0c8c0..5042600 100644 --- a/plugins/stage-a-photodiode/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -13,5 +13,7 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true serialport.workspace = true -stage-a-io = { path = "../../stage-a-io", default-features = false } +# `hardware` brings in the shared platform-aware port discovery; serialport is +# already a direct dependency here, so it adds nothing new to the build. +stage-a-io = { path = "../../stage-a-io" } stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md index 2406154..da03995 100644 --- a/plugins/stage-a-photodiode/README.md +++ b/plugins/stage-a-photodiode/README.md @@ -24,9 +24,12 @@ read-only by construction; the command port belongs to `stage-a-modulation`. ## Ports -**Use `auto` (default recommendation):** it listens briefly on every attached usbmodem/ttyACM -device and connects to the one actually streaming CRC-clean PDA1 sample frames — that is always -the Teensy stream port. `mock` generates a synthetic sine for hardware-free testing. +**Use `auto` (default recommendation):** it listens briefly on every attached USB serial port and +connects to the one actually streaming CRC-clean PDA1 sample frames — that is always the Teensy +stream port. `mock` generates a synthetic sine for hardware-free testing. + +Which ports get listened to is platform-specific: `cu.usbmodem*` on macOS, `ttyACM*` on Linux, +and every USB-classified `COMn` on Windows (ADR 032). ## Owner control service diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 4d4cf3e..4f7ca91 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -556,6 +556,8 @@ impl Reader { ) -> Result { let port = serialport::new(&path, 115_200) .timeout(Duration::from_millis(50)) + // Windows opens with DTR deasserted; see ADR 032. + .dtr_on_open(true) .open() .map_err(|err| format!("open {path}: {err}"))?; let stop = Arc::new(AtomicBool::new(false)); @@ -2527,16 +2529,10 @@ fn rejected_service_reply( } fn serial_ports() -> Vec { - serialport::available_ports() - .map(|ports| { - ports - .into_iter() - .map(|p| p.port_name) - // macOS lists each device twice; use the callout (cu.*) node only. - .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) - .collect() - }) - .unwrap_or_default() + stage_a_io::transport::candidate_ports() + .into_iter() + .map(|port| port.name) + .collect() } /// Finds the Teensy stream port: the dual-serial firmware free-runs PDA1 @@ -2545,7 +2541,7 @@ fn serial_ports() -> Vec { fn resolve_auto_port() -> Result { let candidates = serial_ports(); if candidates.is_empty() { - return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + return Err(stage_a_io::transport::no_candidate_ports_message()); } let mut saw_legacy_ascii = false; for path in &candidates { @@ -2589,6 +2585,8 @@ enum ProbeResult { fn probe_pd_stream(path: &str) -> ProbeResult { let Ok(mut port) = serialport::new(path, 115_200) .timeout(Duration::from_millis(100)) + // Windows opens with DTR deasserted; see ADR 032. + .dtr_on_open(true) .open() else { return ProbeResult::Nothing; @@ -2632,26 +2630,11 @@ fn probe_pd_stream(path: &str) -> ProbeResult { /// host exchanges enum settings as indices into this list. fn port_variants() -> Vec { let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; - for port in serialport::available_ports().unwrap_or_default() { - if !(port.port_name.contains("cu.usbmodem") || port.port_name.contains("ttyACM")) { - continue; - } - let label = match port.port_type { - serialport::SerialPortType::UsbPort(info) => match (info.manufacturer, info.product) { - (Some(manufacturer), Some(product)) if !product.starts_with(&manufacturer) => { - Some(format!("{manufacturer} {product}")) - } - (_, Some(product)) => Some(product), - (Some(manufacturer), None) => Some(manufacturer), - (None, None) => None, - }, - _ => None, - }; - variants.push(match label { - Some(label) => format!("{} ({label})", port.port_name), - None => port.port_name, - }); - } + variants.extend( + stage_a_io::transport::candidate_ports() + .iter() + .map(stage_a_io::transport::PortInfo::variant), + ); variants } diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index e9c9377..5e1b820 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -28,6 +28,10 @@ impl SerialTransport { pub fn open(path: &str, baud: u32, poll_timeout: Duration) -> io::Result { let port = serialport::new(path, baud) .timeout(poll_timeout) + // Windows opens a port with DTR deasserted; macOS and Linux assert + // it for us. A Teensy sketch that gates on `if (Serial)` would stay + // silent there, so assert it everywhere (ADR 032). + .dtr_on_open(true) .open() .map_err(|err| io::Error::other(format!("opening {path} failed: {err}")))?; Ok(Self { port }) @@ -113,33 +117,42 @@ impl Transport for MockTransport { } } -/// Names of serial ports visible to the OS (empty without the `hardware` -/// feature). Used by plugins to offer a port picker. -#[cfg(feature = "hardware")] -pub fn available_port_names() -> Vec { - serialport::available_ports() - .map(|ports| ports.into_iter().map(|p| p.port_name).collect()) - .unwrap_or_default() +/// One serial port as the OS enumerated it. +/// +/// `label` is the human-readable USB manufacturer/product where the OS +/// reports one — e.g. `"Teensyduino Dual Serial"` — so port pickers can show +/// which entry is the Teensy. `is_usb` records whether the OS classified the +/// port as a USB device at all, which is the only device hint Windows gives. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PortInfo { + pub name: String, + pub label: Option, + pub is_usb: bool, } -#[cfg(not(feature = "hardware"))] -pub fn available_port_names() -> Vec { - Vec::new() +impl PortInfo { + /// The port as a picker entry: the path, plus the USB label in parentheses + /// where there is one. Only the leading path is the value — see + /// `variant_path` in the Stage-A plugins. + pub fn variant(&self) -> String { + match &self.label { + Some(label) => format!("{} ({label})", self.name), + None => self.name.clone(), + } + } } -/// Port names plus a human-readable USB label (manufacturer/product) where -/// the OS provides one — e.g. `("/dev/cu.usbmodem…", Some("Teensyduino Dual -/// Serial"))`. Lets port pickers show which entry is the Teensy. +/// Every serial port visible to the OS (empty without the `hardware` feature). #[cfg(feature = "hardware")] -pub fn available_ports_with_labels() -> Vec<(String, Option)> { +pub fn available_ports() -> Vec { serialport::available_ports() .map(|ports| { ports .into_iter() .map(|p| { - let label = match p.port_type { + let (label, is_usb) = match p.port_type { serialport::SerialPortType::UsbPort(info) => { - match (info.manufacturer, info.product) { + let label = match (info.manufacturer, info.product) { (Some(manufacturer), Some(product)) if !product.starts_with(&manufacturer) => { @@ -148,11 +161,16 @@ pub fn available_ports_with_labels() -> Vec<(String, Option)> { (_, Some(product)) => Some(product), (Some(manufacturer), None) => Some(manufacturer), (None, None) => None, - } + }; + (label, true) } - _ => None, + _ => (None, false), }; - (p.port_name, label) + PortInfo { + name: p.port_name, + label, + is_usb, + } }) .collect() }) @@ -160,6 +178,117 @@ pub fn available_ports_with_labels() -> Vec<(String, Option)> { } #[cfg(not(feature = "hardware"))] -pub fn available_ports_with_labels() -> Vec<(String, Option)> { +pub fn available_ports() -> Vec { Vec::new() } + +/// The enumerated ports worth probing for a Teensy. +/// +/// Unix names the device, so the name is the filter: macOS lists every device +/// twice (`tty.*` and `cu.*`) and only the callout node may be opened, and a +/// Teensy's CDC-ACM class node on Linux is `ttyACM*`. Windows names nothing — +/// every port is `COMn` — so USB-ness is the only signal there, and if the OS +/// classified no port at all the whole list is probed rather than none. What +/// actually identifies the Teensy is the probe (HELLO on the command port, +/// PDA1 sample frames on the stream port); this only keeps the probe from +/// stalling on unrelated ports such as Windows' phantom Bluetooth COM entries. +pub fn candidate_ports() -> Vec { + narrow_to_candidates(available_ports(), cfg!(windows)) +} + +fn narrow_to_candidates(ports: Vec, windows: bool) -> Vec { + if !windows { + return ports + .into_iter() + .filter(|port| port.name.contains("cu.usbmodem") || port.name.contains("ttyACM")) + .collect(); + } + if ports.iter().any(|port| port.is_usb) { + return ports.into_iter().filter(|port| port.is_usb).collect(); + } + ports +} + +/// Why there was nothing to probe, naming what the OS did enumerate — the +/// difference between "no device is attached" and "a device is attached but +/// this platform's filter dropped it" is the operator's next step. +pub fn no_candidate_ports_message() -> String { + let ports = available_ports(); + if ports.is_empty() { + return "no serial ports found — check the USB cable and that the Teensy is powered" + .to_owned(); + } + let seen = ports + .iter() + .map(PortInfo::variant) + .collect::>() + .join(", "); + format!("no serial port looked like a Teensy (the OS offered {seen})") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn port(name: &str, is_usb: bool) -> PortInfo { + PortInfo { + name: name.to_owned(), + label: None, + is_usb, + } + } + + fn names(ports: Vec) -> Vec { + ports.into_iter().map(|port| port.name).collect() + } + + #[test] + fn unix_keeps_the_callout_node_and_drops_its_tty_twin() { + let ports = vec![ + port("/dev/tty.usbmodem12345", true), + port("/dev/cu.usbmodem12345", true), + port("/dev/cu.Bluetooth-Incoming-Port", false), + ]; + assert_eq!( + names(narrow_to_candidates(ports, false)), + vec!["/dev/cu.usbmodem12345"] + ); + } + + #[test] + fn unix_keeps_the_linux_cdc_acm_node() { + let ports = vec![port("/dev/ttyACM0", true), port("/dev/ttyS0", false)]; + assert_eq!(names(narrow_to_candidates(ports, false)), vec!["/dev/ttyACM0"]); + } + + #[test] + fn windows_com_ports_survive_the_unix_name_filter() { + // The bug: COMn matches neither `cu.usbmodem` nor `ttyACM`, so the + // Teensy's two ports were filtered out before any probe could run. + let ports = vec![port("COM3", true), port("COM4", true)]; + assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM3", "COM4"]); + } + + #[test] + fn windows_drops_non_usb_ports_when_a_usb_port_exists() { + let ports = vec![port("COM1", false), port("COM7", true)]; + assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM7"]); + } + + #[test] + fn windows_probes_everything_when_the_os_classifies_nothing() { + let ports = vec![port("COM1", false), port("COM3", false)]; + assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM1", "COM3"]); + } + + #[test] + fn a_labelled_port_shows_its_usb_name_in_the_picker() { + let labelled = PortInfo { + name: "COM3".to_owned(), + label: Some("Teensyduino Dual Serial".to_owned()), + is_usb: true, + }; + assert_eq!(labelled.variant(), "COM3 (Teensyduino Dual Serial)"); + assert_eq!(port("COM4", true).variant(), "COM4"); + } +} From 0e07d88a6dd0284d1b488adf71be5a6c79ad462a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 5 Aug 2026 10:14:07 +0200 Subject: [PATCH 43/46] =?UTF-8?q?fix(stage-a-a1):=20=F0=9F=90=9B=20accept?= =?UTF-8?q?=20protocol=20files=20a=20spreadsheet=20saved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Excel's "CSV UTF-8" — the obvious save format on a Windows bench — writes a UTF-8 byte-order mark. Unstripped it becomes part of the first header cell, so `mean_u` stops matching `mean_u` and the protocol is refused for missing a required column that is plainly there. The TOML form fails its parse outright. Neither message points at an invisible character. Strip the BOM once for both readers. CRLF was already handled by `str::lines()`; it now has a test so it stays that way. Refs ADR 027 --- docs/features/stage-a-a1.md | 6 ++++- plugins/stage-a-a1/README.md | 3 +++ plugins/stage-a-a1/src/protocol.rs | 43 +++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 8cf4b67..fe7d18c 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -207,7 +207,11 @@ swept at all, and what a block recorded lived in the panel rather than in anything that travels with the results. A **protocol** is a file naming every axis for every recording. The reader is -chosen by extension, and both produce the same flat list of points. +chosen by extension, and both produce the same flat list of points. Both +tolerate what a spreadsheet writes: CRLF line endings, and the UTF-8 +byte-order mark Excel's "CSV UTF-8" prepends — unstripped, the BOM becomes +part of the first header cell and the file is refused for missing a column it +visibly has. **CSV — one row per recording**, and the one to reach for: it opens in a spreadsheet, comes straight out of a script, and each row carries its own diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 193dbd9..048df5e 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -171,6 +171,9 @@ Columns are found **by name**, so their order does not matter and one can be lef Blank lines and `#` comments are skipped, and a blank cell falls back to the default. Errors carry the **file line number**, which is what your editor and spreadsheet both show. +Files saved by a spreadsheet load as-is: Windows line endings and the byte-order mark that Excel's +"CSV UTF-8" writes are both absorbed, so the first column is not silently reported missing. + Two things the row form gives you that blocks cannot without one block per value: **a different duration per row** (1 Hz needs 40 s of cycles, 200 Hz does not), and **a `role` column**, so a file can open with its own background floor and pilot and then record the points scored against them — diff --git a/plugins/stage-a-a1/src/protocol.rs b/plugins/stage-a-a1/src/protocol.rs index 9434726..2851eba 100644 --- a/plugins/stage-a-a1/src/protocol.rs +++ b/plugins/stage-a-a1/src/protocol.rs @@ -298,10 +298,21 @@ impl Axis { } } +/// Drops a leading UTF-8 byte-order mark. +/// +/// Saving a protocol as "CSV UTF-8" in Excel — the obvious choice on a Windows +/// bench — writes a BOM. Left in place it becomes part of the first header +/// cell, so `mean_u` stops matching `mean_u` and the file is refused for +/// missing a column that is plainly there; in the TOML form it fails the parse +/// outright. Neither message would point at an invisible character. +fn strip_bom(text: &str) -> &str { + text.strip_prefix('\u{feff}').unwrap_or(text) +} + /// Parses a protocol and expands it into the points to record. pub fn parse(text: &str) -> Result { - let doc: ProtocolDoc = - toml::from_str(text).map_err(|error| ProtocolError::Toml(error.to_string()))?; + let doc: ProtocolDoc = toml::from_str(strip_bom(text)) + .map_err(|error| ProtocolError::Toml(error.to_string()))?; let default_duration = doc.defaults.duration_s.unwrap_or(10); let default_settle = doc.defaults.settle_s.unwrap_or(2.0); @@ -444,7 +455,8 @@ pub fn parse_csv(text: &str) -> Result { let mut header: Option> = None; let mut points = Vec::new(); - for (offset, raw) in text.lines().enumerate() { + // `lines()` already absorbs CRLF; the BOM is the part it leaves behind. + for (offset, raw) in strip_bom(text).lines().enumerate() { let line_no = offset + 1; let line = raw.trim(); if line.is_empty() || line.starts_with('#') { @@ -599,6 +611,31 @@ depth_a = 0.8 duration_s = 30 "#; + #[test] + fn a_spreadsheet_bom_does_not_hide_the_first_column() { + // Excel's "CSV UTF-8" writes a BOM. Without stripping it, `mean_u` + // reads as `\u{feff}mean_u` and the file is refused for missing the + // column it visibly has. + let csv = "\u{feff}mean_u,frequency_hz,depth_a\n0.5,10,1.0\n"; + let protocol = parse_file("survey.csv", csv).expect("BOM-prefixed CSV"); + assert_eq!(protocol.points.len(), 1); + assert_eq!(protocol.points[0].mean_u, 0.5); + } + + #[test] + fn a_bom_does_not_break_the_toml_form_either() { + let protocol = parse(&format!("\u{feff}{SAMPLE}")).expect("BOM-prefixed TOML"); + assert_eq!(protocol.name, "sample"); + } + + #[test] + fn a_spreadsheet_crlf_file_parses() { + let csv = "mean_u,frequency_hz,depth_a\r\n0.5,10,1.0\r\n"; + let protocol = parse_file("survey.csv", csv).expect("CRLF CSV"); + assert_eq!(protocol.points.len(), 1); + assert_eq!(protocol.points[0].frequency_hz, 10.0); + } + #[test] fn a_protocol_expands_to_the_product_of_its_axes() { let protocol = parse(SAMPLE).expect("valid protocol"); From eacf2682cdd8709f35ae593763cf00e1e2319fdb Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 5 Aug 2026 14:27:55 +0200 Subject: [PATCH 44/46] =?UTF-8?q?style(stage-a):=20=F0=9F=8E=A8=20apply=20?= =?UTF-8?q?rustfmt=20to=20the=20port-discovery=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Formatting drift left by the two previous commits, split out so the change that follows is only the change it claims to be. Co-Authored-By: Claude Opus 5 --- plugins/stage-a-a1/src/protocol.rs | 4 ++-- stage-a-io/src/transport.rs | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/plugins/stage-a-a1/src/protocol.rs b/plugins/stage-a-a1/src/protocol.rs index 2851eba..f93e443 100644 --- a/plugins/stage-a-a1/src/protocol.rs +++ b/plugins/stage-a-a1/src/protocol.rs @@ -311,8 +311,8 @@ fn strip_bom(text: &str) -> &str { /// Parses a protocol and expands it into the points to record. pub fn parse(text: &str) -> Result { - let doc: ProtocolDoc = toml::from_str(strip_bom(text)) - .map_err(|error| ProtocolError::Toml(error.to_string()))?; + let doc: ProtocolDoc = + toml::from_str(strip_bom(text)).map_err(|error| ProtocolError::Toml(error.to_string()))?; let default_duration = doc.defaults.duration_s.unwrap_or(10); let default_settle = doc.defaults.settle_s.unwrap_or(2.0); diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index 5e1b820..e21832b 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -258,7 +258,10 @@ mod tests { #[test] fn unix_keeps_the_linux_cdc_acm_node() { let ports = vec![port("/dev/ttyACM0", true), port("/dev/ttyS0", false)]; - assert_eq!(names(narrow_to_candidates(ports, false)), vec!["/dev/ttyACM0"]); + assert_eq!( + names(narrow_to_candidates(ports, false)), + vec!["/dev/ttyACM0"] + ); } #[test] @@ -266,7 +269,10 @@ mod tests { // The bug: COMn matches neither `cu.usbmodem` nor `ttyACM`, so the // Teensy's two ports were filtered out before any probe could run. let ports = vec![port("COM3", true), port("COM4", true)]; - assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM3", "COM4"]); + assert_eq!( + names(narrow_to_candidates(ports, true)), + vec!["COM3", "COM4"] + ); } #[test] @@ -278,7 +284,10 @@ mod tests { #[test] fn windows_probes_everything_when_the_os_classifies_nothing() { let ports = vec![port("COM1", false), port("COM3", false)]; - assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM1", "COM3"]); + assert_eq!( + names(narrow_to_candidates(ports, true)), + vec!["COM1", "COM3"] + ); } #[test] From 57e75b37fdbf9715f47837e32eb48526486a84e7 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 5 Aug 2026 14:28:12 +0200 Subject: [PATCH 45/46] =?UTF-8?q?fix(stage-a-a1):=20=F0=9F=90=9B=20make=20?= =?UTF-8?q?every=20protocol=20row=20name=20itself?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A protocol run came off the bench as a folder of files that could not be told apart: same duration on every row, stems differing only in the second the row started. Mapping a .pdq back to the modulation it was recorded under meant sorting by timestamp and counting rows against the protocol file — a join that is only correct if no row was skipped, and a refused point is deliberately non-fatal so an overnight run finishes. A protocol is the one recording path where nothing in the panel is armed at anything that separates its recordings, and three places relied on exactly that: - The file stem took the empty `sweep_tag` branch, which is filled only by the amplitude sweep and the event-count workflow. `ProtocolPoint::tag()` already produced `u500m_f10Hz_a800m` and had a test, but nothing ever called it. It is now appended after the timestamp, behind a 1-based point index whose width follows the point count, so the files sort in protocol order rather than by u-bar. - The A1 sidecar had a section for every other automatic path and none for this one, so a row's `[sweep]` carried only the panel's min_a/max_a and the file did not record what the row had asked for on any axis. It gains `[protocol]`. - The recorder metadata described the drive as a frequency and two DAC codes, which name neither the lobe nor the operating point. u-bar is a whole axis of the survey and appeared nowhere. It now carries the waveform, the optical target, requested/resolved u-bar, the commanded depth, V_null/V_peak and the Pockels calibration id — all of it already published by the modulation owner. The metadata matters most, because the A1 sidecar is not guaranteed to exist: `write_sidecar` refuses without a fresh optical summary, which is what an expired lease or a drive switched off mid-row looks like after the fact. Such a run keeps only its .pdq and _pd.json, so the _pd.json has to stand alone. Free text out of the protocol file is clamped before it enters the map — the photodiode owner refuses the whole recording over an oversized value, and a chatty block name must not cost a run. What is written is the request; `[modulation]` says what the drive reported back and `[optical]` what the photodiode measured, so a row that missed its point is still distinguishable from one that hit it. Refs ADR 033. Co-Authored-By: Claude Opus 5 --- docs/adr/033-a-protocol-row-names-itself.md | 108 ++++++ docs/features/README.md | 2 +- docs/features/stage-a-a1.md | 47 ++- plugins/stage-a-a1/src/runtime.rs | 390 +++++++++++++++++++- 4 files changed, 537 insertions(+), 10 deletions(-) create mode 100644 docs/adr/033-a-protocol-row-names-itself.md diff --git a/docs/adr/033-a-protocol-row-names-itself.md b/docs/adr/033-a-protocol-row-names-itself.md new file mode 100644 index 0000000..312a8b0 --- /dev/null +++ b/docs/adr/033-a-protocol-row-names-itself.md @@ -0,0 +1,108 @@ +# ADR 033 — A protocol row names itself, in its file names and in both sidecars + +**Status:** accepted +**Date:** 2026-08-05 +**Feature briefs:** [Stage-A A1 Analysis](../features/stage-a-a1.md) +**Relates to:** [ADR 027](027-stage-a-a1-declarative-protocols.md) (declarative +protocols), [ADR 015](015-stage-a-a1-recording-robustness.md) (one measurement +folder), [ADR 029](029-stage-a-leases-are-renewed-against-the-granted-deadline.md) +(the sidecar refusal an expired lease produces) + +## Context + +A protocol run came off the bench as a folder of files that could not be told +apart. Every row had the same duration, and the stems differed only in the +second the row started: + +``` +A1-survey_20260805-141201_pd.pdq +A1-survey_20260805-141233_pd.pdq +A1-survey_20260805-141305_pd.pdq +``` + +Mapping a `.pdq` back to the modulation it was recorded under meant sorting by +timestamp and counting rows in the protocol file — a join that is only correct +if no row was skipped, and every row of a survey may be skipped (ADR 027 keeps +a refused point non-fatal, precisely so an overnight run finishes). + +Three things were missing, and each was missing for the same reason: **a +protocol is the one recording path where nothing in the panel is armed at +anything that separates its recordings.** + +- **The file stem had no row tag.** `begin_recording` builds + `_[_role]`, and `sweep_tag` is non-empty only while + an amplitude sweep or the event-count workflow is in its recording phase. A + protocol drives the axes itself and hands off to the same coordinator, so it + took the empty branch. `ProtocolPoint::tag()` — `u500m_f10Hz_a800m` — already + existed and had a test, but nothing ever called it. +- **The A1 sidecar had no `[protocol]` section.** It had one for every other + automatic path (`[sweep]`, `[a0_lock]`, `[frequency_sweep]`). A protocol row + filled in `[sweep]` with the panel's `min_a`/`max_a` and nothing else, so the + file did not record what the row had asked for on any of the three axes. +- **The recorder metadata described the drive only partly.** It carried + `modulation_frequency_hz`, `center_dac` and `amplitude_dac` — a frequency and + two codes, which do not name the lobe or the operating point. `ū` in + particular is a whole axis of the survey and appeared nowhere: two rows can + share `f` and `a` and differ only in the mean illumination they were driven + around. + +The last one matters more than it looks, because the A1 `_config.toml` is not +guaranteed to exist. `write_sidecar` refuses outright without a fresh photodiode +optical summary — deliberately, since it is the quantitative record — and that +refusal is exactly what an expired lease or a drive switched off mid-row looks +like after the fact (ADR 029). A run that hits it keeps its `.pdq` and its +`_pd.json` and loses everything else, so the `_pd.json` has to stand alone. + +## Decision + +**A protocol row is identified on every leg, by the row itself.** + +1. **The file stem carries the point tag.** After the timestamp and role: + `_p_u<ū>m_fHz_am`, e.g. + + ``` + A1-survey_20260805-141233_p03_u500m_f10Hz_a800m_pd.pdq + ``` + + The 1-based index over the whole expanded protocol comes **first**, so the + files sort in protocol order rather than by `ū`, and its width follows the + point count (`_p03` for 12 rows, `_p003` for 400). It is emitted only while + the run is in `ProtocolPhase::Recording`, so a hand-driven recording is + unaffected. + +2. **The A1 sidecar gains a `[protocol]` section**: `name`, `block`, + `point_index`/`point_total`, `point_tag`, `requested_mean_u`, + `requested_frequency_hz`, `requested_depth_a`, `settle_s`, `duration_s`. + +3. **The recorder metadata — which reaches both the `_pd.json` and the camera + leg — carries the same identity** (`protocol_*` keys) **and a full + description of the drive**: `modulation_waveform`, `pockels_calibration_id`, + `optical_target`, `requested_mean_u`, `resolved_mean_u`, `commanded_a`, + `v_null_dac`, `v_peak_dac`. + +**What is written is the request, not the result.** `[protocol]` and the +`protocol_*` keys say what the row asked for; `[modulation]` says what the drive +reported back, and `[optical]` what the photodiode measured. Collapsing them +would make a row that missed its point indistinguishable from one that hit it, +which is the failure the survey exists to detect. + +**Free text from the protocol file is clamped** to 120 characters before it +enters the metadata map. The photodiode owner bounds values at 1 KiB and refuses +the whole `BeginRecording` if one is over — a chatty block name must not be able +to cost a recording. + +## Consequences + +- A `.pdq` names its operating point without any join at all, and survives the + loss of the A1 sidecar with its provenance intact. +- Rows sort in protocol order in a file listing, so a skipped row is visible as + a gap in the indices rather than as a missing timestamp nobody can place. +- Stems are longer. That is the trade: a name that says what the file is beats a + short one that says when it was written. +- Two rows at the same `(ū, f, a)` — a deliberate repeat — still share a tag and + are separated by the point index and the timestamp. +- Nothing changes for hand-driven recordings, sweeps, ladders or event-count + points; their existing tags and sidecar sections are untouched. +- The refusal to write a quantitative sidecar without a measured optical summary + is **kept**. This ADR reduces what is lost when it fires; it does not paper + over the upstream fault that causes it. diff --git a/docs/features/README.md b/docs/features/README.md index 8fbad49..ba82b15 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -9,7 +9,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC from measured `V_null`/`V_peak` endpoints, with target-specific headroom, Bessel-normalized cycle mean `ū`, and an explicit separation from physical flux `I_k`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. Each point is a 20 ms measurement after a 0.1 s settle, and every verdict on the sweep — lobe resolved, cell drifting — is made against the fit's own residual rather than against zero (ADR 019). Applying the fit now actually reaches the panel: the measurement lives on the live worker while the settings snapshot is collected from the UI mirror, so the applied lobe used to be overwritten within one frame (ADR 026). - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. Port discovery is platform-aware and shared with the modulation plugin (ADR 032). -- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). Every leased run heartbeats its modulation and photodiode leases against the deadline the owner actually granted, so a recording longer than the owner's TTL cap no longer loses the drive — and with it the phase-0 trigger and the photodiode's optical summary — in the middle of a point (ADR 029). +- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). Every leased run heartbeats its modulation and photodiode leases against the deadline the owner actually granted, so a recording longer than the owner's TTL cap no longer loses the drive — and with it the phase-0 trigger and the photodiode's optical summary — in the middle of a point (ADR 029). **Each protocol row names itself** — `_p03_u500m_f10Hz_a800m` in the file stem, a `[protocol]` section in the A1 sidecar, `protocol_*` keys in the recorder metadata — so a `.pdq` maps back to its modulation without counting timestamps against the protocol file, and the PDQ sidecar now also describes how the drive was *shaped* (waveform, `ū`, lobe, calibration id) rather than only how fast it ran (ADR 033). - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index fe7d18c..1a7a9d1 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -277,6 +277,22 @@ its three. wording. Because the per-point message is overwritten within the same tick, the reasons are kept on the run and shown in the status pane and the closing summary. +- **Every row names itself, in the file names and in both sidecars** + ([ADR 033](../adr/033-a-protocol-row-names-itself.md)). A protocol is the one + path where nothing in the panel is armed at anything that separates its + recordings, so the files carry the row's own identity: + + | where | what | + |---|---| + | file stem | `_p_u<ū>m_fHz_am` after the timestamp — e.g. `A1-survey_20260805-141233_p03_u500m_f10Hz_a800m` | + | A1 `_config.toml` | a `[protocol]` section: `name`, `block`, `point_index`/`point_total`, `point_tag`, `requested_mean_u`, `requested_frequency_hz`, `requested_depth_a`, `settle_s`, `duration_s` | + | `_pd.json` / camera metadata | `protocol_name`, `protocol_block`, `protocol_point_index`, `protocol_point_total`, `protocol_point_tag`, `protocol_mean_u`, `protocol_frequency_hz`, `protocol_depth_a`, `protocol_settle_s` | + + The index is 1-based over the whole expanded protocol and comes first, so the + files sort in protocol order rather than by `ū`. What the row *asked* for is + written here; what the drive reported back is in `[modulation]`, and what the + photodiode measured is in `[optical]`, so a row that missed its point can + still be told apart from one that hit it. One lease covers the whole file. `plugins/stage-a-a1/protocols/example.toml` is a commented file to copy. @@ -339,14 +355,24 @@ quantitative A1 sidecar without a fresh photodiode optical summary`, and a `Camera: … no trigger signal` line that looked exactly like an unplugged `EXT_TRIGGER` cable but was the drive being off. -**Naming.** Files share an `_[_role]` stem under an `/` subfolder -(`_pilot` / `_background` tag the reference runs, `_ec_fHz` an event-count point): +**Naming.** Files share an `_[_role][_point]` stem under an `/` +subfolder: - `/_.raw` — camera RAW, with the host's own `.toml` sidecar (camera biases, ROI) next to it. - `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar. - `/__config.toml` — the A1 sidecar. +The optional tags say which run within the measurement this is — without one, a +timestamp is the only thing separating two files: + +| tag | written by | +|---|---| +| `_pilot` / `_background` | the reference runs | +| `_p` | an amplitude-sweep point (prefixed `_fHz` when nested in the frequency ladder) | +| `_ec_fHz` | an event-count point | +| `_p_u<ū>m_fHz_am` | a protocol row (ADR 033) | + **Everything lands under `//`** (ADR 015). That folder is the only setting deciding where a measurement ends up — the host output root and the photodiode Data directory no longer have to be kept aligned by hand: @@ -444,6 +470,23 @@ press can start the next recording automatically. (+ the host's `.toml`), `_pd.pdq` + `_pd.json`, and `_config.toml`. +**The PDQ says how the drive was shaped, not only how fast it ran** (ADR 033). +The photodiode's file is the photodiode owner's own, and travels on its own — a +trace whose sidecar names a frequency and two DAC codes cannot say which lobe, +or which operating point `ū`, produced it. Alongside `modulation_frequency_hz`, +`center_dac`, `amplitude_dac`, `depth_a`/`depth_a_source` and `measured_a`, the +recorder metadata therefore also carries `modulation_waveform`, +`pockels_calibration_id`, `optical_target`, `requested_mean_u`, +`resolved_mean_u`, `commanded_a`, `v_null_dac` and `v_peak_dac` — all of it +already published by the modulation owner, and all of it also in the A1 +sidecar's `[modulation]`. + +> The A1 `_config.toml` is the quantitative record and is **refused outright** +> without a fresh photodiode optical summary, which is what an expired lease or +> a drive that was switched off looks like after the fact (ADR 029). The run +> then reports `Recording finished, metadata save failed: …` and only the +> `_pd.json` remains — which is the other reason it has to be self-describing. + ## The two live plots Both fold the camera event stream on `T` (from the firmware phase-0 `EXT_TRIGGER` diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 18e1ead..0e3b6c9 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -56,12 +56,12 @@ use augur_plugin_api::{ use serde::Serialize; use serde_json::{json, Value}; use stage_a_plugin_contract::{ - ClientId, ConnectionStateV1, LeaseId, LeaseSnapshotV1, ModulationCommandV1, ModulationRequestV1, - ModulationStateV1, OpticalTargetV1, PdqReceiptV1, PdqStartSpecV1, PhotodiodeCommandV1, - PhotodiodeOpticalSummaryV1, PhotodiodeRequestV1, PhotodiodeResponseV1, PhotodiodeSummaryV1, - RequestId, RunId, SemanticRevision, WaveformV1, CTX_STAGE_A_MODULATION_STATE_V1, - CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, SERVICE_STAGE_A_MODULATION_CONTROL_V1, - SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, + ClientId, ConnectionStateV1, LeaseId, LeaseSnapshotV1, ModulationCommandV1, + ModulationRequestV1, ModulationStateV1, OpticalTargetV1, PdqReceiptV1, PdqStartSpecV1, + PhotodiodeCommandV1, PhotodiodeOpticalSummaryV1, PhotodiodeRequestV1, PhotodiodeResponseV1, + PhotodiodeSummaryV1, RequestId, RunId, SemanticRevision, WaveformV1, + CTX_STAGE_A_MODULATION_STATE_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, + SERVICE_STAGE_A_MODULATION_CONTROL_V1, SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, }; use crate::phase::{fold_events, fold_events_free_running, MarkerValidationConfig, PhaseFold}; @@ -2109,6 +2109,80 @@ impl StageAA1Plugin { meta.insert("center_dac".into(), config.center_dac.to_string()); meta.insert("amplitude_dac".into(), config.amplitude_dac.to_string()); } + // How the drive was actually shaped, not just how fast it ran. The PDQ + // is the photodiode's own record and travels on its own — a trace whose + // sidecar names a frequency and two DAC codes cannot say which lobe, or + // which operating point `ū`, produced it. All of this is already + // published by the modulation owner; it was simply never written down + // here, so it lived only in the A1 `_config.toml` next door. + if let Some(waveform) = self + .modulation + .as_ref() + .and_then(|state| state.acknowledged.as_ref()) + .and_then(|target| target.waveform.as_ref()) + { + meta.insert("modulation_waveform".into(), waveform_label(waveform)); + } + if let Some(state) = self.modulation.as_ref() { + if let Some(calibration) = state.calibration_id.as_ref() { + meta.insert("pockels_calibration_id".into(), calibration.clone()); + } + if let Some(drive) = state.optical_drive.as_ref() { + meta.insert( + "optical_target".into(), + match drive.target { + OpticalTargetV1::LogSine => "log_sine".into(), + OpticalTargetV1::LinearSine => "linear_sine".into(), + }, + ); + // `ū` is a whole axis of the survey, and the one no other key + // here implies: two rows can share `f` and `a` and differ only + // in the operating point they were driven around. + meta.insert( + "requested_mean_u".into(), + format!("{:.6}", f64::from(drive.requested_mean_u_milli) / 1_000.0), + ); + meta.insert( + "resolved_mean_u".into(), + format!("{:.6}", f64::from(drive.resolved_mean_u_milli) / 1_000.0), + ); + meta.insert( + "commanded_a".into(), + format!("{:.6}", f64::from(drive.depth_a_milli) / 1_000.0), + ); + meta.insert("v_null_dac".into(), drive.v_null_dac.to_string()); + meta.insert("v_peak_dac".into(), drive.v_peak_dac.to_string()); + } + } + // Which row of which protocol this run is. Written on both legs so the + // PDQ and the RAW can each be traced back to the line of the file that + // asked for them, without joining through the A1 sidecar. + if let Some(run) = self + .protocol + .as_ref() + .filter(|run| run.phase == ProtocolPhase::Recording) + { + if let Some(point) = run.point() { + // Both are free text out of the operator's file. The photodiode + // owner rejects the whole recording over an oversized metadata + // value, so a chatty block name must not be able to cost a run. + meta.insert("protocol_name".into(), clamp_metadata(&run.plan.name)); + meta.insert("protocol_block".into(), clamp_metadata(&point.block)); + meta.insert("protocol_point_tag".into(), point.tag()); + meta.insert("protocol_point_index".into(), (run.index + 1).to_string()); + meta.insert( + "protocol_point_total".into(), + run.plan.points.len().to_string(), + ); + meta.insert("protocol_mean_u".into(), format!("{:.6}", point.mean_u)); + meta.insert( + "protocol_frequency_hz".into(), + format!("{:.6}", point.frequency_hz), + ); + meta.insert("protocol_depth_a".into(), format!("{:.6}", point.depth_a)); + meta.insert("protocol_settle_s".into(), format!("{:.3}", point.settle_s)); + } + } if let Some(n) = self.valid_pixel_count() { meta.insert("n_valid".into(), n.to_string()); } @@ -2249,8 +2323,30 @@ impl StageAA1Plugin { } }) .unwrap_or_default(); + // A protocol row is not a sweep point, so `sweep_tag` is empty for one: + // nothing armed in the panel says which of the survey's points this is. + // Without a tag of its own, every row of a protocol lands under the same + // stem but for the second it started, and the operating point a file was + // recorded at can only be recovered by opening its sidecar. The row + // names all three axes, so its files can too — the 1-based point index + // first, so they sort in protocol order rather than by `ū`. + let protocol_tag = self + .protocol + .as_ref() + .filter(|run| run.phase == ProtocolPhase::Recording) + .and_then(|run| { + let point = run.point()?; + let width = run.plan.points.len().to_string().len().max(2); + Some(format!( + "_p{:0width$}_{}", + run.index + 1, + point.tag(), + width = width + )) + }) + .unwrap_or_default(); let stem = format!( - "{id}_{}{}{sweep_tag}", + "{id}_{}{}{sweep_tag}{protocol_tag}", format_compact_utc(now_ms / 1_000), role.suffix() ); @@ -5430,6 +5526,25 @@ impl StageAA1Plugin { requested_frequency_hz: point.frequency_hz, }) }), + protocol: self + .protocol + .as_ref() + .filter(|run| run.phase == ProtocolPhase::Recording) + .and_then(|run| { + let point = run.point()?; + Some(ProtocolSidecar { + name: run.plan.name.clone(), + block: point.block.clone(), + point_index: run.index + 1, + point_total: run.plan.points.len(), + point_tag: point.tag(), + requested_mean_u: point.mean_u, + requested_frequency_hz: point.frequency_hz, + requested_depth_a: point.depth_a, + settle_s: point.settle_s, + duration_s: point.duration_s, + }) + }), pilot: (self.recording.role == RecRole::Pilot) .then_some(self.pilot_windows) .flatten() @@ -5557,6 +5672,10 @@ struct SidecarDoc { /// Present on points recorded by the automatic frequency ladder. #[serde(skip_serializing_if = "Option::is_none")] frequency_sweep: Option, + /// Present on points recorded by a declarative protocol run: the line of + /// the file that asked for this recording, verbatim. + #[serde(skip_serializing_if = "Option::is_none")] + protocol: Option, #[serde(skip_serializing_if = "Option::is_none")] pilot: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -5630,6 +5749,35 @@ struct FreqSweepSidecar { requested_frequency_hz: f64, } +/// The protocol row this recording is, as the file asked for it. +/// +/// A protocol names every axis explicitly, which is exactly what makes its rows +/// indistinguishable once they are on disk: the panel is not armed at anything +/// that would separate them, so without this section the only difference +/// between two rows' sidecars is the second they started. What is written here +/// is the *request* — what the drive reported back is in `[modulation]`, and +/// what the photodiode measured is in `[optical]`, so a row that failed to +/// reach its point can still be told apart from one that hit it. +#[derive(Serialize)] +struct ProtocolSidecar { + /// `name` from the protocol file. + name: String, + /// The `[[block]]` name, or a CSV `label`, this row came from. + block: String, + /// 1-based position within the whole expanded protocol. + point_index: usize, + point_total: usize, + /// The same fragment that appears in this recording's file names. + point_tag: String, + /// Normalized cycle-mean lobe point `ū` — the `I_k` axis. + requested_mean_u: f64, + requested_frequency_hz: f64, + requested_depth_a: f64, + settle_s: f64, + /// The row's own duration, which overrides the panel's for the run. + duration_s: i64, +} + /// Frozen ON/OFF windows written into a **pilot** recording's sidecar and read /// back to reuse them across the row. #[derive(Serialize, serde::Deserialize)] @@ -5819,6 +5967,20 @@ fn points_for(points: &[RollingResponsePoint], first: u64) -> Vec .collect() } +/// Trims free text down to what a recording-metadata value may carry. +/// +/// The photodiode owner bounds every value at 1 KiB and refuses the whole +/// `BeginRecording` if one is over — a limit worth staying well clear of, since +/// these strings come from a protocol file nobody validated for length. +fn clamp_metadata(text: &str) -> String { + const MAX_CHARS: usize = 120; + let trimmed = text.trim(); + if trimmed.chars().count() <= MAX_CHARS { + return trimmed.to_owned(); + } + trimmed.chars().take(MAX_CHARS).collect() +} + fn waveform_label(waveform: &WaveformV1) -> String { match waveform { WaveformV1::Off => "off".into(), @@ -10033,6 +10195,220 @@ mod tests { let _ = std::fs::remove_dir_all(&folder); } + /// Drives a pending protocol up to the tick that starts its first row's + /// recording: take the lease, apply the three retargets, let the dwell pass. + fn run_protocol_to_first_recording(plugin: &mut StageAA1Plugin, sink: &mut ControlSink) { + control_tick(plugin, PluginControlInbox::default(), sink); + let lease_req = sink + .services + .iter() + .find_map(|request| match modulation_command(request) { + Some(ModulationCommandV1::AcquireLease { .. }) => Some(request.request_id), + _ => None, + }) + .expect("the protocol takes a modulation lease"); + sink.services.clear(); + control_tick(plugin, inbox_with(vec![accepted(lease_req)]), sink); + let retargets: Vec = sink + .services + .iter() + .filter(|request| { + matches!( + modulation_command(request), + Some( + ModulationCommandV1::SetOperatingPoint { .. } + | ModulationCommandV1::SetDriveFrequency { .. } + | ModulationCommandV1::SetOpticalDepth { .. } + ) + ) + }) + .map(|request| request.request_id) + .collect(); + sink.services.clear(); + control_tick( + plugin, + inbox_with(retargets.into_iter().map(accepted).collect()), + sink, + ); + // Settle (`settle_s = 0`) then hand off to the recording coordinator. + control_tick(plugin, PluginControlInbox::default(), sink); + } + + /// Every row of a protocol is recorded with the same panel state, so a row + /// that does not name itself is indistinguishable on disk from the row + /// before it but for the second it started. The operating point a `.pdq` + /// was taken at then cannot be recovered from its name at all — which is + /// exactly what a survey's files are for. + #[test] + fn a_protocol_row_names_its_own_point_in_the_file_stem() { + let folder = temp_folder("protocol-file-stem"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + let mut sink = ControlSink::default(); + + run_protocol_to_first_recording(&mut plugin, &mut sink); + + let stem = plugin.recording.stem.clone(); + assert!( + stem.ends_with("_p01_u400m_f25Hz_a700m"), + "the stem does not name the protocol point: {stem}" + ); + // Two rows of the same protocol differ by more than their timestamp. + let second = TWO_POINT_PROTOCOL.replace("mean_u = [0.4, 0.6]", "mean_u = [0.6]"); + let other_folder = temp_folder("protocol-file-stem-2"); + let (mut other, _) = protocol_plugin(&other_folder, &second); + let mut other_sink = ControlSink::default(); + run_protocol_to_first_recording(&mut other, &mut other_sink); + assert!( + other.recording.stem.ends_with("_p01_u600m_f25Hz_a700m"), + "the second row reuses the first row's tag: {}", + other.recording.stem + ); + + let _ = std::fs::remove_dir_all(&folder); + let _ = std::fs::remove_dir_all(&other_folder); + } + + /// The PDQ travels on its own — it is the photodiode owner's file, and an + /// analysis that opens it must be able to say which row of which protocol + /// it is without joining through the A1 sidecar next door. + #[test] + fn a_protocol_row_names_itself_in_the_recording_metadata() { + let folder = temp_folder("protocol-metadata"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + let mut sink = ControlSink::default(); + + run_protocol_to_first_recording(&mut plugin, &mut sink); + let meta = plugin.recording_metadata(); + + assert_eq!( + meta.get("protocol_name").map(String::as_str), + Some("two-point") + ); + assert_eq!(meta.get("protocol_block").map(String::as_str), Some("pair")); + assert_eq!( + meta.get("protocol_point_index").map(String::as_str), + Some("1") + ); + assert_eq!( + meta.get("protocol_point_total").map(String::as_str), + Some("2") + ); + assert_eq!( + meta.get("protocol_point_tag").map(String::as_str), + Some("u400m_f25Hz_a700m") + ); + assert_eq!( + meta.get("protocol_mean_u").map(String::as_str), + Some("0.400000") + ); + assert_eq!( + meta.get("protocol_frequency_hz").map(String::as_str), + Some("25.000000") + ); + assert_eq!( + meta.get("protocol_depth_a").map(String::as_str), + Some("0.700000") + ); + + // The metadata map has to stay inside the photodiode owner's bounds, or + // BeginRecording is refused and the row records nothing at all. + assert!(meta.len() <= 64, "{} metadata keys", meta.len()); + assert!(meta + .iter() + .all(|(key, value)| key.len() <= 128 && value.len() <= 1_024)); + + let _ = std::fs::remove_dir_all(&folder); + } + + /// The A1 sidecar had a section for every automatic path *except* the one + /// that names all three axes: a protocol row's `[sweep]` carries only the + /// panel's `min_a`/`max_a`, so nothing on disk said which line of the file + /// had asked for the recording. + #[test] + fn the_a1_sidecar_names_the_protocol_row() { + let folder = temp_folder("protocol-sidecar"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + let mut sink = ControlSink::default(); + + run_protocol_to_first_recording(&mut plugin, &mut sink); + // The quantitative sidecar needs a measured optical summary; the plan + // itself is admitted against the photodiode the fixture starts with. + plugin.photodiode = Some(fresh_photodiode_summary()); + let path = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&path).expect("read sidecar"); + + assert!(text.contains("[protocol]"), "{text}"); + assert!(text.contains("name = \"two-point\""), "{text}"); + assert!(text.contains("block = \"pair\""), "{text}"); + assert!(text.contains("point_index = 1"), "{text}"); + assert!(text.contains("point_total = 2"), "{text}"); + assert!(text.contains("point_tag = \"u400m_f25Hz_a700m\""), "{text}"); + assert!(text.contains("requested_mean_u = 0.4"), "{text}"); + assert!(text.contains("requested_frequency_hz = 25.0"), "{text}"); + assert!(text.contains("requested_depth_a = 0.7"), "{text}"); + + let _ = std::fs::remove_dir_all(&folder); + } + + /// A run that is not driving anything must not claim a protocol row. + #[test] + fn a_hand_driven_recording_carries_no_protocol_keys() { + let mut plugin = plugin_with_markers(); + plugin.modulation = Some(connected_modulation()); + let meta = plugin.recording_metadata(); + assert!(!meta.contains_key("protocol_name")); + assert!(!meta.contains_key("protocol_point_index")); + } + + /// What the drive was actually doing, in the photodiode's own sidecar. A + /// frequency and two DAC codes do not say which lobe, or which operating + /// point `ū`, produced a trace — and `ū` is a whole axis of the survey. + #[test] + fn the_recording_metadata_describes_how_the_drive_was_shaped() { + let mut plugin = plugin_with_markers(); + let mut state = commanded_modulation(1, 0.7); + state.acknowledged = Some(stage_a_plugin_contract::ModulationTargetV1 { + revision: SemanticRevision(1), + waveform: Some(WaveformV1::Periodic { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Sine, + min_dac: 100, + max_dac: 800, + frequency_millihz: 25_000, + }), + a1_configuration: None, + acquisition_running: true, + board_dac_code: None, + firmware_configuration_revision: None, + }); + plugin.modulation = Some(state); + let meta = plugin.recording_metadata(); + + for key in [ + "modulation_waveform", + "pockels_calibration_id", + "optical_target", + "requested_mean_u", + "resolved_mean_u", + "commanded_a", + "v_null_dac", + "v_peak_dac", + ] { + assert!(meta.contains_key(key), "{key} missing from {meta:?}"); + } + assert_eq!( + meta.get("optical_target").map(String::as_str), + Some("log_sine") + ); + assert_eq!( + meta.get("requested_mean_u").map(String::as_str), + Some("0.500000") + ); + assert_eq!( + meta.get("commanded_a").map(String::as_str), + Some("0.700000") + ); + } + /// A protocol file on disk, and a plugin ready to run it. fn protocol_plugin(folder: &Path, body: &str) -> (StageAA1Plugin, PathBuf) { std::fs::create_dir_all(folder).expect("protocol folder"); From 4bb4826f72291624064393d4ade3619e1dceff92 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 5 Aug 2026 17:44:11 +0200 Subject: [PATCH 46/46] =?UTF-8?q?feat(stage-a-a1):=20=E2=9C=A8=20say=20how?= =?UTF-8?q?=20much=20longer=20a=20run=20has=20to=20go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every A1 run long enough to walk away from — the depth sweep, either mode of the frequency ladder, an a₀ point, a protocol — now states its remaining time on the button press and keeps stating it on the status pane: Estimated time: ≈ 41 min left of ≈ 1 h 5 min — done by 03:41 UTC Only the protocol answered this before, once, from the sum of its rows' duration_s + settle_s. That number is too small — it omits the camera and photodiode handshakes, the drive settling at a new depth, the four marker periods a new frequency is confirmed over and the a₀ lock's trials — and it goes stale the moment the next per-point message overwrites it. So the estimate is the plan's own seconds scaled by the pace the run is actually keeping: the plan alone (plus a fixed handshake allowance) until the first point finishes, saying so in words, and re-scaled by every finished point after that. The pace is clamped so one point that spent its timeouts cannot extrapolate a night onto the rest. It is deliberately not the lease TTL, which is a worst case and would quote half an hour for a five-minute sweep. One line, for the outermost run: a ladder's estimate already covers the depth sweep inside it, and a protocol covers both. --- ...the-plan-corrected-by-the-measured-pace.md | 103 +++++ docs/features/README.md | 2 +- docs/features/stage-a-a1.md | 45 +- plugins/stage-a-a1/README.md | 19 + plugins/stage-a-a1/src/eta.rs | 171 ++++++++ plugins/stage-a-a1/src/lib.rs | 1 + plugins/stage-a-a1/src/protocol.rs | 9 +- plugins/stage-a-a1/src/runtime.rs | 403 +++++++++++++++++- 8 files changed, 739 insertions(+), 14 deletions(-) create mode 100644 docs/adr/034-a1-time-estimates-are-the-plan-corrected-by-the-measured-pace.md create mode 100644 plugins/stage-a-a1/src/eta.rs diff --git a/docs/adr/034-a1-time-estimates-are-the-plan-corrected-by-the-measured-pace.md b/docs/adr/034-a1-time-estimates-are-the-plan-corrected-by-the-measured-pace.md new file mode 100644 index 0000000..4caf9d7 --- /dev/null +++ b/docs/adr/034-a1-time-estimates-are-the-plan-corrected-by-the-measured-pace.md @@ -0,0 +1,103 @@ +# ADR 034 — A run's time estimate is its plan corrected by the pace it is actually keeping + +- **Status:** Accepted +- **Date:** 2026-08-05 +- **Relates to:** ADR 010 (amplitude sweep), ADR 014 (frequency ladder), + ADR 023 (nested depth × frequency sweep), ADR 027 (declarative protocols), + ADR 029 (leases renewed against the granted deadline), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +Three A1 runs are long enough that the operator's next question after pressing +the button is *how long will this take*: the depth sweep, the frequency ladder +(either mode), and a protocol file. Only the protocol answered it, once, on the +button press — `about {minutes:.0} min of bench time` — from +`Protocol::total_seconds()`, the sum of each row's `duration_s + settle_s`. + +That number is wrong in two different ways, and both matter at 23:00. + +**It is too small.** The plan's seconds are the recording and the operator's +settle dwell. They are not what a point costs. Every point also pays: + +- the camera start and stop, and the photodiode connect → lease → start → + finalize handshake (the same handshakes ADR 029 had to widen the lease TTL + for); +- the drive physically settling at a new depth, which the sweep waits for + against the *measured* `a` with a 30 s cap; +- for a ladder rung, the trigger confirming the new frequency — four marker + periods, so 40 s at 0.1 Hz — and, in `a₀` mode, a closed-loop lock of up to + eight trials that appears in no setting at all. + +**It goes stale.** It is printed once and then overwritten by the next +per-point message. A survey that is running at half the expected speed — a +drifting cell that settles slowly, a photodiode window that keeps being +rejected — says nothing until it is still going in the morning. + +The obvious source for a better number is already in the file: the lease TTLs +(`sweep_lease_ttl_ms`, `freq_sweep_lease_ttl_ms`, `protocol_lease_ttl_ms`). +They must not be reused. A TTL is a deliberate **worst case** — it has to cover +the settle timeout of every remaining point or the owner reaps the lease +mid-run — and doubles its input on top. Quoting it would tell the operator that +a five-minute sweep needs half an hour, and an estimate nobody believes is worse +than none. + +## Decision + +**The estimate is the plan's own seconds, scaled by the pace the run has +actually kept, and it is shown for as long as the run lasts.** + +`plugins/stage-a-a1/src/eta.rs` holds the whole mechanism — one `Eta` per run: + +```rust +pub fn point_done(&mut self, now_ms: u64, planned_s: f64); // at each point boundary +pub fn pace(&self) -> Option; // actual / planned, clamped +pub fn remaining_s(&self, now_ms: u64, planned_remaining_s: f64) -> f64; +``` + +- **Until the first point finishes** there is nothing to measure, so the plan + stands alone plus a fixed `POINT_OVERHEAD_S = 5 s` allowance per point for the + handshake. The line says so in words: *(from the plan until the first point + finishes)*. It is a lower bound that improves, not a promise. +- **From then on**, every finished point re-scales what is left. The pace is + clamped to `[0.5, 6.0]`, so one point that spent its timeouts cannot + extrapolate a night onto the rest, and one that was refused before it began + cannot predict the rest away. +- **Skipped points count.** What is being measured is how long this run takes to + get through its list; a point that fails still spends its time. +- **The estimate counts down inside the point in flight**, so it moves between + boundaries instead of standing still for a 40 s row. + +What each run's plan *is* stays with the run, because only it knows its shape: +a protocol reads each row's own `duration_s + settle_s`; a depth sweep multiplies +the panel's duration and dwell by its point count; a ladder rung adds the +frequency retarget, the confirmation (`4 / f`, capped at the 20 s give-up), the +`a₀` lock where one runs at all — three nominal trials, not the cap of eight — +and then whatever the rung records. + +**One estimate, for the outermost run.** A ladder's `Estimated time` line +already contains the inner depth sweep it handed the current rung to, and a +protocol contains both. Three lines would state one fact three times, with the +two inner ones — which end long before the run does — reading as contradictions +of the only one that matters. + +**The finish clock is UTC**, and only appears once more than ten minutes are +left. UTC because every filename, sidecar and timestamp this plugin writes is +UTC, and two clocks in one panel is a bug waiting for a night shift. + +## Consequences + +- The panel gains one line, e.g. + `Estimated time: ≈ 41 min left of ≈ 1 h 5 min — done by 03:41 UTC`, for a + depth sweep, a frequency ladder, an `a₀` point or a protocol. +- The button press states the same number the panel then counts down, so the + figure on the press and the figure a minute later are not two different + answers. The protocol's `about N min` wording is gone. +- The nominal constants (`POINT_OVERHEAD_S`, `A0_LOCK_NOMINAL_TRIALS = 3`, + `A0_LOCK_NOMINAL_TRIAL_S`, `FREQ_RETARGET_NOMINAL_S`) are bench guesses and + are only ever the *first* estimate of a run. They are deliberately not tuned + against any one bench: the measured pace replaces them within one point. +- Nothing here steers a run. No point is skipped, shortened or refused because + of an estimate; it exists so the operator can decide whether to wait. +- `Protocol::total_seconds()` is now `remaining_seconds(0)`, and + `protocol_lease_ttl_ms` reuses the same sum instead of restating it. diff --git a/docs/features/README.md b/docs/features/README.md index ba82b15..b8bf2f3 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -9,7 +9,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC from measured `V_null`/`V_peak` endpoints, with target-specific headroom, Bessel-normalized cycle mean `ū`, and an explicit separation from physical flux `I_k`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. Each point is a 20 ms measurement after a 0.1 s settle, and every verdict on the sweep — lobe resolved, cell drifting — is made against the fit's own residual rather than against zero (ADR 019). Applying the fit now actually reaches the panel: the measurement lives on the live worker while the settings snapshot is collected from the UI mirror, so the applied lobe used to be overwritten within one frame (ADR 026). - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. Port discovery is platform-aware and shared with the modulation plugin (ADR 032). -- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). Every leased run heartbeats its modulation and photodiode leases against the deadline the owner actually granted, so a recording longer than the owner's TTL cap no longer loses the drive — and with it the phase-0 trigger and the photodiode's optical summary — in the middle of a point (ADR 029). **Each protocol row names itself** — `_p03_u500m_f10Hz_a800m` in the file stem, a `[protocol]` section in the A1 sidecar, `protocol_*` keys in the recorder metadata — so a `.pdq` maps back to its modulation without counting timestamps against the protocol file, and the PDQ sidecar now also describes how the drive was *shaped* (waveform, `ū`, lobe, calibration id) rather than only how fast it ran (ADR 033). +- [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). Every leased run heartbeats its modulation and photodiode leases against the deadline the owner actually granted, so a recording longer than the owner's TTL cap no longer loses the drive — and with it the phase-0 trigger and the photodiode's optical summary — in the middle of a point (ADR 029). **Each protocol row names itself** — `_p03_u500m_f10Hz_a800m` in the file stem, a `[protocol]` section in the A1 sidecar, `protocol_*` keys in the recorder metadata — so a `.pdq` maps back to its modulation without counting timestamps against the protocol file, and the PDQ sidecar now also describes how the drive was *shaped* (waveform, `ū`, lobe, calibration id) rather than only how fast it ran (ADR 033). **Every long run says how much longer it has** — a depth sweep, a frequency ladder, an `a₀` point or a protocol — as one status line that starts from the plan's own seconds and is re-scaled by the pace the bench actually keeps, so a survey running at half speed says so within one point instead of in the morning (ADR 034). - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 1a7a9d1..848dc28 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -31,7 +31,10 @@ (the sensor readout travels with the measurement, column-wise), [ADR 029](../adr/029-stage-a-leases-are-renewed-against-the-granted-deadline.md) (a leased run heartbeats against the deadline the owner granted, so a point - longer than the owner's TTL cap no longer loses the drive mid-recording) + longer than the owner's TTL cap no longer loses the drive mid-recording), + [ADR 034](../adr/034-a1-time-estimates-are-the-plan-corrected-by-the-measured-pace.md) + (every long run says how much longer it has: the plan's own seconds, re-scaled + by the pace the bench is actually keeping) - **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) - **Second workflow:** [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — hold one *measured* depth `a₀` across the frequency sweep @@ -297,6 +300,38 @@ its three. One lease covers the whole file. `plugins/stage-a-a1/protocols/example.toml` is a commented file to copy. +### How long will this take? (ADR 034) + +Every run long enough to walk away from — the depth sweep, either mode of the +frequency ladder, an `a₀` point, a protocol — states its remaining time on the +button press and then keeps stating it, in one status line: + +``` +Estimated time: ≈ 41 min left of ≈ 1 h 5 min — done by 03:41 UTC +``` + +- **The plan is the starting point, the bench is the correction.** The first + number is what the run asks for (`duration_s + settle_s` per point, per row + for a protocol) plus a fixed allowance for the start/finalize handshake, and + it says of itself *(from the plan until the first point finishes)*. From the + first finished point on, the estimate is re-scaled by the pace the run is + actually keeping — settling, handshakes, `a₀` trials and all — so a survey + running at half speed says so within one point instead of in the morning. +- **A ladder rung** is costed as the frequency retarget, the trigger + confirmation (four marker periods, so the low frequencies dominate), the `a₀` + lock where one runs at all, and then whatever the rung records. +- **One line, for the run the operator started.** A ladder's estimate already + covers the inner depth sweep it is currently running, and a protocol covers + both. +- **Skipped points count**, the estimate counts down inside the point in + flight, and the finish clock (shown above ten minutes) is UTC like every + filename this plugin writes. + +It is advisory only: nothing is skipped, shortened or refused because of an +estimate. It is deliberately *not* the lease TTL, which is a worst case that +must cover every remaining settle timeout and would quote half an hour for a +five-minute sweep. + ### Bench conditions on every run (ADR 022) Every recording — normal, pilot, background, sweep point, a₀ point — also @@ -621,3 +656,11 @@ Two cover the nested sweep (ADR 023): the whole 2 × 3 block records every depth at every frequency in depth order, on exactly **one** lease acquisition, never entering the search phase and finishing with `2/2 frequencies × 3 depths`; and a nested point's file stem carries both axes (`…_f50Hz_p03`). + +Eight cover the time estimate (ADR 034): duration wording from seconds to hours; +an unmeasured run reporting the plan and still counting down inside the point in +flight; a finished point re-scaling what is left; the bounded pace correction; +elapsed measured from the run start; a protocol stating its bench time on the +button press *and* on the status pane, labelled as the plan; a sweep whose first +point ran at half speed doubling the four points left; and exactly one estimate — +the ladder's, not its inner sweep's — when both runs are live. diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 048df5e..6374b64 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -215,6 +215,25 @@ time are reported first. Use **Stop** in the Record section to end a run early. `~/.augur/plugins/stage-a-a1/protocols/`. See [ADR 027](../../docs/adr/027-stage-a-a1-declarative-protocols.md). +## How long will this take? + +Every run long enough to walk away from — **Sweep a**, **Sweep f**, **Sweep a × f**, an `a₀` point +or a protocol — reports its remaining time on the button press and keeps reporting it on the status +pane: + +``` +Estimated time: ≈ 41 min left of ≈ 1 h 5 min — done by 03:41 UTC +``` + +The first number is the plan's own time (`Duration` + `Settle time` per point, or each protocol +row's own) plus a fixed allowance for the camera/photodiode handshake, and it says so: +*(from the plan until the first point finishes)*. After that, every finished point re-scales what +is left by the pace the bench is actually keeping — settling, handshakes, `a₀` trials and the +marker periods a new frequency has to be confirmed over. Only the outermost run states an estimate: +a ladder's already covers the depth sweep inside it. It is advisory — nothing is skipped or +shortened because of it. See +[ADR 034](../../docs/adr/034-a1-time-estimates-are-the-plan-corrected-by-the-measured-pace.md). + ## Live quicklooks - **Rolling half-period response** `S_p(t) = N_p(t−T/2, t] / N_valid` — events per valid pixel in the diff --git a/plugins/stage-a-a1/src/eta.rs b/plugins/stage-a-a1/src/eta.rs new file mode 100644 index 0000000..8ffae70 --- /dev/null +++ b/plugins/stage-a-a1/src/eta.rs @@ -0,0 +1,171 @@ +//! How much longer an unattended run still has to go. +//! +//! Every long run in this plugin — a depth sweep, a frequency ladder, a +//! protocol — knows what it *asked* for: so many points, so many seconds of +//! recording, so much settle dwell. None of them know what the bench actually +//! charges for a point. The start/finalize handshake, the drive settling at a +//! new depth, the trigger confirming a new frequency, the photodiode's own +//! estimator window: all of that is real time that appears in no setting, and +//! on this bench it is the larger half of a short point. +//! +//! So the estimate is the plan's own numbers scaled by the pace the run is +//! actually keeping. Until the first point finishes there is nothing to +//! measure and the plan stands alone, with a fixed allowance for the handshake; +//! from then on every finished point re-scales what is left. That makes the +//! first number a lower bound that improves rather than a promise — the honest +//! shape for a survey whose settle times are a property of the bench and not +//! of the file it was started from. +//! +//! Nothing here steers a run. The estimate exists so the operator can decide +//! whether to wait, and so a run started at 23:00 says whether it is a +//! coffee-length or a night-length one *before* it is left alone. + +/// Wall-clock cost of the start/finalize handshake around one recording, on +/// top of the time the point itself asks for. Only carries the estimate until +/// the run has measured its own pace. +pub const POINT_OVERHEAD_S: f64 = 5.0; + +/// Bounds on the measured pace correction. A run that lost a lease and spent a +/// point timing out must not extrapolate that point over the whole rest of the +/// survey, and one whose first point was refused before it began must not +/// predict the rest away either. +const PACE_RANGE: (f64, f64) = (0.5, 6.0); + +/// One run's answer to "how much longer?": the plan's own seconds, corrected by +/// the pace the run is keeping. +#[derive(Debug, Clone, Copy)] +pub struct Eta { + /// When the run started, for the elapsed leg of the total. + started_ms: u64, + /// When the point in flight started — the run start, then each boundary. + point_started_ms: u64, + /// Planned seconds of the points that have already finished. + planned_done_s: f64, + /// Wall-clock seconds those points actually took. + actual_done_s: f64, +} + +impl Eta { + pub fn new(now_ms: u64) -> Self { + Self { + started_ms: now_ms, + point_started_ms: now_ms, + planned_done_s: 0.0, + actual_done_s: 0.0, + } + } + + /// One point finished — recorded, skipped or given up — having been planned + /// to cost `planned_s`. + /// + /// Skipped points are counted deliberately: what is being measured is how + /// long this run takes to get through its list, and a point that fails + /// still spends its timeouts. + pub fn point_done(&mut self, now_ms: u64, planned_s: f64) { + let spent_s = now_ms.saturating_sub(self.point_started_ms) as f64 / 1_000.0; + self.point_started_ms = now_ms; + if planned_s > 0.0 { + self.planned_done_s += planned_s; + self.actual_done_s += spent_s; + } + } + + /// Seconds of wall clock the bench spends per planned second, from the + /// points that have finished. `None` until there is one to measure. + pub fn pace(&self) -> Option { + (self.planned_done_s > 0.0 && self.actual_done_s > 0.0) + .then(|| (self.actual_done_s / self.planned_done_s).clamp(PACE_RANGE.0, PACE_RANGE.1)) + } + + /// Wall-clock seconds still to go, for a plan that has `planned_remaining_s` + /// left *including* the point currently in flight. + pub fn remaining_s(&self, now_ms: u64, planned_remaining_s: f64) -> f64 { + let scaled = planned_remaining_s.max(0.0) * self.pace().unwrap_or(1.0); + let in_flight_s = now_ms.saturating_sub(self.point_started_ms) as f64 / 1_000.0; + (scaled - in_flight_s).max(0.0) + } + + pub fn elapsed_s(&self, now_ms: u64) -> f64 { + now_ms.saturating_sub(self.started_ms) as f64 / 1_000.0 + } +} + +/// A duration as the operator would say it: seconds while that is meaningful, +/// then minutes, then hours. Never more than two units — the point is the size +/// of the wait, not its last second. +pub fn format_duration(seconds: f64) -> String { + let total = seconds.max(0.0).round() as u64; + if total < 90 { + return format!("{total} s"); + } + let minutes = total / 60; + if minutes < 60 { + let rest = total % 60; + return if minutes < 10 && rest > 0 { + format!("{minutes} min {rest} s") + } else { + format!("{minutes} min") + }; + } + let hours = minutes / 60; + let rest = minutes % 60; + if rest > 0 { + format!("{hours} h {rest} min") + } else { + format!("{hours} h") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn durations_read_as_a_person_would_say_them() { + assert_eq!(format_duration(0.0), "0 s"); + assert_eq!(format_duration(45.4), "45 s"); + assert_eq!(format_duration(89.0), "89 s"); + assert_eq!(format_duration(200.0), "3 min 20 s"); + assert_eq!(format_duration(1_800.0), "30 min"); + assert_eq!(format_duration(3_600.0), "1 h"); + assert_eq!(format_duration(4_500.0), "1 h 15 min"); + } + + /// Before anything has finished the plan is all there is, and it counts + /// down inside the point in flight rather than standing still. + #[test] + fn an_unmeasured_run_reports_the_plan_and_still_counts_down() { + let eta = Eta::new(0); + assert!(eta.pace().is_none()); + assert!((eta.remaining_s(0, 100.0) - 100.0).abs() < 1e-9); + assert!((eta.remaining_s(30_000, 100.0) - 70.0).abs() < 1e-9); + } + + /// A point that took twice its planned time says the rest will too. + #[test] + fn a_finished_point_rescales_what_is_left() { + let mut eta = Eta::new(0); + eta.point_done(20_000, 10.0); + assert_eq!(eta.pace(), Some(2.0)); + // Four points of 10 planned seconds left, at 2 s of bench per planned + // second, with nothing spent in the current point yet. + assert!((eta.remaining_s(20_000, 40.0) - 80.0).abs() < 1e-9); + } + + /// One pathological point must not extrapolate over the whole survey. + #[test] + fn the_pace_correction_is_bounded() { + let mut eta = Eta::new(0); + eta.point_done(600_000, 1.0); + assert_eq!(eta.pace(), Some(PACE_RANGE.1)); + } + + /// The run's own clock keeps running across points, so the total the panel + /// shows (elapsed + remaining) is wall clock and not a sum of estimates. + #[test] + fn elapsed_is_measured_from_the_run_start() { + let mut eta = Eta::new(1_000); + eta.point_done(11_000, 10.0); + assert!((eta.elapsed_s(31_000) - 30.0).abs() < 1e-9); + } +} diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs index 14acc6a..5f4db9c 100644 --- a/plugins/stage-a-a1/src/lib.rs +++ b/plugins/stage-a-a1/src/lib.rs @@ -5,6 +5,7 @@ //! photodiode plugins; this code only validates and analyses immutable inputs. mod csv; +pub mod eta; pub mod phase; pub mod protocol; pub mod rates; diff --git a/plugins/stage-a-a1/src/protocol.rs b/plugins/stage-a-a1/src/protocol.rs index f93e443..152e476 100644 --- a/plugins/stage-a-a1/src/protocol.rs +++ b/plugins/stage-a-a1/src/protocol.rs @@ -141,7 +141,14 @@ impl Protocol { /// Total bench time the protocol asks for, settling included. pub fn total_seconds(&self) -> f64 { - self.points + self.remaining_seconds(0) + } + + /// Bench time the points from `from` onwards still ask for, settling + /// included — what a run in progress has left of its *plan*, before the + /// bench's own overheads are added to it. + pub fn remaining_seconds(&self, from: usize) -> f64 { + self.points[from.min(self.points.len())..] .iter() .map(|point| point.duration_s as f64 + point.settle_s) .sum() diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 0e3b6c9..af02122 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -64,6 +64,7 @@ use stage_a_plugin_contract::{ SERVICE_STAGE_A_MODULATION_CONTROL_V1, SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, }; +use crate::eta::{format_duration, Eta}; use crate::phase::{fold_events, fold_events_free_running, MarkerValidationConfig, PhaseFold}; use crate::protocol; use crate::rates::{rolling_half_period_response, RollingResponsePoint}; @@ -122,6 +123,17 @@ const A0_LOCK_SAMPLE_SPACING: f64 = 0.5; /// operating point is called unstable instead of locked. A drifting `a` that /// happens to cross the target on one reading is not a lock. const A0_LOCK_MAX_SPREAD_TOLERANCES: f64 = 2.0; +/// Trials one `a₀` lock is *expected* to spend, for the time estimate only. +/// The cap above is [`A0_LOCK_MAX_TRIALS`], but a calibrated drive lands inside +/// the tolerance in two or three, and estimating every rung of a ladder at the +/// cap would put a night on a run that takes an hour. +const A0_LOCK_NOMINAL_TRIALS: f64 = 3.0; +/// Nominal measuring time of one lock trial on top of the operator's settle +/// dwell: the independent photodiode readings it takes a window apart. +const A0_LOCK_NOMINAL_TRIAL_S: f64 = 4.0; +/// Nominal firmware/table cost of retargeting the drive to a new frequency, on +/// top of the marker periods the confirmation then waits for. +const FREQ_RETARGET_NOMINAL_S: f64 = 2.0; /// Clipping fraction above which a lock's measured `a` is called out as /// unreliable in the operator message. /// @@ -462,6 +474,8 @@ struct Sweep { completed_ok: bool, last_activity_ms: u64, stop_requested: bool, + /// How much longer this run has to go — see [`crate::eta`]. + eta: Eta, } impl Sweep { @@ -790,6 +804,9 @@ struct FreqSweep { seed: u64, last_activity_ms: u64, stop_requested: bool, + /// How much longer the *ladder* has to go — see [`crate::eta`]. The inner + /// run keeps one of its own; only the outermost is ever shown. + eta: Eta, } impl FreqSweep { @@ -851,6 +868,8 @@ struct ProtocolRun { /// Why the current point is being given up, when that was decided in a /// service reply rather than in the tick. skip_reason: Option, + /// How much longer the survey has to go — see [`crate::eta`]. + eta: Eta, } impl ProtocolRun { @@ -2758,6 +2777,123 @@ impl StageAA1Plugin { .collect() } + // ---- how long a run is planned to take ----------------------------- + // + // These are deliberately *not* the lease TTLs below. A TTL is a worst case + // — it has to cover the settle timeout of every point, or the owner takes + // the drive back mid-run — while an estimate the operator plans an evening + // around has to be the likely case. Quoting the TTL would tell someone + // their five-minute sweep needs half an hour. The measured pace in + // [`crate::eta`] closes whatever gap is left. + + /// What one recorded point costs: the recording itself, the operator's + /// settle dwell, and the start/finalize handshake around it. + fn planned_point_s(&self) -> f64 { + self.duration_s.max(1) as f64 + self.settle_s.max(0.0) + crate::eta::POINT_OVERHEAD_S + } + + /// What one closed-loop `a₀` lock costs, or zero where the depth is + /// commanded and no lock runs at all (ADR 021). + fn planned_a0_lock_s(&self) -> f64 { + if !self.depth_source.needs_a0_lock() { + return 0.0; + } + A0_LOCK_NOMINAL_TRIALS * (self.settle_s.max(0.0) + A0_LOCK_NOMINAL_TRIAL_S) + } + + /// What one rung of the frequency ladder costs at `frequency_hz`: + /// retargeting the drive, waiting for the trigger to confirm the new period + /// — which is a few cycles, so it is the low frequencies that dominate a + /// ladder — and then whatever that rung records. + fn planned_rung_s(&self, mode: FreqSweepMode, frequency_hz: f64) -> f64 { + let confirm_s = if frequency_hz > 0.0 { + (FREQ_CONFIRM_CYCLES / frequency_hz).min(FREQ_CONFIRM_BASE_MS as f64 / 1_000.0) + } else { + 0.0 + }; + let inner_s = match mode { + FreqSweepMode::A0Point => self.planned_a0_lock_s() + self.planned_point_s(), + FreqSweepMode::DepthSweep => { + self.sweep_count.clamp(2, 64) as f64 * self.planned_point_s() + } + }; + FREQ_RETARGET_NOMINAL_S + confirm_s + inner_s + } + + /// Planned seconds a run still has left, counting the point in flight. + /// `None` when nothing is running. + fn planned_remaining_s(&self) -> Option { + // Outermost first: a ladder's estimate already contains the inner + // sweep it handed the current rung to, and a protocol contains both. + if let Some(run) = self.protocol.as_ref() { + let remaining_points = run.plan.points.len().saturating_sub(run.index); + return Some( + run.plan.remaining_seconds(run.index) + + remaining_points as f64 * crate::eta::POINT_OVERHEAD_S, + ); + } + if let Some(sweep) = self.freq_sweep.as_ref() { + let from = sweep.index.min(sweep.points.len()); + return Some( + sweep.points[from..] + .iter() + .map(|point| self.planned_rung_s(sweep.mode, point.frequency_hz)) + .sum(), + ); + } + if let Some(sweep) = self.sweep.as_ref() { + let remaining = sweep.total().saturating_sub(sweep.index); + return Some(remaining as f64 * self.planned_point_s()); + } + None + } + + /// The estimate itself: the [`Eta`] of the outermost run in flight, paired + /// with what its plan still asks for. + fn run_eta(&self) -> Option<(Eta, f64)> { + let planned_remaining_s = self.planned_remaining_s()?; + let eta = if let Some(run) = self.protocol.as_ref() { + run.eta + } else if let Some(sweep) = self.freq_sweep.as_ref() { + sweep.eta + } else { + self.sweep.as_ref()?.eta + }; + Some((eta, planned_remaining_s)) + } + + /// One line of "how much longer", for whichever run the operator started. + /// + /// The total is elapsed + remaining rather than a stored plan number, so it + /// is wall clock throughout and grows honestly when a run runs long. + fn estimated_time_line(&self, now_ms: u64) -> Option { + let (eta, planned_remaining_s) = self.run_eta()?; + let remaining_s = eta.remaining_s(now_ms, planned_remaining_s); + let total_s = eta.elapsed_s(now_ms) + remaining_s; + let mut line = format!( + "Estimated time: ≈ {} left of ≈ {}", + format_duration(remaining_s), + format_duration(total_s) + ); + // Long enough that the operator will leave the bench: say when to come + // back, in the same UTC every file of the run is stamped with. Minutes + // are noise below that — a five-minute sweep is watched, not planned + // around. + if remaining_s >= 600.0 { + line.push_str(&format!( + " — done by {}", + format_clock_utc(now_ms / 1_000 + remaining_s as u64) + )); + } + // Nothing has finished yet, so this is the plan's own time with a fixed + // allowance for the bench. Say so, rather than let a number that is + // about to grow look like a measurement. + if eta.pace().is_none() { + line.push_str(" (from the plan until the first point finishes)"); + } + Some(line) + } + /// Worst-case sweep duration, used as the modulation lease TTL. fn sweep_lease_ttl_ms(&self, remaining_points: usize) -> u64 { let per_point_ms = (self.duration_s.max(1) as u64) @@ -2824,8 +2960,9 @@ impl StageAA1Plugin { } let points = self.sweep_points(); let message = format!( - "Sweep: acquiring modulation lease for {} points…", - points.len() + "Sweep: acquiring modulation lease for {} points, ≈ {} of bench time…", + points.len(), + format_duration(points.len() as f64 * self.planned_point_s()) ); self.begin_leased_sweep(context, SweepKind::Amplitude, points, None, None, message); } @@ -2896,6 +3033,7 @@ impl StageAA1Plugin { completed_ok: false, last_activity_ms: now_ms, stop_requested: false, + eta: Eta::new(now_ms), }); self.message = message; } @@ -3168,7 +3306,9 @@ impl StageAA1Plugin { } self.finish_sweep(context, message); } else { + let planned_s = self.planned_point_s(); if let Some(sweep) = self.sweep.as_mut() { + sweep.eta.point_done(now_ms, planned_s); sweep.index += 1; } self.send_sweep_depth(context); @@ -4220,6 +4360,10 @@ impl StageAA1Plugin { let lease_req = request.request_id; context.request_service(&request); let total = points.len(); + let planned_s: f64 = points + .iter() + .map(|point| self.planned_rung_s(mode, point.frequency_hz)) + .sum(); self.freq_sweep = Some(FreqSweep { phase: FreqSweepPhase::AcquiringLease, mode, @@ -4238,15 +4382,18 @@ impl StageAA1Plugin { seed: self.freq_seed, last_activity_ms: now_ms, stop_requested: false, + eta: Eta::new(now_ms), }); + let estimate = format_duration(planned_s); self.message = match mode { FreqSweepMode::A0Point => format!( - "Frequency sweep: acquiring the modulation lease for {total} points ({} order)…", + "Frequency sweep: acquiring the modulation lease for {total} points ({} order), \ + ≈ {estimate} of bench time…", self.freq_order.label() ), FreqSweepMode::DepthSweep => format!( "Depth sweep at every frequency: acquiring the modulation lease for {total} × {} \ - recordings ({} order)…", + recordings ({} order), ≈ {estimate} of bench time…", self.sweep_count.clamp(2, 64), self.freq_order.label() ), @@ -4417,8 +4564,18 @@ impl StageAA1Plugin { /// Move to the next ladder point, or finish with a summary. fn advance_freq_sweep(&mut self, context: &mut impl RecordingControl) { + // The rung that just ended is what the estimate learns from, so its + // planned cost has to be read before the index moves past it. + let planned_s = self + .freq_sweep + .as_ref() + .and_then(|sweep| sweep.point().map(|point| (sweep.mode, point.frequency_hz))) + .map(|(mode, frequency_hz)| self.planned_rung_s(mode, frequency_hz)) + .unwrap_or(0.0); + let now_ms = now_unix_ms(); let done = match self.freq_sweep.as_mut() { Some(sweep) => { + sweep.eta.point_done(now_ms, planned_s); sweep.index += 1; sweep.index >= sweep.points.len() } @@ -4485,10 +4642,7 @@ impl StageAA1Plugin { /// A1 *asks* for, not what it gets — the owner caps the TTL it grants, and /// [`Self::drive_lease_heartbeat`] is what actually keeps the lease alive. fn protocol_lease_ttl_ms(plan: &protocol::Protocol, from: usize) -> u64 { - let remaining: f64 = plan.points[from.min(plan.points.len())..] - .iter() - .map(|point| point.duration_s as f64 + point.settle_s) - .sum(); + let remaining = plan.remaining_seconds(from); // Doubled: every point also spends time on the start/finalize // handshake, which is not in the protocol's own numbers. ((remaining * 2_000.0) as u64).saturating_add(60_000) @@ -4572,10 +4726,14 @@ impl StageAA1Plugin { let (means, frequencies, depths) = plan.axis_counts(); let total = plan.points.len(); - let minutes = plan.total_seconds() / 60.0; + // The plan's own seconds plus what the bench charges per point — the + // same number the panel then counts down, so the estimate on the button + // press and the estimate a minute later are not two different figures. + let estimate = + format_duration(plan.total_seconds() + total as f64 * crate::eta::POINT_OVERHEAD_S); self.message = format!( "Protocol '{}': {total} recordings ({means} × ū, {frequencies} × f, {depths} × a), \ - about {minutes:.0} min of bench time — acquiring the modulation lease…", + ≈ {estimate} of bench time — acquiring the modulation lease…", plan.name ); self.protocol = Some(ProtocolRun { @@ -4592,6 +4750,7 @@ impl StageAA1Plugin { last_activity_ms: now_ms, stop_requested: false, skip_reason: None, + eta: Eta::new(now_ms), }); } @@ -4696,8 +4855,16 @@ impl StageAA1Plugin { let Some(run) = self.protocol.as_mut() else { return; }; + let now_ms = now_unix_ms(); + // What the row that just ended was planned to cost, before the index + // moves past it — that difference is the whole estimate. + let planned_s = run + .point() + .map(|point| point.duration_s as f64 + point.settle_s + crate::eta::POINT_OVERHEAD_S) + .unwrap_or(0.0); + run.eta.point_done(now_ms, planned_s); run.index += 1; - run.last_activity_ms = now_unix_ms(); + run.last_activity_ms = now_ms; if run.index < run.plan.points.len() && !run.stop_requested { self.send_protocol_point(context); return; @@ -6142,6 +6309,14 @@ fn format_compact_utc(unix_secs: u64) -> String { format!("{y:04}{m:02}{d:02}-{hh:02}{mm:02}{ss:02}") } +/// Wall-clock time of day, for "come back at". UTC like every other timestamp +/// this plugin writes — a local time here and UTC in the filenames would be two +/// clocks in one panel. +fn format_clock_utc(unix_secs: u64) -> String { + let (_, _, _, hh, mm, _) = ymd_hms(unix_secs); + format!("{hh:02}:{mm:02} UTC") +} + fn format_iso_utc(unix_secs: u64) -> String { let (y, m, d, hh, mm, ss) = ymd_hms(unix_secs); format!("{y:04}-{m:02}-{d:02}T{hh:02}:{mm:02}:{ss:02}Z") @@ -7307,6 +7482,12 @@ impl Plugin for StageAA1Plugin { lock.samples.len() ))); } + // One estimate, for the run the operator started: a ladder's own line + // already covers the inner sweep it is currently running, and a + // protocol covers both, so three of these would be one fact three times. + if let Some(line) = self.estimated_time_line(now_unix_ms()) { + entries.push(StatusEntry::Text(line)); + } if !self.message.is_empty() { entries.push(StatusEntry::Text(self.message.clone())); } @@ -8330,6 +8511,7 @@ mod tests { seed: 1, last_activity_ms: 0, stop_requested: false, + eta: Eta::new(0), }); plugin.sweep = Some(Sweep { phase: SweepPhase::Recording, @@ -8355,6 +8537,7 @@ mod tests { completed_ok: false, last_activity_ms: 0, stop_requested: false, + eta: Eta::new(0), }); let mut sink = ControlSink::default(); @@ -9094,6 +9277,7 @@ mod tests { completed_ok: false, last_activity_ms: 0, stop_requested: false, + eta: Eta::new(0), }); plugin.recording.id = "A1-sweeprow".into(); plugin.recording.stem = "A1-sweeprow_20260723-000000_p02".into(); @@ -9579,6 +9763,7 @@ mod tests { seed: 1, last_activity_ms: now_unix_ms(), stop_requested: false, + eta: Eta::new(0), }); let mut sink = ControlSink::default(); plugin.send_freq_sweep_frequency(&mut sink); @@ -9904,6 +10089,7 @@ mod tests { completed_ok: false, last_activity_ms: 0, stop_requested: false, + eta: Eta::new(0), }); // The stem carries the frequency instead of a sweep-point index. @@ -10852,6 +11038,201 @@ depth_a = 99.0 let _ = std::fs::remove_dir_all(&folder); } + /// Every text line of the status pane, joined. + fn status_text(plugin: &StageAA1Plugin) -> String { + plugin + .status_entries() + .iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) => Some(text.clone()), + _ => None, + }) + .collect::>() + .join(" | ") + } + + /// A survey started at 23:00 has to say whether it is a coffee-length or a + /// night-length one — on the button press, before it is left alone. + #[test] + fn a_protocol_says_how_long_it_will_take_before_it_starts() { + let folder = temp_folder("protocol-estimate"); + let (mut plugin, _) = protocol_plugin(&folder, TWO_POINT_PROTOCOL); + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + // Two rows of 3 s with no settle, plus the handshake around each. + let expected = format_duration(2.0 * (3.0 + crate::eta::POINT_OVERHEAD_S)); + assert!( + plugin.message.contains(&format!("≈ {expected}")), + "the button press does not say how long the survey takes: {}", + plugin.message + ); + // And the panel keeps saying it while the run walks the file. + let status = status_text(&plugin); + assert!( + status.contains("Estimated time:"), + "no estimate on the status pane: {status}" + ); + assert!( + status.contains("from the plan"), + "an estimate with nothing measured yet claims to be a measurement: {status}" + ); + + let _ = std::fs::remove_dir_all(&folder); + } + + /// The plan is what the operator asked for; the bench charges more for it + /// (handshakes, settling, a photodiode window). Once a point has actually + /// finished, the estimate has to follow the bench rather than the file — + /// otherwise a run that is running at half speed keeps promising the + /// original finish time right up to the end. + #[test] + fn the_estimate_follows_the_measured_pace_not_the_plan() { + let mut plugin = StageAA1Plugin { + duration_s: 10, + settle_s: 0.0, + sweep_count: 5, + ..StageAA1Plugin::default() + }; + let planned_point_s = plugin.planned_point_s(); + let start_ms = 1_000_000; + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + kind: SweepKind::Amplitude, + points: plugin.sweep_points(), + lock: None, + index: 0, + lease_id: LeaseId::new("test"), + lease_granted: true, + lease_req: 0, + owns_lease: true, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: true, + completed_ok: false, + last_activity_ms: start_ms, + stop_requested: false, + eta: Eta::new(start_ms), + }); + + // Nothing measured yet: five points at their planned cost. + let planned_total = 5.0 * planned_point_s; + assert!( + plugin + .estimated_time_line(start_ms) + .expect("a running sweep has an estimate") + .contains(&format_duration(planned_total)), + "the first estimate is not the plan's own time" + ); + + // The first point took twice as long as planned. Four points are left, + // so the estimate has to double them too. + let after_ms = start_ms + (2.0 * planned_point_s * 1_000.0) as u64; + if let Some(sweep) = plugin.sweep.as_mut() { + sweep.eta.point_done(after_ms, planned_point_s); + sweep.index = 1; + } + let line = plugin + .estimated_time_line(after_ms) + .expect("a running sweep has an estimate"); + assert!( + line.contains(&format_duration(8.0 * planned_point_s)), + "the estimate ignored how long the first point actually took: {line}" + ); + assert!( + !line.contains("from the plan"), + "a measured estimate still calls itself a plan: {line}" + ); + } + + /// The ladder, its inner depth sweep and a protocol are three nested runs, + /// and each knows its own remaining time. Printing all three would state + /// one fact three times, with the two inner ones — which end long before + /// the run does — reading as contradictions of the one that matters. + #[test] + fn only_the_outermost_run_states_an_estimate() { + let mut plugin = StageAA1Plugin { + duration_s: 5, + sweep_count: 3, + ..StageAA1Plugin::default() + }; + // The status pane reads the wall clock, so this run has to start on it. + let now_ms = now_unix_ms(); + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + kind: SweepKind::Amplitude, + points: plugin.sweep_points(), + lock: None, + index: 0, + lease_id: LeaseId::new("test"), + lease_granted: true, + lease_req: 0, + owns_lease: false, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: true, + completed_ok: false, + last_activity_ms: now_ms, + stop_requested: false, + eta: Eta::new(now_ms), + }); + plugin.freq_sweep = Some(FreqSweep { + phase: FreqSweepPhase::Recording, + mode: FreqSweepMode::DepthSweep, + points: vec![ + FreqSweepPoint { + frequency_hz: 10.0, + is_reference: false, + }, + FreqSweepPoint { + frequency_hz: 20.0, + is_reference: false, + }, + ], + index: 0, + lease_id: LeaseId::new("test-ladder"), + lease_granted: true, + lease_req: 0, + freq_req: 0, + freq_applied: true, + confirm_deadline_ms: 0, + skip_reason: None, + failed: Vec::new(), + recorded: 0, + order: FreqOrder::default(), + seed: 1, + last_activity_ms: now_ms, + stop_requested: false, + eta: Eta::new(now_ms), + }); + + let lines: Vec = plugin + .status_entries() + .iter() + .filter_map(|entry| match entry { + StatusEntry::Text(text) if text.starts_with("Estimated time:") => { + Some(text.clone()) + } + _ => None, + }) + .collect(); + assert_eq!(lines.len(), 1, "more than one estimate: {lines:?}"); + // And it is the ladder's: both rungs, not just the three depths of the + // rung currently running. + let both_rungs = plugin.planned_rung_s(FreqSweepMode::DepthSweep, 10.0) + + plugin.planned_rung_s(FreqSweepMode::DepthSweep, 20.0); + assert!( + lines[0].contains(&format_duration(both_rungs)), + "the inner sweep's estimate won over the ladder's: {}", + lines[0] + ); + } + /// The host writes its sensor telemetry beside the RAW and A1 moves the /// recording somewhere else, so the conditions a run was taken under used /// to be separated from the run itself at the first gather. It has to