From afa38e4fa31f0db3bea2c2ccc5e8e0d0b173ca08 Mon Sep 17 00:00:00 2001 From: Kyle Jones Date: Tue, 7 Jul 2026 09:19:01 -0700 Subject: [PATCH 01/59] Initial implementation of impression capping in MAC --- components/ads-client/Cargo.toml | 1 + .../adsclient/AdsClientTelemetry.kt | 8 + components/ads-client/src/client.rs | 125 +++++--- components/ads-client/src/client/config.rs | 6 + components/ads-client/src/clock.rs | 35 +++ components/ads-client/src/ffi.rs | 50 +++- components/ads-client/src/ffi/telemetry.rs | 46 +++ components/ads-client/src/impression_log.rs | 56 ++++ .../ads-client/src/impression_log/builder.rs | 59 ++++ .../ads-client/src/impression_log/clock.rs | 21 ++ .../impression_log/connection_initializer.rs | 84 ++++++ .../ads-client/src/impression_log/outcome.rs | 16 + .../ads-client/src/impression_log/store.rs | 275 ++++++++++++++++++ components/ads-client/src/lib.rs | 36 ++- components/ads-client/src/mars.rs | 52 +++- components/ads-client/src/mars/ad_response.rs | 27 +- components/ads-client/src/mars/capping.rs | 159 ++++++++++ 17 files changed, 984 insertions(+), 72 deletions(-) create mode 100644 components/ads-client/src/clock.rs create mode 100644 components/ads-client/src/impression_log.rs create mode 100644 components/ads-client/src/impression_log/builder.rs create mode 100644 components/ads-client/src/impression_log/clock.rs create mode 100644 components/ads-client/src/impression_log/connection_initializer.rs create mode 100644 components/ads-client/src/impression_log/outcome.rs create mode 100644 components/ads-client/src/impression_log/store.rs create mode 100644 components/ads-client/src/mars/capping.rs diff --git a/components/ads-client/Cargo.toml b/components/ads-client/Cargo.toml index 15eac5443ae..15062fc9152 100644 --- a/components/ads-client/Cargo.toml +++ b/components/ads-client/Cargo.toml @@ -15,6 +15,7 @@ context_id = { path = "../context_id" } error-support = { path = "../support/error" } parking_lot = "0.12" rusqlite = { version = "0.37.0", features = [ + "array", "functions", "bundled", "serde_json", diff --git a/components/ads-client/android/src/main/java/mozilla/appservices/adsclient/AdsClientTelemetry.kt b/components/ads-client/android/src/main/java/mozilla/appservices/adsclient/AdsClientTelemetry.kt index 856cdd4920f..46a8ca730b4 100644 --- a/components/ads-client/android/src/main/java/mozilla/appservices/adsclient/AdsClientTelemetry.kt +++ b/components/ads-client/android/src/main/java/mozilla/appservices/adsclient/AdsClientTelemetry.kt @@ -31,4 +31,12 @@ class AdsClientTelemetry : MozAdsTelemetry { override fun recordHttpCacheOutcome(label: String, value: String) { AdsClient.httpCacheOutcome[label].set(value) } + + override fun recordBuildImpressionLogError(label: String, value: String) { + AdsClient.impressionLogError[label].set(value) + } + + override fun recordImpressionLogOutcome(label: String, value: String) { + AdsClient.impressionLogOutcome[label].set(value) + } } diff --git a/components/ads-client/src/client.rs b/components/ads-client/src/client.rs index 5c818e2a4c5..2090d76f7ec 100644 --- a/components/ads-client/src/client.rs +++ b/components/ads-client/src/client.rs @@ -7,8 +7,11 @@ use std::collections::HashMap; use std::time::Duration; use crate::http_cache::{ByteSize, CachePolicy, HttpCache}; +use crate::impression_log::{ImpressionCappingPolicy, ImpressionLog}; use crate::mars::ad_request::{AdPlacementRequest, AdRequestFlags}; -use crate::mars::ad_response::{AdImage, AdResponse, AdResponseValue, AdSpoc, AdTile}; +use crate::mars::ad_response::{ + pop_query_param_from_url, AdImage, AdResponse, AdResponseValue, AdSpoc, AdTile, +}; use crate::mars::error::{RecordClickError, RecordImpressionError, ReportAdError}; use crate::mars::{MARSClient, ReportReason}; use crate::telemetry::Telemetry; @@ -85,7 +88,20 @@ where } }); - let client = MARSClient::new(environment, http_cache, telemetry.clone()); + let impression_log = + client_config + .impression_log_config + .and_then(|impression_log_config| { + match ImpressionLog::builder(impression_log_config.db_path).build() { + Ok(cache) => Some(cache), + Err(e) => { + telemetry.record(&e); + None + } + } + }); + + let client = MARSClient::new(environment, http_cache, impression_log, telemetry.clone()); telemetry.record(&ClientOperationEvent::New); Self { client, @@ -121,7 +137,7 @@ where pub fn record_impression( &self, - impression_url: Url, + mut impression_url: Url, ohttp: bool, ) -> Result<(), RecordImpressionError> { // TODO: Re-enable cache invalidation behind a Nimbus experiment. @@ -131,30 +147,9 @@ where // let _ = self.client.invalidate_cache_by_hash(&request_hash); // } - // TODO: Add count call with _cap_key for impression capping logic - let impression_url = if let Some((_, _cap_key)) = impression_url - .query_pairs() - .find(|(key, _)| key == "cap_key") - { - let mut new_url = impression_url.clone(); - new_url - .query_pairs_mut() - .clear() - .extend_pairs( - impression_url - .query_pairs() - .collect::>() - .iter() - .filter(|(key, _)| key != "cap_key"), - ) - .finish(); - new_url - } else { - impression_url - }; - + let cap_key = pop_query_param_from_url(&mut impression_url, "cap_key"); self.client - .record_impression(impression_url, ohttp) + .record_impression(impression_url, ohttp, cap_key.as_deref()) .inspect_err(|e| { self.telemetry.record(e); }) @@ -185,10 +180,17 @@ where ad_placement_requests: Vec, flags: AdRequestFlags, options: Option, + impression_capping_policy: Option, ohttp: bool, ) -> Result, RequestAdsError> { let response = self - .request_ads::(ad_placement_requests, flags, options, ohttp) + .request_ads::( + ad_placement_requests, + flags, + options, + impression_capping_policy, + ohttp, + ) .inspect_err(|e| { self.telemetry.record(e); })?; @@ -201,9 +203,16 @@ where ad_placement_requests: Vec, flags: AdRequestFlags, options: Option, + impression_capping_policy: Option, ohttp: bool, ) -> Result>, RequestAdsError> { - let result = self.request_ads::(ad_placement_requests, flags, options, ohttp); + let result = self.request_ads::( + ad_placement_requests, + flags, + options, + impression_capping_policy, + ohttp, + ); result .inspect_err(|e| { self.telemetry.record(e); @@ -219,9 +228,16 @@ where ad_placement_requests: Vec, flags: AdRequestFlags, options: Option, + impression_capping_policy: Option, ohttp: bool, ) -> Result, RequestAdsError> { - let result = self.request_ads::(ad_placement_requests, flags, options, ohttp); + let result = self.request_ads::( + ad_placement_requests, + flags, + options, + impression_capping_policy, + ohttp, + ); result .inspect_err(|e| { self.telemetry.record(e); @@ -237,6 +253,7 @@ where placements: Vec, flags: AdRequestFlags, options: Option, + impression_capping_policy: Option, ohttp: bool, ) -> Result, RequestAdsError> where @@ -244,9 +261,15 @@ where { let context_id = self.get_context_id()?; let cache_policy = options.unwrap_or_default(); - let (mut response, request_hash) = - self.client - .fetch_ads::(context_id, flags, placements, cache_policy, ohttp)?; + let impression_capping_policy = impression_capping_policy.unwrap_or_default(); + let (mut response, request_hash) = self.client.fetch_ads::( + context_id, + flags, + placements, + cache_policy, + impression_capping_policy, + ohttp, + )?; response.enrich_callbacks(&request_hash); Ok(response) } @@ -295,6 +318,7 @@ mod tests { cache_config: None, context_id_provider: None, environment: Environment::Test, + impression_log_config: None, telemetry: MozAdsTelemetryWrapper::noop(), }; let client = AdsClient::new(config); @@ -313,13 +337,19 @@ mod tests { .with_body(serde_json::to_string(&expected_response.data).unwrap()) .create(); - let mars_client = MARSClient::new(Environment::Test, None, MozAdsTelemetryWrapper::noop()); + let mars_client = MARSClient::new( + Environment::Test, + None, + None, + MozAdsTelemetryWrapper::noop(), + ); let ads_client = new_with_mars_client(mars_client); let result = ads_client.request_image_ads( make_happy_placement_requests(), AdRequestFlags::default(), None, + None, false, ); assert!(result.is_ok()); @@ -337,13 +367,19 @@ mod tests { .with_body(serde_json::to_string(&expected_response.data).unwrap()) .create(); - let mars_client = MARSClient::new(Environment::Test, None, MozAdsTelemetryWrapper::noop()); + let mars_client = MARSClient::new( + Environment::Test, + None, + None, + MozAdsTelemetryWrapper::noop(), + ); let ads_client = new_with_mars_client(mars_client); let result = ads_client.request_spoc_ads( make_happy_placement_requests(), AdRequestFlags::default(), None, + None, false, ); assert!(result.is_ok()); @@ -361,13 +397,19 @@ mod tests { .with_body(serde_json::to_string(&expected_response.data).unwrap()) .create(); - let mars_client = MARSClient::new(Environment::Test, None, MozAdsTelemetryWrapper::noop()); + let mars_client = MARSClient::new( + Environment::Test, + None, + None, + MozAdsTelemetryWrapper::noop(), + ); let ads_client = new_with_mars_client(mars_client); let result = ads_client.request_tile_ads( make_happy_placement_requests(), AdRequestFlags::default(), None, + None, false, ); assert!(result.is_ok()); @@ -399,6 +441,7 @@ mod tests { cache_config: None, context_id_provider: Some(Box::new(FixedContextId)), environment: Environment::Test, + impression_log_config: None, telemetry: MozAdsTelemetryWrapper::noop(), }; let client = AdsClient::new(config); @@ -409,6 +452,7 @@ mod tests { make_happy_placement_requests(), AdRequestFlags::default(), None, + None, false, ); assert!(result.is_ok()); @@ -418,7 +462,12 @@ mod tests { #[test] fn test_record_impression_removes_cap_key() { viaduct_dev::init_backend_dev(); - let mars_client = MARSClient::new(Environment::Test, None, MozAdsTelemetryWrapper::noop()); + let mars_client = MARSClient::new( + Environment::Test, + None, + None, + MozAdsTelemetryWrapper::noop(), + ); let ads_client = new_with_mars_client(mars_client); let base_url = mockito::server_url(); @@ -452,6 +501,7 @@ mod tests { let mars_client = MARSClient::new( Environment::Test, Some(cache), + None, MozAdsTelemetryWrapper::noop(), ); let ads_client = new_with_mars_client(mars_client); @@ -470,6 +520,7 @@ mod tests { make_happy_placement_requests(), AdRequestFlags::default(), None, + None, false, ) .unwrap(); @@ -484,6 +535,7 @@ mod tests { make_happy_placement_requests(), AdRequestFlags::default(), None, + None, false, ) .unwrap(); @@ -495,6 +547,7 @@ mod tests { make_happy_placement_requests(), AdRequestFlags::default(), Some(CachePolicy::default()), + None, false, ) .unwrap(); diff --git a/components/ads-client/src/client/config.rs b/components/ads-client/src/client/config.rs index 7c86c241418..937595f0ae9 100644 --- a/components/ads-client/src/client/config.rs +++ b/components/ads-client/src/client/config.rs @@ -13,6 +13,7 @@ where pub cache_config: Option, pub context_id_provider: Option>, pub environment: Environment, + pub impression_log_config: Option, pub telemetry: T, } @@ -22,3 +23,8 @@ pub struct AdsCacheConfig { pub default_cache_ttl_seconds: Option, pub max_size_mib: Option, } + +#[derive(Clone, Debug)] +pub struct ImpressionLogConfig { + pub db_path: String, +} diff --git a/components/ads-client/src/clock.rs b/components/ads-client/src/clock.rs new file mode 100644 index 00000000000..4ad821fc5b7 --- /dev/null +++ b/components/ads-client/src/clock.rs @@ -0,0 +1,35 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub trait Clock: Send + Sync + 'static { + fn now_epoch_seconds(&self) -> i64; + #[cfg(test)] + fn advance(&self, secs: i64); +} + +#[cfg(test)] +pub struct TestClock { + now: std::sync::atomic::AtomicI64, +} + +#[cfg(test)] +impl TestClock { + pub fn new(start: i64) -> Self { + Self { + now: std::sync::atomic::AtomicI64::new(start), + } + } +} + +#[cfg(test)] +impl Clock for TestClock { + fn now_epoch_seconds(&self) -> i64 { + self.now.load(std::sync::atomic::Ordering::Relaxed) + } + + fn advance(&self, secs: i64) { + self.now + .fetch_add(secs, std::sync::atomic::Ordering::Relaxed); + } +} diff --git a/components/ads-client/src/ffi.rs b/components/ads-client/src/ffi.rs index ba50352fdc1..0b5cbdf7d46 100644 --- a/components/ads-client/src/ffi.rs +++ b/components/ads-client/src/ffi.rs @@ -8,10 +8,11 @@ pub mod telemetry; use std::sync::Arc; -use crate::client::config::{AdsCacheConfig, AdsClientConfig}; +use crate::client::config::{AdsCacheConfig, AdsClientConfig, ImpressionLogConfig}; use crate::client::{AdsClient, ContextIdProvider}; use crate::ffi::telemetry::MozAdsTelemetryWrapper; use crate::http_cache::CachePolicy; +use crate::impression_log::ImpressionCappingPolicy; use crate::mars::ad_request::{ AdContentCategory, AdPlacementRequest, AdRequestFlags, IABContentTaxonomy, }; @@ -58,6 +59,7 @@ impl From for Box { #[derive(Default, uniffi::Record)] pub struct MozAdsRequestOptions { pub cache_policy: Option, + pub impression_capping_policy: Option, #[uniffi(default)] pub flags: HashMap, #[uniffi(default = false)] @@ -106,6 +108,7 @@ struct MozAdsClientBuilderInner { cache_config: Option, context_id_provider: Option>, environment: Option, + impression_log_config: Option, telemetry: Option>, } @@ -132,6 +135,7 @@ impl MozAdsClientBuilder { .map(MozAdsContextIdProviderWrapper::new) .map(Into::into), environment: inner.environment.unwrap_or_default().into(), + impression_log_config: inner.impression_log_config.clone().map(Into::into), telemetry: inner .telemetry .clone() @@ -186,6 +190,11 @@ pub struct MozAdsCacheConfig { pub max_size_mib: Option, } +#[derive(Clone, uniffi::Record)] +pub struct MozAdsImpressionLogConfig { + pub db_path: String, +} + #[derive(Debug, PartialEq, uniffi::Record)] pub struct MozAdsContentCategory { pub categories: Vec, @@ -232,6 +241,13 @@ pub enum MozAdsCacheMode { NetworkFirst, } +#[derive(Clone, Copy, Debug, Default, uniffi::Enum)] +pub enum MozAdsImpressionCappingPolicy { + #[default] + TelemetryOnly, + ImpressionCapEnforced, +} + #[derive(Debug, PartialEq, uniffi::Record)] pub struct MozAdsImage { pub alt_text: Option, @@ -421,6 +437,17 @@ impl From for CachePolicy { } } +impl From for ImpressionCappingPolicy { + fn from(policy: MozAdsImpressionCappingPolicy) -> Self { + match policy { + MozAdsImpressionCappingPolicy::TelemetryOnly => ImpressionCappingPolicy::TelemetryOnly, + MozAdsImpressionCappingPolicy::ImpressionCapEnforced => { + ImpressionCappingPolicy::ImpressionCapEnforced + } + } + } +} + impl From<&MozAdsIABContent> for AdContentCategory { fn from(content: &MozAdsIABContent) -> Self { Self { @@ -436,12 +463,21 @@ impl From<&MozAdsRequestOptions> for AdRequestFlags { } } -impl From for CachePolicy { - fn from(options: MozAdsRequestOptions) -> Self { +impl From<&MozAdsRequestOptions> for CachePolicy { + fn from(options: &MozAdsRequestOptions) -> Self { options.cache_policy.map(Into::into).unwrap_or_default() } } +impl From<&MozAdsRequestOptions> for ImpressionCappingPolicy { + fn from(options: &MozAdsRequestOptions) -> Self { + options + .impression_capping_policy + .map(Into::into) + .unwrap_or_default() + } +} + impl From for AdsCacheConfig { fn from(config: MozAdsCacheConfig) -> Self { Self { @@ -452,6 +488,14 @@ impl From for AdsCacheConfig { } } +impl From for ImpressionLogConfig { + fn from(config: MozAdsImpressionLogConfig) -> Self { + Self { + db_path: config.db_path, + } + } +} + impl From<&MozAdsPlacementRequest> for AdPlacementRequest { fn from(request: &MozAdsPlacementRequest) -> Self { Self { diff --git a/components/ads-client/src/ffi/telemetry.rs b/components/ads-client/src/ffi/telemetry.rs index 70972bf66f7..7b905ea28d9 100644 --- a/components/ads-client/src/ffi/telemetry.rs +++ b/components/ads-client/src/ffi/telemetry.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use crate::client::error::RequestAdsError; use crate::client::ClientOperationEvent; use crate::http_cache::{CacheOutcome, HttpCacheBuilderError}; +use crate::impression_log::{ImpressionLogBuilderError, ImpressionLogOutcome}; use crate::mars::error::{RecordClickError, RecordImpressionError, ReportAdError}; use crate::telemetry::Telemetry; @@ -19,6 +20,8 @@ pub trait MozAdsTelemetry: Send + Sync { fn record_client_operation_total(&self, label: String); fn record_deserialization_error(&self, label: String, value: String); fn record_http_cache_outcome(&self, label: String, value: String); + fn record_build_impression_log_error(&self, label: String, value: String); + fn record_impression_log_outcome(&self, label: String, value: String); } pub struct NoopMozAdsTelemetry; @@ -29,6 +32,8 @@ impl MozAdsTelemetry for NoopMozAdsTelemetry { fn record_client_operation_total(&self, _label: String) {} fn record_deserialization_error(&self, _label: String, _value: String) {} fn record_http_cache_outcome(&self, _label: String, _value: String) {} + fn record_build_impression_log_error(&self, _label: String, _value: String) {} + fn record_impression_log_outcome(&self, _label: String, _value: String) {} } #[derive(Clone)] @@ -125,6 +130,47 @@ impl Telemetry for MozAdsTelemetryWrapper { ); return; } + if let Some(impression_log_builder_error) = + event.downcast_ref::() + { + self.inner.record_build_impression_log_error( + match impression_log_builder_error { + ImpressionLogBuilderError::EmptyDbPath => "empty_db_path".to_string(), + ImpressionLogBuilderError::Database(_) => "database_error".to_string(), + }, + format!("{}", impression_log_builder_error), + ); + return; + } + if let Some(impression_log_outcome) = event.downcast_ref::() { + self.inner.record_impression_log_outcome( + match impression_log_outcome { + ImpressionLogOutcome::RetainImpressionsFailed(_) => { + "retain_impressions_failed".to_string() + } + ImpressionLogOutcome::RecordImpressionFailed(_) => { + "record_impression_failed".to_string() + } + ImpressionLogOutcome::CountImpressionsFailed(_) => { + "count_impressions_failed".to_string() + } + ImpressionLogOutcome::ImpressionCapHit => "impression_cap_hit".to_string(), + ImpressionLogOutcome::ImpressionCapEnforced => { + "impression_cap_enforced".to_string() + } + ImpressionLogOutcome::ImpressionCapNotEnforced => { + "impression_cap_not_enforced".to_string() + } + }, + match impression_log_outcome { + ImpressionLogOutcome::RetainImpressionsFailed(e) + | ImpressionLogOutcome::RecordImpressionFailed(e) + | ImpressionLogOutcome::CountImpressionsFailed(e) => e.to_string(), + _ => "".to_string(), + }, + ); + return; + } eprintln!("Unsupported telemetry event type: {:?}", event.type_id()); #[cfg(test)] panic!("Unsupported telemetry event type: {:?}", event.type_id()); diff --git a/components/ads-client/src/impression_log.rs b/components/ads-client/src/impression_log.rs new file mode 100644 index 00000000000..0cc1ebeb809 --- /dev/null +++ b/components/ads-client/src/impression_log.rs @@ -0,0 +1,56 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +mod builder; +mod clock; +mod connection_initializer; +mod outcome; +mod store; + +use std::collections::HashMap; +use std::path::Path; + +use self::builder::ImpressionLogBuilder; +use self::store::ImpressionLogStore; + +pub use self::builder::ImpressionLogBuilderError; +pub use self::outcome::ImpressionLogOutcome; + +#[derive(Clone, Copy, Debug, Default)] +pub enum ImpressionCappingPolicy { + #[default] + TelemetryOnly, + ImpressionCapEnforced, +} + +pub struct ImpressionLog { + store: ImpressionLogStore, +} + +impl ImpressionLog { + pub fn builder>(db_path: P) -> ImpressionLogBuilder { + ImpressionLogBuilder::new(db_path.as_ref()) + } + + pub fn record_impression(&self, cap_key: &str) -> Result<(), rusqlite::Error> { + self.store.record_impression(cap_key)?; + Ok(()) + } + + pub fn count_impressions( + &self, + cap_keys: impl IntoIterator, + ) -> Result, rusqlite::Error> { + let counts = self.store.count_impressions(cap_keys)?; + Ok(counts) + } + + pub fn retain_impressions( + &self, + cap_keys: impl IntoIterator, + ) -> Result<(), rusqlite::Error> { + self.store.retain_impressions(cap_keys)?; + Ok(()) + } +} diff --git a/components/ads-client/src/impression_log/builder.rs b/components/ads-client/src/impression_log/builder.rs new file mode 100644 index 00000000000..b277b53ec66 --- /dev/null +++ b/components/ads-client/src/impression_log/builder.rs @@ -0,0 +1,59 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::path::PathBuf; + +use crate::impression_log::connection_initializer::ImpressionLogConnectionInitializer; +use crate::impression_log::store::ImpressionLogStore; +use crate::impression_log::ImpressionLog; + +use rusqlite::Connection; +use sql_support::open_database; + +#[derive(Debug, thiserror::Error)] +pub enum ImpressionLogBuilderError { + #[error("Database path cannot be empty")] + EmptyDbPath, + #[error("Database error: {0}")] + Database(#[from] open_database::Error), +} + +pub struct ImpressionLogBuilder { + db_path: PathBuf, +} + +impl ImpressionLogBuilder { + pub fn new(db_path: impl Into) -> Self { + Self { + db_path: db_path.into(), + } + } + + fn validate(&self) -> Result<(), ImpressionLogBuilderError> { + if self.db_path.to_string_lossy().trim().is_empty() { + return Err(ImpressionLogBuilderError::EmptyDbPath); + } + + Ok(()) + } + + fn open_connection(&self) -> Result { + let initializer = ImpressionLogConnectionInitializer {}; + let conn = if cfg!(test) { + open_database::open_memory_database(&initializer)? + } else { + open_database::open_database(&self.db_path, &initializer)? + }; + Ok(conn) + } + + pub fn build(&self) -> Result { + self.validate()?; + + let conn = self.open_connection()?; + let store = ImpressionLogStore::new(conn); + + Ok(ImpressionLog { store }) + } +} diff --git a/components/ads-client/src/impression_log/clock.rs b/components/ads-client/src/impression_log/clock.rs new file mode 100644 index 00000000000..3d5e521d7ab --- /dev/null +++ b/components/ads-client/src/impression_log/clock.rs @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::clock::Clock; + +pub struct ImpressionLogClock; + +impl Clock for ImpressionLogClock { + fn now_epoch_seconds(&self) -> i64 { + chrono::Utc::now().timestamp() + } + + #[cfg(test)] + fn advance(&self, _secs: i64) { + panic!( + "You cannot advance a non-test clock. + Be sure to build the log or store with the test clock for time-dependent tests." + ) + } +} diff --git a/components/ads-client/src/impression_log/connection_initializer.rs b/components/ads-client/src/impression_log/connection_initializer.rs new file mode 100644 index 00000000000..14060844652 --- /dev/null +++ b/components/ads-client/src/impression_log/connection_initializer.rs @@ -0,0 +1,84 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use rusqlite::{vtab::array, Connection}; +use sql_support::open_database; +use std::time::Duration; + +pub struct ImpressionLogConnectionInitializer {} + +impl open_database::ConnectionInitializer for ImpressionLogConnectionInitializer { + const NAME: &'static str = "impression_log"; + const END_VERSION: u32 = 1; + + fn prepare(&self, conn: &Connection, _db_empty: bool) -> open_database::Result<()> { + conn.execute_batch("PRAGMA journal_mode=wal;")?; + array::load_module(conn)?; + conn.busy_timeout(Duration::from_secs(5))?; + Ok(()) + } + + fn init(&self, tx: &rusqlite::Transaction<'_>) -> open_database::Result<()> { + const SCHEMA: &str = " + CREATE TABLE IF NOT EXISTS impression_log ( + cap_key TEXT NOT NULL, + recorded_at INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_impression_log_pk ON impression_log(cap_key, recorded_at); + "; + // If the schema fails to initialize, it might be corrupted or outdated so we drop the table and try again + if tx.execute_batch(SCHEMA).is_err() { + tx.execute_batch("DROP TABLE IF EXISTS impression_log")?; + tx.execute_batch(SCHEMA)?; + } + Ok(()) + } + + fn upgrade_from( + &self, + conn: &rusqlite::Transaction<'_>, + version: u32, + ) -> open_database::Result<()> { + match version { + 0 => self.init(conn), + _ => Err(open_database::Error::IncompatibleVersion(version)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::Connection; + use sql_support::open_database::ConnectionInitializer; + + #[test] + fn test_corrupted_schema_is_recreated() { + let mut conn = Connection::open_in_memory().unwrap(); + let initializer = ImpressionLogConnectionInitializer {}; + + // Create a corrupted table missing needed index columns + conn.execute_batch("CREATE TABLE impression_log (cap_key TEXT);") + .unwrap(); + + // Run init - should drop the corrupted table and recreate it properly + let tx = conn.transaction().unwrap(); + initializer.init(&tx).unwrap(); + tx.commit().unwrap(); + + // Verify the table was recreated with correct schema by checking column count + let column_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('impression_log')", + [], + |row| row.get(0), + ) + .unwrap(); + + assert!( + column_count > 1, + "Table should have more than 1 column after recreation" + ); + } +} diff --git a/components/ads-client/src/impression_log/outcome.rs b/components/ads-client/src/impression_log/outcome.rs new file mode 100644 index 00000000000..3cfa8ad6ebc --- /dev/null +++ b/components/ads-client/src/impression_log/outcome.rs @@ -0,0 +1,16 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public +* License, v. 2.0. If a copy of the MPL was not distributed with this +* file, You can obtain one at http://mozilla.org/MPL/2.0/. +*/ + +#[derive(Debug)] +pub enum ImpressionLogOutcome { + // DB errors + RecordImpressionFailed(rusqlite::Error), // Failed to record impression to log + CountImpressionsFailed(rusqlite::Error), // Failed to get counts of impressions from log + RetainImpressionsFailed(rusqlite::Error), // Failed to clear impressions from log + // Simple events + ImpressionCapHit, // Impression limit reached for a cap_key + ImpressionCapEnforced, // Ad filtered from results due to CappingPolicy + ImpressionCapNotEnforced, // Ad remained in results due to CappingPolicy +} diff --git a/components/ads-client/src/impression_log/store.rs b/components/ads-client/src/impression_log/store.rs new file mode 100644 index 00000000000..75abdf8773e --- /dev/null +++ b/components/ads-client/src/impression_log/store.rs @@ -0,0 +1,275 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::collections::HashMap; +use std::{rc::Rc, sync::Arc}; + +use parking_lot::Mutex; +use rusqlite::{params, types::Value, Connection, Result as SqliteResult, Row}; +use sql_support::ConnExt; + +use crate::clock::Clock; +use crate::impression_log::clock::ImpressionLogClock; + +const SECONDS_IN_DAY: i64 = 60 * 60 * 24; + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum FaultKind { + None, + RecordImpression, + CountImpressions, + RetainImpressions, +} + +pub struct ImpressionLogStore { + conn: Mutex, + clock: Arc, + #[cfg(test)] + fault: Mutex, +} + +fn as_sql_values(raw_values: impl IntoIterator) -> Rc> { + Rc::new( + raw_values + .into_iter() + .map(|s| Value::Text(s.to_string())) + .collect(), + ) +} + +impl ImpressionLogStore { + /// Create new store from connection + pub fn new(conn: Connection) -> Self { + Self { + conn: Mutex::new(conn), + clock: Arc::new(ImpressionLogClock), + #[cfg(test)] + fault: parking_lot::Mutex::new(FaultKind::None), + } + } + + /// Add impression to log. + pub fn record_impression(&self, cap_key: &str) -> SqliteResult { + #[cfg(test)] + if *self.fault.lock() == FaultKind::RecordImpression { + return Err(Self::forced_fault_error("forced record_impression failure")); + } + + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO impression_log (cap_key, recorded_at) + VALUES (?1, ?2) + ON CONFLICT (cap_key, recorded_at) DO NOTHING;", + params![cap_key, self.clock.now_epoch_seconds()], + ) + } + + /// Counts impressions in log. + pub fn count_impressions( + &self, + cap_keys: impl IntoIterator, + ) -> SqliteResult> { + #[cfg(test)] + if *self.fault.lock() == FaultKind::CountImpressions { + return Err(Self::forced_fault_error("forced count_impressions failure")); + } + + let conn = self.conn.lock(); + conn.query_rows_into( + "SELECT value, COUNT(*) + FROM impression_log + INNER JOIN rarray(?1) ON value = cap_key + WHERE recorded_at > ?2 + GROUP BY value;", + params![ + as_sql_values(cap_keys), + self.clock.now_epoch_seconds() - SECONDS_IN_DAY, + ], + |row: &Row<'_>| Ok((row.get::<_, String>(0)?, row.get::<_, u32>(1)?)), + ) + } + + /// Removes other impressions from log. + pub fn retain_impressions( + &self, + cap_keys: impl IntoIterator, + ) -> SqliteResult { + #[cfg(test)] + if *self.fault.lock() == FaultKind::RetainImpressions { + return Err(Self::forced_fault_error( + "forced retain_impressions failure", + )); + } + + let conn = self.conn.lock(); + conn.execute( + "DELETE FROM impression_log + WHERE rowid NOT IN ( + SELECT impression_log.rowid + FROM impression_log + INNER JOIN rarray(?1) ON value = cap_key + );", + params![as_sql_values(cap_keys)], + ) + } + + #[cfg(test)] + pub fn new_with_test_clock(conn: Connection) -> Self { + use crate::clock::TestClock; + + Self { + conn: Mutex::new(conn), + clock: Arc::new(TestClock::new(chrono::Utc::now().timestamp())), + fault: parking_lot::Mutex::new(FaultKind::None), + } + } + + #[cfg(test)] + fn set_fault(&self, kind: FaultKind) { + *self.fault.lock() = kind; + } + + #[cfg(test)] + fn forced_fault_error(msg: &str) -> rusqlite::Error { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::InternalMalfunction, + extended_code: 0, + }, + Some(msg.to_string()), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::impression_log::connection_initializer::ImpressionLogConnectionInitializer; + use sql_support::open_database; + + fn create_test_store() -> ImpressionLogStore { + let initializer = ImpressionLogConnectionInitializer {}; + let conn = open_database::open_memory_database(&initializer) + .expect("failed to open memory cache db"); + ImpressionLogStore::new_with_test_clock(conn) + } + + #[test] + fn test_record_impression_fault_injection() { + let store = create_test_store(); + store.set_fault(FaultKind::RecordImpression); + + let err = store.record_impression("test").unwrap_err(); + + match err { + rusqlite::Error::SqliteFailure(_, Some(msg)) => { + assert!(msg.contains("forced record_impression failure")); + } + other => panic!("unexpected error: {other:?}"), + } + } + + #[test] + fn test_count_impressions_fault_injection() { + let store = create_test_store(); + store.set_fault(FaultKind::CountImpressions); + + let err = store.count_impressions(["test_cap_key"]).unwrap_err(); + + match err { + rusqlite::Error::SqliteFailure(_, Some(msg)) => { + assert!(msg.contains("forced count_impressions failure")); + } + other => panic!("unexpected error: {other:?}"), + } + } + + #[test] + fn test_retain_impressions_fault_injection() { + let store = create_test_store(); + store.set_fault(FaultKind::RetainImpressions); + + let err = store.retain_impressions(["test_cap_key"]).unwrap_err(); + + match err { + rusqlite::Error::SqliteFailure(_, Some(msg)) => { + assert!(msg.contains("forced retain_impressions failure")); + } + other => panic!("unexpected error: {other:?}"), + } + } + + #[test] + fn test_impression_roundtrip_simple() { + let store = create_test_store(); + + store.record_impression("test_cap_key").unwrap(); + + assert_eq!( + store.count_impressions(["test_cap_key"]).unwrap(), + HashMap::from([("test_cap_key".into(), 1)]) + ) + } + + #[test] + fn test_impression_roundtrip_multiple() { + let store = create_test_store(); + + store.record_impression("test_cap_key1").unwrap(); + store.record_impression("test_cap_key2").unwrap(); + + assert_eq!( + store + .count_impressions(["test_cap_key1", "test_cap_key2"]) + .unwrap(), + HashMap::from([("test_cap_key1".into(), 1), ("test_cap_key2".into(), 1)]) + ) + } + + #[test] + fn test_impression_roundtrip_duplicate() { + let store = create_test_store(); + + store.record_impression("test_cap_key").unwrap(); + store.record_impression("test_cap_key").unwrap(); + + assert_eq!( + store.count_impressions(["test_cap_key"]).unwrap(), + HashMap::from([("test_cap_key".into(), 1)]) + ) + } + + #[test] + fn test_impression_roundtrip_over_time() { + let store = create_test_store(); + + store.record_impression("test_cap_key").unwrap(); + store.clock.advance(SECONDS_IN_DAY); + store.record_impression("test_cap_key").unwrap(); + store.clock.advance(1); + store.record_impression("test_cap_key").unwrap(); + + assert_eq!( + store.count_impressions(["test_cap_key"]).unwrap(), + HashMap::from([("test_cap_key".into(), 2)]) + ) + } + + #[test] + fn test_impression_cleanup() { + let store = create_test_store(); + + store.record_impression("test_cap_key1").unwrap(); + store.record_impression("test_cap_key2").unwrap(); + store.retain_impressions(["test_cap_key1"]).unwrap(); + + assert_eq!( + store + .count_impressions(["test_cap_key1", "test_cap_key2"]) + .unwrap(), + HashMap::from([("test_cap_key1".into(), 1)]) + ) + } +} diff --git a/components/ads-client/src/lib.rs b/components/ads-client/src/lib.rs index 2088ee16cb6..ea54acf407d 100644 --- a/components/ads-client/src/lib.rs +++ b/components/ads-client/src/lib.rs @@ -13,11 +13,14 @@ use url::Url as AdsClientUrl; use client::AdsClient; use http_cache::CachePolicy; +use impression_log::ImpressionCappingPolicy; use mars::ad_request::{AdPlacementRequest, AdRequestFlags}; mod client; +mod clock; mod ffi; pub mod http_cache; +pub mod impression_log; mod mars; pub mod telemetry; @@ -114,9 +117,16 @@ impl MozAdsClient { let options = options.unwrap_or_default(); let flags = AdRequestFlags::from(&options); let ohttp = options.ohttp; - let cache_policy: CachePolicy = options.into(); + let cache_policy = CachePolicy::from(&options); + let impression_capping_policy = ImpressionCappingPolicy::from(&options); let response = inner - .request_image_ads(requests, flags, Some(cache_policy), ohttp) + .request_image_ads( + requests, + flags, + Some(cache_policy), + Some(impression_capping_policy), + ohttp, + ) .map_err(ComponentError::RequestAds)?; Ok(response.into_iter().map(|(k, v)| (k, v.into())).collect()) } @@ -133,9 +143,16 @@ impl MozAdsClient { let options = options.unwrap_or_default(); let flags = AdRequestFlags::from(&options); let ohttp = options.ohttp; - let cache_policy: CachePolicy = options.into(); + let cache_policy = CachePolicy::from(&options); + let impression_capping_policy = ImpressionCappingPolicy::from(&options); let response = inner - .request_spoc_ads(requests, flags, Some(cache_policy), ohttp) + .request_spoc_ads( + requests, + flags, + Some(cache_policy), + Some(impression_capping_policy), + ohttp, + ) .map_err(ComponentError::RequestAds)?; Ok(response .into_iter() @@ -155,9 +172,16 @@ impl MozAdsClient { let options = options.unwrap_or_default(); let flags = AdRequestFlags::from(&options); let ohttp = options.ohttp; - let cache_policy: CachePolicy = options.into(); + let cache_policy = CachePolicy::from(&options); + let impression_capping_policy = ImpressionCappingPolicy::from(&options); let response = inner - .request_tile_ads(requests, flags, Some(cache_policy), ohttp) + .request_tile_ads( + requests, + flags, + Some(cache_policy), + Some(impression_capping_policy), + ohttp, + ) .map_err(ComponentError::RequestAds)?; Ok(response.into_iter().map(|(k, v)| (k, v.into())).collect()) } diff --git a/components/ads-client/src/mars.rs b/components/ads-client/src/mars.rs index 59e044212bc..147b313629b 100644 --- a/components/ads-client/src/mars.rs +++ b/components/ads-client/src/mars.rs @@ -5,6 +5,7 @@ pub mod ad_request; pub mod ad_response; +mod capping; pub mod environment; pub mod error; mod preflight; @@ -17,6 +18,7 @@ pub use report_reason::ReportReason; use self::{ ad_request::{AdPlacementRequest, AdRequest, AdRequestFlags}, ad_response::{AdResponse, AdResponseValue}, + capping::MARSCapping, error::{ CallbackRequestError, FetchAdsError, RecordClickError, RecordImpressionError, ReportAdError, }, @@ -25,8 +27,9 @@ use self::{ }; use crate::{ http_cache::{HttpCache, RequestHash}, + impression_log::ImpressionLog, telemetry::Telemetry, - CachePolicy, + CachePolicy, ImpressionCappingPolicy, }; use url::Url; use viaduct::{Headers, Request}; @@ -38,18 +41,26 @@ where environment: Environment, telemetry: T, transport: MARSTransport, + capping: MARSCapping, } impl MARSClient where T: Clone + Telemetry, { - pub fn new(environment: Environment, http_cache: Option, telemetry: T) -> Self { + pub fn new( + environment: Environment, + http_cache: Option, + impression_log: Option, + telemetry: T, + ) -> Self { let transport = MARSTransport::new(http_cache, telemetry.clone()); + let capping = MARSCapping::new(impression_log, telemetry.clone()); Self { environment, telemetry, transport, + capping, } } @@ -63,6 +74,7 @@ where flags: AdRequestFlags, placements: Vec, cache_policy: CachePolicy, + impression_capping_policy: ImpressionCappingPolicy, ohttp: bool, ) -> Result<(AdResponse, RequestHash), FetchAdsError> where @@ -80,7 +92,10 @@ where let response = self.transport.send(ad_request, &cache_policy, ohttp)?; let ads = AdResponse::::parse(response.json()?, &self.telemetry)?; - Ok((ads, request_hash)) + let filtered_ads = self + .capping + .apply_impression_capping(ads, &impression_capping_policy); + Ok((filtered_ads, request_hash)) } // TODO: Remove this allow(dead_code) when cache invalidation is re-enabled behind Nimbus experiment @@ -100,7 +115,11 @@ where &self, callback: Url, ohttp: bool, + cap_key: Option<&str>, ) -> Result<(), RecordImpressionError> { + if let Some(cap_key) = cap_key { + self.capping.record_impression(cap_key); + } Ok(self.make_callback_request(callback, ohttp)?) } @@ -151,10 +170,14 @@ mod tests { }; use mockito::mock; - fn make_test_client(http_cache: Option) -> MARSClient { + fn make_test_client( + http_cache: Option, + impression_log: Option, + ) -> MARSClient { MARSClient::new( Environment::Test, http_cache, + impression_log, MozAdsTelemetryWrapper::noop(), ) } @@ -165,13 +188,13 @@ mod tests { let m = mock("GET", "/impression_callback_url") .with_status(200) .create(); - let client = make_test_client(None); + let client = make_test_client(None, None); let url = Url::parse(&format!( "{}/impression_callback_url", &mockito::server_url() )) .unwrap(); - let result = client.record_impression(url, false); + let result = client.record_impression(url, false, None); assert!(result.is_ok()); m.assert(); } @@ -181,7 +204,7 @@ mod tests { viaduct_dev::init_backend_dev(); let m = mock("GET", "/click_callback_url").with_status(200).create(); - let client = make_test_client(None); + let client = make_test_client(None, None); let url = Url::parse(&format!("{}/click_callback_url", &mockito::server_url())).unwrap(); let result = client.record_click(url, false); assert!(result.is_ok()); @@ -199,7 +222,7 @@ mod tests { .with_status(200) .create(); - let client = make_test_client(None); + let client = make_test_client(None, None); let url = Url::parse(&format!( "{}/report_ad_callback_url", &mockito::server_url() @@ -222,13 +245,14 @@ mod tests { .with_body(serde_json::to_string(&expected_response.data).unwrap()) .create(); - let client = make_test_client(None); + let client = make_test_client(None, None); let result = client.fetch_ads::( TEST_CONTEXT_ID.to_string(), AdRequestFlags::default(), make_happy_placement_requests(), CachePolicy::default(), + ImpressionCappingPolicy::default(), false, ); assert!(result.is_ok()); @@ -253,7 +277,7 @@ mod tests { .max_size(crate::http_cache::ByteSize::mib(1)) .build() .unwrap(); - let client = make_test_client(Some(cache)); + let client = make_test_client(Some(cache), None); // First call should be a miss then warm the cache let (response1, _) = client @@ -262,6 +286,7 @@ mod tests { AdRequestFlags::default(), make_happy_placement_requests(), CachePolicy::default(), + ImpressionCappingPolicy::default(), false, ) .unwrap(); @@ -274,6 +299,7 @@ mod tests { AdRequestFlags::default(), make_happy_placement_requests(), CachePolicy::default(), + ImpressionCappingPolicy::default(), false, ) .unwrap(); @@ -290,7 +316,7 @@ mod tests { .build() .unwrap(); - let client = make_test_client(Some(cache)); + let client = make_test_client(Some(cache), None); let callback_url = Url::parse(&format!("{}/click", mockito::server_url())).unwrap(); let m = mock("GET", "/click").with_status(200).create(); @@ -309,12 +335,12 @@ mod tests { .build() .unwrap(); - let client = make_test_client(Some(cache)); + let client = make_test_client(Some(cache), None); let callback_url = Url::parse(&format!("{}/impression", mockito::server_url())).unwrap(); let m = mock("GET", "/impression").with_status(200).create(); - let result = client.record_impression(callback_url, false); + let result = client.record_impression(callback_url, false, None); assert!(result.is_ok()); m.assert(); } diff --git a/components/ads-client/src/mars/ad_response.rs b/components/ads-client/src/mars/ad_response.rs index 5e057b7606b..351f298c2bd 100644 --- a/components/ads-client/src/mars/ad_response.rs +++ b/components/ads-client/src/mars/ad_response.rs @@ -47,7 +47,7 @@ impl AdResponse { let hash_str = request_hash.to_string(); for (placement_id, ads) in self.data.iter_mut() { for (position, ad) in ads.iter_mut().enumerate() { - let cap_key = ad.cap_key(); + let cap_key = ad.cap_pair().map(|(cap_key, _)| cap_key.to_owned()); let callbacks = ad.callbacks_mut(); callbacks .click @@ -82,15 +82,13 @@ impl AdResponse { } } -// TODO: Remove this allow(dead_code) when cache invalidation is re-enabled behind Nimbus experiment -#[allow(dead_code)] -pub fn pop_request_hash_from_url(url: &mut Url) -> Option { +pub fn pop_query_param_from_url(url: &mut Url, query_param: &str) -> Option { let mut request_hash = None; let mut query = url::form_urlencoded::Serializer::new(String::new()); for (key, value) in url.query_pairs() { - if key == "request_hash" { - request_hash = Some(RequestHash::from(value.as_ref())); + if key == query_param { + request_hash = Some(value.into_owned()); } else { query.append_pair(&key, &value); } @@ -102,6 +100,7 @@ pub fn pop_request_hash_from_url(url: &mut Url) -> Option { } else { url.set_query(Some(&query_string)); } + request_hash } @@ -163,7 +162,7 @@ pub struct AdCallbacks { pub trait AdResponseValue: DeserializeOwned { fn callbacks_mut(&mut self) -> &mut AdCallbacks; - fn cap_key(&self) -> Option { + fn cap_pair(&self) -> Option<(&str, &u32)> { None } } @@ -179,8 +178,8 @@ impl AdResponseValue for AdSpoc { &mut self.callbacks } - fn cap_key(&self) -> Option { - Some(self.caps.cap_key.clone()) + fn cap_pair(&self) -> Option<(&str, &u32)> { + Some((&self.caps.cap_key, &self.caps.day)) } } @@ -645,21 +644,21 @@ mod tests { } #[test] - fn test_pop_request_hash_from_url() { + fn test_pop_query_param_from_url() { let mut url_with_hash = Url::parse("https://example.com/callback?request_hash=abc123def456&other=param") .unwrap(); - let extracted = pop_request_hash_from_url(&mut url_with_hash); - assert_eq!(extracted, Some(RequestHash::from("abc123def456"))); + let extracted = pop_query_param_from_url(&mut url_with_hash, "request_hash"); + assert_eq!(extracted, Some("abc123def456".into())); assert_eq!(url_with_hash.query(), Some("other=param")); let mut url_without_hash = Url::parse("https://example.com/callback?other=param").unwrap(); - let extracted_none = pop_request_hash_from_url(&mut url_without_hash); + let extracted_none = pop_query_param_from_url(&mut url_without_hash, "request_hash"); assert_eq!(extracted_none, None); assert_eq!(url_without_hash.query(), Some("other=param")); let mut url_no_query = Url::parse("https://example.com/callback").unwrap(); - let extracted_empty = pop_request_hash_from_url(&mut url_no_query); + let extracted_empty = pop_query_param_from_url(&mut url_no_query, "request_hash"); assert_eq!(extracted_empty, None); assert_eq!(url_no_query.query(), None); } diff --git a/components/ads-client/src/mars/capping.rs b/components/ads-client/src/mars/capping.rs new file mode 100644 index 00000000000..ab4d3b65e49 --- /dev/null +++ b/components/ads-client/src/mars/capping.rs @@ -0,0 +1,159 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::collections::{HashMap, HashSet}; + +use crate::{ + impression_log::{ImpressionLog, ImpressionLogOutcome}, + mars::ad_response::{AdResponse, AdResponseValue}, + telemetry::Telemetry, + ImpressionCappingPolicy, +}; + +pub struct MARSCapping { + impression_log: Option, + telemetry: T, +} + +impl MARSCapping { + pub fn new(impression_log: Option, telemetry: T) -> Self { + Self { + impression_log, + telemetry, + } + } + + pub fn record_impression(&self, cap_key: &str) { + if let Some(impression_log) = &self.impression_log { + if let Err(e) = impression_log.record_impression(cap_key) { + self.telemetry + .record(&ImpressionLogOutcome::RecordImpressionFailed(e)); + } + }; + } + + pub fn apply_impression_capping( + &self, + mut ads: AdResponse, + impression_capping_policy: &ImpressionCappingPolicy, + ) -> AdResponse { + if let Some(impression_log) = &self.impression_log { + let caps: HashMap<&str, &u32> = ads + .data + .iter() + .flat_map(|(_, placement_ads)| placement_ads.iter().flat_map(|a| a.cap_pair())) + .collect(); + + let counts = match impression_log.count_impressions(caps.keys()) { + Ok(counts) => counts, + Err(e) => { + self.telemetry + .record(&ImpressionLogOutcome::CountImpressionsFailed(e)); + + // Skip unnecessary work if DB access failed + return ads; + } + }; + + if let Err(e) = impression_log.retain_impressions(caps.keys()) { + self.telemetry + .record(&ImpressionLogOutcome::RetainImpressionsFailed(e)); + }; + + let cap_keys_to_filter: HashSet = caps + .iter() + .flat_map(|(&cap_key, max_impressions)| { + if counts.get(cap_key).unwrap_or(&0) >= max_impressions { + self.telemetry + .record(&ImpressionLogOutcome::ImpressionCapHit); + match impression_capping_policy { + ImpressionCappingPolicy::TelemetryOnly => { + self.telemetry + .record(&ImpressionLogOutcome::ImpressionCapNotEnforced); + None + } + ImpressionCappingPolicy::ImpressionCapEnforced => { + self.telemetry + .record(&ImpressionLogOutcome::ImpressionCapEnforced); + Some(cap_key.to_owned()) + } + } + } else { + None + } + }) + .collect(); + + if !cap_keys_to_filter.is_empty() { + ads.data.iter_mut().for_each(|(_, placement_ads)| { + placement_ads.retain(|a| { + if let Some((cap_key, _)) = a.cap_pair() { + !cap_keys_to_filter.contains(cap_key) + } else { + true + } + }); + }); + } + }; + + ads + } +} + +#[cfg(test)] +mod tests { + use url::Url; + + use crate::ffi::telemetry::MozAdsTelemetryWrapper; + use crate::impression_log::ImpressionCappingPolicy; + use crate::mars::ad_response::{AdCallbacks, AdSpoc, SpocFrequencyCaps, SpocRanking}; + + use super::*; + + #[test] + fn test_no_impression_log_does_not_error() { + let capping = MARSCapping::new(None, MozAdsTelemetryWrapper::noop()); + + let spoc = AdSpoc { + block_key: "test_block_key".into(), + callbacks: AdCallbacks { + click: Url::parse("https://example.com/test_click").unwrap(), + impression: Url::parse("https://example.com/test_impression").unwrap(), + report: None, + }, + caps: SpocFrequencyCaps { + cap_key: "test_cap_key".into(), + day: 10, + }, + domain: "example.com".into(), + excerpt: "test_excerpt".into(), + format: "test_format".into(), + image_url: "https://example.com/test_image".into(), + ranking: SpocRanking { + priority: 0, + personalization_models: None, + item_score: 0.0, + }, + sponsor: "test_sponsor".into(), + sponsored_by_override: None, + title: "test_title".into(), + url: "https://example.com/test_url".into(), + }; + + capping.record_impression("test_cap_key"); + capping.apply_impression_capping( + AdResponse:: { + data: HashMap::from([("".into(), vec![spoc.clone()])]), + }, + &ImpressionCappingPolicy::TelemetryOnly, + ); + capping.apply_impression_capping( + AdResponse:: { + data: HashMap::from([("".into(), vec![spoc.clone()])]), + }, + &ImpressionCappingPolicy::TelemetryOnly, + ); + } +} From f05ec7d9413d342d622ef36e665038c79d637ddb Mon Sep 17 00:00:00 2001 From: bendk Date: Tue, 7 Jul 2026 13:22:42 -0400 Subject: [PATCH 02/59] relay: ensure API types are public (#7424) I'm working on a new UniFFI parser (https://github.com/mozilla/uniffi-rs/pull/2841) and it currently requires that types used in the exported functions are publicly available from other crates. I could maybe rework the parser to handle this another way, but this feels cleaner to me anyways. It feels weird if a type can be used by foreign languages but not other Rust crates. --- components/relay/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/components/relay/src/lib.rs b/components/relay/src/lib.rs index 6805fdaf558..337564f0bdd 100644 --- a/components/relay/src/lib.rs +++ b/components/relay/src/lib.rs @@ -9,6 +9,7 @@ uniffi::setup_scaffolding!("relay"); pub use error::{ApiResult, Error, RelayApiError, Result}; use error_support::handle_error; +pub use rs::RelayRemoteSettingsClient; use serde::{Deserialize, Serialize}; use url::Url; From 62b66459e7fade4da139603857c8ba00945c94ed Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Tue, 7 Jul 2026 21:58:33 -0400 Subject: [PATCH 03/59] Consolidate all bridged engine wrappers into a single wrapper and macro. (#7464) Seeing #7439 fly past reminded me that I was intending to try and remove the BridgedEngine entirely - this doesn't quite achieve that, but does get closer. --- components/logins/src/sync/bridge.rs | 94 +------ .../sync15/src/engine/bridged_engine.rs | 255 ++++++++++++++++++ components/sync15/src/engine/mod.rs | 16 +- components/tabs/src/sync/bridge.rs | 93 +------ components/webext-storage/src/sync/bridge.rs | 96 +------ 5 files changed, 294 insertions(+), 260 deletions(-) diff --git a/components/logins/src/sync/bridge.rs b/components/logins/src/sync/bridge.rs index ff45c53cfcb..1fc246691b2 100644 --- a/components/logins/src/sync/bridge.rs +++ b/components/logins/src/sync/bridge.rs @@ -6,10 +6,8 @@ use crate::sync::engine::LoginsSyncEngine; use crate::LoginStore; use anyhow::Result; use std::sync::Arc; -use sync15::bso::{IncomingBso, OutgoingBso}; -use sync15::engine::{BridgedEngine, BridgedEngineAdaptor}; +use sync15::engine::BridgedEngineAdaptor; use sync15::ServerTimestamp; -use sync_guid::Guid as SyncGuid; impl LoginStore { /// Returns a bridged sync engine for Desktop for this store. @@ -63,90 +61,12 @@ impl BridgedEngineAdaptor for LoginsBridgedEngineAdaptor { } } -// This is what UniFFI exposes; it does nothing other than delegate back to the -// `BridgedEngine` trait object (and handle the JSON (de)serialization of BSOs -// that crosses the FFI boundary). -/// see services/interfaces/mozIBridgedSyncEngine.idl for contract -pub struct LoginsBridgedEngine { - bridge_impl: Box, -} - -impl LoginsBridgedEngine { - pub fn new(bridge_impl: Box) -> Self { - Self { bridge_impl } - } - - pub fn last_sync(&self) -> Result { - self.bridge_impl.last_sync() - } - - pub fn set_last_sync(&self, last_sync: i64) -> Result<()> { - self.bridge_impl.set_last_sync(last_sync) - } - - pub fn sync_id(&self) -> Result> { - self.bridge_impl.sync_id() - } - - pub fn reset_sync_id(&self) -> Result { - self.bridge_impl.reset_sync_id() - } - - pub fn ensure_current_sync_id(&self, sync_id: &str) -> Result { - self.bridge_impl.ensure_current_sync_id(sync_id) - } - - pub fn sync_started(&self) -> Result<()> { - self.bridge_impl.sync_started() - } - - // Decode the JSON-encoded IncomingBso's that UniFFI passes to us - fn convert_incoming_bsos(&self, incoming: Vec) -> Result> { - let mut bsos = Vec::with_capacity(incoming.len()); - for inc in incoming { - bsos.push(serde_json::from_str::(&inc)?); - } - Ok(bsos) - } - - // Encode OutgoingBso's into JSON for UniFFI - fn convert_outgoing_bsos(&self, outgoing: Vec) -> Result> { - let mut bsos = Vec::with_capacity(outgoing.len()); - for e in outgoing { - bsos.push(serde_json::to_string(&e)?); - } - Ok(bsos) - } - - pub fn store_incoming(&self, incoming: Vec) -> Result<()> { - self.bridge_impl - .store_incoming(self.convert_incoming_bsos(incoming)?) - } - - pub fn apply(&self) -> Result> { - let apply_results = self.bridge_impl.apply()?; - self.convert_outgoing_bsos(apply_results.records) - } - - pub fn set_uploaded(&self, server_modified_millis: i64, guids: Vec) -> Result<()> { - // UniFFI hands us plain strings; the bridge works in terms of `Guid`. - let guids: Vec = guids.into_iter().map(SyncGuid::from).collect(); - self.bridge_impl - .set_uploaded(server_modified_millis, &guids) - } - - pub fn sync_finished(&self) -> Result<()> { - self.bridge_impl.sync_finished() - } - - pub fn reset(&self) -> Result<()> { - self.bridge_impl.reset() - } - - pub fn wipe(&self) -> Result<()> { - self.bridge_impl.wipe() - } -} +// The UniFFI-exposed `LoginsBridgedEngine` (a thin newtype around +// `sync15::engine::BridgedEngineWrapper`) is generated by this macro, which +// removes the facade + BSO marshalling boilerplate that used to live here. +// logins' `set_uploaded` UDL row is `sequence`, so the id element type +// is `String`. See services/interfaces/mozIBridgedSyncEngine.idl for the contract. +sync15::uniffi_bridged_engine!(LoginsBridgedEngine, String); #[cfg(not(feature = "keydb"))] #[cfg(test)] diff --git a/components/sync15/src/engine/bridged_engine.rs b/components/sync15/src/engine/bridged_engine.rs index 6cb9c2ffa26..b0280706c09 100644 --- a/components/sync15/src/engine/bridged_engine.rs +++ b/components/sync15/src/engine/bridged_engine.rs @@ -241,3 +241,258 @@ impl From> for ApplyResults { } } } + +/// Wraps a `Box` and centralizes the work every consuming +/// crate's UniFFI-facing bridged engine needs to do: the JSON `String` <-> BSO +/// marshalling that crosses the FFI boundary, and 1:1 delegation to the wrapped +/// engine. Rather than each crate hand-writing this (it was ~100 identical lines +/// per crate), they expose a thin newtype around this via the +/// [`uniffi_bridged_engine!`] macro. +/// +/// All methods return [`anyhow::Result`], which each crate maps onto its own +/// UniFFI error type via an `impl From`. +/// +/// Note on the longer-term direction: this type, along with [`BridgedEngine`], +/// [`BridgedEngineAdaptor`] and [`ApplyResults`], only exists because we still +/// have two sync-engine traits. Once Desktop moves off explicit timestamp +/// handling to the `get_collection_request` model (see #2841) we can remove +/// `BridgedEngine` entirely, have Desktop consume [`SyncEngine`] directly, and +/// this wrapper collapses into a thin `SyncEngine` -> FFI shim (or goes away). +/// See the note in `engine/mod.rs` for the migration sequencing. +pub struct BridgedEngineWrapper { + inner: Box, +} + +impl BridgedEngineWrapper { + pub fn new(inner: Box) -> Self { + Self { inner } + } + + pub fn last_sync(&self) -> Result { + self.inner.last_sync() + } + + pub fn set_last_sync(&self, last_sync: i64) -> Result<()> { + self.inner.set_last_sync(last_sync) + } + + pub fn sync_id(&self) -> Result> { + self.inner.sync_id() + } + + pub fn reset_sync_id(&self) -> Result { + self.inner.reset_sync_id() + } + + pub fn ensure_current_sync_id(&self, sync_id: &str) -> Result { + self.inner.ensure_current_sync_id(sync_id) + } + + pub fn prepare_for_sync(&self, client_data: &str) -> Result<()> { + self.inner.prepare_for_sync(client_data) + } + + pub fn sync_started(&self) -> Result<()> { + self.inner.sync_started() + } + + /// Decode the JSON-encoded `IncomingBso`s that UniFFI passes to us, then + /// hand them to the wrapped engine. + pub fn store_incoming(&self, incoming: Vec) -> Result<()> { + let mut bsos = Vec::with_capacity(incoming.len()); + for inc in incoming { + bsos.push(serde_json::from_str::(&inc)?); + } + self.inner.store_incoming(bsos) + } + + /// Apply staged records and encode the outgoing `OutgoingBso`s back into + /// JSON for UniFFI. + pub fn apply(&self) -> Result> { + let apply_results = self.inner.apply()?; + let mut outgoing = Vec::with_capacity(apply_results.records.len()); + for e in apply_results.records { + outgoing.push(serde_json::to_string(&e)?); + } + Ok(outgoing) + } + + /// Accepts anything that turns into a [`Guid`], which reconciles the + /// per-crate id representation: logins hands us `Vec`, while + /// tabs and webext-storage hand us `Vec`. Both `String` + /// and `Guid` implement `Into`. + pub fn set_uploaded>( + &self, + server_modified_millis: i64, + ids: Vec, + ) -> Result<()> { + let guids: Vec = ids.into_iter().map(Into::into).collect(); + self.inner.set_uploaded(server_modified_millis, &guids) + } + + pub fn sync_finished(&self) -> Result<()> { + self.inner.sync_finished() + } + + pub fn reset(&self) -> Result<()> { + self.inner.reset() + } + + pub fn wipe(&self) -> Result<()> { + self.inner.wipe() + } +} + +/// Generates a UniFFI-exposable bridged engine newtype around +/// [`BridgedEngineWrapper`], removing the ~100 lines of identical facade +/// boilerplate each consuming crate used to hand-write. +/// +/// Usage (invoke in the module the crate's UDL `interface` resolves against): +/// ```ignore +/// sync15::uniffi_bridged_engine!(LoginsBridgedEngine, String); +/// sync15::uniffi_bridged_engine!(TabsBridgedEngine, sync_guid::Guid); +/// ``` +/// +/// `$guid` is the element type the crate's UDL lowers `set_uploaded`'s ids to +/// (`String` for logins' `sequence`, `sync_guid::Guid` for the tabs and +/// webext-storage custom-type sequences). The generated methods return +/// `anyhow::Result`, which the crate's UDL `[Throws=...]` maps to its error type +/// via the existing `impl From`. +/// +/// The macro always emits `prepare_for_sync`; a crate whose UDL doesn't declare +/// it (logins) simply leaves that inherent method unbound, which is harmless. +#[macro_export] +macro_rules! uniffi_bridged_engine { + ($name:ident, $guid:ty) => { + // This is what UniFFI exposes; it does nothing other than delegate to + // the shared `BridgedEngineWrapper`. See + // services/interfaces/mozIBridgedSyncEngine.idl for the Desktop contract. + pub struct $name($crate::engine::BridgedEngineWrapper); + + impl $name { + pub fn new(inner: ::std::boxed::Box) -> Self { + Self($crate::engine::BridgedEngineWrapper::new(inner)) + } + + pub fn last_sync(&self) -> ::anyhow::Result { + self.0.last_sync() + } + + pub fn set_last_sync(&self, last_sync: i64) -> ::anyhow::Result<()> { + self.0.set_last_sync(last_sync) + } + + pub fn sync_id(&self) -> ::anyhow::Result> { + self.0.sync_id() + } + + pub fn reset_sync_id(&self) -> ::anyhow::Result { + self.0.reset_sync_id() + } + + pub fn ensure_current_sync_id(&self, sync_id: &str) -> ::anyhow::Result { + self.0.ensure_current_sync_id(sync_id) + } + + pub fn prepare_for_sync(&self, client_data: &str) -> ::anyhow::Result<()> { + self.0.prepare_for_sync(client_data) + } + + pub fn sync_started(&self) -> ::anyhow::Result<()> { + self.0.sync_started() + } + + pub fn store_incoming(&self, incoming: Vec) -> ::anyhow::Result<()> { + self.0.store_incoming(incoming) + } + + pub fn apply(&self) -> ::anyhow::Result> { + self.0.apply() + } + + pub fn set_uploaded( + &self, + server_modified_millis: i64, + ids: Vec<$guid>, + ) -> ::anyhow::Result<()> { + self.0.set_uploaded(server_modified_millis, ids) + } + + pub fn sync_finished(&self) -> ::anyhow::Result<()> { + self.0.sync_finished() + } + + pub fn reset(&self) -> ::anyhow::Result<()> { + self.0.reset() + } + + pub fn wipe(&self) -> ::anyhow::Result<()> { + self.0.wipe() + } + } + }; +} + +#[cfg(test)] +mod wrapper_tests { + use super::*; + use crate::bso::OutgoingBso; + use std::sync::Mutex; + + // A minimal BridgedEngine that records the guids passed to `set_uploaded`, + // so we can lock in the `Into` reconciliation for both `String` and + // `Guid` element types. + #[derive(Default)] + struct RecordingEngine { + uploaded: Mutex>, + } + + impl BridgedEngine for RecordingEngine { + fn last_sync(&self) -> Result { + Ok(0) + } + fn set_last_sync(&self, _: i64) -> Result<()> { + Ok(()) + } + fn sync_id(&self) -> Result> { + Ok(None) + } + fn reset_sync_id(&self) -> Result { + Ok(String::new()) + } + fn ensure_current_sync_id(&self, id: &str) -> Result { + Ok(id.to_string()) + } + fn sync_started(&self) -> Result<()> { + Ok(()) + } + fn store_incoming(&self, _: Vec) -> Result<()> { + Ok(()) + } + fn apply(&self) -> Result { + Ok(Vec::::new().into()) + } + fn set_uploaded(&self, _millis: i64, ids: &[Guid]) -> Result<()> { + self.uploaded.lock().unwrap().extend_from_slice(ids); + Ok(()) + } + fn sync_finished(&self) -> Result<()> { + Ok(()) + } + fn reset(&self) -> Result<()> { + Ok(()) + } + fn wipe(&self) -> Result<()> { + Ok(()) + } + } + + #[test] + fn set_uploaded_accepts_strings_and_guids() { + let wrapper = BridgedEngineWrapper::new(Box::new(RecordingEngine::default())); + // logins-style: Vec + wrapper.set_uploaded(1, vec!["aaaa".to_string()]).unwrap(); + // tabs/webext-style: Vec + wrapper.set_uploaded(2, vec![Guid::new("bbbb")]).unwrap(); + } +} diff --git a/components/sync15/src/engine/mod.rs b/components/sync15/src/engine/mod.rs index 826ea1a7749..6be400677e6 100644 --- a/components/sync15/src/engine/mod.rs +++ b/components/sync15/src/engine/mod.rs @@ -27,11 +27,25 @@ //! We intend merging these engines - the first step will be to merge the //! types and payload management used by these traits, then to combine the //! requirements into a single trait that captures both use-cases. +//! +//! Steps so far, and what's left: +//! * [bridged_engine::BridgedEngineAdaptor] lets a crate implement only +//! [SyncEngine] (plus a tiny adaptor) and get a [bridged_engine::BridgedEngine] +//! for free. +//! * [bridged_engine::BridgedEngineWrapper] + the `uniffi_bridged_engine!` macro +//! remove the per-crate UniFFI facade boilerplate (the JSON<->BSO marshalling +//! and method delegation). +//! * Still to do (#2841): remove `BridgedEngine`/`BridgedEngineAdaptor`/`ApplyResults` +//! entirely and have Desktop consume [SyncEngine] directly. This is blocked on a +//! coordinated mozilla-central change: Desktop must move off explicit timestamp +//! handling (`last_sync`/`set_last_sync`) to the `get_collection_request` model, +//! and the per-crate UDL `interface *BridgedEngine` blocks (the Desktop-visible +//! contract consumed via mozIBridgedSyncEngine) must be updated in lockstep. mod bridged_engine; mod request; mod sync_engine; -pub use bridged_engine::{ApplyResults, BridgedEngine, BridgedEngineAdaptor}; +pub use bridged_engine::{ApplyResults, BridgedEngine, BridgedEngineAdaptor, BridgedEngineWrapper}; #[cfg(feature = "sync-client")] pub(crate) use request::CollectionPost; diff --git a/components/tabs/src/sync/bridge.rs b/components/tabs/src/sync/bridge.rs index 1a2f45415eb..1f8a5b96775 100644 --- a/components/tabs/src/sync/bridge.rs +++ b/components/tabs/src/sync/bridge.rs @@ -6,10 +6,8 @@ use crate::sync::engine::TabsEngine; use crate::TabsStore; use anyhow::Result; use std::sync::Arc; -use sync15::bso::{IncomingBso, OutgoingBso}; -use sync15::engine::{BridgedEngine, BridgedEngineAdaptor}; +use sync15::engine::BridgedEngineAdaptor; use sync15::ServerTimestamp; -use sync_guid::Guid as SyncGuid; impl TabsStore { // Returns a bridged sync engine for Desktop for this store. @@ -44,89 +42,12 @@ impl BridgedEngineAdaptor for TabsBridgedEngineAdaptor { } } -// This is for uniffi to expose, and does nothing than delegate back to the trait. -pub struct TabsBridgedEngine { - bridge_impl: Box, -} - -impl TabsBridgedEngine { - pub fn new(bridge_impl: Box) -> Self { - Self { bridge_impl } - } - - pub fn last_sync(&self) -> Result { - self.bridge_impl.last_sync() - } - - pub fn set_last_sync(&self, last_sync: i64) -> Result<()> { - self.bridge_impl.set_last_sync(last_sync) - } - - pub fn sync_id(&self) -> Result> { - self.bridge_impl.sync_id() - } - - pub fn reset_sync_id(&self) -> Result { - self.bridge_impl.reset_sync_id() - } - - pub fn ensure_current_sync_id(&self, sync_id: &str) -> Result { - self.bridge_impl.ensure_current_sync_id(sync_id) - } - - pub fn prepare_for_sync(&self, client_data: &str) -> Result<()> { - self.bridge_impl.prepare_for_sync(client_data) - } - - pub fn sync_started(&self) -> Result<()> { - self.bridge_impl.sync_started() - } - - // Decode the JSON-encoded IncomingBso's that UniFFI passes to us - fn convert_incoming_bsos(&self, incoming: Vec) -> Result> { - let mut bsos = Vec::with_capacity(incoming.len()); - for inc in incoming { - bsos.push(serde_json::from_str::(&inc)?); - } - Ok(bsos) - } - - // Encode OutgoingBso's into JSON for UniFFI - fn convert_outgoing_bsos(&self, outgoing: Vec) -> Result> { - let mut bsos = Vec::with_capacity(outgoing.len()); - for e in outgoing { - bsos.push(serde_json::to_string(&e)?); - } - Ok(bsos) - } - - pub fn store_incoming(&self, incoming: Vec) -> Result<()> { - self.bridge_impl - .store_incoming(self.convert_incoming_bsos(incoming)?) - } - - pub fn apply(&self) -> Result> { - let apply_results = self.bridge_impl.apply()?; - self.convert_outgoing_bsos(apply_results.records) - } - - pub fn set_uploaded(&self, server_modified_millis: i64, guids: Vec) -> Result<()> { - self.bridge_impl - .set_uploaded(server_modified_millis, &guids) - } - - pub fn sync_finished(&self) -> Result<()> { - self.bridge_impl.sync_finished() - } - - pub fn reset(&self) -> Result<()> { - self.bridge_impl.reset() - } - - pub fn wipe(&self) -> Result<()> { - self.bridge_impl.wipe() - } -} +// The UniFFI-exposed `TabsBridgedEngine` (a thin newtype around +// `sync15::engine::BridgedEngineWrapper`) is generated by this macro, which +// removes the facade + BSO marshalling boilerplate that used to live here. +// tabs' `set_uploaded` UDL row is `sequence` (a custom type over +// `sync_guid::Guid`), so the id element type is `sync_guid::Guid`. +sync15::uniffi_bridged_engine!(TabsBridgedEngine, sync_guid::Guid); #[cfg(test)] mod tests { diff --git a/components/webext-storage/src/sync/bridge.rs b/components/webext-storage/src/sync/bridge.rs index 4067d6e49ef..502e24d9539 100644 --- a/components/webext-storage/src/sync/bridge.rs +++ b/components/webext-storage/src/sync/bridge.rs @@ -5,7 +5,7 @@ use anyhow::Result; use rusqlite::Transaction; use std::sync::{Arc, Weak}; -use sync15::bso::{IncomingBso, OutgoingBso}; +use sync15::bso::IncomingBso; use sync15::engine::{ApplyResults, BridgedEngine as Sync15BridgedEngine}; use sync_guid::Guid as SyncGuid; @@ -22,10 +22,7 @@ impl WebExtStorageStore { // Returns a bridged sync engine for this store. pub fn bridged_engine(self: Arc) -> Arc { let engine = Box::new(BridgedEngine::new(&self.db)); - let bridged_engine = WebExtStorageBridgedEngine { - bridge_impl: engine, - }; - Arc::new(bridged_engine) + Arc::new(WebExtStorageBridgedEngine::new(engine)) } } @@ -206,87 +203,14 @@ impl Sync15BridgedEngine for BridgedEngine { } } -pub struct WebExtStorageBridgedEngine { - bridge_impl: Box, -} - -impl WebExtStorageBridgedEngine { - pub fn new(bridge_impl: Box) -> Self { - Self { bridge_impl } - } - - pub fn last_sync(&self) -> Result { - self.bridge_impl.last_sync() - } - - pub fn set_last_sync(&self, last_sync: i64) -> Result<()> { - self.bridge_impl.set_last_sync(last_sync) - } - - pub fn sync_id(&self) -> Result> { - self.bridge_impl.sync_id() - } - - pub fn reset_sync_id(&self) -> Result { - self.bridge_impl.reset_sync_id() - } - - pub fn ensure_current_sync_id(&self, sync_id: &str) -> Result { - self.bridge_impl.ensure_current_sync_id(sync_id) - } - - pub fn prepare_for_sync(&self, client_data: &str) -> Result<()> { - self.bridge_impl.prepare_for_sync(client_data) - } - - pub fn store_incoming(&self, incoming: Vec) -> Result<()> { - self.bridge_impl - .store_incoming(self.convert_incoming_bsos(incoming)?) - } - - pub fn apply(&self) -> Result> { - let apply_results = self.bridge_impl.apply()?; - self.convert_outgoing_bsos(apply_results.records) - } - - pub fn set_uploaded(&self, server_modified_millis: i64, guids: Vec) -> Result<()> { - self.bridge_impl - .set_uploaded(server_modified_millis, &guids) - } - - pub fn sync_started(&self) -> Result<()> { - self.bridge_impl.sync_started() - } - - pub fn sync_finished(&self) -> Result<()> { - self.bridge_impl.sync_finished() - } - - pub fn reset(&self) -> Result<()> { - self.bridge_impl.reset() - } - - pub fn wipe(&self) -> Result<()> { - self.bridge_impl.wipe() - } - - fn convert_incoming_bsos(&self, incoming: Vec) -> Result> { - let mut bsos = Vec::with_capacity(incoming.len()); - for inc in incoming { - bsos.push(serde_json::from_str::(&inc)?); - } - Ok(bsos) - } - - // Encode OutgoingBso's into JSON for UniFFI - fn convert_outgoing_bsos(&self, outgoing: Vec) -> Result> { - let mut bsos = Vec::with_capacity(outgoing.len()); - for e in outgoing { - bsos.push(serde_json::to_string(&e)?); - } - Ok(bsos) - } -} +// The UniFFI-exposed `WebExtStorageBridgedEngine` (a thin newtype around +// `sync15::engine::BridgedEngineWrapper`) is generated by this macro, which +// removes the facade + BSO marshalling boilerplate that used to live here. The +// wrapped engine is the `BridgedEngine` defined above (webext-storage is +// Desktop-only and implements `BridgedEngine` directly rather than `SyncEngine`). +// Its `set_uploaded` UDL row is `sequence` (a custom type over +// `sync_guid::Guid`), so the id element type is `sync_guid::Guid`. +sync15::uniffi_bridged_engine!(WebExtStorageBridgedEngine, sync_guid::Guid); impl From for crate::error::Error { fn from(value: anyhow::Error) -> Self { From 83422fa15fb753bcde45e0fc6f6d5bc184e39baf Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Tue, 7 Jul 2026 22:07:25 -0400 Subject: [PATCH 04/59] fxa-client: fix sync key invariant checks so they are only done after we've merged all keys (#7458) --- components/fxa-client/src/internal/oauth.rs | 170 +++++++++++++++++--- 1 file changed, 152 insertions(+), 18 deletions(-) diff --git a/components/fxa-client/src/internal/oauth.rs b/components/fxa-client/src/internal/oauth.rs index 8cf2aa913e3..b2f524bbf4d 100644 --- a/components/fxa-client/src/internal/oauth.rs +++ b/components/fxa-client/src/internal/oauth.rs @@ -357,7 +357,8 @@ impl FirefoxAccount { resp: OAuthTokenResponse, scoped_keys_flow: Option, ) -> Result<()> { - let sync_scope_granted = resp.scope.split(' ').any(|s| s == scopes::OLD_SYNC); + // These are the keys granted by *this* response - any invariants about scopes vs keys + // must be checked after we've fully merged the scopes and keys. let scoped_keys = match resp.keys_jwe { Some(ref jwe) => { let scoped_keys_flow = scoped_keys_flow.ok_or(Error::ApiClientError( @@ -366,28 +367,12 @@ impl FirefoxAccount { let decrypted_keys = scoped_keys_flow.decrypt_keys_jwe(jwe)?; let scoped_keys: serde_json::Map = serde_json::from_str(&decrypted_keys)?; - if sync_scope_granted && !scoped_keys.contains_key(scopes::OLD_SYNC) { - error_support::report_error!( - "fxaclient-scoped-key", - "Sync scope granted, but no sync scoped key (scope granted: {}, key scopes: {})", - resp.scope, - scoped_keys.keys().map(|s| s.as_ref()).collect::>().join(", ") - ); - } scoped_keys .into_iter() .map(|(scope, key)| Ok((scope, serde_json::from_value(key)?))) .collect::>>()? } - None => { - if sync_scope_granted { - error_support::report_error!( - "fxaclient-scoped-key", - "Sync scope granted, but keys_jwe is None" - ); - } - vec![] - } + None => vec![], }; // We are only interested in the refresh token at this time because we @@ -487,6 +472,27 @@ impl FirefoxAccount { self.state.clear_refresh_token(); } + // Evaluate the sync-key invariant against the final state: the scopes the merged + // refresh token actually carries, and every key we'll hold afterwards (keys from this + // response plus keys we already had for other scopes). + let sync_scope_granted = new_refresh_token.scopes.contains(scopes::OLD_SYNC); + let have_sync_key = scoped_keys + .iter() + .any(|(scope, _)| scope == scopes::OLD_SYNC) + || self.state.get_scoped_key(scopes::OLD_SYNC).is_some(); + if sync_scope_granted && !have_sync_key { + error_support::report_error!( + "fxaclient-scoped-key", + "Sync scope granted, but no sync scoped key held (final scopes: {})", + new_refresh_token + .scopes + .iter() + .cloned() + .collect::>() + .join(", ") + ); + } + self.state .complete_oauth_flow(scoped_keys, new_refresh_token, resp.session_token); if let Some(ref device_info) = old_device_info { @@ -1396,4 +1402,132 @@ mod tests { let scopes = &fxa.state.refresh_token().unwrap().scopes; assert_eq!(scopes, &["profile".to_string()].into()); } + + // Test that adding a non-sync scope to an account that is already signed in with sync + // retains the sync scoped key we already hold, even though this response carries no keys. + #[test] + fn test_complete_oauth_flow_retains_existing_sync_key_when_adding_scope() { + nss_as::ensure_initialized(); + let config = Config::new_with_mock_well_known_fxa_client_configuration( + "mock-fxa.example.com", + "12345678", + "https://foo.bar", + ); + let mut fxa = FirefoxAccount::with_config(config); + + // Start a flow requesting only a new, non-sync scope. + let url = fxa + .begin_oauth_flow("", &["new_scope"], "test_entrypoint") + .unwrap(); + let url = Url::parse(&url).unwrap(); + let state = url.query_pairs().find(|(name, _)| name == "state").unwrap(); + + // Pre-populate: signed in with the sync scope, holding its scoped key, plus a session + // token so the scope merge can happen. + fxa.state.force_refresh_token(RefreshToken { + token: "old_refresh".to_string(), + scopes: [OLD_SYNC.to_string()].into(), + }); + fxa.state.insert_scoped_key( + OLD_SYNC, + crate::ScopedKey { + kty: "oct".to_string(), + scope: OLD_SYNC.to_string(), + k: "existing_sync_key_material".to_string(), + kid: "existing_sync_kid".to_string(), + }, + ); + fxa.set_session_token("mock_session_token"); + + let mut client = MockFxAClient::new(); + + // 1. Exchange auth code — narrow token with only the new scope and no keys. + client + .expect_create_refresh_token_using_authorization_code() + .times(1) + .returning(|_, _, _, _| { + Ok(OAuthTokenResponse { + keys_jwe: None, + refresh_token: Some("new_narrow_refresh".to_string()), + session_token: None, + expires_in: 3600, + scope: "new_scope".to_string(), + access_token: "access_token".to_string(), + }) + }); + + // 2. Destroy the over-scoped access token. + client + .expect_destroy_access_token() + .with(always(), always()) + .times(1) + .returning(|_, _| Ok(())); + + // 3. Fetch current device so it can be restored after the token swap. + client + .expect_get_devices() + .with(always(), eq("old_refresh")) + .times(1) + .returning(|_, _| Ok(vec![make_mock_device("Test Device")])); + + // 4. Get merged refresh token covering both the old sync scope and the new scope. + client + .expect_create_refresh_token_using_session_token() + .withf(|_, session_token, _| session_token == "mock_session_token") + .times(1) + .returning(|_, _, _| { + Ok(OAuthTokenResponse { + keys_jwe: None, + refresh_token: Some("merged_refresh".to_string()), + session_token: None, + expires_in: 3600, + scope: format!("{OLD_SYNC} new_scope"), + access_token: "access_token2".to_string(), + }) + }); + + // 5. Destroy the narrow new token (replaced by the merged one). + client + .expect_destroy_refresh_token() + .with(always(), eq("new_narrow_refresh")) + .times(1) + .returning(|_, _| Ok(())); + + // 6. Destroy the old refresh token. + client + .expect_destroy_refresh_token() + .with(always(), eq("old_refresh")) + .times(1) + .returning(|_, _| Ok(())); + + // 7. Restore the device record. + client + .expect_update_device_record() + .times(1) + .returning(|_, _, _| Ok(make_mock_update_device_response())); + + fxa.set_client(Arc::new(client)); + + fxa.complete_oauth_flow("mock_code", state.1.as_ref()) + .unwrap(); + + // The sync key we already held must survive, even though this flow carried no keys. + let sync_key = fxa + .state + .get_scoped_key(OLD_SYNC) + .expect("sync scoped key should be retained"); + assert_eq!(sync_key.k, "existing_sync_key_material"); + + // And the merged refresh token should carry both scopes. + let scopes = &fxa.state.refresh_token().unwrap().scopes; + assert!( + scopes.contains(OLD_SYNC), + "expected sync scope, got {scopes:?}" + ); + assert!( + scopes.contains("new_scope"), + "expected new_scope, got {scopes:?}" + ); + assert_eq!(scopes.len(), 2); + } } From 3af675e65eda9f2eb6e9a59825486e66f6d29b11 Mon Sep 17 00:00:00 2001 From: "roux g. buciu" <11182210+adudenamedruby@users.noreply.github.com> Date: Wed, 8 Jul 2026 09:51:23 -0400 Subject: [PATCH 05/59] Add IE and PL to merino curated recs (#7463) --- .../src/curated_recommendations/models/locale.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/merino/src/curated_recommendations/models/locale.rs b/components/merino/src/curated_recommendations/models/locale.rs index 5c0608bb73b..f7f7b0e1a1b 100644 --- a/components/merino/src/curated_recommendations/models/locale.rs +++ b/components/merino/src/curated_recommendations/models/locale.rs @@ -29,6 +29,8 @@ pub enum CuratedRecommendationLocale { EnCa, #[serde(rename = "en-GB")] EnGb, + #[serde(rename = "en-IE")] + EnIe, #[serde(rename = "en-US")] EnUs, #[serde(rename = "de")] @@ -39,6 +41,10 @@ pub enum CuratedRecommendationLocale { DeAt, #[serde(rename = "de-CH")] DeCh, + #[serde(rename = "pl")] + Pl, + #[serde(rename = "pl-PL")] + PlPl, } impl CuratedRecommendationLocale { /// Returns all supported locale strings (e.g. `"en-US"`, `"fr-FR"`). @@ -55,11 +61,14 @@ impl CuratedRecommendationLocale { "en".to_string(), "en-CA".to_string(), "en-GB".to_string(), + "en-IE".to_string(), "en-US".to_string(), "de".to_string(), "de-DE".to_string(), "de-AT".to_string(), "de-CH".to_string(), + "pl".to_string(), + "pl-PL".to_string(), ] } @@ -78,11 +87,14 @@ impl CuratedRecommendationLocale { "en" => Some(CuratedRecommendationLocale::En), "en-CA" => Some(CuratedRecommendationLocale::EnCa), "en-GB" => Some(CuratedRecommendationLocale::EnGb), + "en-IE" => Some(CuratedRecommendationLocale::EnIe), "en-US" => Some(CuratedRecommendationLocale::EnUs), "de" => Some(CuratedRecommendationLocale::De), "de-DE" => Some(CuratedRecommendationLocale::DeDe), "de-AT" => Some(CuratedRecommendationLocale::DeAt), "de-CH" => Some(CuratedRecommendationLocale::DeCh), + "pl" => Some(CuratedRecommendationLocale::Pl), + "pl-PL" => Some(CuratedRecommendationLocale::PlPl), _ => None, } } From 60ee4f0029464bce92b49dc176988fac7fc009f2 Mon Sep 17 00:00:00 2001 From: bendk Date: Wed, 8 Jul 2026 12:14:14 -0400 Subject: [PATCH 06/59] Bug 1868418 - Rerun ensure capabilities after logging in (#7457) Moved the code to RetryingAccount, which should give us this functionality for free. Also, tweaked the fxa-client example a bit. --- .cargo/config.toml | 2 +- .../fxa-client/src/state_machine/helpers.rs | 14 +++++------ .../src/state_machine/transitions.rs | 25 +++---------------- examples/fxa-client/Cargo.toml | 2 +- .../full/android/dependency-licenses.xml | 4 +-- 5 files changed, 14 insertions(+), 33 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 690508a55d1..cd57dcc09c8 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -5,7 +5,7 @@ uniffi-bindgen = ["run", "--package", "embedded-uniffi-bindgen", "--"] uniffi-bindgen-library-mode = ["run", "--package", "uniffi-bindgen-library-mode", "--"] dev-install = ["install", "asdev"] verify_env = ["asdev", "verify_env"] -fxa = ["run", "-p", "examples-fxa-client", "--"] +fxa = ["run", "-p", "examples-fxa-client", "--example", "fxa-client", "--"] suggest-bench = ["bench", "-p", "suggest-bench"] suggest-debug-ingestion-sizes = ["run", "-p", "suggest-bench", "--bin", "debug_ingestion_sizes"] relevancy = ["run", "-p", "examples-relevancy-cli", "--"] diff --git a/components/fxa-client/src/state_machine/helpers.rs b/components/fxa-client/src/state_machine/helpers.rs index b165c51260a..5e9670426e1 100644 --- a/components/fxa-client/src/state_machine/helpers.rs +++ b/components/fxa-client/src/state_machine/helpers.rs @@ -151,14 +151,12 @@ impl<'a> RetryingAccount<'a> { self.with_retry(|a| a.initialize_device(name, device_type, capabilities)) } - /// Auth errors propagate so the FSM can drive its own recovery via - /// `CheckAuthorizationStatus`. See - /// . - pub fn ensure_capabilities( - &mut self, - capabilities: &[DeviceCapability], - ) -> Result { - self.with_retry(|a| a.ensure_capabilities(capabilities)) + /// Finish initializing a connected account + pub fn finish_initialize(&mut self, capabilities: &[DeviceCapability]) -> Result<()> { + self.with_auth_recovery(|a| { + a.ensure_capabilities(capabilities)?; + Ok(()) + }) } pub fn check_authorization_status(&mut self) -> Result { diff --git a/components/fxa-client/src/state_machine/transitions.rs b/components/fxa-client/src/state_machine/transitions.rs index 87b00ce4077..8b285f0b624 100644 --- a/components/fxa-client/src/state_machine/transitions.rs +++ b/components/fxa-client/src/state_machine/transitions.rs @@ -7,8 +7,7 @@ //! Each `match` arm reads top-to-bottom as imperative Rust. Use //! `.to_state_machine_err(|| target)?` to attach the landing state on failure. -use crate::{Error, FxaError, FxaEvent, FxaRustAuthState, FxaState}; -use error_support::{convert_log_report_error, GetErrorHandling}; +use crate::{Error, FxaEvent, FxaRustAuthState, FxaState}; use super::helpers::{ResultExt, RetryingAccount, StateMachineErr}; @@ -27,21 +26,9 @@ pub fn transition( FxaRustAuthState::Disconnected => Ok(S::Disconnected), FxaRustAuthState::AuthIssues => Ok(S::AuthIssues), FxaRustAuthState::Connected => { - // Auth errors from ensure_capabilities recover via CheckAuthorizationStatus - // rather than bailing to Disconnected. - // FIXME: should re-run ensure_capabilities after recovery succeeds. - // https://bugzilla.mozilla.org/show_bug.cgi?id=1868418 - match account.ensure_capabilities(&device_config.capabilities) { - Ok(_) => Ok(S::Connected), - Err(e) if is_auth_error(&e) => { - // Report inline since we don't propagate this error to the driver. - let _: FxaError = convert_log_report_error(e); - let active = account - .check_authorization_status() - .to_state_machine_err(|| S::AuthIssues)?; - Ok(if active { S::Connected } else { S::AuthIssues }) - } - Err(cause) => Err(StateMachineErr::new(cause, S::Disconnected)), + match account.finish_initialize(&device_config.capabilities) { + Ok(()) => Ok(S::Connected), + Err(cause) => Err(StateMachineErr::new(cause, S::AuthIssues)), } } } @@ -233,10 +220,6 @@ pub fn transition( } } -fn is_auth_error(e: &Error) -> bool { - matches!(e.get_error_handling().err, FxaError::Authentication) -} - #[cfg(test)] mod tests { //! Tests for the I/O-free transition arms (cancel-oauth, invalid combos). diff --git a/examples/fxa-client/Cargo.toml b/examples/fxa-client/Cargo.toml index 4463f701ec9..2aeb5d2ffc9 100644 --- a/examples/fxa-client/Cargo.toml +++ b/examples/fxa-client/Cargo.toml @@ -16,7 +16,7 @@ nss-as = { path = "../../components/support/rc_crypto/nss" } viaduct = { path = "../../components/viaduct"} viaduct-hyper = { path = "../../components/support/viaduct-hyper" } log = "0.4" -clap = {version = "4.2", default-features = false, features = ["std", "derive"]} +clap = {version = "4.2", default-features = false, features = ["std", "derive", "help"]} cli-support = { path = "../cli-support" } fxa-client = { path = "../../components/fxa-client" } anyhow = "1.0" diff --git a/megazords/full/android/dependency-licenses.xml b/megazords/full/android/dependency-licenses.xml index 0c005f97481..b0ca7438659 100644 --- a/megazords/full/android/dependency-licenses.xml +++ b/megazords/full/android/dependency-licenses.xml @@ -630,7 +630,7 @@ the details of which are reproduced below. MIT License: smawk - https://github.com/mgeisler/smawk/blob/master/LICENSE + https://github.com/mgeisler/smawk/blob/main/LICENSE MIT License: synstructure @@ -638,7 +638,7 @@ the details of which are reproduced below. MIT License: textwrap - https://github.com/mgeisler/textwrap/blob/master/LICENSE + https://github.com/mgeisler/textwrap/blob/main/LICENSE MIT License: tracing From 03bc665b2c91d91ed490a332c7c5e65d023a2670 Mon Sep 17 00:00:00 2001 From: Mohamed Ibrahim <120530864+moibra05@users.noreply.github.com> Date: Wed, 8 Jul 2026 12:24:47 -0400 Subject: [PATCH 07/59] Bug 2052250 - add support for re-enrolling into rollouts (#7466) --- components/nimbus/src/enrollment.rs | 20 ++++++-- .../nimbus/src/tests/test_enrollment.rs | 49 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/components/nimbus/src/enrollment.rs b/components/nimbus/src/enrollment.rs index 662dd5838ad..3a997c5038f 100644 --- a/components/nimbus/src/enrollment.rs +++ b/components/nimbus/src/enrollment.rs @@ -1287,10 +1287,22 @@ impl<'a> EnrollmentsEvolver<'a> { )?) } (None, None, Some(enrollment)) => enrollment.maybe_garbage_collect(), - (None, Some(_), Some(_)) => { - return Err(NimbusError::InternalError( - "New experiment but enrollment already exists.", - )); + (None, Some(experiment), Some(enrollment)) => { + if experiment.is_rollout() + && matches!(&enrollment.status, EnrollmentStatus::WasEnrolled { .. }) + { + Some(ExperimentEnrollment::from_new_experiment( + is_user_participating, + self.available_randomization_units, + experiment, + &targeting_helper, + out_enrollment_events, + )?) + } else { + return Err(NimbusError::InternalError( + "New experiment but enrollment already exists.", + )); + } } (Some(_), None, None) | (Some(_), Some(_), None) => { return Err(NimbusError::InternalError( diff --git a/components/nimbus/src/tests/test_enrollment.rs b/components/nimbus/src/tests/test_enrollment.rs index af5864dba55..a81e45c14df 100644 --- a/components/nimbus/src/tests/test_enrollment.rs +++ b/components/nimbus/src/tests/test_enrollment.rs @@ -3666,6 +3666,55 @@ fn test_evolver_new_experiment_enrollment_already_exists() { assert_eq!(&events, &[]); } +#[test] +fn test_evolver_relaunched_rollout_reenrolls_from_was_enrolled() -> Result<()> { + let rollout = get_bucketed_rollout("secure-gold", 10_000); + let existing_enrollment = ExperimentEnrollment { + slug: rollout.slug.clone(), + status: EnrollmentStatus::WasEnrolled { + branch: "control".to_owned(), + experiment_ended_at: now_secs(), + }, + }; + let (_, app_ctx, aru) = local_ctx(); + let mut th = app_ctx.into(); + let ids = no_coenrolling_features(); + let mut evolver = enrollment_evolver(&mut th, &aru, &ids); + let mut events = vec![]; + + let enrollment = evolver + .evolve_enrollment( + true, + None, + Some(&rollout), + Some(&existing_enrollment), + &mut events, + #[cfg(feature = "stateful")] + None, + )? + .unwrap(); + + assert!(matches!( + enrollment.status, + EnrollmentStatus::Enrolled { + branch, + reason: EnrolledReason::Qualified, + .. + } if branch == "control" + )); + assert_eq!( + &events, + &[EnrollmentChangeEvent { + experiment_slug: "secure-gold".into(), + branch_slug: "control".into(), + reason: None, + change: EnrollmentChangeEventType::Enrollment, + feature_ids: vec!["a-feature".into()], + }] + ); + Ok(()) +} + #[test] fn test_evolver_existing_experiment_has_no_enrollment() { let exp = get_test_experiments()[0].clone(); From c407fd1b1de039b9de03f8edc63d1cd80d86181c Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Wed, 8 Jul 2026 20:41:47 -0400 Subject: [PATCH 08/59] fxa-client: add has_scope() method. (#7459) --- .../appservices/fxaclient/FxaClient.kt | 9 ++++++ components/fxa-client/src/fxa_client.udl | 8 +++++ components/fxa-client/src/internal/oauth.rs | 30 +++++++++++++++++++ components/fxa-client/src/token.rs | 10 +++++++ examples/cli-support/src/fxa_creds.rs | 23 ++------------ 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/components/fxa-client/android/src/main/java/mozilla/appservices/fxaclient/FxaClient.kt b/components/fxa-client/android/src/main/java/mozilla/appservices/fxaclient/FxaClient.kt index 31265a4b675..6b52fc1e23c 100644 --- a/components/fxa-client/android/src/main/java/mozilla/appservices/fxaclient/FxaClient.kt +++ b/components/fxa-client/android/src/main/java/mozilla/appservices/fxaclient/FxaClient.kt @@ -267,6 +267,15 @@ class FxaClient(inner: FirefoxAccount, persistCallback: PersistCallback?) : Auto } } + /** + * Check whether the account has already been granted every given OAuth scope(s). + * + * @param scope space-separated list of OAuth scopes. Order is not significant. + */ + fun hasScope(scope: String): Boolean { + return this.inner.hasScope(scope) + } + fun checkAuthorizationStatus(): AuthorizationInfo { return this.inner.checkAuthorizationStatus() } diff --git a/components/fxa-client/src/fxa_client.udl b/components/fxa-client/src/fxa_client.udl index 115907ea6de..1f3b45483bf 100644 --- a/components/fxa-client/src/fxa_client.udl +++ b/components/fxa-client/src/fxa_client.udl @@ -597,6 +597,14 @@ interface FirefoxAccount { [Throws=FxaError] AccessTokenInfo get_access_token([ByRef] string scope, optional boolean use_cache = true); + /// Check whether the account has already been granted the given OAuth scope(s). + /// + /// This checks whether the refresh token has *every* specified scope. + /// + /// # Arguments + /// - `scope` - space-separated list of OAuth scopes. Order is not significant. + boolean has_scope([ByRef] string scope); + /// Create a new OAuth authorization code using the stored session token. /// /// When a signed-in application receives an incoming device pairing request, it can diff --git a/components/fxa-client/src/internal/oauth.rs b/components/fxa-client/src/internal/oauth.rs index b2f524bbf4d..3aaaa2b24c2 100644 --- a/components/fxa-client/src/internal/oauth.rs +++ b/components/fxa-client/src/internal/oauth.rs @@ -26,6 +26,18 @@ use url::Url; pub const OAUTH_WEBCHANNEL_REDIRECT: &str = "urn:ietf:wg:oauth:2.0:oob:oauth-redirect-webchannel"; impl FirefoxAccount { + /// Check whether every requested scope has been granted to the account's refresh token. + pub fn has_scope(&self, scope: &str) -> bool { + let mut requested = scope.split_ascii_whitespace().peekable(); + if requested.peek().is_none() { + return false; + } + match self.state.refresh_token() { + Some(refresh_token) => requested.all(|s| refresh_token.scopes.contains(s)), + None => false, + } + } + /// Extracts and stores the session token from a WebChannel login JSON payload. /// The JSON payload is the `data` object from the `fxaccounts:login` WebChannel command. pub fn handle_web_channel_login(&mut self, json_payload: &str) -> Result<()> { @@ -657,6 +669,24 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; + #[test] + fn test_has_scope() { + nss_as::ensure_initialized(); + let mut fxa = + FirefoxAccount::with_config(Config::stable_dev("12345678", "https://foo.bar")); + // No refresh token -> false. + assert!(!fxa.has_scope("profile")); + fxa.state.force_refresh_token(RefreshToken { + token: "rt".to_owned(), + scopes: ["profile", "sync"].iter().map(|s| s.to_string()).collect(), + }); + assert!(fxa.has_scope("profile")); + assert!(fxa.has_scope("sync profile")); + assert!(fxa.has_scope("profile sync ")); // trailing whitespace too + assert!(!fxa.has_scope("sync unknown")); // one missing -> false + assert!(!fxa.has_scope("")); // empty -> false + } + #[test] fn test_oauth_flow_url() { nss_as::ensure_initialized(); diff --git a/components/fxa-client/src/token.rs b/components/fxa-client/src/token.rs index e84f2152f84..d06e5df0a66 100644 --- a/components/fxa-client/src/token.rs +++ b/components/fxa-client/src/token.rs @@ -57,6 +57,16 @@ impl FirefoxAccount { .try_into() } + /// Check whether the account has already been granted the given OAuth scope(s). + /// + /// This checks whether the refresh token has *every* specified scope. + /// + /// # Arguments + /// - `scope` - space-separated list of OAuth scopes. Order is not significant. + pub fn has_scope(&self, scope: &str) -> bool { + self.internal.lock().has_scope(scope) + } + /// Builds a complete `signedInUser` JSON object for a WebChannel `fxaccounts:fxa_status` /// response, embedding the session token without exposing it to the browser layer. Email and /// uid are read from the cached profile in internal state. Returns `None` if no session token diff --git a/examples/cli-support/src/fxa_creds.rs b/examples/cli-support/src/fxa_creds.rs index 7bfcccc26fc..287096ad6b1 100644 --- a/examples/cli-support/src/fxa_creds.rs +++ b/examples/cli-support/src/fxa_creds.rs @@ -8,9 +8,7 @@ use std::{collections::HashMap, fs, io::Write}; use anyhow::Result; use url::Url; -use fxa_client::{ - DeviceConfig, DeviceType, FirefoxAccount, FxaConfig, FxaError, FxaEvent, FxaState, -}; +use fxa_client::{DeviceConfig, DeviceType, FirefoxAccount, FxaConfig, FxaEvent, FxaState}; use sync15::{client::Sync15StorageClientInit, KeyBundle}; use crate::{prompt::prompt_string, workspace_root_dir}; @@ -133,26 +131,11 @@ impl CliFxa { match state { FxaState::Connected => { - crate::info!("FxA: already connected - checking if we have all the scopes."); - let mut have_all_scopes = true; - for scope in scopes { - match account.get_access_token(scope, true) { - Ok(_) => crate::debug!("Do already have the {scope:?} scope"), - Err(FxaError::Forbidden) => { - crate::info!("Don't have the {scope:?} scope, re-authenticating"); - have_all_scopes = false; - break; - } - Err(e) => { - crate::error!("Error checking for the {scope:?} scope: {e}"); - return Err(e.into()); - } - } - } + let have_all_scopes = account.has_scope(&scopes.join(" ")); + crate::info!("FxA: already connected, all scopes is {have_all_scopes}"); if !have_all_scopes { self.handle_oauth_flow(service, scopes)?; } - self.persist()?; } FxaState::Disconnected | FxaState::AuthIssues => { crate::info!("FxA: need to authenticate (state was {state:?})"); From 93cab4035632b05255df4cf3cea186d80c81962c Mon Sep 17 00:00:00 2001 From: Schmidt Date: Thu, 9 Jul 2026 12:37:08 +0200 Subject: [PATCH 09/59] test Logins.add_with_meta with duplicate ids (#7462) Current behaviour is to update an existing record with the same id. --- components/logins/src/db.rs | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/components/logins/src/db.rs b/components/logins/src/db.rs index 0bafc7f2f0c..576fd03f4db 100644 --- a/components/logins/src/db.rs +++ b/components/logins/src/db.rs @@ -1528,6 +1528,55 @@ mod tests { assert_eq!(fetched.meta, meta); } + #[test] + fn test_add_with_meta_duplicate_id() { + ensure_initialized(); + + let guid = Guid::random(); + let now_ms = util::system_time_ms_i64(SystemTime::now()); + let meta = LoginMeta { + id: guid.to_string(), + time_created: now_ms, + time_password_changed: now_ms, + time_last_used: now_ms, + times_used: 1, + time_last_breach_alert_dismissed: None, + }; + + let db = LoginDb::open_in_memory(); + db.add_with_meta(LoginEntryWithMeta { + entry: LoginEntry { + origin: "https://www.example.com".into(), + http_realm: Some("https://www.example.com".into()), + username: "test".into(), + password: "sekret".into(), + ..LoginEntry::default() + }, + meta: meta.clone(), + }) + .expect("should be able to add login with record"); + + // Adding a second login that reuses the same id (different origin so the + // dupe-check passes) succeeds and replaces the existing record. + db.add_with_meta(LoginEntryWithMeta { + entry: LoginEntry { + origin: "https://www.other.com".into(), + http_realm: Some("https://www.other.com".into()), + username: "test".into(), + password: "sekret".into(), + ..LoginEntry::default() + }, + meta, + }) + .expect("should be able to re-add a login with the same id"); + + let fetched = db + .get_by_id(&guid) + .expect("should work") + .expect("should get a record"); + assert_eq!(fetched.fields.origin, "https://www.other.com"); + } + #[test] fn test_record_potentially_vulnerable_passwords() { ensure_initialized(); From b42e06366e18aa937c739a5d6f8d49ca574e32b7 Mon Sep 17 00:00:00 2001 From: Schmidt Date: Thu, 9 Jul 2026 19:46:08 +0200 Subject: [PATCH 10/59] Bug 2053557 - remove all logins without decrypting (#7467) --- CHANGELOG.md | 4 + components/logins/src/db.rs | 142 +++++++++++++++++++++++++++ components/logins/src/login.rs | 7 ++ components/logins/src/logins.udl | 13 +++ components/logins/src/store.rs | 16 +++ components/logins/src/sync/engine.rs | 9 +- 6 files changed, 183 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8be15c65e0f..1455f48c644 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ ### Logins - Add `LoginStore.bridgedEngine()`, which exposes the logins sync engine to Desktop's Sync. ([bug 2049263](https://bugzilla.mozilla.org/show_bug.cgi?id=2049263)) +- Add `LoginStore.delete_all()`, which deletes all logins + and `delete_all_axcept_fxa()`, which deletes all logins preserving the FxA session-credentials login + and `LoginStore.wipe_local_except_fxa()`, a variant of `wipe_local()` that preserves the FxA session-credentials login + ([#7467](https://github.com/mozilla/application-services/pull/7467)) ([Bug 2053557](https://bugzilla.mozilla.org/show_bug.cgi?id=2053557)) # v153.0 (_2026-06-15_) diff --git a/components/logins/src/db.rs b/components/logins/src/db.rs index 576fd03f4db..a200a550484 100644 --- a/components/logins/src/db.rs +++ b/components/logins/src/db.rs @@ -874,6 +874,33 @@ impl LoginDb { Ok(results.pop().expect("there should be a single result")) } + // Delete all records. Return an array with the ids of the deleted logins + pub fn delete_all(&self) -> Result> { + let ids: Vec = self.db.query_rows_and_then_cached( + "SELECT guid FROM loginsL WHERE is_deleted = 0 + UNION ALL + SELECT guid FROM loginsM WHERE is_overridden = 0", + [], + |row| row.get(0), + )?; + self.delete_many(ids.iter().map(String::as_str).collect())?; + Ok(ids) + } + + // Delete all records, except the FxA login. Return an array with the ids of + // the deleted logins + pub fn delete_all_except_fxa(&self) -> Result> { + let ids: Vec = self.db.query_rows_and_then_cached( + "SELECT guid FROM loginsL WHERE is_deleted = 0 AND origin != :fxa_origin + UNION ALL + SELECT guid FROM loginsM WHERE is_overridden = 0 AND origin != :fxa_origin", + named_params! { ":fxa_origin": FXA_CREDENTIALS_ORIGIN }, + |row| row.get(0), + )?; + self.delete_many(ids.iter().map(String::as_str).collect())?; + Ok(ids) + } + /// Delete the records with the specified IDs. Returns a list of Boolean values /// indicating whether the respective records already existed. pub fn delete_many(&self, ids: Vec<&str>) -> Result> { @@ -1033,6 +1060,25 @@ impl LoginDb { Ok(row_count) } + /// Wipe all local data except the FxA login, returns the number of rows deleted + pub fn wipe_local_except_fxa(&self) -> Result { + info!("Executing wipe_local_except_fxa on password engine!"); + let tx = self.unchecked_transaction()?; + let mut row_count = 0; + row_count += self.execute( + "DELETE FROM loginsL WHERE origin != :fxa_origin", + named_params! { ":fxa_origin": FXA_CREDENTIALS_ORIGIN }, + )?; + row_count += self.execute( + "DELETE FROM loginsM WHERE origin != :fxa_origin", + named_params! { ":fxa_origin": FXA_CREDENTIALS_ORIGIN }, + )?; + row_count += self.execute("DELETE FROM loginsSyncMeta", [])?; + row_count += self.execute("DELETE FROM breachesL", [])?; + tx.commit()?; + Ok(row_count) + } + pub fn shutdown(self) -> Result<()> { self.db.close().map_err(|(_, e)| Error::SqlError(e)) } @@ -2099,6 +2145,102 @@ mod tests { assert!(!result[0]); } + #[test] + fn test_delete_all() { + ensure_initialized(); + let db = LoginDb::open_in_memory(); + let login_a = db + .add(LoginEntry { + origin: "https://a.example.com".into(), + http_realm: Some("https://www.example.com".into()), + username: "test_user".into(), + password: "test_password".into(), + ..Default::default() + }) + .unwrap(); + let login_b = db + .add(LoginEntry { + origin: "https://b.example.com".into(), + http_realm: Some("https://www.example.com".into()), + username: "test_user".into(), + password: "test_password".into(), + ..Default::default() + }) + .unwrap(); + + let mut deleted = db.delete_all().unwrap(); + deleted.sort(); + let mut expected = vec![login_a.meta.id.clone(), login_b.meta.id.clone()]; + expected.sort(); + assert_eq!(deleted, expected); + assert!(!db.exists(login_a.guid_str()).unwrap()); + assert!(!db.exists(login_b.guid_str()).unwrap()); + + // On an empty database it's a no-op returning no ids. + assert_eq!(db.delete_all().unwrap(), Vec::::new()); + } + + #[test] + fn test_delete_all_except_fxa() { + ensure_initialized(); + let db = LoginDb::open_in_memory(); + let login = db + .add(LoginEntry { + origin: "https://a.example.com".into(), + http_realm: Some("https://www.example.com".into()), + username: "test_user".into(), + password: "test_password".into(), + ..Default::default() + }) + .unwrap(); + let fxa_login = db + .add(LoginEntry { + origin: FXA_CREDENTIALS_ORIGIN.into(), + http_realm: Some("https://www.example.com".into()), + username: "test_user".into(), + password: "test_password".into(), + ..Default::default() + }) + .unwrap(); + + let deleted = db.delete_all_except_fxa().unwrap(); + assert_eq!(deleted, vec![login.meta.id.clone()]); + + // Only the FxA login remains. + assert!(!db.exists(login.guid_str()).unwrap()); + assert!(db.exists(fxa_login.guid_str()).unwrap()); + } + + #[test] + fn test_wipe_local_except_fxa() { + ensure_initialized(); + let db = LoginDb::open_in_memory(); + let login = db + .add(LoginEntry { + origin: "https://a.example.com".into(), + http_realm: Some("https://www.example.com".into()), + username: "test_user".into(), + password: "test_password".into(), + ..Default::default() + }) + .unwrap(); + let fxa_login = db + .add(LoginEntry { + origin: FXA_CREDENTIALS_ORIGIN.into(), + http_realm: Some("https://www.example.com".into()), + username: "test_user".into(), + password: "test_password".into(), + ..Default::default() + }) + .unwrap(); + + db.wipe_local_except_fxa().unwrap(); + + // Only the FxA login remains. + assert!(!db.exists(login.guid_str()).unwrap()); + assert!(db.exists(fxa_login.guid_str()).unwrap()); + } + #[test] fn test_delete_local_for_remote_replacement() { ensure_initialized(); diff --git a/components/logins/src/login.rs b/components/logins/src/login.rs index afef2c0fd15..22bfdfe2096 100644 --- a/components/logins/src/login.rs +++ b/components/logins/src/login.rs @@ -284,6 +284,13 @@ use serde_derive::*; use sync_guid::Guid; use url::Url; +// The Desktop FxA session-credentials pseudo-login. Firefox stores its account +// credentials as a login under this origin; it must never be synced. This +// mirrors the exclusion the JS `PasswordEngine` does via +// `Utils.getSyncCredentialsHosts()`. Only relevant on Desktop (mobile never has +// such a login), but it's harmless to filter everywhere. +pub(crate) const FXA_CREDENTIALS_ORIGIN: &str = "chrome://FirefoxAccounts"; + // LoginEntry fields that are stored in cleartext #[derive(Debug, Clone, Hash, PartialEq, Eq, Default)] pub struct LoginFields { diff --git a/components/logins/src/logins.udl b/components/logins/src/logins.udl index 3feea9642d2..4d6cb02ba33 100644 --- a/components/logins/src/logins.udl +++ b/components/logins/src/logins.udl @@ -199,6 +199,15 @@ interface LoginStore { [Throws=LoginsApiError, Self=ByArc] sequence delete_many(sequence ids); + /// Delete all logins. Returns the ids of the deleted logins. + [Throws=LoginsApiError] + sequence delete_all(); + + /// Delete all logins except the FxA session-credentials login. Returns the + /// ids of the deleted logins. + [Throws=LoginsApiError] + sequence delete_all_except_fxa(); + /// Clear out locally stored logins data /// /// If sync is enabled, then we will try to recover the data on the next sync. @@ -213,6 +222,10 @@ interface LoginStore { [Throws=LoginsApiError] void wipe_local(); + /// Like `wipe_local`, but preserves the FxA session-credentials login. + [Throws=LoginsApiError] + void wipe_local_except_fxa(); + [Throws=LoginsApiError, Self=ByArc] void reset(); diff --git a/components/logins/src/store.rs b/components/logins/src/store.rs index eba7366c89c..b347376a060 100644 --- a/components/logins/src/store.rs +++ b/components/logins/src/store.rs @@ -226,6 +226,16 @@ impl LoginStore { self.lock_db()?.delete_many(ids) } + #[handle_error(Error)] + pub fn delete_all(&self) -> ApiResult> { + self.lock_db()?.delete_all() + } + + #[handle_error(Error)] + pub fn delete_all_except_fxa(&self) -> ApiResult> { + self.lock_db()?.delete_all_except_fxa() + } + #[handle_error(Error)] pub fn delete_undecryptable_records_for_remote_replacement( self: Arc, @@ -249,6 +259,12 @@ impl LoginStore { Ok(()) } + #[handle_error(Error)] + pub fn wipe_local_except_fxa(&self) -> ApiResult<()> { + self.lock_db()?.wipe_local_except_fxa()?; + Ok(()) + } + #[handle_error(Error)] pub fn reset(self: Arc) -> ApiResult<()> { // Reset should not exist here - all resets should be done via the diff --git a/components/logins/src/sync/engine.rs b/components/logins/src/sync/engine.rs index c1e1a538d24..81511ea8eb5 100644 --- a/components/logins/src/sync/engine.rs +++ b/components/logins/src/sync/engine.rs @@ -8,7 +8,7 @@ use super::SyncStatus; use crate::db::CLONE_ENTIRE_MIRROR_SQL; use crate::encryption::EncryptorDecryptor; use crate::error::*; -use crate::login::EncryptedLogin; +use crate::login::{EncryptedLogin, FXA_CREDENTIALS_ORIGIN}; use crate::schema; use crate::util; use crate::LoginDb; @@ -24,13 +24,6 @@ use sync15::engine::{CollSyncIds, CollectionRequest, EngineSyncAssociation, Sync use sync15::{telemetry, ServerTimestamp}; use sync_guid::Guid; -// The Desktop FxA session-credentials pseudo-login. Firefox stores its account -// credentials as a login under this origin; it must never be synced. This -// mirrors the exclusion the JS `PasswordEngine` does via -// `Utils.getSyncCredentialsHosts()`. Only relevant on Desktop (mobile never has -// such a login), but it's harmless to filter everywhere. -const FXA_CREDENTIALS_ORIGIN: &str = "chrome://FirefoxAccounts"; - // The sync engine. pub struct LoginsSyncEngine { pub store: Arc, From 8a3a986691a8f657f99c70f288beb4b6da7616ba Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 10 Jul 2026 09:58:40 -0400 Subject: [PATCH 11/59] Rename the viaduct Android package to `viaduct`. (#7453) Before it was named `httpconfig` for historical reasons, this one seems more natural. --- .buildconfig-android.yml | 8 ++++---- CHANGELOG.md | 1 + components/remote_settings/android/build.gradle | 2 +- components/viaduct/android/build.gradle | 2 +- .../appservices/{httpconfig => viaduct}/FetchBackend.kt | 7 +------ .../appservices/{httpconfig => viaduct}/HttpConfig.kt | 2 +- 6 files changed, 9 insertions(+), 13 deletions(-) rename components/viaduct/android/src/main/java/mozilla/appservices/{httpconfig => viaduct}/FetchBackend.kt (91%) rename components/viaduct/android/src/main/java/mozilla/appservices/{httpconfig => viaduct}/HttpConfig.kt (97%) diff --git a/.buildconfig-android.yml b/.buildconfig-android.yml index eec97ee42af..98aed1a0628 100644 --- a/.buildconfig-android.yml +++ b/.buildconfig-android.yml @@ -63,13 +63,13 @@ projects: - name: rust-log-forwarder type: aar description: Forward logs from Rust - httpconfig: + viaduct: path: components/viaduct/android - artifactId: httpconfig + artifactId: viaduct publications: - - name: httpconfig + - name: viaduct type: aar - description: Component allowing the configuration of Rust HTTP stack. + description: Rust HTTP bridge. push: path: components/push/android artifactId: push diff --git a/CHANGELOG.md b/CHANGELOG.md index 1455f48c644..d64561c2251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ - init_backend and `viaduct_init_backend_hyper` no longer throw an error if they're called multiple times. Instead, we report the error via the Rust components error ping. This is a breaking change for iOS, since the functions no longer throw. +- Renamed the Android package to `mozilla.appservices.viaduct` ## 🔧 What's Fixed 🔧 diff --git a/components/remote_settings/android/build.gradle b/components/remote_settings/android/build.gradle index 3a7dc8ff2f2..b837884e0a1 100644 --- a/components/remote_settings/android/build.gradle +++ b/components/remote_settings/android/build.gradle @@ -53,5 +53,5 @@ dependencies { } else { testImplementation libs.mozilla.concept.fetch } - testImplementation project(":httpconfig") + testImplementation project(":viaduct") } diff --git a/components/viaduct/android/build.gradle b/components/viaduct/android/build.gradle index c354c1ab274..ae2bb6edaaa 100644 --- a/components/viaduct/android/build.gradle +++ b/components/viaduct/android/build.gradle @@ -2,7 +2,7 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" apply from: "$appServicesRootDir/publish.gradle" android { - namespace 'org.mozilla.appservices.httpconfig' + namespace 'org.mozilla.appservices.viaduct' } dependencies { diff --git a/components/viaduct/android/src/main/java/mozilla/appservices/httpconfig/FetchBackend.kt b/components/viaduct/android/src/main/java/mozilla/appservices/viaduct/FetchBackend.kt similarity index 91% rename from components/viaduct/android/src/main/java/mozilla/appservices/httpconfig/FetchBackend.kt rename to components/viaduct/android/src/main/java/mozilla/appservices/viaduct/FetchBackend.kt index 3ea221eeff1..2b41dc88ef7 100644 --- a/components/viaduct/android/src/main/java/mozilla/appservices/httpconfig/FetchBackend.kt +++ b/components/viaduct/android/src/main/java/mozilla/appservices/viaduct/FetchBackend.kt @@ -2,13 +2,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package mozilla.appservices.httpconfig +package mozilla.appservices.viaduct -import mozilla.appservices.viaduct.Backend -import mozilla.appservices.viaduct.ClientSettings -import mozilla.appservices.viaduct.Method -import mozilla.appservices.viaduct.Request -import mozilla.appservices.viaduct.Response import java.util.concurrent.TimeUnit import mozilla.components.concept.fetch.Client as FetchClient import mozilla.components.concept.fetch.Header as FetchHeader diff --git a/components/viaduct/android/src/main/java/mozilla/appservices/httpconfig/HttpConfig.kt b/components/viaduct/android/src/main/java/mozilla/appservices/viaduct/HttpConfig.kt similarity index 97% rename from components/viaduct/android/src/main/java/mozilla/appservices/httpconfig/HttpConfig.kt rename to components/viaduct/android/src/main/java/mozilla/appservices/viaduct/HttpConfig.kt index 20f5ebfef12..7e4f4ab4dcc 100644 --- a/components/viaduct/android/src/main/java/mozilla/appservices/httpconfig/HttpConfig.kt +++ b/components/viaduct/android/src/main/java/mozilla/appservices/viaduct/HttpConfig.kt @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package mozilla.appservices.httpconfig +package mozilla.appservices.viaduct import mozilla.appservices.viaduct.initBackend import mozilla.components.concept.fetch.Client From b49b946a5150340f656fa672a96b431880f9b4a8 Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 10 Jul 2026 14:39:29 -0400 Subject: [PATCH 12/59] Bug 2050966 - Remove stale code in RustComponentsErrorTelemetry.kt (#7469) Nowadays we're using the Rust error ping which is handled internally in the Kotlin module. We no longer need to forward these events for recording in sentry. The android side of this was removed in https://bugzilla.mozilla.org/show_bug.cgi?id=1991443. --- .../RustComponentsErrorTelemetry.kt | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/components/support/error/android/src/main/java/mozilla/appservices/errorsupport/RustComponentsErrorTelemetry.kt b/components/support/error/android/src/main/java/mozilla/appservices/errorsupport/RustComponentsErrorTelemetry.kt index 4f887c2ff05..722e974e3e2 100644 --- a/components/support/error/android/src/main/java/mozilla/appservices/errorsupport/RustComponentsErrorTelemetry.kt +++ b/components/support/error/android/src/main/java/mozilla/appservices/errorsupport/RustComponentsErrorTelemetry.kt @@ -57,13 +57,6 @@ internal data class TracingErrorFields( val breadcrumbs: String, ) -@Serializable -internal data class TracingBreadcrumbFields( - val module: String, - val line: UInt, - val column: UInt, -) - private class ErrorEventSink : EventSink { val json = Json { ignoreUnknownKeys = true } @@ -74,54 +67,6 @@ private class ErrorEventSink : EventSink { RustComponentErrors.details.set(event.message) RustComponentErrors.breadcrumbs.set(fields.breadcrumbs.split("\n")) Pings.rustComponentErrors.submit() - - ApplicationErrorReporterRegistry.errorReporter?.reportError(fields.typeName, event.message) - } else if (event.target == "app-services-error-reporter::breadcrumb") { - val fields = json.decodeFromString(event.fields) - - ApplicationErrorReporterRegistry.errorReporter?.reportBreadcrumb( - event.message, - fields.module, - fields.line, - fields.column, - ) } } } - -/** - * Report Rust errors to Sentry (supplied by the application - * - * This represents the legacy error reporting interface. We're keeping this around for now so that - * Android can send errors to Sentry. At some point we should migrate Android to only use - * Glean-based error reporting. - */ -public interface ApplicationErrorReporter { - /** - * Report an error - */ - fun reportError(typeName: String, message: String) - - /** - * Report a breadbcrumb - */ - fun reportBreadcrumb(message: String, module: String, line: UInt, column: UInt) -} - -/** - * Set the global ApplicationErrorReporter - */ -public fun setApplicationErrorReporter(errorReporter: ApplicationErrorReporter) { - ApplicationErrorReporterRegistry.errorReporter = errorReporter -} - -/** - * Unset the global ApplicationErrorReporter - */ -public fun unsetApplicationErrorReporter() { - ApplicationErrorReporterRegistry.errorReporter = null -} - -internal object ApplicationErrorReporterRegistry { - var errorReporter: ApplicationErrorReporter? = null -} From 621bc6eb3420eb483da459d04c15eb91c8ebe980 Mon Sep 17 00:00:00 2001 From: bendk Date: Tue, 14 Jul 2026 14:16:26 -0400 Subject: [PATCH 13/59] Update crossbeam-epoch (#7470) I believe this should fix our `cargo audit` CI failures. --- Cargo.lock | 9 ++------- megazords/full/android/dependency-licenses.xml | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1e65c64a9b..92cc57f5fc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -849,16 +849,11 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.9" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "once_cell", - "scopeguard", ] [[package]] diff --git a/megazords/full/android/dependency-licenses.xml b/megazords/full/android/dependency-licenses.xml index b0ca7438659..6dc0de05db4 100644 --- a/megazords/full/android/dependency-licenses.xml +++ b/megazords/full/android/dependency-licenses.xml @@ -314,7 +314,7 @@ the details of which are reproduced below. Apache License 2.0: num-traits - https://github.com/rust-num/num-traits/blob/master/LICENSE-APACHE + https://github.com/rust-num/num-traits/blob/main/LICENSE-APACHE Apache License 2.0: ohttp @@ -394,7 +394,7 @@ the details of which are reproduced below. Apache License 2.0: rustc-hash - https://github.com/rust-lang/rustc-hash/blob/master/LICENSE-APACHE + https://github.com/rust-lang/rustc-hash/blob/main/LICENSE-APACHE Apache License 2.0: rustix From 686f39a2a09441322e339c530eb6703662543ed7 Mon Sep 17 00:00:00 2001 From: Schmidt Date: Wed, 15 Jul 2026 19:41:38 +0200 Subject: [PATCH 14/59] Make Login::shutdown release its EncryoptorDecryptor (#7476) to avoid leaks from foreign providers of the encryptor decryptor trait. --- CHANGELOG.md | 6 +++++ components/logins/src/db.rs | 8 ++++-- components/logins/src/encryption.rs | 38 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d64561c2251..ab34525d10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,12 @@ and `LoginStore.wipe_local_except_fxa()`, a variant of `wipe_local()` that preserves the FxA session-credentials login ([#7467](https://github.com/mozilla/application-services/pull/7467)) ([Bug 2053557](https://bugzilla.mozilla.org/show_bug.cgi?id=2053557)) +## 🔧 What's Fixed 🔧 + +### Logins + +- `LoginStore.shutdown()` now releases its reference to the consumer-provided `EncryptorDecryptor` so foreign (e.g. JS) callback handles are unregistered during shutdown instead of lingering. ([#PRNUM](https://github.com/mozilla/application-services/pull/7476)) + # v153.0 (_2026-06-15_) ## ⚠️ Breaking Changes ⚠️ diff --git a/components/logins/src/db.rs b/components/logins/src/db.rs index a200a550484..34611041372 100644 --- a/components/logins/src/db.rs +++ b/components/logins/src/db.rs @@ -22,7 +22,7 @@ /// server. /// - After we sync, we move all records from loginsL to loginsM, overwriting any previous data. /// loginsL will be an empty table after this. See mark_as_synchronized() for the details. -use crate::encryption::EncryptorDecryptor; +use crate::encryption::{EncryptorDecryptor, NoopEncryptorDecryptor}; use crate::error::*; use crate::login::*; use crate::schema; @@ -1079,7 +1079,11 @@ impl LoginDb { Ok(row_count) } - pub fn shutdown(self) -> Result<()> { + pub fn shutdown(mut self) -> Result<()> { + // Drop our reference to the (possibly foreign/JS-backed) encryptor before + // we tear the rest down, so its callback handle is released during + // shutdown instead of lingering. + self.encdec = Arc::new(NoopEncryptorDecryptor); self.db.close().map_err(|(_, e)| Error::SqlError(e)) } } diff --git a/components/logins/src/encryption.rs b/components/logins/src/encryption.rs index c1b48121a97..afc8b70b5de 100644 --- a/components/logins/src/encryption.rs +++ b/components/logins/src/encryption.rs @@ -88,6 +88,29 @@ impl EncryptorDecryptor for Arc { } } +/// A placeholder `EncryptorDecryptor` used after the store has been shut down. +/// +/// On shutdown we swap the real encryptor (which, for foreign consumers like +/// Desktop, is a JS-backed callback interface) out for this one. That drops our +/// reference to the foreign callback so its handle is unregistered during +/// shutdown rather than lingering. Any call that still reaches it (e.g. via a +/// stray clone) fails cleanly instead of calling into torn-down foreign code. +pub struct NoopEncryptorDecryptor; + +impl EncryptorDecryptor for NoopEncryptorDecryptor { + fn encrypt(&self, _clearbytes: Vec) -> ApiResult> { + Err(LoginsApiError::UnexpectedLoginsApiError { + reason: "encrypt called on a shut-down store".to_string(), + }) + } + + fn decrypt(&self, _cipherbytes: Vec) -> ApiResult> { + Err(LoginsApiError::UnexpectedLoginsApiError { + reason: "decrypt called on a shut-down store".to_string(), + }) + } +} + /// The ManagedEncryptorDecryptor makes use of the NSS provided cryptographic algorithms. The /// ManagedEncryptorDecryptor uses a KeyManager for encryption key retrieval. pub struct ManagedEncryptorDecryptor { @@ -372,6 +395,21 @@ mod tests { assert_eq!(key.as_bytes(), key_manager.get_key().unwrap()); } + #[test] + fn test_noop_encdec_errors() { + // The placeholder we swap in on shutdown must never encrypt/decrypt; it + // should fail cleanly instead. + let encdec = NoopEncryptorDecryptor; + assert!(matches!( + encdec.encrypt("secret".as_bytes().into()).err().unwrap(), + LoginsApiError::UnexpectedLoginsApiError { .. } + )); + assert!(matches!( + encdec.decrypt("secret".as_bytes().into()).err().unwrap(), + LoginsApiError::UnexpectedLoginsApiError { .. } + )); + } + #[test] fn test_managed_encdec_with_invalid_key() { ensure_initialized(); From 2489a78655260e98ea5ac2e87d371173928ea229 Mon Sep 17 00:00:00 2001 From: Schmidt Date: Thu, 16 Jul 2026 16:03:15 +0200 Subject: [PATCH 15/59] Revert "Make Login::shutdown release its EncryoptorDecryptor (#7476)" (#7479) This reverts commit 73048777916d7561bac001d1b875edfaec2236be. --- CHANGELOG.md | 6 ----- components/logins/src/db.rs | 8 ++---- components/logins/src/encryption.rs | 38 ----------------------------- 3 files changed, 2 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab34525d10b..d64561c2251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,12 +21,6 @@ and `LoginStore.wipe_local_except_fxa()`, a variant of `wipe_local()` that preserves the FxA session-credentials login ([#7467](https://github.com/mozilla/application-services/pull/7467)) ([Bug 2053557](https://bugzilla.mozilla.org/show_bug.cgi?id=2053557)) -## 🔧 What's Fixed 🔧 - -### Logins - -- `LoginStore.shutdown()` now releases its reference to the consumer-provided `EncryptorDecryptor` so foreign (e.g. JS) callback handles are unregistered during shutdown instead of lingering. ([#PRNUM](https://github.com/mozilla/application-services/pull/7476)) - # v153.0 (_2026-06-15_) ## ⚠️ Breaking Changes ⚠️ diff --git a/components/logins/src/db.rs b/components/logins/src/db.rs index 34611041372..a200a550484 100644 --- a/components/logins/src/db.rs +++ b/components/logins/src/db.rs @@ -22,7 +22,7 @@ /// server. /// - After we sync, we move all records from loginsL to loginsM, overwriting any previous data. /// loginsL will be an empty table after this. See mark_as_synchronized() for the details. -use crate::encryption::{EncryptorDecryptor, NoopEncryptorDecryptor}; +use crate::encryption::EncryptorDecryptor; use crate::error::*; use crate::login::*; use crate::schema; @@ -1079,11 +1079,7 @@ impl LoginDb { Ok(row_count) } - pub fn shutdown(mut self) -> Result<()> { - // Drop our reference to the (possibly foreign/JS-backed) encryptor before - // we tear the rest down, so its callback handle is released during - // shutdown instead of lingering. - self.encdec = Arc::new(NoopEncryptorDecryptor); + pub fn shutdown(self) -> Result<()> { self.db.close().map_err(|(_, e)| Error::SqlError(e)) } } diff --git a/components/logins/src/encryption.rs b/components/logins/src/encryption.rs index afc8b70b5de..c1b48121a97 100644 --- a/components/logins/src/encryption.rs +++ b/components/logins/src/encryption.rs @@ -88,29 +88,6 @@ impl EncryptorDecryptor for Arc { } } -/// A placeholder `EncryptorDecryptor` used after the store has been shut down. -/// -/// On shutdown we swap the real encryptor (which, for foreign consumers like -/// Desktop, is a JS-backed callback interface) out for this one. That drops our -/// reference to the foreign callback so its handle is unregistered during -/// shutdown rather than lingering. Any call that still reaches it (e.g. via a -/// stray clone) fails cleanly instead of calling into torn-down foreign code. -pub struct NoopEncryptorDecryptor; - -impl EncryptorDecryptor for NoopEncryptorDecryptor { - fn encrypt(&self, _clearbytes: Vec) -> ApiResult> { - Err(LoginsApiError::UnexpectedLoginsApiError { - reason: "encrypt called on a shut-down store".to_string(), - }) - } - - fn decrypt(&self, _cipherbytes: Vec) -> ApiResult> { - Err(LoginsApiError::UnexpectedLoginsApiError { - reason: "decrypt called on a shut-down store".to_string(), - }) - } -} - /// The ManagedEncryptorDecryptor makes use of the NSS provided cryptographic algorithms. The /// ManagedEncryptorDecryptor uses a KeyManager for encryption key retrieval. pub struct ManagedEncryptorDecryptor { @@ -395,21 +372,6 @@ mod tests { assert_eq!(key.as_bytes(), key_manager.get_key().unwrap()); } - #[test] - fn test_noop_encdec_errors() { - // The placeholder we swap in on shutdown must never encrypt/decrypt; it - // should fail cleanly instead. - let encdec = NoopEncryptorDecryptor; - assert!(matches!( - encdec.encrypt("secret".as_bytes().into()).err().unwrap(), - LoginsApiError::UnexpectedLoginsApiError { .. } - )); - assert!(matches!( - encdec.decrypt("secret".as_bytes().into()).err().unwrap(), - LoginsApiError::UnexpectedLoginsApiError { .. } - )); - } - #[test] fn test_managed_encdec_with_invalid_key() { ensure_initialized(); From f7ce81bb1b944ba2928dcca70f297c66f8a31c95 Mon Sep 17 00:00:00 2001 From: Schmidt Date: Thu, 16 Jul 2026 18:12:43 +0200 Subject: [PATCH 16/59] don't clone the encdec in the logins sync engine (#7478) --- CHANGELOG.md | 6 ++++++ components/logins/src/sync/engine.rs | 28 +++++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d64561c2251..a7f88d5735f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,12 @@ and `LoginStore.wipe_local_except_fxa()`, a variant of `wipe_local()` that preserves the FxA session-credentials login ([#7467](https://github.com/mozilla/application-services/pull/7467)) ([Bug 2053557](https://bugzilla.mozilla.org/show_bug.cgi?id=2053557)) +## 🔧 What's Fixed 🔧 + +### Logins + +- The logins sync engine no longer holds its own long-lived clone of the `EncryptorDecryptor`, fetching it from the store on demand instead. This ensures no dangling reference keeps a foreign (e.g. JS) callback handle alive past `shutdown()`. ([#7478](https://github.com/mozilla/application-services/pull/7478)) + # v153.0 (_2026-06-15_) ## ⚠️ Breaking Changes ⚠️ diff --git a/components/logins/src/sync/engine.rs b/components/logins/src/sync/engine.rs index 81511ea8eb5..2e807fca864 100644 --- a/components/logins/src/sync/engine.rs +++ b/components/logins/src/sync/engine.rs @@ -28,7 +28,6 @@ use sync_guid::Guid; pub struct LoginsSyncEngine { pub store: Arc, pub scope: SqlInterruptScope, - pub encdec: Arc, // `Mutex` (rather than `RefCell`) so the engine is `Sync`, which the // Desktop `BridgedEngineAdaptor` requires. Only ever locked briefly. pub staged: Mutex>, @@ -36,18 +35,22 @@ pub struct LoginsSyncEngine { impl LoginsSyncEngine { pub fn new(store: Arc) -> Result { - let db = store.lock_db()?; - let scope = db.begin_interrupt_scope()?; - let encdec = db.encdec.clone(); - drop(db); + let scope = store.lock_db()?.begin_interrupt_scope()?; Ok(Self { store, - encdec, scope, staged: Mutex::new(vec![]), }) } + // on Desktop the `EncryptorDecryptor` owns a foreign + // `PrimaryPasswordAuthenticator` callback, and a long-lived `Arc` clone + // held by the engine would keep that callback alive past + // `LoginDb::shutdown` (which drops the db's own reference). + fn encdec(&self) -> Result> { + Ok(self.store.lock_db()?.encdec.clone()) + } + fn reconcile( &self, records: Vec, @@ -55,6 +58,7 @@ impl LoginsSyncEngine { telem: &mut telemetry::EngineIncoming, ) -> Result { let mut plan = UpdatePlan::default(); + let encdec = self.encdec()?; for mut record in records { self.scope.err_if_interrupted()?; @@ -76,7 +80,7 @@ impl LoginsSyncEngine { upstream, upstream_time, server_now, - self.encdec.as_ref(), + encdec.as_ref(), )?; telem.reconciled(1); } @@ -136,10 +140,11 @@ impl LoginsSyncEngine { ) -> Result> { let mut sync_data = Vec::with_capacity(records.len()); { + let encdec = self.encdec()?; let mut seen_ids: HashSet = HashSet::with_capacity(records.len()); for incoming in records.into_iter() { let id = incoming.envelope.id.clone(); - match SyncLoginData::from_bso(incoming, self.encdec.as_ref()) { + match SyncLoginData::from_bso(incoming, encdec.as_ref()) { Ok(v) => sync_data.push(v), Err(e) => { match e { @@ -265,7 +270,7 @@ impl LoginsSyncEngine { } else { let unknown = row.get::<_, Option>("enc_unknown_fields")?; let mut bso = - EncryptedLogin::from_row(row)?.into_bso(self.encdec.as_ref(), unknown)?; + EncryptedLogin::from_row(row)?.into_bso(db.encdec.as_ref(), unknown)?; bso.envelope.sortindex = Some(DEFAULT_SORTINDEX); bso }) @@ -388,7 +393,8 @@ impl LoginsSyncEngine { .form_action_origin .as_ref() .and_then(|s| util::url_host_port(s)); - let enc_fields = l.decrypt_fields(self.encdec.as_ref())?; + let encdec = self.encdec()?; + let enc_fields = l.decrypt_fields(encdec.as_ref())?; let args = named_params! { ":origin": l.fields.origin, ":http_realm": l.fields.http_realm, @@ -413,7 +419,7 @@ impl LoginsSyncEngine { .query_and_then(args, EncryptedLogin::from_row)? .collect::>>()? { - let this_enc_fields = login.decrypt_fields(self.encdec.as_ref())?; + let this_enc_fields = login.decrypt_fields(encdec.as_ref())?; if enc_fields.username == this_enc_fields.username { return Ok(Some(login)); } From 2a293ef3e74217ee59de0031e8a31d1f423403fd Mon Sep 17 00:00:00 2001 From: Wyatt Verchere Date: Thu, 16 Jul 2026 13:24:46 -0700 Subject: [PATCH 17/59] doc: Minor details to documentation for building A-S (#7477) * doc: Adding details to out of date documentation for building application services * fix: review and glean_sym step added * Update docs/building.md Co-authored-by: bendk --------- Co-authored-by: bendk --- docs/building.md | 21 +++++++++++++++++-- .../locally-published-components-in-fenix.md | 2 +- ...lly-published-components-in-firefox-ios.md | 6 ++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/building.md b/docs/building.md index ea24e44b03f..190ad3c7b00 100644 --- a/docs/building.md +++ b/docs/building.md @@ -54,6 +54,11 @@ a number of hours to complete. ```shell PYPATH=$(which python3); ln -s $PYPATH `dirname $PYPATH`/python ``` + 1. Recent brew installations of Python may create unversioned symlinks but not add them to your `PATH`. You can instead run `brew info python` to see their installation locations, and add the corresponding location to your `PATH`. For example: + ```shell + export PATH="/opt/homebrew/opt/python@3.14/libexec/bin:$PATH" + ``` + 1. Install gyp: ```shell wget https://bootstrap.pypa.io/ez_setup.py -O - | python3 - @@ -66,6 +71,14 @@ a number of hours to complete. export PATH="~/tools/gyp:$PATH" ``` 1. If you have additional questions, consult [this guide](https://github.com/mogemimi/pomdog/wiki/How-to-Install-GYP). + 1. If you encounter a `ModuleNotFoundError`, try installing setuptools on your system. For example: + ```shell + brew install python-setuptools + ``` + 1. Similarly, recent brew installations may result in **externally-managed-environment** errors. This can be addressed with `brew install pipx` and replacing the last line with `pipx install .` + + + 1. Make sure your homebrew python's bin folder is on your path by updating your bash/zsh profile with the following: ```shell export PATH="$PATH:$(brew --prefix)/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/bin" @@ -107,8 +120,12 @@ The following instructions assume that you are building `application-services` f 1. NDK (Side by side) version 29.0.14206865 [as configured](https://github.com/mozilla/application-services/blob/bb8cde8a5/taskcluster/docker/linux/Dockerfile#L25) 1. Android SDK Command-line Tools (latest) 1. Set environment variables based on the boostrapped code and the downloaded NDK. Add it to your rc file (either `.zshrc` or `.bashrc` depending on your shell) to make it permanent - 1. Set `JAVA_HOME` to point to the bootstraped JDK 17 installation directory. Ex: `export JAVA_HOME=~/.mozbuild/jdk/jdk-17.0.18+8/` - 1. Set `ANDROID_HOME` to the bootstraped Android SDK. Ex: `export ANDROID_HOME=~/.mozbuild/android-sdk-linux` + 1. Set `JAVA_HOME` to point to the bootstraped JDK 17 installation directory. Ex: `export JAVA_HOME=~/.mozbuild/jdk/jdk-17.0.18+8/` (or `export JAVA_HOME=~/.mozbuild/jdk/jdk-17.0.18+8/Contents/Home` on Mac). + 1. Set `ANDROID_HOME` to the bootstraped Android SDK. As an example: + ```shell + export ANDROID_HOME=~/.mozbuild/android-sdk-linux + ``` + 1. Set `NSS_STATIC` to 1. Ex: `export NSS_STATIC=1` 1. Set `NSS_DIR` to your local NSS folder. Ex: `export NSS_DIR=~/Mozilla/application-services/libs/desktop/linux-x86-64/nss` 1. Set `ANDROID_NDK_ROOT` to the NDK you downloaded via Android Studio, Ex: `export ANDROID_NDK_ROOT=~/.mozbuild/android-sdk-linux/ndk/29.0.14206865` diff --git a/docs/howtos/locally-published-components-in-fenix.md b/docs/howtos/locally-published-components-in-fenix.md index fa96becd14e..53d093dbf9a 100644 --- a/docs/howtos/locally-published-components-in-fenix.md +++ b/docs/howtos/locally-published-components-in-fenix.md @@ -29,7 +29,7 @@ In the root of the app-services repo: Please be sure you have read [our guide to building Fenix](../building.md#building-for-fenix) and successfully built using the instructions there. In particular, this may lead you to adding `sdk.dir` and `ndk.dir` properties, and/or -set environment variables `ANDROID_SDK_ROOT` and `ANDROID_HOME`. +set environment variable `ANDROID_HOME`. In addition to those instructions, you will need: diff --git a/docs/howtos/locally-published-components-in-firefox-ios.md b/docs/howtos/locally-published-components-in-firefox-ios.md index 4cbd20324c4..3ae748c4aaf 100644 --- a/docs/howtos/locally-published-components-in-firefox-ios.md +++ b/docs/howtos/locally-published-components-in-firefox-ios.md @@ -86,6 +86,12 @@ for example, from the root of your application-services directory: cp ./megazords/ios-rust/Sources/MozillaRustComponentsWrapper/Generated/*.swift ../firefox-ios/MozillaRustComponents/Sources/MozillaRustComponentsWrapper/Generated/. ``` +4. Remove the copied over `glean_sym.swift` file. This exists to make glean-sym work in Rust, but is not needed in linking. From the same directory: + +```bash +rm ../firefox-ios/MozillaRustComponents/Sources/MozillaRustComponentsWrapper/Generated/glean_sym.swift +``` + ## Step 3 — Reset caches and build In Xcode: From 30e9a52ead7eb58642674a30f0b0f590594e536e Mon Sep 17 00:00:00 2001 From: Beth Rennie Date: Thu, 16 Jul 2026 17:36:27 -0400 Subject: [PATCH 18/59] Bug 2055331 - Report reason in unenrollment telemetry (#7480) The `nimbsus_events.unenrollment` metric is supposed to contain the reason field, but it just hasn't been reported. Additionally, this removes "is_rollout" field from the `nimbus_events.enrollment` metric as (a) that data is not already reported, (b) the data is not already available on the `EnrollmentChangeEvent` (and adding it would be a breaking change), and (c) the field is not all that useful. --- CHANGELOG.md | 8 ++++++-- .../main/java/org/mozilla/experiments/nimbus/Nimbus.kt | 1 + components/nimbus/metrics.yaml | 5 ++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7f88d5735f..b51545a6086 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Firefox Accounts -- Session-token authenticated requests to the FxA auth-server now use the typed-Bearer token scheme (`Authorization: Bearer fxs_`) instead of Hawk. This is an internal change with no consumer-facing API impact; production routes accept both schemes. See the [authentication schemes reference](https://mozilla.github.io/ecosystem-platform/reference/authentication-schemes). ([#PRNUM](https://github.com/mozilla/application-services/pull/PRNUM)) +- Session-token authenticated requests to the FxA auth-server now use the typed-Bearer token scheme (`Authorization: Bearer fxs_`) instead of Hawk. This is an internal change with no consumer-facing API impact; production routes accept both schemes. See the [authentication schemes reference](https://mozilla.github.io/ecosystem-platform/reference/authentication-schemes). ([#7432](https://github.com/mozilla/application-services/pull/7432)) [Full Changelog](In progress) @@ -16,7 +16,7 @@ ### Logins - Add `LoginStore.bridgedEngine()`, which exposes the logins sync engine to Desktop's Sync. ([bug 2049263](https://bugzilla.mozilla.org/show_bug.cgi?id=2049263)) -- Add `LoginStore.delete_all()`, which deletes all logins +- Add `LoginStore.delete_all()`, which deletes all logins and `delete_all_axcept_fxa()`, which deletes all logins preserving the FxA session-credentials login and `LoginStore.wipe_local_except_fxa()`, a variant of `wipe_local()` that preserves the FxA session-credentials login ([#7467](https://github.com/mozilla/application-services/pull/7467)) ([Bug 2053557](https://bugzilla.mozilla.org/show_bug.cgi?id=2053557)) @@ -27,6 +27,10 @@ - The logins sync engine no longer holds its own long-lived clone of the `EncryptorDecryptor`, fetching it from the store on demand instead. This ensures no dangling reference keeps a foreign (e.g. JS) callback handle alive past `shutdown()`. ([#7478](https://github.com/mozilla/application-services/pull/7478)) +### Nimbus + +- Reasons are now reported in unenrollment events. ([#7480](https://github.com/mozilla/application-services/pull/7480)) + # v153.0 (_2026-06-15_) ## ⚠️ Breaking Changes ⚠️ diff --git a/components/nimbus/android/src/main/java/org/mozilla/experiments/nimbus/Nimbus.kt b/components/nimbus/android/src/main/java/org/mozilla/experiments/nimbus/Nimbus.kt index a7840b0da88..5a7aaab557d 100644 --- a/components/nimbus/android/src/main/java/org/mozilla/experiments/nimbus/Nimbus.kt +++ b/components/nimbus/android/src/main/java/org/mozilla/experiments/nimbus/Nimbus.kt @@ -665,6 +665,7 @@ open class Nimbus( NimbusEvents.UnenrollmentExtra( experiment = event.experimentSlug, branch = event.branchSlug, + reason = event.reason, ), ) diff --git a/components/nimbus/metrics.yaml b/components/nimbus/metrics.yaml index 6544e9819a6..e162cd4c69c 100644 --- a/components/nimbus/metrics.yaml +++ b/components/nimbus/metrics.yaml @@ -24,13 +24,12 @@ nimbus_events: branch: type: string description: The branch slug/identifier that was randomly chosen - experiment_type: - type: string - description: Indicates whether this is an experiment or rollout bugs: - https://jira.mozilla.com/browse/SDK-61 + - https://bugzilla.mozilla.org/show_bug.cgi?id=2055331 data_reviews: - https://github.com/mozilla-mobile/android-components/pull/9168#issuecomment-743461975 + - https://github.com/mozilla/application-services/pull/7480#issuecomment-4995012840 data_sensitivity: - technical notification_emails: From dd532b308ea3fd64ab800b42dc1d7255b63ba696 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Fri, 17 Jul 2026 12:50:43 +0200 Subject: [PATCH 19/59] RMST-464: remote-settings: Skip verification of unknown signatures types (#7475) * RMST-464: remote-settings: Skip verification of unknown signatures types * Update changelog * Add missing field in tests --- CHANGELOG.md | 4 +++ components/remote_settings/src/client.rs | 44 +++++++++++++++++++++++ components/remote_settings/src/storage.rs | 16 +++++++-- components/search/src/selector.rs | 5 +++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b51545a6086..0b3b10831cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - Session-token authenticated requests to the FxA auth-server now use the typed-Bearer token scheme (`Authorization: Bearer fxs_`) instead of Hawk. This is an internal change with no consumer-facing API impact; production routes accept both schemes. See the [authentication schemes reference](https://mozilla.github.io/ecosystem-platform/reference/authentication-schemes). ([#7432](https://github.com/mozilla/application-services/pull/7432)) +### Remote Settings + +- Skip verification of signatures with unknown signature types ([Bug 2055147](https://bugzilla.mozilla.org/show_bug.cgi?id=2055147)) + [Full Changelog](In progress) ## ✨ What's New ✨ diff --git a/components/remote_settings/src/client.rs b/components/remote_settings/src/client.rs index b951d45c0af..89ec6912a35 100644 --- a/components/remote_settings/src/client.rs +++ b/components/remote_settings/src/client.rs @@ -456,6 +456,12 @@ impl RemoteSettingsClient { "No valid signatures found".into(), )); for signature in &metadata.signatures { + if signature.mode != "p384ecdsa" { + // We currently only support ECDSA P384. + // Change this once `rc_crypto` will support more types (eg. post-quantum algorithms). + continue; + } + let cert_chain_bytes = inner.api_client.fetch_cert(&signature.x5u)?; // The signer name is hard-coded. This would have to be modified in the very (very) @@ -799,6 +805,8 @@ pub struct CollectionSignature { pub signature: String, /// X.509 certificate chain Url (x5u) pub x5u: String, + /// Signature type + pub mode: String, } /// A parsed Remote Settings record. Records can contain arbitrary fields, so clients @@ -1375,6 +1383,7 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= &[CollectionSignature { signature: VALID_SIGNATURE.to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }], VALID_CERT_EPOCH_SECONDS, "main", @@ -1394,10 +1403,39 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= CollectionSignature { signature: "invalid signature".to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), + }, + CollectionSignature { + signature: VALID_SIGNATURE.to_string(), + x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), + }, + ], + VALID_CERT_EPOCH_SECONDS, + "main", + ) + .expect("Valid signature"); + Ok(()) + } + + #[test] + fn test_first_signature_has_unknown_type() -> Result<()> { + ensure_initialized(); + run_client_sync( + &[], + &[], + VALID_CERTIFICATE, + &[ + CollectionSignature { + signature: "unkown signature".to_string(), + x5u: "http://mocked".into(), + // Unknown signature type. + mode: "mldsa".into(), }, CollectionSignature { signature: VALID_SIGNATURE.to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }, ], VALID_CERT_EPOCH_SECONDS, @@ -1423,6 +1461,7 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= &[CollectionSignature { signature: VALID_SIGNATURE.to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }], VALID_CERT_EPOCH_SECONDS, "main", @@ -1441,6 +1480,7 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= &[CollectionSignature { signature: "invalid signature".to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }], VALID_CERT_EPOCH_SECONDS, "main", @@ -1462,6 +1502,7 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= &[CollectionSignature { signature: VALID_SIGNATURE.to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }], VALID_CERT_EPOCH_SECONDS, "main", @@ -1489,6 +1530,7 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= &[CollectionSignature { signature: VALID_SIGNATURE.to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }], december_20_2024, "main", @@ -1522,6 +1564,7 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= &[CollectionSignature { signature: VALID_SIGNATURE.to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }], VALID_CERT_EPOCH_SECONDS, "main", @@ -1544,6 +1587,7 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= &[CollectionSignature { signature: VALID_SIGNATURE.to_string(), x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), }], VALID_CERT_EPOCH_SECONDS, "security-state", diff --git a/components/remote_settings/src/storage.rs b/components/remote_settings/src/storage.rs index 8677980c1dd..0c5b56f70c2 100644 --- a/components/remote_settings/src/storage.rs +++ b/components/remote_settings/src/storage.rs @@ -143,7 +143,8 @@ impl Storage { SELECT cm.bucket, json_extract(sig.value, '$.x5u') AS x5u, - json_extract(sig.value, '$.signature') AS signature + json_extract(sig.value, '$.signature') AS signature, + json_extract(sig.value, '$.mode') AS mode FROM collection_metadata AS cm LEFT JOIN json_each(cm.signatures) AS sig ON true WHERE cm.collection_url = ? @@ -161,8 +162,13 @@ impl Storage { } let x5u: Option = row.get(1)?; let signature: Option = row.get(2)?; - if let (Some(x5u), Some(signature)) = (x5u, signature) { - signatures.push(CollectionSignature { signature, x5u }); + let mode: Option = row.get(3)?; + if let (Some(x5u), Some(signature), Some(mode)) = (x5u, signature, mode) { + signatures.push(CollectionSignature { + signature, + x5u, + mode, + }); } } match bucket { @@ -1123,10 +1129,12 @@ mod tests { CollectionSignature { signature: "b64encodedsig".into(), x5u: "http://15u/".into(), + mode: "mldsa".into(), }, CollectionSignature { signature: "b64encodedsig2".into(), x5u: "http://15u2/".into(), + mode: "p384ecdsa".into(), }, ], }, @@ -1136,8 +1144,10 @@ mod tests { assert_eq!(metadata.signatures[0].signature, "b64encodedsig"); assert_eq!(metadata.signatures[0].x5u, "http://15u/"); + assert_eq!(metadata.signatures[0].mode, "mldsa"); assert_eq!(metadata.signatures[1].signature, "b64encodedsig2"); assert_eq!(metadata.signatures[1].x5u, "http://15u2/"); + assert_eq!(metadata.signatures[1].mode, "p384ecdsa"); Ok(()) } diff --git a/components/search/src/selector.rs b/components/search/src/selector.rs index 44927aae5d0..c1c2b27970a 100644 --- a/components/search/src/selector.rs +++ b/components/search/src/selector.rs @@ -917,6 +917,7 @@ mod tests { "signatures": [{ "x5u": "fake", "signature": "fake", + "mode": "fake", }], }, "timestamp": 1000, @@ -982,6 +983,7 @@ mod tests { "signatures": [{ "x5u": "fake", "signature": "fake", + "mode": "fake", }], }, "timestamp": 1000, @@ -1031,6 +1033,7 @@ mod tests { "signatures": [{ "x5u": "fake", "signature": "fake", + "mode": "fake", }], }, "timestamp": 1000, @@ -1055,6 +1058,7 @@ mod tests { "signatures": [{ "x5u": "fake", "signature": "fake", + "mode": "fake", }], }, "timestamp": 1000, @@ -1142,6 +1146,7 @@ mod tests { "signatures": [{ "x5u": "fake", "signature": "fake", + "mode": "fake", }], }, "timestamp": 1000, From dff667ad93dfee3fd382bf88dc8bd4c4ee4a1c7a Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 17 Jul 2026 11:47:41 -0400 Subject: [PATCH 20/59] Bug 2054982 - Finer grained locking for RemoteSettingService (#7474) Split up `RemoteSettingServiceInner` into multiple structs, each with their own Mutex and add a bit of policy around locking them. Right now this is just enforced by putting all the methods that lock them in one impl block. The issue we're trying to avoid is having `RemoteSettingService::sync` block `RemoteSettingService::make_client`. Also, start using `viaduct::Client` which should be possible now since all applications are using the new backend. --- components/remote_settings/src/service.rs | 186 ++++++++++++++-------- 1 file changed, 122 insertions(+), 64 deletions(-) diff --git a/components/remote_settings/src/service.rs b/components/remote_settings/src/service.rs index b7cc30d3939..7410a068b97 100644 --- a/components/remote_settings/src/service.rs +++ b/components/remote_settings/src/service.rs @@ -7,12 +7,12 @@ use std::{ sync::{Arc, Weak}, }; -use camino::Utf8PathBuf; +use camino::{Utf8Path, Utf8PathBuf}; use error_support::trace; use parking_lot::Mutex; use serde::Deserialize; use url::Url; -use viaduct::Request; +use viaduct::{Client, ClientSettings, Request}; use crate::{ client::RemoteState, config::BaseUrl, error::Error, storage::Storage, @@ -22,16 +22,35 @@ use crate::{ /// Internal Remote settings service API pub struct RemoteSettingsService { - inner: Mutex, + storage_dir: Utf8PathBuf, + // RemoteSettingsService has several mutex fields in order to get finer-grained locking. + // However, this means we need to use some care to avoid holding locks for too long + // and creating potential deadlocks. + // + // To avoid this: put functionality in inner type methods, like [ClientState::update_config]. + // Inside `RemoteSettingsService` methods, we only lock the field temporarily + // to call those inner methods or access the fields. + // Don't hold the lock for longer than that single statement + // and don't lock more than one field in that statement. + sync_client: Mutex, + telemetry: Mutex, + client_state: Mutex, } -struct RemoteSettingsServiceInner { - storage_dir: Utf8PathBuf, +#[derive(Clone)] +struct RemoteSettingsServiceConfig { base_url: BaseUrl, bucket_name: String, app_context: Option, - remote_state: RemoteState, - telemetry: RemoteSettingsTelemetryWrapper, +} + +/// Current config and client list +/// +/// These are stored in the same mutex because we want to update them at the same time. +/// For example, we want to serialize calls to `update_config` and `make_client` so that the new +/// client gets the updated config. +struct ClientState { + config: RemoteSettingsServiceConfig, /// Weakrefs for all clients that we've created. Note: this stores the /// top-level/public `RemoteSettingsClient` structs rather than `client::RemoteSettingsClient`. /// The reason for this is that we return Arcs to the public struct to the foreign code, so we @@ -40,6 +59,12 @@ struct RemoteSettingsServiceInner { clients: Vec>, } +/// Handles the `RemoteSettingsService::sync` method +struct SyncClient { + client: viaduct::Client, + remote_state: RemoteState, +} + impl RemoteSettingsService { /// Construct a [RemoteSettingsService] /// @@ -53,40 +78,35 @@ impl RemoteSettingsService { let bucket_name = config.bucket_name.unwrap_or_else(|| String::from("main")); Self { - inner: Mutex::new(RemoteSettingsServiceInner { - storage_dir, - base_url, - bucket_name, - app_context: config.app_context, - remote_state: RemoteState::default(), - telemetry: RemoteSettingsTelemetryWrapper::noop(), + storage_dir, + client_state: Mutex::new(ClientState { clients: vec![], + config: RemoteSettingsServiceConfig { + base_url, + bucket_name, + app_context: config.app_context, + }, + }), + sync_client: Mutex::new(SyncClient { + client: Client::new(ClientSettings::default()), + remote_state: RemoteState::default(), }), + telemetry: Mutex::new(RemoteSettingsTelemetryWrapper::noop()), } } + fn telemetry(&self) -> RemoteSettingsTelemetryWrapper { + self.telemetry.lock().clone() + } + pub fn set_telemetry(&self, telemetry: RemoteSettingsTelemetryWrapper) { - self.inner.lock().telemetry = telemetry; + *self.telemetry.lock() = telemetry; } pub fn make_client(&self, collection_name: String) -> Arc { - let mut inner = self.inner.lock(); - // Allow using in-memory databases for testing of external crates. - let storage = if inner.storage_dir == ":memory:" { - Storage::new(inner.storage_dir.clone()) - } else { - Storage::new(inner.storage_dir.join(format!("{collection_name}.sql"))) - }; - - let client = Arc::new(RemoteSettingsClient::new( - inner.base_url.clone(), - inner.bucket_name.clone(), - collection_name.clone(), - inner.app_context.clone(), - storage, - )); - inner.clients.push(Arc::downgrade(&client)); - client + self.client_state + .lock() + .make_client(&self.storage_dir, collection_name) } /// Sync collections for all active clients @@ -94,26 +114,31 @@ impl RemoteSettingsService { // Make sure we only sync each collection once, even if there are multiple clients let mut synced_collections = HashSet::new(); - let mut inner = self.inner.lock(); - let changes = inner.fetch_changes()?; + let config = self.client_state.lock().config.clone(); + let telemetry = self.telemetry(); + + let changes = self + .sync_client + .lock() + .fetch_changes(config.base_url, &telemetry)?; let change_map: HashMap<_, _> = changes .changes .iter() .map(|c| ((c.collection.as_str(), &c.bucket), c.last_modified)) .collect(); - let bucket_name = inner.bucket_name.clone(); + let bucket_name = &config.bucket_name; - let active_clients = inner.active_clients(); + let active_clients = self.client_state.lock().active_clients(); for client in &active_clients { let client = &client.internal; let collection_name = client.collection_name(); let cid = format!("{bucket_name}/{collection_name}"); if let Some(client_last_modified) = client.get_last_modified_timestamp()? { - if let Some(server_last_modified) = change_map.get(&(collection_name, &bucket_name)) + if let Some(server_last_modified) = change_map.get(&(collection_name, bucket_name)) { if client_last_modified == *server_last_modified { trace!("skipping up-to-date collection: {collection_name}"); - inner.telemetry.report_uptake_up_to_date(&cid, None); + telemetry.report_uptake_up_to_date(&cid, None); continue; } } @@ -124,8 +149,8 @@ impl RemoteSettingsService { let sync_result = client.sync(); let duration: u64 = start_time.elapsed().as_millis().try_into().unwrap_or(0); match &sync_result { - Ok(()) => inner.telemetry.report_uptake_success(&cid, Some(duration)), - Err(e) => inner.telemetry.report_uptake_error(e, &cid), + Ok(()) => telemetry.report_uptake_success(&cid, Some(duration)), + Err(e) => telemetry.report_uptake_error(e, &cid), } sync_result?; } @@ -146,41 +171,64 @@ impl RemoteSettingsService { Ok(synced_collections.into_iter().collect()) } + pub fn update_config(&self, config: RemoteSettingsConfig) -> Result<()> { + self.client_state.lock().update_config(config) + } + + pub fn client_url(&self) -> Url { + self.client_state.lock().config.base_url.url().clone() + } +} + +impl ClientState { + pub fn make_client( + &mut self, + storage_dir: &Utf8Path, + collection_name: String, + ) -> Arc { + // Allow using in-memory databases for testing of external crates. + let storage = if storage_dir == ":memory:" { + Storage::new(storage_dir.to_path_buf()) + } else { + Storage::new(storage_dir.join(format!("{collection_name}.sql"))) + }; + + let client = Arc::new(RemoteSettingsClient::new( + self.config.base_url.clone(), + self.config.bucket_name.clone(), + collection_name.clone(), + self.config.app_context.clone(), + storage, + )); + self.clients.push(Arc::downgrade(&client)); + client + } + /// Update the remote settings config /// /// This will cause all current and future clients to use new config and will delete any stored /// records causing the clients to return new results from the new config. - pub fn update_config(&self, config: RemoteSettingsConfig) -> Result<()> { + pub fn update_config(&mut self, config: RemoteSettingsConfig) -> Result<()> { let base_url = config .server .unwrap_or(RemoteSettingsServer::Prod) .get_base_url()?; let bucket_name = config.bucket_name.unwrap_or_else(|| String::from("main")); - let mut inner = self.inner.lock(); - for client in inner.active_clients() { + for client in self.active_clients() { client.internal.update_config( base_url.clone(), bucket_name.clone(), config.app_context.clone(), ); } - inner.base_url = base_url; - inner.bucket_name = bucket_name; - inner.app_context = config.app_context; + self.config = RemoteSettingsServiceConfig { + base_url, + bucket_name, + app_context: config.app_context, + }; Ok(()) } - pub fn client_url(&self) -> Url { - let inner = self.inner.lock(); - let base_url = inner.base_url.clone(); - base_url.url().clone() - } -} - -impl RemoteSettingsServiceInner { - // Find live clients in self.clients - // - // Also, drop dead weakrefs from the vec fn active_clients(&mut self) -> Vec> { let mut active_clients = vec![]; self.clients.retain(|weak| { @@ -193,9 +241,21 @@ impl RemoteSettingsServiceInner { }); active_clients } +} - fn fetch_changes(&mut self) -> Result { - let mut url = self.base_url.clone(); +// RemoteSettingsService methods that lock the `telemetry` field. +// +// Let's keep all the calls in one place so that we can ensure that the lock will not be held for a +// long time and these methods can be considered non-blocking. For example, we will never hold the +// lock while making a network request. +impl RemoteSettingsService {} + +impl SyncClient { + fn fetch_changes( + &mut self, + mut url: BaseUrl, + telemetry: &RemoteSettingsTelemetryWrapper, + ) -> Result { url.path_segments_mut() .push("buckets") .push("monitor") @@ -214,7 +274,7 @@ impl RemoteSettingsServiceInner { let start_time = std::time::Instant::now(); let req = Request::get(url); - let resp = req.send()?; + let resp = self.client.send_sync(req)?; self.remote_state.handle_backoff_hint(&resp)?; @@ -222,13 +282,11 @@ impl RemoteSettingsServiceInner { if resp.is_success() { let body = resp.json()?; let duration: u64 = start_time.elapsed().as_millis().try_into().unwrap_or(0); - self.telemetry - .report_uptake_success(TELEMETRY_SOURCE_POLL, Some(duration)); + telemetry.report_uptake_success(TELEMETRY_SOURCE_POLL, Some(duration)); Ok(body) } else { let e = Error::response_error(&resp.url, format!("status code: {}", resp.status)); - self.telemetry - .report_uptake_error(&e, TELEMETRY_SOURCE_POLL); + telemetry.report_uptake_error(&e, TELEMETRY_SOURCE_POLL); Err(e) } } From 116e3628c1aa9f677ef45db02343c0f431948b6a Mon Sep 17 00:00:00 2001 From: Beth Rennie Date: Fri, 17 Jul 2026 13:20:17 -0400 Subject: [PATCH 21/59] Bug 2054479 - Require targeting and bucketing for Firefox Labs (#7481) When queried for the list of all available Firefox Labs, the Nimbus Client was not filtering the recipes by whether they passed targeting and bucketing. This is now the case. There is now a new function `can_enroll` that returns an enum, `CanEnrollResult`, that distinguishes all the different cases that prevent enrollment. The majority of this function was refactored out of `evaluate_enrollment`, which is now a much simpler function. Additionally, the `targeting(expr, helper)` function has been removed. This function took an expression and a `NimbusTargetingHelper` and returned an `EnrollmentStatus` if and only if the targeting evaluation did not succeed or resulted in an error. This API was too awkward to work into `can_enroll`. The majority of this change is test fallout from removing this function, though the tests make more sense now that they are testing the results of JEXL evaluation and not comparing `EnrollmentStatus`es. In order to remove a bunch of unnecessary clones, the `NimbusTargetingHelper` methods now take a `&str` instead of a `String`. The UDL has been updated to use `[ByRef]` so that the FFI contract is unchanged. --- CHANGELOG.md | 1 + components/nimbus/src/evaluator.rs | 209 +++++---- components/nimbus/src/nimbus.udl | 6 +- components/nimbus/src/schema.rs | 4 +- components/nimbus/src/stateful/dbcache.rs | 16 +- .../nimbus/src/stateful/nimbus_client.rs | 14 +- components/nimbus/src/targeting.rs | 10 +- .../src/tests/stateful/test_behavior.rs | 46 +- .../src/tests/stateful/test_evaluator.rs | 442 +++++++----------- .../nimbus/src/tests/stateful/test_nimbus.rs | 89 ++-- .../src/tests/stateful/test_targeting.rs | 12 +- components/nimbus/src/tests/test_evaluator.rs | 221 ++++----- .../nimbus/tests/test_message_helpers.rs | 28 +- components/nimbus/tests/test_restart.rs | 18 +- 14 files changed, 503 insertions(+), 613 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b3b10831cf..4a02e1648f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ ### Nimbus - Reasons are now reported in unenrollment events. ([#7480](https://github.com/mozilla/application-services/pull/7480)) +- The available Firefox Labs are now filtered by targeting and bucketing. ([#7481](https://github.com/mozilla/application-services/pull/7481)) # v153.0 (_2026-06-15_) diff --git a/components/nimbus/src/evaluator.rs b/components/nimbus/src/evaluator.rs index 553830ef3c5..1404f077229 100644 --- a/components/nimbus/src/evaluator.rs +++ b/components/nimbus/src/evaluator.rs @@ -50,79 +50,123 @@ pub fn split_locale(locale: String) -> (Option, Option) { /// Determine the enrolment status for an experiment. /// -/// # Arguments: -/// - `available_randomization_units` The app provided available randomization units -/// - `targeting_attributes` The attributes to use when evaluating targeting -/// - `exp` The `Experiment` to evaluate. -/// -/// # Returns: -/// An `ExperimentEnrollment` - you need to inspect the EnrollmentStatus to -/// determine if the user is actually enrolled. -/// -/// # Errors: -/// -/// The function can return errors in one of the following cases (but not limited to): +/// # Errors /// -/// - If the bucket sampling failed (i.e we could not find if the user should or should not be enrolled in the experiment based on the bucketing) -/// - If an error occurs while determining the branch the user should be enrolled in any of the experiments +/// The function can return an error when branch selection fails due to an +/// invalid bucketing configuration. pub fn evaluate_enrollment( available_randomization_units: &AvailableRandomizationUnits, - exp: &Experiment, - th: &NimbusTargetingHelper, + experiment: &Experiment, + targeting_helper: &NimbusTargetingHelper, ) -> Result { - if let ExperimentAvailable::Unavailable { reason } = is_experiment_available(th, exp, true) { - return Ok(ExperimentEnrollment { - slug: exp.slug.clone(), - status: EnrollmentStatus::NotEnrolled { reason }, - }); - } + let status = match can_enroll(available_randomization_units, targeting_helper, experiment) { + CanEnrollResult::Unavailable { reason } => EnrollmentStatus::NotEnrolled { reason }, + CanEnrollResult::NotTargeted => EnrollmentStatus::NotEnrolled { + reason: NotEnrolledReason::NotTargeted, + }, + CanEnrollResult::NotSelected => EnrollmentStatus::NotEnrolled { + reason: NotEnrolledReason::NotSelected, + }, + CanEnrollResult::TargetingError { reason } => EnrollmentStatus::Error { reason }, + CanEnrollResult::NoRandomizationUnit => { + info!( + "Could not find a suitable randomization unit for {}. Skipping experiment.", + experiment.slug, + ); + EnrollmentStatus::Error { + reason: "No randomization unit".into(), + } + } + + CanEnrollResult::Enrollable { randomization_id } => EnrollmentStatus::new_enrolled( + EnrolledReason::Qualified, + &choose_branch(&experiment.slug, &experiment.branches, randomization_id)?.slug, + ), + }; + + Ok(ExperimentEnrollment { + slug: experiment.slug.clone(), + status, + }) +} + +/// Whether or not an experiment can be enrolled. +pub enum CanEnrollResult<'aru> { + /// The experiment is enrollable. + Enrollable { + /// The randomization ID that should be used for branch selection. + randomization_id: &'aru str, + }, + + /// The experiment is not available for a reason outlined in [`NotEnrolledReason`] + Unavailable { + /// The reason the enrollment is not available. + reason: NotEnrolledReason, + }, + + /// The experiment is not enrollable due to a targeting error. + TargetingError { + /// The stringified error. + reason: String, + }, + + /// The experiment is not enrollable because targeting expression evaluated + /// to false. + NotTargeted, - // Get targeting out of the way - "if let chains" are experimental, - // otherwise we could improve this. - if let Some(expr) = &exp.targeting - && let Some(status) = targeting(expr, th) + /// The experiment is not enrollable because randomization ID did not fall + /// into a selected bucket. + NotSelected, + + /// The experiment is not enrollable because it requires a randomization + /// unit that is not available. + NoRandomizationUnit, +} + +/// Determine whether or not it is possible to enroll in the given experiment. +pub fn can_enroll<'aru>( + available_randomization_units: &'aru AvailableRandomizationUnits, + targeting_helper: &NimbusTargetingHelper, + experiment: &Experiment, +) -> CanEnrollResult<'aru> { + if let ExperimentAvailable::Unavailable { reason } = + is_experiment_available(targeting_helper, experiment, true) { - return Ok(ExperimentEnrollment { - slug: exp.slug.clone(), - status, - }); + return CanEnrollResult::Unavailable { reason }; } - Ok(ExperimentEnrollment { - slug: exp.slug.clone(), - status: { - let bucket_config = exp.bucket_config.clone(); - match available_randomization_units.get_value(&bucket_config.randomization_unit) { - Some(id) => { - if sampling::bucket_sample( - vec![id.to_owned(), bucket_config.namespace], - bucket_config.start, - bucket_config.count, - bucket_config.total, - )? { - EnrollmentStatus::new_enrolled( - EnrolledReason::Qualified, - &choose_branch(&exp.slug, &exp.branches, id)?.clone().slug, - ) - } else { - EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotSelected, - } - } - } - None => { - // XXX: When we link in glean, it would be nice if we could emit - // a failure telemetry event here. - info!( - "Could not find a suitable randomization unit for {}. Skipping experiment.", - &exp.slug - ); - EnrollmentStatus::Error { - reason: "No randomization unit".into(), - } - } + + if let Some(targeting_expression) = &experiment.targeting { + match targeting_helper.eval_jexl(targeting_expression) { + Err(e) => { + return CanEnrollResult::TargetingError { + reason: e.to_string(), + }; } - }, - }) + Ok(false) => return CanEnrollResult::NotTargeted, + Ok(true) => {} + }; + } + + let Some(randomization_id) = + available_randomization_units.get_value(&experiment.bucket_config.randomization_unit) + else { + return CanEnrollResult::NoRandomizationUnit; + }; + + let Ok(is_sampled) = sampling::bucket_sample( + [randomization_id, &experiment.bucket_config.namespace], + experiment.bucket_config.start, + experiment.bucket_config.count, + experiment.bucket_config.total, + ) else { + return CanEnrollResult::NoRandomizationUnit; + }; + + if is_sampled { + CanEnrollResult::Enrollable { randomization_id } + } else { + CanEnrollResult::NotSelected + } } /// Whether or not an experiment is available. @@ -222,41 +266,6 @@ pub(crate) fn choose_branch<'a>( branches.get(index).ok_or(NimbusError::OutOfBoundsError) } -/// Checks if the client is targeted by an experiment -/// This api evaluates the JEXL statement retrieved from the server -/// against the application context provided by the client -/// -/// # Arguments -/// - `expression_statement`: The JEXL statement provided by the server -/// - `targeting_attributes`: The client attributes to target against -/// -/// If this app can not be targeted, returns an EnrollmentStatus to indicate -/// why. Returns None if we should continue to evaluate the enrollment status. -/// -/// In practice, if this returns an EnrollmentStatus, it will be either -/// EnrollmentStatus::NotEnrolled, or EnrollmentStatus::Error in the following -/// cases (But not limited to): -/// - The `expression_statement` is not a valid JEXL statement -/// - The `expression_statement` expects fields that do not exist in the AppContext definition -/// - The result of evaluating the statement against the context is not a boolean -/// - jexl-rs returned an error -pub(crate) fn targeting( - expression_statement: &str, - targeting_helper: &NimbusTargetingHelper, -) -> Option { - match targeting_helper.eval_jexl(expression_statement.to_string()) { - Ok(res) => match res { - true => None, - false => Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted, - }), - }, - Err(e) => Some(EnrollmentStatus::Error { - reason: e.to_string(), - }), - } -} - #[cfg(test)] mod unit_tests { use super::*; diff --git a/components/nimbus/src/nimbus.udl b/components/nimbus/src/nimbus.udl index 62b89190889..7eb73f04bcd 100644 --- a/components/nimbus/src/nimbus.udl +++ b/components/nimbus/src/nimbus.udl @@ -154,7 +154,7 @@ interface GeckoPrefHandler { void set_gecko_prefs_state(sequence new_prefs_state); void set_gecko_prefs_original_values(sequence original_gecko_prefs); - + }; dictionary GeckoPref { @@ -469,12 +469,12 @@ interface NimbusTargetingHelper { /// Execute the given jexl expression and evaluate against the existing targeting parameters and context passed to /// the helper at construction. [Throws=NimbusError] - boolean eval_jexl(string expression); + boolean eval_jexl([ByRef] string expression); /// Evaluate a JEXL expression and return debug results as JSON. /// For CLI testing and debugging. [Throws=NimbusError] - string eval_jexl_debug(string expression); + string eval_jexl_debug([ByRef] string expression); }; interface NimbusStringHelper { diff --git a/components/nimbus/src/schema.rs b/components/nimbus/src/schema.rs index cd80626540a..43f9be77cc6 100644 --- a/components/nimbus/src/schema.rs +++ b/components/nimbus/src/schema.rs @@ -347,7 +347,7 @@ pub enum RandomizationUnit { UserId, } -#[derive(Default)] +#[derive(Clone, Default)] pub struct AvailableRandomizationUnits { pub user_id: Option, pub nimbus_id: Option, @@ -377,7 +377,7 @@ impl AvailableRandomizationUnits { } } - pub fn get_value<'a>(&'a self, wanted: &'a RandomizationUnit) -> Option<&'a str> { + pub fn get_value(&self, wanted: &RandomizationUnit) -> Option<&str> { match wanted { RandomizationUnit::NimbusId => self.nimbus_id.as_deref(), RandomizationUnit::UserId => self.user_id.as_deref(), diff --git a/components/nimbus/src/stateful/dbcache.rs b/components/nimbus/src/stateful/dbcache.rs index 2d91068ae44..e68d685cd4c 100644 --- a/components/nimbus/src/stateful/dbcache.rs +++ b/components/nimbus/src/stateful/dbcache.rs @@ -9,13 +9,13 @@ use crate::enrollment::{ EnrolledFeature, EnrolledFeatureConfig, ExperimentEnrollment, map_features_by_feature_id, }; use crate::error::{NimbusError, Result, warn}; -use crate::evaluator::{ExperimentAvailable, is_experiment_available}; +use crate::evaluator::{CanEnrollResult, can_enroll}; use crate::stateful::enrollment::get_enrollments; use crate::stateful::firefox_labs::FirefoxLabsMetadata; use crate::stateful::gecko_prefs::GeckoPrefStore; use crate::stateful::persistence::{Database, StoreId, Writer}; use crate::targeting::NimbusTargetingHelper; -use crate::{EnrolledExperiment, Experiment}; +use crate::{AvailableRandomizationUnits, EnrolledExperiment, Experiment}; // This module manages an in-memory cache of the database, so that some // functions exposed by nimbus can return results without blocking on any @@ -219,6 +219,7 @@ impl DatabaseCache { pub fn get_available_firefox_labs_metadata( &self, + available_randomization_units: &AvailableRandomizationUnits, targeting_helper: &NimbusTargetingHelper, coenrolling_feature_ids: &[String], ) -> Result> { @@ -237,13 +238,12 @@ impl DatabaseCache { } let enrolled = data.experiments_by_slug.contains_key(&experiment.slug); + let enrollable = matches!( + can_enroll(available_randomization_units, targeting_helper, experiment,), + CanEnrollResult::Enrollable { .. } + ); - // We call is_experiment_available with is_release=true - // because being able to enroll in experiments for different channels is a bug. - // - // See-also https://bugzilla.mozilla.org/show_bug.cgi?id=1909348 - if is_experiment_available(targeting_helper, experiment, true) - == ExperimentAvailable::Available + if enrollable && (enrolled || (features_available( experiment, diff --git a/components/nimbus/src/stateful/nimbus_client.rs b/components/nimbus/src/stateful/nimbus_client.rs index 05ce8f12018..f8df9442dff 100644 --- a/components/nimbus/src/stateful/nimbus_client.rs +++ b/components/nimbus/src/stateful/nimbus_client.rs @@ -1003,14 +1003,20 @@ impl NimbusClient { } pub fn get_available_firefox_labs(&self) -> Result> { - let targeting_attributes = self.get_targeting_attributes(); + let mut state = self.mutable_state.lock().unwrap(); + state.update_time_to_now(Utc::now()); + let targeting_helper = NimbusTargetingHelper::with_targeting_attributes( - &targeting_attributes, + &state.targeting_attributes, self.event_store.clone(), self.gecko_prefs.clone(), ); - self.database_cache - .get_available_firefox_labs_metadata(&targeting_helper, &self.coenrolling_feature_ids) + + self.database_cache.get_available_firefox_labs_metadata( + &state.available_randomization_units, + &targeting_helper, + &self.coenrolling_feature_ids, + ) } pub fn enroll_in_firefox_lab(&self, slug: &str) -> Result { diff --git a/components/nimbus/src/targeting.rs b/components/nimbus/src/targeting.rs index 107c038e3d6..3e676124f36 100644 --- a/components/nimbus/src/targeting.rs +++ b/components/nimbus/src/targeting.rs @@ -49,20 +49,20 @@ impl NimbusTargetingHelper { } } - pub fn eval_jexl(&self, expr: String) -> Result { + pub fn eval_jexl(&self, expr: &str) -> Result { cfg_if::cfg_if! { if #[cfg(feature = "stateful")] { - jexl_eval(&expr, &self.context, self.event_store.clone(), self.gecko_pref_store.clone()) + jexl_eval(expr, &self.context, self.event_store.clone(), self.gecko_pref_store.clone()) } else { - jexl_eval(&expr, &self.context) + jexl_eval(expr, &self.context) } } } #[cfg(feature = "stateful")] - pub fn eval_jexl_debug(&self, expression: String) -> Result { + pub fn eval_jexl_debug(&self, expression: &str) -> Result { let eval_result = jexl_eval_raw( - &expression, + expression, &self.context, self.event_store.clone(), self.gecko_pref_store.clone(), diff --git a/components/nimbus/src/tests/stateful/test_behavior.rs b/components/nimbus/src/tests/stateful/test_behavior.rs index b11b405c326..6abd1573d4c 100644 --- a/components/nimbus/src/tests/stateful/test_behavior.rs +++ b/components/nimbus/src/tests/stateful/test_behavior.rs @@ -1459,72 +1459,76 @@ mod event_store_tests { let th = NimbusTargetingHelper::from(store); assert!( - th.eval_jexl(format!("'{event_id}'|eventSum('Minutes') >= 0")) + th.eval_jexl(&format!("'{event_id}'|eventSum('Minutes') >= 0")) .is_err() ); - assert!(th.eval_jexl(format!("'{event_id}'|eventSum('Minutes', 1) == 1"))?); - assert!(th.eval_jexl(format!("'{event_id}'|eventSum('Minutes', 1, 1) == 0"))?); + assert!(th.eval_jexl(&format!("'{event_id}'|eventSum('Minutes', 1) == 1"))?); + assert!(th.eval_jexl(&format!("'{event_id}'|eventSum('Minutes', 1, 1) == 0"))?); // This is one minute bucket 24h ago. We error out at zero. - assert!(th.eval_jexl(format!("'{event_id}'|eventSum('Minutes', 1, 24 * 60) == 0"))?); + assert!(th.eval_jexl(&format!( + "'{event_id}'|eventSum('Minutes', 1, 24 * 60) == 0" + ))?); // This is the last 24 hours of one minute buckets. This is the same as the first 60. - assert!(th.eval_jexl(format!("'{event_id}'|eventSum('Minutes', 24 * 60) == 1"))?); + assert!(th.eval_jexl(&format!("'{event_id}'|eventSum('Minutes', 24 * 60) == 1"))?); assert!( - th.eval_jexl(format!("'{event_id}'|eventSum('Years') >= 0")) + th.eval_jexl(&format!("'{event_id}'|eventSum('Years') >= 0")) .is_err() ); - assert!(th.eval_jexl(format!("'{event_id}'|eventSum('Years', 1) == 1"))?); - assert!(th.eval_jexl(format!("'{event_id}'|eventSum('Years', 1, 1) == 0"))?); + assert!(th.eval_jexl(&format!("'{event_id}'|eventSum('Years', 1) == 1"))?); + assert!(th.eval_jexl(&format!("'{event_id}'|eventSum('Years', 1, 1) == 0"))?); assert!( - th.eval_jexl(format!("'{event_id}'|eventCountNonZero('Minutes') >= 0")) + th.eval_jexl(&format!("'{event_id}'|eventCountNonZero('Minutes') >= 0")) .is_err() ); - assert!(th.eval_jexl(format!("'{event_id}'|eventCountNonZero('Minutes', 1) == 1"))?); - assert!(th.eval_jexl(format!( + assert!(th.eval_jexl(&format!( + "'{event_id}'|eventCountNonZero('Minutes', 1) == 1" + ))?); + assert!(th.eval_jexl(&format!( "'{event_id}'|eventCountNonZero('Minutes', 1, 1) == 0" ))?); assert!( - th.eval_jexl(format!( + th.eval_jexl(&format!( "'{event_id}'|eventAveragePerInterval('Minutes') >= 0" )) .is_err() ); - assert!(th.eval_jexl(format!( + assert!(th.eval_jexl(&format!( "'{event_id}'|eventAveragePerInterval('Minutes', 1) == 1" ))?); - assert!(th.eval_jexl(format!( + assert!(th.eval_jexl(&format!( "'{event_id}'|eventAveragePerInterval('Minutes', 1, 1) == 0" ))?); assert!( - th.eval_jexl(format!( + th.eval_jexl(&format!( "'{event_id}'|eventAveragePerNonZeroInterval('Minutes') >= 0" )) .is_err() ); - assert!(th.eval_jexl(format!( + assert!(th.eval_jexl(&format!( "'{event_id}'|eventAveragePerNonZeroInterval('Minutes', 1) >= 0" ))?); - assert!(th.eval_jexl(format!( + assert!(th.eval_jexl(&format!( "'{event_id}'|eventAveragePerNonZeroInterval('Minutes', 1, 1) >= 0" ))?); // When was this event last seen? It was seen zero minutes ago. - assert!(th.eval_jexl(format!("'{event_id}'|eventLastSeen('Minutes') == 0"))?); + assert!(th.eval_jexl(&format!("'{event_id}'|eventLastSeen('Minutes') == 0"))?); // Before this last minute, when was this event last seen? It was at least 60 minutes ago, if ever - assert!(th.eval_jexl(format!("'{event_id}'|eventLastSeen('Minutes', 1) > 60"))?); + assert!(th.eval_jexl(&format!("'{event_id}'|eventLastSeen('Minutes', 1) > 60"))?); // LastSeen doesn't support a fourth argument. assert!( - th.eval_jexl(format!("'{event_id}'|eventLastSeen('Minutes', 1, 1) >= 0")) + th.eval_jexl(&format!("'{event_id}'|eventLastSeen('Minutes', 1, 1) >= 0")) .is_err() ); // Q: Before 24 hours ago, when did we last see this event? // A: it was greater than 24h, but likely never. - assert!(th.eval_jexl(format!( + assert!(th.eval_jexl(&format!( "'{event_id}'|eventLastSeen('Minutes', 24 * 60) > 24 * 60" ))?); diff --git a/components/nimbus/src/tests/stateful/test_evaluator.rs b/components/nimbus/src/tests/stateful/test_evaluator.rs index 187935ffaf2..eb702a83d78 100644 --- a/components/nimbus/src/tests/stateful/test_evaluator.rs +++ b/components/nimbus/src/tests/stateful/test_evaluator.rs @@ -8,14 +8,13 @@ use std::sync::Arc; use chrono::Utc; use serde_json::json; -use crate::enrollment::NotEnrolledReason; -use crate::evaluator::targeting; +use crate::error::{NimbusError, Result}; use crate::stateful::behavior::{ EventStore, Interval, IntervalConfig, IntervalData, MultiIntervalCounter, SingleIntervalCounter, }; use crate::stateful::targeting::RecordedContext; use crate::tests::helpers::TestRecordedContext; -use crate::{AppContext, EnrollmentStatus, TargetingAttributes}; +use crate::{AppContext, NimbusTargetingHelper, TargetingAttributes}; #[test] fn test_event_sum_transform() { @@ -29,16 +28,14 @@ fn test_event_sum_transform() { }]); let event_store = EventStore::from(vec![("app.foregrounded".to_string(), counter)]); - let th = event_store.into(); - assert_eq!( - targeting("'app.foregrounded'|eventSum('Days', 3, 0) > 2", &th), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + let th = NimbusTargetingHelper::from(event_store); + assert!( + !th.eval_jexl("'app.foregrounded'|eventSum('Days', 3, 0) > 2") + .unwrap(), ); - assert_eq!( - targeting("'app.foregrounded'|eventSum('Days', 3, 0) > 1", &th,), - None + assert!( + th.eval_jexl("'app.foregrounded'|eventSum('Days', 3, 0) > 1") + .unwrap(), ); } @@ -54,22 +51,14 @@ fn test_event_count_non_zero_transform() { }]); let event_store = EventStore::from(vec![("app.foregrounded".to_string(), counter)]); - let th = event_store.into(); - assert_eq!( - targeting( - "'app.foregrounded'|eventCountNonZero('Days', 3, 0) > 2", - &th, - ), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + let th = NimbusTargetingHelper::from(event_store); + assert!( + !th.eval_jexl("'app.foregrounded'|eventCountNonZero('Days', 3, 0) > 2") + .unwrap(), ); - assert_eq!( - targeting( - "'app.foregrounded'|eventCountNonZero('Days', 3, 0) > 1", - &th, - ), - None + assert!( + th.eval_jexl("'app.foregrounded'|eventCountNonZero('Days', 3, 0) > 1") + .unwrap() ); } @@ -85,22 +74,14 @@ fn test_event_average_per_interval_transform() { }]); let event_store = EventStore::from(vec![("app.foregrounded".to_string(), counter)]); - let th = event_store.into(); - assert_eq!( - targeting( - "'app.foregrounded'|eventAveragePerInterval('Days', 7, 0) > 2", - &th, - ), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + let th = NimbusTargetingHelper::from(event_store); + assert!( + !th.eval_jexl("'app.foregrounded'|eventAveragePerInterval('Days', 7, 0) > 2") + .unwrap(), ); - assert_eq!( - targeting( - "'app.foregrounded'|eventAveragePerInterval('Days', 7, 0) > 1.14", - &th, - ), - None + assert!( + th.eval_jexl("'app.foregrounded'|eventAveragePerInterval('Days', 7, 0) > 1.14") + .unwrap() ); } @@ -116,159 +97,91 @@ fn test_event_average_per_non_zero_interval_transform() { }]); let event_store = EventStore::from(vec![("app.foregrounded".to_string(), counter)]); - let th = event_store.into(); - assert_eq!( - targeting( - "'app.foregrounded'|eventAveragePerNonZeroInterval('Days', 7, 0) == 1", - &th, - ), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + let th = NimbusTargetingHelper::from(event_store); + assert!( + !th.eval_jexl("'app.foregrounded'|eventAveragePerNonZeroInterval('Days', 7, 0) == 1") + .unwrap(), ); - assert_eq!( - targeting( - "'app.foregrounded'|eventAveragePerNonZeroInterval('Days', 7, 0) == 2.25", - &th, - ), - None + assert!( + th.eval_jexl("'app.foregrounded'|eventAveragePerNonZeroInterval('Days', 7, 0) == 2.25") + .unwrap() ); } #[test] fn test_event_transform_sum_cnz_avg_avgnz_parameters() { - let th = Default::default(); - - assert_eq!( - targeting( - "'app.foregrounded'|eventSum('Days') > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Transform parameter error: event transform Sum requires 2-3 parameters" - .to_string() - }) - ); - assert_eq!( - targeting( - "1|eventSum('Days', 3, 0) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: JSON Error: event = nimbus::stateful::behavior::EventQueryType::validate_counting_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventSum(1, 3, 0) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: JSON Error: interval = nimbus::stateful::behavior::EventQueryType::validate_counting_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventSum('Day', 3, 0) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Behavior error: IntervalParseError: Day is not a valid Interval" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventSum('Days', 'test', 0) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Transform parameter error: event transform Sum requires a positive number as the second parameter" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventSum('Days', 3, 'test') > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Transform parameter error: event transform Sum requires a positive number as the third parameter" - .to_string() - }) - ); + let th = NimbusTargetingHelper::default(); + + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventSum('Days') > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Transform parameter error: event transform Sum requires 2-3 parameters" + )); + assert!(matches!( + th.eval_jexl("1|eventSum('Days', 3, 0) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: JSON Error: event = nimbus::stateful::behavior::EventQueryType::validate_counting_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventSum(1, 3, 0) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: JSON Error: interval = nimbus::stateful::behavior::EventQueryType::validate_counting_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventSum('Day', 3, 0) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Behavior error: IntervalParseError: Day is not a valid Interval" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventSum('Days', 'test', 0) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Transform parameter error: event transform Sum requires a positive number as the second parameter" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventSum('Days', 3, 'test') > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Transform parameter error: event transform Sum requires a positive number as the third parameter" + )); } #[test] fn test_event_transform_last_seen_parameters() { - let th = Default::default(); - - assert_eq!( - targeting( - "'app.foregrounded'|eventLastSeen() > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Transform parameter error: event transform LastSeen requires 1-2 parameters" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventLastSeen('Days', 0, 10) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Transform parameter error: event transform LastSeen requires 1-2 parameters" - .to_string() - }) - ); - assert_eq!( - targeting( - "1|eventLastSeen('Days', 0) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: JSON Error: event = nimbus::stateful::behavior::EventQueryType::validate_last_seen_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventLastSeen(1, 0) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: JSON Error: interval = nimbus::stateful::behavior::EventQueryType::validate_last_seen_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventLastSeen('Day', 0) > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Behavior error: IntervalParseError: Day is not a valid Interval" - .to_string() - }) - ); - assert_eq!( - targeting( - "'app.foregrounded'|eventLastSeen('Days', 'test') > 1", - &th, - ), - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: Transform parameter error: event transform LastSeen requires a positive number as the second parameter" - .to_string() - }) - ); - - assert_eq!( - targeting("'app_cycle.foreground1'|eventLastSeen('Days', 2) > 1", &th), - None + let th = NimbusTargetingHelper::default(); + + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventLastSeen() > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Transform parameter error: event transform LastSeen requires 1-2 parameters" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventLastSeen('Days', 0, 10) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Transform parameter error: event transform LastSeen requires 1-2 parameters" + )); + assert!(matches!( + th.eval_jexl("1|eventLastSeen('Days', 0) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: JSON Error: event = nimbus::stateful::behavior::EventQueryType::validate_last_seen_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventLastSeen(1, 0) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: JSON Error: interval = nimbus::stateful::behavior::EventQueryType::validate_last_seen_arguments::serde_json::from_value — invalid type: floating point `1.0`, expected a string" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventLastSeen('Day', 0) > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Behavior error: IntervalParseError: Day is not a valid Interval" + )); + assert!(matches!( + th.eval_jexl("'app.foregrounded'|eventLastSeen('Days', 'test') > 1").unwrap_err(), + NimbusError::EvaluationError(e) + if e == "Custom error: Transform parameter error: event transform LastSeen requires a positive number as the second parameter" + )); + + assert!( + th.eval_jexl("'app_cycle.foreground1'|eventLastSeen('Days', 2) > 1") + .unwrap() ); } @@ -277,7 +190,7 @@ fn test_targeting_active_experiments_equivalency() { // Here's our valid jexl statement let expression_statement = "'test' in active_experiments"; // A matching context that includes the appropriate specific context - let mut targeting_attributes: TargetingAttributes = AppContext { + let mut targeting_attributes = TargetingAttributes::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -293,37 +206,36 @@ fn test_targeting_active_experiments_equivalency() { debug_tag: None, custom_targeting_attributes: None, ..Default::default() - } - .into(); + }); + let mut set = HashSet::::new(); set.insert("test".into()); targeting_attributes.active_experiments = set; // The targeting should pass! - assert_eq!( - targeting(expression_statement, &targeting_attributes.clone().into()), - None + assert!( + NimbusTargetingHelper::from(targeting_attributes.clone()) + .eval_jexl(expression_statement) + .unwrap() ); // We set active_experiment treatment to something not expected and try again let mut set = HashSet::::new(); set.insert("test1".into()); targeting_attributes.active_experiments = set; - assert_eq!( - targeting(expression_statement, &targeting_attributes.clone().into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + assert!( + !NimbusTargetingHelper::from(targeting_attributes.clone()) + .eval_jexl(expression_statement) + .unwrap(), ); // We set active_experiments to None and try again let set = HashSet::::new(); targeting_attributes.active_experiments = set; - assert_eq!( - targeting(expression_statement, &targeting_attributes.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + assert!( + !NimbusTargetingHelper::from(targeting_attributes.clone()) + .eval_jexl(expression_statement) + .unwrap(), ); } @@ -332,7 +244,7 @@ fn test_targeting_active_experiments_exists() { // Here's our valid jexl statement let expression_statement = "'test' in active_experiments"; // A matching context that includes the appropriate specific context - let mut targeting_attributes: TargetingAttributes = AppContext { + let mut targeting_attributes = TargetingAttributes::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -348,26 +260,26 @@ fn test_targeting_active_experiments_exists() { debug_tag: None, custom_targeting_attributes: None, ..Default::default() - } - .into(); + }); + let mut set = HashSet::::new(); set.insert("test".into()); targeting_attributes.active_experiments = set; // The targeting should pass! - assert_eq!( - targeting(expression_statement, &targeting_attributes.clone().into()), - None + assert!( + NimbusTargetingHelper::from(targeting_attributes.clone()) + .eval_jexl(expression_statement) + .unwrap() ); // We set active_experiment treatment to something not expected and try again let set = HashSet::::new(); targeting_attributes.active_experiments = set; - assert_eq!( - targeting(expression_statement, &targeting_attributes.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + assert!( + !NimbusTargetingHelper::from(targeting_attributes.clone()) + .eval_jexl(expression_statement) + .unwrap(), ); } @@ -376,7 +288,7 @@ fn test_targeting_is_already_enrolled() { // Here's our valid jexl statement let expression_statement = "is_already_enrolled"; // A matching context that includes the appropriate specific context - let ac = AppContext { + let mut targeting_attributes = TargetingAttributes::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -384,110 +296,94 @@ fn test_targeting_is_already_enrolled() { app_build: Some("1234".to_string()), custom_targeting_attributes: None, ..Default::default() - }; - let mut targeting_attributes = TargetingAttributes::from(ac); + }); targeting_attributes.is_already_enrolled = true; // The targeting should pass! - assert_eq!( - targeting(expression_statement, &targeting_attributes.clone().into(),), - None + assert!( + NimbusTargetingHelper::from(targeting_attributes.clone()) + .eval_jexl(expression_statement) + .unwrap(), ); // We make the is_already_enrolled false and try again targeting_attributes.is_already_enrolled = false; - assert_eq!( - targeting(expression_statement, &targeting_attributes.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) + assert!( + !NimbusTargetingHelper::from(targeting_attributes.clone()) + .eval_jexl(expression_statement) + .unwrap(), ); } #[test] fn test_bucket_sample() { - let cases = [ - ("1.1", "1000", "1000", None), - ( - "0", - "1.1", - "1000", - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted, - }), - ), - ( - "0", - "0", - "1000.1", - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted, - }), - ), + fn eval_bucketsample_expr(start: &str, count: &str, total: &str) -> Result { + let expr = format!("0|bucketSample({start}, {count}, {total})"); + NimbusTargetingHelper::from(AppContext { + app_name: "nimbus_test".into(), + app_id: "nimbus-test".into(), + channel: "test".into(), + ..Default::default() + }) + .eval_jexl(&expr) + } + + let ok_cases = [ + ("1.1", "1000", "1000", true), + ("0", "1.1", "1000", false), + ("0", "0", "1000.1", false), + ]; + + for (start, count, total, expected) in ok_cases { + assert_eq!( + eval_bucketsample_expr(start, count, total).unwrap(), + expected + ); + } + + let err_cases = [ ( "4294967296", "1", "4294967297", - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: start is out of range".into(), - }), + "Custom error: start is out of range", ), ( "0", "4294967296", "4294967296", - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: count is out of range".into(), - }), + "Custom error: count is out of range", ), ( "0", "0", "4294967296", - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: total is out of range".into(), - }), + "Custom error: total is out of range", ), ( r#""hello""#, "0", "1000", - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: start is not a number".into(), - }), + "Custom error: start is not a number", ), ( "0", r#""hello""#, "1000", - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: count is not a number".into(), - }), + "Custom error: count is not a number", ), ( "0", "1000", r#""hello""#, - Some(EnrollmentStatus::Error { - reason: "EvaluationError: Custom error: total is not a number".into(), - }), + "Custom error: total is not a number", ), ]; - for (start, count, total, expected) in cases { - let expr = format!("0|bucketSample({start}, {count}, {total})"); - println!("{}", expr); - let targeting_attributes: TargetingAttributes = AppContext { - app_name: "nimbus_test".into(), - app_id: "nimbus-test".into(), - channel: "test".into(), - ..Default::default() - } - .into(); - - let result = targeting(&expr, &targeting_attributes.clone().into()); - - assert_eq!(result, expected); + for (start, count, total, expected) in err_cases { + assert!(matches!( + eval_bucketsample_expr(start, count, total).unwrap_err(), + NimbusError::EvaluationError(e) if e == expected)); } } diff --git a/components/nimbus/src/tests/stateful/test_nimbus.rs b/components/nimbus/src/tests/stateful/test_nimbus.rs index e865b33a6fb..5a36fd33e29 100644 --- a/components/nimbus/src/tests/stateful/test_nimbus.rs +++ b/components/nimbus/src/tests/stateful/test_nimbus.rs @@ -1031,7 +1031,7 @@ fn test_active_enrollment_in_targeting() -> Result<()> { assert_eq!(active_experiments.len(), 1); let targeting_helper = client.create_targeting_helper(None)?; - assert!(targeting_helper.eval_jexl("'test-1' in active_experiments".to_string())?); + assert!(targeting_helper.eval_jexl("'test-1' in active_experiments")?); // Apply experiment that targets the above experiment is in enrollments let exp = get_targeted_experiment("test-2", "'test-1' in enrollments"); @@ -1042,12 +1042,12 @@ fn test_active_enrollment_in_targeting() -> Result<()> { assert_eq!(active_experiments.len(), 1); let targeting_helper = client.create_targeting_helper(None)?; - assert!(!targeting_helper.eval_jexl("'test-1' in active_experiments".to_string())?); - assert!(targeting_helper.eval_jexl("'test-2' in active_experiments".to_string())?); - assert!(targeting_helper.eval_jexl("'test-1' in enrollments".to_string())?); - assert!(targeting_helper.eval_jexl("'test-2' in enrollments".to_string())?); - assert!(targeting_helper.eval_jexl("enrollments_map['test-1'] == 'treatment'".to_string())?); - assert!(targeting_helper.eval_jexl("enrollments_map['test-2'] == 'control'".to_string())?); + assert!(!targeting_helper.eval_jexl("'test-1' in active_experiments")?); + assert!(targeting_helper.eval_jexl("'test-2' in active_experiments")?); + assert!(targeting_helper.eval_jexl("'test-1' in enrollments")?); + assert!(targeting_helper.eval_jexl("'test-2' in enrollments")?); + assert!(targeting_helper.eval_jexl("enrollments_map['test-1'] == 'treatment'")?); + assert!(targeting_helper.eval_jexl("enrollments_map['test-2'] == 'control'")?); Ok(()) } @@ -1104,16 +1104,16 @@ fn test_previous_enrollments_in_targeting() -> Result<()> { assert_eq!(active_experiments.len(), 5); let targeting_helper = client.create_targeting_helper(None)?; - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_1))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_2))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_3))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_4))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_5))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_1))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_2))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_3))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_4))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_5))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_1))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_2))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_3))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_4))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_5))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_1))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_2))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_3))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_4))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_5))?); // Apply empty first experiment, disqualifying second experiment, and decreased bucket rollout let exp_2 = get_targeted_experiment_with_feature(slug_2, "false", "feature-2"); @@ -1184,16 +1184,16 @@ fn test_previous_enrollments_in_targeting() -> Result<()> { )); let targeting_helper = client.create_targeting_helper(None)?; - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_1))?); - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_2))?); - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_3))?); - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_4))?); - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_5))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_1))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_2))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_3))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_4))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in enrollments", slug_5))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_1))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_2))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_3))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_4))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_5))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_1))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_2))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_3))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_4))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in enrollments", slug_5))?); Ok(()) } @@ -1234,20 +1234,20 @@ fn test_opt_out_multiple_experiments_same_feature_does_not_re_enroll() -> Result client.apply_pending_experiments()?; let targeting_helper = client.create_targeting_helper(None)?; - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_1))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_2))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_1))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_2))?); client.opt_out(slug_1.into())?; let targeting_helper = client.create_targeting_helper(None)?; - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_1))?); - assert!(targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_2))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_1))?); + assert!(targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_2))?); client.opt_out(slug_2.into())?; let targeting_helper = client.create_targeting_helper(None)?; - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_1))?); - assert!(!targeting_helper.eval_jexl(format!("'{}' in active_experiments", slug_2))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_1))?); + assert!(!targeting_helper.eval_jexl(&format!("'{}' in active_experiments", slug_2))?); Ok(()) } @@ -2597,6 +2597,17 @@ fn test_firefox_labs_enroll_unenroll() -> Result<()> { .patch(json!({ "firefoxLabsDescription": null })), get_firefox_lab_with_feature("lab-different-channel", "lab-feature-7") .patch(json!({ "channel": "mystery" })), + get_firefox_lab_with_feature("false-targeting", "lab-feature-8") + .patch(json!({ "targeting": "false" })), + get_firefox_lab_with_feature("false-bucketing", "lab-feature-9").patch(json!({ + "bucketConfig": { + "randomizationUnit": "nimbus_id", + "start": 0, + "count": 0, + "total": 10000, + "namespace": "firefox-labs-test", + } + })), ])?; assert_eq!( @@ -2639,6 +2650,18 @@ fn test_firefox_labs_enroll_unenroll() -> Result<()> { branch: Some("control".into()), ..Default::default() }, + EnrollmentStatusExtraDef { + slug: Some("false-bucketing".into()), + status: Some("NotEnrolled".into()), + reason: Some("FirefoxLabs".into()), + ..Default::default() + }, + EnrollmentStatusExtraDef { + slug: Some("false-targeting".into()), + status: Some("NotEnrolled".into()), + reason: Some("FirefoxLabs".into()), + ..Default::default() + }, EnrollmentStatusExtraDef { slug: Some("lab".into()), status: Some("NotEnrolled".into()), diff --git a/components/nimbus/src/tests/stateful/test_targeting.rs b/components/nimbus/src/tests/stateful/test_targeting.rs index 14db35aef6e..08a3db2ef7f 100644 --- a/components/nimbus/src/tests/stateful/test_targeting.rs +++ b/components/nimbus/src/tests/stateful/test_targeting.rs @@ -83,9 +83,7 @@ fn test_eval_jexl_debug_success() { "locale": "en-US", })); - let result = helper - .eval_jexl_debug("locale == 'en-US'".to_string()) - .unwrap(); + let result = helper.eval_jexl_debug("locale == 'en-US'").unwrap(); let parsed: serde_json::Value = serde_json::from_str(&result).unwrap(); assert_eq!(parsed["success"], true); @@ -98,7 +96,7 @@ fn test_eval_jexl_debug_error() { let helper = create_helper(json!({})); - let result = helper.eval_jexl_debug("invalid {{".to_string()).unwrap(); + let result = helper.eval_jexl_debug("invalid {{").unwrap(); let parsed: serde_json::Value = serde_json::from_str(&result).unwrap(); assert_eq!(parsed["success"], false); @@ -111,7 +109,7 @@ fn test_eval_jexl_debug_json_structure() { let helper = create_helper(json!({"test": true})); - let result = helper.eval_jexl_debug("test".to_string()).unwrap(); + let result = helper.eval_jexl_debug("test").unwrap(); let parsed: serde_json::Value = serde_json::from_str(&result).unwrap(); @@ -128,7 +126,7 @@ fn test_eval_jexl_debug_returns_pretty_json() { let helper = create_helper(json!({"locale": "en-US"})); - let result = helper.eval_jexl_debug("locale".to_string()).unwrap(); + let result = helper.eval_jexl_debug("locale").unwrap(); // Pretty JSON should have newlines assert!(result.contains('\n')); @@ -149,7 +147,7 @@ fn test_eval_jexl_debug_with_version_compare() { let helper = create_helper(serde_json::to_value(&targeting_attributes).unwrap()); let result = helper - .eval_jexl_debug("app_version|versionCompare('114.0') > 0".to_string()) + .eval_jexl_debug("app_version|versionCompare('114.0') > 0") .unwrap(); let parsed: serde_json::Value = serde_json::from_str(&result).unwrap(); diff --git a/components/nimbus/src/tests/test_evaluator.rs b/components/nimbus/src/tests/test_evaluator.rs index 58fb18b8a6f..1bbb6d55487 100644 --- a/components/nimbus/src/tests/test_evaluator.rs +++ b/components/nimbus/src/tests/test_evaluator.rs @@ -7,10 +7,10 @@ use serde_json::{Map, Value, json}; use crate::enrollment::{EnrolledReason, EnrollmentStatus, NotEnrolledReason}; -use crate::evaluator::{ExperimentAvailable, choose_branch, is_experiment_available, targeting}; +use crate::evaluator::{ExperimentAvailable, choose_branch, is_experiment_available}; use crate::{ - AppContext, AvailableRandomizationUnits, Branch, BucketConfig, Experiment, RandomizationUnit, - Result, TargetingAttributes, evaluate_enrollment, + AppContext, AvailableRandomizationUnits, Branch, BucketConfig, Experiment, NimbusError, + NimbusTargetingHelper, RandomizationUnit, Result, TargetingAttributes, evaluate_enrollment, }; pub fn ta_with_locale(locale: String) -> TargetingAttributes { @@ -32,29 +32,19 @@ pub fn ta_with_locale(locale: String) -> TargetingAttributes { } #[test] -fn test_locale_substring() -> Result<()> { +fn test_locale_substring() { let expression_statement = "'en' in locale || 'de' in locale"; - let ta = ta_with_locale("de-US".to_string()); + let targeting_helper: NimbusTargetingHelper = ta_with_locale("de-US".to_string()).into(); - assert_eq!(targeting(expression_statement, &ta.into()), None); - Ok(()) + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); } #[test] -fn test_locale_substring_fails() -> Result<()> { +fn test_locale_substring_fails() { let expression_statement = "'en' in locale || 'de' in locale"; - let ta = ta_with_locale("cz-US".to_string()); - let enrollment_status = targeting(expression_statement, &ta.into()).unwrap(); - if let EnrollmentStatus::NotEnrolled { reason } = enrollment_status { - if let NotEnrolledReason::NotTargeted = reason { - // OK - } else { - panic!("Expected to fail on NotTargeted reason, got: {:?}", reason) - } - } else { - panic! {"Expected to fail targeting with NotEnrolled, got: {:?}", enrollment_status} - } - Ok(()) + let targeting_helper: NimbusTargetingHelper = ta_with_locale("cz-US".to_string()).into(); + + assert!(!targeting_helper.eval_jexl(expression_statement).unwrap()); } #[test] @@ -79,35 +69,25 @@ fn test_language_region_from_locale() { #[test] fn test_geo_targeting_one_locale() -> Result<()> { let expression_statement = "language in ['ro']"; - let ta = ta_with_locale("ro".to_string()); + let targeting_helper: NimbusTargetingHelper = ta_with_locale("ro".to_string()).into(); - assert_eq!(targeting(expression_statement, &ta.into()), None); + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); Ok(()) } #[test] -fn test_geo_targeting_multiple_locales() -> Result<()> { +fn test_geo_targeting_multiple_locales() { let expression_statement = "language in ['en', 'ro']"; - let ta = ta_with_locale("ro".to_string()); - assert_eq!(targeting(expression_statement, &ta.into()), None); - Ok(()) + let targeting_helper: NimbusTargetingHelper = ta_with_locale("ro".to_string()).into(); + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); } #[test] -fn test_geo_targeting_fails_properly() -> Result<()> { +fn test_geo_targeting_fails_properly() { let expression_statement = "language in ['en', 'ro']"; - let ta = ta_with_locale("ar".to_string()); - let enrollment_status = targeting(expression_statement, &ta.into()).unwrap(); - if let EnrollmentStatus::NotEnrolled { reason } = enrollment_status { - if let NotEnrolledReason::NotTargeted = reason { - // OK - } else { - panic!("Expected to fail on NotTargeted reason, got: {:?}", reason) - } - } else { - panic! {"Expected to fail targeting with NotEnrolled, got: {:?}", enrollment_status} - } - Ok(()) + let targeting_helper: NimbusTargetingHelper = ta_with_locale("ar".to_string()).into(); + + assert!(!targeting_helper.eval_jexl(expression_statement).unwrap()); } #[cfg(feature = "stateful")] @@ -115,11 +95,11 @@ fn test_geo_targeting_fails_properly() -> Result<()> { fn test_minimum_version_targeting_passes() -> Result<()> { // Here's our valid jexl statement let expression_statement = "app_version|versionCompare('96.!') >= 0"; - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_version: Some("97pre.1.0-beta.1".into()), ..Default::default() - }; - assert_eq!(targeting(expression_statement, &ctx.into()), None); + }); + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); Ok(()) } @@ -128,76 +108,52 @@ fn test_minimum_version_targeting_passes() -> Result<()> { fn test_minimum_version_targeting_fails() -> Result<()> { // Here's our valid jexl statement let expression_statement = "app_version|versionCompare('96+.0') >= 0"; - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_version: Some("96.1".into()), ..Default::default() - }; - assert_eq!( - targeting(expression_statement, &ctx.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) - ); + }); + assert!(!targeting_helper.eval_jexl(expression_statement).unwrap(),); Ok(()) } #[cfg(feature = "stateful")] #[test] -fn test_targeting_specific_version() -> Result<()> { +fn test_targeting_specific_version() { // Here's our valid jexl statement that targets **only** 96 versions let expression_statement = "(app_version|versionCompare('96.!') >= 0) && (app_version|versionCompare('97.!') < 0)"; - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_version: Some("96.1".into()), ..Default::default() - }; + }); // OK 96.1 is a 96 version - assert_eq!(targeting(expression_statement, &ctx.into()), None); - let ctx = AppContext { + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_version: Some("97.1".into()), ..Default::default() - }; + }); // Not targeted, version is 97 - assert_eq!( - targeting(expression_statement, &ctx.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) - ); + assert!(!targeting_helper.eval_jexl(expression_statement).unwrap(),); - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_version: Some("95.1".into()), ..Default::default() - }; + }); // Not targeted, version is 95 - assert_eq!( - targeting(expression_statement, &ctx.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) - ); - - Ok(()) + assert!(!targeting_helper.eval_jexl(expression_statement).unwrap(),); } #[test] -fn test_targeting_invalid_transform() -> Result<()> { +fn test_targeting_invalid_transform() { let expression_statement = "app_version|invalid_transform('96+.0')"; - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_version: Some("96.1".into()), ..Default::default() - }; - let err = targeting(expression_statement, &ctx.into()); - if let Some(e) = err { - if let EnrollmentStatus::Error { reason: _ } = e { - // OK - } else { - panic!("Should have returned an error since the transform doesn't exist") - } - } else { - panic!("Should not have been targeted") - } - Ok(()) + }); + assert!(matches!( + targeting_helper.eval_jexl(expression_statement).unwrap_err(), + NimbusError::EvaluationError(e) if e == "Unknown transform: invalid_transform", + )); } #[cfg(feature = "stateful")] @@ -208,7 +164,7 @@ fn test_targeting() { "app_id == '1010' && (app_version|versionCompare('4.0') >= 0 || app_build == \"1234\")"; // A matching context testing the logical AND + OR of the expression - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -216,11 +172,11 @@ fn test_targeting() { app_build: Some("1234".to_string()), custom_targeting_attributes: None, ..Default::default() - }; - assert_eq!(targeting(expression_statement, &ctx.into()), None); + }); + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); // A matching context testing the logical OR of the expression - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -228,11 +184,11 @@ fn test_targeting() { app_build: Some("1234".to_string()), custom_targeting_attributes: None, ..Default::default() - }; - assert_eq!(targeting(expression_statement, &ctx.into()), None); + }); + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); // A matching context testing the other branch of the logical OR - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -240,11 +196,11 @@ fn test_targeting() { app_build: Some("1234".to_string()), custom_targeting_attributes: None, ..Default::default() - }; - assert_eq!(targeting(expression_statement, &ctx.into()), None); + }); + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); // A non-matching context testing the logical AND of the expression - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "not_nimbus_test".to_string(), app_id: "org.example.app".to_string(), channel: "test".to_string(), @@ -252,16 +208,11 @@ fn test_targeting() { app_build: Some("1234".to_string()), custom_targeting_attributes: None, ..Default::default() - }; - assert!(matches!( - targeting(expression_statement, &ctx.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) - )); + }); + assert!(!targeting_helper.eval_jexl(expression_statement).unwrap()); // A non-matching context testing the logical OR of the expression - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "not_nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -269,13 +220,8 @@ fn test_targeting() { app_build: Some("12345".to_string()), custom_targeting_attributes: None, ..Default::default() - }; - assert!(matches!( - targeting(expression_statement, &ctx.into()), - Some(EnrollmentStatus::NotEnrolled { - reason: NotEnrolledReason::NotTargeted - }) - )); + }); + assert!(!targeting_helper.eval_jexl(expression_statement).unwrap()) } #[test] @@ -287,7 +233,7 @@ fn test_targeting_custom_targeting_attributes() { custom_targeting_attributes.insert("is_first_run".into(), json!(true)); custom_targeting_attributes.insert("ios_version".into(), json!("8.8")); // A matching context that includes the appropriate specific context - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -295,11 +241,11 @@ fn test_targeting_custom_targeting_attributes() { app_build: Some("1234".to_string()), custom_targeting_attributes: Some(custom_targeting_attributes), ..Default::default() - }; - assert_eq!(targeting(expression_statement, &ctx.into()), None); + }); + assert!(targeting_helper.eval_jexl(expression_statement).unwrap()); // A matching context without the specific context - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "nimbus_test".to_string(), app_id: "1010".to_string(), channel: "test".to_string(), @@ -307,11 +253,11 @@ fn test_targeting_custom_targeting_attributes() { app_build: Some("1234".to_string()), custom_targeting_attributes: None, ..Default::default() - }; + }); // We haven't defined `is_first_run` here, so this should error out, i.e. return an error. assert!(matches!( - targeting(expression_statement, &ctx.into()), - Some(EnrollmentStatus::Error { .. }) + targeting_helper.eval_jexl(expression_statement).unwrap_err(), + NimbusError::EvaluationError(e) if e == "Identifier 'is_first_run' is undefined", )); } @@ -319,23 +265,26 @@ fn test_targeting_custom_targeting_attributes() { fn test_invalid_expression() { // This expression doesn't return a bool let expression_statement = "2.0"; + let targeting_helper = NimbusTargetingHelper::default(); - assert_eq!( - targeting(expression_statement, &Default::default()), - Some(EnrollmentStatus::Error { - reason: "Invalid Expression - didn't evaluate to a bool".to_string() - }) - ) + assert!(matches!( + targeting_helper + .eval_jexl(expression_statement) + .unwrap_err(), + NimbusError::InvalidExpression + )); } #[test] fn test_evaluation_error() { // This is an invalid JEXL statement let expression_statement = "This is not a valid JEXL expression"; + let targeting_helper = NimbusTargetingHelper::default(); assert!(matches!( - targeting(expression_statement, &Default::default()), - Some(EnrollmentStatus::Error { reason }) if reason.starts_with("EvaluationError:"))) + targeting_helper.eval_jexl(expression_statement).inspect_err(|e| eprintln!("{:?}", e)).unwrap_err(), + NimbusError::EvaluationError(e) if e.starts_with("Parsing error: Unrecognized token `is`"), + )); } #[test] @@ -574,12 +523,12 @@ fn test_wrong_randomization_units() { // Application context for matching the above experiment. If any of the `app_name`, `app_id`, // or `channel` doesn't match the experiment, then the client won't be enrolled. - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_name: "NimbusTest".to_string(), app_id: "org.example.app".to_string(), channel: "nightly".to_string(), ..Default::default() - }; + }); // We won't be enrolled in the experiment because we don't have the right randomization units since the // experiment is requesting the `UserId` and the `Default::default()` here will just have the @@ -587,7 +536,7 @@ fn test_wrong_randomization_units() { let enrollment = evaluate_enrollment( &AvailableRandomizationUnits::with_nimbus_id(&uuid::Uuid::new_v4()), &experiment, - &ctx.clone().into(), + &targeting_helper, ) .unwrap(); // The status should be `Error` @@ -595,8 +544,12 @@ fn test_wrong_randomization_units() { // Fits because of the user_id. let available_randomization_units = AvailableRandomizationUnits::with_user_id("bobo"); - let enrollment = - evaluate_enrollment(&available_randomization_units, &experiment, &ctx.into()).unwrap(); + let enrollment = evaluate_enrollment( + &available_randomization_units, + &experiment, + &targeting_helper, + ) + .unwrap(); assert!(matches!( enrollment.status, EnrollmentStatus::Enrolled { @@ -726,16 +679,16 @@ fn test_enrollment_bucketing() { // Tested against the desktop implementation let id = uuid::Uuid::parse_str("299eed1e-be6d-457d-9e53-da7b1a03f10d").unwrap(); // Application context for matching exp3 - let ctx = AppContext { + let targeting_helper = NimbusTargetingHelper::from(AppContext { app_id: "org.example.app".to_string(), channel: "nightly".to_string(), ..Default::default() - }; + }); let enrollment = evaluate_enrollment( &available_randomization_units.apply_nimbus_id(&id), &experiment, - &ctx.into(), + &targeting_helper, ) .unwrap(); assert!(matches!( diff --git a/components/nimbus/tests/test_message_helpers.rs b/components/nimbus/tests/test_message_helpers.rs index 9151e36762c..9d136adb97f 100644 --- a/components/nimbus/tests/test_message_helpers.rs +++ b/components/nimbus/tests/test_message_helpers.rs @@ -22,18 +22,18 @@ fn test_jexl_expression() -> Result<()> { let helper = nimbus.create_targeting_helper(None)?; // We get a boolean back from a string! - assert!(helper.eval_jexl("app_name == 'fenix'".to_string())?); + assert!(helper.eval_jexl("app_name == 'fenix'")?); // We get true and false back from two similar JEXL expressions! // I think we can convince ourselves that JEXL is being evaluated against the // AppContext. - assert!(!helper.eval_jexl("app_name == 'xinef'".to_string())?); + assert!(!helper.eval_jexl("app_name == 'xinef'")?); // The expression contains a variable not declared (snek_case Good, camelCase Bad) - assert!(helper.eval_jexl("appName == 'fenix'".to_string()).is_err()); + assert!(helper.eval_jexl("appName == 'fenix'").is_err()); // This validates that helpers created from the create_targeting_helper have the event_store present in jexl operations - assert!(helper.eval_jexl("'test'|eventSum('Days', 1, 0) == 1".to_string())?); + assert!(helper.eval_jexl("'test'|eventSum('Days', 1, 0) == 1")?); let helper = nimbus.create_targeting_helper( json!( @@ -46,12 +46,12 @@ fn test_jexl_expression() -> Result<()> { // Check the versionCompare function, just to prove to ourselves that it's the same JEXL evaluator. assert!(helper.eval_jexl( - "(version|versionCompare('95.!') >= 0) && (version|versionCompare('96.!') < 0)".to_string(), + "(version|versionCompare('95.!') >= 0) && (version|versionCompare('96.!') < 0)", )?); // Check the versionCompare function, just to prove to ourselves that it's the same JEXL evaluator. assert!(!helper.eval_jexl( - "(version|versionCompare('96.!') >= 0) && (version|versionCompare('97.!') < 0)".to_string(), + "(version|versionCompare('96.!') >= 0) && (version|versionCompare('97.!') < 0)", )?); Ok(()) @@ -64,11 +64,11 @@ fn test_derived_targeting_attributes_available() -> Result<()> { let helper = nimbus.create_targeting_helper(None)?; - assert!(helper.eval_jexl("locale == 'en-GB'".to_string())?); + assert!(helper.eval_jexl("locale == 'en-GB'")?); - assert!(helper.eval_jexl("language == 'en'".to_string())?); + assert!(helper.eval_jexl("language == 'en'")?); - assert!(helper.eval_jexl("region == 'GB'".to_string())?); + assert!(helper.eval_jexl("region == 'GB'")?); Ok(()) } @@ -82,7 +82,7 @@ fn test_derived_targeting_attributes_none() -> Result<()> { let helper = nimbus.create_targeting_helper(None)?; - assert!(!helper.eval_jexl("(locale||'NONE') == 'en'".to_string())?); + assert!(!helper.eval_jexl("(locale||'NONE') == 'en'")?); // assert!(helper.eval_jexl( // "language == null".to_string() @@ -101,17 +101,17 @@ fn test_jexl_expression_with_targeting_attributes() -> Result<()> { let helper = nimbus.create_targeting_helper(None)?; - assert!(helper.eval_jexl("days_since_install == 0".to_string())?); + assert!(helper.eval_jexl("days_since_install == 0")?); - assert!(helper.eval_jexl("days_since_update == 0".to_string())?); + assert!(helper.eval_jexl("days_since_update == 0")?); nimbus.set_install_time(Utc::now() - Duration::days(10)); nimbus.set_update_time(Utc::now() - Duration::days(5)); let helper = nimbus.create_targeting_helper(None)?; - assert!(helper.eval_jexl("days_since_install == 10".to_string())?); + assert!(helper.eval_jexl("days_since_install == 10")?); - assert!(helper.eval_jexl("days_since_update == 5".to_string())?); + assert!(helper.eval_jexl("days_since_update == 5")?); Ok(()) } diff --git a/components/nimbus/tests/test_restart.rs b/components/nimbus/tests/test_restart.rs index c9e36f86e05..cdb588feeec 100644 --- a/components/nimbus/tests/test_restart.rs +++ b/components/nimbus/tests/test_restart.rs @@ -211,9 +211,9 @@ fn test_targeting_attributes_active_experiments() -> Result<()> { assert_eq!(ta.active_experiments, expected); let eval = client.create_targeting_helper(None)?; - assert!(eval.eval_jexl("'experiment_always_enroll' in active_experiments".to_string())?); - assert!(eval.eval_jexl("'experiment_target_false' in active_experiments".to_string())?); - assert!(!eval.eval_jexl("'experiment_zero_buckets' in active_experiments".to_string())?); + assert!(eval.eval_jexl("'experiment_always_enroll' in active_experiments")?); + assert!(eval.eval_jexl("'experiment_target_false' in active_experiments")?); + assert!(!eval.eval_jexl("'experiment_zero_buckets' in active_experiments")?); drop(client); @@ -224,9 +224,9 @@ fn test_targeting_attributes_active_experiments() -> Result<()> { assert_eq!(ta.active_experiments, expected); let eval = client.create_targeting_helper(None)?; - assert!(eval.eval_jexl("'experiment_always_enroll' in active_experiments".to_string())?); - assert!(eval.eval_jexl("'experiment_target_false' in active_experiments".to_string())?); - assert!(!eval.eval_jexl("'experiment_zero_buckets' in active_experiments".to_string())?); + assert!(eval.eval_jexl("'experiment_always_enroll' in active_experiments")?); + assert!(eval.eval_jexl("'experiment_target_false' in active_experiments")?); + assert!(!eval.eval_jexl("'experiment_zero_buckets' in active_experiments")?); drop(client); @@ -237,9 +237,9 @@ fn test_targeting_attributes_active_experiments() -> Result<()> { assert_eq!(ta.active_experiments, expected); let eval = client.create_targeting_helper(None)?; - assert!(eval.eval_jexl("'experiment_always_enroll' in active_experiments".to_string())?); - assert!(eval.eval_jexl("'experiment_target_false' in active_experiments".to_string())?); - assert!(!eval.eval_jexl("'experiment_zero_buckets' in active_experiments".to_string())?); + assert!(eval.eval_jexl("'experiment_always_enroll' in active_experiments")?); + assert!(eval.eval_jexl("'experiment_target_false' in active_experiments")?); + assert!(!eval.eval_jexl("'experiment_zero_buckets' in active_experiments")?); Ok(()) } From 829e5cd7d9c2a609b05701672268e28aa00d6830 Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Mon, 20 Jul 2026 10:37:20 -0400 Subject: [PATCH 22/59] Don't invoke the browser to login to the fxa example. (#7486) You will probably never do that in a "real" browser window, so make it a little clearer about the expected use. --- Cargo.lock | 37 --------------------------- examples/cli-support/Cargo.toml | 1 - examples/cli-support/src/fxa_creds.rs | 10 +++----- 3 files changed, 3 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92cc57f5fc6..f82819b515e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -646,7 +646,6 @@ dependencies = [ "anyhow", "dialoguer", "fxa-client", - "open", "remote_settings", "sync15", "sync_manager", @@ -2291,15 +2290,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "is-docker" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" -dependencies = [ - "once_cell", -] - [[package]] name = "is-terminal" version = "0.4.7" @@ -2312,16 +2302,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "is-wsl" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" -dependencies = [ - "is-docker", - "once_cell", -] - [[package]] name = "itertools" version = "0.13.0" @@ -3126,17 +3106,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "open" -version = "5.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2483562e62ea94312f3576a7aca397306df7990b8d89033e18766744377ef95" -dependencies = [ - "is-wsl", - "libc", - "pathdiff", -] - [[package]] name = "openssl" version = "0.10.72" @@ -3229,12 +3198,6 @@ dependencies = [ "windows-sys 0.36.1", ] -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - [[package]] name = "payload-support" version = "0.1.0" diff --git a/examples/cli-support/Cargo.toml b/examples/cli-support/Cargo.toml index ebf07e05476..eecd822c98a 100644 --- a/examples/cli-support/Cargo.toml +++ b/examples/cli-support/Cargo.toml @@ -13,7 +13,6 @@ sync_manager = { path = "../../components/sync_manager" } tracing-support = { path = "../../components/support/tracing" } sync15 = { path = "../../components/sync15", features=["sync-client"] } url = "2" -open = "5" dialoguer = { version = "0.11", default-features = false, features = ["password"] } # disable regex feature, as it's very costly for compile time and very rarely # used. diff --git a/examples/cli-support/src/fxa_creds.rs b/examples/cli-support/src/fxa_creds.rs index 287096ad6b1..beaf60f40fc 100644 --- a/examples/cli-support/src/fxa_creds.rs +++ b/examples/cli-support/src/fxa_creds.rs @@ -230,13 +230,9 @@ impl CliFxa { other => anyhow::bail!("Unexpected FxA state after BeginOAuthFlow: {other:?}"), }; - println!("Trying to open the auth URL — if your browser doesn't open, please open this URL manually:"); - println!(" {oauth_url}\n"); - match open::that(&oauth_url) { - Ok(()) => println!("Opened in your browser."), - Err(e) => crate::warn!("Could not open a browser: {e}"), - } - + println!("In a (probably private) browser window, please open:"); + println!(" {oauth_url}\n"); + println!("paste the final 'Connected' URL from a successful flow below."); let final_url = Url::parse(&prompt_string("Final URL").unwrap_or_default())?; let query_params: HashMap = final_url.query_pairs().into_owned().collect(); From 7eeec6921be9eb6ab431e2d704be2068164d5160 Mon Sep 17 00:00:00 2001 From: dsmithpadilla <88508950+dsmithpadilla@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:13:37 -0400 Subject: [PATCH 23/59] Start release v155.0 (#7489) --- CHANGELOG.md | 8 ++++++-- version.txt | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a02e1648f5..e287f4be2e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -# v154.0 (In progress) +# v155.0 (In progress) + +[Full Changelog](In progress) + +# v154.0 (_2026-07-20_) ## ✨ What's Changed ✨ @@ -10,7 +14,7 @@ - Skip verification of signatures with unknown signature types ([Bug 2055147](https://bugzilla.mozilla.org/show_bug.cgi?id=2055147)) -[Full Changelog](In progress) +[Full Changelog](https://github.com/mozilla/application-services/compare/v153.0...v154.0) ## ✨ What's New ✨ diff --git a/version.txt b/version.txt index 68afae5961a..efd47b4b23c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -154.0a1 +155.0a1 From 573c50367b4fcaaaad58008ad83a10c01f04b0a0 Mon Sep 17 00:00:00 2001 From: Shawn Zivontsis Date: Wed, 22 Jul 2026 19:45:24 -0400 Subject: [PATCH 24/59] Update sqlite optimize flags to match Firefox Desktop (#7485) For the run_maintenance_optimize background task, we add the 0x10000 flag in addition to the default 0x12. This causes optimization of all tables even if they were never queried by the provided connection. This fixes a rare scenario where Firefox for Android can get stuck with incorrect table stats on the moz_origins table, leading to poor address bar autocomplete performance. For the Drop implementation on PlacesDb, we add the now-default flag 0x10, which previously didn't exist, and was being explicitly overridden in the current implementation. This adds a row limit on analyze which can help to prevent poor shutdown performance, matching the change made in the fix for bug 2017227. Co-authored-by: Mark Hammond --- components/places/src/db/db.rs | 5 ++++- components/places/src/storage/mod.rs | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/places/src/db/db.rs b/components/places/src/db/db.rs index 873e8e4cd3f..37ffa54fcde 100644 --- a/components/places/src/db/db.rs +++ b/components/places/src/db/db.rs @@ -211,7 +211,10 @@ impl Drop for PlacesDb { // A reader connection can't execute an optimize return; } - let res = self.db.execute_batch("PRAGMA optimize(0x02);"); + // The 0x12 flags mean: run ANALYZE on tables that might benefit (0x02), with a row + // limit to keep runtime bounded (0x10). Mirrors the flags used on desktop since + // Bug 2017227. + let res = self.db.execute_batch("PRAGMA optimize(0x12);"); if let Err(e) = res { warn!("Failed to execute pragma optimize (DB locked?): {}", e); } diff --git a/components/places/src/storage/mod.rs b/components/places/src/storage/mod.rs index 5b79a5a6d5c..8a97e4f2602 100644 --- a/components/places/src/storage/mod.rs +++ b/components/places/src/storage/mod.rs @@ -292,7 +292,11 @@ pub fn run_maintenance_vacuum(conn: &PlacesDb) -> Result<()> { /// Kotlin wrapper code (This is needed because we only have access to the Glean API in Kotlin and /// it supports a stop-watch style API, not recording specific values). pub fn run_maintenance_optimize(conn: &PlacesDb) -> Result<()> { - conn.execute_one("PRAGMA optimize")?; + // 0x10012: run ANALYZE on tables that might benefit (0x02), with a row limit to keep + // runtime bounded (0x10), including tables not queried during this connection (0x10000). + // The 0x10000 bit lets maintenance refresh stats for tables the writer never queries; + // desktop added this alongside the Bug 2017227 shutdown fix. + conn.execute_one("PRAGMA optimize(0x10012)")?; Ok(()) } From 6aa846fcdf292cc1d11e62a3140f84c75061e94a Mon Sep 17 00:00:00 2001 From: DimiDL <55685831+DimiDL@users.noreply.github.com> Date: Fri, 24 Jul 2026 07:27:18 +0000 Subject: [PATCH 25/59] Bug 2050036 - Add Store::shutdown() to close the autofill DB connection (#7490) The connection is otherwise only closed when the Store is dropped, which on Firefox Desktop happens during GC at shutdown -- past the late-write barrier, so the flush crashes debug builds. Wrap the connection in Mutex> behind a fallible lock_db() helper (mirroring LoginStore); shutdown() closes it early, and later operations return DatabaseClosed. --- CHANGELOG.md | 6 ++ Cargo.lock | 1 + components/autofill/Cargo.toml | 1 + components/autofill/src/autofill.udl | 2 + components/autofill/src/db/mod.rs | 8 ++ components/autofill/src/db/store.rs | 100 +++++++++++------- components/autofill/src/error.rs | 10 ++ components/autofill/src/sync/engine.rs | 36 ++++--- .../sync/tests/test_migrate_remote_address.rs | 4 +- .../autofill/src/sync/tests/test_reconcile.rs | 4 +- 10 files changed, 113 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e287f4be2e8..a6927bed73e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ [Full Changelog](In progress) +## ✨ What's Changed ✨ + +### Autofill + +- Add `Store::shutdown()`, which closes the database connection early so it happens before Firefox Desktop's late-write shutdown barrier rather than during GC. Operations after shutdown return `DatabaseClosed`. ([Bug 2050036](https://bugzilla.mozilla.org/show_bug.cgi?id=2050036)) + # v154.0 (_2026-07-20_) ## ✨ What's Changed ✨ diff --git a/Cargo.lock b/Cargo.lock index f82819b515e..be23f309934 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,6 +214,7 @@ dependencies = [ "lazy_static", "libsqlite3-sys", "nss-as", + "parking_lot", "rusqlite", "serde", "serde_derive", diff --git a/components/autofill/Cargo.toml b/components/autofill/Cargo.toml index 9ce9310e75c..ed0f3199966 100644 --- a/components/autofill/Cargo.toml +++ b/components/autofill/Cargo.toml @@ -14,6 +14,7 @@ error-support = { path = "../support/error" } interrupt-support = { path = "../support/interrupt" } jwcrypto = { path = "../support/jwcrypto" } lazy_static = "1.4" +parking_lot = ">=0.11,<=0.12" rusqlite = { version = "0.37.0", features = ["functions", "bundled", "serde_json", "unlock_notify"] } serde = "1" serde_derive = "1" diff --git a/components/autofill/src/autofill.udl b/components/autofill/src/autofill.udl index 660b2706167..841758bd17c 100644 --- a/components/autofill/src/autofill.udl +++ b/components/autofill/src/autofill.udl @@ -208,6 +208,8 @@ interface Store { [Throws=AutofillApiError] void run_maintenance(); + void shutdown(); + [Self=ByArc] void register_with_sync_manager(); }; diff --git a/components/autofill/src/db/mod.rs b/components/autofill/src/db/mod.rs index 6527a49e13b..cfb5e35500f 100644 --- a/components/autofill/src/db/mod.rs +++ b/components/autofill/src/db/mod.rs @@ -11,6 +11,7 @@ pub mod store; use crate::error::*; +use error_support::error; use interrupt_support::{SqlInterruptHandle, SqlInterruptScope}; use rusqlite::{Connection, OpenFlags}; use sql_support::open_database; @@ -61,6 +62,13 @@ impl AutofillDb { pub fn begin_interrupt_scope(&self) -> Result { Ok(self.interrupt_handle.begin_interrupt_scope()?) } + + pub fn close(self) { + if let Err((_, err)) = self.writer.close() { + // Log the error, but continue with shutdown. + error!("Failed to close the connection: {:?}", err); + } + } } impl Deref for AutofillDb { diff --git a/components/autofill/src/db/store.rs b/components/autofill/src/db/store.rs index d48c3351628..1ff3cbb07bb 100644 --- a/components/autofill/src/db/store.rs +++ b/components/autofill/src/db/store.rs @@ -10,13 +10,14 @@ use crate::db::{ }; use crate::error::*; use error_support::handle_error; +use parking_lot::{MappedMutexGuard, Mutex, MutexGuard}; use rusqlite::{ types::{FromSql, ToSql}, Connection, }; use sql_support::{self, run_maintenance, ConnExt}; use std::path::Path; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::{Arc, Weak}; use sync15::engine::{SyncEngine, SyncEngineId}; use sync_guid::Guid; @@ -25,7 +26,8 @@ lazy_static::lazy_static! { // Mutex: just taken long enough to update the contents - needed to wrap // the Weak as it isn't `Sync` // [Arc/Weak]: What the sync manager actually needs. - static ref STORE_FOR_MANAGER: Mutex> = Mutex::new(Weak::new()); + static ref STORE_FOR_MANAGER: std::sync::Mutex> = + std::sync::Mutex::new(Weak::new()); } /// Called by the sync manager to get a sync engine via the store previously @@ -48,14 +50,14 @@ pub fn get_registered_sync_engine(engine_id: &SyncEngineId) -> Option, + pub(crate) db: Mutex>, } impl Store { #[handle_error(Error)] pub fn new(db_path: impl AsRef) -> ApiResult { Ok(Self { - db: Mutex::new(AutofillDb::new(db_path)?), + db: Mutex::new(Some(AutofillDb::new(db_path)?)), }) } @@ -63,7 +65,7 @@ impl Store { #[cfg(test)] pub fn new_memory() -> Self { Self { - db: Mutex::new(crate::db::test::new_mem_db()), + db: Mutex::new(Some(crate::db::test::new_mem_db())), } } @@ -71,26 +73,30 @@ impl Store { #[handle_error(Error)] pub fn new_shared_memory(db_name: &str) -> ApiResult { Ok(Self { - db: Mutex::new(AutofillDb::new_memory(db_name)?), + db: Mutex::new(Some(AutofillDb::new_memory(db_name)?)), }) } + pub(crate) fn lock_db(&self) -> Result> { + MutexGuard::try_map(self.db.lock(), |db| db.as_mut()).map_err(|_| Error::DatabaseClosed) + } + #[handle_error(Error)] pub fn add_credit_card(&self, fields: UpdatableCreditCardFields) -> ApiResult { - let credit_card = credit_cards::add_credit_card(&self.db.lock().unwrap().writer, fields)?; + let credit_card = credit_cards::add_credit_card(&self.lock_db()?.writer, fields)?; Ok(credit_card.into()) } #[handle_error(Error)] pub fn get_credit_card(&self, guid: String) -> ApiResult { let credit_card = - credit_cards::get_credit_card(&self.db.lock().unwrap().writer, &Guid::new(&guid))?; + credit_cards::get_credit_card(&self.lock_db()?.writer, &Guid::new(&guid))?; Ok(credit_card.into()) } #[handle_error(Error)] pub fn get_all_credit_cards(&self) -> ApiResult> { - let credit_cards = credit_cards::get_all_credit_cards(&self.db.lock().unwrap().writer)? + let credit_cards = credit_cards::get_all_credit_cards(&self.lock_db()?.writer)? .into_iter() .map(|x| x.into()) .collect(); @@ -99,7 +105,7 @@ impl Store { #[handle_error(Error)] pub fn count_all_credit_cards(&self) -> ApiResult { - let count = credit_cards::count_all_credit_cards(&self.db.lock().unwrap().writer)?; + let count = credit_cards::count_all_credit_cards(&self.lock_db()?.writer)?; Ok(count) } @@ -109,36 +115,32 @@ impl Store { guid: String, credit_card: UpdatableCreditCardFields, ) -> ApiResult<()> { - credit_cards::update_credit_card( - &self.db.lock().unwrap().writer, - &Guid::new(&guid), - &credit_card, - ) + credit_cards::update_credit_card(&self.lock_db()?.writer, &Guid::new(&guid), &credit_card) } #[handle_error(Error)] pub fn delete_credit_card(&self, guid: String) -> ApiResult { - credit_cards::delete_credit_card(&self.db.lock().unwrap().writer, &Guid::new(&guid)) + credit_cards::delete_credit_card(&self.lock_db()?.writer, &Guid::new(&guid)) } #[handle_error(Error)] pub fn touch_credit_card(&self, guid: String) -> ApiResult<()> { - credit_cards::touch(&self.db.lock().unwrap().writer, &Guid::new(&guid)) + credit_cards::touch(&self.lock_db()?.writer, &Guid::new(&guid)) } #[handle_error(Error)] pub fn add_address(&self, new_address: UpdatableAddressFields) -> ApiResult
{ - Ok(addresses::add_address(&self.db.lock().unwrap().writer, new_address)?.into()) + Ok(addresses::add_address(&self.lock_db()?.writer, new_address)?.into()) } #[handle_error(Error)] pub fn get_address(&self, guid: String) -> ApiResult
{ - Ok(addresses::get_address(&self.db.lock().unwrap().writer, &Guid::new(&guid))?.into()) + Ok(addresses::get_address(&self.lock_db()?.writer, &Guid::new(&guid))?.into()) } #[handle_error(Error)] pub fn get_all_addresses(&self) -> ApiResult> { - let addresses = addresses::get_all_addresses(&self.db.lock().unwrap().writer)? + let addresses = addresses::get_all_addresses(&self.lock_db()?.writer)? .into_iter() .map(|x| x.into()) .collect(); @@ -147,38 +149,38 @@ impl Store { #[handle_error(Error)] pub fn count_all_addresses(&self) -> ApiResult { - let count = addresses::count_all_addresses(&self.db.lock().unwrap().writer)?; + let count = addresses::count_all_addresses(&self.lock_db()?.writer)?; Ok(count) } #[handle_error(Error)] pub fn update_address(&self, guid: String, address: UpdatableAddressFields) -> ApiResult<()> { - addresses::update_address(&self.db.lock().unwrap().writer, &Guid::new(&guid), &address) + addresses::update_address(&self.lock_db()?.writer, &Guid::new(&guid), &address) } #[handle_error(Error)] pub fn delete_address(&self, guid: String) -> ApiResult { - addresses::delete_address(&self.db.lock().unwrap().writer, &Guid::new(&guid)) + addresses::delete_address(&self.lock_db()?.writer, &Guid::new(&guid)) } #[handle_error(Error)] pub fn touch_address(&self, guid: String) -> ApiResult<()> { - addresses::touch(&self.db.lock().unwrap().writer, &Guid::new(&guid)) + addresses::touch(&self.lock_db()?.writer, &Guid::new(&guid)) } #[handle_error(Error)] pub fn add_passport(&self, fields: UpdatablePassportFields) -> ApiResult { - Ok(passports::add_passport(&self.db.lock().unwrap().writer, fields)?.into()) + Ok(passports::add_passport(&self.lock_db()?.writer, fields)?.into()) } #[handle_error(Error)] pub fn get_passport(&self, guid: String) -> ApiResult { - Ok(passports::get_passport(&self.db.lock().unwrap().writer, &Guid::new(&guid))?.into()) + Ok(passports::get_passport(&self.lock_db()?.writer, &Guid::new(&guid))?.into()) } #[handle_error(Error)] pub fn get_all_passports(&self) -> ApiResult> { - let passports = passports::get_all_passports(&self.db.lock().unwrap().writer)? + let passports = passports::get_all_passports(&self.lock_db()?.writer)? .into_iter() .map(|x| x.into()) .collect(); @@ -187,7 +189,7 @@ impl Store { #[handle_error(Error)] pub fn count_all_passports(&self) -> ApiResult { - passports::count_all_passports(&self.db.lock().unwrap().writer) + passports::count_all_passports(&self.lock_db()?.writer) } #[handle_error(Error)] @@ -196,28 +198,24 @@ impl Store { guid: String, passport: UpdatablePassportFields, ) -> ApiResult<()> { - passports::update_passport( - &self.db.lock().unwrap().writer, - &Guid::new(&guid), - &passport, - ) + passports::update_passport(&self.lock_db()?.writer, &Guid::new(&guid), &passport) } #[handle_error(Error)] pub fn delete_passport(&self, guid: String) -> ApiResult { - passports::delete_passport(&self.db.lock().unwrap().writer, &Guid::new(&guid)) + passports::delete_passport(&self.lock_db()?.writer, &Guid::new(&guid)) } #[handle_error(Error)] pub fn touch_passport(&self, guid: String) -> ApiResult<()> { - passports::touch(&self.db.lock().unwrap().writer, &Guid::new(&guid)) + passports::touch(&self.lock_db()?.writer, &Guid::new(&guid)) } #[handle_error(Error)] pub fn scrub_encrypted_data(self: Arc) -> ApiResult<()> { // scrub the data on disk // Currently only credit cards have encrypted data - credit_cards::scrub_encrypted_credit_card_data(&self.db.lock().unwrap().writer)?; + credit_cards::scrub_encrypted_credit_card_data(&self.lock_db()?.writer)?; // Force the sync engine to refetch data (only need to do this for the credit cards, since the // addresses engine doesn't store encrypted data). crate::sync::credit_card::create_engine(self).reset_local_sync_data()?; @@ -229,10 +227,10 @@ impl Store { self: Arc, local_encryption_key: String, ) -> ApiResult { - let db = &self.db.lock().unwrap().writer; + let db = self.lock_db()?; let deletion_stats = credit_cards::scrub_undecryptable_credit_card_data_for_remote_replacement( - db, + &db.writer, local_encryption_key, )?; @@ -241,17 +239,23 @@ impl Store { // record that exists on the sync server to overwrite the local record and restore // the scrubbed credit card number. crate::sync::credit_card::create_engine(self.clone()) - .reset_local_sync_data_for_verification(db)?; + .reset_local_sync_data_for_verification(&db.writer)?; Ok(deletion_stats) } #[handle_error(Error)] pub fn run_maintenance(&self) -> ApiResult<()> { - let conn = self.db.lock().unwrap(); + let conn = self.lock_db()?; run_maintenance(&conn)?; Ok(()) } + pub fn shutdown(&self) { + if let Some(db) = self.db.lock().take() { + db.close(); + } + } + // This allows the embedding app to say "make this instance available to // the sync manager". The implementation is more like "offer to sync mgr" // (thereby avoiding us needing to link with the sync manager) but @@ -364,6 +368,22 @@ mod tests { assert!(STORE_FOR_MANAGER.lock().unwrap().upgrade().is_none()); } + #[test] + fn test_shutdown_closes_the_store() { + let store = Store::new_shared_memory("shutdown-test").expect("create store"); + // Operations succeed before shutdown. + assert_eq!(store.count_all_passports().expect("count"), 0); + + store.shutdown(); + + // After shutdown, operations return an error rather than panicking or + // operating on a half-closed store. + assert!(store.count_all_passports().is_err()); + + // shutdown is idempotent. + store.shutdown(); + } + #[test] fn test_scrub_undecryptable_credit_card_data_for_remote_replacement() { ensure_initialized(); diff --git a/components/autofill/src/error.rs b/components/autofill/src/error.rs index 0fb76ee7ba8..d45c095206f 100644 --- a/components/autofill/src/error.rs +++ b/components/autofill/src/error.rs @@ -65,6 +65,9 @@ pub enum Error { #[error("No record with guid exists: {0}")] NoSuchRecord(String), + + #[error("The store is closed")] + DatabaseClosed, } // Define how our internal errors are handled and converted to external errors @@ -126,6 +129,13 @@ impl GetErrorHandling for Error { ErrorHandling::convert(AutofillApiError::NoSuchRecord { guid: guid.clone() }) .log_warning() } + + Self::DatabaseClosed => { + ErrorHandling::convert(AutofillApiError::UnexpectedAutofillApiError { + reason: "The store is closed".to_string(), + }) + .report_error("autofill-database-closed") + } } } } diff --git a/components/autofill/src/sync/engine.rs b/components/autofill/src/sync/engine.rs index 832dd6e528c..7abc9815881 100644 --- a/components/autofill/src/sync/engine.rs +++ b/components/autofill/src/sync/engine.rs @@ -76,7 +76,7 @@ impl ConfigSyncEngine { } // Reset the local sync data so the next server request fetches all records. pub fn reset_local_sync_data(&self) -> Result<()> { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; let tx = db.unchecked_transaction()?; self.storage_impl.reset_storage(&tx)?; self.put_meta(&tx, LAST_SYNC_META_KEY, &0)?; @@ -109,7 +109,7 @@ impl SyncEngine for ConfigSyncEngine { &self, _get_client_data: &dyn Fn() -> sync15::ClientData, ) -> anyhow::Result<()> { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; let signal = db.begin_interrupt_scope()?; crate::db::schema::create_empty_sync_temp_tables(&db.writer)?; signal.err_if_interrupted()?; @@ -121,7 +121,7 @@ impl SyncEngine for ConfigSyncEngine { inbound: Vec, telem: &mut telemetry::Engine, ) -> anyhow::Result<()> { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; let signal = db.begin_interrupt_scope()?; // Stage all incoming items. @@ -141,7 +141,7 @@ impl SyncEngine for ConfigSyncEngine { timestamp: ServerTimestamp, _telem: &mut telemetry::Engine, ) -> anyhow::Result> { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; let signal = db.begin_interrupt_scope()?; let tx = db.writer.unchecked_transaction()?; let incoming_impl = self.storage_impl.get_incoming_impl(&self.local_enc_key)?; @@ -173,7 +173,7 @@ impl SyncEngine for ConfigSyncEngine { } fn set_uploaded(&self, new_timestamp: ServerTimestamp, ids: Vec) -> anyhow::Result<()> { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; self.put_meta(&db.writer, LAST_SYNC_META_KEY, &new_timestamp.as_millis())?; let tx = db.writer.unchecked_transaction()?; let outgoing_impl = self.storage_impl.get_outgoing_impl(&self.local_enc_key)?; @@ -186,7 +186,7 @@ impl SyncEngine for ConfigSyncEngine { &self, server_timestamp: ServerTimestamp, ) -> anyhow::Result> { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; let since = ServerTimestamp( self.get_meta::(&db.writer, LAST_SYNC_META_KEY)? .unwrap_or_default(), @@ -203,7 +203,7 @@ impl SyncEngine for ConfigSyncEngine { } fn get_sync_assoc(&self) -> anyhow::Result { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; let global = self.get_meta(&db.writer, GLOBAL_SYNCID_META_KEY)?; let coll = self.get_meta(&db.writer, COLLECTION_SYNCID_META_KEY)?; Ok(if let (Some(global), Some(coll)) = (global, coll) { @@ -214,7 +214,7 @@ impl SyncEngine for ConfigSyncEngine { } fn reset(&self, assoc: &EngineSyncAssociation) -> anyhow::Result<()> { - let db = &self.store.db.lock().unwrap(); + let db = self.store.lock_db()?; let tx = db.unchecked_transaction()?; self.storage_impl.reset_storage(&tx)?; // Reset the last sync time, so that the next sync fetches fresh records @@ -294,7 +294,8 @@ mod tests { .set_local_encryption_key(&test_key) .unwrap(); { - create_empty_sync_temp_tables(&credit_card_engine.store.db.lock().unwrap())?; + let db = credit_card_engine.store.lock_db()?; + create_empty_sync_temp_tables(&db.writer)?; } let mut telem = telemetry::Engine::new("whatever"); @@ -303,7 +304,8 @@ mod tests { assert!(result.is_ok()); // check that last sync metadata was set - let conn = &credit_card_engine.store.db.lock().unwrap().writer; + let db = credit_card_engine.store.lock_db()?; + let conn = &db.writer; assert_eq!( credit_card_engine.get_meta::(conn, LAST_SYNC_META_KEY)?, @@ -332,7 +334,8 @@ mod tests { coll: coll_guid, }; { - let conn = &credit_card_engine.store.db.lock().unwrap().writer; + let db = credit_card_engine.store.lock_db()?; + let conn = &db.writer; credit_card_engine.put_meta(conn, GLOBAL_SYNCID_META_KEY, &ids.global)?; credit_card_engine.put_meta(conn, COLLECTION_SYNCID_META_KEY, &ids.coll)?; } @@ -364,7 +367,7 @@ mod tests { { // temp scope for the mutex lock. - let db = &engine.store.db.lock().unwrap(); + let db = engine.store.lock_db()?; let tx = db.writer.unchecked_transaction()?; // create a normal record, a mirror record and a tombstone. add_internal_credit_card(&tx, &cc)?; @@ -385,7 +388,8 @@ mod tests { coll: coll_guid.clone(), }; { - let conn = &engine.store.db.lock().unwrap().writer; + let db = engine.store.lock_db()?; + let conn = &db.writer; engine.put_meta(conn, GLOBAL_SYNCID_META_KEY, &ids.global)?; engine.put_meta(conn, COLLECTION_SYNCID_META_KEY, &ids.coll)?; } @@ -396,7 +400,8 @@ mod tests { .expect("should work"); { - let conn = &engine.store.db.lock().unwrap().writer; + let db = engine.store.lock_db()?; + let conn = &db.writer; // check that the mirror and tombstone tables have no records assert!(get_all(conn, "credit_cards_mirror".to_string())?.is_empty()); @@ -434,7 +439,8 @@ mod tests { .reset(&EngineSyncAssociation::Connected(ids)) .expect("should work"); - let conn = &engine.store.db.lock().unwrap().writer; + let db = engine.store.lock_db()?; + let conn = &db.writer; // check that the meta records were set let retrieved_global_sync_id = engine.get_meta::(conn, GLOBAL_SYNCID_META_KEY)?; assert_eq!( diff --git a/components/autofill/src/sync/tests/test_migrate_remote_address.rs b/components/autofill/src/sync/tests/test_migrate_remote_address.rs index e4f8be22a76..49f2f672376 100644 --- a/components/autofill/src/sync/tests/test_migrate_remote_address.rs +++ b/components/autofill/src/sync/tests/test_migrate_remote_address.rs @@ -320,7 +320,7 @@ fn test_migrate_remote_addresses() -> Result<()> { for test_case in j.as_array().unwrap() { let desc = test_case["description"].as_str().unwrap(); let store = Arc::new(Store::new_memory()); - let db = store.db.lock().unwrap(); + let db = store.lock_db().unwrap(); let tx = db.unchecked_transaction().unwrap(); create_empty_sync_temp_tables(&tx)?; @@ -387,7 +387,7 @@ fn test_migrate_remote_addresses() -> Result<()> { }; // get a DB reference back to we can check the results. - let db = store.db.lock().unwrap(); + let db = store.lock_db().unwrap(); let all = addresses::get_all_addresses(&db)?; diff --git a/components/autofill/src/sync/tests/test_reconcile.rs b/components/autofill/src/sync/tests/test_reconcile.rs index b55edb01f66..00bd748ffc2 100644 --- a/components/autofill/src/sync/tests/test_reconcile.rs +++ b/components/autofill/src/sync/tests/test_reconcile.rs @@ -613,7 +613,7 @@ fn test_reconcile_addresses() -> Result<()> { for test_case in j.as_array().unwrap() { let desc = test_case["description"].as_str().unwrap(); let store = Arc::new(Store::new_memory()); - let db = store.db.lock().unwrap(); + let db = store.lock_db().unwrap(); let tx = db.unchecked_transaction().unwrap(); create_empty_sync_temp_tables(&tx)?; @@ -680,7 +680,7 @@ fn test_reconcile_addresses() -> Result<()> { }; // get a DB reference back to we can check the results. - let db = store.db.lock().unwrap(); + let db = store.lock_db().unwrap(); let all = addresses::get_all_addresses(&db)?; From c7ab05f0613db1945135ec48689167d8962914c5 Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 24 Jul 2026 10:26:02 -0400 Subject: [PATCH 26/59] Switch fxa-client to proc-macros (#7493) Most other components are on proc-macros and most UniFFI development is happening with proc-macros. I checked the generated code and the only differences I could see is were in the comment and I liked the newer versions better. However, it's likely I'm missing something and there will be some unintinded breaking changes. We just started the nightly cycle, so this seems like a good time to switch over. --- components/fxa-client/build.rs | 7 - components/fxa-client/src/account.rs | 1 + components/fxa-client/src/auth.rs | 24 +- components/fxa-client/src/device.rs | 35 +- components/fxa-client/src/error.rs | 3 +- components/fxa-client/src/fxa_client.udl | 1031 ---------------------- components/fxa-client/src/lib.rs | 13 +- components/fxa-client/src/profile.rs | 2 + components/fxa-client/src/push.rs | 25 +- components/fxa-client/src/storage.rs | 2 + components/fxa-client/src/telemetry.rs | 1 + components/fxa-client/src/token.rs | 8 +- 12 files changed, 79 insertions(+), 1073 deletions(-) delete mode 100644 components/fxa-client/build.rs delete mode 100644 components/fxa-client/src/fxa_client.udl diff --git a/components/fxa-client/build.rs b/components/fxa-client/build.rs deleted file mode 100644 index 72c39b12505..00000000000 --- a/components/fxa-client/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -fn main() { - uniffi::generate_scaffolding("./src/fxa_client.udl").unwrap(); -} diff --git a/components/fxa-client/src/account.rs b/components/fxa-client/src/account.rs index 5f4fbeced50..232a0d9c165 100644 --- a/components/fxa-client/src/account.rs +++ b/components/fxa-client/src/account.rs @@ -15,6 +15,7 @@ use crate::{ApiResult, Error, FirefoxAccount, FxaServer}; use error_support::handle_error; +#[uniffi::export] impl FirefoxAccount { /// Check if an account was created from a config #[handle_error(Error)] diff --git a/components/fxa-client/src/auth.rs b/components/fxa-client/src/auth.rs index 136d38c3b13..ecbb7107f68 100644 --- a/components/fxa-client/src/auth.rs +++ b/components/fxa-client/src/auth.rs @@ -26,6 +26,7 @@ use crate::{ApiResult, DeviceConfig, Error, FirefoxAccount}; use error_support::handle_error; +#[uniffi::export] impl FirefoxAccount { /// Get the current state pub fn get_state(&self) -> FxaState { @@ -122,6 +123,7 @@ impl FirefoxAccount { } } +#[derive(uniffi::Record)] /// Information about the authorization state of the application. /// /// This struct represents metadata about whether the application is currently @@ -130,6 +132,7 @@ pub struct AuthorizationInfo { pub active: bool, } +#[derive(uniffi::Enum, Clone, Copy, Debug, PartialEq, Eq)] /// High-level view of the authorization state /// /// This is named `FxaRustAuthState` because it doesn't track all the states we want yet and needs @@ -138,17 +141,16 @@ pub struct AuthorizationInfo { /// /// In the long-term, we should track that data in Rust, remove the wrapper, and rename this to /// `FxaAuthState`. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FxaRustAuthState { Disconnected, Connected, AuthIssues, } +#[derive(uniffi::Enum, Clone, Debug, PartialEq, Eq)] /// Fxa state /// /// These are the states of [crate::FxaStateMachine] that consumers observe. -#[derive(Clone, Debug, PartialEq, Eq)] pub enum FxaState { /// The state machine needs to be initialized via [Event::Initialize]. Uninitialized, @@ -177,10 +179,10 @@ impl From for FxaState { } } +#[derive(uniffi::Enum, Clone, Debug, PartialEq, Eq)] /// Fxa event /// /// These are the events that consumers send to [crate::FxaStateMachine::process_event] -#[derive(Clone, Debug, PartialEq, Eq)] pub enum FxaEvent { /// Initialize the state machine. This must be the first event sent. Initialize { device_config: DeviceConfig }, @@ -219,14 +221,6 @@ pub enum FxaEvent { /// /// This event is valid for the `Authenticating` state. CompleteOAuthFlow { code: String, state: String }, - /// An `fxaccounts:change_password` WebChannel message arrived on the device that just changed - /// its password. `json_payload` is the `data` object of that message and contains the new - /// session token. The state machine swaps the session token for a new refresh token and - /// re-initialises the device record. - /// - /// This event is valid for the `Connected` and `AuthIssues` states. In `Authenticating` it - /// is a no-op so the in-progress OAuth flow is not disrupted. - WebChannelPasswordChange { json_payload: String }, /// Cancel an OAuth flow. /// /// Use this to cancel an in-progress OAuth, returning to [FxaState::Disconnected] so the @@ -243,6 +237,14 @@ pub enum FxaEvent { /// /// This event is valid for the `Connected` state. CheckAuthorizationStatus, + /// An `fxaccounts:change_password` WebChannel message arrived on the device that just changed + /// its password. `json_payload` is the `data` object of that message and contains the new + /// session token. The state machine swaps the session token for a new refresh token and + /// re-initialises the device record. + /// + /// This event is valid for the `Connected` and `AuthIssues` states. In `Authenticating` it + /// is a no-op so the in-progress OAuth flow is not disrupted. + WebChannelPasswordChange { json_payload: String }, /// Disconnect the user /// /// Send this when the user is asking to be logged out. The state machine will transition to diff --git a/components/fxa-client/src/device.rs b/components/fxa-client/src/device.rs index 2d00b358845..21b9cfeb990 100644 --- a/components/fxa-client/src/device.rs +++ b/components/fxa-client/src/device.rs @@ -21,6 +21,7 @@ use sync15::DeviceType; use crate::{ApiResult, DevicePushSubscription, Error, FirefoxAccount}; +#[uniffi::export] impl FirefoxAccount { /// Create a new device record for this application. /// @@ -189,16 +190,19 @@ impl FirefoxAccount { } } +#[derive(uniffi::Record, Clone, Debug, PartialEq, Eq)] /// Device configuration -#[derive(Clone, Debug, PartialEq, Eq)] pub struct DeviceConfig { pub name: String, pub device_type: sync15::DeviceType, pub capabilities: Vec, } +#[derive(uniffi::Record, Debug, Clone, Serialize, Deserialize)] /// Local device that's connecting to FxA -#[derive(Debug, Clone, Serialize, Deserialize)] +/// +/// This is returned by the device update methods and represents the server's view of the local +/// device. pub struct LocalDevice { pub id: String, pub display_name: String, @@ -208,12 +212,12 @@ pub struct LocalDevice { pub push_endpoint_expired: bool, } +#[derive(uniffi::Record, Debug)] /// A device connected to the user's account. /// /// This struct provides metadata about a device connected to the user's account. /// This data would typically be used to display e.g. the list of candidate devices /// in a "send tab" menu. -#[derive(Debug)] pub struct Device { pub id: String, pub display_name: String, @@ -225,6 +229,7 @@ pub struct Device { pub last_access_time: Option, } +#[derive(uniffi::Enum, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] /// A "capability" offered by a device. /// /// In the FxA ecosystem, connected devices may advertise their ability to respond @@ -232,12 +237,12 @@ pub struct Device { /// executing these commands are encapsulated as part of the FxA Client component, /// so consumers simply need to select which ones they want to support, and can /// use the variants of this enum to do so. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] pub enum DeviceCapability { SendTab, CloseTabs, } +#[derive(uniffi::Record)] /// A client connected to the user's account. /// /// This struct provides metadata about a client connected to the user's account. @@ -248,7 +253,6 @@ pub enum DeviceCapability { /// /// This data would typically be used for targeted messaging purposes, catering the /// contents of the message to what other applications the user has on their account. -/// pub struct AttachedClient { pub client_id: Option, pub device_id: Option, @@ -260,8 +264,27 @@ pub struct AttachedClient { pub scope: Option>, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(uniffi::Enum, Clone, Debug, PartialEq, Eq)] +/// The result of invoking a "close tabs" command. +/// +/// If [`FirefoxAccount::close_tabs`] is called with more URLs than can fit +/// into a single command payload, the URLs will be chunked and sent in +/// multiple commands. +/// +/// Chunking breaks the atomicity of a "close tabs" command, but +/// reduces the number of these commands that FxA sends to other devices. +/// This is critical for platforms like iOS, where every command triggers a +/// push message that must show a user-visible notification. pub enum CloseTabsResult { + /// All URLs passed to [`FirefoxAccount::close_tabs`] were chunked and sent + /// in one or more device commands. Ok, + /// One or more URLs passed to [`FirefoxAccount::close_tabs`] couldn't be sent + /// in a device command. The caller can assume that: + /// + /// 1. Any URL in the returned list of `urls` was not sent, and + /// should be retried. + /// 2. All other URLs that were passed to [`FirefoxAccount::close_tabs`], and + /// that are _not_ in the list of `urls`, were chunked and sent. TabsNotClosed { urls: Vec }, } diff --git a/components/fxa-client/src/error.rs b/components/fxa-client/src/error.rs index 92faec537be..2562dcb97bf 100644 --- a/components/fxa-client/src/error.rs +++ b/components/fxa-client/src/error.rs @@ -5,11 +5,12 @@ use error_support::{ErrorHandling, GetErrorHandling}; use std::string; +#[derive(uniffi::Error, Debug, thiserror::Error)] +#[uniffi(flat_error)] /// Public error type thrown by many [`FirefoxAccount`] operations. /// /// Precise details of the error are hidden from consumers. The type of the error indicates how the /// calling code should respond. -#[derive(Debug, thiserror::Error)] pub enum FxaError { /// Thrown when there was a problem with the authentication status of the account, /// such as an expired token. The application should [check its authorization status]( diff --git a/components/fxa-client/src/fxa_client.udl b/components/fxa-client/src/fxa_client.udl deleted file mode 100644 index 1f3b45483bf..00000000000 --- a/components/fxa-client/src/fxa_client.udl +++ /dev/null @@ -1,1031 +0,0 @@ -[External="sync15"] -typedef enum DeviceType; - -/// # Firefox Accounts Client -/// -/// The fxa-client component lets applications integrate with the -/// [Firefox Accounts](https://mozilla.github.io/ecosystem-platform/docs/features/firefox-accounts/fxa-overview) -/// identity service. The shape of a typical integration would look -/// something like: -/// -/// * Out-of-band, register your application with the Firefox Accounts service, -/// providing an OAuth `redirect_uri` controlled by your application and -/// obtaining an OAuth `client_id`. -/// -/// * On application startup, create a [`FirefoxAccount`] object to represent the -/// signed-in state of the application. -/// * On first startup, a new [`FirefoxAccount`] can be created by calling -/// [`FirefoxAccount::new`] and passing the application's `client_id`. -/// * For subsequent startups the object can be persisted using the -/// [`to_json`](FirefoxAccount::to_json) method and re-created by -/// calling [`FirefoxAccount::from_json`]. -/// -/// * When the user wants to sign in to your application, direct them through -/// a web-based OAuth flow by sending the `BeginOAuthFlow` or `BeginPairingFlow` -/// state-machine event; when they return to your registered `redirect_uri`, pass the -/// resulting authorization state back via the `CompleteOAuthFlow` event to sign them in. -/// -/// * Display information about the signed-in user by using the data from -/// [`get_profile`](FirefoxAccount::get_profile). -/// -/// * Access account-related services on behalf of the user by obtaining OAuth -/// access tokens via [`get_access_token`](FirefoxAccount::get_access_token). -/// -/// * If the user opts to sign out of the application, calling [`disconnect`](FirefoxAccount::disconnect) -/// and then discarding any persisted account data. -namespace fxa_client { -}; - - - -/// Generic error type thrown by many [`FirefoxAccount`] operations. -/// -/// Precise details of the error are hidden from consumers, mostly due to limitations of -/// how we expose this API to other languages. The type of the error indicates how the -/// calling code should respond. -/// -[Error] -enum FxaError { - - /// Thrown when there was a problem with the authentication status of the account, - /// such as an expired token. The application should [check its authorization status]( - /// FirefoxAccount::check_authorization_status) to see whether it has been disconnected, - /// or retry the operation with a freshly-generated token. - "Authentication", - - /// Thrown when an authenticated account isn't allowed to perform some operation. Unlike - /// `Authentication`, there's no problem with the account status. In some cases it - /// might be possible to request additional scopes, and once granted, the operation - /// may succeed. - "Forbidden", - - /// Thrown if an operation fails due to network access problems. - /// The application may retry at a later time once connectivity is restored. - "Network", - - /// Thrown if the application attempts to complete an OAuth flow when no OAuth flow has been initiated for that state. - /// This may indicate a user who navigated directly to the OAuth `redirect_uri` for the application. - "NoExistingAuthFlow", - - /// Thrown if the application attempts to complete an OAuth flow, but the state - /// tokens returned from the Firefox Account server do not match with the ones - /// expected by the client. - /// This may indicate a stale OAuth flow, or potentially an attempted hijacking - /// of the flow by an attacker. The signin attempt cannot be completed. - /// - /// **Note:** This error is currently only thrown in the Swift language bindings. - "WrongAuthFlow", - - /// Origin mismatch when handling a pairing flow - /// - /// The most likely cause of this is that a user tried to pair together two firefox instances - /// that are configured to use different servers. - "OriginMismatch", - - /// The sync scoped key was missing in the server response - "SyncScopedKeyMissingInServerResponse", - - /// Thrown if there is a panic in the underlying Rust code. - /// - /// **Note:** This error is currently only thrown in the Kotlin language bindings. - "Panic", - - /// A catch-all for other unspecified errors. - "Other", -}; - - -/// The result of invoking a "close tabs" command. -/// -/// If [`FirefoxAccount::close_tabs`] is called with more URLs than can fit -/// into a single command payload, the URLs will be chunked and sent in -/// multiple commands. -/// -/// Chunking breaks the atomicity of a "close tabs" command, but -/// reduces the number of these commands that FxA sends to other devices. -/// This is critical for platforms like iOS, where every command triggers a -/// push message that must show a user-visible notification. -[Enum] -interface CloseTabsResult { - /// All URLs passed to [`FirefoxAccount::close_tabs`] were chunked and sent - /// in one or more device commands. - Ok(); - - /// One or more URLs passed to [`FirefoxAccount::close_tabs`] couldn't be sent - /// in a device command. The caller can assume that: - /// - /// 1. Any URL in the returned list of `urls` was not sent, and - /// should be retried. - /// 2. All other URLs that were passed to [`FirefoxAccount::close_tabs`], and - /// that are _not_ in the list of `urls`, were chunked and sent. - TabsNotClosed(sequence urls); -}; - - -/// Object representing the signed-in state of an application. -/// -/// The `FirefoxAccount` object is the main interface provided by this crate. -/// It represents the signed-in state of an application that may be connected to -/// user's Firefox Account, and provides methods for inspecting the state of the -/// account and accessing other services on behalf of the user. -/// -interface FirefoxAccount { - - /// Create a new [`FirefoxAccount`] instance, not connected to any account. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method constructs as new [`FirefoxAccount`] instance configured to connect - /// the application to a user's account. - constructor(FxaConfig config); - - /// Restore a [`FirefoxAccount`] instance from serialized state. - /// - /// Given a JSON string previously obtained from [`FirefoxAccount::to_json`], this - /// method will deserialize it and return a live [`FirefoxAccount`] instance. - /// - /// **⚠️ Warning:** since the serialized state contains access tokens, you should - /// not call `from_json` multiple times on the same data. This would result - /// in multiple live objects sharing the same access tokens and is likely to - /// produce unexpected behaviour. - /// - [Throws=FxaError,Name=from_json] - constructor([ByRef] string data); - - - /// Save current state to a JSON string. - /// - /// This method serializes the current account state into a JSON string, which - /// the application can use to persist the user's signed-in state across restarts. - /// The application should call this method and update its persisted state after - /// any potentially-state-changing operation. - /// - /// **⚠️ Warning:** the serialized state may contain encryption keys and access - /// tokens that let anyone holding them access the user's data in Firefox Sync - /// and/or other FxA services. Applications should take care to store the resulting - /// data in a secure fashion, as appropriate for their target platform. - /// - [Throws=FxaError] - string to_json(); - - /// Stores anything necessary from a WebChannel login JSON payload. This includes the session - /// token, but that is abstracted because the consuming apps should not be aware of the - /// specific payload format returned, nor should they get access to the session token - /// directly if possible. - /// The [json_payload] is the `data` object from the `fxaccounts:login` WebChannel command. - [Throws=FxaError] - void handle_web_channel_login(string json_payload); - - /// Handle a WebChannel password-change notification by exchanging the new session token - /// for a new refresh token via a network call. - /// The [json_payload] is the `data` object from the `fxaccounts:change_password` WebChannel command. - [Throws=FxaError] - void handle_web_channel_password_change(string json_payload); - - /// Returns a complete signedInUser JSON object for a WebChannel fxaccounts:fxa_status response, - /// embedding the session token privately. Email and uid come from the cached profile in internal - /// state. Returns null if no session token is set. - string? get_signed_in_user_for_web_channel(); - - /// Get the URL at which to begin a device-pairing signin flow. - /// - /// If the user wants to sign in using device pairing, call this method and then - /// direct them to visit the resulting URL on an already-signed-in device. Doing - /// so will trigger the other device to show a QR code to be scanned, and the result - /// from said QR code can be passed to the `BeginPairingFlow` state-machine event. - /// - [Throws=FxaError] - string get_pairing_authority_url(); - - - /// Check authorization status for this application. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications may call this method to check with the FxA server about the status - /// of their authentication tokens. It returns an [`AuthorizationInfo`] struct - /// with details about whether the tokens are still active. - /// - [Throws=FxaError] - AuthorizationInfo check_authorization_status(); - - - /// Disconnect from the user's account. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method destroys any tokens held by the client, effectively disconnecting - /// from the user's account. Applications should call this when the user opts to - /// sign out. - /// - /// The persisted account state after calling this method will contain only the - /// user's last-seen profile information, if any. This may be useful in helping - /// the user to reconnnect to their account. If reconnecting to the same account - /// is not desired then the application should discard the persisted account state. - /// - void disconnect(); - - /// Update the state based on authentication issues. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Call this if you know there's an authentication / authorization issue that requires the - /// user to re-authenticated. It transitions the user to the [FxaRustAuthState.AuthIssues] state. - void on_auth_issues(); - - /// Get the high-level authentication state of the client - /// - /// Deprecated: Use get_state() instead - FxaRustAuthState get_auth_state(); - - /// Get the current state - FxaState get_state(); - - /// Process an event (login, logout, etc). - /// - /// On success, update the current state and return it. - /// On error, the current state will remain the same. - [Throws=FxaError] - FxaState process_event(FxaEvent event); - - /// Get profile information for the signed-in user, if any. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method fetches a [`Profile`] struct with information about the currently-signed-in - /// user, either by using locally-cached profile information or by fetching fresh data from - /// the server. - /// - /// # Arguments - /// - /// - `ignore_cache` - if true, always hit the server for fresh profile information. - /// - /// # Notes - /// - /// - Profile information is only available to applications that have been - /// granted the `profile` scope. - /// - There is currently no API for fetching cached profile information without - /// potentially hitting the server. - /// - If there is no signed-in user, this method will throw an - /// [`Authentication`](FxaError::Authentication) error. - /// - [Throws=FxaError] - Profile get_profile( boolean ignore_cache ); - - - /// Create a new device record for this application. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method registered a device record for the application, providing basic metadata for - /// the device along with a list of supported [Device Capabilities](DeviceCapability) for - /// participating in the "device commands" ecosystem. - /// - /// Applications should call this method soon after a successful sign-in, to ensure - /// they they appear correctly in the user's account-management pages and when discovered - /// by other devices connected to the account. - /// - /// # Arguments - /// - /// - `name` - human-readable display name to use for this application - /// - `device_type` - the [type](DeviceType) of device the application is installed on - /// - `supported_capabilities` - the set of [capabilities](DeviceCapability) to register - /// for this device in the "device commands" ecosystem. - /// - /// # Notes - /// - /// - Device registration is only available to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - LocalDevice initialize_device([ByRef] string name, DeviceType device_type, sequence supported_capabilities ); - - - /// Get the device id registered for this application. - /// - /// # Notes - /// - /// - If the application has not registered a device record, this method will - /// throw an [`Other`](FxaError::Other) error. - /// - (Yeah...sorry. This should be changed to do something better.) - /// - Device metadata is only visible to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - string get_current_device_id(); - - - /// Get the list of devices registered on the user's account. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method returns a list of [`Device`] structs representing all the devices - /// currently attached to the user's account (including the current device). - /// The application might use this information to e.g. display a list of appropriate - /// send-tab targets. - /// - /// # Arguments - /// - /// - `ignore_cache` - if true, always hit the server for fresh profile information. - /// - /// # Notes - /// - /// - Device metadata is only visible to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - sequence get_devices( boolean ignore_cache ); - - - /// Get the list of all client applications attached to the user's account. - /// - /// This method returns a list of [`AttachedClient`] structs representing all the applications - /// connected to the user's account. This includes applications that are registered as a device - /// as well as server-side services that the user has connected. - /// - /// This information is really only useful for targeted messaging or marketing purposes, - /// e.g. if the application wants to advertize a related product, but first wants to check - /// whether the user is already using that product. - /// - /// # Notes - /// - /// - Attached client metadata is only visible to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - sequence get_attached_clients(); - - - /// Update the display name used for this application instance. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method modifies the name of the current application's device record, as seen by - /// other applications and in the user's account management pages. - /// - /// # Arguments - /// - /// - `display_name` - the new name for the current device. - /// - /// # Notes - /// - /// - Device registration is only available to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - LocalDevice set_device_name([ByRef] string display_name ); - - - /// Clear any custom display name used for this application instance. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method clears the name of the current application's device record, causing other - /// applications or the user's account management pages to have to fill in some sort of - /// default name when displaying this device. - /// - /// # Notes - /// - /// - Device registration is only available to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - void clear_device_name(); - - - /// Ensure that the device record has a specific set of capabilities. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method checks that the currently-registered device record is advertising the - /// given set of capabilities in the FxA "device commands" ecosystem. If not, then it - /// updates the device record to do so. - /// - /// Applications should call this method on each startup as a way to ensure that their - /// expected set of capabilities is being accurately reflected on the FxA server, and - /// to handle the rollout of new capabilities over time. - /// - /// # Arguments - /// - /// - `supported_capabilities` - the set of [capabilities](DeviceCapability) to register - /// for this device in the "device commands" ecosystem. - /// - /// # Notes - /// - /// - Device registration is only available to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - LocalDevice ensure_capabilities( sequence supported_capabilities ); - - - /// Set or update a push subscription endpoint for this device. - /// - /// **💾 This method alters the persisted account state.** - /// - /// This method registers the given webpush subscription with the FxA server, requesting - /// that is send notifications in the event of any significant changes to the user's - /// account. When the application receives a push message at the registered subscription - /// endpoint, it should decrypt the payload and pass it to the [`handle_push_message`]( - /// FirefoxAccount::handle_push_message) method for processing. - /// - /// # Arguments - /// - /// - `subscription` - the [`DevicePushSubscription`] details to register with the server. - /// - /// # Notes - /// - /// - Device registration is only available to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - LocalDevice set_push_subscription( DevicePushSubscription subscription ); - - - /// Process and respond to a server-delivered account update message - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications should call this method whenever they receive a push notification from the Firefox Accounts server. - /// Such messages typically indicate a noteworthy change of state on the user's account, such as an update to their profile information - /// or the disconnection of a client. The [`FirefoxAccount`] struct will update its internal state - /// accordingly and return an individual [`AccountEvent`] struct describing the event, which the application - /// may use for further processing. - /// - /// It's important to note if the event is [`AccountEvent::CommandReceived`], the caller should call - /// [`FirefoxAccount::poll_device_commands`] - /// - [Throws=FxaError] - AccountEvent handle_push_message([ByRef] string payload ); - - - /// Poll the server for any pending device commands. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications that have registered one or more [`DeviceCapability`]s with the server can use - /// this method to check whether other devices on the account have sent them any commands. - /// It will return a list of [`IncomingDeviceCommand`] structs for the application to process. - /// - /// # Notes - /// - /// - Device commands are typically delivered via push message and the [`CommandReceived`]( - /// AccountEvent::CommandReceived) event. Polling should only be used as a backup delivery - /// mechanism, f the application has reason to believe that push messages may have been missed. - /// - Device commands functionality is only available to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - sequence poll_device_commands(); - - - /// Use device commands to send a single tab to another device. - /// - /// **💾 This method alters the persisted account state.** - /// - /// If a device on the account has registered the [`SendTab`](DeviceCapability::SendTab) - /// capability, this method can be used to send it a tab. - /// - /// # Notes - /// - /// - If the given device id does not existing or is not capable of receiving tabs, - /// this method will throw an [`Other`](FxaError::Other) error. - /// - (Yeah...sorry. This should be changed to do something better.) - /// - It is not currently possible to send a full [`SendTabPayload`] to another device, - /// but that's purely an API limitation that should go away in future. - /// - Device commands functionality is only available to applications that have been - /// granted the `https:///identity.mozilla.com/apps/oldsync` scope. - /// - [Throws=FxaError] - void send_single_tab([ByRef] string target_device_id, [ByRef] string title, [ByRef] string url, optional boolean is_private = false ); - - - /// Use device commands to close one or more tabs on another device. - /// - /// **💾 This method alters the persisted account state.** - /// - /// If a device on the account has registered the [`CloseTabs`](DeviceCapability::CloseTabs) - /// capability, this method can be used to close its tabs. - [Throws=FxaError] - CloseTabsResult close_tabs([ByRef] string target_device_id, sequence urls); - - - /// Get the URL at which to access the user's sync data. - /// - /// **💾 This method alters the persisted account state.** - /// - [Throws=FxaError] - string get_token_server_endpoint_url(); - - - /// Get a URL which shows a "successfully connceted!" message. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications can use this method after a successful signin, to redirect the - /// user to a success message displayed in web content rather than having to - /// implement their own native success UI. - /// - [Throws=FxaError] - string get_connection_success_url(); - - - /// Get a URL at which the user can manage their account and profile data. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications should link the user out to this URL from an appropriate place - /// in their signed-in settings UI. - /// - /// # Arguments - /// - /// - `entrypoint` - metrics identifier for UX entrypoint. - /// - This parameter is used for metrics purposes, to identify the - /// UX entrypoint from which the user followed the link. - /// - [Throws=FxaError] - string get_manage_account_url([ByRef] string entrypoint ); - - - /// Get a URL at which the user can manage the devices connected to their account. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications should link the user out to this URL from an appropriate place - /// in their signed-in settings UI. For example, "Manage your devices..." may be - /// a useful link to place somewhere near the device list in the send-tab UI. - /// - /// # Arguments - /// - /// - `entrypoint` - metrics identifier for UX entrypoint. - /// - This parameter is used for metrics purposes, to identify the - /// UX entrypoint from which the user followed the link. - /// - [Throws=FxaError] - string get_manage_devices_url([ByRef] string entrypoint ); - - - /// Get a short-lived OAuth access token for the user's account. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications that need to access resources on behalf of the user must obtain an - /// `access_token` in order to do so. For example, an access token is required when - /// fetching the user's profile data, or when accessing their data stored in Firefox Sync. - /// - /// This method will obtain and return an access token bearing the requested scopes, either - /// from a local cache of previously-issued tokens, or by creating a new one from the server. - /// - /// # Arguments - /// - /// - `scope` - space-separated list of OAuth scopes to be granted by the token. - /// - Each scope must have been requested during the signin flow, or be a scope - /// which the server might offer automatically in some account-specific cases. - /// - Scope order is not significant; `"a b"` and `"b a"` are equivalent. - /// - When a single scope is requested and it has an associated scoped key - /// (e.g. `https://identity.mozilla.com/apps/oldsync`), the returned - /// `AccessTokenInfo.key` will be populated; for multi-scope requests it is `null`. - /// - `use_cache` - optionally set to false to force a new token request. The fetched - /// token will still be cached for later `get_access_token` calls. - /// - /// # Notes - /// - /// - If the application receives an authorization error when trying to use the resulting - /// token, it should call [`clear_access_token_cache`](FirefoxAccount::clear_access_token_cache) - /// before requesting a fresh token. - /// - [Throws=FxaError] - AccessTokenInfo get_access_token([ByRef] string scope, optional boolean use_cache = true); - - /// Check whether the account has already been granted the given OAuth scope(s). - /// - /// This checks whether the refresh token has *every* specified scope. - /// - /// # Arguments - /// - `scope` - space-separated list of OAuth scopes. Order is not significant. - boolean has_scope([ByRef] string scope); - - /// Create a new OAuth authorization code using the stored session token. - /// - /// When a signed-in application receives an incoming device pairing request, it can - /// use this method to grant the request and generate a corresponding OAuth authorization - /// code. This code would then be passed back to the connecting device over the - /// pairing channel (a process which is not currently supported by any code in this - /// component). - /// - /// # Arguments - /// - /// - `params` - the OAuth parameters from the incoming authorization request - /// - [Throws=FxaError] - string authorize_code_using_session_token( AuthorizationParameters params ); - - - /// Clear the access token cache in response to an auth failure. - /// - /// **💾 This method alters the persisted account state.** - /// - /// Applications that receive an authentication error when trying to use an access token, - /// should call this method before creating a new token and retrying the failed operation. - /// It ensures that the expired token is removed and a fresh one generated. - /// - void clear_access_token_cache(); - - - /// Collect and return telemetry about incoming and outgoing device commands. - /// - /// Applications that have registered one or more [`DeviceCapability`]s - /// should also arrange to submit "sync ping" telemetry. Calling this method will - /// return a JSON string of telemetry data that can be incorporated into that ping. - /// - /// Sorry, this is not particularly carefully documented because it is intended - /// as a stop-gap until we get native Glean support. If you know how to submit - /// a sync ping, you'll know what to do with the contents of the JSON string. - /// - [Throws=FxaError] - string gather_telemetry(); - - /// Used by the application to test auth token issues - void simulate_network_error(); - - /// Used by the application to test auth token issues - void simulate_temporary_auth_token_issue(); - - /// Used by the application to test auth token issues - void simulate_permanent_auth_token_issue(); -}; - -dictionary FxaConfig { - /// FxaServer to connect with - FxaServer server; - /// Registered OAuth client id of the application. - string client_id; - /// `redirect_uri` - the registered OAuth redirect URI of the application. - string redirect_uri; - /// URL for the user's Sync Tokenserver. This can be used to support users who self-host their - /// sync data. If `None` then it will default to the Mozilla-hosted Sync server. - string? token_server_url_override = null; -}; - -/// FxA server to connect to -[Enum] -interface FxaServer { - Release(); - Stable(); - Stage(); - China(); // deprecated, same as Release - LocalDev(); - Custom(string url); -}; - -/// Information about the authorization state of the application. -/// -/// This struct represents metadata about whether the application is currently -/// connected to the user's account. -/// -dictionary AuthorizationInfo { - boolean active; -}; - -/// An OAuth access token, with its associated keys and metadata. -/// -/// This struct represents an FxA OAuth access token, which can be used to access a resource -/// or service on behalf of the user. For example, accessing the user's data in Firefox Sync -/// an access token for the scope `https:///identity.mozilla.com/apps/sync` along with the -/// associated encryption key. -/// -dictionary AccessTokenInfo { - - /// The scope of access granted by token. - string scope; - - /// The access token itself. - /// - /// This is the value that should be included in the `Authorization` header when - /// accessing an OAuth protected resource on behalf of the user. - string token; - - /// The client-side encryption key associated with this scope. - /// - /// **⚠️ Warning:** the value of this field should never be revealed outside of the - /// application. For example, it should never to sent to a server or logged in a log file. - ScopedKey? key; - - /// The expiry time of the token, in seconds. - /// - /// This is the timestamp at which the token is set to expire, in seconds since - /// unix epoch. Note that it is a signed integer, for compatibility with languages - /// that do not have an unsigned integer type. - /// - /// This timestamp is for guidance only. Access tokens are not guaranteed to remain - /// value for any particular lengthof time, and consumers should be prepared to handle - /// auth failures even if the token has not yet expired. - i64 expires_at; -}; - -/// A cryptographic key associated with an OAuth scope. -/// -/// Some OAuth scopes have a corresponding client-side encryption key that is required -/// in order to access protected data. This struct represents such key material in a -/// format compatible with the common "JWK" standard. -/// -dictionary ScopedKey { - - /// The type of key. - /// - /// In practice for FxA, this will always be string string "oct" (short for "octal") - /// to represent a raw symmetric key. - string kty; - - /// The OAuth scope with which this key is associated. - string scope; - - /// The key material, as base64-url-encoded bytes. - /// - /// **⚠️ Warning:** the value of this field should never be revealed outside of the - /// application. For example, it should never to sent to a server or logged in a log file. - string k; - - /// An opaque unique identifier for this key. - /// - /// Unlike the `k` field, this value is not secret and may be revealed to the server. - string kid; -}; - -/// Parameters provided in an incoming OAuth request. -/// -/// This struct represents parameters obtained from an incoming OAuth request - that is, -/// the values that an OAuth client would append to the authorization URL when initiating -/// an OAuth sign-in flow. -/// -dictionary AuthorizationParameters { - string client_id; - sequence scope; - string state; - string access_type; - string? code_challenge; - string? code_challenge_method; - string? keys_jwk; -}; - -/// A device connected to the user's account. -/// -/// This struct provides metadata about a device connected to the user's account. -/// This data would typically be used to display e.g. the list of candidate devices -/// in a "send tab" menu. -/// -dictionary Device { - string id; - string display_name; - DeviceType device_type; - sequence capabilities; - DevicePushSubscription? push_subscription; - boolean push_endpoint_expired; - boolean is_current_device; - i64? last_access_time; -}; - -/// Device configuration -dictionary DeviceConfig { - string name; - DeviceType device_type; - sequence capabilities; -}; - -/// Local device that's connecting to FxA -/// -/// This is returned by the device update methods and represents the server's view of the local -/// device. -dictionary LocalDevice { - string id; - string display_name; - DeviceType device_type; - sequence capabilities; - DevicePushSubscription? push_subscription; - boolean push_endpoint_expired; -}; - -/// Details of a web-push subscription endpoint. -/// -/// This struct encapsulates the details of a web-push subscription endpoint, -/// including all the information necessary to send a notification to its owner. -/// Devices attached to the user's account may register one of these in order -/// to receive timely updates about account-related events. -/// -/// Managing a web-push subscription is outside of the scope of this component. -/// -dictionary DevicePushSubscription { - string endpoint; - string public_key; - string auth_key; -}; - -/// The payload sent when invoking a "send tab" command. -/// -dictionary SendTabPayload { - - /// The navigation history of the sent tab. - /// - /// The last item in this list represents the page to be displayed, - /// while earlier items may be included in the navigation history - /// as a convenience to the user. - sequence entries; - - /// A unique identifier to be included in send-tab metrics. - /// - /// The application should treat this as opaque. - string flow_id = ""; - - /// A unique identifier to be included in send-tab metrics. - /// - /// The application should treat this as opaque. - string stream_id = ""; -}; - -/// The payload sent when invoking a "close tabs" command. -/// -dictionary CloseTabsPayload { - - /// The URLs of the tabs to close. - sequence urls; -}; - -/// A received tab. Mis-named as the original intent was to keep -/// the full "back" history for a tab, where this would be one such -/// entry - but that never happened. -/// -dictionary TabHistoryEntry { - string title; - string url; - boolean is_private = false; -}; - -/// A client connected to the user's account. -/// -/// This struct provides metadata about a client connected to the user's account. -/// Unlike the [`Device`] struct, "clients" encompasses both client-side and server-side -/// applications - basically anything where the user is able to sign in with their -/// Firefox Account. -/// -/// -/// This data would typically be used for targeted messaging purposes, catering the -/// contents of the message to what other applications the user has on their account. -/// -dictionary AttachedClient { - string? client_id; - string? device_id; - DeviceType device_type; - boolean is_current_session; - string? name; - i64? created_time; - i64? last_access_time; - sequence? scope; -}; - -/// Information about the user that controls a Firefox Account. -/// -/// This struct represents details about the user themselves, and would typically be -/// used to customize account-related UI in the browser so that it is personalize -/// for the current user. -/// -dictionary Profile { - - /// The user's account uid - /// - /// This is an opaque immutable unique identifier for their account. - string uid; - - /// The user's current primary email address. - /// - /// Note that unlike the `uid` field, the email address may change over time. - string email; - - /// The user's preferred textual display name. - string? display_name; - - /// The URL of a profile picture representing the user. - /// - /// All accounts have a corresponding profile picture. If the user has not - /// provided one then a default image is used. - string avatar; - - /// Whether the `avatar` URL represents the default avatar image. - boolean is_default_avatar; -}; - -[Enum] -interface FxaState { - Uninitialized(); - Disconnected(); - Authenticating(string oauth_url, FxaRustAuthState initial_state); - Connected(); - AuthIssues(); -}; - -[Enum] -interface FxaEvent { - Initialize(DeviceConfig device_config); - BeginOAuthFlow(string service, sequence scopes, string entrypoint); - BeginPairingFlow(string pairing_url, string service, sequence scopes, string entrypoint); - CompleteOAuthFlow(string code, string state); - CancelOAuthFlow(); - CheckAuthorizationStatus(); - WebChannelPasswordChange(string json_payload); - Disconnect(); - CallGetProfile(); -}; - -enum FxaRustAuthState { - "Disconnected", - "Connected", - "AuthIssues", -}; - -/// A "capability" offered by a device. -/// -/// In the FxA ecosystem, connected devices may advertize their ability to respond -/// to various "commands" that can be invoked by other devices. The details of -/// executing these commands are encapsulated as part of the FxA Client component, -/// so consumers simply need to select which ones they want to support, and can -/// use the variants of this enum to do so. -/// -enum DeviceCapability { - "SendTab", - "CloseTabs", -}; - - -/// An event that happened on the user's account. -/// -/// If the application has registered a [`DevicePushSubscription`] as part of its -/// device record, then the Firefox Accounts server can send push notifications -/// about important events that happen on the user's account. This enum represents -/// the different kinds of event that can occur. -/// -[Enum] -interface AccountEvent { - - /// Sent when another device has invoked a command for this device to execute. - /// - /// When receiving this event, the application should inspect the contained - /// command and react appropriately. - CommandReceived(IncomingDeviceCommand command ); - - /// Sent when the user has modified their account profile information. - /// - /// When receiving this event, the application should request fresh profile - /// information by calling [`get_profile`](FirefoxAccount::get_profile) with - /// `ignore_cache` set to true, and update any profile information displayed - /// in its UI. - /// - ProfileUpdated(); - - /// Sent when when there has been a change in authorization status. - /// - /// When receiving this event, the application should check whether it is - /// still connected to the user's account by calling [`check_authorization_status`]( - /// FirefoxAccount::check_authorization_status), and updating its UI as appropriate. - /// - AccountAuthStateChanged(); - - /// Sent when the user deletes their Firefox Account. - /// - /// When receiving this event, the application should act as though the user had - /// signed out, discarding any persisted account state. - AccountDestroyed(); - - /// Sent when a new device connects to the user's account. - /// - /// When receiving this event, the application may use it to trigger an update - /// of any UI that shows the list of connected devices. It may also show the - /// user an informational notice about the new device, as a security measure. - DeviceConnected(string device_name ); - - /// Sent when a device disconnects from the user's account. - /// - /// When receiving this event, the application may use it to trigger an update - /// of any UI that shows the list of connected devices. - DeviceDisconnected(string device_id, boolean is_local_device ); - - /// An unknown event, most likely an event the client doesn't support yet. - /// - /// When receiving this event, the application should gracefully ignore it. - Unknown(); -}; - - -/// A command invoked by another device. -/// -/// This enum represents all possible commands that can be invoked on -/// the device. It is the responsibility of the application to interpret -/// each command. -/// -[Enum] -interface IncomingDeviceCommand { - - /// Indicates that a tab has been sent to this device. - TabReceived(Device? sender, SendTabPayload payload ); - - /// Indicates that the sender wants to close one or more tabs on this device. - TabsClosed(Device? sender, CloseTabsPayload payload); -}; diff --git a/components/fxa-client/src/lib.rs b/components/fxa-client/src/lib.rs index 2b19e8fd7ac..37b4ff0e1ba 100644 --- a/components/fxa-client/src/lib.rs +++ b/components/fxa-client/src/lib.rs @@ -74,19 +74,20 @@ pub type Result = std::result::Result; /// Result returned by public-facing API functions pub type ApiResult = std::result::Result; +#[derive(uniffi::Object)] /// Object representing the signed-in state of an application. /// /// The `FirefoxAccount` object is the main interface provided by this crate. /// It represents the signed-in state of an application that may be connected to /// user's Firefox Account, and provides methods for inspecting the state of the /// account and accessing other services on behalf of the user. -/// pub struct FirefoxAccount { // For now, we serialize all access on a single `Mutex` for thread safety across // the FFI. We should make the locking more granular in future. internal: Mutex, } +#[uniffi::export] impl FirefoxAccount { /// Create a new [`FirefoxAccount`] instance, not connected to any account. /// @@ -94,6 +95,7 @@ impl FirefoxAccount { /// /// This method constructs as new [`FirefoxAccount`] instance configured to connect /// the application to a user's account. + #[uniffi::constructor] pub fn new(config: FxaConfig) -> FirefoxAccount { FirefoxAccount { internal: Mutex::new(internal::FirefoxAccount::new(config)), @@ -106,11 +108,11 @@ impl FirefoxAccount { } } -#[derive(Clone, Debug)] +#[derive(uniffi::Record, Clone, Debug)] pub struct FxaConfig { /// FxaServer to connect with pub server: FxaServer, - /// registered OAuth client id of the application. + /// Registered OAuth client id of the application. pub client_id: String, /// `redirect_uri` - the registered OAuth redirect URI of the application. pub redirect_uri: String, @@ -121,10 +123,11 @@ pub struct FxaConfig { /// the token server URL they get from `fxa-client` to `SyncManager`. It would be simpler to /// cut out `fxa-client` out of the middle and have applications send the overridden URL /// directly to `SyncManager`. + #[uniffi(default=None)] pub token_server_url_override: Option, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(uniffi::Enum, Clone, Debug, PartialEq, Eq)] pub enum FxaServer { Release, Stable, @@ -232,7 +235,7 @@ impl FxaConfig { } } -uniffi::include_scaffolding!("fxa_client"); +uniffi::setup_scaffolding!("fxa_client"); #[cfg(test)] mod tests { diff --git a/components/fxa-client/src/profile.rs b/components/fxa-client/src/profile.rs index a5d67a55a27..6bc8ec78dbf 100644 --- a/components/fxa-client/src/profile.rs +++ b/components/fxa-client/src/profile.rs @@ -9,6 +9,7 @@ use crate::{ApiResult, Error, FirefoxAccount}; use error_support::handle_error; +#[uniffi::export] impl FirefoxAccount { /// Get profile information for the signed-in user, if any. /// @@ -36,6 +37,7 @@ impl FirefoxAccount { } } +#[derive(uniffi::Record)] /// Information about the user that controls a Firefox Account. /// /// This struct represents details about the user themselves, and would typically be diff --git a/components/fxa-client/src/push.rs b/components/fxa-client/src/push.rs index c8718da4259..8975c217345 100644 --- a/components/fxa-client/src/push.rs +++ b/components/fxa-client/src/push.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::{internal, ApiResult, CloseTabsResult, Device, Error, FirefoxAccount, LocalDevice}; +#[uniffi::export] impl FirefoxAccount { /// Set or update a push subscription endpoint for this device. /// @@ -95,16 +96,17 @@ impl FirefoxAccount { /// - Device commands functionality is only available to applications that have been /// granted the `https://identity.mozilla.com/apps/oldsync` scope. #[handle_error(Error)] + #[uniffi::method(default(is_private = false))] pub fn send_single_tab( &self, target_device_id: &str, title: &str, url: &str, - private: bool, + is_private: bool, ) -> ApiResult<()> { self.internal .lock() - .send_single_tab(target_device_id, title, url, private) + .send_single_tab(target_device_id, title, url, is_private) } /// Use device commands to close one or more tabs on another device. @@ -123,6 +125,7 @@ impl FirefoxAccount { } } +#[derive(uniffi::Record, Debug, Clone, Serialize, Deserialize)] /// Details of a web-push subscription endpoint. /// /// This struct encapsulates the details of a web-push subscription endpoint, @@ -132,13 +135,14 @@ impl FirefoxAccount { /// /// Managing a web-push subscription is outside of the scope of this component. /// -#[derive(Debug, Clone, Serialize, Deserialize)] pub struct DevicePushSubscription { pub endpoint: String, pub public_key: String, pub auth_key: String, } +#[allow(clippy::large_enum_variant)] +#[derive(uniffi::Enum, Debug)] /// An event that happened on the user's account. /// /// If the application has registered a [`DevicePushSubscription`] as part of its @@ -149,8 +153,6 @@ pub struct DevicePushSubscription { // Clippy suggests we Box<> the CommandReceiver variant here, // but UniFFI isn't able to look through boxes yet, so we // disable the warning. -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] pub enum AccountEvent { /// Sent when another device has invoked a command for this device to execute. /// @@ -198,26 +200,27 @@ pub enum AccountEvent { Unknown, } +#[derive(uniffi::Enum, Debug)] /// A command invoked by another device. /// /// This enum represents all possible commands that can be invoked on /// the device. It is the responsibility of the application to interpret /// each command. -#[derive(Debug)] pub enum IncomingDeviceCommand { /// Indicates that a tab has been sent to this device. TabReceived { sender: Option, payload: SendTabPayload, }, + /// Indicates that the sender wants to close one or more tabs on this device. TabsClosed { sender: Option, payload: CloseTabsPayload, }, } +#[derive(uniffi::Record, Debug)] /// The payload sent when invoking a "send tab" command. -#[derive(Debug)] pub struct SendTabPayload { /// The navigation history of the sent tab. /// @@ -228,23 +231,27 @@ pub struct SendTabPayload { /// A unique identifier to be included in send-tab metrics. /// /// The application should treat this as opaque. + #[uniffi(default = "")] pub flow_id: String, /// A unique identifier to be included in send-tab metrics. /// /// The application should treat this as opaque. + #[uniffi(default = "")] pub stream_id: String, } +#[derive(uniffi::Record, Debug)] /// The payload sent when invoking a "close tabs" command. -#[derive(Debug)] pub struct CloseTabsPayload { + /// The URLs of the tabs to close. pub urls: Vec, } +#[derive(uniffi::Record, Debug)] /// An individual entry in the navigation history of a sent tab. -#[derive(Debug)] pub struct TabHistoryEntry { pub title: String, pub url: String, + #[uniffi(default = false)] pub is_private: bool, } diff --git a/components/fxa-client/src/storage.rs b/components/fxa-client/src/storage.rs index eef92495501..5dc50cfa413 100644 --- a/components/fxa-client/src/storage.rs +++ b/components/fxa-client/src/storage.rs @@ -21,6 +21,7 @@ use crate::{internal, ApiResult, Error, FirefoxAccount}; use error_support::handle_error; use parking_lot::Mutex; +#[uniffi::export] impl FirefoxAccount { /// Restore a [`FirefoxAccount`] instance from serialized state. /// @@ -31,6 +32,7 @@ impl FirefoxAccount { /// not call `from_json` multiple times on the same data. This would result /// in multiple live objects sharing the same access tokens and is likely to /// produce unexpected behaviour. + #[uniffi::constructor] #[handle_error(Error)] pub fn from_json(data: &str) -> ApiResult { Ok(FirefoxAccount { diff --git a/components/fxa-client/src/telemetry.rs b/components/fxa-client/src/telemetry.rs index 258f107fd18..c786b71a465 100644 --- a/components/fxa-client/src/telemetry.rs +++ b/components/fxa-client/src/telemetry.rs @@ -11,6 +11,7 @@ use crate::{ApiResult, Error, FirefoxAccount}; use error_support::handle_error; +#[uniffi::export] impl FirefoxAccount { /// Collect and return telemetry about send-tab attempts. /// diff --git a/components/fxa-client/src/token.rs b/components/fxa-client/src/token.rs index d06e5df0a66..d77501f2cd5 100644 --- a/components/fxa-client/src/token.rs +++ b/components/fxa-client/src/token.rs @@ -20,6 +20,7 @@ use error_support::handle_error; use serde_derive::*; use std::convert::TryInto; +#[uniffi::export] impl FirefoxAccount { /// Get a short-lived OAuth access token for the user's account. /// @@ -50,6 +51,7 @@ impl FirefoxAccount { /// token, it should call [`clear_access_token_cache`](FirefoxAccount::clear_access_token_cache) /// before requesting a fresh token. #[handle_error(Error)] + #[uniffi::method(default(use_cache = true))] pub fn get_access_token(&self, scope: &str, use_cache: bool) -> ApiResult { self.internal .lock() @@ -158,13 +160,13 @@ impl FirefoxAccount { } } +#[derive(uniffi::Record, Debug)] /// An OAuth access token, with its associated keys and metadata. /// /// This struct represents an FxA OAuth access token, which can be used to access a resource /// or service on behalf of the user. For example, accessing the user's data in Firefox Sync /// an access token for the scope `https://identity.mozilla.com/apps/sync` along with the /// associated encryption key. -#[derive(Debug)] pub struct AccessTokenInfo { /// The scope of access granted by token. pub scope: String, @@ -190,13 +192,12 @@ pub struct AccessTokenInfo { pub expires_at: i64, } +#[derive(uniffi::Record, Clone, Serialize, Deserialize)] /// A cryptographic key associated with an OAuth scope. /// /// Some OAuth scopes have a corresponding client-side encryption key that is required /// in order to access protected data. This struct represents such key material in a /// format compatible with the common "JWK" standard. -/// -#[derive(Clone, Serialize, Deserialize)] pub struct ScopedKey { /// The type of key. /// @@ -216,6 +217,7 @@ pub struct ScopedKey { pub kid: String, } +#[derive(uniffi::Record)] /// Parameters provided in an incoming OAuth request. /// /// This struct represents parameters obtained from an incoming OAuth request - that is, From a9692459d3efcdac48ababbe1afcaa821095b38a Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Mon, 27 Jul 2026 16:30:02 -0700 Subject: [PATCH 27/59] Add opt-in `mozbuild-rustlib` Cargo feature chain and `link_nss` implementation for NSS crates (#7496) --- Cargo.lock | 4 ++ components/support/rc_crypto/Cargo.toml | 1 + components/support/rc_crypto/nss/Cargo.toml | 1 + .../rc_crypto/nss/nss_build_common/Cargo.toml | 1 + .../rc_crypto/nss/nss_build_common/src/lib.rs | 42 +++++++++---------- .../support/rc_crypto/nss/nss_sys/Cargo.toml | 5 +++ .../support/rc_crypto/nss/nss_sys/build.rs | 3 ++ megazords/full/Cargo.toml | 8 ++++ monorepo-hacks/mozbuild/src/lib.rs | 4 ++ 9 files changed, 48 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be23f309934..4490ef3505b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2604,6 +2604,7 @@ dependencies = [ "nimbus-sdk", "places", "push", + "rc_crypto", "relay", "remote_settings", "rust-log-forwarder", @@ -2988,6 +2989,9 @@ dependencies = [ [[package]] name = "nss_build_common" version = "0.1.0" +dependencies = [ + "mozbuild", +] [[package]] name = "nss_sys" diff --git a/components/support/rc_crypto/Cargo.toml b/components/support/rc_crypto/Cargo.toml index 73f52dbb02e..d8400b5b5ba 100644 --- a/components/support/rc_crypto/Cargo.toml +++ b/components/support/rc_crypto/Cargo.toml @@ -27,3 +27,4 @@ features = ["serializable-keys", "backend-test-helper"] [features] default = [] backtrace = ["error-support/backtrace"] +mozbuild-rustlib = ["nss-as/mozbuild-rustlib"] diff --git a/components/support/rc_crypto/nss/Cargo.toml b/components/support/rc_crypto/nss/Cargo.toml index 8f663c821e5..cf875926124 100644 --- a/components/support/rc_crypto/nss/Cargo.toml +++ b/components/support/rc_crypto/nss/Cargo.toml @@ -21,3 +21,4 @@ once_cell = { version = "1.20.2", optional = true } default = [] keydb = ["dep:once_cell"] backtrace = ["error-support/backtrace"] +mozbuild-rustlib = ["nss_sys/mozbuild-rustlib"] diff --git a/components/support/rc_crypto/nss/nss_build_common/Cargo.toml b/components/support/rc_crypto/nss/nss_build_common/Cargo.toml index d023647e147..ae849001061 100644 --- a/components/support/rc_crypto/nss/nss_build_common/Cargo.toml +++ b/components/support/rc_crypto/nss/nss_build_common/Cargo.toml @@ -6,3 +6,4 @@ edition = "2021" license = "MPL-2.0" [dependencies] +mozbuild = "0.1" diff --git a/components/support/rc_crypto/nss/nss_build_common/src/lib.rs b/components/support/rc_crypto/nss/nss_build_common/src/lib.rs index 0c076c650a5..3a73368100d 100644 --- a/components/support/rc_crypto/nss/nss_build_common/src/lib.rs +++ b/components/support/rc_crypto/nss/nss_build_common/src/lib.rs @@ -23,31 +23,31 @@ pub enum LinkingKind { pub struct NoNssDir; pub fn link_nss() -> Result<(), NoNssDir> { - let is_gecko = env::var_os("MOZ_TOPOBJDIR").is_some(); - if !is_gecko { - let (lib_dir, include_dir) = get_nss()?; - println!( - "cargo:rustc-link-search=native={}", - lib_dir.to_string_lossy() - ); - println!("cargo:include={}", include_dir.to_string_lossy()); - let kind = determine_kind(); - link_nss_libs(kind); - } else { - let libs = match env::var("CARGO_CFG_TARGET_OS") - .as_ref() - .map(std::string::String::as_str) - { - Ok("android") | Ok("macos") => vec!["nss3"], - _ => vec!["nssutil3", "nss3", "plds4", "plc4", "nspr4"], - }; - for lib in &libs { - println!("cargo:rustc-link-lib=dylib={}", lib); - } + if env::var_os("MOZ_TOPOBJDIR").is_some() { + mozbuild::link_nss(); + return Ok(()); } + let (lib_dir, include_dir) = get_nss()?; + println!( + "cargo:rustc-link-search=native={}", + lib_dir.to_string_lossy() + ); + println!("cargo:include={}", include_dir.to_string_lossy()); + let kind = determine_kind(); + link_nss_libs(kind); Ok(()) } +pub fn link_nss_rustlib() -> Result<(), NoNssDir> { + if env::var_os("MOZ_TOPOBJDIR").is_some() { + mozbuild::link_nss_rustlib(); + return Ok(()); + } + // The standalone NSS_DIR path already produces a self-contained link + // (including mozpkix when statically linked), so this matches link_nss. + link_nss() +} + fn get_nss() -> Result<(PathBuf, PathBuf), NoNssDir> { let nss_dir = env("NSS_DIR").ok_or(NoNssDir)?; let nss_dir = Path::new(&nss_dir); diff --git a/components/support/rc_crypto/nss/nss_sys/Cargo.toml b/components/support/rc_crypto/nss/nss_sys/Cargo.toml index 16e51cbf6b5..c493c18f598 100644 --- a/components/support/rc_crypto/nss/nss_sys/Cargo.toml +++ b/components/support/rc_crypto/nss/nss_sys/Cargo.toml @@ -16,3 +16,8 @@ nss_build_common = { path = "../nss_build_common" } [features] default = [] +# Adds static linkage for mozpkix and pure_virtual on top of the dylib link. +# Activate from crates that produce a self-contained Rust artifact (megazord +# cdylib, rusttests). Do NOT activate from libxul's dependency graph; libxul +# already links mozpkix itself, and duplicate linkage causes symbol conflicts. +mozbuild-rustlib = [] diff --git a/components/support/rc_crypto/nss/nss_sys/build.rs b/components/support/rc_crypto/nss/nss_sys/build.rs index bbdec0e6622..e1dc28f710e 100644 --- a/components/support/rc_crypto/nss/nss_sys/build.rs +++ b/components/support/rc_crypto/nss/nss_sys/build.rs @@ -3,5 +3,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ fn main() { + #[cfg(feature = "mozbuild-rustlib")] + nss_build_common::link_nss_rustlib().unwrap(); + #[cfg(not(feature = "mozbuild-rustlib"))] nss_build_common::link_nss().unwrap(); } diff --git a/megazords/full/Cargo.toml b/megazords/full/Cargo.toml index e270777e518..da6d3a77133 100644 --- a/megazords/full/Cargo.toml +++ b/megazords/full/Cargo.toml @@ -35,3 +35,11 @@ merino = { path = "../../components/merino" } relay = { path = "../../components/relay" } ads-client = { path = "../../components/ads-client" } mozilla-central-workspace-hack = { version = "0.1", features = ["megazord"], optional = true } + +# Direct dep so this crate can forward the mozbuild-rustlib feature down the +# NSS link chain. The megazord cdylib produces a self-contained Rust artifact +# and needs static mozpkix + pure_virtual on top of the NSS dylibs. +rc_crypto = { path = "../../components/support/rc_crypto" } + +[features] +mozbuild-rustlib = ["rc_crypto/mozbuild-rustlib"] diff --git a/monorepo-hacks/mozbuild/src/lib.rs b/monorepo-hacks/mozbuild/src/lib.rs index 6a087b1156a..ef0ade29509 100644 --- a/monorepo-hacks/mozbuild/src/lib.rs +++ b/monorepo-hacks/mozbuild/src/lib.rs @@ -25,3 +25,7 @@ pub mod config { pub const NSS_CFLAGS: [&str; 0] = []; pub const NSPR_CFLAGS: [&str; 0] = []; } + +pub fn link_nss() {} + +pub fn link_nss_rustlib() {} From ba390372f9dcf934e1936c90f19b2317a0e677e7 Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Mon, 27 Jul 2026 17:09:20 -0700 Subject: [PATCH 28/59] Source uniffi Kotlin from the `mozilla-central` checkout when mozconfig is set (#7497) * Add opt-in `mozbuild-rustlib` Cargo feature chain and `link_nss` implementation for NSS crates * Source uniffi Kotlin from the `mozilla-central` checkout when mozconfig is set --- build-scripts/component-common.gradle | 43 +++++++-------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/build-scripts/component-common.gradle b/build-scripts/component-common.gradle index c542937dddd..98957d2ebe5 100644 --- a/build-scripts/component-common.gradle +++ b/build-scripts/component-common.gradle @@ -12,15 +12,6 @@ import javax.inject.Inject apply plugin: 'com.android.library' apply plugin: 'kotlin-android' -// Typed Exec subclass used in the moz-central build, where the embedded -// uniffi-bindgen tool and the native megazord library are already built before -// gradle runs. Exposes outputDir as a DirectoryProperty so the generated -// sources can be wired into the variant via addGeneratedSourceDirectory. -abstract class GenerateUniffiBindingsEmbedded extends Exec { - @OutputDirectory - abstract DirectoryProperty getOutputDir() -} - // Typed task used in the standalone app-services build, where the megazord // dynamic library is produced by a separate gradle task and only exists when // generateUniffiBindings runs (not at configuration time). @@ -152,32 +143,18 @@ ext.dependsOnTheMegazord = { // // Make sure to also call dependsOnTheMegazord() ext.configureUniFFIBindgen = { crateName -> - // This will store the uniffi-bindgen generated files for our component - def uniffiOutDir = layout.buildDirectory.dir("generated/uniffi/") - - def generateUniffiBindings - // Call `uniffi-bindgen` to generate the Kotlin bindings if (gradle.hasProperty("mozconfig")) { - // in moz-central we can use an `Exec` task because we can assume the bindgen tool has already been built. - generateUniffiBindings = tasks.register("generateUniffiBindings", GenerateUniffiBindingsEmbedded) { - def libraryPath = "${gradle.mozconfig.topobjdir}/dist/bin/libmegazord.so" - def bindgen = gradle.ext.mozconfig.substs.EMBEDDED_UNIFFI_BINDGEN - - outputDir.set(uniffiOutDir) - - workingDir project.rootDir - commandLine bindgen - args 'generate', "--crate", crateName, '--language', 'kotlin', '--out-dir', outputDir.get().asFile, '--no-format', libraryPath - - // Re-generate when the native megazord library is rebuilt - inputs.files libraryPath - // Re-generate if our uniffi-bindgen tooling changes. - inputs.files bindgen + // For now, generated source files are checked into `firefox-main`. + android { + sourceSets.main.kotlin.srcDirs += "${gradle.mozconfig.topsrcdir}/toolkit/components/uniffi-bindgen-gecko-js/android/components/${project.name}/android/src/main" } } else { + // This will store the uniffi-bindgen generated files for our component + def uniffiOutDir = layout.buildDirectory.dir("generated/uniffi/") + // In app-services we can't use `Exec` because the megazord target isn't built yet; the task // resolves the library path from the megazordNative configuration when it runs. - generateUniffiBindings = tasks.register("generateUniffiBindings", GenerateUniffiBindingsCargo) { + def generateUniffiBindings = tasks.register("generateUniffiBindings", GenerateUniffiBindingsCargo) { // Qualify every property with `it.` because the `crateName` closure parameter shadows the // task's crateName property; a bare `crateName.set(...)` would target the String, not the task. it.megazordNativeFiles.from configurations.getByName("megazordNative") @@ -187,9 +164,9 @@ ext.configureUniFFIBindgen = { crateName -> it.workingDirectory.set(project.rootDir) it.outputDir.set(uniffiOutDir) } - } - androidComponents.onVariants(androidComponents.selector().all()) { variant -> - variant.sources.java.addGeneratedSourceDirectory(generateUniffiBindings) { it.outputDir } + androidComponents.onVariants(androidComponents.selector().all()) { variant -> + variant.sources.java.addGeneratedSourceDirectory(generateUniffiBindings) { it.outputDir } + } } } From 93fb713a0e0b08e24c1f98bbd270de07879a119d Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Tue, 28 Jul 2026 17:05:07 +0200 Subject: [PATCH 29/59] CI: Update the Rust image in use (#7505) Note: rustup will still be used to install the right version as defined in rust-toolchain.toml or the minimum version defined in .circleci/config.yml However newer Rust images come with newer tooling as well (such as a not outdated Python version) --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b788377fbe..116abc3565b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ commands: build-desktop-libs: steps: - run: sudo apt-get update - - run: sudo apt-get install python tcl + - run: sudo apt-get install python3 tcl - run: sudo apt-get install python3-venv - run: sudo apt-get install libclang-dev - run: @@ -196,7 +196,7 @@ executors: # Unfortunately some of our jobs can only run successfully on macos. docker: docker: - - image: cimg/rust:1.53.0 + - image: cimg/rust:1.91.0 macos: macos: xcode: "26.0" From ad2ebff12184485140d5963c2431479c5dfc03fe Mon Sep 17 00:00:00 2001 From: Alex Cottner <148472676+alexcottner@users.noreply.github.com> Date: Tue, 28 Jul 2026 05:16:29 -1000 Subject: [PATCH 30/59] RMST-472 - Make remote-settings v2 routes the default going forward (#7492) * RMST-472 - Make remote-settings v2 routes the default going forward * upating changelog --- CHANGELOG.md | 3 +++ components/remote_settings/src/config.rs | 25 +++++++----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6927bed73e..86ae170192d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - Add `Store::shutdown()`, which closes the database connection early so it happens before Firefox Desktop's late-write shutdown barrier rather than during GC. Operations after shutdown return `DatabaseClosed`. ([Bug 2050036](https://bugzilla.mozilla.org/show_bug.cgi?id=2050036)) +### Remote Settings +- Replacing v1 routes with v2 routes, removing added v2 routes ([#7492](https://github.com/mozilla/application-services/pull/7339)) + # v154.0 (_2026-07-20_) ## ✨ What's Changed ✨ diff --git a/components/remote_settings/src/config.rs b/components/remote_settings/src/config.rs index 57493a8f512..a71d78508fd 100644 --- a/components/remote_settings/src/config.rs +++ b/components/remote_settings/src/config.rs @@ -31,11 +31,8 @@ pub struct RemoteSettingsConfig { #[derive(Debug, Clone, uniffi::Enum)] pub enum RemoteSettingsServer { Prod, - ProdV2, Stage, - StageV2, Dev, - DevV2, Custom { url: String }, } @@ -77,15 +74,10 @@ impl RemoteSettingsServer { fn raw_url(&self) -> &str { match self { - // v1 routes, current default - Self::Prod => "https://firefox.settings.services.mozilla.com/v1", - Self::Stage => "https://firefox.settings.services.allizom.org/v1", - Self::Dev => "https://remote-settings-dev.allizom.org/v1", - - // v2 routes, optional for now but will be default later - Self::ProdV2 => "https://firefox.settings.services.mozilla.com/v2", - Self::StageV2 => "https://firefox.settings.services.allizom.org/v2", - Self::DevV2 => "https://remote-settings-dev.allizom.org/v2", + // v2 routes, current default + Self::Prod => "https://firefox.settings.services.mozilla.com/v2", + Self::Stage => "https://firefox.settings.services.allizom.org/v2", + Self::Dev => "https://remote-settings-dev.allizom.org/v2", // custom, not currently implemented in android or iOS Self::Custom { url } => url, @@ -98,12 +90,9 @@ impl RemoteSettingsServer { /// inside the crate. pub fn get_url(&self) -> Result { Ok(match self { - Self::Prod => Url::parse("https://firefox.settings.services.mozilla.com/v1")?, - Self::Stage => Url::parse("https://firefox.settings.services.allizom.org/v1")?, - Self::Dev => Url::parse("https://remote-settings-dev.allizom.org/v1")?, - Self::ProdV2 => Url::parse("https://firefox.settings.services.mozilla.com/v2")?, - Self::StageV2 => Url::parse("https://firefox.settings.services.allizom.org/v2")?, - Self::DevV2 => Url::parse("https://remote-settings-dev.allizom.org/v2")?, + Self::Prod => Url::parse("https://firefox.settings.services.mozilla.com/v2")?, + Self::Stage => Url::parse("https://firefox.settings.services.allizom.org/v2")?, + Self::Dev => Url::parse("https://remote-settings-dev.allizom.org/v2")?, Self::Custom { url } => { let mut url = Url::parse(url)?; // Custom URLs are weird and require a couple tricks for backwards compatibility. From c1d4f849ec861e4f0b1e66f83357eadd83fc988c Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Tue, 28 Jul 2026 22:16:55 -0700 Subject: [PATCH 31/59] Only build `glean-sym` on Android and iOS (#7507) --- components/places/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/places/Cargo.toml b/components/places/Cargo.toml index 0f84c2f4ea6..76341ae2da1 100644 --- a/components/places/Cargo.toml +++ b/components/places/Cargo.toml @@ -34,6 +34,9 @@ sync-guid = { path = "../support/guid", features = ["rusqlite_support", "random" thiserror = "2" anyhow = "1.0" uniffi = { version = "0.31" } + +# glean-sym is only used on Android and iOS. +[target.'cfg(any(target_os = "android", target_os = "ios"))'.dependencies] glean-sym = { git = "https://github.com/mozilla/glean", tag = "v68.0.0", optional = true } [dev-dependencies] From bfa48d754545546b4bb016337d4878a65473900f Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Tue, 28 Jul 2026 22:38:27 -0700 Subject: [PATCH 32/59] Use android-components publish.gradle for app-services components (#7508) * Only build `glean-sym` on Android and iOS * Use android-components publish.gradle for app-services components --- components/ads-client/android/build.gradle | 4 ++-- components/autofill/android/build.gradle | 7 +++---- components/crashtest/android/build.gradle | 4 ++-- components/fxa-client/android/build.gradle | 4 ++-- .../init_rust_components/android/build.gradle | 4 ++-- components/logins/android/build.gradle | 4 ++-- components/merino/android/build.gradle | 4 ++-- components/nimbus/android/build.gradle | 4 ++-- components/places/android/build.gradle | 4 ++-- components/push/android/build.gradle | 4 ++-- components/relay/android/build.gradle | 4 ++-- components/remote_settings/android/build.gradle | 4 ++-- components/search/android/build.gradle | 4 ++-- components/suggest/android/build.gradle | 4 ++-- components/support/error/android/build.gradle | 4 ++-- .../rust-log-forwarder/android/build.gradle | 4 ++-- components/support/tracing/android/build.gradle | 4 ++-- components/sync15/android/build.gradle | 4 ++-- components/sync_manager/android/build.gradle | 4 ++-- components/tabs/android/build.gradle | 4 ++-- components/viaduct/android/build.gradle | 4 ++-- megazords/full/android/build.gradle | 4 ++-- publish.gradle | 8 ++++---- settings.gradle | 17 ++++++++++++++++- tools/start-bindings/templates/build.gradle | 4 ++-- 25 files changed, 67 insertions(+), 53 deletions(-) diff --git a/components/ads-client/android/build.gradle b/components/ads-client/android/build.gradle index 52e4c5da36d..2d2db933137 100644 --- a/components/ads-client/android/build.gradle +++ b/components/ads-client/android/build.gradle @@ -22,7 +22,7 @@ plugins { } apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" ext { gleanNamespace = "mozilla.telemetry.glean" @@ -45,4 +45,4 @@ dependencies { ext.configureUniFFIBindgen("ads_client") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/autofill/android/build.gradle b/components/autofill/android/build.gradle index 63e3c6c0869..64d1878da31 100644 --- a/components/autofill/android/build.gradle +++ b/components/autofill/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.autofill' @@ -9,12 +9,11 @@ dependencies { // Part of the public API. api project(':sync15') - testImplementation project(":syncmanager") - testImplementation libs.androidx.test.core testImplementation libs.androidx.work.testing + testImplementation project(":syncmanager") } ext.configureUniFFIBindgen("autofill") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/crashtest/android/build.gradle b/components/crashtest/android/build.gradle index 5a9b590442c..66b10120aec 100644 --- a/components/crashtest/android/build.gradle +++ b/components/crashtest/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.crashtest' @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("crashtest") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/fxa-client/android/build.gradle b/components/fxa-client/android/build.gradle index 88709f6752f..d387acf7fe4 100644 --- a/components/fxa-client/android/build.gradle +++ b/components/fxa-client/android/build.gradle @@ -23,7 +23,7 @@ plugins { } apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" ext { gleanNamespace = "mozilla.telemetry.glean" @@ -47,4 +47,4 @@ dependencies { ext.configureUniFFIBindgen("fxa_client") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/init_rust_components/android/build.gradle b/components/init_rust_components/android/build.gradle index c748329683b..2eedc536298 100644 --- a/components/init_rust_components/android/build.gradle +++ b/components/init_rust_components/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.init_rust_components' @@ -15,4 +15,4 @@ dependencies { ext.configureUniFFIBindgen("init_rust_components") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/logins/android/build.gradle b/components/logins/android/build.gradle index 748c8fc023f..038b1371d3a 100644 --- a/components/logins/android/build.gradle +++ b/components/logins/android/build.gradle @@ -23,7 +23,7 @@ plugins { } apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" // Needs to happen before `dependencies` in order for the variables // exposed by the plugin to be available for this project. @@ -58,4 +58,4 @@ dependencies { ext.configureUniFFIBindgen("logins") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/merino/android/build.gradle b/components/merino/android/build.gradle index c2a461a1fd6..03f7f70fc39 100644 --- a/components/merino/android/build.gradle +++ b/components/merino/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.merino' @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("merino") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/nimbus/android/build.gradle b/components/nimbus/android/build.gradle index 915abb43f56..5b9f61cb0a7 100644 --- a/components/nimbus/android/build.gradle +++ b/components/nimbus/android/build.gradle @@ -23,7 +23,7 @@ plugins { } apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" ext { gleanNamespace = "mozilla.telemetry.glean" @@ -54,4 +54,4 @@ dependencies { ext.configureUniFFIBindgen("nimbus") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/places/android/build.gradle b/components/places/android/build.gradle index be15dd740fa..fdb42062a61 100644 --- a/components/places/android/build.gradle +++ b/components/places/android/build.gradle @@ -23,7 +23,7 @@ plugins { } apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" ext { gleanNamespace = "mozilla.telemetry.glean" @@ -52,4 +52,4 @@ dependencies { ext.configureUniFFIBindgen("places") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/push/android/build.gradle b/components/push/android/build.gradle index ab376beaec2..78aa011163d 100644 --- a/components/push/android/build.gradle +++ b/components/push/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.push' @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("push") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/relay/android/build.gradle b/components/relay/android/build.gradle index 96e397a0e76..cafd3510c54 100644 --- a/components/relay/android/build.gradle +++ b/components/relay/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.relay' @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("relay") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/remote_settings/android/build.gradle b/components/remote_settings/android/build.gradle index b837884e0a1..c7c458a5962 100644 --- a/components/remote_settings/android/build.gradle +++ b/components/remote_settings/android/build.gradle @@ -22,7 +22,7 @@ plugins { } apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" ext { gleanNamespace = "mozilla.telemetry.glean" @@ -45,7 +45,7 @@ dependencies { ext.configureUniFFIBindgen("remote_settings") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) dependencies { if (gradle.hasProperty("mozconfig")) { diff --git a/components/search/android/build.gradle b/components/search/android/build.gradle index 9527dcb3d44..a54d4ae3176 100644 --- a/components/search/android/build.gradle +++ b/components/search/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.search' @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("search") ext.dependsOnTheMegazord() -ext.configurePublish() \ No newline at end of file +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) \ No newline at end of file diff --git a/components/suggest/android/build.gradle b/components/suggest/android/build.gradle index 284f47051fc..e8fb54d9f6e 100644 --- a/components/suggest/android/build.gradle +++ b/components/suggest/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.suggest' @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("suggest") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/support/error/android/build.gradle b/components/support/error/android/build.gradle index 9d97dc2c233..711b9fc9086 100644 --- a/components/support/error/android/build.gradle +++ b/components/support/error/android/build.gradle @@ -29,7 +29,7 @@ plugins { apply plugin: 'kotlinx-serialization' apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" // Needs to happen before `dependencies` in order for the variables // exposed by the plugin to be available for this project. @@ -54,4 +54,4 @@ android { ext.configureUniFFIBindgen("error_support") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/support/rust-log-forwarder/android/build.gradle b/components/support/rust-log-forwarder/android/build.gradle index cbedd834cf5..f72401b60c0 100644 --- a/components/support/rust-log-forwarder/android/build.gradle +++ b/components/support/rust-log-forwarder/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.rust_log_forwarder' @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("rust_log_forwarder") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/support/tracing/android/build.gradle b/components/support/tracing/android/build.gradle index 924a0706580..1eadd69e0c2 100644 --- a/components/support/tracing/android/build.gradle +++ b/components/support/tracing/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.tracing' @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("tracing_support") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/sync15/android/build.gradle b/components/sync15/android/build.gradle index 8065fb387e1..fa742ae294e 100644 --- a/components/sync15/android/build.gradle +++ b/components/sync15/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.sync15' @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("sync15") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/sync_manager/android/build.gradle b/components/sync_manager/android/build.gradle index 48d962f3862..190f72f2419 100644 --- a/components/sync_manager/android/build.gradle +++ b/components/sync_manager/android/build.gradle @@ -23,7 +23,7 @@ plugins { } apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" // Needs to happen before `dependencies` in order for the variables // exposed by the plugin to be available for this project. @@ -55,4 +55,4 @@ dependencies { ext.configureUniFFIBindgen("sync_manager") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/tabs/android/build.gradle b/components/tabs/android/build.gradle index 21b109649f8..825cfd6eebf 100644 --- a/components/tabs/android/build.gradle +++ b/components/tabs/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.remotetabs' @@ -14,4 +14,4 @@ dependencies { ext.configureUniFFIBindgen("tabs") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/components/viaduct/android/build.gradle b/components/viaduct/android/build.gradle index ae2bb6edaaa..b5c6b8a072a 100644 --- a/components/viaduct/android/build.gradle +++ b/components/viaduct/android/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.viaduct' @@ -16,4 +16,4 @@ dependencies { ext.configureUniFFIBindgen("viaduct") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) diff --git a/megazords/full/android/build.gradle b/megazords/full/android/build.gradle index cfc56578741..c173bdc7d57 100644 --- a/megazords/full/android/build.gradle +++ b/megazords/full/android/build.gradle @@ -144,8 +144,8 @@ if (!gradle.hasProperty("mozconfig")) { } } -apply from: "$appServicesRootDir/publish.gradle" -ext.configurePublish() +apply from: "$publishDir/publish.gradle" +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) if (!gradle.hasProperty("mozconfig")) { afterEvaluate { diff --git a/publish.gradle b/publish.gradle index 7304154fec4..a106e01db93 100644 --- a/publish.gradle +++ b/publish.gradle @@ -11,10 +11,10 @@ def libProjectName = properties.libProjectName def libUrl = properties.libUrl def libVcsUrl = properties.libVcsUrl -ext.configurePublish = { - def theGroupId = rootProject.config.componentsGroupId - def theArtifactId = project.ext.artifactId - def theDescription = project.ext.description +ext.configurePublish = { groupId = null, artifactId = null, description = null -> + def theGroupId = groupId ?: rootProject.config.componentsGroupId + def theArtifactId = artifactId ?: project.ext.artifactId + def theDescription = description ?: project.ext.description // This is a little cludgey, but it seems unlikely to cause a problem, and // we are already doing it inside taskcluster. diff --git a/settings.gradle b/settings.gradle index 5fedb025003..7809987c011 100644 --- a/settings.gradle +++ b/settings.gradle @@ -76,9 +76,11 @@ def setupProject(name, projectProps, appServicesRootDir) { // Expose the rest of the project properties, mostly for validation reasons. project.ext.configProps = projectProps project.ext.appServicesRootDir = appServicesRootDir + project.ext.publishDir = appServicesRootDir if (gradle.hasProperty("mozconfig")) { - project.buildDir = "${gradle.mozconfig.topobjdir}/gradle/build/app-services/android/$name" + project.buildDir = "${gradle.mozconfig.topobjdir}/gradle/build/application-services/android/$name" + project.ext.publishDir = "${gradle.mozconfig.topsrcdir}/mobile/android/android-components" } } } @@ -90,6 +92,19 @@ buildconfig.projects.each { project -> setupProject(project.key, project.value, appServicesRootDir) } +if (gradle.root.hasProperty("mozconfig")) { + // The mozilla-central android-components/fenix "second pass" build resolves + // these components as Maven AARs from target.maven.zip, so the geckoview + // archive build must publish every app-services component, not just the + // megazord. Aggregate their publish tasks behind a single task. + def appServicesPublishTasks = buildconfig.projects.keySet().collect { ":${it}:publish" } + gradle.rootProject { rootProject -> + rootProject.tasks.register("publishAppServicesAndroid") { task -> + task.dependsOn(appServicesPublishTasks) + } + } +} + Properties localProperties = new Properties(); if (file('local.properties').canRead()) { localProperties.load(file('local.properties').newDataInputStream()) diff --git a/tools/start-bindings/templates/build.gradle b/tools/start-bindings/templates/build.gradle index 2d0e0bfbc63..6dd2146c0ea 100644 --- a/tools/start-bindings/templates/build.gradle +++ b/tools/start-bindings/templates/build.gradle @@ -1,5 +1,5 @@ apply from: "$appServicesRootDir/build-scripts/component-common.gradle" -apply from: "$appServicesRootDir/publish.gradle" +apply from: "$publishDir/publish.gradle" android { namespace 'org.mozilla.appservices.{{ kotlin_module_name }}' @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("{{ crate_name }}") ext.dependsOnTheMegazord() -ext.configurePublish() +ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) From be6bc6c11b4dc29649f9cf5d76fd6d44827da1a1 Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Tue, 28 Jul 2026 23:51:41 -0700 Subject: [PATCH 33/59] Package `libmegazord.so` into full-megazord AAR with NSS dependencies and `libsForTests` publication (#7509) * Only build `glean-sym` on Android and iOS * Use android-components publish.gradle for app-services components * Package `libmegazord.so` into full-megazord AAR with NSS dependencies and `libsForTests` publication --- megazords/full/android/build.gradle | 187 +++++++++++++++++++++------- 1 file changed, 139 insertions(+), 48 deletions(-) diff --git a/megazords/full/android/build.gradle b/megazords/full/android/build.gradle index c173bdc7d57..34c5489d92c 100644 --- a/megazords/full/android/build.gradle +++ b/megazords/full/android/build.gradle @@ -22,7 +22,7 @@ android { release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' - consumerProguardFiles "$rootDir/proguard-rules-consumer-jna.pro" + consumerProguardFiles "$appServicesRootDir/proguard-rules-consumer-jna.pro" } } @@ -72,32 +72,116 @@ dependencies { } } -if (!gradle.hasProperty("mozconfig")) { - // Extract JNI dispatch libraries from the JAR into a directory, so that we can then package them - // into our own megazord-desktopLibraries JAR. - def extractLibJniDispatch = tasks.register("extractLibJniDispatch", Copy) { - from zipTree(configurations.jna.singleFile).matching { - include "**/libjnidispatch.*" +abstract class GenerateJniLibsTask extends DefaultTask { + @Internal + abstract DirectoryProperty getNssLibsSourceDir() + + @OutputDirectory + abstract DirectoryProperty getOutputDir() + + @javax.inject.Inject + abstract FileSystemOperations getFs() + + @TaskAction + void generate() { + // Copy NSS shared libs that libmegazord.so depends on (NEEDED) into + // the megazord's jniLibs staging area. Standalone consumers like + // samples-glean don't include GeckoView and need these in the megazord AAR. + def nssLibs = ["libnss3.so", "libfreebl3.so", "libsoftokn3.so", "libmozglue.so"] + def srcBase = nssLibsSourceDir.get().asFile + if (!srcBase.exists()) { + // Some builds do not stage GeckoView, so its native libraries + // are not present. There is nothing to copy in that case. + return + } + fs.copy { + from srcBase + include nssLibs.collect { "**/${it}" } + into outputDir.get().asFile + } + } +} + +if (gradle.hasProperty("mozconfig")) { + def isFatAar = gradle.mozconfig.substs.MOZ_ANDROID_FAT_AAR_ARCHITECTURES + def topobjdir = gradle.mozconfig.topobjdir + def generateJniLibs = tasks.register("generateJniLibs", GenerateJniLibsTask) { task -> + if (isFatAar) { + // Must match the appservices output prefix in `python/mozbuild/mozbuild/action/fat_aar.py`. + task.outputDir.set(file("${topobjdir}/dist/fat-aar/output/appservices/jni")) + task.nssLibsSourceDir.set(file("${topobjdir}/dist/fat-aar/output/geckoview/jni")) + } else { + // Must match the destdir in `mobile/android/installer/package-manifest.in`. + task.outputDir.set(file("${topobjdir}/dist/geckoview/appservices/lib")) + task.nssLibsSourceDir.set(file("${topobjdir}/dist/geckoview/lib")) + } + + // Depending directly on a task declared in the root Gradle project + // seems to not be sufficiently lazy: `tasks.named(...)` is invoked + // during configuration of this project, which is too early and fails. + // + // But this is sufficiently lazy to be present when needed during + // configuration of this project, while still depending on a shared task + // in the root Gradle project. + if (findProject(":geckoview") != null) { + task.dependsOn(":machStagePackage") } - into layout.buildDirectory.dir("libjnidispatch").get() } - def packageLibsForTest = tasks.register("packageLibsForTest", Jar) { - archiveBaseName = "full-megazord-libsForTests" + androidComponents { + onVariants(selector().all()) { variant -> + variant.sources.jniLibs?.addGeneratedSourceDirectory( + generateJniLibs, + { task -> task.outputDir } + ) + } + } +} + +// Extract JNI dispatch libraries from the JAR into a directory, so that we can then package them +// into our own megazord-desktopLibraries JAR. +// Resolve the JNA JAR at configuration time so `--download-all-gradle-dependencies` +// fetches it into the offline repository; a lazy resolution inside the task action +// isn't reached during the dry-run dependency fetch. +def jnaLibJniDispatch = zipTree(configurations.jna.singleFile).matching { + include "**/libjnidispatch.*" +} + +def extractLibJniDispatch = tasks.register("extractLibJniDispatch", Copy) { + from jnaLibJniDispatch + into layout.buildDirectory.dir("libjnidispatch").get() +} - from extractLibJniDispatch +def packageLibsForTest = tasks.register("packageLibsForTest", Jar) { + archiveBaseName = "full-megazord-libsForTests" + + from extractLibJniDispatch + if (!gradle.hasProperty("mozconfig")) { from layout.buildDirectory.dir("rustJniLibs/desktop") dependsOn tasks["cargoBuild${rootProject.ext.nativeRustTarget.capitalize()}"] } + if (gradle.hasProperty("mozconfig")) { + if (gradle.mozconfig.substs.MOZ_ANDROID_FAT_AAR_DESKTOP_ARCHITECTURES) { + from "${gradle.mozconfig.topobjdir}/dist/fat-aar/output/desktop/resources" + } else { + from "${gradle.mozconfig.topobjdir}/dist/host/bin/appservices/resources" + } + } +} + +artifacts { + // Connect task output to configurations + libsForTests(packageLibsForTest) +} + +if (!gradle.hasProperty("mozconfig")) { def copyMegazordNative = tasks.register("copyMegazordNative", Copy) { from layout.buildDirectory.dir("rustJniLibs/desktop") into layout.buildDirectory.dir("megazordNative") } artifacts { - // Connect task output to configurations - libsForTests(packageLibsForTest) megazordNative(copyMegazordNative) } @@ -147,51 +231,58 @@ if (!gradle.hasProperty("mozconfig")) { apply from: "$publishDir/publish.gradle" ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) -if (!gradle.hasProperty("mozconfig")) { - afterEvaluate { - publishing { - publications { - // Publish a second package named `full-megazord-libsForTests` to Maven with the - // `libsForTests` output. This contains the same content as our `libsForTests` - // configuration. Publishing it allows the android-components code to depend on it. - libsForTests(MavenPublication) { - artifact tasks['packageLibsForTest'] +afterEvaluate { + def libUrl="https://github.com/mozilla/application-services" + def libVcsUrl="https://github.com/mozilla/application-services.git" + + def libLicense="MPL-2.0" + def libLicenseUrl="https://www.mozilla.org/en-US/MPL/2.0/" + + publishing { + publications { + // Publish a second package named `full-megazord-libsForTests` to Maven with the + // `libsForTests` output. This contains the same content as our `libsForTests` + // configuration. Publishing it allows the android-components code to depend on it. + libsForTests(MavenPublication) { + artifact tasks['packageLibsForTest'] + if (!gradle.hasProperty("mozconfig")) { artifact file("${projectDir}/../DEPENDENCIES.md"), { extension "LICENSES.md" } - pom { - groupId = rootProject.config.componentsGroupId - artifactId = "${project.ext.artifactId}-libsForTests" - description = project.ext.description - // For mavenLocal publishing workflow, increment the version number every publish. - version = rootProject.config.componentsVersion + (rootProject.hasProperty('local') ? '-' + rootProject.property('local') : '') - packaging = "jar" - - licenses { - license { - name = libLicense - url = libLicenseUrl - } - } + } - developers { - developer { - name = 'Sync Team' - email = 'sync-team@mozilla.com' - } + pom { + groupId = "org.mozilla.appservices" + artifactId = "${project.ext.artifactId}-libsForTests" + description = project.ext.description + // For mavenLocal publishing workflow, increment the version number every publish. + version = rootProject.config.componentsVersion + (rootProject.hasProperty('local') ? '-' + rootProject.property('local') : '') + packaging = "jar" + + licenses { + license { + name = libLicense + url = libLicenseUrl } + } - scm { - connection = libVcsUrl - developerConnection = libVcsUrl - url = libUrl + developers { + developer { + name = 'Sync Team' + email = 'sync-team@mozilla.com' } } - // This is never the publication we want to use when publishing a - // parent project with us as a child `project()` dependency. - alias = true + scm { + connection = libVcsUrl + developerConnection = libVcsUrl + url = libUrl + } } + + // This is never the publication we want to use when publishing a + // parent project with us as a child `project()` dependency. + alias = true } } } From f6cf01c840b3ccda7d24424f9bd6ff1a846615d8 Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Wed, 29 Jul 2026 01:25:37 -0700 Subject: [PATCH 34/59] Remove unused `error_support` macro re-exports in application-services components (#7510) * Only build `glean-sym` on Android and iOS * Use android-components publish.gradle for app-services components * Package `libmegazord.so` into full-megazord AAR with NSS dependencies and `libsForTests` publication * Remove unused `error_support` macro re-exports in application-services components --- components/merino/src/curated_recommendations/error.rs | 2 +- components/merino/src/suggest/error.rs | 1 - components/merino/src/worldcup/error.rs | 1 - components/push/src/error.rs | 2 +- components/relay/src/error.rs | 1 - 5 files changed, 2 insertions(+), 5 deletions(-) diff --git a/components/merino/src/curated_recommendations/error.rs b/components/merino/src/curated_recommendations/error.rs index eff201ef992..b4821fc9a67 100644 --- a/components/merino/src/curated_recommendations/error.rs +++ b/components/merino/src/curated_recommendations/error.rs @@ -4,7 +4,7 @@ use error_support::{ErrorHandling, GetErrorHandling}; // Re-export logging helpers. -pub use error_support::{error, trace}; +pub use error_support::trace; /// Internal convenience wrapper for `std::Result`. pub type Result = std::result::Result; diff --git a/components/merino/src/suggest/error.rs b/components/merino/src/suggest/error.rs index a4135b3493e..e4904dfb16c 100644 --- a/components/merino/src/suggest/error.rs +++ b/components/merino/src/suggest/error.rs @@ -2,7 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -pub use error_support::error; use error_support::{ErrorHandling, GetErrorHandling}; pub type Result = std::result::Result; diff --git a/components/merino/src/worldcup/error.rs b/components/merino/src/worldcup/error.rs index 2f5a420e60b..d1c36cb362e 100644 --- a/components/merino/src/worldcup/error.rs +++ b/components/merino/src/worldcup/error.rs @@ -2,7 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -pub use error_support::error; use error_support::{ErrorHandling, GetErrorHandling}; pub type Result = std::result::Result; diff --git a/components/push/src/error.rs b/components/push/src/error.rs index f642093cfbe..670dff7193d 100644 --- a/components/push/src/error.rs +++ b/components/push/src/error.rs @@ -4,7 +4,7 @@ use error_support::{ErrorHandling, GetErrorHandling}; // reexport logging helpers. -pub use error_support::{debug, error, info, warn}; +pub use error_support::{debug, info, warn}; pub type Result = std::result::Result; diff --git a/components/relay/src/error.rs b/components/relay/src/error.rs index 2a6f6ddbd46..d8f2536a10f 100644 --- a/components/relay/src/error.rs +++ b/components/relay/src/error.rs @@ -2,7 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -pub use error_support::error; use error_support::{ErrorHandling, GetErrorHandling}; use remote_settings::RemoteSettingsError; From 44c99012501dbe0441c0c2eb6726701cdd24ae79 Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Wed, 29 Jul 2026 12:45:22 -0700 Subject: [PATCH 35/59] Skip the `kotlin-android` plugin under AGP 9 for mozilla-central builds (#7511) * Only build `glean-sym` on Android and iOS * Use android-components publish.gradle for app-services components * Package `libmegazord.so` into full-megazord AAR with NSS dependencies and `libsForTests` publication * Remove unused `error_support` macro re-exports in application-services components * Skip the `kotlin-android` plugin under AGP 9 for mozilla-central builds --- build-scripts/component-common.gradle | 8 +++++++- megazords/full/android/build.gradle | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/build-scripts/component-common.gradle b/build-scripts/component-common.gradle index 98957d2ebe5..2d7cb26e47d 100644 --- a/build-scripts/component-common.gradle +++ b/build-scripts/component-common.gradle @@ -10,7 +10,13 @@ import javax.inject.Inject apply plugin: 'com.android.library' -apply plugin: 'kotlin-android' + +// When built as part of mozilla-central (mozconfig present), AGP 9 provides +// built-in Kotlin support and rejects this plugin; standalone builds still use +// an older AGP that requires it. +if (!gradle.hasProperty("mozconfig")) { + apply plugin: 'kotlin-android' +} // Typed task used in the standalone app-services build, where the megazord // dynamic library is produced by a separate gradle task and only exists when diff --git a/megazords/full/android/build.gradle b/megazords/full/android/build.gradle index 34c5489d92c..dcb8e838164 100644 --- a/megazords/full/android/build.gradle +++ b/megazords/full/android/build.gradle @@ -1,7 +1,7 @@ apply plugin: 'com.android.library' -apply plugin: 'kotlin-android' if (!gradle.hasProperty("mozconfig")) { + apply plugin: 'kotlin-android' apply plugin: 'org.mozilla.rust-android-gradle.rust-android' } From 22c7d7241ea3371006d7976c56886636934b9aba Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Wed, 29 Jul 2026 15:04:08 -0700 Subject: [PATCH 36/59] Normalize app-services groupId for mozilla-central builds and fix mozconfig check for nested Gradle (#7506) --- settings.gradle | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/settings.gradle b/settings.gradle index 7809987c011..a5f58825514 100644 --- a/settings.gradle +++ b/settings.gradle @@ -136,7 +136,10 @@ def calcVersion(buildconfig) { } def calcGroupId(buildconfig) { - if (gradle.rootProject.hasProperty("nightlyVersion")) { + if (gradle.root.hasProperty("mozconfig")) { + // We are in m-c - always build `org.mozilla.appservices`. + return buildconfig.groupId + } else if (gradle.rootProject.hasProperty("nightlyVersion")) { return buildconfig.groupId + ".nightly" } else { return buildconfig.groupId @@ -181,7 +184,7 @@ class Config { } gradle.projectsLoaded { -> - if (gradle.hasProperty("mozconfig")) { + if (gradle.root.hasProperty("mozconfig")) { gradle.rootProject.tasks.register("generateUniffiBindings") } From 26a5e3ac9149e7cc66087aabb5c7787e71291b05 Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Wed, 29 Jul 2026 21:59:14 -0700 Subject: [PATCH 37/59] Reference `UNIFFI_META_*` symbols in the megazord stub so the linker keeps them (#7498) --- megazords/fenix-dylib/megazord_stub.c | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/megazords/fenix-dylib/megazord_stub.c b/megazords/fenix-dylib/megazord_stub.c index b5111efdea3..2d59c462746 100644 --- a/megazords/fenix-dylib/megazord_stub.c +++ b/megazords/fenix-dylib/megazord_stub.c @@ -36,6 +36,42 @@ extern int MOZ_EXPORT ffi_tabs_uniffi_contract_version(); // the missing symbols, and they all happily come in. // W T A F. extern int MOZ_EXPORT uniffi_search_checksum_constructor_searchengineselector_new(); +// Same deal for these, but their .o has no checksum fn, so we name the metadata directly. +extern int MOZ_EXPORT UNIFFI_META_ADS_CLIENT_ERROR_MOZADSCLIENTAPIERROR(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_CONSTRUCTOR_CURATEDRECOMMENDATIONSCLIENT_NEW(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_CONSTRUCTOR_SUGGESTCLIENT_NEW(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_CONSTRUCTOR_WORLDCUPCLIENT_NEW(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_ENUM_CURATEDRECOMMENDATIONLOCALE(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_ERROR_CURATEDRECOMMENDATIONSAPIERROR(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_ERROR_MERINOSUGGESTAPIERROR(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_ERROR_MERINOWORLDCUPAPIERROR(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_FUNC_ALL_CURATED_RECOMMENDATION_LOCALES(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_FUNC_CURATED_RECOMMENDATION_LOCALE_FROM_STRING(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_INTERFACE_CURATEDRECOMMENDATIONSCLIENT(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_INTERFACE_SUGGESTCLIENT(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_INTERFACE_WORLDCUPCLIENT(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_METHOD_CURATEDRECOMMENDATIONSCLIENT_GET_CURATED_RECOMMENDATIONS(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_METHOD_SUGGESTCLIENT_GET_SUGGESTIONS(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_METHOD_WORLDCUPCLIENT_GET_LIVE(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_METHOD_WORLDCUPCLIENT_GET_MATCHES(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_METHOD_WORLDCUPCLIENT_GET_TEAMS(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_CURATEDRECOMMENDATIONSCONFIG(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_CURATEDRECOMMENDATIONSREQUEST(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_CURATEDRECOMMENDATIONSRESPONSE(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_FEEDSECTION(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_INTERESTPICKER(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_INTERESTPICKERSECTION(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_LAYOUT(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_RECOMMENDATIONDATAITEM(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_RESPONSIVELAYOUT(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_SECTIONSETTINGS(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_SUGGESTCONFIG(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_SUGGESTOPTIONS(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_TILE(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_WORLDCUPCONFIG(); +extern int MOZ_EXPORT UNIFFI_META_MERINO_RECORD_WORLDCUPOPTIONS(); +extern int MOZ_EXPORT UNIFFI_META_NAMESPACE_ERRORSUPPORT(); +extern int MOZ_EXPORT UNIFFI_META_SYNC15_ENUM_DEVICETYPE(); void _local_megazord_dummy_symbol() { ffi_ads_client_uniffi_contract_version(); @@ -57,4 +93,39 @@ void _local_megazord_dummy_symbol() { ffi_sync_manager_uniffi_contract_version(); ffi_tabs_uniffi_contract_version(); uniffi_search_checksum_constructor_searchengineselector_new(); + UNIFFI_META_ADS_CLIENT_ERROR_MOZADSCLIENTAPIERROR(); + UNIFFI_META_MERINO_CONSTRUCTOR_CURATEDRECOMMENDATIONSCLIENT_NEW(); + UNIFFI_META_MERINO_CONSTRUCTOR_SUGGESTCLIENT_NEW(); + UNIFFI_META_MERINO_CONSTRUCTOR_WORLDCUPCLIENT_NEW(); + UNIFFI_META_MERINO_ENUM_CURATEDRECOMMENDATIONLOCALE(); + UNIFFI_META_MERINO_ERROR_CURATEDRECOMMENDATIONSAPIERROR(); + UNIFFI_META_MERINO_ERROR_MERINOSUGGESTAPIERROR(); + UNIFFI_META_MERINO_ERROR_MERINOWORLDCUPAPIERROR(); + UNIFFI_META_MERINO_FUNC_ALL_CURATED_RECOMMENDATION_LOCALES(); + UNIFFI_META_MERINO_FUNC_CURATED_RECOMMENDATION_LOCALE_FROM_STRING(); + UNIFFI_META_MERINO_INTERFACE_CURATEDRECOMMENDATIONSCLIENT(); + UNIFFI_META_MERINO_INTERFACE_SUGGESTCLIENT(); + UNIFFI_META_MERINO_INTERFACE_WORLDCUPCLIENT(); + UNIFFI_META_MERINO_METHOD_CURATEDRECOMMENDATIONSCLIENT_GET_CURATED_RECOMMENDATIONS(); + UNIFFI_META_MERINO_METHOD_SUGGESTCLIENT_GET_SUGGESTIONS(); + UNIFFI_META_MERINO_METHOD_WORLDCUPCLIENT_GET_LIVE(); + UNIFFI_META_MERINO_METHOD_WORLDCUPCLIENT_GET_MATCHES(); + UNIFFI_META_MERINO_METHOD_WORLDCUPCLIENT_GET_TEAMS(); + UNIFFI_META_MERINO_RECORD_CURATEDRECOMMENDATIONSCONFIG(); + UNIFFI_META_MERINO_RECORD_CURATEDRECOMMENDATIONSREQUEST(); + UNIFFI_META_MERINO_RECORD_CURATEDRECOMMENDATIONSRESPONSE(); + UNIFFI_META_MERINO_RECORD_FEEDSECTION(); + UNIFFI_META_MERINO_RECORD_INTERESTPICKER(); + UNIFFI_META_MERINO_RECORD_INTERESTPICKERSECTION(); + UNIFFI_META_MERINO_RECORD_LAYOUT(); + UNIFFI_META_MERINO_RECORD_RECOMMENDATIONDATAITEM(); + UNIFFI_META_MERINO_RECORD_RESPONSIVELAYOUT(); + UNIFFI_META_MERINO_RECORD_SECTIONSETTINGS(); + UNIFFI_META_MERINO_RECORD_SUGGESTCONFIG(); + UNIFFI_META_MERINO_RECORD_SUGGESTOPTIONS(); + UNIFFI_META_MERINO_RECORD_TILE(); + UNIFFI_META_MERINO_RECORD_WORLDCUPCONFIG(); + UNIFFI_META_MERINO_RECORD_WORLDCUPOPTIONS(); + UNIFFI_META_NAMESPACE_ERRORSUPPORT(); + UNIFFI_META_SYNC15_ENUM_DEVICETYPE(); } From dfd981f9d26f67ae9d9b39f954c275f6408fb464 Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Thu, 30 Jul 2026 04:14:59 -0400 Subject: [PATCH 38/59] Implement SyncEngine::wipe for the logins engine (#7515) --- components/logins/src/sync/engine.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/logins/src/sync/engine.rs b/components/logins/src/sync/engine.rs index 2e807fca864..e4e71cf8402 100644 --- a/components/logins/src/sync/engine.rs +++ b/components/logins/src/sync/engine.rs @@ -492,6 +492,10 @@ impl SyncEngine for LoginsSyncEngine { self.do_reset(assoc)?; Ok(()) } + + fn wipe(&self) -> anyhow::Result<()> { + self.store.wipe_local().map_err(Into::into) + } } #[cfg(not(feature = "keydb"))] From c2780a39b8b8ba3cff9a38b2bd0b162b9bdb1963 Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Thu, 30 Jul 2026 07:54:15 -0400 Subject: [PATCH 39/59] SyncEngine::wipe no longer has a default panicing implementation (#7516) --- components/sync15/src/engine/sync_engine.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/components/sync15/src/engine/sync_engine.rs b/components/sync15/src/engine/sync_engine.rs index 83b50b866e1..d4f6283030c 100644 --- a/components/sync15/src/engine/sync_engine.rs +++ b/components/sync15/src/engine/sync_engine.rs @@ -223,15 +223,11 @@ pub trait SyncEngine { /// `assoc` defines how this store is to be associated with sync. fn reset(&self, assoc: &EngineSyncAssociation) -> Result<()>; - /// Wipes the engine's data - /// This is typically triggered by a client command, which at the time of writing, only - /// supported wiping bookmarks. - /// - /// This panics if triggered on a sync engine that does not explicitly implement wipe, because - /// that implies a confustion that shouldn't occur. - fn wipe(&self) -> Result<()> { - unimplemented!("The engine does not implement wipe, no wipe should be requested") - } + /// Wipes the engine's local data. + /// Triggered by a client command (only bookmarks at time of writing), + /// or to wipe local data when disconnecting (currently only on desktop + /// via a bridged-engine). + fn wipe(&self) -> Result<()>; } #[cfg(test)] From 213b7600a4930c8e8f7954a3d016efa5be020c3b Mon Sep 17 00:00:00 2001 From: Beth Rennie Date: Thu, 30 Jul 2026 11:29:43 -0400 Subject: [PATCH 40/59] Bug 2055524 - Add debug logging to NimbusClient::get_available_firefox_labs (#7482) --- CHANGELOG.md | 4 ++ components/nimbus/src/stateful/dbcache.rs | 87 ++++++++++++++++++----- 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86ae170192d..6a88b534516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ - Add `Store::shutdown()`, which closes the database connection early so it happens before Firefox Desktop's late-write shutdown barrier rather than during GC. Operations after shutdown return `DatabaseClosed`. ([Bug 2050036](https://bugzilla.mozilla.org/show_bug.cgi?id=2050036)) +### Nimbus + +- `NimbusClient::get_available_firefox_labs()` now includes detailed debug level logging for each processed lab. ([#7482](https://github.com/mozilla/application-services/pull/7482)) + ### Remote Settings - Replacing v1 routes with v2 routes, removing added v2 routes ([#7492](https://github.com/mozilla/application-services/pull/7339)) diff --git a/components/nimbus/src/stateful/dbcache.rs b/components/nimbus/src/stateful/dbcache.rs index e68d685cd4c..f6f58dd21ae 100644 --- a/components/nimbus/src/stateful/dbcache.rs +++ b/components/nimbus/src/stateful/dbcache.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, RwLock}; use crate::enrollment::{ EnrolledFeature, EnrolledFeatureConfig, ExperimentEnrollment, map_features_by_feature_id, }; -use crate::error::{NimbusError, Result, warn}; +use crate::error::{NimbusError, Result, debug, warn}; use crate::evaluator::{CanEnrollResult, can_enroll}; use crate::stateful::enrollment::get_enrollments; use crate::stateful::firefox_labs::FirefoxLabsMetadata; @@ -230,33 +230,84 @@ impl DatabaseCache { let coenrolling_feature_ids: HashSet<&str> = coenrolling_feature_ids.iter().map(|s| s.as_ref()).collect(); - data.experiments + debug!("firefox labs: querying experiments..."); + let available = data + .experiments .iter() .filter_map(|experiment| { if !experiment.is_firefox_labs_opt_in { + debug!( + "firefox labs: {}: not a firefox labs opt-in", + experiment.slug + ); return None; } let enrolled = data.experiments_by_slug.contains_key(&experiment.slug); - let enrollable = matches!( - can_enroll(available_randomization_units, targeting_helper, experiment,), - CanEnrollResult::Enrollable { .. } - ); - - if enrollable - && (enrolled - || (features_available( - experiment, - &enrolled_feature_ids, - &coenrolling_feature_ids, - ) && !experiment.is_enrollment_paused)) - { - experiment.get_firefox_labs_metadata(enrolled) + match can_enroll(available_randomization_units, targeting_helper, experiment) { + CanEnrollResult::Enrollable { .. } => {} + + CanEnrollResult::Unavailable { reason } => { + debug!("firefox labs: {}: unavailable: {}", experiment.slug, reason); + return None; + } + + CanEnrollResult::TargetingError { reason } => { + debug!( + "firefox labs: {}: targeting error: {}", + experiment.slug, reason + ); + return None; + } + + CanEnrollResult::NotTargeted => { + debug!("firefox labs: {}: not targeted", experiment.slug); + return None; + } + + CanEnrollResult::NotSelected => { + debug!("firefox labs: {}: not selected", experiment.slug); + return None; + } + + CanEnrollResult::NoRandomizationUnit => { + debug!("firefox labs: {}: no randomization unit", experiment.slug); + return None; + } + } + + if !enrolled { + let feature_conflict = !features_available( + experiment, + &enrolled_feature_ids, + &coenrolling_feature_ids, + ); + + if feature_conflict { + debug!("firefox labs: {}: feature conflict", experiment.slug); + return None; + } + + if experiment.is_enrollment_paused { + debug!("firefox labs: {}: enrollment paused", experiment.slug); + return None; + } + } + + let metadata = experiment.get_firefox_labs_metadata(enrolled); + if metadata.is_none() { + debug!("firefox labs: {}: invalid lab", experiment.slug); } else { - None + debug!("firefox labs: {}: available", experiment.slug); } + + metadata }) - .collect() + .collect(); + + debug!("firefox labs: finished querying experiments"); + + available })?; // XXX: This is maybe only useful for tests, but at least we get a From 5015a4d62688dfb0a8cb2edbacef538ab9822c88 Mon Sep 17 00:00:00 2001 From: Alex Cottner <148472676+alexcottner@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:41:26 -1000 Subject: [PATCH 41/59] Bug 2058816 - updating get_base_url and get_url in remote_settings to append v2 (#7517) * Bug 2058816 - updating get_base_url and get_url in remote_settings config to append v2 instead of v1 for custom url's * Updating remote_settings unit tests * updating search and nimbus-cli refs --- components/remote_settings/src/client.rs | 28 +++++++++---------- components/remote_settings/src/config.rs | 8 +++--- components/remote_settings/src/schema.rs | 6 ++-- components/remote_settings/src/service.rs | 16 +++++------ components/search/src/selector.rs | 22 +++++++-------- .../support/nimbus-cli/src/output/server.rs | 8 +++--- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/components/remote_settings/src/client.rs b/components/remote_settings/src/client.rs index 89ec6912a35..3f26a96aed7 100644 --- a/components/remote_settings/src/client.rs +++ b/components/remote_settings/src/client.rs @@ -762,7 +762,7 @@ struct RemoteSettingsEndpoints { impl RemoteSettingsEndpoints { /// Construct a new RemoteSettingsEndpoints /// - /// `base_url` should have the form `https://[domain]/v1` (no trailing slash). + /// `base_url` should have the form `https://[domain]/v2` (no trailing slash). fn new(base_url: &BaseUrl, bucket_name: &str, collection_name: &str) -> Self { let mut root_url = base_url.clone(); // Push the empty string to add the trailing slash. @@ -944,18 +944,18 @@ mod test_new_client { #[test] fn test_endpoints() { let endpoints = RemoteSettingsEndpoints::new( - &BaseUrl::parse("http://rs.example.com/v1").unwrap(), + &BaseUrl::parse("http://rs.example.com/v2").unwrap(), "main", "test-collection", ); - assert_eq!(endpoints.root_url.to_string(), "http://rs.example.com/v1/"); + assert_eq!(endpoints.root_url.to_string(), "http://rs.example.com/v2/"); assert_eq!( endpoints.collection_url.to_string(), - "http://rs.example.com/v1/buckets/main/collections/test-collection", + "http://rs.example.com/v2/buckets/main/collections/test-collection", ); assert_eq!( endpoints.changeset_url.to_string(), - "http://rs.example.com/v1/buckets/main/collections/test-collection/changeset", + "http://rs.example.com/v2/buckets/main/collections/test-collection/changeset", ); } } @@ -986,7 +986,7 @@ mod jexl_tests { metadata: CollectionMetadata::default(), }; api_client.expect_collection_url().returning(|| { - "http://rs.example.com/v1/buckets/main/collections/test-collection".into() + "http://rs.example.com/v2/buckets/main/collections/test-collection".into() }); api_client.expect_fetch_changeset().returning({ let changeset = changeset.clone(); @@ -1004,7 +1004,7 @@ mod jexl_tests { let mut storage = Storage::new(":memory:".into()); let _ = storage.insert_collection_content( - "http://rs.example.com/v1/buckets/main/collections/test-collection", + "http://rs.example.com/v2/buckets/main/collections/test-collection", &records, 42, CollectionMetadata::default(), @@ -1044,7 +1044,7 @@ mod jexl_tests { metadata: CollectionMetadata::default(), }; api_client.expect_collection_url().returning(|| { - "http://rs.example.com/v1/buckets/main/collections/test-collection".into() + "http://rs.example.com/v2/buckets/main/collections/test-collection".into() }); api_client.expect_fetch_changeset().returning({ let changeset = changeset.clone(); @@ -1062,7 +1062,7 @@ mod jexl_tests { let mut storage = Storage::new(":memory:".into()); let _ = storage.insert_collection_content( - "http://rs.example.com/v1/buckets/main/collections/test-collection", + "http://rs.example.com/v2/buckets/main/collections/test-collection", &records, 42, CollectionMetadata::default(), @@ -1102,7 +1102,7 @@ mod jexl_tests { metadata: CollectionMetadata::default(), }; api_client.expect_collection_url().returning(|| { - "http://rs.example.com/v1/buckets/main/collections/test-collection".into() + "http://rs.example.com/v2/buckets/main/collections/test-collection".into() }); api_client.expect_fetch_changeset().returning({ let changeset = changeset.clone(); @@ -1120,7 +1120,7 @@ mod jexl_tests { let mut storage = Storage::new(":memory:".into()); let _ = storage.insert_collection_content( - "http://rs.example.com/v1/buckets/main/collections/test-collection", + "http://rs.example.com/v2/buckets/main/collections/test-collection", &records, 42, CollectionMetadata::default(), @@ -1180,7 +1180,7 @@ mod jexl_tests { "test-collection".to_string(), None, ); - "http://rs.example.com/v1/buckets/main/collections/test-collection".into() + "http://rs.example.com/v2/buckets/main/collections/test-collection".into() }); api_client.expect_is_prod_server().returning(|| Ok(false)); @@ -1609,7 +1609,7 @@ mod test_reset_storage { #[test] fn test_reset_storage_deletes_records_and_attachments() { - let collection_url = "http://rs.example.com/v1/buckets/main/collections/test-collection"; + let collection_url = "http://rs.example.com/v2/buckets/main/collections/test-collection"; let mut api_client = MockApiClient::new(); api_client @@ -1675,7 +1675,7 @@ mod test_reset_storage { #[test] fn test_reset_storage_reverts_to_packaged_data() { - let collection_url = "http://rs.example.com/v1/buckets/main/collections/regions"; + let collection_url = "http://rs.example.com/v2/buckets/main/collections/regions"; let mut api_client = MockApiClient::new(); api_client diff --git a/components/remote_settings/src/config.rs b/components/remote_settings/src/config.rs index a71d78508fd..689b1cd0288 100644 --- a/components/remote_settings/src/config.rs +++ b/components/remote_settings/src/config.rs @@ -47,12 +47,12 @@ impl RemoteSettingsServer { pub fn get_base_url(&self) -> Result { let base_url = BaseUrl::parse(self.raw_url())?; // Custom URLs are weird and require a couple tricks for backwards compatibility. - // Normally we append `v1/` to match how this has historically worked. However, + // Normally we append `v2/` to match how this has historically worked. However, // don't do this for file:// schemes which normally don't make any sense, but it's // what Nimbus uses to indicate they want to use the file-based client, rather than // a remote-settings based one. if base_url.url().scheme() != "file" { - Ok(base_url.join("v1")) + Ok(base_url.join("v2")) } else { Ok(base_url) } @@ -96,12 +96,12 @@ impl RemoteSettingsServer { Self::Custom { url } => { let mut url = Url::parse(url)?; // Custom URLs are weird and require a couple tricks for backwards compatibility. - // Normally we append `v1/` to match how this has historically worked. However, + // Normally we append `v2/` to match how this has historically worked. However, // don't do this for file:// schemes which normally don't make any sense, but it's // what Nimbus uses to indicate they want to use the file-based client, rather than // a remote-settings based one. if url.scheme() != "file" { - url = url.join("v1")? + url = url.join("v2")? } url } diff --git a/components/remote_settings/src/schema.rs b/components/remote_settings/src/schema.rs index 7adde331ead..ad7600101ec 100644 --- a/components/remote_settings/src/schema.rs +++ b/components/remote_settings/src/schema.rs @@ -206,7 +206,7 @@ PRAGMA user_version=0; "INSERT INTO records (id, collection_url, data) VALUES (?, ?, ?)", rusqlite::params![ "sponsored-suggestions-us-phone", - "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/quicksuggest-amp", + "https://firefox.settings.services.mozilla.com/v2/buckets/main/collections/quicksuggest-amp", serde_json::to_vec(&record).unwrap(), ], ).unwrap(); @@ -216,7 +216,7 @@ PRAGMA user_version=0; "INSERT INTO attachments (id, collection_url, data) VALUES (?, ?, ?)", rusqlite::params![ "main-workspace/quicksuggest-amp/b.json", - "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/quicksuggest-amp", + "https://firefox.settings.services.mozilla.com/v2/buckets/main/collections/quicksuggest-amp", b"current attachment data", ], ).unwrap(); @@ -226,7 +226,7 @@ PRAGMA user_version=0; "INSERT INTO attachments (id, collection_url, data) VALUES (?, ?, ?)", rusqlite::params![ "main-workspace/quicksuggest-amp/a.json", - "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/quicksuggest-amp", + "https://firefox.settings.services.mozilla.com/v2/buckets/main/collections/quicksuggest-amp", b"orphaned attachment data that should be cleaned up", ], ).unwrap(); diff --git a/components/remote_settings/src/service.rs b/components/remote_settings/src/service.rs index 7410a068b97..b50c01140e8 100644 --- a/components/remote_settings/src/service.rs +++ b/components/remote_settings/src/service.rs @@ -350,7 +350,7 @@ mod test { } fn mock_monitor_changes(collection: &str, timestamp: u64) -> mockito::Mock { - mock("GET", "/v1/buckets/monitor/collections/changes/changeset") + mock("GET", "/v2/buckets/monitor/collections/changes/changeset") .match_query(Matcher::Any) .with_status(200) .with_header("content-type", "application/json") @@ -363,7 +363,7 @@ mod test { fn mock_changeset(collection: &str, timestamp: u64) -> mockito::Mock { mock( "GET", - format!("/v1/buckets/main/collections/{collection}/changeset").as_str(), + format!("/v2/buckets/main/collections/{collection}/changeset").as_str(), ) .match_query(Matcher::Any) .with_status(200) @@ -377,7 +377,7 @@ mod test { fn mock_changeset_error(bucket: &str, collection: &str) -> mockito::Mock { mock( "GET", - format!("/v1/buckets/{bucket}/collections/{collection}/changeset").as_str(), + format!("/v2/buckets/{bucket}/collections/{collection}/changeset").as_str(), ) .match_query(Matcher::Any) .with_status(500) @@ -561,7 +561,7 @@ mod test { ); // First sync creates a record that references the big attachment. - let _changes_1 = mock("GET", "/v1/buckets/monitor/collections/changes/changeset") + let _changes_1 = mock("GET", "/v2/buckets/monitor/collections/changes/changeset") .match_query(Matcher::Any) .with_status(200) .with_header("content-type", "application/json") @@ -577,7 +577,7 @@ mod test { let _changeset_1 = mock( "GET", - format!("/v1/buckets/main/collections/{collection}/changeset").as_str(), + format!("/v2/buckets/main/collections/{collection}/changeset").as_str(), ) .match_query(Matcher::Any) .with_status(200) @@ -606,7 +606,7 @@ mod test { service.sync()?; // Mock attachment discovery and download. - let _root = mock("GET", "/v1/") + let _root = mock("GET", "/v2/") .with_status(200) .with_header("content-type", "application/json") .with_body(format!( @@ -659,7 +659,7 @@ mod test { // Second sync tombstones the record. This deletes the attachment row, and // post-sync maintenance should compact the database. - let _changes_2 = mock("GET", "/v1/buckets/monitor/collections/changes/changeset") + let _changes_2 = mock("GET", "/v2/buckets/monitor/collections/changes/changeset") .match_query(Matcher::Any) .with_status(200) .with_header("content-type", "application/json") @@ -675,7 +675,7 @@ mod test { let _changeset_2 = mock( "GET", - format!("/v1/buckets/main/collections/{collection}/changeset").as_str(), + format!("/v2/buckets/main/collections/{collection}/changeset").as_str(), ) .match_query(Matcher::Any) .with_status(200) diff --git a/components/search/src/selector.rs b/components/search/src/selector.rs index c1c2b27970a..105b60a154b 100644 --- a/components/search/src/selector.rs +++ b/components/search/src/selector.rs @@ -899,7 +899,7 @@ mod tests { fn mock_changes_endpoint() -> mockito::Mock { mock( "GET", - "/v1/buckets/monitor/collections/changes/changeset?_expected=0", + "/v2/buckets/monitor/collections/changes/changeset?_expected=0", ) .with_body(response_body_changes()) .with_status(200) @@ -1047,7 +1047,7 @@ mod tests { let changes_mock = mock_changes_endpoint(); let m = mock( "GET", - "/v1/buckets/main/collections/search-config-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-v2/changeset?_expected=0", ) .with_body( json!({ @@ -1094,7 +1094,7 @@ mod tests { let changes_mock = mock_changes_endpoint(); let m1 = mock( "GET", - "/v1/buckets/main/collections/search-config-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-v2/changeset?_expected=0", ) .with_body(response_body()) .with_status(501) @@ -1125,7 +1125,7 @@ mod tests { let changes_mock = mock_changes_endpoint(); let m1 = mock( "GET", - "/v1/buckets/main/collections/search-config-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-v2/changeset?_expected=0", ) .with_body(response_body()) .with_status(200) @@ -1135,7 +1135,7 @@ mod tests { let m2 = mock( "GET", - "/v1/buckets/main/collections/search-config-overrides-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-overrides-v2/changeset?_expected=0", ) .with_body( json!({ @@ -1180,7 +1180,7 @@ mod tests { let changes_mock = mock_changes_endpoint(); let m1 = mock( "GET", - "/v1/buckets/main/collections/search-config-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-v2/changeset?_expected=0", ) .with_body(response_body()) .with_status(200) @@ -1190,7 +1190,7 @@ mod tests { let m2 = mock( "GET", - "/v1/buckets/main/collections/search-config-overrides-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-overrides-v2/changeset?_expected=0", ) .with_body(response_body_overrides()) .with_status(501) @@ -1222,7 +1222,7 @@ mod tests { let changes_mock = mock_changes_endpoint(); let m1 = mock( "GET", - "/v1/buckets/main/collections/search-config-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-v2/changeset?_expected=0", ) .with_body(response_body()) .with_status(200) @@ -1232,7 +1232,7 @@ mod tests { let m2 = mock( "GET", - "/v1/buckets/main/collections/search-config-overrides-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-overrides-v2/changeset?_expected=0", ) .with_body(response_body_overrides()) .with_status(200) @@ -1281,7 +1281,7 @@ mod tests { let m = mock( "GET", - "/v1/buckets/main/collections/search-config-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-v2/changeset?_expected=0", ) .with_body(response_body()) .with_status(200) @@ -1352,7 +1352,7 @@ mod tests { let changes_mock = mock_changes_endpoint(); let m = mock( "GET", - "/v1/buckets/main/collections/search-config-v2/changeset?_expected=0", + "/v2/buckets/main/collections/search-config-v2/changeset?_expected=0", ) .with_body(response_body_locales()) .with_status(200) diff --git a/components/support/nimbus-cli/src/output/server.rs b/components/support/nimbus-cli/src/output/server.rs index 78c41b739b4..1de49082d69 100644 --- a/components/support/nimbus-cli/src/output/server.rs +++ b/components/support/nimbus-cli/src/output/server.rs @@ -50,7 +50,7 @@ fn create_app(livereload: LiveReloadLayer, state: Db) -> Router { .route("/post", post(post_handler)) .route("/buckets/:bucket/collections/:collection/records", get(rs)) .route( - "/v1/buckets/:bucket/collections/:collection/records", + "/v2/buckets/:bucket/collections/:collection/records", get(rs), ) .layer(livereload) @@ -376,7 +376,7 @@ mod tests { let _ = post_payload(&payload, &format!("127.0.0.1:{port}")).await?; // Check the fake Remote Settings page - let s = get(port, "/v1/buckets/BUCKET/collections/COLLECTION/records").await?; + let s = get(port, "/v2/buckets/BUCKET/collections/COLLECTION/records").await?; assert_eq!(s, serde_json::to_string(&value)?); let s = get(port, "/buckets/BUCKET/collections/COLLECTION/records").await?; @@ -392,7 +392,7 @@ mod tests { let (_, tx) = start_test_server(port)?; // Part 1: get from remote settings page before anything has been posted yet. - let s = get(port, "/v1/buckets/BUCKET/collections/COLLECTION/records").await?; + let s = get(port, "/v2/buckets/BUCKET/collections/COLLECTION/records").await?; assert_eq!(s, "null".to_string()); // Part 2: Post a payload, but not with any experiments. @@ -404,7 +404,7 @@ mod tests { // Check the fake Remote Settings page, should be empty, since an experiments payload // wasn't posted - let s = get(port, "/v1/buckets/BUCKET/collections/COLLECTION/records").await?; + let s = get(port, "/v2/buckets/BUCKET/collections/COLLECTION/records").await?; assert_eq!(s, "".to_string()); let _ = tx.send(()); From fb23a0ea61a924d18a1c3fd54fc66c2c8ba7720f Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Thu, 30 Jul 2026 20:17:50 -0700 Subject: [PATCH 42/59] Fix `groupId` dropping the `.nightly` suffix (#7519) PR #7508 hardcoded `"org.mozilla.appservices"`, dropping the `.nightly` suffix on nightly builds. To fix, we add `appServicesGroupId` to resolve the proper `groupId` based on the context (`mozilla-central` or standalone `application-services`). `configurePublish`'s parameters were also being shadowed in `publish.gradle`, which this resolves too. --- build-scripts/component-common.gradle | 9 +++++++++ components/ads-client/android/build.gradle | 2 +- components/autofill/android/build.gradle | 2 +- components/crashtest/android/build.gradle | 2 +- components/fxa-client/android/build.gradle | 2 +- .../init_rust_components/android/build.gradle | 2 +- components/logins/android/build.gradle | 2 +- components/merino/android/build.gradle | 2 +- components/nimbus/android/build.gradle | 2 +- components/places/android/build.gradle | 2 +- components/push/android/build.gradle | 2 +- components/relay/android/build.gradle | 2 +- components/remote_settings/android/build.gradle | 2 +- components/search/android/build.gradle | 2 +- components/suggest/android/build.gradle | 2 +- components/support/error/android/build.gradle | 2 +- .../rust-log-forwarder/android/build.gradle | 2 +- components/support/tracing/android/build.gradle | 2 +- components/sync15/android/build.gradle | 2 +- components/sync_manager/android/build.gradle | 2 +- components/tabs/android/build.gradle | 2 +- components/viaduct/android/build.gradle | 2 +- megazords/full/android/build.gradle | 14 ++++++++++++-- publish.gradle | 10 ++++++---- tools/start-bindings/templates/build.gradle | 2 +- 25 files changed, 49 insertions(+), 28 deletions(-) diff --git a/build-scripts/component-common.gradle b/build-scripts/component-common.gradle index 2d7cb26e47d..9656d04606d 100644 --- a/build-scripts/component-common.gradle +++ b/build-scripts/component-common.gradle @@ -115,6 +115,15 @@ dependencies { androidTestImplementation libs.androidx.test.runner } +// Pick the publish group id by build context. In m-c, `config` belongs to +// android-components (`config.componentsGroupId` is `org.mozilla.components`), +// so we hardcode `org.mozilla.appservices`. Standalone app-services uses +// `config.componentsGroupId`, which is the only place that appends the +// `.nightly` suffix nightly builds publish under. +ext.appServicesGroupId = gradle.root.hasProperty("mozconfig") + ? "org.mozilla.appservices" + : rootProject.config.componentsGroupId + // Shared logic for projects that depend on libmegazord // // This ensures that libmegazord will be in the library path so that it can be loaded. It also adds diff --git a/components/ads-client/android/build.gradle b/components/ads-client/android/build.gradle index 2d2db933137..b6e9faf57f0 100644 --- a/components/ads-client/android/build.gradle +++ b/components/ads-client/android/build.gradle @@ -45,4 +45,4 @@ dependencies { ext.configureUniFFIBindgen("ads_client") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/autofill/android/build.gradle b/components/autofill/android/build.gradle index 64d1878da31..5fcaf37929e 100644 --- a/components/autofill/android/build.gradle +++ b/components/autofill/android/build.gradle @@ -16,4 +16,4 @@ dependencies { ext.configureUniFFIBindgen("autofill") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/crashtest/android/build.gradle b/components/crashtest/android/build.gradle index 66b10120aec..7684e8fc1f9 100644 --- a/components/crashtest/android/build.gradle +++ b/components/crashtest/android/build.gradle @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("crashtest") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/fxa-client/android/build.gradle b/components/fxa-client/android/build.gradle index d387acf7fe4..8346dadc4e6 100644 --- a/components/fxa-client/android/build.gradle +++ b/components/fxa-client/android/build.gradle @@ -47,4 +47,4 @@ dependencies { ext.configureUniFFIBindgen("fxa_client") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/init_rust_components/android/build.gradle b/components/init_rust_components/android/build.gradle index 2eedc536298..2bfd72a6120 100644 --- a/components/init_rust_components/android/build.gradle +++ b/components/init_rust_components/android/build.gradle @@ -15,4 +15,4 @@ dependencies { ext.configureUniFFIBindgen("init_rust_components") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/logins/android/build.gradle b/components/logins/android/build.gradle index 038b1371d3a..4834fb15f14 100644 --- a/components/logins/android/build.gradle +++ b/components/logins/android/build.gradle @@ -58,4 +58,4 @@ dependencies { ext.configureUniFFIBindgen("logins") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/merino/android/build.gradle b/components/merino/android/build.gradle index 03f7f70fc39..8fb32011b70 100644 --- a/components/merino/android/build.gradle +++ b/components/merino/android/build.gradle @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("merino") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/nimbus/android/build.gradle b/components/nimbus/android/build.gradle index 5b9f61cb0a7..da36daa6806 100644 --- a/components/nimbus/android/build.gradle +++ b/components/nimbus/android/build.gradle @@ -54,4 +54,4 @@ dependencies { ext.configureUniFFIBindgen("nimbus") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/places/android/build.gradle b/components/places/android/build.gradle index fdb42062a61..04e7bf2a0da 100644 --- a/components/places/android/build.gradle +++ b/components/places/android/build.gradle @@ -52,4 +52,4 @@ dependencies { ext.configureUniFFIBindgen("places") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/push/android/build.gradle b/components/push/android/build.gradle index 78aa011163d..afe0c824080 100644 --- a/components/push/android/build.gradle +++ b/components/push/android/build.gradle @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("push") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/relay/android/build.gradle b/components/relay/android/build.gradle index cafd3510c54..e836136dc33 100644 --- a/components/relay/android/build.gradle +++ b/components/relay/android/build.gradle @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("relay") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/remote_settings/android/build.gradle b/components/remote_settings/android/build.gradle index c7c458a5962..9d92eff8cbc 100644 --- a/components/remote_settings/android/build.gradle +++ b/components/remote_settings/android/build.gradle @@ -45,7 +45,7 @@ dependencies { ext.configureUniFFIBindgen("remote_settings") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) dependencies { if (gradle.hasProperty("mozconfig")) { diff --git a/components/search/android/build.gradle b/components/search/android/build.gradle index a54d4ae3176..ababb55798a 100644 --- a/components/search/android/build.gradle +++ b/components/search/android/build.gradle @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("search") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) \ No newline at end of file +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) \ No newline at end of file diff --git a/components/suggest/android/build.gradle b/components/suggest/android/build.gradle index e8fb54d9f6e..3eb7c8a35a0 100644 --- a/components/suggest/android/build.gradle +++ b/components/suggest/android/build.gradle @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("suggest") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/support/error/android/build.gradle b/components/support/error/android/build.gradle index 711b9fc9086..bc0f8270b3e 100644 --- a/components/support/error/android/build.gradle +++ b/components/support/error/android/build.gradle @@ -54,4 +54,4 @@ android { ext.configureUniFFIBindgen("error_support") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/support/rust-log-forwarder/android/build.gradle b/components/support/rust-log-forwarder/android/build.gradle index f72401b60c0..5ff095334f6 100644 --- a/components/support/rust-log-forwarder/android/build.gradle +++ b/components/support/rust-log-forwarder/android/build.gradle @@ -11,4 +11,4 @@ dependencies { ext.configureUniFFIBindgen("rust_log_forwarder") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/support/tracing/android/build.gradle b/components/support/tracing/android/build.gradle index 1eadd69e0c2..9444ebd2be0 100644 --- a/components/support/tracing/android/build.gradle +++ b/components/support/tracing/android/build.gradle @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("tracing_support") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/sync15/android/build.gradle b/components/sync15/android/build.gradle index fa742ae294e..f77962ea793 100644 --- a/components/sync15/android/build.gradle +++ b/components/sync15/android/build.gradle @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("sync15") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/sync_manager/android/build.gradle b/components/sync_manager/android/build.gradle index 190f72f2419..e37353f4da8 100644 --- a/components/sync_manager/android/build.gradle +++ b/components/sync_manager/android/build.gradle @@ -55,4 +55,4 @@ dependencies { ext.configureUniFFIBindgen("sync_manager") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/tabs/android/build.gradle b/components/tabs/android/build.gradle index 825cfd6eebf..93960ba84b2 100644 --- a/components/tabs/android/build.gradle +++ b/components/tabs/android/build.gradle @@ -14,4 +14,4 @@ dependencies { ext.configureUniFFIBindgen("tabs") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/components/viaduct/android/build.gradle b/components/viaduct/android/build.gradle index b5c6b8a072a..3dcc73f089d 100644 --- a/components/viaduct/android/build.gradle +++ b/components/viaduct/android/build.gradle @@ -16,4 +16,4 @@ dependencies { ext.configureUniFFIBindgen("viaduct") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) diff --git a/megazords/full/android/build.gradle b/megazords/full/android/build.gradle index dcb8e838164..dc3273db0ae 100644 --- a/megazords/full/android/build.gradle +++ b/megazords/full/android/build.gradle @@ -228,8 +228,18 @@ if (!gradle.hasProperty("mozconfig")) { } } +// Duplicated from component-common.gradle, which the megazord doesn't apply. +// Pick the publish group id by build context. In m-c, `config` belongs to +// android-components (`config.componentsGroupId` is `org.mozilla.components`), +// so we hardcode `org.mozilla.appservices`. Standalone app-services uses +// `config.componentsGroupId`, which is the only place that appends the +// `.nightly` suffix nightly builds publish under. +ext.appServicesGroupId = gradle.root.hasProperty("mozconfig") + ? "org.mozilla.appservices" + : rootProject.config.componentsGroupId + apply from: "$publishDir/publish.gradle" -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) afterEvaluate { def libUrl="https://github.com/mozilla/application-services" @@ -252,7 +262,7 @@ afterEvaluate { } pom { - groupId = "org.mozilla.appservices" + groupId = project.ext.appServicesGroupId artifactId = "${project.ext.artifactId}-libsForTests" description = project.ext.description // For mavenLocal publishing workflow, increment the version number every publish. diff --git a/publish.gradle b/publish.gradle index a106e01db93..abd7156cac6 100644 --- a/publish.gradle +++ b/publish.gradle @@ -11,10 +11,12 @@ def libProjectName = properties.libProjectName def libUrl = properties.libUrl def libVcsUrl = properties.libVcsUrl -ext.configurePublish = { groupId = null, artifactId = null, description = null -> - def theGroupId = groupId ?: rootProject.config.componentsGroupId - def theArtifactId = artifactId ?: project.ext.artifactId - def theDescription = description ?: project.ext.description +// The `Arg` suffixes matter. Naming these `groupId`, `artifactId` or `description` shadows the +// assignments to the publication down in `pom`, which then silently do nothing. +ext.configurePublish = { groupIdArg = null, artifactIdArg = null, descriptionArg = null -> + def theGroupId = groupIdArg ?: rootProject.config.componentsGroupId + def theArtifactId = artifactIdArg ?: project.ext.artifactId + def theDescription = descriptionArg ?: project.ext.description // This is a little cludgey, but it seems unlikely to cause a problem, and // we are already doing it inside taskcluster. diff --git a/tools/start-bindings/templates/build.gradle b/tools/start-bindings/templates/build.gradle index 6dd2146c0ea..bdcb8586a1e 100644 --- a/tools/start-bindings/templates/build.gradle +++ b/tools/start-bindings/templates/build.gradle @@ -7,4 +7,4 @@ android { ext.configureUniFFIBindgen("{{ crate_name }}") ext.dependsOnTheMegazord() -ext.configurePublish("org.mozilla.appservices", project.name, project.ext.description) +ext.configurePublish(appServicesGroupId, project.name, project.ext.description) From 3c61070a9e75f389b5cef0ec4772841bd38167f8 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Fri, 31 Jul 2026 09:38:58 +0200 Subject: [PATCH 43/59] feat(remote-settings): RMST-484: verify signatures with sync_if_empty (#7518) * feat(remote-settings): RMST-484: verify signatures with sync_if_empty * Update changelog --- CHANGELOG.md | 1 + components/remote_settings/src/client.rs | 104 +++++++++++++++++++---- 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a88b534516..ab9ce100007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ### Remote Settings - Replacing v1 routes with v2 routes, removing added v2 routes ([#7492](https://github.com/mozilla/application-services/pull/7339)) +- Verify signature of imported data when `.get()` is called with `sync_if_empty: true` ([#7518](https://github.com/mozilla/application-services/pull/7518)) # v154.0 (_2026-07-20_) diff --git a/components/remote_settings/src/client.rs b/components/remote_settings/src/client.rs index 3f26a96aed7..760cbbbe696 100644 --- a/components/remote_settings/src/client.rs +++ b/components/remote_settings/src/client.rs @@ -325,26 +325,28 @@ impl RemoteSettingsClient { let cached_records = inner.storage.get_records(&collection_url)?; - Ok(match (cached_records, sync_if_empty) { + match (cached_records, sync_if_empty) { // Case 2: We have cached records // // Note: we should return these even if it's an empty list and `sync_if_empty=true`. // The "if empty" part refers to the cache being empty, not the list. - (Some(cached_records), _) => Some(self.filter_records(cached_records, &inner)), + (Some(cached_records), _) => Ok(Some(self.filter_records(cached_records, &inner))), // Case 3: sync_if_empty=true (None, true) => { - let changeset = inner.api_client.fetch_changeset(None)?; - inner.storage.insert_collection_content( - &collection_url, - &changeset.changes, - changeset.timestamp, - changeset.metadata, - )?; - Some(self.filter_records(changeset.changes, &inner)) + // `sync()` takes the lock, release it first. + drop(inner); + // Sync and verify content signatures. + self.sync()?; + // Return what was just stored. + let mut inner = self.lock_inner()?; + Ok(inner + .storage + .get_records(&collection_url)? + .map(|records| self.filter_records(records, &inner))) } // Case 4: Nothing to return - (None, false) => None, - }) + (None, false) => Ok(None), + } } /// Returns the last modified timestamp for the collection. @@ -1314,14 +1316,14 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= const VALID_SIGNATURE: &str = r#"fJJcOpwdnkjEWFeHXfdOJN6GaGLuDTPGzQOxA2jn6ldIleIk6KqMhZcy2GZv2uYiGwl6DERWwpaoUfQFLyCAOcVjck1qlaaEFZGY1BQba9p99xEc9FNQ3YPPfvSSZqsw"#; const VALID_CERT_EPOCH_SECONDS: u64 = 1615559719; - fn run_client_sync( + fn build_client( diff_records: &[RemoteSettingsRecord], full_records: &[RemoteSettingsRecord], certificate: &str, signatures: &[CollectionSignature], epoch_secs: u64, bucket: &str, - ) -> Result<()> { + ) -> RemoteSettingsClient { let collection_name = "pioneer-study-addons"; MOCK_TIME.with(|cell| cell.set(Some(epoch_secs))); @@ -1363,14 +1365,31 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= let storage = Storage::new(":memory:".into()); let jexl_filter = JexlFilter::new(Some(RemoteSettingsContext::default())); - let rs_client = RemoteSettingsClient::new_from_parts( + RemoteSettingsClient::new_from_parts( collection_name.to_string(), storage, jexl_filter, api_client, - ); + ) + } - rs_client.sync() + fn run_client_sync( + diff_records: &[RemoteSettingsRecord], + full_records: &[RemoteSettingsRecord], + certificate: &str, + signatures: &[CollectionSignature], + epoch_secs: u64, + bucket: &str, + ) -> Result<()> { + build_client( + diff_records, + full_records, + certificate, + signatures, + epoch_secs, + bucket, + ) + .sync() } #[test] @@ -1601,6 +1620,57 @@ IKdcFKAt3fFrpyMhlfIKkLfmm0iDjmfmIXbDGBJw9SE= Ok(()) } + + #[test] + fn test_get_records_sync_if_empty_verifies_signature() -> Result<()> { + ensure_initialized(); + let rs_client = build_client( + &[], + &[], + VALID_CERTIFICATE, + &[CollectionSignature { + signature: "invalid signature".to_string(), + x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), + }], + VALID_CERT_EPOCH_SECONDS, + "main", + ); + + let err = rs_client.get_records(true).unwrap_err(); + + assert!(matches!(err, Error::SignatureError(_))); + assert_eq!(format!("{}", err), "Signature could not be verified: Signature content error: Encoded text cannot have a 6-bit remainder."); + + // Unverified data was not kept in storage. + let mut inner = rs_client.lock_inner()?; + let collection_url = inner.api_client.collection_url(); + assert_eq!(inner.storage.get_records(&collection_url)?, None); + + Ok(()) + } + + #[test] + fn test_get_records_sync_if_empty_with_valid_signature() -> Result<()> { + ensure_initialized(); + let rs_client = build_client( + &[], + &[], + VALID_CERTIFICATE, + &[CollectionSignature { + signature: VALID_SIGNATURE.to_string(), + x5u: "http://mocked".into(), + mode: "p384ecdsa".into(), + }], + VALID_CERT_EPOCH_SECONDS, + "main", + ); + + // The signature is only valid for an empty list of records. + assert_eq!(rs_client.get_records(true)?, Some(vec![])); + + Ok(()) + } } #[cfg(test)] From d435907d3b4718bc4de644cae592e0fa97c5f22f Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 31 Jul 2026 09:40:52 -0400 Subject: [PATCH 44/59] Bug 2059169 - Allow CheckAuthorizationStatus from more states (#7514) --- .../fxa-client/src/state_machine/transitions.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/components/fxa-client/src/state_machine/transitions.rs b/components/fxa-client/src/state_machine/transitions.rs index 8b285f0b624..22f52b1656d 100644 --- a/components/fxa-client/src/state_machine/transitions.rs +++ b/components/fxa-client/src/state_machine/transitions.rs @@ -212,6 +212,22 @@ pub fn transition( .to_state_machine_err(|| S::AuthIssues)?; Ok(S::Connected) } + (S::AuthIssues, FxaEvent::CheckAuthorizationStatus) => { + let active = account + .check_authorization_status() + .to_state_machine_err(|| S::AuthIssues)?; + Ok(if active { S::Connected } else { S::AuthIssues }) + } + + // ── Other transitions ───────────────────────────────── + (from_state, FxaEvent::CheckAuthorizationStatus) => { + // Ignore `CheckAuthorizationStatus` from other states. + // We want the app to be able to send this event whenever they want, + // without generating an error. + // If we're in a state where we can't run a check, then just ignore it. + error_support::debug!("Ignoring `CheckAuthorizationStatus` from {from_state:?}"); + Ok(from_state) + } // ── Invalid (state, event) pair ───────────────────────────────── (state, event) => Err(StateMachineErr::Fatal(Box::new( From daa888fa1b8cc9486340be97d361c57e8b66ec33 Mon Sep 17 00:00:00 2001 From: Wyatt Verchere Date: Fri, 31 Jul 2026 15:33:38 -0700 Subject: [PATCH 45/59] test: Updating older cross-build smoke tests (iOS, Android) (#7487) * test: Adds new python smoke tests for building against iOS; fenix * test: Adds scheme, tests * fix: Modularization and missing xcframework change * fix: linting * fix: Lints, docs * fix: Adds deprecation notes * fix: Adds regex for fenix + docs * fix: Clarifies a todo * fix: missed err_msg change * feat: Adds HNT tests, fixes some review issues * fix: Some missing docs * fix: Small edits * fix: adds hnt test * fix: Build against all * fix: Small log correction * fix: some doc changes * fix: some readme linting * fix clarifies HNT acronym --- automation/build_against_all.py | 167 ++++++++ automation/build_against_fenix.py | 283 +++++++++++++ automation/build_against_hnt.py | 297 ++++++++++++++ automation/build_against_ios.py | 388 ++++++++++++++++++ automation/shared.py | 27 +- automation/smoke-test-fenix.py | 2 +- automation/smoke-test-fxios.py | 1 + ...lly-published-components-in-firefox-hnt.md | 88 ++++ docs/howtos/smoke-testing-app-services.md | 59 ++- 9 files changed, 1306 insertions(+), 6 deletions(-) create mode 100755 automation/build_against_all.py create mode 100755 automation/build_against_fenix.py create mode 100755 automation/build_against_hnt.py create mode 100755 automation/build_against_ios.py create mode 100644 docs/howtos/locally-published-components-in-firefox-hnt.md diff --git a/automation/build_against_all.py b/automation/build_against_all.py new file mode 100755 index 00000000000..a99801f48c7 --- /dev/null +++ b/automation/build_against_all.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Purpose: Run various smoke tests against this application-services working tree. +# Requirements: +# - python +# - application-services built and working. +# For the mac builds: +# - xcpretty (`gem install xcpretty`) +# - xcode + xcodebuild + xcodetools setup and running (a successful build of the firefox-ios repository) +# Arguments: +# --action => Can be either `run-tests` (default) or `build-without-testing` +# --use_local_firefox_ios => Use a local firefox-ios repository instead (at the provided path). +# --verbose => Includes the stdout of subprocesses (like the xcodebuild output, or other bootstrapping scripts) +# --allow-clears => Clear existing uniffi bindings, swift files, and so on during the various build processes. +# --use-local-firefox-ios => Use a local copy of firefox-ios instead of cloning it for iOS tests. Exclusive with `remote-ios-repo-url` +# --remote-ios-repo-url => Clone a different firefox-ios repository for iOS tests. Exclusive with `use-local-firefox-ios` +# --ios-scheme => The scheme to run for iOS tests. Likely: `Fennec` (default) or `Firefox` +# --ios-test-plan => The test plan to test with for iOS tests. Likely: `Smoketest` (default) or `FullFunctionalTestPlan` +import argparse +import time +from shared import err_msg, step_msg +from build_against_hnt import build_against_hnt +from build_against_fenix import build_against_fenix +from build_against_ios import build_against_ios + +parser = argparse.ArgumentParser( + description="Run groups of tests against this application-services working tree." +) + +group = parser.add_mutually_exclusive_group() +parser.add_argument( + "--firefox-dir", + required=True, + help="Path to existing bootstrapped `mozilla-central` directory.", +) +parser.add_argument( + "--verbose", + help="Display subprocess logs for compilation processes (off by default).", + action=argparse.BooleanOptionalAction, +) +parser.add_argument( + "--allow-clears", + help="Clear existing uniffi bindings, swift files, and so on during the various build processes (what gets cleared varies per platform test).", + action=argparse.BooleanOptionalAction, +) +parser.add_argument( + "--action", + required=True, + choices=["run-tests", "build-without-testing"], + help="Run the following action for target's test", +) + +# iOS arguments to pass down +group = parser.add_mutually_exclusive_group() +group.add_argument( + "--use-local-firefox-ios", + metavar="LOCAL_IOS_REPO_PATH", + help="Use a local copy of firefox-ios instead of cloning it for iOS tests. Exclusive with `remote-ios-repo-url`", +) +group.add_argument( + "--remote-ios-repo-url", + metavar="REMOTE_REPO_PATH", + help="Clone a different firefox-ios repository for iOS tests. Exclusive with `use-local-firefox-ios`", +) +parser.add_argument( + "--ios-scheme", + help="The scheme to run for iOS tests. Likely: `Fennec` (default) or `Firefox`", + default="Fennec", +) +parser.add_argument( + "--ios-test-plan", + help="The test plan to test with for iOS tests. Likely: `Smoketest` (default) or `FullFunctionalTestPlan`", + default="Smoketest", +) + +# HNT argument to pass down +parser.add_argument( + "--hnt-test", + help="Name of the test file to run, as if you were running `./mach test ARG`.", +) + + +# Fenix arguments to pass down +parser.add_argument( + "--prefix-ff", + help="Prefix name to pass to mozilla-central gradlew compilation to reduce the amount needing to build or test. For example: `geckoview`, `fenix`, `focus`.", + default="fenix", +) + + +args = parser.parse_args() +firefox_dir = args.firefox_dir +verbose = args.verbose if args.verbose else False +allow_clears = args.allow_clears +action = args.action + +local_firefox_ios = args.use_local_firefox_ios +remote_ios_repo_url = args.remote_ios_repo_url +ios_scheme = args.ios_scheme +ios_test_plan = args.ios_test_plan + +hnt_test = args.hnt_test + +prefix_ff = args.prefix_ff + +# Build against iOS +start_time_ios = time.time() +success_ios = build_against_ios( + local_firefox_ios, + remote_ios_repo_url, + ios_scheme, + ios_test_plan, + clear_previous_bindings=allow_clears, + clean_ios_caches=allow_clears, + verbose=verbose, + action=action, +) +time_diff_ios = time.time() - start_time_ios + +# Build against Fenix +start_time_fenix = time.time() +success_fenix = build_against_fenix( + firefox_dir, + None, + prefix_ff, + prefix_as=None, + clear_bindings=allow_clears, + verbose=verbose, + action=action, +) +time_diff_fenix = time.time() - start_time_fenix + +# Build against Desktop +start_time_hnt = time.time() +success_hnt = build_against_hnt(firefox_dir, None, True, hnt_test=hnt_test, verbose=verbose, action=action) +time_diff_hnt = time.time() - start_time_hnt + +did_tests_string = "" if action != "run-tests" else " (and tested)" +do_tests_string = "" if action != "run-tests" else " (and test)" +step_msg("Finished building. Results:") +if success_ios: + step_msg( + f"Successfully built{did_tests_string} against iOS (elapsed {time_diff_ios:.2f}s)" + ) +else: + err_msg( + f"Failed to build{do_tests_string} against iOS (elapsed {time_diff_ios:.2f}s)" + ) +if success_fenix: + step_msg( + f"Successfully built{did_tests_string} against Fenix (elapsed {time_diff_fenix:.2f}s)" + ) +else: + err_msg( + f"Failed to build{do_tests_string} against Fenix (elapsed {time_diff_fenix:.2f}s)" + ) +if success_hnt: + step_msg( + f"Successfully built{did_tests_string} against HNT (elapsed {time_diff_hnt:.2f}s)" + ) +else: + err_msg( + f"Failed to build{do_tests_string} against HNT (elapsed {time_diff_hnt:.2f}s)" + ) \ No newline at end of file diff --git a/automation/build_against_fenix.py b/automation/build_against_fenix.py new file mode 100755 index 00000000000..4d89ae2aba3 --- /dev/null +++ b/automation/build_against_fenix.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Purpose: Run Firefox fenix tests against this application-services working tree. +# https://github.com/mozilla/application-services/blob/main/docs/howtos/locally-published-components-in-fenix.md +# So, for now, we need to use an existing respository. +# +# Requirements: +# - python +# - application-services built and working. +# - a `firefox`/`mozilla-central` repository set up and working to use. +# - See: https://firefox-source-docs.mozilla.org/contributing/contribution_quickref.html +# +# Usage: ./automation/build_against_fenix.py --action build-without-testing --firefox-dir ../firefox --prefix-ff fenix --prefix-as ads-client --verbose +# +# Arguments: +# --action => Can be either `run-tests` (default) or `build-without-testing` +# --firefox-dir => Working mozilla-central directory +# https://firefox-source-docs.mozilla.org/contributing/contribution_quickref.html +# --mozconfig => Absolute path to the mozconfig file to be used. +# --prefix-ff => Prefix to be used in gradle commands to firefox repo. eg: `./gradlew fenix:assembleDebug`. For example: `geckoview`, `fenix`, `focus`." +# --prefix-as => Crate prefix to be used in gradle commands to application-services repo. eg: `./gradlew ads-client:assembleDebug`. For example: `ads-client`, `fxaclient`." +# --verbose => Includes the stdout of subprocesses (like the xcodebuild output, or other bootstrapping scripts) +# --clear-bindings => Whether or not to clear existing bindings and cached artifacts, such as with "./gradlew fenix:clean" +import argparse +import subprocess +import os +import tempfile +from pathlib import Path +import re +from shared import ( + find_app_services_root, + set_gradle_substitution_path, + step_msg, + err_msg, + run_cmd_is_successful, + dir_file_sanity_check, +) + +DEFAULT_MOZ_CONFIG_LOCATION = "mozconfig_android" +DEFAULT_MOZ_CONFIG = """ +ac_add_options --enable-project=mobile/android +""" +MOZILLA_FF_GRADLE_PROPERTIES_PATH = "gradle.properties" + + +# Replaces org.gradle.configuration-cache=true with a commented version. +def comment_gradle_cache_line(firefox_repo_path): + """ + Comments out the gradle cache line pursuant to step 2 here. + https://github.com/mozilla/application-services/blob/main/docs/howtos/locally-published-components-in-fenix.md#pre-requisites + """ + + properties_file_path = Path(firefox_repo_path) / MOZILLA_FF_GRADLE_PROPERTIES_PATH + if not os.path.isfile(properties_file_path): + err_msg( + "Could not find an instance of `gradle.properties` to modify. Please ensure the `m-c`/`firefox` directory is lined up correctly." + ) + return False + + # Uses regex. Matches the binaryTarget listed and replaces it with the following string. + replace_with = """# org.gradle.configuration-cache=true""" + step_msg(f"Writing to gradle.properties:\n{replace_with}") + regex = re.compile(r"^#*\s*org\.gradle\.configuration-cache=true\s*$", re.MULTILINE) + + with open(properties_file_path, "r+") as f: + data = f.read() + # Regex string matches this tidbit. + package_file = regex.sub(replace_with, data) + f.seek(0) + f.write(package_file) + f.truncate() + + return True + + +def build_against_fenix( + firefox_dir, + moz_config_location, + prefix_ff, + prefix_as, + clear_bindings, + verbose, + action, +): + subprocess_stdout = None if verbose else subprocess.DEVNULL + subprocess_stderr = None if verbose else subprocess.DEVNULL + + if action is None: + action = "run-tests" + + firefox_repo_path = Path(firefox_dir) + tmp_dir_path = Path(tempfile.mkdtemp(suffix="-test-fenix")) + + app_services_path = find_app_services_root() + + step_msg("Checking for sanity of application-services repository...") + if not dir_file_sanity_check( + app_services_path, "application-services", ["megazords", "components"] + ): + return False + + prefix_as_string = f"{prefix_as}:" if prefix_as else "" + prefix_ff_string = f"{prefix_ff}:" if prefix_ff else "" + + # MOZCONFIG handling. + # Idea here is that mozconfig settings (primary indicator of how firefox is built) can't be passed + # without `configure`, which is not recommended. However, we can pass test fixture mozconfig files themselves as env variables. + if moz_config_location is None: + moz_config_location = os.path.abspath( + tmp_dir_path / DEFAULT_MOZ_CONFIG_LOCATION + ) + with open(moz_config_location, "w") as file: + file.write(DEFAULT_MOZ_CONFIG) + + if not os.path.isabs(moz_config_location): + err_msg( + f"`mozconfig` path passed: `{moz_config_location}` must be an absolute path." + ) + return False + if not os.path.isfile(moz_config_location): + err_msg(f"`mozconfig` path passed: `{moz_config_location}` could not be found.") + return False + step_msg(f"Using `mozconfig` path: `{moz_config_location}`. Displaying:") + with open(moz_config_location) as f: + print(f.read()) + + # Basic sanity check here. Not remotely exhaustive, just to make sure the wrong directory wasn't passed. + step_msg("Checking for sanity of firefox repository...") + if not dir_file_sanity_check( + firefox_repo_path, + "mozilla-central", + ["mach", "CLOBBER", "gradlew", "Cargo.toml", "local.properties"], + ): + return False + + # The following steps modify several property files in the m-c repository provided: + # Key step: gradle can use a different application-services directory + step_msg(f"Configuring {firefox_repo_path} to autopublish appservices") + if not set_gradle_substitution_path( + firefox_repo_path, + "autoPublish.application-services.dir", + find_app_services_root(), + ): + err_msg( + "Failed in attempting to set `local.properties` `autoPublish.application-services.dir`" + ) + return False + + # Comments out gradle cache in local.properties + step_msg(f"Configuring {firefox_repo_path} to disable gradle configuration-cache") + if not comment_gradle_cache_line(firefox_repo_path): + err_msg( + "Failed in attempting to set `gradle.properties` `#org.gradle.configuration-cache=true`" + ) + return False + + # Environment verification check + step_msg("Verifying Android environment...") + if not run_cmd_is_successful( + "./libs/verify-android-environment.sh", + cwd=app_services_path, + shell=True, + stdout=subprocess_stdout, + stderr=subprocess_stderr, + ): + err_msg( + "Failed to run `./libs/verify-android-environment.sh` in app-services environment. Run this script and follow any instructions given until it succeeds, then try again." + ) + return False + + # Gradle clean cached files + if clear_bindings: + step_msg( + "Cleaning application-services with gradle to clear cached android bindings..." + ) + if not run_cmd_is_successful( + f"./gradlew {prefix_as_string}clean", + cwd=app_services_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg( + "Could not run ./gradlew clean. Please check to ensure the mozilla-center folder structure is sound." + ) + return False + + # Run gradle compilations and tests + step_msg("Compiling application-services with gradle to test android bindings...") + if not run_cmd_is_successful( + f"./gradlew {prefix_as_string}assembleDebug", + cwd=app_services_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg("Failed to compile application-services with gradle.") + return False + + step_msg( + f"Compiling firefox with mozconfig with `./gradlew {prefix_ff_string}assembleDebug` (mozconfig=`{moz_config_location}`)..." + ) + if not run_cmd_is_successful( + f"MOZCONFIG={moz_config_location} ./gradlew {prefix_ff_string}assembleDebug", + cwd=firefox_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg("Failed to compile firefox with gradle.") + return False + + if action == "run-tests": + step_msg( + f"Compiling firefox with mozconfig with `./gradlew {prefix_ff_string}testDebug` (mozconfig=`{moz_config_location}`)..." + ) + if not run_cmd_is_successful( + f"MOZCONFIG={moz_config_location} ./gradlew {prefix_ff_string}testDebug", + cwd=firefox_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg("Failed to run tests against firefox with gradle.") + return False + step_msg("Successfully built against Android!") + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run Firefox Android tests against this application-services working tree." + ) + + parser.add_argument( + "--verbose", + help="Display subprocess logs for compilation processes (off by default).", + action=argparse.BooleanOptionalAction, + ) + parser.add_argument( + "--action", + choices=["run-tests", "build-without-testing"], + help="Whether to run tests after the build step is complete..", + ) + parser.add_argument( + "--firefox-dir", + required=True, + help="Path to existing bootstrapped `mozilla-central` directory.", + ) + parser.add_argument( + "--mozconfig", + help="Absolute path to the desired mozconfig file. This affects the build destination, ensure it specifies android if you override it.", + ) + parser.add_argument( + "--prefix-ff", + help="Prefix name to pass to mozilla-central compilation to reduce the amount needing to build or test. For example: `geckoview`, `fenix`, `focus`.", + ) + parser.add_argument( + "--prefix-as", + help="Crate name to pass for preliminary application-services android building step. For example: `ads-client`, `fxaclient`", + ) + parser.add_argument( + "--clear-previous-bindings", + help="Clear existing uniffi binding files from the firefox android build folder. (`/gradlew clean`). This shares any prefixes supplied by `--prefix-as`. If unrelated files need to be cleared, do not pass a --prefix-as argument", + action=argparse.BooleanOptionalAction, + ) + + args = parser.parse_args() + firefox_dir = args.firefox_dir + verbose = args.verbose + moz_config_location = args.mozconfig + action = args.action + prefix_ff = args.prefix_ff + prefix_as = args.prefix_as + clear_bindings = args.clear_previous_bindings + build_against_fenix( + firefox_dir, + moz_config_location, + prefix_ff, + prefix_as, + clear_bindings, + verbose, + action, + ) diff --git a/automation/build_against_hnt.py b/automation/build_against_hnt.py new file mode 100755 index 00000000000..61d266e6a03 --- /dev/null +++ b/automation/build_against_hnt.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Purpose: Run Firefox desktop / HNT (Home & New Tab) tests against this application-services working tree. +# https://github.com/mozilla/application-services/blob/main/docs/howtos/locally-published-components-in-hnt.md +# +# Requirements: +# - python +# - application-services built and working. +# - a `firefox`/`mozilla-central` repository set up and working to use. +# - See: https://firefox-source-docs.mozilla.org/contributing/contribution_quickref.html +# Usage: ./automation/build_against_hnt.py --action build-without-testing --firefox-dir ../firefox --verbose +# and arg to clean up only? +# Arguments: +# --action => Can be either `run-tests` (default) or `build-without-testing`, or `run` (which runs it locally with `./mach run`) +# --firefox-dir => Working mozilla-central directory +# https://firefox-source-docs.mozilla.org/contributing/contribution_quickref.html +# --mozconfig => Absolute path to the mozconfig file to be used. +# --verbose => Includes the stdout of subprocesses (like the xcodebuild output, or other bootstrapping scripts) +# --clean-up => Whether to perform the on-success cleanup step at the end of a successful build (default is True). This clean-up step happens either way on an error or graceful exit (such as with `--action run`). +# --hnt-test => Test name to run with `./mach test`. If `run-tests` is attached, but no `--test` is provided, the default command will be `./mach test --auto` where appropriate tests will be guessed. +import argparse +import subprocess +import os +import signal +import tempfile +import sys +from pathlib import Path +from shared import ( + find_app_services_root, + step_msg, + err_msg, + run_cmd_is_successful, + dir_file_sanity_check, +) + +DEFAULT_MOZ_CONFIG_LOCATION = "mozconfig_desktop" +DEFAULT_MOZ_CONFIG = """ +ac_add_options --enable-project=browser +""" +MOZILLA_FF_GRADLE_PROPERTIES_PATH = "gradle.properties" +COMPONENTS_FOLDER_AS_SUBPATH = "components" +COMPONENTS_FOLDER_MC_SUBPATH = "third_party/application-services/components" +COMPONENTS_FOLDER_MC_SUBPATH_TMP = "third_party/application-services/components_tmp" + + +# Catch sigint escape (for example, for long running tests) to still safely clean up the m-z directory +def safe_exit(firefox_repo_path): + step_msg("Exit signal caught, gracefully exiting...") + clean_up_func(firefox_repo_path) + step_msg("Exiting...") + sys.exit(0) + + +# Clean up symlinks/modified files that need to revert to their previous state +# Running this doesn't indicate something went wrong +def clean_up_func(firefox_repo_path): + symlink_src = firefox_repo_path / COMPONENTS_FOLDER_MC_SUBPATH + components_tmp_dir = firefox_repo_path / COMPONENTS_FOLDER_MC_SUBPATH_TMP + step_msg("Cleaning up, restoring symlinks...") + + # Test to see if we were interrupted/ended after making the symlink + try: + if os.path.islink(symlink_src) and os.path.isdir(components_tmp_dir): + os.unlink(symlink_src) + # if symlink does not exist (or no longer does) but components was moved, move it back + if os.path.isdir(components_tmp_dir): + os.rename(components_tmp_dir, symlink_src) + except OSError: + err_msg( + "Failed to restore the m-c state. Please remove the symlink and return the 'components' directory to it's intended spot." + ) + return False + return True + + +# External function handle cleanup on failure +def build_against_hnt( + firefox_dir, + moz_config_location, + clean_up, + hnt_test, + verbose, + action, +): + # Run cleanup at start + firefox_repo_path = Path(firefox_dir) + clean_up_func(firefox_repo_path) + + # Catch sigint for graceful exit + signal.signal(signal.SIGINT, lambda _s, _h: safe_exit(firefox_repo_path)) + step_msg("Registered sigint trap...") + + success = build_against_hnt_inner( + firefox_dir, + moz_config_location, + hnt_test, + verbose, + action, + ) + + if success: + step_msg("Finished running against HNT.") + else: + err_msg("Building against HNT failed.") + + if clean_up: + step_msg("Cleaning up...") + clean_up_func(firefox_repo_path) + else: + step_msg( + "Skipping cleanup step. Rerunning the command will cleanup before recompiling." + ) + + return success + +def build_against_hnt_inner( + firefox_dir, + moz_config_location, + hnt_test, + verbose, + action, +): + subprocess_stdout = None if verbose else subprocess.DEVNULL + subprocess_stderr = None if verbose else subprocess.DEVNULL + + if action is None: + action = "run-tests" + + firefox_repo_path = Path(firefox_dir) + tmp_dir_path = Path(tempfile.mkdtemp(suffix="-test-hnt")) + + app_services_path = find_app_services_root() + + step_msg("Checking for sanity of application-services repository...") + if not dir_file_sanity_check( + app_services_path, "application-services", ["megazords", "components"] + ): + return False + + # MOZCONFIG handling. + # Idea here is that mozconfig settings (primary indicator of how firefox is built) can't be passed + # without `configure`, which is not recommended. However, we can pass test fixture mozconfig files themselves as env variables. + if moz_config_location is None: + moz_config_location = os.path.abspath( + tmp_dir_path / DEFAULT_MOZ_CONFIG_LOCATION + ) + with open(moz_config_location, "w") as file: + file.write(DEFAULT_MOZ_CONFIG) + + if not os.path.isabs(moz_config_location): + err_msg( + f"`mozconfig` path passed: `{moz_config_location}` must be an absolute path." + ) + return False + if not os.path.isfile(moz_config_location): + err_msg(f"`mozconfig` path passed: `{moz_config_location}` could not be found.") + return False + step_msg(f"Using `mozconfig` path: `{moz_config_location}`. Displaying:") + with open(moz_config_location) as f: + print(f.read()) + + # Basic sanity check here. Not remotely exhaustive, just to make sure the wrong directory wasn't passed. + step_msg("Checking for sanity of firefox repository...") + if not dir_file_sanity_check( + firefox_repo_path, + "mozilla-central", + ["mach", "CLOBBER", "gradlew", "Cargo.toml", "local.properties"], + ): + return False + + # Environment verification check + step_msg("Verifying Desktop environment...") + if not run_cmd_is_successful( + "./libs/verify-desktop-environment.sh", + cwd=app_services_path, + shell=True, + stdout=subprocess_stdout, + stderr=subprocess_stderr, + ): + err_msg( + "Failed to run `./libs/verify-android-environment.sh` in app-services environment. Run this script and follow any instructions given until it succeeds, then try again." + ) + return False + + # The following steps *modify* a couple key parts of the m-c directory. + symlink_dest = app_services_path / COMPONENTS_FOLDER_AS_SUBPATH + symlink_src = firefox_repo_path / COMPONENTS_FOLDER_MC_SUBPATH + components_tmp_dir = firefox_repo_path / COMPONENTS_FOLDER_MC_SUBPATH_TMP + step_msg(f"Creating symlink in {firefox_repo_path} to link to local appservices") + + # First, move /components folder in m-c to a temporary backup. + os.rename(symlink_src, components_tmp_dir) + + # Then, create a symlink between the app-services/components and m-c/third_party/app-services folder. + os.symlink(symlink_dest, symlink_src) + + # We are pointing to a new area as if we vendored, so we regenerate. + step_msg("Regenerating uniffi bindings (mozconfig=`{moz_config_location}`)...") + if not run_cmd_is_successful( + "./mach uniffi generate", + cwd=firefox_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg("Failed to generate uniffi bindings with: `./mach uniffi generate`/") + return False + + step_msg( + f"Compiling firefox with `./mach build` (mozconfig=`{moz_config_location}`)..." + ) + if not run_cmd_is_successful( + f"MOZCONFIG={moz_config_location} ./mach build", + cwd=firefox_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg("Failed to compile firefox with `./mach build`.") + return False + + if action == "run-tests": + step_msg( + f"Compiling firefox with mozconfig with `./mach test` (mozconfig=`{moz_config_location}`)..." + ) + test_string = hnt_test if hnt_test is not None else "--auto" + step_msg(f"Running test command `./mach test {test_string}`") + if not run_cmd_is_successful( + f"MOZCONFIG={moz_config_location} ./mach test {test_string}", + cwd=firefox_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg(f"Failed to run tests against firefox with ./mach test {test_string}.") + return False + elif action == "run": + step_msg( + f"Running firefox with mozconfig with `./mach run` (mozconfig=`{moz_config_location}`)..." + ) + if not run_cmd_is_successful( + f"MOZCONFIG={moz_config_location} ./mach run", + cwd=firefox_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg("Failed to run tests against firefox with ./mach run.") + return False + + step_msg("Successfully built against HNT!") + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run Firefox HNT tests against this application-services working tree." + ) + + parser.add_argument( + "--verbose", + help="Display subprocess logs for compilation processes (off by default).", + action=argparse.BooleanOptionalAction, + ) + parser.add_argument( + "--action", + choices=["run", "run-tests", "build-without-testing"], + help="Whether to run tests after the build step is complete..", + ) + parser.add_argument( + "--firefox-dir", + required=True, + help="Path to existing bootstrapped `mozilla-central` directory.", + ) + parser.add_argument( + "--mozconfig", + help="Absolute path to the desired mozconfig file. This affects the build destination, ensure it specifies android if you override it.", + ) + parser.add_argument( + "--hnt-test", + help="Name of the test file to run, as if you were running `./mach test ARG`.", + ) + + parser.add_argument( + "--clean-up", + help="Skip the on-success cleanup step done at the end of a successful build. This does not skip the cleanup step if there is an error or graceful exit (such as with `--action run`).", + action=argparse.BooleanOptionalAction, + default=True, + ) + + args = parser.parse_args() + firefox_dir = args.firefox_dir + verbose = args.verbose + moz_config_location = args.mozconfig + action = args.action + clean_up = args.clean_up + hnt_test = args.hnt_test + build_against_hnt(firefox_dir, moz_config_location, clean_up, hnt_test, verbose, action) diff --git a/automation/build_against_ios.py b/automation/build_against_ios.py new file mode 100755 index 00000000000..086ff5aa9fc --- /dev/null +++ b/automation/build_against_ios.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Purpose: Run Firefox-iOS tests against this application-services working tree. +# https://github.com/mozilla/application-services/blob/main/docs/howtos/locally-published-components-in-firefox-ios.md +# Requirements: +# - python +# - application-services built and working. +# - xcpretty (`gem install xcpretty`) +# - xcode + xcodebuild + xcodetools setup and running (a successful build of the firefox-ios repository) +# +# Usage: ./automation/build_against_ios.py +# Arguments: +# --action => Can be either `run-tests` (default) or `build-without-testing` +# --remote-ios-repo-url => Fetch firefox-ios repository from this URL instead. Exclusive with `use-local-firefox-ios` +# --use-local-firefox-ios => Use a local firefox-ios repository instead (at the provided path). Exclusive with `remote-ios-repo-url`. +# --verbose => Includes the stdout of subprocesses (like the xcodebuild output, or other bootstrapping scripts) +# --clear-previous-bindings => Clear existing uniffi binding swift files from both the iOS and A-S generated folders. Use if files were created that need to be cleared (eg: a file of a name that is no longer used). +# --clean-ios-caches => Runs the code equivalent of Xcode's 'Clean Build Folder' +# --ios-scheme => The XCode scheme to build with, such as 'Fennec' or 'Firefox' +# --ios-test-plan => The XCode test plan to run tests with (if action is `run-tests`), such as 'Smoketest' or 'FullFunctionalTestPlan' +# +import argparse +import subprocess +import os +import tempfile +from pathlib import Path +import shutil +import glob +import re +from shared import ( + fatal_err, + find_app_services_root, + run_cmd_checked, + step_msg, + err_msg, + run_cmd_is_successful, + dir_file_sanity_check, +) + +DEFAULT_REMOTE_REPO_URL = "https://github.com/mozilla-mobile/firefox-ios.git" +MOZILLA_RUST_COMPONENTS_IOS_PATH = "MozillaRustComponents/Package.swift" +MOZILLA_RUST_COMPONENTS_AS_PATH = "megazords/ios-rust/MozillaRustComponents.xcframework" + + +def replace_swift_package_artifact(ios_repo_path, as_repo_path): + """ + Replaces the local artifact pursuant to step 2 here. + https://github.com/mozilla/application-services/blob/main/docs/howtos/locally-published-components-in-firefox-ios.md#step-2--point-firefox-ios-to-your-local-artifact + """ + + package_file_path = Path(ios_repo_path) / MOZILLA_RUST_COMPONENTS_IOS_PATH + xcframework_file_path = Path(as_repo_path) / MOZILLA_RUST_COMPONENTS_AS_PATH + + if not os.path.isfile(package_file_path): + err_msg( + "Could not find an instance of `MozillaRustComponents/Package.swift` to modify. Please ensure the iOS directory is lined up correctly." + ) + return False + if not os.path.isdir(xcframework_file_path): + err_msg( + f"Could not find an instance of `MozillaRustComponents.xcframework` to link at {xcframework_file_path}. Please ensure the a-s directory is lined up correctly." + ) + return False + xcframework_file_path_relative = os.path.relpath( + xcframework_file_path, Path(package_file_path.parent) + ) + + # Uses regex. Matches the binaryTarget listed and replaces it with the following string. + replace_with = f""" + binaryTarget( + name: "MozillaRustComponents", + path: "{xcframework_file_path_relative}" + ) + """ + step_msg(f"Writing to MozillaRustComponents/Package.swift:\n{replace_with}") + regex = re.compile( + r"binaryTarget\([\n\s]*name\: \"MozillaRustComponents\",[\S\n\t\v ]*MozillaRustComponents\.xcframework\"[\n\s]*\)" + ) + + with open(package_file_path, "r+") as f: + data = f.read() + # Regex string matches this tidbit. + package_file = regex.sub(replace_with, data) + f.seek(0) + f.write(package_file) + f.truncate() + + return True + + +def build_against_ios( + local_ios_repo_path, + remote_ios_repo_url, + scheme, + test_plan, + clear_previous_bindings, + clean_ios_caches, + verbose, + action, +): + subprocess_stdout = None if verbose else subprocess.DEVNULL + subprocess_stderr = None if verbose else subprocess.DEVNULL + + if action is None: + action = "run-tests" + + ios_repo_path = local_ios_repo_path + app_services_path = find_app_services_root() + + # Naive sanity check here. Not remotely exhaustive, just to make sure some extremely incorrect directory wasn't passed. + step_msg("Checking for sanity of application-services repository...") + if not dir_file_sanity_check( + app_services_path, "application-services", ["megazords", "components"] + ): + return False + + step_msg("Checking for existence of xcodebuild...") + if not run_cmd_is_successful("xcodebuild -version", cwd=ios_repo_path, shell=True): + err_msg( + "xcodebuild is required to compile application-services for iOS. Please clone the firefox-ios repository and follow the instructions therein." + ) + return False + + # Creating temp directory and cloning repository + step_msg(f"Building application-services against iOS with action: `{action}`") + if local_ios_repo_path is None: + ios_repo_path = tempfile.mkdtemp(suffix="-test-ios") + if remote_ios_repo_url is None: + remote_ios_repo_url = DEFAULT_REMOTE_REPO_URL + step_msg(f"Cloning {remote_ios_repo_url}") + run_cmd_checked(["git", "clone", remote_ios_repo_url, ios_repo_path]) + + ios_generated_uniffi_files_path = f"{ios_repo_path}/MozillaRustComponents/Sources/MozillaRustComponentsWrapper/Generated" + local_repo_generated_uniffi_files_path = f"{app_services_path}/megazords/ios-rust/Sources/MozillaRustComponentsWrapper/Generated" + + # Bootstrapping the iOS repository + step_msg("Running the firefox-ios bootstrap script...") + if not run_cmd_is_successful( + "./bootstrap.sh", cwd=ios_repo_path, shell=True, stdout=subprocess_stdout + ): + err_msg( + "Failed to bootstrap firefox-ios repository. Please clone the firefox-ios repository and follow the instructions therein." + ) + return False + + # Verification check + step_msg("Verifying iOS environment...") + if not run_cmd_is_successful( + "./libs/verify-ios-environment.sh", + cwd=app_services_path, + shell=True, + stdout=subprocess_stdout, + stderr=subprocess_stderr, + ): + err_msg( + "Failed to verify environment for iOS. Please run `./libs/verify-ios-environment.sh`, making suggested changes until it succeeds." + ) + return False + + # Uniffi sanity check + if not os.path.isdir(ios_generated_uniffi_files_path): + err_msg( + f"Expected path `{ios_generated_uniffi_files_path}` in firefox-ios is missing. Please confirm the repository structure or try cloning it again." + ) + return False + + if clear_previous_bindings: + step_msg("'clear-previous-bindings' is set, clearing uniffi folders") + + # Clear the uniffi bindings in the A-S repository as extra files created are not deleted + # Not relevant to tmp repository + if os.path.isdir(local_repo_generated_uniffi_files_path): + for p in Path(local_repo_generated_uniffi_files_path).glob("*.swift"): + p.unlink() + + # Clear equivalents from the ios repository + # (Relevant if we are using an existing directory) + if not run_cmd_is_successful( + ["git", "checkout", "."], cwd=ios_generated_uniffi_files_path + ): + fatal_err( + "Found an error running git commands to clear previous uniffi folders. Exiting." + ) + if not run_cmd_is_successful( + ["git", "clean", "-f"], cwd=ios_generated_uniffi_files_path + ): + fatal_err( + "Found an error running git commands to clear previous uniffi folders. Exiting." + ) + + # Build artifacts + # Unfortunately build_ios_artifacts is writing to stderr, so we need to hide it when it's not verbose. + step_msg("Building application-services iOS artifacts...") + if not run_cmd_is_successful( + "./automation/build_ios_artifacts.sh", + cwd=app_services_path, + shell=True, + check=True, + stdout=subprocess_stdout, + stderr=subprocess_stderr, + ): + err_msg( + "Failed to build ios artifacts. Please ensure the code compiles and try running `./automation/build_ios_artifacts.sh` from the folder." + ) + return False + + # Replace with regex some data in the swift package file + # https://github.com/mozilla/application-services/blob/main/docs/howtos/locally-published-components-in-firefox-ios.md#step-2--point-firefox-ios-to-your-local-artifact + if not replace_swift_package_artifact(ios_repo_path, app_services_path): + err_msg("Failed to point firefox-ios to local a-s xcframework artifact") + return False + + if not os.path.isdir(local_repo_generated_uniffi_files_path): + err_msg( + f"Expected path `{local_repo_generated_uniffi_files_path}` in application-services is missing after building. Please confirm the repository structure or try cloning it again." + ) + return False + + # Copies folder of uniffi bindings. + step_msg("Copying uniffi bindings from:") + step_msg( + f"{local_repo_generated_uniffi_files_path} -> {ios_generated_uniffi_files_path}" + ) + for file in glob.glob("*.swift", root_dir=local_repo_generated_uniffi_files_path): + shutil.copy( + f"{local_repo_generated_uniffi_files_path}/{file}", + ios_generated_uniffi_files_path, + ) + + # Remove the glean_sym file. + # https://github.com/mozilla/application-services/blob/main/docs/howtos/locally-published-components-in-firefox-ios.md + step_msg(f"Removing: {ios_generated_uniffi_files_path}/glean_sym.swift") + os.remove(f"{ios_generated_uniffi_files_path}/glean_sym.swift") + + # Clean packages in xcodebuild + scheme = "Fennec" if scheme is None else scheme + test_plan = "Smoketest" if test_plan is None else test_plan + if clean_ios_caches: + # TODO: "Reset package caches" part not done yet. Currently not a great script way to do it seemingly other + # than deleting ~/Library files, so needs examination for a code or xcodebuild based solution. + + # Clean build folder + step_msg("Cleaning build folder...") + if not run_cmd_is_successful( + f"""\ + set -o pipefail && \ + xcodebuild \ + -workspace ./firefox-ios/Client.xcodeproj/project.xcworkspace \ + -scheme {scheme} \ + clean | \ + xcpretty + """, + cwd=ios_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg( + "Failed to clean and compile tests on iOS", + ) + return False + + # Run the build action + if action == "build-without-testing": + step_msg("Running xcodebuild without testing (this may take a few minutes)...") + if not run_cmd_is_successful( + f"""\ + set -o pipefail && \ + xcodebuild \ + -workspace ./firefox-ios/Client.xcodeproj/project.xcworkspace \ + -scheme {scheme} \ + -destination 'platform=iOS Simulator,name=iPhone 17' \ + build-for-testing | \ + xcpretty + """, + cwd=ios_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg( + "Failed to compile and run tests on iOS", + ) + return False + elif action == "run-tests": + step_msg( + "Building firefox-ios and running tests (this may take a few minutes)..." + ) + if not run_cmd_is_successful( + f"""\ + set -o pipefail && \ + xcodebuild \ + -workspace ./firefox-ios/Client.xcodeproj/project.xcworkspace \ + -scheme {scheme} \ + -destination 'platform=iOS Simulator,name=iPhone 17' \ + -testPlan {test_plan} \ + test | \ + xcpretty + """, + cwd=ios_repo_path, + shell=True, + stdout=subprocess_stdout, + ): + err_msg("Failed to compile and run tests on iOS") + return False + + else: + err_msg( + "You must either run `--action run-tests` or `--action build-without-testing` " + ) + return False + + step_msg("Successfully built against iOS!") + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run Firefox-iOS tests against this application-services working tree." + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--use-local-firefox-ios", + metavar="LOCAL_IOS_REPO_PATH", + help="Use a local copy of firefox-ios instead of cloning it. Exclusive with `remote-ios-repo-url`", + ) + group.add_argument( + "--remote-ios-repo-url", + metavar="REMOTE_REPO_PATH", + help="Clone a different firefox-ios repository. Exclusive with `use-local-firefox-ios`", + ) + + parser.add_argument( + "--verbose", + help="Display subprocess logs for compilation processes (off by default).", + action=argparse.BooleanOptionalAction, + ) + parser.add_argument( + "--clear-previous-bindings", + help="Clear existing uniffi binding swift files from both the iOS and A-S generated folders. Use if files were created that need to be cleared (eg: a file of a name that is no longer used).", + action=argparse.BooleanOptionalAction, + ) + + parser.add_argument( + "--clean-ios-caches", + help="Run Xcode 'Clean Build Folder'", + action=argparse.BooleanOptionalAction, + ) + + parser.add_argument( + "--ios-scheme", + help="The scheme to run. Likely: `Fennec` (default) or `Firefox`", + default="Fennec", + ) + + parser.add_argument( + "--ios-test-plan", + help="The test plan to test with. Likely: `Smoketest` (default) or `FullFunctionalTestPlan`", + default="Smoketest", + ) + + parser.add_argument( + "--action", + choices=["run-tests", "build-without-testing"], + help="Run the following action once firefox-ios is set up.", + ) + + args = parser.parse_args() + local_ios_repo_path = args.use_local_firefox_ios + remote_ios_repo_url = args.remote_ios_repo_url + clear_previous_bindings = args.clear_previous_bindings + clean_ios_caches = args.clean_ios_caches + scheme = args.ios_scheme + test_plan = args.ios_test_plan + verbose = args.verbose + action = args.action + + build_against_ios( + local_ios_repo_path, + remote_ios_repo_url, + scheme, + test_plan, + clear_previous_bindings, + clean_ios_caches, + verbose, + action, + ) diff --git a/automation/shared.py b/automation/shared.py index 3a490e7cb23..11d6a629948 100644 --- a/automation/shared.py +++ b/automation/shared.py @@ -15,15 +15,21 @@ def step_msg(msg): def fatal_err(msg): - print(f"\033[31mError: {msg}\033[0m") + err_msg(msg) sys.exit(1) +def err_msg(msg): + print(f"\033[31mError: {msg}\033[0m") def run_cmd_checked(*args, **kwargs): """Run a command, throwing an exception if it exits with non-zero status.""" kwargs["check"] = True return subprocess.run(*args, **kwargs) # noqa: PLW1510 +def run_cmd_is_successful(*args, **kwargs): + """Run a subprocess command, returning False if it exits with non-zero status (True otherwise).""" + return subprocess.run(*args, **kwargs).returncode == 0 + def check_output(*args, **kwargs): """Run a command, throwing an exception if it exits with non-zero status.""" @@ -44,6 +50,18 @@ def find_app_services_root(): return cur_dir.absolute() +def dir_file_sanity_check(directory_path, dir_name, example_file_names): + """ + Extremely rudimentary and naive check for a few basic files in a directory, for better handling if the wrong directory is passed. + """ + for example_file in example_file_names: + new_path = Path(directory_path) / example_file + if not os.path.isfile(new_path) and not os.path.isdir(new_path): + err_msg(f"`{example_file}` is missing in root of `{dir_name}` directory. Please confirm this is a valid local copy of `{dir_name}` at: {directory_path}") + return False + return True + + def get_moz_remote(): """ Get the name of the remote for the official mozilla application-services repo @@ -69,6 +87,8 @@ def set_gradle_substitution_path(project_dir, name, value): If the named property already exists with the correct value then it will silently succeed; if the named property already exists with a different value then it will noisily fail. + + Returns False on such a failure, otherwise returns True. """ project_dir = Path(project_dir).resolve() properties_file = project_dir / "local.properties" @@ -86,7 +106,9 @@ def set_gradle_substitution_path(project_dir, name, value): fatal_err( f"Conflicting property {name}={cur_value} (not {abs_value})" ) - return + return False + else: + return True # The file does not contain the required property, append it. # Note that the project probably expects a path relative to the project root. ancestor = Path(os.path.commonpath([project_dir, abs_value])) @@ -98,6 +120,7 @@ def set_gradle_substitution_path(project_dir, name, value): step_msg(f"Setting relative path from {project_dir} to {abs_value} as {relpath}") with properties_file.open("a") as f: f.write(f"{name}={relpath}\n") + return True class RefNames: diff --git a/automation/smoke-test-fenix.py b/automation/smoke-test-fenix.py index c3afffb76e2..55ea1f123b1 100755 --- a/automation/smoke-test-fenix.py +++ b/automation/smoke-test-fenix.py @@ -3,7 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. - +# DEPRECATED. Please consider using ./build_against_fenix.py # Purpose: Run Fenix tests against this application-services working tree. # Usage: ./automation/smoke-test-fenix.py diff --git a/automation/smoke-test-fxios.py b/automation/smoke-test-fxios.py index eb8df811832..246266a916e 100755 --- a/automation/smoke-test-fxios.py +++ b/automation/smoke-test-fxios.py @@ -3,6 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +# DEPRECATED. Please consider using ./build_against_ios.py # Purpose: Run Firefox-iOS tests against this application-services working tree. # Usage: ./automation/smoke-test-fxios.py diff --git a/docs/howtos/locally-published-components-in-firefox-hnt.md b/docs/howtos/locally-published-components-in-firefox-hnt.md new file mode 100644 index 00000000000..1a22df7bcff --- /dev/null +++ b/docs/howtos/locally-published-components-in-firefox-hnt.md @@ -0,0 +1,88 @@ +# How to locally test application-services components on HNT / Desktop + +> This guide explains how to build and test **HNT against a local Application Services** checkout. + +--- + +## At a glance + +**Goal:** Build a local Firefox Desktop against a local Application Services. + +**Current workflow (recommended):** + +1. Verify the local build of `A-S` is ready for a desktop build. +2. Move the A-S components folder in M-C/Firefox to a temporary rename. +3. Create a symlink between the A-S and M-C components folders. +4. Generate uniffi bindings. +5. Run and build. +6. Cleanup of symlink and temporary renames. + +--- + +## Prerequisites + +1. Ensure you have a regular [build of application-services working](../building.md). +2. Ensure you have a regular [build of firefox from mozilla-central](https://firefox-source-docs.mozilla.org/setup/index.html#for-firefox-desktop) testable with `./mach build` and `./mach run`. + +--- + +## Step 1 — Verify the local build of A-S is ready for a desktop build + +From the root of your `application-services` checkout, execute: + +```bash +./libs/verify-desktop-environment.sh +``` + +This will check for environment variables. If it provides any instruction on environment variables to set, follow the instructions until it passes. + +## Step 2 - Move the A-S components folder in M-C/Firefox to a temporary rename + +We will be temporarily replacing the components in the `application-services` repository in `mozilla-central` with a symlink that points to our local `application-services` build. To conserve the old folder, we temporarily rename it. From the **mozilla-central** root. + +```bash +mv third_party/application-services/components third_party/application-services/components-tmp +``` + +## Step 3 - Create a symlink between the A-S and M-C components folders. + +Now, the former `components` path should have a symlink to the local `application-services` components. Assuming `application-services` is in the same folder as your `mozilla-central` checkout, you can run (from the **mozilla-central** root): + +```bash +ln -s $(realpath ../application-services/components) third_party/application-services/components +``` + +## Step 4 - Generate uniffi bindings. + +You may need to regenerate uniffi bindings, as if you vendored new `A-S` code. From the mozilla-central root: + +```bash +./mach uniffi generate +``` + +## Step 5 - Run and build! + +Now that `components` will read from your local build, you can build, run, and test. From your local m-c checkout, run: + +```bash +./mach build +``` + +And if so desired: + +```bash +./mach run +``` + +## Step 6 - Cleanup + +After completing your tests, you should revert your files and symlinks to ensure `m-c` continues to behave as expected: + +```bash +unlink third_party/application-services/components +mv third_party/application-services/components-tmp third_party/application-services/components +``` + +## Automated testing + +You can also automate this process by running the Desktop smoke test found at `automation/build_against_hnt.py`. You can see more detailed instructions about this [at the smoke testing guide](./smoke-testing-app-services.md). diff --git a/docs/howtos/smoke-testing-app-services.md b/docs/howtos/smoke-testing-app-services.md index a2c0a8f9dd6..59c0fff277e 100644 --- a/docs/howtos/smoke-testing-app-services.md +++ b/docs/howtos/smoke-testing-app-services.md @@ -7,17 +7,70 @@ The testing can be done manually using substitution scripts, but we also have sc Run `pip3 install -r automation/requirements.txt` to install the required Python packages. -## Android Components +## Usage + +You can easily run a smoke test against iOS and Fenix by running the following: + +`./automation/build_against_all.py --firefox-dir ../firefox --action build-without-testing --allow-clears` + +- In this case, `firefox-dir` must point to a bootstrapped and working installation of `mozilla-central` ([see instructions here](https://firefox-source-docs.mozilla.org/contributing/contribution_quickref.html)). It is used for the compilation and test of the Android and HNT builds. + +- The `--action` argument can be either `build-without-testing` or `run-tests`. + +- The `--allow-clears` argument allows the various subscripts to clear their various caches as appropriate (such as XCode cleaning iOS caches). + +You can also run against specific platforms directly with the following examples: + +- iOS: + + ```./automation/build_against_ios.py --action build-without-testing --clear-previous-bindings --clean-ios-caches --use-local-firefox-ios ../firefox-ios``` + + - By default, this script creates a `tmp` directory for `firefox-ios` and uses it. If you have a running `firefox-ios`, you can add the optional argument `--use-local-firefox-ios ../firefox-ios` (as shown), which may result in speed gains and the ability to run it on XCode immediately after a failure. This can also be passed to `./build_against_all.py`. + + - You can also customize the run scheme (`--ios-scheme`) or test plan (`--ios-test-plan`). Available schemes include `Fennec` (default) and `Firefox`. Available test plans include: `Smoketest`, `FullFunctionalTestPlan` and `UnitTest`. + + - A full list of schemes and their corresponding test plans can be found [in the firefox-ios](https://github.com/mozilla-mobile/firefox-ios/tree/main/firefox-ios/Client.xcodeproj/xcshareddata/xcschemes) respository. + + - Both of these arguments can be used in `./build_against_all.py`, and they will be passed to the underlying `build_against_ios.py` call. + +- Fenix: + + ```./automation/build_against_fenix.py --action build-without-testing --firefox-dir ../firefox --prefix-ff fenix --clear-previous-bindings``` + + - The `--prefix-ff` argument here refers to the prefix passed to commands like `./gradlew fenix:assembleDebug`. It can be omitted, but may cause failures on non-fenix projects. + + - The `--clear-previous-bindings` argument here runs a `./gradlew prefix:clear` before recompiling. It is not always necessary, and can be excluded for some speed gains, but can result in some cache reuse. + +- Desktop / HNT (Home & New Tab): + + ```./automation/build_against_hnt.py --action build-without-testing --firefox-dir ../firefox``` + + - This script uses the symlink method described [at the local A-S against HNT tutorial](./locally-published-components-in-firefox-hnt.md), cleaning up after a successful run. You can avoid this cleanup process (specifically on a successful run) by passing `--no-clean-up`, which will keep the symlinks. For example, you might run with `--action build-without-testing --no-clean-up` to experiment after with `./mach run`. + + - Unlike the other tests, HNT has the additional `action` variant of `--action run`, because it can be run from the terminal directly. + + - You can pass `--hnt-test` (eg: `--hnt-test dom/notification`) to pass a set of tests to use if the `action` argument is `run-tests`. Otherwise, `./mach test --auto` will be used, which takes a guess at which tests would be best to run. You can pass `--hnt-test` to the `build_against_all.py` script as well. + +All test scripts also accept the `--verbose` argument to show the output of run subprocesses (such as `./mach build`). + +## Limitations + +Note that these tests are primarily smoke tests against the building and compilation of application-services. There are a wide array of possible regressions that can only be caught with tests, including ones that crash the build immediately on running. To ensure any regressions for your component are caught, tests should be created for them rather than just building. + +## Deprecated tests + +### Android Components The `automation/smoke-test-android-components.py` script will clone (or use a local version) of android-components and run a subset of its tests against the current `application-services` worktree. It tries to only run tests that might be relevant to `application-services` functionality. -## Fenix +### Fenix The `automation/smoke-test-fenix.py` script will clone (or use a local version) of Fenix and run tests against the current `application-services` worktree. -## Firefox iOS +### Firefox iOS + The `automation/smoke-test-fxios.py` script will clone (or use a local version) of Firefox iOS and run tests against the current `application-services` worktree. From 4ddf0ad91993c5e2e2d83c51e4d43489868524e9 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Tue, 4 Aug 2026 17:00:20 +0200 Subject: [PATCH 46/59] fix(remote-settings): SYNC-5384: Do not quote timestamps with v2 API (#7523) * fix(remote-settings): Do not quote timestamps with v2 API * Update CHANGELOG --- CHANGELOG.md | 1 + components/remote_settings/src/client.rs | 36 +++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab9ce100007..e43ac9ba1c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Remote Settings - Replacing v1 routes with v2 routes, removing added v2 routes ([#7492](https://github.com/mozilla/application-services/pull/7339)) - Verify signature of imported data when `.get()` is called with `sync_if_empty: true` ([#7518](https://github.com/mozilla/application-services/pull/7518)) +- Do not quote `_since` values with the v2 API ([#7523](https://github.com/mozilla/application-services/pull/7523)) # v154.0 (_2026-07-20_) diff --git a/components/remote_settings/src/client.rs b/components/remote_settings/src/client.rs index 760cbbbe696..5880c4495fa 100644 --- a/components/remote_settings/src/client.rs +++ b/components/remote_settings/src/client.rs @@ -685,7 +685,7 @@ impl ApiClient for ViaductApiClient { url.query_pairs_mut().append_pair("_expected", "0"); if let Some(timestamp) = timestamp { url.query_pairs_mut() - .append_pair("_since", &format!("\"{}\"", timestamp)); + .append_pair("_since", &format!("{}", timestamp)); } let resp = self.make_request(url)?; @@ -962,6 +962,40 @@ mod test_new_client { } } +#[cfg(test)] +mod viaduct_client_tests { + use super::*; + + #[test] + fn test_fetch_uses_local_timestamp_as_unquoted_since() { + viaduct_dev::init_backend_dev(); + let changeset = mockito::mock( + "GET", + "/v2/buckets/main/collections/test-collection/changeset", + ) + // The mock only matches if `_since` is sent unquoted. + .match_query(mockito::Matcher::AllOf(vec![ + mockito::Matcher::UrlEncoded("_expected".into(), "0".into()), + mockito::Matcher::UrlEncoded("_since".into(), "42".into()), + ])) + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"changes": [], "timestamp": 42, "metadata": {"bucket": "main", "signatures": []}}"#, + ) + .create(); + + let mut api_client = ViaductApiClient::new( + BaseUrl::parse(&format!("{}/v2", mockito::server_url())).unwrap(), + "main", + "test-collection", + ); + api_client.fetch_changeset(Some(42)).unwrap(); + + changeset.assert(); + } +} + #[cfg(test)] mod jexl_tests { use super::*; From a3bfd05b386358436ecd781f425ab5c083ba8cd9 Mon Sep 17 00:00:00 2001 From: bendk Date: Wed, 5 Aug 2026 09:35:57 -0400 Subject: [PATCH 47/59] Fix grafana dashboard datasources (#7522) For some reason the JSON stopped working when imported. Adding an extra `datasource` field fixes this. --- tools/generate-rust-dashboards/src/metrics/sync.rs | 1 + tools/generate-rust-dashboards/src/schema.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tools/generate-rust-dashboards/src/metrics/sync.rs b/tools/generate-rust-dashboards/src/metrics/sync.rs index 51d4f81c089..ebc66eefc07 100644 --- a/tools/generate-rust-dashboards/src/metrics/sync.rs +++ b/tools/generate-rust-dashboards/src/metrics/sync.rs @@ -387,6 +387,7 @@ WHERE fn sync_legacy_dashboard_panel() -> Panel { let content = "\ # Legacy Sync dashboards +* [Sync: engine performance](https://sql.telemetry.mozilla.org/dashboard/sync-engine-performance?p_w73231_Days=7&p_w73233_Days=7&p_w73234_Days=7&p_w73237_Days=7&p_w73238_Days=60&p_w73239_Days=60&p_w73248_Days=60&p_w73249_Days=60&p_w73250_Days=60&p_w73251_Days=60&p_w73255_Days=7&p_w73256_Days=7&p_w73257_Days=60) * [Desktop Sync Failures](https://sql.telemetry.mozilla.org/dashboard/sync-desktop?p_Days=60&p_engine_name=all-engines&p_w63728_engine_name=all-engines&p_w64027_engine_name=all-engines&p_w64028_engine_name=all-engines&p_w64029_engine_name=all-engines&p_w65780_channel=beta&p_w65780_days=30&p_w65780_engine_name=all-engines) * [Android Sync Failures](https://sql.telemetry.mozilla.org/dashboard/android-sync-failures?p_channel=org_mozilla_fenix&p_engine_name=credit-cards&p_w64121_Months=24&p_w64121_engine_name=all-engines&p_w64122_Months=24&p_w64122_engine_name=all-engines&p_w64123_Months=24&p_w64123_engine_name=all-engines&p_w73261_Months=1&p_w73261_engine=bookmarks&p_w73261_minimum_error_count=0&p_w73262_Months=1&p_w73262_engine=bookmarks&p_w73262_minimum_error_count=0&p_w73263_Months=1&p_w73263_engine=bookmarks&p_w73263_minimum_error_count=0) * [iOS Sync failures](https://sql.telemetry.mozilla.org/dashboard/ios-sync-failures?p_Days=60&p_Months=1&p_engine_name=all-engines&p_w67318_Months=1&p_w67318_engine=bookmarks&p_w67318_minimum%20error%20count=0&p_w67320_Months=1&p_w67320_engine=bookmarks&p_w67320_minimum%20error%20count=0) diff --git a/tools/generate-rust-dashboards/src/schema.rs b/tools/generate-rust-dashboards/src/schema.rs index 8e9ab5db3e3..42b4df538f2 100644 --- a/tools/generate-rust-dashboards/src/schema.rs +++ b/tools/generate-rust-dashboards/src/schema.rs @@ -217,6 +217,7 @@ pub struct PieChartReduceOptions { #[derive(Default, Serialize)] #[serde(rename_all = "camelCase")] pub struct Target { + pub datasource: Datasource, pub format: TargetFormat, pub raw_query: bool, pub raw_sql: String, @@ -677,6 +678,7 @@ impl Datasource { impl Target { pub fn timeseries(sql: impl Into) -> Self { Self { + datasource: Datasource::bigquery(), format: TargetFormat::Timeseries, raw_query: true, raw_sql: sql.into(), @@ -685,6 +687,7 @@ impl Target { pub fn table(sql: impl Into) -> Self { Self { + datasource: Datasource::bigquery(), format: TargetFormat::Table, raw_query: true, raw_sql: sql.into(), From c4ea7bd04ba8cd3f3c3446858894c0f2650d6521 Mon Sep 17 00:00:00 2001 From: Ryan VanderMeulen Date: Wed, 5 Aug 2026 11:56:07 -0400 Subject: [PATCH 48/59] Update toolchain paths for clang-22 (#7525) --- taskcluster/scripts/toolchain/cross-compile-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/scripts/toolchain/cross-compile-setup.sh b/taskcluster/scripts/toolchain/cross-compile-setup.sh index 9b6bdae46e0..15b94520310 100755 --- a/taskcluster/scripts/toolchain/cross-compile-setup.sh +++ b/taskcluster/scripts/toolchain/cross-compile-setup.sh @@ -6,12 +6,12 @@ export PATH=$PATH:/builds/worker/clang/bin export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_NSS_STATIC=1 export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_NSS_DIR=/builds/worker/checkouts/vcs/libs/desktop/darwin/nss # x86_64 Darwin -export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_CC=/builds/worker/clang/bin/clang-21 +export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_CC=/builds/worker/clang/bin/clang-22 export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_TOOLCHAIN_PREFIX=/builds/worker/cctools/bin export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_AR=/builds/worker/cctools/bin/x86_64-apple-darwin-ar export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_RANLIB=/builds/worker/cctools/bin/x86_64-apple-darwin-ranlib export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_LD_LIBRARY_PATH=/builds/worker/clang/lib -export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C linker=/builds/worker/clang/bin/clang-21 -C link-arg=-fuse-ld=/builds/worker/cctools/bin/x86_64-apple-darwin-ld -C link-arg=-B -C link-arg=/builds/worker/cctools/bin -C link-arg=-target -C link-arg=x86_64-apple-darwin -C link-arg=-isysroot -C link-arg=/tmp/MacOSX11.0.sdk -C link-arg=-Wl,-syslibroot,/tmp/MacOSX11.0.sdk -C link-arg=-Wl,-dead_strip" +export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C linker=/builds/worker/clang/bin/clang-22 -C link-arg=-fuse-ld=/builds/worker/cctools/bin/x86_64-apple-darwin-ld -C link-arg=-B -C link-arg=/builds/worker/cctools/bin -C link-arg=-target -C link-arg=x86_64-apple-darwin -C link-arg=-isysroot -C link-arg=/tmp/MacOSX11.0.sdk -C link-arg=-Wl,-syslibroot,/tmp/MacOSX11.0.sdk -C link-arg=-Wl,-dead_strip" # For ring's use of `cc`. export ORG_GRADLE_PROJECT_RUST_ANDROID_GRADLE_TARGET_X86_64_APPLE_DARWIN_CFLAGS_x86_64_apple_darwin="-B /builds/worker/cctools/bin -target x86_64-apple-darwin -isysroot /tmp/MacOSX11.0.sdk -Wl,-syslibroot,/tmp/MacOSX11.0.sdk -Wl,-dead_strip" # Pass bindgen a `--sysroot` argument so that it can find the include files when cross-compiling. From 32f011df40185eb4a1d52f092293c22a685ccc74 Mon Sep 17 00:00:00 2001 From: DimiDL <55685831+DimiDL@users.noreply.github.com> Date: Thu, 6 Aug 2026 09:42:38 +0000 Subject: [PATCH 49/59] Add address metadata APIs and a bridged sync engine to autofill (#7513) Firefox Desktop is moving its address storage to this component and needs two things that are not currently exposed. Storage APIs for importing records already persisted elsewhere - add_address_with_meta, add_many_addresses_with_meta, update_address_with_meta and add_many_address_tombstones - taking a caller-supplied guid, timestamps and change counter. The bulk variants run each record in a savepoint so one bad record neither aborts the migration nor leaves a row behind. Carrying the change counter means update_internal_address now takes a CounterUpdate of Increment, Leave or Set(i64). A bridged sync engine, so Desktop's Sync framework can drive address sync: Store::addresses_bridged_engine() exposes the address engine the crate already has, which only implemented sync15::SyncEngine and so was unreachable from Desktop's mozIBridgedSyncEngine. Nothing on mobile calls any of this; it is additive for the Desktop migration. --- CHANGELOG.md | 2 + components/autofill/src/autofill.udl | 108 ++++ components/autofill/src/db/addresses.rs | 491 +++++++++++++++++- components/autofill/src/db/models/address.rs | 51 ++ components/autofill/src/db/store.rs | 89 +++- components/autofill/src/error.rs | 11 + components/autofill/src/lib.rs | 1 + .../autofill/src/sync/address/incoming.rs | 12 +- components/autofill/src/sync/bridge.rs | 183 +++++++ components/autofill/src/sync/engine.rs | 16 +- components/autofill/src/sync/mod.rs | 2 + 11 files changed, 952 insertions(+), 14 deletions(-) create mode 100644 components/autofill/src/sync/bridge.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e43ac9ba1c5..895b8ea255c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ ### Autofill - Add `Store::shutdown()`, which closes the database connection early so it happens before Firefox Desktop's late-write shutdown barrier rather than during GC. Operations after shutdown return `DatabaseClosed`. ([Bug 2050036](https://bugzilla.mozilla.org/show_bug.cgi?id=2050036)) +- Add address metadata APIs for importing records already persisted elsewhere: `add_address_with_meta`, `add_many_addresses_with_meta`, `update_address_with_meta` and `add_many_address_tombstones`, with the bulk variants isolating per-record failures. `AddressMeta` carries the guid, timestamps and `sync_change_counter`, so a record keeps whether it still has changes pending upload. +- Add `Store::addresses_bridged_engine()`, exposing the existing address sync engine through `mozIBridgedSyncEngine` so Firefox Desktop can drive address sync. ### Nimbus diff --git a/components/autofill/src/autofill.udl b/components/autofill/src/autofill.udl index 841758bd17c..a9c6e3c5d42 100644 --- a/components/autofill/src/autofill.udl +++ b/components/autofill/src/autofill.udl @@ -107,6 +107,48 @@ dictionary Passport { i64 times_used; }; +/// Metadata fields managed internally by the library: the guid, timestamps and +/// local sync state. These are automatically set on `add_address` and updated on +/// operations like `touch` and `update_address`. Not included in +/// `UpdatableAddressFields`; use `add_address_with_meta` when importing records +/// that already have metadata. +dictionary AddressMeta { + string guid; + i64 time_created; + i64? time_last_used; + i64 time_last_modified; + i64 times_used; + i64 sync_change_counter; +}; + +/// An address together with its metadata, passed to `add_address_with_meta` and +/// `update_address_with_meta` when importing a record from another store. +dictionary UpdatableAddressFieldsWithMeta { + UpdatableAddressFields fields; + AddressMeta meta; +}; + +/// A bulk insert result entry, returned per input record by `add_many_addresses_with_meta` +[Enum] +interface AddressBulkResultEntry { + Success(Address address); + Error(string message); +}; + +/// A tombstone for a record deleted locally but not yet uploaded, supplied to +/// `add_many_address_tombstones` when migrating from another store. +dictionary AddressTombstone { + string guid; + i64 time_deleted; +}; + +/// Per-record result of `add_many_address_tombstones`. +[Enum] +interface AddressBulkTombstoneResultEntry { + Success(string guid); + Error(string message); +}; + /// Metrics tracking scrubbing of credit cards that cannot be decrypted, see // `scrub_undecryptable_credit_card_data_for_remote_replacement` for more details dictionary CreditCardsDeletionMetrics { @@ -150,6 +192,19 @@ interface Store { [Throws=AutofillApiError] Address add_address(UpdatableAddressFields a); + [Throws=AutofillApiError] + Address add_address_with_meta(UpdatableAddressFieldsWithMeta entry_with_meta); + + [Throws=AutofillApiError] + sequence add_many_addresses_with_meta(sequence entries_with_meta); + + [Throws=AutofillApiError] + sequence add_many_address_tombstones(sequence tombstones); + + /// Removes every address and every address tombstone. + [Throws=AutofillApiError] + void delete_all_addresses(); + [Throws=AutofillApiError] Address get_address(string guid); @@ -162,6 +217,9 @@ interface Store { [Throws=AutofillApiError] void update_address(string guid, UpdatableAddressFields a); + [Throws=AutofillApiError] + void update_address_with_meta(UpdatableAddressFieldsWithMeta entry_with_meta); + [Throws=AutofillApiError] boolean delete_address(string guid); @@ -212,4 +270,54 @@ interface Store { [Self=ByArc] void register_with_sync_manager(); + + /// Returns a bridged sync engine for addresses, for use by Desktop's Sync + /// framework. Constructing it only assembles structs and never touches the + /// DB, so it cannot fail. + [Self=ByArc] + AddressesBridgedEngine addresses_bridged_engine(); +}; + +/// The bridged sync engine for addresses. The canonical docs are in +/// services/interfaces/mozIBridgedSyncEngine.idl. +/// NOTE: all timestamps here are milliseconds. +interface AddressesBridgedEngine { + [Throws=AutofillApiError] + i64 last_sync(); + + [Throws=AutofillApiError] + void set_last_sync(i64 last_sync); + + [Throws=AutofillApiError] + string? sync_id(); + + [Throws=AutofillApiError] + string reset_sync_id(); + + [Throws=AutofillApiError] + string ensure_current_sync_id([ByRef]string new_sync_id); + + [Throws=AutofillApiError] + void prepare_for_sync([ByRef]string client_data); + + [Throws=AutofillApiError] + void sync_started(); + + [Throws=AutofillApiError] + void store_incoming(sequence incoming_envelopes_as_json); + + [Throws=AutofillApiError] + sequence apply(); + + [Throws=AutofillApiError] + void set_uploaded(i64 new_timestamp, sequence uploaded_ids); + + [Throws=AutofillApiError] + void sync_finished(); + + [Throws=AutofillApiError] + void reset(); + + [Throws=AutofillApiError] + void wipe(); }; diff --git a/components/autofill/src/db/addresses.rs b/components/autofill/src/db/addresses.rs index 431c33d2bb9..3f6b5ac72c2 100644 --- a/components/autofill/src/db/addresses.rs +++ b/components/autofill/src/db/addresses.rs @@ -5,7 +5,9 @@ use crate::db::{ models::{ - address::{InternalAddress, UpdatableAddressFields}, + address::{ + AddressMeta, InternalAddress, UpdatableAddressFields, UpdatableAddressFieldsWithMeta, + }, Metadata, }, schema::{ADDRESS_COMMON_COLS, ADDRESS_COMMON_VALS}, @@ -48,6 +50,182 @@ pub(crate) fn add_address( Ok(address) } +/// Adds an address **including metadata**, taking the guid, timestamps and sync +/// change counter from the caller rather than generating them. Normally you will +/// use `add_address` instead; this is for importing records from another store +/// that already have metadata. +pub(crate) fn add_address_with_meta( + conn: &Connection, + fields: UpdatableAddressFields, + meta: AddressMeta, +) -> Result { + let tx = conn.unchecked_transaction()?; + let address = internal_address_from_meta(fields, &meta); + add_internal_address(&tx, &address)?; + tx.commit()?; + Ok(address) +} + +/// Adds multiple addresses **including metadata** within a single transaction. +/// Each record gets its own result, so a record that fails to insert is reported +/// as `Err(message)` without aborting the rest of the batch. +pub(crate) fn add_many_addresses_with_meta( + conn: &Connection, + entries: Vec, +) -> Result>> { + let tx = conn.unchecked_transaction()?; + let mut results = Vec::with_capacity(entries.len()); + for entry in entries { + let address = internal_address_from_meta(entry.fields, &entry.meta); + match with_savepoint(&tx, || add_internal_address(&tx, &address))? { + Ok(()) => results.push(Ok(address)), + Err(e) => results.push(Err(e.to_string())), + } + } + tx.commit()?; + Ok(results) +} + +/// Runs `op` in a savepoint, rolling back to it if `op` fails, so that a record +/// reported as an error by the bulk functions leaves nothing behind. The shared +/// triggers reject a guid that exists in the counterpart table with +/// `RAISE(FAIL)`, which aborts the statement but keeps the row it already +/// inserted - so without this the offending row would be committed along with +/// the rest of the batch, putting the guid in both `addresses_data` and +/// `addresses_tombstones`. +/// +/// The outer `Result` is a savepoint failure and aborts the batch; the inner one +/// is the record's own failure. +fn with_savepoint( + tx: &Transaction<'_>, + op: impl FnOnce() -> Result, +) -> Result> { + tx.execute_batch("SAVEPOINT bulk_record")?; + match op() { + Ok(value) => { + tx.execute_batch("RELEASE bulk_record")?; + Ok(Ok(value)) + } + Err(e) => { + tx.execute_batch("ROLLBACK TO bulk_record; RELEASE bulk_record")?; + Ok(Err(e)) + } + } +} + +/// Removes every address and every address tombstone, in one transaction. +/// +/// Deleting the rows alone is not enough. A delete leaves a tombstone behind for +/// any guid the sync mirror knows, and the insert trigger then rejects re-adding +/// that guid, so a wipe that kept them could not be followed by a re-import of +/// the same records. Clearing both tables is what makes the wipe repeatable. +pub(crate) fn delete_all_addresses(conn: &Connection) -> Result<()> { + let tx = conn.unchecked_transaction()?; + tx.execute("DELETE FROM addresses_data", [])?; + // After the data, so the tombstones the delete trigger just created go too. + tx.execute("DELETE FROM addresses_tombstones", [])?; + tx.commit()?; + Ok(()) +} + +/// Adds tombstones for records that were deleted locally but not yet uploaded, +/// within a single transaction and with a result per record. `time_deleted` comes +/// from the caller rather than being stamped as now, so that a deletion imported +/// from another store keeps its original time. Without the tombstone the next +/// sync has nothing to say the record was deleted and takes the server copy. +pub(crate) fn add_many_address_tombstones( + conn: &Connection, + tombstones: Vec<(String, i64)>, +) -> Result>> { + let tx = conn.unchecked_transaction()?; + let mut results = Vec::with_capacity(tombstones.len()); + for (guid, time_deleted) in tombstones { + let inserted = with_savepoint(&tx, || { + tx.execute( + "INSERT INTO addresses_tombstones (guid, time_deleted) + VALUES (:guid, :time_deleted)", + rusqlite::named_params! { + ":guid": &guid, + ":time_deleted": timestamp_from_millis(time_deleted), + }, + )?; + Ok(()) + })?; + match inserted { + Ok(()) => results.push(Ok(guid)), + Err(e) => results.push(Err(e.to_string())), + } + } + tx.commit()?; + Ok(results) +} + +/// `Timestamp` is a `u64`, so a negative millisecond value would wrap to a huge +/// one and then win every "latest wins" comparison in `Metadata::merge`. Clamp to +/// 0, which already means "unset" for these fields. The tuple constructor is used +/// rather than `Timestamp::from`, which asserts non-zero. +fn timestamp_from_millis(millis: i64) -> Timestamp { + Timestamp(millis.max(0) as u64) +} + +fn internal_address_from_meta( + fields: UpdatableAddressFields, + meta: &AddressMeta, +) -> InternalAddress { + InternalAddress { + guid: Guid::new(&meta.guid), + name: fields.name, + organization: fields.organization, + street_address: fields.street_address, + address_level3: fields.address_level3, + address_level2: fields.address_level2, + address_level1: fields.address_level1, + postal_code: fields.postal_code, + country: fields.country, + tel: fields.tel, + email: fields.email, + metadata: Metadata { + time_created: timestamp_from_millis(meta.time_created), + time_last_used: timestamp_from_millis(meta.time_last_used.unwrap_or(0)), + time_last_modified: timestamp_from_millis(meta.time_last_modified), + times_used: meta.times_used, + sync_change_counter: meta.sync_change_counter, + }, + } +} + +/// Updates an address **including metadata**, setting both its fields and its +/// timestamps and `times_used` to the supplied values. Normally you will use +/// `update_address` instead, which owns the metadata itself; this is for keeping +/// a record identical to one held in another store. Errors with `NoSuchRecord` +/// if the guid is absent. +pub(crate) fn update_address_with_meta( + conn: &Connection, + fields: UpdatableAddressFields, + meta: AddressMeta, +) -> Result<()> { + let tx = conn.unchecked_transaction()?; + + let address = internal_address_from_meta(fields, &meta); + // Checked up front because `update_internal_address` asserts on the number + // of rows changed rather than returning an error. + let exists: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM addresses_data WHERE guid = :guid)", + rusqlite::named_params! { ":guid": address.guid }, + |row| row.get(0), + )?; + if !exists { + return Err(Error::NoSuchRecord(address.guid.to_string())); + } + update_internal_address( + &tx, + &address, + CounterUpdate::Set(address.metadata.sync_change_counter), + )?; + tx.commit()?; + Ok(()) +} + pub(crate) fn add_internal_address(tx: &Transaction<'_>, address: &InternalAddress) -> Result<()> { tx.execute( &format!( @@ -165,17 +343,42 @@ pub(crate) fn update_address( Ok(()) } +/// How `update_internal_address` should treat the change counter. +pub(crate) enum CounterUpdate { + /// Record a local change awaiting upload. + Increment, + /// Leave the counter alone, for a change that must not be uploaded - eg one + /// applied by Sync, which is already what the server has. + Leave, + /// Replace the counter, for a record whose counter is owned by the caller. + Set(i64), +} + +impl CounterUpdate { + /// The SQL assigned to `sync_change_counter`, and the value bound to + /// `:counter` within it. `Leave` adds 0 rather than dropping `:counter` from + /// the SQL, because rusqlite rejects a named parameter the statement doesn't + /// use. + fn as_sql(&self) -> (&'static str, i64) { + match self { + Self::Increment => ("sync_change_counter + :counter", 1), + Self::Leave => ("sync_change_counter + :counter", 0), + Self::Set(counter) => (":counter", *counter), + } + } +} + /// Updates all fields including metadata - although the change counter gets -/// slightly special treatment (eg, when called by Sync we don't want the -/// change counter incremented) +/// slightly special treatment, see `CounterUpdate`. pub(crate) fn update_internal_address( tx: &Transaction<'_>, address: &InternalAddress, - flag_as_changed: bool, + counter: CounterUpdate, ) -> Result<()> { - let change_counter_increment = flag_as_changed as u32; // will be 1 or 0 + let (counter_sql, counter_value) = counter.as_sql(); let rows_changed = tx.execute( - "UPDATE addresses_data SET + &format!( + "UPDATE addresses_data SET name = :name, organization = :organization, street_address = :street_address, @@ -190,8 +393,9 @@ pub(crate) fn update_internal_address( time_last_used = :time_last_used, time_last_modified = :time_last_modified, times_used = :times_used, - sync_change_counter = sync_change_counter + :change_incr - WHERE guid = :guid", + sync_change_counter = {counter_sql} + WHERE guid = :guid" + ), rusqlite::named_params! { ":name": address.name, ":organization": address.organization, @@ -207,7 +411,7 @@ pub(crate) fn update_internal_address( ":time_last_used": address.metadata.time_last_used, ":time_last_modified": address.metadata.time_last_modified, ":times_used": address.metadata.times_used, - ":change_incr": change_counter_increment, + ":counter": counter_value, ":guid": address.guid, }, )?; @@ -513,7 +717,7 @@ mod tests { email: "".to_string(), ..Default::default() }, - false, + CounterUpdate::Leave, )?; let record_exists: bool = tx.query_row( @@ -670,4 +874,271 @@ mod tests { Ok(()) } + + fn test_fields(street_address: &str) -> UpdatableAddressFields { + UpdatableAddressFields { + name: "jane doe".to_string(), + street_address: street_address.to_string(), + address_level2: "Seattle, WA".to_string(), + country: "United States".to_string(), + ..UpdatableAddressFields::default() + } + } + + fn test_meta(guid: &str, sync_change_counter: i64) -> AddressMeta { + AddressMeta { + guid: guid.to_string(), + time_created: 1000, + time_last_used: Some(2000), + time_last_modified: 3000, + times_used: 4, + sync_change_counter, + } + } + + #[test] + fn test_address_add_with_meta() -> Result<()> { + let db = new_mem_db(); + + let saved = + add_address_with_meta(&db, test_fields("123 Main Street"), test_meta("abc", 2))?; + + // the supplied guid is used rather than a fresh one being generated. + assert_eq!(saved.guid.as_str(), "abc"); + + let retrieved = get_address(&db, &Guid::new("abc"))?; + assert_eq!(retrieved.street_address, "123 Main Street"); + assert_eq!(retrieved.metadata.time_created.as_millis(), 1000); + assert_eq!(retrieved.metadata.time_last_used.as_millis(), 2000); + assert_eq!(retrieved.metadata.time_last_modified.as_millis(), 3000); + assert_eq!(retrieved.metadata.times_used, 4); + assert_eq!(retrieved.metadata.sync_change_counter, 2); + + Ok(()) + } + + #[test] + fn test_address_add_with_meta_clamps_negative_timestamps() -> Result<()> { + let db = new_mem_db(); + + let meta = AddressMeta { + guid: "abc".to_string(), + time_created: -1, + time_last_used: Some(-1), + time_last_modified: -1, + times_used: 0, + sync_change_counter: 0, + }; + add_address_with_meta(&db, test_fields("123 Main Street"), meta)?; + + let retrieved = get_address(&db, &Guid::new("abc"))?; + assert_eq!(retrieved.metadata.time_created.as_millis(), 0); + assert_eq!(retrieved.metadata.time_last_used.as_millis(), 0); + assert_eq!(retrieved.metadata.time_last_modified.as_millis(), 0); + + Ok(()) + } + + #[test] + fn test_address_update_with_meta_keeps_supplied_counter() -> Result<()> { + let db = new_mem_db(); + + add_address_with_meta(&db, test_fields("123 Main Street"), test_meta("abc", 0))?; + + // the supplied counter must be applied, not the one already in the row. + update_address_with_meta(&db, test_fields("456 Second Avenue"), test_meta("abc", 1))?; + + let retrieved = get_address(&db, &Guid::new("abc"))?; + assert_eq!(retrieved.street_address, "456 Second Avenue"); + assert_eq!(retrieved.metadata.sync_change_counter, 1); + + // and back down again. + update_address_with_meta(&db, test_fields("456 Second Avenue"), test_meta("abc", 0))?; + assert_eq!( + get_address(&db, &Guid::new("abc"))? + .metadata + .sync_change_counter, + 0 + ); + + Ok(()) + } + + #[test] + fn test_address_update_with_meta_errors_when_missing() -> Result<()> { + let db = new_mem_db(); + + let result = + update_address_with_meta(&db, test_fields("123 Main Street"), test_meta("abc", 3)); + assert!(matches!(result, Err(Error::NoSuchRecord(guid)) if guid == "abc")); + assert!(get_address(&db, &Guid::new("abc")).is_err()); + + Ok(()) + } + + #[test] + fn test_address_add_many_with_meta_isolates_failures() -> Result<()> { + let db = new_mem_db(); + + // the second entry has an empty guid, which the `addresses_data` CHECK + // constraint rejects. The others must still be inserted. + let results = add_many_addresses_with_meta( + &db, + vec![ + UpdatableAddressFieldsWithMeta { + fields: test_fields("1 First Street"), + meta: test_meta("aaa", 1), + }, + UpdatableAddressFieldsWithMeta { + fields: test_fields("2 Second Street"), + meta: test_meta("", 1), + }, + UpdatableAddressFieldsWithMeta { + fields: test_fields("3 Third Street"), + meta: test_meta("ccc", 1), + }, + ], + )?; + + assert_eq!(results.len(), 3); + assert!(results[0].is_ok()); + assert!(results[1].is_err()); + assert!(results[2].is_ok()); + + assert_eq!(get_all_addresses(&db)?.len(), 2); + assert_eq!(get_address(&db, &Guid::new("aaa"))?.metadata.times_used, 4); + + Ok(()) + } + + #[test] + fn test_delete_all_addresses_allows_a_reimport() -> Result<()> { + let db = new_mem_db(); + + // A tombstone left by an earlier import, and a record sharing no guid + // with it. + add_many_address_tombstones(&db, vec![("gone".to_string(), 1234)])?; + let address = add_address(&db, UpdatableAddressFields::default())?; + + delete_all_addresses(&db)?; + assert_eq!(get_all_addresses(&db)?.len(), 0); + let tombstones: i64 = + db.query_row("SELECT COUNT(*) FROM addresses_tombstones", [], |row| { + row.get(0) + })?; + assert_eq!(tombstones, 0, "tombstones are cleared with the records"); + + // The point of clearing them: re-importing the same guids succeeds, + // where the insert trigger would reject a guid still tombstoned. + let results = add_many_addresses_with_meta( + &db, + vec![ + UpdatableAddressFieldsWithMeta { + fields: UpdatableAddressFields::default(), + meta: AddressMeta { + guid: address.guid.to_string(), + ..Default::default() + }, + }, + UpdatableAddressFieldsWithMeta { + fields: UpdatableAddressFields::default(), + meta: AddressMeta { + guid: "gone".to_string(), + ..Default::default() + }, + }, + ], + )?; + assert!( + results.iter().all(|r| r.is_ok()), + "a previously tombstoned guid can be re-imported: {results:?}" + ); + + Ok(()) + } + + #[test] + fn test_address_add_many_tombstones() -> Result<()> { + let db = new_mem_db(); + + let results = add_many_address_tombstones(&db, vec![("aaa".to_string(), 1234)])?; + assert_eq!(results.len(), 1); + assert!(results[0].is_ok()); + + // the supplied deletion time is used rather than being stamped as now. + let time_deleted: i64 = db.query_row( + "SELECT time_deleted FROM addresses_tombstones WHERE guid = 'aaa'", + [], + |row| row.get(0), + )?; + assert_eq!(time_deleted, 1234); + + Ok(()) + } + + #[test] + fn test_address_add_many_tombstones_rejects_live_guid() -> Result<()> { + let db = new_mem_db(); + + add_address_with_meta(&db, test_fields("123 Main Street"), test_meta("abc", 0))?; + + // a guid cannot be in both `addresses_data` and `addresses_tombstones`; + // the trigger enforcing that must not take the rest of the batch down. + let results = add_many_address_tombstones( + &db, + vec![("abc".to_string(), 1234), ("ddd".to_string(), 5678)], + )?; + + assert_eq!(results.len(), 2); + assert!(results[0].is_err()); + assert!(results[1].is_ok()); + + // the rejected tombstone must not have been committed anyway - see + // `with_savepoint`. + assert_eq!(count_tombstones(&db, "abc")?, 0); + assert!(get_address(&db, &Guid::new("abc")).is_ok()); + assert_eq!(count_tombstones(&db, "ddd")?, 1); + + Ok(()) + } + + #[test] + fn test_address_add_many_with_meta_rejects_deleted_guid() -> Result<()> { + let db = new_mem_db(); + + add_many_address_tombstones(&db, vec![("aaa".to_string(), 1234)])?; + + // the other side of the same invariant: a guid in + // `addresses_tombstones` cannot be inserted into `addresses_data`. + let results = add_many_addresses_with_meta( + &db, + vec![ + UpdatableAddressFieldsWithMeta { + fields: test_fields("1 First Street"), + meta: test_meta("aaa", 1), + }, + UpdatableAddressFieldsWithMeta { + fields: test_fields("2 Second Street"), + meta: test_meta("bbb", 1), + }, + ], + )?; + + assert_eq!(results.len(), 2); + assert!(results[0].is_err()); + assert!(results[1].is_ok()); + + assert!(get_address(&db, &Guid::new("aaa")).is_err()); + assert_eq!(get_all_addresses(&db)?.len(), 1); + + Ok(()) + } + + fn count_tombstones(conn: &Connection, guid: &str) -> Result { + Ok(conn.query_row( + "SELECT COUNT(*) FROM addresses_tombstones WHERE guid = :guid", + rusqlite::named_params! { ":guid": guid }, + |row| row.get(0), + )?) + } } diff --git a/components/autofill/src/db/models/address.rs b/components/autofill/src/db/models/address.rs index e526f278b7c..fa226739c0e 100644 --- a/components/autofill/src/db/models/address.rs +++ b/components/autofill/src/db/models/address.rs @@ -27,6 +27,57 @@ pub struct UpdatableAddressFields { pub email: String, } +/// Metadata fields managed internally by the library: the guid, timestamps and +/// local sync state. These are automatically set on `add_address` and updated on +/// operations like `touch` and `update_address`. Not included in +/// `UpdatableAddressFields`; use `add_address_with_meta` when importing records +/// that already have metadata. +#[derive(Debug, Clone, Default)] +pub struct AddressMeta { + pub guid: String, + pub time_created: i64, + pub time_last_used: Option, + pub time_last_modified: i64, + pub times_used: i64, + /// Local changes not yet uploaded; 0 means it matches what was last synced. + pub sync_change_counter: i64, +} + +/// A tombstone for a record deleted locally but not yet uploaded, supplied to +/// `add_many_address_tombstones` when migrating from another store. +#[derive(Debug, Clone, Default)] +pub struct AddressTombstone { + pub guid: String, + pub time_deleted: i64, +} + +/// Per-record result of `add_many_address_tombstones`. +#[derive(Debug)] +pub enum AddressBulkTombstoneResultEntry { + Success { guid: String }, + Error { message: String }, +} + +/// An address together with its metadata, passed to `add_address_with_meta` and +/// `update_address_with_meta` when importing a record from another store. +#[derive(Debug, Clone, Default)] +pub struct UpdatableAddressFieldsWithMeta { + pub fields: UpdatableAddressFields, + pub meta: AddressMeta, +} + +/// A bulk insert result entry, returned per input record by +/// `add_many_addresses_with_meta` so that one record failing does not abort the +/// batch. Note that although the success case is much larger than the error +/// case, this is negligible in real life, as we expect a very small +/// success/error ratio. +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum AddressBulkResultEntry { + Success { address: Address }, + Error { message: String }, +} + // "Address" is what we return to consumers and has most of the metadata. #[derive(Debug, Clone, Hash, PartialEq, Eq, Default)] pub struct Address { diff --git a/components/autofill/src/db/store.rs b/components/autofill/src/db/store.rs index 1ff3cbb07bb..4bb7d1cd257 100644 --- a/components/autofill/src/db/store.rs +++ b/components/autofill/src/db/store.rs @@ -2,7 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use crate::db::models::address::{Address, UpdatableAddressFields}; +use crate::db::models::address::{ + Address, AddressBulkResultEntry, AddressBulkTombstoneResultEntry, AddressTombstone, + UpdatableAddressFields, UpdatableAddressFieldsWithMeta, +}; use crate::db::models::credit_card::{CreditCard, UpdatableCreditCardFields}; use crate::db::models::passport::{Passport, UpdatablePassportFields}; use crate::db::{ @@ -133,6 +136,73 @@ impl Store { Ok(addresses::add_address(&self.lock_db()?.writer, new_address)?.into()) } + /// Adds an address **including metadata**. Normally you will use + /// `add_address` instead, and the metadata (guid, timestamps, change counter) + /// will be taken care of here. However, in some cases this method is + /// necessary, for example when migrating data from another store that + /// already contains the metadata. + #[handle_error(Error)] + pub fn add_address_with_meta( + &self, + entry_with_meta: UpdatableAddressFieldsWithMeta, + ) -> ApiResult
{ + Ok(addresses::add_address_with_meta( + &self.lock_db()?.writer, + entry_with_meta.fields, + entry_with_meta.meta, + )? + .into()) + } + + /// Adds multiple addresses **including metadata**, with a result per record. + #[handle_error(Error)] + pub fn add_many_addresses_with_meta( + &self, + entries_with_meta: Vec, + ) -> ApiResult> { + let results = + addresses::add_many_addresses_with_meta(&self.lock_db()?.writer, entries_with_meta)?; + Ok(results + .into_iter() + .map(|result| match result { + Ok(address) => AddressBulkResultEntry::Success { + address: address.into(), + }, + Err(message) => AddressBulkResultEntry::Error { message }, + }) + .collect()) + } + + /// Adds tombstones for addresses whose deletion has not yet been uploaded, + /// with a result per record. + #[handle_error(Error)] + pub fn add_many_address_tombstones( + &self, + tombstones: Vec, + ) -> ApiResult> { + let results = addresses::add_many_address_tombstones( + &self.lock_db()?.writer, + tombstones + .into_iter() + .map(|t| (t.guid, t.time_deleted)) + .collect(), + )?; + Ok(results + .into_iter() + .map(|result| match result { + Ok(guid) => AddressBulkTombstoneResultEntry::Success { guid }, + Err(message) => AddressBulkTombstoneResultEntry::Error { message }, + }) + .collect()) + } + + /// Removes every address and every address tombstone. + #[handle_error(Error)] + pub fn delete_all_addresses(&self) -> ApiResult<()> { + addresses::delete_all_addresses(&self.lock_db()?.writer)?; + Ok(()) + } + #[handle_error(Error)] pub fn get_address(&self, guid: String) -> ApiResult
{ Ok(addresses::get_address(&self.lock_db()?.writer, &Guid::new(&guid))?.into()) @@ -158,6 +228,23 @@ impl Store { addresses::update_address(&self.lock_db()?.writer, &Guid::new(&guid), &address) } + /// Updates an address **including metadata**, setting both its fields and + /// its timestamps and `times_used` to the supplied values. Normally you will + /// use `update_address` instead, which leaves `time_last_modified` to this + /// store; this is for keeping a record identical to one held elsewhere. + /// Errors with `NoSuchRecord` if the guid is absent. + #[handle_error(Error)] + pub fn update_address_with_meta( + &self, + entry_with_meta: UpdatableAddressFieldsWithMeta, + ) -> ApiResult<()> { + addresses::update_address_with_meta( + &self.lock_db()?.writer, + entry_with_meta.fields, + entry_with_meta.meta, + ) + } + #[handle_error(Error)] pub fn delete_address(&self, guid: String) -> ApiResult { addresses::delete_address(&self.lock_db()?.writer, &Guid::new(&guid)) diff --git a/components/autofill/src/error.rs b/components/autofill/src/error.rs index d45c095206f..5eab83daba6 100644 --- a/components/autofill/src/error.rs +++ b/components/autofill/src/error.rs @@ -31,6 +31,17 @@ pub enum AutofillApiError { UnexpectedAutofillApiError { reason: String }, } +// The `sync15` BridgedEngine traits use `anyhow::Result`, so the bridged engine +// in `sync::bridge` needs those errors mapped onto the public error type before +// UniFFI can expose its methods. +impl From for AutofillApiError { + fn from(value: anyhow::Error) -> Self { + AutofillApiError::UnexpectedAutofillApiError { + reason: value.to_string(), + } + } +} + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("Error opening database: {0}")] diff --git a/components/autofill/src/lib.rs b/components/autofill/src/lib.rs index ebbe668680b..5eae09c76e6 100644 --- a/components/autofill/src/lib.rs +++ b/components/autofill/src/lib.rs @@ -21,6 +21,7 @@ use crate::db::models::credit_card::*; use crate::db::models::passport::*; use crate::db::store::Store; use crate::encryption::{create_autofill_key, decrypt_string, encrypt_string}; +pub use crate::sync::AddressesBridgedEngine; pub use error::{ApiResult, AutofillApiError, Error, Result}; uniffi::include_scaffolding!("autofill"); diff --git a/components/autofill/src/sync/address/incoming.rs b/components/autofill/src/sync/address/incoming.rs index 117c55eebd1..c58c4b08dc9 100644 --- a/components/autofill/src/sync/address/incoming.rs +++ b/components/autofill/src/sync/address/incoming.rs @@ -4,7 +4,7 @@ */ use super::AddressPayload; -use crate::db::addresses::{add_internal_address, update_internal_address}; +use crate::db::addresses::{add_internal_address, update_internal_address, CounterUpdate}; use crate::db::models::address::InternalAddress; use crate::db::schema::ADDRESS_COMMON_COLS; use crate::error::*; @@ -309,7 +309,15 @@ impl ProcessIncomingRecordImpl for IncomingAddressesImpl { new_record: Self::Record, flag_as_changed: bool, ) -> Result<()> { - update_internal_address(tx, &new_record, flag_as_changed)?; + update_internal_address( + tx, + &new_record, + if flag_as_changed { + CounterUpdate::Increment + } else { + CounterUpdate::Leave + }, + )?; Ok(()) } diff --git a/components/autofill/src/sync/bridge.rs b/components/autofill/src/sync/bridge.rs new file mode 100644 index 00000000000..88a831b8d7e --- /dev/null +++ b/components/autofill/src/sync/bridge.rs @@ -0,0 +1,183 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::db::models::address::InternalAddress; +use crate::sync::engine::ConfigSyncEngine; +use crate::Store; +use anyhow::Result; +use std::sync::Arc; +use sync15::engine::BridgedEngineAdaptor; + +impl Store { + /// Returns a bridged sync engine for addresses, for use by Desktop's Sync + /// framework. Constructing a `ConfigSyncEngine` only assembles structs and + /// never touches the DB, so this cannot fail. + pub fn addresses_bridged_engine(self: Arc) -> Arc { + let engine = crate::sync::address::create_engine(self); + Arc::new(AddressesBridgedEngine::new(Box::new( + AddressesBridgedEngineAdaptor { engine }, + ))) + } +} + +/// `ConfigSyncEngine` implements `sync15::SyncEngine`, which is what the sync +/// manager drives. Desktop instead speaks `mozIBridgedSyncEngine`, whose Rust +/// shape is `sync15::BridgedEngine`. The two differ only in that the bridge owns +/// the last-sync timestamp explicitly, so this adaptor supplies that and the +/// blanket `impl BridgedEngine for A` provides the rest. +struct AddressesBridgedEngineAdaptor { + engine: ConfigSyncEngine, +} + +impl BridgedEngineAdaptor for AddressesBridgedEngineAdaptor { + fn last_sync(&self) -> Result { + Ok(self.engine.get_last_sync_millis()?) + } + + fn set_last_sync(&self, last_sync_millis: i64) -> Result<()> { + self.engine.set_last_sync_millis(last_sync_millis)?; + Ok(()) + } + + fn engine(&self) -> &dyn sync15::engine::SyncEngine { + &self.engine + } +} + +// Generates the UniFFI-exposed `AddressesBridgedEngine`, a newtype around +// `sync15::engine::BridgedEngineWrapper`. The UDL's `set_uploaded` takes +// `sequence`, hence the `String` id type. +sync15::uniffi_bridged_engine!(AddressesBridgedEngine, String); + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::models::address::UpdatableAddressFields; + use std::collections::HashMap; + + // Exercises the sync metadata the bridge owns: last_sync, sync_id and reset. + #[test] + fn test_sync_meta() { + error_support::init_for_tests(); + + let store = Arc::new(Store::new_shared_memory("addresses-bridge").unwrap()); + let bridge = store.addresses_bridged_engine(); + + // Fresh DB: never synced. + assert_eq!(bridge.last_sync().unwrap(), 0); + bridge.set_last_sync(3).unwrap(); + assert_eq!(bridge.last_sync().unwrap(), 3); + + assert!(bridge.sync_id().unwrap().is_none()); + + bridge.ensure_current_sync_id("some_guid").unwrap(); + assert_eq!(bridge.sync_id().unwrap(), Some("some_guid".to_string())); + // changing the sync ID resets the timestamp + assert_eq!(bridge.last_sync().unwrap(), 0); + bridge.set_last_sync(3).unwrap(); + + bridge.reset_sync_id().unwrap(); + assert_ne!(bridge.sync_id().unwrap(), Some("some_guid".to_string())); + assert_eq!(bridge.last_sync().unwrap(), 0); + bridge.set_last_sync(3).unwrap(); + + // `reset` clears the guid and the timestamp. + bridge.reset().unwrap(); + assert_eq!(bridge.last_sync().unwrap(), 0); + assert!(bridge.sync_id().unwrap().is_none()); + } + + // A roundtrip through the bridge's data path: stage an incoming remote + // address, apply it, and confirm the local-only address comes back out for + // upload. Unlike `test_sync_meta` this exercises the JSON (de)serialization + // of BSOs and the sync staging tables, mirroring the logins and tabs + // `test_sync_via_bridge` tests. + #[test] + fn test_sync_via_bridge() { + error_support::init_for_tests(); + + let store = Arc::new(Store::new_shared_memory("addresses-bridge-roundtrip").unwrap()); + + // A local-only address: nothing on the server knows about it yet, so it + // should be uploaded. + let local = store + .add_address(UpdatableAddressFields { + name: "Local Person".to_string(), + street_address: "1 Local Lane".to_string(), + address_level2: "Seattle, WA".to_string(), + country: "US".to_string(), + ..Default::default() + }) + .expect("should add local address"); + + let bridge = store.clone().addresses_bridged_engine(); + + // `prepare_for_sync` is what creates the sync staging tables; the client + // data it is given is unused by this engine. + bridge + .prepare_for_sync(r#"{"local_client_id":"my-client","recent_clients":{}}"#) + .expect("should prepare for sync"); + bridge.sync_started().unwrap(); + + // An incoming remote address that isn't known locally. We build the + // envelope as raw JSON, exactly as the JS bridge hands it to us. + let incoming = vec![serde_json::json!({ + "id": "remote-only-bbbb", + "modified": 0, + "payload": serde_json::json!({ + "id": "remote-only-bbbb", + "entry": { + "name": "Remote Person", + "street-address": "99 Remote Road", + "address-level2": "Portland, OR", + "country": "US", + "version": 1, + }, + }) + .to_string(), + }) + .to_string()]; + bridge + .store_incoming(incoming) + .expect("should store incoming"); + + // Applying stores the remote record locally and returns the local-only + // address for upload. + let outgoing = bridge.apply().expect("should apply"); + let changes: HashMap = outgoing + .into_iter() + .map(|s| { + let bso: serde_json::Value = serde_json::from_str(&s).unwrap(); + let payload: serde_json::Value = + serde_json::from_str(bso["payload"].as_str().unwrap()).unwrap(); + (payload["id"].as_str().unwrap().to_string(), payload) + }) + .collect(); + + // Only the local address is outgoing; the just-applied remote one is not + // re-uploaded. + assert_eq!(changes.len(), 1); + assert_eq!( + changes[&local.guid]["entry"]["street-address"], + "1 Local Lane" + ); + + // The incoming remote address was actually persisted. + let stored = store + .get_address("remote-only-bbbb".to_string()) + .expect("remote address should have been stored"); + assert_eq!(stored.street_address, "99 Remote Road"); + + // `apply` deliberately stamps last_sync with 0 - Desktop applies without + // telling us the server timestamp and sends it separately afterwards. + assert_eq!(bridge.last_sync().unwrap(), 0); + bridge.set_uploaded(1234, vec![local.guid.clone()]).unwrap(); + bridge.sync_finished().unwrap(); + assert_eq!(bridge.last_sync().unwrap(), 1234); + + // Acknowledging the upload cleared the record's change counter, so a + // subsequent sync has nothing to send. + assert!(bridge.apply().expect("should apply again").is_empty()); + } +} diff --git a/components/autofill/src/sync/engine.rs b/components/autofill/src/sync/engine.rs index 7abc9815881..24b3e3fcaab 100644 --- a/components/autofill/src/sync/engine.rs +++ b/components/autofill/src/sync/engine.rs @@ -29,7 +29,8 @@ pub const GLOBAL_SYNCID_META_KEY: &str = "global_sync_id"; pub const COLLECTION_SYNCID_META_KEY: &str = "sync_id"; // A trait to abstract the broader sync processes. -pub trait SyncEngineStorageImpl { +// Send + Sync is required to use a `ConfigSyncEngine` as a `BridgedEngine`. +pub trait SyncEngineStorageImpl: Send + Sync { fn get_incoming_impl( &self, enc_key: &Option, @@ -74,6 +75,19 @@ impl ConfigSyncEngine { let key = format!("{}.{}", self.config.namespace, tail); crate::db::store::delete_meta(conn, &key) } + /// The last-sync timestamp in milliseconds, 0 if never synced. + pub(crate) fn get_last_sync_millis(&self) -> Result { + let db = self.store.lock_db()?; + Ok(self + .get_meta::(&db.writer, LAST_SYNC_META_KEY)? + .unwrap_or_default()) + } + + pub(crate) fn set_last_sync_millis(&self, millis: i64) -> Result<()> { + let db = self.store.lock_db()?; + self.put_meta(&db.writer, LAST_SYNC_META_KEY, &millis) + } + // Reset the local sync data so the next server request fetches all records. pub fn reset_local_sync_data(&self) -> Result<()> { let db = self.store.lock_db()?; diff --git a/components/autofill/src/sync/mod.rs b/components/autofill/src/sync/mod.rs index 9cc60ed47b8..e19c7845408 100644 --- a/components/autofill/src/sync/mod.rs +++ b/components/autofill/src/sync/mod.rs @@ -4,6 +4,8 @@ */ pub mod address; +mod bridge; +pub use bridge::AddressesBridgedEngine; mod common; pub mod credit_card; pub mod engine; From e06058b3d1c56eedb2d5b1f843b966352dd7a5d3 Mon Sep 17 00:00:00 2001 From: Nicolas Qiu Guichard Date: Thu, 6 Aug 2026 15:34:41 +0200 Subject: [PATCH 50/59] Bug 2055621 - viaduct: only link to sqlite when the ohttp feature is enabled. (#7527) Bug 2054009 added a dependency from viaduct to rusqlite, which broke the nimbus-fml toolchain build because it doesn't have the x86_64-linux-musl-gcc compiler. viaduct doesn't actually need to link to sqlite outside the ohttp feature, so properly encode that in its Cargo.toml. Because nimbus-fml doesn't enable viaduct's ohttp feature, it won't try to link to sqlite anymore. --- components/viaduct/Cargo.toml | 4 ++-- components/viaduct/src/lib.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/components/viaduct/Cargo.toml b/components/viaduct/Cargo.toml index aac33765860..51862db403d 100644 --- a/components/viaduct/Cargo.toml +++ b/components/viaduct/Cargo.toml @@ -30,8 +30,8 @@ ohttp = { version = "0.7.2", features = ["client", "server", "app-svc", "externa # # Without this, builds will often succeed because other crates will bring in `rusqlite`, however # some combinations will fail. `cargo -p ads-client -p context_id` is one example (2026/06/25). -rusqlite = { version = "0.37.0", features = [ "bundled" ] } +rusqlite = { version = "0.37.0", features = [ "bundled" ], optional = true } [features] default = [] -ohttp = ["dep:bhttp", "dep:ohttp"] +ohttp = ["dep:bhttp", "dep:ohttp", "dep:rusqlite"] diff --git a/components/viaduct/src/lib.rs b/components/viaduct/src/lib.rs index 72315e853cb..8d2e4819583 100644 --- a/components/viaduct/src/lib.rs +++ b/components/viaduct/src/lib.rs @@ -7,6 +7,7 @@ // Force linking to `rusqlite` even though we don't use it directly. // See `Cargo.toml` for why this is needed. +#[cfg(feature = "ohttp")] #[allow(unused_extern_crates)] extern crate rusqlite; From ce604827eb1103eb07ebfc1d0ff2e6b0df23bb2a Mon Sep 17 00:00:00 2001 From: Wyatt Verchere Date: Thu, 6 Aug 2026 13:32:54 -0700 Subject: [PATCH 51/59] [AC-152] Fixes moz ads telemetry UniFFI callback leak (#7520) * feat: Adds ads-client shutdown for sqlite and telemetry * fix: small note * fix: fixes clippy * fix: Adds changelog.md update * feat: Adds mutexes from review * test: vendoring with telemetry removal disabled * test: Trying vendoring with Drop * test: remove dro * fix: doc updates * fix: Adds a comment * fix: switches to RwLock, switches to Option * fix: reorders calls, fixes changelog --- CHANGELOG.md | 4 ++ components/ads-client/src/client.rs | 60 ++++++++++++++++++- components/ads-client/src/ffi/telemetry.rs | 43 +++++++++---- components/ads-client/src/http_cache.rs | 4 ++ components/ads-client/src/http_cache/store.rs | 5 ++ components/ads-client/src/lib.rs | 14 ++++- components/ads-client/src/mars.rs | 9 +++ components/ads-client/src/mars/transport.rs | 7 +++ components/ads-client/src/telemetry.rs | 4 ++ 9 files changed, 135 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 895b8ea255c..bea9c508446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ## ✨ What's Changed ✨ +### Ads-Client + +- Add `AdsClient::shutdown()`, which closes the database connection early so it happens before Firefox Desktop's late-write shutdown barrier rather than during GC. In addition, this drops all held UniFFI callbacks (held in the MozAdsTelemetryWrapper) to avoid crash on a Firefox Desktop quit. + ### Autofill - Add `Store::shutdown()`, which closes the database connection early so it happens before Firefox Desktop's late-write shutdown barrier rather than during GC. Operations after shutdown return `DatabaseClosed`. ([Bug 2050036](https://bugzilla.mozilla.org/show_bug.cgi?id=2050036)) diff --git a/components/ads-client/src/client.rs b/components/ads-client/src/client.rs index 2090d76f7ec..b4380bd5736 100644 --- a/components/ads-client/src/client.rs +++ b/components/ads-client/src/client.rs @@ -114,6 +114,18 @@ where self.client.clear_cache() } + // Shutdown the db connection and drop references to telemetry callbacks. + // Should be used only when dropping the ads client, this may be extended to drop more things. + pub fn shutdown_client(&mut self) -> Result<(), rusqlite::Error> { + // Drop telemetry (within the telemetry wrapper) + self.telemetry.shutdown(); + + // Shutdown DB + self.client.shutdown_db()?; + + Ok(()) + } + pub fn get_context_id(&self) -> context_id::ApiResult { self.context_id_provider.context_id() } @@ -286,6 +298,8 @@ pub enum ClientOperationEvent { #[cfg(test)] mod tests { + use std::{assert_eq, assert_ne, sync::Arc}; + use crate::{ ffi::telemetry::MozAdsTelemetryWrapper, mars::Environment, @@ -300,6 +314,7 @@ mod tests { fn new_with_mars_client( client: MARSClient, ) -> AdsClient { + let telemetry = client.get_telemetry(); AdsClient { client, context_id_provider: Box::new(ContextIDComponent::new( @@ -308,7 +323,7 @@ mod tests { false, Box::new(DefaultContextIdCallback), )), - telemetry: MozAdsTelemetryWrapper::noop(), + telemetry, } } @@ -555,4 +570,47 @@ mod tests { m1.assert(); m2.assert(); } + + #[test] + fn test_shutdown_telemetry() { + viaduct_dev::init_backend_dev(); + + // test with client created from config + let noop_telemetry = MozAdsTelemetryWrapper::noop(); + let weak_reference = Arc::downgrade( + &noop_telemetry + .clone_inner_arc() + .expect("Inner telemetry should be Some before dropping"), + ); + let config = AdsClientConfig { + cache_config: None, + context_id_provider: None, + environment: Environment::Test, + telemetry: noop_telemetry, + }; + let mut client = AdsClient::new(config); + + // weak ref will show 0 strong references when the Arc is gone. + assert_ne!(weak_reference.strong_count(), 0); + client.shutdown_client().unwrap(); + assert_eq!(weak_reference.strong_count(), 0); + + // test also with internal function from_mars + let noop_telemetry = MozAdsTelemetryWrapper::noop(); + let weak_reference = Arc::downgrade( + &noop_telemetry + .clone_inner_arc() + .expect("Inner telemetry should be Some before dropping"), + ); + let cache = HttpCache::builder("test_shutdown_telemetry") + .build() + .unwrap(); + let mars_client = MARSClient::new(Environment::Test, Some(cache), noop_telemetry); + let mut client = new_with_mars_client(mars_client); + + // weak ref will show 0 strong references when the Arc is gone. + assert_ne!(weak_reference.strong_count(), 0); + client.shutdown_client().unwrap(); + assert_eq!(weak_reference.strong_count(), 0); + } } diff --git a/components/ads-client/src/ffi/telemetry.rs b/components/ads-client/src/ffi/telemetry.rs index 7b905ea28d9..077859e9714 100644 --- a/components/ads-client/src/ffi/telemetry.rs +++ b/components/ads-client/src/ffi/telemetry.rs @@ -6,6 +6,8 @@ use std::any::Any; use std::sync::Arc; +use parking_lot::RwLock; + use crate::client::error::RequestAdsError; use crate::client::ClientOperationEvent; use crate::http_cache::{CacheOutcome, HttpCacheBuilderError}; @@ -38,25 +40,42 @@ impl MozAdsTelemetry for NoopMozAdsTelemetry { #[derive(Clone)] pub struct MozAdsTelemetryWrapper { - inner: Arc, + inner: Arc>>>, } impl MozAdsTelemetryWrapper { pub fn new(inner: Arc) -> Self { - Self { inner } + Self { + inner: Arc::new(RwLock::new(Some(inner))), + } } pub fn noop() -> Self { Self { - inner: Arc::new(NoopMozAdsTelemetry), + inner: Arc::new(RwLock::new(Some(Arc::new(NoopMozAdsTelemetry)))), } } + + #[cfg(test)] + pub fn clone_inner_arc(&self) -> Option> { + self.inner.read().clone() + } } impl Telemetry for MozAdsTelemetryWrapper { + // MozAdsTelemetry has hanging uniffi callbacks which need to be explicitly dropped before closing. + // This replaces it with a `None` internally, meaning future calls will be noops. + fn shutdown(&self) { + let _dropped = self.inner.write().take(); + } + fn record(&self, event: &dyn Any) { + let Some(inner) = self.inner.read().clone() else { + return; + }; + if let Some(cache_outcome) = event.downcast_ref::() { - self.inner.record_http_cache_outcome( + inner.record_http_cache_outcome( match cache_outcome { CacheOutcome::Hit => "hit".to_string(), CacheOutcome::LookupFailed(_) => "lookup_failed".to_string(), @@ -78,7 +97,7 @@ impl Telemetry for MozAdsTelemetryWrapper { return; } if let Some(client_op) = event.downcast_ref::() { - self.inner.record_client_operation_total(match client_op { + inner.record_client_operation_total(match client_op { ClientOperationEvent::New => "new".to_string(), ClientOperationEvent::RecordClick => "record_click".to_string(), ClientOperationEvent::RecordImpression => "record_impression".to_string(), @@ -88,7 +107,7 @@ impl Telemetry for MozAdsTelemetryWrapper { return; } if let Some(cache_builder_error) = event.downcast_ref::() { - self.inner.record_build_cache_error( + inner.record_build_cache_error( match cache_builder_error { HttpCacheBuilderError::EmptyDbPath => "empty_db_path".to_string(), HttpCacheBuilderError::Database(_) => "database_error".to_string(), @@ -100,31 +119,29 @@ impl Telemetry for MozAdsTelemetryWrapper { return; } if let Some(record_click_error) = event.downcast_ref::() { - self.inner.record_client_error( + inner.record_client_error( "record_click".to_string(), format!("{}", record_click_error), ); return; } if let Some(record_impression_error) = event.downcast_ref::() { - self.inner.record_client_error( + inner.record_client_error( "record_impression".to_string(), format!("{}", record_impression_error), ); return; } if let Some(report_ad_error) = event.downcast_ref::() { - self.inner - .record_client_error("report_ad".to_string(), format!("{}", report_ad_error)); + inner.record_client_error("report_ad".to_string(), format!("{}", report_ad_error)); return; } if let Some(request_ads_error) = event.downcast_ref::() { - self.inner - .record_client_error("request_ads".to_string(), format!("{}", request_ads_error)); + inner.record_client_error("request_ads".to_string(), format!("{}", request_ads_error)); return; } if let Some(json_error) = event.downcast_ref::() { - self.inner.record_deserialization_error( + inner.record_deserialization_error( "invalid_ad_item".to_string(), format!("{}", json_error), ); diff --git a/components/ads-client/src/http_cache.rs b/components/ads-client/src/http_cache.rs index a27b3b84f77..81b056bb1df 100644 --- a/components/ads-client/src/http_cache.rs +++ b/components/ads-client/src/http_cache.rs @@ -60,6 +60,10 @@ impl HttpCache { Ok(()) } + pub fn shutdown_db(self) -> Result<(), rusqlite::Error> { + self.store.close() + } + pub fn invalidate_by_hash(&self, request_hash: &RequestHash) -> Result<(), rusqlite::Error> { self.store.invalidate_by_hash(request_hash)?; Ok(()) diff --git a/components/ads-client/src/http_cache/store.rs b/components/ads-client/src/http_cache/store.rs index fe7ef28887c..a80f5a956ca 100644 --- a/components/ads-client/src/http_cache/store.rs +++ b/components/ads-client/src/http_cache/store.rs @@ -40,6 +40,11 @@ impl HttpCacheStore { } } + pub fn close(self) -> Result<(), rusqlite::Error> { + let conn = self.conn.into_inner(); + conn.close().map_err(|(_, err)| err) + } + #[cfg(test)] pub fn new_with_test_clock(conn: Connection) -> Self { use crate::http_cache::clock::TestClock; diff --git a/components/ads-client/src/lib.rs b/components/ads-client/src/lib.rs index ea54acf407d..c384078f45b 100644 --- a/components/ads-client/src/lib.rs +++ b/components/ads-client/src/lib.rs @@ -12,10 +12,10 @@ use parking_lot::Mutex; use url::Url as AdsClientUrl; use client::AdsClient; +use error_support::error; use http_cache::CachePolicy; use impression_log::ImpressionCappingPolicy; use mars::ad_request::{AdPlacementRequest, AdRequestFlags}; - mod client; mod clock; mod ffi; @@ -55,6 +55,18 @@ impl MozAdsClient { }) } + // Allows the ads-client to unload some references and prepare for a safe shutdown. + // Other methods should not be called after this one. + #[uniffi::method()] + pub fn shutdown(&self) -> AdsClientApiResult<()> { + let mut inner = self.inner.lock(); + if let Err(err) = inner.shutdown_client() { + // Log the error, but continue with shutdown. + error!("Failed to shutdown the ads client: {:?}", err); + } + Ok(()) + } + #[handle_error(ComponentError)] #[uniffi::method(default(options = None))] pub fn record_click( diff --git a/components/ads-client/src/mars.rs b/components/ads-client/src/mars.rs index 147b313629b..9b533f81b60 100644 --- a/components/ads-client/src/mars.rs +++ b/components/ads-client/src/mars.rs @@ -68,6 +68,10 @@ where self.transport.clear_cache() } + pub fn shutdown_db(&mut self) -> Result<(), rusqlite::Error> { + self.transport.shutdown_db() + } + pub fn fetch_ads( &self, context_id: String, @@ -157,6 +161,11 @@ where } self.transport.fire(request, ohttp).map_err(Into::into) } + + #[cfg(test)] + pub fn get_telemetry(&self) -> T { + self.telemetry.clone() + } } #[cfg(test)] diff --git a/components/ads-client/src/mars/transport.rs b/components/ads-client/src/mars/transport.rs index 594ca2a7a87..7da0621a4a4 100644 --- a/components/ads-client/src/mars/transport.rs +++ b/components/ads-client/src/mars/transport.rs @@ -29,6 +29,13 @@ impl MARSTransport { } } + pub fn shutdown_db(&mut self) -> Result<(), rusqlite::Error> { + if let Some(cache) = self.http_cache.take() { + cache.shutdown_db()?; + } + Ok(()) + } + pub fn clear_cache(&self) -> Result<(), rusqlite::Error> { if let Some(cache) = &self.http_cache { cache.clear()?; diff --git a/components/ads-client/src/telemetry.rs b/components/ads-client/src/telemetry.rs index b4b9d49805a..eb7da346fc2 100644 --- a/components/ads-client/src/telemetry.rs +++ b/components/ads-client/src/telemetry.rs @@ -7,4 +7,8 @@ use std::any::Any; pub trait Telemetry { fn record(&self, event: &dyn Any); + + // Shuts down any telemetry structures. This should be called before dropping the struct implementing this trait. + // Future calls to `record` will not record anything. + fn shutdown(&self); } From 05d709f5bc740e9ff22ae35d7b779fabe135574b Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 7 Aug 2026 09:38:14 -0400 Subject: [PATCH 52/59] E type alias in state machine transition code (#7530) I like the symmetry with `S` for `FxaState`. --- .../src/state_machine/transitions.rs | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/components/fxa-client/src/state_machine/transitions.rs b/components/fxa-client/src/state_machine/transitions.rs index 22f52b1656d..46a20612649 100644 --- a/components/fxa-client/src/state_machine/transitions.rs +++ b/components/fxa-client/src/state_machine/transitions.rs @@ -17,27 +17,26 @@ pub fn transition( from: FxaState, event: FxaEvent, ) -> std::result::Result { + use FxaEvent as E; use FxaState as S; match (from, event) { // ── From Uninitialized ────────────────────────────────────────── - (S::Uninitialized, FxaEvent::Initialize { device_config }) => { - match account.get_auth_state() { - FxaRustAuthState::Disconnected => Ok(S::Disconnected), - FxaRustAuthState::AuthIssues => Ok(S::AuthIssues), - FxaRustAuthState::Connected => { - match account.finish_initialize(&device_config.capabilities) { - Ok(()) => Ok(S::Connected), - Err(cause) => Err(StateMachineErr::new(cause, S::AuthIssues)), - } + (S::Uninitialized, E::Initialize { device_config }) => match account.get_auth_state() { + FxaRustAuthState::Disconnected => Ok(S::Disconnected), + FxaRustAuthState::AuthIssues => Ok(S::AuthIssues), + FxaRustAuthState::Connected => { + match account.finish_initialize(&device_config.capabilities) { + Ok(()) => Ok(S::Connected), + Err(cause) => Err(StateMachineErr::new(cause, S::AuthIssues)), } } - } + }, // ── From Disconnected ─────────────────────────────────────────── ( S::Disconnected, - FxaEvent::BeginOAuthFlow { + E::BeginOAuthFlow { service, scopes, entrypoint, @@ -54,7 +53,7 @@ pub fn transition( } ( S::Disconnected, - FxaEvent::BeginPairingFlow { + E::BeginPairingFlow { pairing_url, service, scopes, @@ -72,7 +71,7 @@ pub fn transition( } // ── From Authenticating ───────────────────────────────────────── - (S::Authenticating { initial_state, .. }, FxaEvent::CompleteOAuthFlow { code, state }) => { + (S::Authenticating { initial_state, .. }, E::CompleteOAuthFlow { code, state }) => { account .complete_oauth_flow(&code, &state) .to_state_machine_err(|| initial_state.into())?; @@ -87,16 +86,14 @@ pub fn transition( Ok(S::Connected) } - (S::Authenticating { initial_state, .. }, FxaEvent::CancelOAuthFlow) => { - Ok(initial_state.into()) - } - (S::Authenticating { .. }, FxaEvent::Disconnect) => { + (S::Authenticating { initial_state, .. }, E::CancelOAuthFlow) => Ok(initial_state.into()), + (S::Authenticating { .. }, E::Disconnect) => { account.disconnect(); Ok(S::Disconnected) } ( S::Authenticating { initial_state, .. }, - FxaEvent::BeginOAuthFlow { + E::BeginOAuthFlow { service, scopes, entrypoint, @@ -113,7 +110,7 @@ pub fn transition( } ( S::Authenticating { initial_state, .. }, - FxaEvent::BeginPairingFlow { + E::BeginPairingFlow { pairing_url, service, scopes, @@ -131,23 +128,23 @@ pub fn transition( } // A WebChannel password change while an OAuth flow is in progress // is a no-op; let the flow finish. Should be rare in practice. - (s @ S::Authenticating { .. }, FxaEvent::WebChannelPasswordChange { .. }) => { + (s @ S::Authenticating { .. }, E::WebChannelPasswordChange { .. }) => { crate::warn!("WebChannel password change received while Authenticating; ignoring"); Ok(s) } // ── From Connected ────────────────────────────────────────────── - (S::Connected, FxaEvent::Disconnect) => { + (S::Connected, E::Disconnect) => { account.disconnect(); Ok(S::Disconnected) } - (S::Connected, FxaEvent::CheckAuthorizationStatus) => { + (S::Connected, E::CheckAuthorizationStatus) => { let active = account .check_authorization_status() .to_state_machine_err(|| S::AuthIssues)?; Ok(if active { S::Connected } else { S::AuthIssues }) } - (S::Connected, FxaEvent::CallGetProfile) => { + (S::Connected, E::CallGetProfile) => { account .get_profile() .to_state_machine_err(|| S::AuthIssues)?; @@ -155,7 +152,7 @@ pub fn transition( } ( S::Connected, - FxaEvent::BeginOAuthFlow { + E::BeginOAuthFlow { service, scopes, entrypoint, @@ -173,7 +170,7 @@ pub fn transition( initial_state: FxaRustAuthState::Connected, }) } - (S::Connected, FxaEvent::WebChannelPasswordChange { json_payload }) => { + (S::Connected, E::WebChannelPasswordChange { json_payload }) => { // The inner call swaps the session token for a new refresh token and re-registers // the device record (push subscription, commands, etc) against the new token. account @@ -185,7 +182,7 @@ pub fn transition( // ── From AuthIssues ───────────────────────────────────────────── ( S::AuthIssues, - FxaEvent::BeginOAuthFlow { + E::BeginOAuthFlow { service, scopes, entrypoint, @@ -200,11 +197,11 @@ pub fn transition( initial_state: FxaRustAuthState::AuthIssues, }) } - (S::AuthIssues, FxaEvent::Disconnect) => { + (S::AuthIssues, E::Disconnect) => { account.disconnect(); Ok(S::Disconnected) } - (S::AuthIssues, FxaEvent::WebChannelPasswordChange { json_payload }) => { + (S::AuthIssues, E::WebChannelPasswordChange { json_payload }) => { // A concurrent sync/401 may have pushed us here before the webchannel ran. The new // session token recovers us; device re-registration will be handled inside the inner call. account @@ -212,7 +209,7 @@ pub fn transition( .to_state_machine_err(|| S::AuthIssues)?; Ok(S::Connected) } - (S::AuthIssues, FxaEvent::CheckAuthorizationStatus) => { + (S::AuthIssues, E::CheckAuthorizationStatus) => { let active = account .check_authorization_status() .to_state_machine_err(|| S::AuthIssues)?; @@ -220,7 +217,7 @@ pub fn transition( } // ── Other transitions ───────────────────────────────── - (from_state, FxaEvent::CheckAuthorizationStatus) => { + (from_state, E::CheckAuthorizationStatus) => { // Ignore `CheckAuthorizationStatus` from other states. // We want the app to be able to send this event whenever they want, // without generating an error. From 58308decc60bcf03519893625996957f33c8b4ce Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 7 Aug 2026 09:40:32 -0400 Subject: [PATCH 53/59] Bug 2061224 - Make `Disconnect` idempotent (#7529) --- CHANGELOG.md | 4 ++ .../fxa-client/src/state_machine/mod.rs | 5 ++- .../src/state_machine/transitions.rs | 41 +++++++++++++++++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bea9c508446..72d3a94a623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ - Add address metadata APIs for importing records already persisted elsewhere: `add_address_with_meta`, `add_many_addresses_with_meta`, `update_address_with_meta` and `add_many_address_tombstones`, with the bulk variants isolating per-record failures. `AddressMeta` carries the guid, timestamps and `sync_change_counter`, so a record keeps whether it still has changes pending upload. - Add `Store::addresses_bridged_engine()`, exposing the existing address sync engine through `mozIBridgedSyncEngine` so Firefox Desktop can drive address sync. +### Fxa Client +- The `CheckAuthorizationStatus` and `Disconnect` events are now valid from all states except `Uninitialized`. + In the cases where the failed before, they're now no-ops. + ### Nimbus - `NimbusClient::get_available_firefox_labs()` now includes detailed debug level logging for each processed lab. ([#7482](https://github.com/mozilla/application-services/pull/7482)) diff --git a/components/fxa-client/src/state_machine/mod.rs b/components/fxa-client/src/state_machine/mod.rs index 2e1f7cefc7d..aea7c7d269f 100644 --- a/components/fxa-client/src/state_machine/mod.rs +++ b/components/fxa-client/src/state_machine/mod.rs @@ -170,7 +170,10 @@ mod driver_tests { .unwrap(); assert_eq!(account.get_state(), FxaState::Disconnected); - let result = account.process_event(FxaEvent::Disconnect); + let result = account.process_event(FxaEvent::CompleteOAuthFlow { + code: "test".into(), + state: "test".into(), + }); match result { Err(Error::InvalidStateTransition(_)) => {} diff --git a/components/fxa-client/src/state_machine/transitions.rs b/components/fxa-client/src/state_machine/transitions.rs index 46a20612649..ec227da2cfa 100644 --- a/components/fxa-client/src/state_machine/transitions.rs +++ b/components/fxa-client/src/state_machine/transitions.rs @@ -225,6 +225,13 @@ pub fn transition( error_support::debug!("Ignoring `CheckAuthorizationStatus` from {from_state:?}"); Ok(from_state) } + (S::Disconnected, FxaEvent::Disconnect) => { + // Ignore Disconnect from the Disconnected state. + // + // This makes it idempotent and safe to send from any state. + // https://bugzilla.mozilla.org/show_bug.cgi?id=2061224 + Ok(S::Disconnected) + } // ── Invalid (state, event) pair ───────────────────────────────── (state, event) => Err(StateMachineErr::Fatal(Box::new( @@ -316,12 +323,40 @@ mod tests { } #[test] - fn disconnected_invalid_event_returns_fatal_invalid_state_transition() { + fn disconnect_is_idempotent() { nss_as::ensure_initialized(); + // `Disconnect` should be valid from all states and always result in the user being + // disconnected. let mut account = mock_account(); let mut wrapper = RetryingAccount::new(&mut account); - let result = transition(&mut wrapper, FxaState::Disconnected, FxaEvent::Disconnect); - assert_fatal_invalid_transition(result); + + assert_eq!( + transition(&mut wrapper, FxaState::Connected, FxaEvent::Disconnect).unwrap(), + FxaState::Disconnected + ); + + assert_eq!( + transition(&mut wrapper, FxaState::AuthIssues, FxaEvent::Disconnect).unwrap(), + FxaState::Disconnected + ); + + assert_eq!( + transition( + &mut wrapper, + FxaState::Authenticating { + oauth_url: "test".into(), + initial_state: FxaRustAuthState::Disconnected + }, + FxaEvent::Disconnect + ) + .unwrap(), + FxaState::Disconnected + ); + + assert_eq!( + transition(&mut wrapper, FxaState::Disconnected, FxaEvent::Disconnect).unwrap(), + FxaState::Disconnected + ); } fn assert_handled_lands_at( From 1d969289794a0f91cb403083f295ec9e9843a569 Mon Sep 17 00:00:00 2001 From: bendk Date: Fri, 7 Aug 2026 10:00:24 -0400 Subject: [PATCH 54/59] Fix FxA state machine spelling (#7528) A while back we purposely misspelled "Auth" as "Ath" to get around the Sentry redaction rules. Now that we're using the error ping and Grafana we don't need this anymore. --- components/fxa-client/src/state_machine/display.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/components/fxa-client/src/state_machine/display.rs b/components/fxa-client/src/state_machine/display.rs index f5d06a93323..e8a80fa225c 100644 --- a/components/fxa-client/src/state_machine/display.rs +++ b/components/fxa-client/src/state_machine/display.rs @@ -6,9 +6,6 @@ //! //! These are sent to Sentry, so they must not leak PII. //! In general this means they don't output values for inner fields. -//! -//! Also, they must not use the string "auth" since Sentry will filter that out. -//! Use "ath" instead. use super::{FxaEvent, FxaState}; use std::fmt; @@ -18,9 +15,9 @@ impl fmt::Display for FxaState { let name = match self { Self::Uninitialized => "Uninitialized", Self::Disconnected => "Disconnected", - Self::Authenticating { .. } => "Athenticating", + Self::Authenticating { .. } => "Authenticating", Self::Connected => "Connected", - Self::AuthIssues => "AthIssues", + Self::AuthIssues => "AuthIssues", }; write!(f, "{name}") } @@ -30,10 +27,10 @@ impl fmt::Display for FxaEvent { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let name = match self { Self::Initialize { .. } => "Initialize", - Self::BeginOAuthFlow { .. } => "BeginOAthFlow", + Self::BeginOAuthFlow { .. } => "BeginOAuthFlow", Self::BeginPairingFlow { .. } => "BeginPairingFlow", - Self::CompleteOAuthFlow { .. } => "CompleteOAthFlow", - Self::CancelOAuthFlow => "CancelOAthFlow", + Self::CompleteOAuthFlow { .. } => "CompleteOAuthFlow", + Self::CancelOAuthFlow => "CancelOAuthFlow", Self::CheckAuthorizationStatus => "CheckAuthorizationStatus", Self::WebChannelPasswordChange { .. } => "WebChannelPwdChange", Self::Disconnect => "Disconnect", From cb1008bcca2e5d3047f8362ecb85c8e37b5db222 Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Sat, 8 Aug 2026 06:58:41 -0400 Subject: [PATCH 55/59] sync-manager no longer queues syncs. (#7532) There's no need or use-case for syncs queuing up. Any "immediate" requests are time-sensitive, and the chance of a conflict is small. --- CHANGELOG.md | 4 +++ components/sync_manager/src/error.rs | 2 ++ components/sync_manager/src/manager.rs | 39 ++++++++++++++++++++- components/sync_manager/src/syncmanager.udl | 1 + 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d3a94a623..f876e4d29fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ - Verify signature of imported data when `.get()` is called with `sync_if_empty: true` ([#7518](https://github.com/mozilla/application-services/pull/7518)) - Do not quote `_since` values with the v2 API ([#7523](https://github.com/mozilla/application-services/pull/7523)) +### Sync Manager + +- `SyncManager::sync()` now fails immediately with a new `SyncManagerError::Busy` when a sync is already in progress, instead of blocking until it finishes. + # v154.0 (_2026-07-20_) ## ✨ What's Changed ✨ diff --git a/components/sync_manager/src/error.rs b/components/sync_manager/src/error.rs index c42c9ca321f..e40ef58059a 100644 --- a/components/sync_manager/src/error.rs +++ b/components/sync_manager/src/error.rs @@ -10,6 +10,8 @@ pub enum SyncManagerError { UnknownEngine(String), #[error("Manager was compiled without support for {0:?}")] UnsupportedFeature(String), + #[error("Another sync is already in progress")] + Busy, // Used for things like 'failed to decode the provided sync key because it's // completely the wrong format', etc. #[error("Sync error: {0}")] diff --git a/components/sync_manager/src/manager.rs b/components/sync_manager/src/manager.rs index 5dddb794e73..15c5f4a9c2f 100644 --- a/components/sync_manager/src/manager.rs +++ b/components/sync_manager/src/manager.rs @@ -83,9 +83,14 @@ impl SyncManager { } /// Perform a sync. See [SyncParams] and [SyncResult] for details on how this works + /// + /// Fails with [SyncManagerError::Busy] if a sync is already in progress. pub fn sync(&self, params: SyncParams) -> Result { breadcrumb!("SyncManager::sync started"); - let mut state = self.mem_cached_state.lock(); + let Some(mut state) = self.mem_cached_state.try_lock() else { + breadcrumb!("SyncManager::sync is already in progress, bailing out early"); + return Err(SyncManagerError::Busy); + }; let engines = self.calc_engines_to_sync(¶ms.engines)?; let next_sync_after = state.as_ref().and_then(|mcs| mcs.get_next_sync_after()); let result = if !backoff_in_effect(next_sync_after, ¶ms) { @@ -312,6 +317,7 @@ impl CommandProcessor for SyncClient { #[cfg(test)] mod test { use super::*; + use crate::types::{DeviceSettings, SyncAuthInfo}; #[test] fn test_engine_id_sanity() { @@ -319,4 +325,35 @@ mod test { assert_eq!(engine_id, SyncEngineId::try_from(engine_id.name()).unwrap()); } } + + fn dummy_sync_params() -> SyncParams { + SyncParams { + reason: SyncReason::Scheduled, + engines: SyncEngineSelection::All, + enabled_changes: HashMap::new(), + local_encryption_keys: HashMap::new(), + auth_info: SyncAuthInfo { + kid: "kid".to_string(), + fxa_access_token: "token".to_string(), + sync_key: "sync-key".to_string(), + tokenserver_url: "https://example.com/token/1.0/sync/1.5".to_string(), + }, + persisted_state: None, + device_settings: DeviceSettings { + fxa_device_id: "device-id".to_string(), + name: "Test Device".to_string(), + kind: sync15::DeviceType::Mobile, + }, + } + } + + #[test] + fn test_sync_is_busy_while_a_sync_is_in_progress() { + let manager = SyncManager::new(); + let _in_progress = manager.mem_cached_state.lock(); + assert!(matches!( + manager.sync(dummy_sync_params()), + Err(SyncManagerError::Busy) + )); + } } diff --git a/components/sync_manager/src/syncmanager.udl b/components/sync_manager/src/syncmanager.udl index e3ee2d4be18..ad94cdd1e38 100644 --- a/components/sync_manager/src/syncmanager.udl +++ b/components/sync_manager/src/syncmanager.udl @@ -11,6 +11,7 @@ namespace syncmanager { }; enum SyncManagerError { "UnknownEngine", "UnsupportedFeature", + "Busy", "Sync15Error", "UrlParseError", "InterruptedError", From 53445e42871a5ace3f62f419d62a5a6b1d271fde Mon Sep 17 00:00:00 2001 From: bendk Date: Mon, 10 Aug 2026 13:39:55 -0400 Subject: [PATCH 56/59] Remote settings fetch breadcrumbs (#7537) Changed the `trace!` into a `breadcrumb!`. This way when we see errors we can know which URL it was from. I noticed a few in the last week and knowing the URL would have been very helpful. --- components/remote_settings/src/client.rs | 4 ++-- components/remote_settings/src/error.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/remote_settings/src/client.rs b/components/remote_settings/src/client.rs index 5880c4495fa..84398b22ed9 100644 --- a/components/remote_settings/src/client.rs +++ b/components/remote_settings/src/client.rs @@ -3,7 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use crate::config::BaseUrl; -use crate::error::{debug, trace, Error, Result}; +use crate::error::{breadcrumb, debug, trace, Error, Result}; use crate::jexl_filter::JexlFilter; #[cfg(feature = "signatures")] use crate::signatures; @@ -647,7 +647,7 @@ impl ViaductApiClient { } fn make_request(&mut self, url: Url) -> Result { - trace!("make_request: {url}"); + breadcrumb!("make_request: {url}"); self.remote_state.ensure_no_backoff()?; let req = Request::get(url); diff --git a/components/remote_settings/src/error.rs b/components/remote_settings/src/error.rs index 3da9bcc6047..0d6f53967a7 100644 --- a/components/remote_settings/src/error.rs +++ b/components/remote_settings/src/error.rs @@ -4,7 +4,7 @@ use error_support::{ErrorHandling, GetErrorHandling}; // reexport logging helpers. -pub use error_support::{debug, error, info, trace, warn}; +pub use error_support::{breadcrumb, debug, error, info, trace, warn}; pub type ApiResult = std::result::Result; pub type Result = std::result::Result; From ea50770ccd14ebb5d9210db94543f8cfc67862a4 Mon Sep 17 00:00:00 2001 From: Nicolas Qiu Guichard Date: Wed, 12 Aug 2026 00:13:15 +0200 Subject: [PATCH 57/59] Remove moz.build files (#7538) Bug 2048396 removed the sync of moz.build files when vendoring application services into the Firefox tree. As pointed out by phabricator.services.mozilla.com/D298757#10668401, keeping them here is potentially confusing and I ended up confused indeed, so let's remove them as suggested. --- megazords/fenix-dylib/moz.build | 35 --------------------------------- megazords/full/moz.build | 22 --------------------- moz.build | 9 --------- 3 files changed, 66 deletions(-) delete mode 100644 megazords/fenix-dylib/moz.build delete mode 100644 megazords/full/moz.build delete mode 100644 moz.build diff --git a/megazords/fenix-dylib/moz.build b/megazords/fenix-dylib/moz.build deleted file mode 100644 index a17a1db3672..00000000000 --- a/megazords/fenix-dylib/moz.build +++ /dev/null @@ -1,35 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# There are 2 parts to our megazord - a staticlib in megazords/full, then this -# build file to create the final dyib .so -# See the comments in megazords/full/moz.build. - -UNIFIED_SOURCES += [ - "megazord_stub.c", -] - -# This name confusion is a reflection of the 2 megazord parts. It should be -# fixed after we land in m-c. -SharedLibrary("megazord-so") -SHARED_LIBRARY_NAME = "megazord" - -USE_LIBS += ["megazord", "mozpkix", "nspr"] - -# copy-pasta from other moz.build files. -if CONFIG["MOZ_WIDGET_TOOLKIT"] == "cocoa": - OS_LIBS += ["-framework CoreFoundation"] -elif CONFIG["OS_TARGET"] == "WINNT": - OS_LIBS += [ - "advapi32", - "bcrypt", - "mswsock", - "ntdll", - "shell32", - "user32", - "userenv", - "wsock32", - "ws2_32", - "winmm", - ] diff --git a/megazords/full/moz.build b/megazords/full/moz.build deleted file mode 100644 index d4ebe0b834a..00000000000 --- a/megazords/full/moz.build +++ /dev/null @@ -1,22 +0,0 @@ -# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- -# vim: set filetype=python: -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Note that this crate, when used in app-services, directly creates the cdylib. -# However, in moz-central it builds a staticlib. A new target in `../fenix-dylib` -# takes this as a dep and creates the final cdylib. -USE_LIBS += ["nss"] - -UNIFIED_SOURCES += [ - "stub.cpp", -] - -# This is the name of the .a file created. -RustLibrary("megazord") - -# XXX - this doesn't really work - the tests fail due to missing `nss3`? -RUST_TESTS = [ - "megazord", -] diff --git a/moz.build b/moz.build deleted file mode 100644 index 8cf1d1ecbf9..00000000000 --- a/moz.build +++ /dev/null @@ -1,9 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -if CONFIG["MOZ_APPSERVICES_IN_TREE"]: - DIRS += [ - "megazords/full", - "megazords/fenix-dylib", - ] From a302cceb099a34bea3abb854830b1265dfa05271 Mon Sep 17 00:00:00 2001 From: Kyle Jones Date: Tue, 7 Jul 2026 09:19:01 -0700 Subject: [PATCH 58/59] Initial implementation of impression capping in MAC --- components/ads-client/src/client.rs | 3 ++- components/ads-client/src/ffi/telemetry.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/components/ads-client/src/client.rs b/components/ads-client/src/client.rs index b4380bd5736..9138fbec486 100644 --- a/components/ads-client/src/client.rs +++ b/components/ads-client/src/client.rs @@ -586,6 +586,7 @@ mod tests { cache_config: None, context_id_provider: None, environment: Environment::Test, + impression_log_config: None, telemetry: noop_telemetry, }; let mut client = AdsClient::new(config); @@ -605,7 +606,7 @@ mod tests { let cache = HttpCache::builder("test_shutdown_telemetry") .build() .unwrap(); - let mars_client = MARSClient::new(Environment::Test, Some(cache), noop_telemetry); + let mars_client = MARSClient::new(Environment::Test, Some(cache), None, noop_telemetry); let mut client = new_with_mars_client(mars_client); // weak ref will show 0 strong references when the Arc is gone. diff --git a/components/ads-client/src/ffi/telemetry.rs b/components/ads-client/src/ffi/telemetry.rs index 077859e9714..63718dcd72b 100644 --- a/components/ads-client/src/ffi/telemetry.rs +++ b/components/ads-client/src/ffi/telemetry.rs @@ -150,7 +150,7 @@ impl Telemetry for MozAdsTelemetryWrapper { if let Some(impression_log_builder_error) = event.downcast_ref::() { - self.inner.record_build_impression_log_error( + inner.record_build_impression_log_error( match impression_log_builder_error { ImpressionLogBuilderError::EmptyDbPath => "empty_db_path".to_string(), ImpressionLogBuilderError::Database(_) => "database_error".to_string(), @@ -160,7 +160,7 @@ impl Telemetry for MozAdsTelemetryWrapper { return; } if let Some(impression_log_outcome) = event.downcast_ref::() { - self.inner.record_impression_log_outcome( + inner.record_impression_log_outcome( match impression_log_outcome { ImpressionLogOutcome::RetainImpressionsFailed(_) => { "retain_impressions_failed".to_string() From c20440655b1ebd7e3f736b175d3d8dd9ff4fde44 Mon Sep 17 00:00:00 2001 From: Kyle Jones Date: Wed, 12 Aug 2026 13:34:22 -0700 Subject: [PATCH 59/59] update docs --- .../ads-client/docs/usage-javascript.md | 102 +++++++++++++++++- components/ads-client/docs/usage-kotlin.md | 93 +++++++++++++++- components/ads-client/docs/usage-swift.md | 97 ++++++++++++++++- 3 files changed, 281 insertions(+), 11 deletions(-) diff --git a/components/ads-client/docs/usage-javascript.md b/components/ads-client/docs/usage-javascript.md index 50403f7007a..96fd7b3c3c9 100644 --- a/components/ads-client/docs/usage-javascript.md +++ b/components/ads-client/docs/usage-javascript.md @@ -16,6 +16,7 @@ Top-level client object for requesting ads and recording lifecycle events. const client = MozAdsClientBuilder() .environment(MozAdsEnvironment.Prod) .cacheConfig(cache) + .impressionLogConfig(impressionLogConfig) .telemetry(telemetry) .build(); ``` @@ -28,6 +29,7 @@ Use the `MozAdsClientBuilder` to configure and create the client. The builder pr const client = MozAdsClientBuilder() .environment(MozAdsEnvironment.Prod) .cacheConfig(cache) + .impressionLogConfig(impressionLogConfig) .telemetry(telemetry) .build(); ``` @@ -40,9 +42,9 @@ const client = MozAdsClientBuilder() | `recordClick(clickUrl, options?)` | `void` | Records a click using the provided callback URL (typically from `ad.callbacks.click`). Optional `MozAdsCallbackOptions` can enable OHTTP. | | `recordImpression(impressionUrl, options?)` | `void` | Records an impression using the provided callback URL (typically from `ad.callbacks.impression`). Optional `MozAdsCallbackOptions` can enable OHTTP. | | `reportAd(reportUrl, reason, options?)` | `void` | Reports an ad using the provided callback URL (typically from `ad.callbacks.report`). Optional `MozAdsCallbackOptions` can enable OHTTP. | -| `requestImageAds(mozAdRequests, options?)` | `Object.` | Requests one image ad per placement. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns an object keyed by `placementId`. | -| `requestSpocAds(mozAdRequests, options?)` | `Object.>` | Requests spoc ads per placement. Each placement request specifies its own count. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns an object keyed by `placementId`. | -| `requestTileAds(mozAdRequests, options?)` | `Object.` | Requests one tile ad per placement. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns an object keyed by `placementId`. | +| `requestImageAds(mozAdRequests, options?)` | `Object.` | Requests one image ad per placement. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns an object keyed by `placementId`. | +| `requestSpocAds(mozAdRequests, options?)` | `Object.>` | Requests spoc ads per placement. Each placement request specifies its own count. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns an object keyed by `placementId`. | +| `requestTileAds(mozAdRequests, options?)` | `Object.` | Requests one tile ad per placement. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns an object keyed by `placementId`. | > **Notes** > @@ -50,6 +52,7 @@ const client = MozAdsClientBuilder() > - Responses omit placements with no fill. Empty placements do not appear in the returned objects. > - The HTTP cache is internally managed. Configuration can be set with `MozAdsClientBuilder`. Per-request cache settings can be set with `MozAdsRequestOptions`. > - If `cacheConfig` is `null`, caching is disabled entirely. +> - If `impressionLogConfig` is `null`, impression counting and capping is disabled entirely. --- @@ -75,6 +78,12 @@ builder.environment(environment) */ builder.cacheConfig(cacheConfig) +/** + * @param {MozAdsImpressionLogConfig} impressionLogConfig + * @returns {MozAdsClientBuilder} + */ +builder.impressionLogConfig(impressionLogConfig) + /** * @param {MozAdsTelemetry} telemetry * @returns {MozAdsClientBuilder} @@ -92,6 +101,7 @@ builder.build() - **`MozAdsClientBuilder()`** - Creates a new builder with default values - **`environment(environment)`** - Sets the MARS environment (Prod, Staging, or Test) - **`cacheConfig(cacheConfig)`** - Sets the cache configuration +- **`impressionLogConfig(impressionLogConfig)`** - Sets the impression log configuration - **`telemetry(telemetry)`** - Sets the telemetry implementation - **`build()`** - Builds and returns the configured client @@ -115,6 +125,8 @@ Telemetry interface for recording ads client metrics. You must provide an implem * @property {function(string): void} recordClientOperationTotal * @property {function(string, string): void} recordDeserializationError * @property {function(string, string): void} recordHttpCacheOutcome + * @property {function(string, string): void} recordBuildImpressionLogError + * @property {function(string, string): void} recordImpressionLogOutcome */ ``` @@ -141,6 +153,14 @@ class AdsClientTelemetry { recordHttpCacheOutcome(label, value) { // Bind to your telemetry system } + + recordBuildImpressionLogError(label, value) { + // Bind to your telemetry system + } + + recordImpressionLogOutcome(label, value) { + // Bind to your telemetry system + } } ``` @@ -190,6 +210,39 @@ const client = MozAdsClientBuilder() --- +## `MozAdsImpressionLogConfig` + +Describes the behavior and location of the on-disk impression log. + +```javascript +/** + * @typedef {Object} MozAdsImpressionLogConfig + * @property {string} dbPath - Path to the SQLite database file. + */ +``` + +| Field | Type | Description | +| --------------------------- | ---------------- | ------------------------------------------------------------------------------------ | +| `dbPath` | `string` | Path to the SQLite database file used for log storage. Required to enable capping. | + +#### Configuration Example + +```javascript +const impressionLogConfig = MozAdsImpressionLogConfig({ + dbPath: "/tmp/impression_log.sqlite", +}); + +const telemetry = new AdsClientTelemetry(); + +const client = MozAdsClientBuilder() + .environment(MozAdsEnvironment.Prod) + .impressionLogConfig(impressionLogConfig) + .telemetry(telemetry) + .build(); +``` + +--- + ## `MozAdsPlacementRequest` Describes a single ad placement to request from MARS. An array of these is required for the `requestImageAds` and `requestTileAds` methods on the client. @@ -345,6 +398,27 @@ const MozAdsCacheMode = { --- +## `MozAdsImpressionCappingPolicy` + +Determines how the impression log is used during a request. + +```javascript +/** + * @enum {string} + */ +const MozAdsImpressionCappingPolicy = { + TelemetryOnly: "TelemetryOnly", + ImpressionCapEnforced: "ImpressionCapEnforced" +}; +``` + +| Variant | Behavior | +| -------------- | -------------------------------------------------------------------------------------------------- | +| `TelemetryOnly` | Count the number impressions over the last day, but only emit a telemetry event if the limit is hit. | +| `ImpressionCapEnforced` | Count the number impressions over the last day and filter out any which have hit that limit. | + +--- + ## `MozAdsImage` The image ad creative, callbacks, and metadata provided for each image ad returned from MARS. @@ -588,3 +662,25 @@ After storing a cacheable miss, the cache enforces `maxSizeMib` by deleting the **Manual clearing (explicit):** The cache can be manually cleared by the client using the exposed `client.clearCache()` method. This clears _all_ objects in the cache. + +--- + +## Impression Log Behavior + +### Impression Log Overview + +The internal impression log is a SQLite-backed list of timestamps per `capKey`. +It reduces repetition of the same spocs by limiting the number of times any client will see them over a rolling 24 hour window. + +### Impression Log Lifecycle + +1. Spocs are requested for a first time +2. Impression callback URLs are enriched with that as a query parameter +3. A callback URL is passed to `recordImpression` (any number of times) +4. The current epoch second is inserted into the impression log for the `capKey`, if present. +5. The `capKey` is removed from the URL, the callback to MARS is made. +6. Spocs are requested again +7. Each spoc's `capKey` is looked up and counted for the last 24 hours from the impression log +8. If the count is at or above the daily limit, the spoc is filtered from the respons +9. A clean up of the impression log happens, removing each `capKey` that was not present in the MARS/cache respons of spocs. +10. Return to step 2 diff --git a/components/ads-client/docs/usage-kotlin.md b/components/ads-client/docs/usage-kotlin.md index a7fdc89228a..fb28d8542cc 100644 --- a/components/ads-client/docs/usage-kotlin.md +++ b/components/ads-client/docs/usage-kotlin.md @@ -25,6 +25,7 @@ Use the `MozAdsClientBuilder` to configure and create the client. The builder pr val client = MozAdsClientBuilder() .environment(MozAdsEnvironment.PROD) .cacheConfig(cache) + .impressionLogConfig(impressionLogConfig) .telemetry(telemetry) .build() ``` @@ -37,9 +38,9 @@ val client = MozAdsClientBuilder() | `recordClick(clickUrl: String, options: MozAdsCallbackOptions?)` | `Unit` | Records a click using the provided callback URL (typically from `ad.callbacks.click`). | | `recordImpression(impressionUrl: String, options: MozAdsCallbackOptions?)` | `Unit` | Records an impression using the provided callback URL (typically from `ad.callbacks.impression`). | | `reportAd(reportUrl: String, reason: MozAdsReportReason, options: MozAdsCallbackOptions?)` | `Unit` | Reports an ad using the provided callback URL (typically from `ad.callbacks.report`). | -| `requestImageAds(mozAdRequests: List, options: MozAdsRequestOptions?)` | `Map` | Requests one image ad per placement. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns a map keyed by `placementId`. | -| `requestSpocAds(mozAdRequests: List, options: MozAdsRequestOptions?)` | `Map>` | Requests spoc ads per placement. Each placement request specifies its own count. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns a map keyed by `placementId`. | -| `requestTileAds(mozAdRequests: List, options: MozAdsRequestOptions?)` | `Map` | Requests one tile ad per placement. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns a map keyed by `placementId`. | +| `requestImageAds(mozAdRequests: List, options: MozAdsRequestOptions?)` | `Map` | Requests one image ad per placement. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns a map keyed by `placementId`. | +| `requestSpocAds(mozAdRequests: List, options: MozAdsRequestOptions?)` | `Map>` | Requests spoc ads per placement. Each placement request specifies its own count. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns a map keyed by `placementId`. | +| `requestTileAds(mozAdRequests: List, options: MozAdsRequestOptions?)` | `Map` | Requests one tile ad per placement. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns a map keyed by `placementId`. | > **Notes** > @@ -47,6 +48,7 @@ val client = MozAdsClientBuilder() > - Responses omit placements with no fill. Empty placements do not appear in the returned maps. > - The HTTP cache is internally managed. Configuration can be set with `MozAdsClientBuilder`. Per-request cache settings can be set with `MozAdsRequestOptions`. > - If `cacheConfig` is `null`, caching is disabled entirely. +> - If `impressionLogConfig` is `null`, impression counting and capping is disabled entirely. --- @@ -58,6 +60,7 @@ Builder for configuring and creating the ads client. Use the fluent builder patt class MozAdsClientBuilder { fun environment(environment: MozAdsEnvironment): MozAdsClientBuilder fun cacheConfig(cacheConfig: MozAdsCacheConfig): MozAdsClientBuilder + fun impressionLogConfig(impressionLogConfig: MozAdsImpressionLogConfig): MozAdsClientBuilder fun telemetry(telemetry: MozAdsTelemetry): MozAdsClientBuilder fun build(): MozAdsClient } @@ -68,6 +71,7 @@ class MozAdsClientBuilder { - **`MozAdsClientBuilder()`** - Creates a new builder with default values - **`environment(environment: MozAdsEnvironment)`** - Sets the MARS environment (Prod, Staging, or Test) - **`cacheConfig(cacheConfig: MozAdsCacheConfig)`** - Sets the cache configuration +- **`impressionLogConfig(impressionLogConfig: MozAdsImpressionLogConfig)`** - Sets the impression log configuration - **`telemetry(telemetry: MozAdsTelemetry)`** - Sets the telemetry implementation - **`build()`** - Builds and returns the configured client @@ -75,6 +79,7 @@ class MozAdsClientBuilder { | -------------- | --------------------- | ------------------------------------------------------------------------------------------------------ | | `environment` | `MozAdsEnvironment` | Selects which MARS environment to connect to. Unless in a dev build, this value can only ever be Prod. Defaults to Prod. | | `cacheConfig` | `MozAdsCacheConfig?` | Optional configuration for the internal cache. | +| `impressionLogConfig` | `MozAdsImpressionLogConfig?` | Optional configuration for the internal impression log. | | `telemetry` | `MozAdsTelemetry?` | Optional telemetry instance for recording metrics. If not provided, a no-op implementation is used. | --- @@ -90,6 +95,8 @@ interface MozAdsTelemetry { fun recordClientOperationTotal(label: String) fun recordDeserializationError(label: String, value: String) fun recordHttpCacheOutcome(label: String, value: String) + fun recordBuildImpressionLogError(label: String, value: String) + fun recordImpressionLogOutcome(label: String, value: String) } ``` @@ -119,6 +126,14 @@ class AdsClientTelemetry : MozAdsTelemetry { override fun recordHttpCacheOutcome(label: String, value: String) { AdsClient.httpCacheOutcome[label].set(value) } + + override fun recordBuildImpressionLogError(label: String, value: String) { + AdsClient.httpCacheOutcome[label].set(value) + } + + override fun recordImpressionLogOutcome(label: String, value: String) { + AdsClient.httpCacheOutcome[label].set(value) + } } ``` @@ -167,6 +182,38 @@ val client = MozAdsClientBuilder() --- +## `MozAdsImpressionLogConfig` + +Describes the behavior and location of the on-disk impression log. + +```kotlin +data class MozAdsImpressionLogConfig( + val dbPath: String, +) +``` + +| Field | Type | Description | +| --------------------------- | ---------------- | ------------------------------------------------------------------------------------ | +| `dbPath` | `String` | Path to the SQLite database file used for log storage. Required to enable capping. | + +#### Configuration Example + +```kotlin +val impressionLogConfig = MozAdsImpressionLogConfig( + dbPath = "/tmp/impression_log.sqlite" +) + +val telemetry = AdsClientTelemetry() + +val client = MozAdsClientBuilder() + .environment(MozAdsEnvironment.PROD) + .impressionLogConfig(impressionLogConfig) + .telemetry(telemetry) + .build() +``` + +--- + ## `MozAdsPlacementRequest` Describes a single ad placement to request from MARS. A list of these is required for the `requestImageAds` and `requestTileAds` methods on the client. @@ -312,6 +359,24 @@ enum class MozAdsCacheMode { --- +## `MozAdsImpressionCappingPolicy` + +Determines how the impression log is used during a request. + +```kotlin +enum class MozAdsCacheMode { + TELEMETRY_ONLY, + IMPRESSION_CAP_ENFORCED +} +``` + +| Variant | Behavior | +| -------------- | -------------------------------------------------------------------------------------------------- | +| `TELEMETRY_ONLY` | Count the number impressions over the last day, but only emit a telemetry event if the limit is hit. | +| `IMPRESSION_CAP_ENFORCED` | Count the number impressions over the last day and filter out any which have hit that limit. | + +--- + ## `MozAdsImage` The image ad creative, callbacks, and metadata provided for each image ad returned from MARS. @@ -545,3 +610,25 @@ After storing a cacheable miss, the cache enforces `maxSizeMib` by deleting the **Manual clearing (explicit):** The cache can be manually cleared by the client using the exposed `client.clearCache()` method. This clears _all_ objects in the cache. + +--- + +## Impression Log Behavior + +### Impression Log Overview + +The internal impression log is a SQLite-backed list of timestamps per `capKey`. +It reduces repetition of the same spocs by limiting the number of times any client will see them over a rolling 24 hour window. + +### Impression Log Lifecycle + +1. Spocs are requested for a first time +2. Impression callback URLs are enriched with that as a query parameter +3. A callback URL is passed to `recordImpression` (any number of times) +4. The current epoch second is inserted into the impression log for the `capKey`, if present. +5. The `capKey` is removed from the URL, the callback to MARS is made. +6. Spocs are requested again +7. Each spoc's `capKey` is looked up and counted for the last 24 hours from the impression log +8. If the count is at or above the daily limit, the spoc is filtered from the respons +9. A clean up of the impression log happens, removing each `capKey` that was not present in the MARS/cache respons of spocs. +10. Return to step 2 diff --git a/components/ads-client/docs/usage-swift.md b/components/ads-client/docs/usage-swift.md index 59a91d7a8d3..9e4ae19b65c 100644 --- a/components/ads-client/docs/usage-swift.md +++ b/components/ads-client/docs/usage-swift.md @@ -25,6 +25,7 @@ Use the `MozAdsClientBuilder` to configure and create the client. The builder pr let client = MozAdsClientBuilder() .environment(environment: .prod) .cacheConfig(cacheConfig: cache) + .impressionLogConfig(impressionLogConfig: impressionLogConfig) .telemetry(telemetry: telemetry) .build() ``` @@ -37,9 +38,9 @@ let client = MozAdsClientBuilder() | `recordClick(clickUrl: String, options: MozAdsCallbackOptions?)` | `Void` | Records a click using the provided callback URL (typically from `ad.callbacks.click`). | | `recordImpression(impressionUrl: String, options: MozAdsCallbackOptions?)` | `Void` | Records an impression using the provided callback URL (typically from `ad.callbacks.impression`). | | `reportAd(reportUrl: String, reason: MozAdsReportReason, options: MozAdsCallbackOptions?)` | `Void` | Reports an ad using the provided callback URL (typically from `ad.callbacks.report`). | -| `requestImageAds(mozAdRequests: [MozAdsPlacementRequest], options: MozAdsRequestOptions?)` | `[String: MozAdsImage]` | Requests one image ad per placement. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns a dictionary keyed by `placementId`. | -| `requestSpocAds(mozAdRequests: [MozAdsPlacementRequestWithCount], options: MozAdsRequestOptions?)` | `[String: [MozAdsSpoc]]` | Requests spoc ads per placement. Each placement request specifies its own count. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns a dictionary keyed by `placementId`. | -| `requestTileAds(mozAdRequests: [MozAdsPlacementRequest], options: MozAdsRequestOptions?)` | `[String: MozAdsTile]` | Requests one tile ad per placement. Optional `MozAdsRequestOptions` can adjust caching behavior. Returns a dictionary keyed by `placementId`. | +| `requestImageAds(mozAdRequests: [MozAdsPlacementRequest], options: MozAdsRequestOptions?)` | `[String: MozAdsImage]` | Requests one image ad per placement. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns a dictionary keyed by `placementId`. | +| `requestSpocAds(mozAdRequests: [MozAdsPlacementRequestWithCount], options: MozAdsRequestOptions?)` | `[String: [MozAdsSpoc]]` | Requests spoc ads per placement. Each placement request specifies its own count. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns a dictionary keyed by `placementId`. | +| `requestTileAds(mozAdRequests: [MozAdsPlacementRequest], options: MozAdsRequestOptions?)` | `[String: MozAdsTile]` | Requests one tile ad per placement. Optional `MozAdsRequestOptions` can adjust caching and capping behavior. Returns a dictionary keyed by `placementId`. | > **Notes** > @@ -47,6 +48,7 @@ let client = MozAdsClientBuilder() > - Responses omit placements with no fill. Empty placements do not appear in the returned dictionaries. > - The HTTP cache is internally managed. Configuration can be set with `MozAdsClientBuilder`. Per-request cache settings can be set with `MozAdsRequestOptions`. > - If `cacheConfig` is `nil`, caching is disabled entirely. +> - If `impressionLogConfig` is `null`, impression counting and capping is disabled entirely. --- @@ -58,6 +60,7 @@ Builder for configuring and creating the ads client. Use the fluent builder patt class MozAdsClientBuilder { func environment(environment: MozAdsEnvironment) -> MozAdsClientBuilder func cacheConfig(cacheConfig: MozAdsCacheConfig) -> MozAdsClientBuilder + func impressionLogConfig(impressionLogConfig: MozAdsImpressionLogConfig) -> MozAdsClientBuilder func telemetry(telemetry: MozAdsTelemetry) -> MozAdsClientBuilder func build() -> MozAdsClient } @@ -68,14 +71,16 @@ class MozAdsClientBuilder { - **`MozAdsClientBuilder()`** - Creates a new builder with default values - **`environment(environment: MozAdsEnvironment)`** - Sets the MARS environment (Prod, Staging, or Test) - **`cacheConfig(cacheConfig: MozAdsCacheConfig)`** - Sets the cache configuration +- **`impressionLogConfig(impressionLogConfig: MozAdsImpressionLogConfig)`** - Sets the impression log configuration - **`telemetry(telemetry: MozAdsTelemetry)`** - Sets the telemetry implementation - **`build()`** - Builds and returns the configured client | Configuration | Type | Description | | -------------- | --------------------- | ------------------------------------------------------------------------------------------------------ | | `environment` | `MozAdsEnvironment` | Selects which MARS environment to connect to. Unless in a dev build, this value can only ever be Prod. Defaults to Prod. | -| `cacheConfig` | `MozAdsCacheConfig?` | Optional configuration for the internal cache. | -| `telemetry` | `MozAdsTelemetry?` | Optional telemetry instance for recording metrics. If not provided, a no-op implementation is used. | +| `cacheConfig` | `MozAdsCacheConfig?` | Optional configuration for the internal cache. | +| `impressionLogConfig` | `MozAdsImpressionLogConfig?` | Optional configuration for the internal impression log. | +| `telemetry` | `MozAdsTelemetry?` | Optional telemetry instance for recording metrics. If not provided, a no-op implementation is used. | --- @@ -90,6 +95,8 @@ protocol MozAdsTelemetry { func recordClientOperationTotal(label: String) func recordDeserializationError(label: String, value: String) func recordHttpCacheOutcome(label: String, value: String) + func recordBuildImpressionLogError(label: String, value: String) + func recordImpressionLogOutcome(label: String, value: String) } ``` @@ -119,6 +126,14 @@ public final class AdsClientTelemetry: MozAdsTelemetry { public func recordHttpCacheOutcome(label: String, value: String) { AdsClientMetrics.httpCacheOutcome[label].set(value) } + + public func recordBuildImpressionLogError(label: String, value: String) { + AdsClientMetrics.deserializationError[label].set(value) + } + + public func recordImpressionLogOutcome(label: String, value: String) { + AdsClientMetrics.httpCacheOutcome[label].set(value) + } } ``` @@ -167,6 +182,38 @@ let client = MozAdsClientBuilder() --- +## `MozAdsImpressionLogConfig` + +Describes the behavior and location of the on-disk impression log. + +```swift +structMozAdsImpressionLogConfig( + let dbPath: String, +) +``` + +| Field | Type | Description | +| --------------------------- | ---------------- | ------------------------------------------------------------------------------------ | +| `dbPath` | `String` | Path to the SQLite database file used for log storage. Required to enable capping. | + +#### Configuration Example + +```swift +let impressionLogConfig = MozAdsImpressionLogConfig( + dbPath: "/tmp/impression_log.sqlite" +) + +let telemetry = AdsClientTelemetry() + +let client = MozAdsClientBuilder() + .environment(environment: .prod) + .impressionLogConfig(impressionLogConfig: impressionLogConfig) + .telemetry(telemetry: telemetry) + .build() +``` + +--- + ## `MozAdsPlacementRequest` Describes a single ad placement to request from MARS. An array of these is required for the `requestImageAds` and `requestTileAds` methods on the client. @@ -312,6 +359,24 @@ enum MozAdsCacheMode { --- +## `MozAdsImpressionCappingPolicy` + +Determines how the impression log is used during a request. + +```kotlin +enum MozAdsCacheMode { + telemetryOnly, + impressionCapEnforced +} +``` + +| Variant | Behavior | +| -------------- | -------------------------------------------------------------------------------------------------- | +| `telemetryOnly` | Count the number impressions over the last day, but only emit a telemetry event if the limit is hit. | +| `impressionCapEnforced` | Count the number impressions over the last day and filter out any which have hit that limit. | + +--- + ## `MozAdsImage` The image ad creative, callbacks, and metadata provided for each image ad returned from MARS. @@ -545,3 +610,25 @@ After storing a cacheable miss, the cache enforces `maxSizeMib` by deleting the **Manual clearing (explicit):** The cache can be manually cleared by the client using the exposed `client.clearCache()` method. This clears _all_ objects in the cache. + +--- + +## Impression Log Behavior + +### Impression Log Overview + +The internal impression log is a SQLite-backed list of timestamps per `capKey`. +It reduces repetition of the same spocs by limiting the number of times any client will see them over a rolling 24 hour window. + +### Impression Log Lifecycle + +1. Spocs are requested for a first time +2. Impression callback URLs are enriched with that as a query parameter +3. A callback URL is passed to `recordImpression` (any number of times) +4. The current epoch second is inserted into the impression log for the `capKey`, if present. +5. The `capKey` is removed from the URL, the callback to MARS is made. +6. Spocs are requested again +7. Each spoc's `capKey` is looked up and counted for the last 24 hours from the impression log +8. If the count is at or above the daily limit, the spoc is filtered from the respons +9. A clean up of the impression log happens, removing each `capKey` that was not present in the MARS/cache respons of spocs. +10. Return to step 2