From 0c5f6183111026ce3f6f30eb488d1d87569e328f Mon Sep 17 00:00:00 2001 From: Ben Dean-Kawamura Date: Mon, 10 Aug 2026 15:02:46 -0400 Subject: [PATCH] Track secondary sync metrics on our dashboard. Added panels for total sync counts and average sync time on the main dashboard. Added panels for average sync time to the details page. Both of these use a 7 day moving average. Refactored the code so that it will hopefully be easier to add more of these metrics if we want to later on. Also: - Added `rust-logins` engine for Desktop. This captures users using the Rust logins component, which is what nightly is on nowadays. - Added CC key regeneration metrics for iOs. - Put the error panel below the sync metric panels - Changed the slug for the details page from `sync-sync-details` to `sync-sync-extra`. I'm not sure why, but the old slug started getting import errors. I believe was was related to me uploading it so many times. --- .../src/component_config.rs | 2 +- tools/generate-rust-dashboards/src/main.rs | 10 +- .../src/metrics/sync.rs | 356 +++++++++++------- tools/generate-rust-dashboards/src/schema.rs | 2 + .../src/team_config.rs | 10 +- 5 files changed, 233 insertions(+), 147 deletions(-) diff --git a/tools/generate-rust-dashboards/src/component_config.rs b/tools/generate-rust-dashboards/src/component_config.rs index f9dc2a0b294..3391f939ee6 100644 --- a/tools/generate-rust-dashboards/src/component_config.rs +++ b/tools/generate-rust-dashboards/src/component_config.rs @@ -69,7 +69,7 @@ impl Component { match self { Self::Autofill => &["addresses", "creditcards"], Self::Fxa => &[], - Self::Logins => &["logins"], + Self::Logins => &["logins", "rust-logins"], Self::Places => &["bookmarks", "history"], Self::RemoteSettings => &[], Self::Suggest => &[], diff --git a/tools/generate-rust-dashboards/src/main.rs b/tools/generate-rust-dashboards/src/main.rs index 254aaf47661..6bb9371ad54 100644 --- a/tools/generate-rust-dashboards/src/main.rs +++ b/tools/generate-rust-dashboards/src/main.rs @@ -41,16 +41,16 @@ fn main() -> Result<()> { let mut main_dashboard_builder = main_dashboard::start_dashboard(&config); let mut extra_dashboards: Vec = vec![]; - if config.component_errors { - metrics::rust_component_errors::add_to_dashboard(&mut main_dashboard_builder, &config)?; - extra_dashboards.push(metrics::rust_component_errors::extra_dashboard(&config)?); - } - if config.sync_metrics { metrics::sync::add_to_main_dashboard(&mut main_dashboard_builder, &config)?; extra_dashboards.push(metrics::sync::extra_dashboard(&config)?); } + if config.component_errors { + metrics::rust_component_errors::add_to_dashboard(&mut main_dashboard_builder, &config)?; + extra_dashboards.push(metrics::rust_component_errors::extra_dashboard(&config)?); + } + for metric in config.main_dashboard_metrics.iter() { metric.add_to_dashboard(&mut main_dashboard_builder, &config)?; } diff --git a/tools/generate-rust-dashboards/src/metrics/sync.rs b/tools/generate-rust-dashboards/src/metrics/sync.rs index ebc66eefc07..76faf67d9a7 100644 --- a/tools/generate-rust-dashboards/src/metrics/sync.rs +++ b/tools/generate-rust-dashboards/src/metrics/sync.rs @@ -3,11 +3,11 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use crate::{ - config::{Application, ReleaseChannel, TeamConfig}, + config::{Application, ReleaseChannel, TeamConfig, Unit}, schema::{ - CalculateFieldOptions, CustomVariable, Dashboard, DashboardBuilder, DataLink, Datasource, - FieldConfig, FieldConfigCustom, FieldConfigDefaults, GridPos, LogOptions, LogPanel, Panel, - Target, TextPanel, TimeSeriesPanel, Transformation, WindowFunctionWindow, + CustomVariable, Dashboard, DashboardBuilder, DataLink, Datasource, FieldConfig, + FieldConfigCustom, FieldConfigDefaults, GridPos, LogOptions, LogPanel, Panel, + ScaleDistribution, Target, TextPanel, TimeSeriesPanel, Transformation, }, sql::Query, util::{Join, UrlBuilder}, @@ -15,14 +15,27 @@ use crate::{ }; pub fn add_to_main_dashboard(builder: &mut DashboardBuilder, config: &TeamConfig) -> Result<()> { - builder.add_panel_title("Sync"); + add_overview_panels( + builder, + "Sync success rate", + config, + SyncMetric::SuccessRate, + ); + add_overview_panels( + builder, + "Sync: total counts", + config, + SyncMetric::TotalCounts, + ); + add_overview_panels( + builder, + "Sync: average time", + config, + SyncMetric::AverageTime, + ); - for app in config.applications().iter() { - builder.add_panel_third(overview_count_panel(config, *app, ReleaseChannel::Nightly)); - builder.add_panel_third(overview_count_panel(config, *app, ReleaseChannel::Beta)); - builder.add_panel_third(overview_count_panel(config, *app, ReleaseChannel::Release)); - } if config.team_name == "SYNC" { + builder.add_panel_title("Legacy dashboards"); builder.add_panel_full(sync_legacy_dashboard_panel()); } @@ -32,7 +45,7 @@ pub fn add_to_main_dashboard(builder: &mut DashboardBuilder, config: &TeamConfig pub fn extra_dashboard(config: &TeamConfig) -> Result { let mut builder = DashboardBuilder::new( format!("{} - Sync Details", config.team_name), - format!("{}-sync-details", config.team_slug()), + format!("{}-sync-extra", config.team_slug()), ); builder.add_application_variable(config)?; builder.add_channel_variable(); @@ -51,13 +64,15 @@ pub fn extra_dashboard(config: &TeamConfig) -> Result { builder.add_panel_title("Metrics"); builder.add_panel_full(details_dash_count_panel( "Success Rate", - "success_rate", - false, + SyncMetric::SuccessRate, )); builder.add_panel_full(details_dash_count_panel( "Total counts (7 day moving average)", - "count_total", - true, + SyncMetric::TotalCounts, + )); + builder.add_panel_full(details_dash_count_panel( + "Average sync time (7 day moving average)", + SyncMetric::AverageTime, )); builder.add_panel_title("Errors"); builder.add_panel_full(details_dash_error_count_panel(config)); @@ -66,10 +81,99 @@ pub fn extra_dashboard(config: &TeamConfig) -> Result { Ok(builder.dashboard) } -fn overview_count_panel( +#[derive(Clone, Copy)] +enum SyncMetric { + SuccessRate, + TotalCounts, + AverageTime, +} + +impl SyncMetric { + fn column_name(&self) -> &'static str { + match &self { + SyncMetric::SuccessRate => "success_rate", + SyncMetric::TotalCounts => "count_total", + SyncMetric::AverageTime => "avg_sync_time", + } + } + + fn moving_average(&self) -> bool { + matches!(self, SyncMetric::TotalCounts | SyncMetric::AverageTime) + } + + fn field_config_custom(&self) -> FieldConfigCustom { + match self { + SyncMetric::SuccessRate => FieldConfigCustom { + axis_label: "success rate".into(), + axis_soft_min: 99, + axis_soft_max: 100, + ..FieldConfigCustom::default() + }, + SyncMetric::TotalCounts => FieldConfigCustom { + scale_distribution: ScaleDistribution { + type_: "log".into(), + log: Some(10), + }, + ..FieldConfigCustom::default() + }, + SyncMetric::AverageTime => FieldConfigCustom::default(), + } + } + + fn unit(&self) -> Option { + match self { + SyncMetric::SuccessRate => None, + SyncMetric::TotalCounts => Some(Unit::SiShort), + SyncMetric::AverageTime => Some(Unit::Seconds), + } + } + + fn column_expr(&self) -> String { + let column_name = self.column_name(); + if !self.moving_average() { + column_name.into() + } else { + format!( + "AVG({column_name}) OVER ( + PARTITION BY engine_name + ORDER BY submission_date + ROWS BETWEEN 6 PRECEDING AND CURRENT ROW + ) AS {column_name}" + ) + } + } +} + +fn add_overview_panels( + builder: &mut DashboardBuilder, + title: &str, + config: &TeamConfig, + metric: SyncMetric, +) { + builder.add_panel_title(title); + + for app in config.applications().iter() { + builder.add_panel_third(overview_panel( + config, + *app, + ReleaseChannel::Nightly, + metric, + )); + builder.add_panel_third(overview_panel(config, *app, ReleaseChannel::Beta, metric)); + builder.add_panel_third(overview_panel( + config, + *app, + ReleaseChannel::Release, + metric, + )); + } +} + +fn overview_panel( config: &TeamConfig, application: Application, channel: ReleaseChannel, + metric: SyncMetric, ) -> Panel { if application == Application::Ios && channel == ReleaseChannel::Nightly { return TextPanel { @@ -80,7 +184,28 @@ fn overview_count_panel( .into(); } - let query = count_query(config, application, format!("'{channel}'")); + let column_name = metric.column_name(); + let query = Query { + select: vec![ + "TIMESTAMP(submission_date) as time".into(), + "engine_name".into(), + metric.column_expr(), + ], + from: format!("({COMBINED_SUBQUERY})"), + where_: vec![ + "$__timeFilter(TIMESTAMP(submission_date))".into(), + format!("channel = '{channel}'"), + match application { + Application::Desktop => "application = 'desktop'", + Application::Ios => "application = 'firefox-ios'", + Application::Android => "application = 'firefox-android'", + } + .into(), + engine_where_clause(config), + ], + order_by: Some("time".into()), + ..Query::default() + }; Panel::from(TimeSeriesPanel { title: application.display_name(channel), @@ -88,11 +213,11 @@ fn overview_count_panel( datasource: Datasource::bigquery(), // needs to be fairly large since the total sync count can be low on mobile/nightly interval: "1d".into(), - targets: vec![Target::table(query)], + targets: vec![Target::table(query.sql())], field_config: FieldConfig { defaults: FieldConfigDefaults { links: vec![DataLink { - url: UrlBuilder::new_dashboard(format!("{}-sync-details", config.team_slug())) + url: UrlBuilder::new_dashboard(format!("{}-sync-extra", config.team_slug())) .with_time_range_param() .with_param("var-application", application.slug()) .with_param("var-channel", channel.to_string()) @@ -102,13 +227,8 @@ fn overview_count_panel( one_click: true, title: "Errors".into(), }], - custom: FieldConfigCustom { - axis_label: "success rate".into(), - axis_soft_min: 99, - axis_soft_max: 100, - ..FieldConfigCustom::default() - }, - unit: None, + custom: metric.field_config_custom(), + unit: metric.unit(), }, }, transformations: vec![ @@ -118,7 +238,7 @@ fn overview_count_panel( }, // Fixup the field names for better legend labels Transformation::RenameByRegex { - regex: "success_rate (.*)".into(), + regex: format!("{column_name} (.*)"), rename_pattern: "$1".into(), }, ], @@ -126,71 +246,28 @@ fn overview_count_panel( }) } -/// Query to fetch sync success rates -fn count_query(config: &TeamConfig, application: Application, channel_expr: String) -> String { - let table_name = if application == Application::Desktop { - "desktop_v1" - } else { - "mobile_v1" - }; - let application_where = match application { - Application::Desktop => "application = 'desktop'", - Application::Ios => "application = 'firefox-ios'", - Application::Android => "application = 'firefox-android'", - }; - - let mut engines: Vec<_> = config - .components - .iter() - .flat_map(|c| c.sync_engines()) - .map(|e| format!("'{e}'")) - .collect(); - engines.sort_unstable(); - engines.dedup(); - let engines_where = format!("engine_name IN ({})", engines.join(", ")); - - format!( - "\ -SELECT - TIMESTAMP(submission_date) as time, - engine_name, - success_rate -FROM - moz-fx-data-shared-prod.sync_derived.{table_name} -WHERE - channel = {channel_expr} - AND $__timeFilter(TIMESTAMP(submission_date)) - AND {application_where} - AND {engines_where} -ORDER BY time" - ) -} - -fn details_dash_count_panel(title: &str, column_name: &str, moving_average: bool) -> Panel { +fn details_dash_count_panel(title: &str, metric: SyncMetric) -> Panel { let query = Query { - select: vec!["time".into(), column_name.into()], - from: format!("(\n{}\n)", details_dash_count_query()), - group_by: Some("1, 2".into()), + select: vec![ + "TIMESTAMP(submission_date) as time".into(), + metric.column_expr(), + ], + from: format!("(\n{COMBINED_SUBQUERY}\n)"), + where_: vec![ + "$__timeFilter(TIMESTAMP(submission_date))".into(), + "channel = '${channel}'".into(), + "application=CASE '${application}' + WHEN 'firefox_desktop' THEN 'desktop' + WHEN 'firefox_android' THEN 'firefox-android' + WHEN 'firefox_ios' THEN 'firefox-ios' + ELSE '${application}' + END" + .into(), + "engine_name = '${engine}'".into(), + ], ..Query::default() }; - let transformations = if moving_average { - vec![Transformation::CalculateField( - CalculateFieldOptions::WindowFunctions { - replace_fields: true, - window: WindowFunctionWindow { - field: "count_total".into(), - reducer: "mean".into(), - window_alignment: "centered".into(), - window_size: 7.0, - window_size_mode: "fixed".into(), - }, - }, - )] - } else { - vec![] - }; - TimeSeriesPanel { title: title.into(), grid_pos: GridPos::height(10), @@ -198,54 +275,19 @@ fn details_dash_count_panel(title: &str, column_name: &str, moving_average: bool // needs to be fairly large since the total sync count can be low on mobile/nightly interval: "1d".into(), targets: vec![Target::table(query.sql())], - transformations, + transformations: vec![], + field_config: FieldConfig { + defaults: FieldConfigDefaults { + links: vec![], + custom: metric.field_config_custom(), + unit: metric.unit(), + }, + }, ..TimeSeriesPanel::default() } .into() } -/// Query to count metrics for the details dashboard -fn details_dash_count_query() -> String { - "\ -SELECT - TIMESTAMP(submission_date) as time, - success_rate, - count_total -FROM - moz-fx-data-shared-prod.sync_derived.desktop_v1 -WHERE - $__timeFilter(TIMESTAMP(submission_date)) - AND channel = '${channel}' - AND application=CASE '${application}' - WHEN 'firefox_desktop' THEN 'desktop' - WHEN 'firefox_android' THEN 'firefox-android' - WHEN 'firefox_ios' THEN 'firefox-ios' - ELSE '${application}' - END - AND engine_name = '${engine}' - -UNION ALL - -SELECT - TIMESTAMP(submission_date) as time, - success_rate, - count_total -FROM - moz-fx-data-shared-prod.sync_derived.mobile_v1 -WHERE - $__timeFilter(TIMESTAMP(submission_date)) - AND channel = '${channel}' - AND application=CASE '${application}' - WHEN 'firefox_desktop' THEN 'desktop' - WHEN 'firefox_android' THEN 'firefox-android' - WHEN 'firefox_ios' THEN 'firefox-ios' - ELSE '${application}' - END - AND engine_name = '${engine}' -ORDER BY time" - .to_string() -} - fn details_dash_error_count_panel(config: &TeamConfig) -> Panel { let query = Query { select: vec![ @@ -255,7 +297,7 @@ fn details_dash_error_count_panel(config: &TeamConfig) -> Panel { ], where_: vec![ "application='${application}'".into(), - "engine_name='${engine}'".into(), + "engine_name = '${engine}'".into(), "normalized_channel = '${channel}'".into(), "$__timeFilter(submission_timestamp)".into(), ], @@ -295,7 +337,7 @@ fn details_dash_error_log_panel(config: &TeamConfig) -> Panel { ], from: format!("(\n{}\n)", error_subquery(config)), where_: vec![ - "engine_name='${engine}'".into(), + "engine_name = '${engine}'".into(), "normalized_channel = '${channel}'".into(), "application='${application}'".into(), "$__timeFilter(submission_timestamp)".into(), @@ -351,6 +393,8 @@ WHERE .components .iter() .flat_map(|c| c.sync_engines()) + // filter out desktop-only engines + .filter(|c| **c != "rust-logins") .flat_map(|engine_name| { [ format!( @@ -387,13 +431,6 @@ WHERE fn sync_legacy_dashboard_panel() -> Panel { let content = "\ # Legacy Sync dashboards -* [Sync: engine performance](https://sql.telemetry.mozilla.org/dashboard/sync-engine-performance?p_w73231_Days=7&p_w73233_Days=7&p_w73234_Days=7&p_w73237_Days=7&p_w73238_Days=60&p_w73239_Days=60&p_w73248_Days=60&p_w73249_Days=60&p_w73250_Days=60&p_w73251_Days=60&p_w73255_Days=7&p_w73256_Days=7&p_w73257_Days=60) -* [Desktop Sync Failures](https://sql.telemetry.mozilla.org/dashboard/sync-desktop?p_Days=60&p_engine_name=all-engines&p_w63728_engine_name=all-engines&p_w64027_engine_name=all-engines&p_w64028_engine_name=all-engines&p_w64029_engine_name=all-engines&p_w65780_channel=beta&p_w65780_days=30&p_w65780_engine_name=all-engines) -* [Android Sync Failures](https://sql.telemetry.mozilla.org/dashboard/android-sync-failures?p_channel=org_mozilla_fenix&p_engine_name=credit-cards&p_w64121_Months=24&p_w64121_engine_name=all-engines&p_w64122_Months=24&p_w64122_engine_name=all-engines&p_w64123_Months=24&p_w64123_engine_name=all-engines&p_w73261_Months=1&p_w73261_engine=bookmarks&p_w73261_minimum_error_count=0&p_w73262_Months=1&p_w73262_engine=bookmarks&p_w73262_minimum_error_count=0&p_w73263_Months=1&p_w73263_engine=bookmarks&p_w73263_minimum_error_count=0) -* [iOS Sync failures](https://sql.telemetry.mozilla.org/dashboard/ios-sync-failures?p_Days=60&p_Months=1&p_engine_name=all-engines&p_w67318_Months=1&p_w67318_engine=bookmarks&p_w67318_minimum%20error%20count=0&p_w67320_Months=1&p_w67320_engine=bookmarks&p_w67320_minimum%20error%20count=0) -* [Android Logins key regeneration errors](https://sql.telemetry.mozilla.org/queries/83554#207048) -* [iOS Logins Key Regeneration Metrics](https://sql.telemetry.mozilla.org/dashboard/ios-logins-key-regeneration-metrics) -* [iOS Credit Cards Key Regeneration Metrics](https://sql.telemetry.mozilla.org/dashboard/ios-credit-cards-key-regeneration-metrics) * [iOS Credit Cards Verification Usage](https://sql.telemetry.mozilla.org/dashboard/ios-credit-cards-verification-usage) * [Mobile Logins Verification Usage](https://sql.telemetry.mozilla.org/dashboard/mobile-logins-verification-usage?p_channel=org_mozilla_ios_firefox) * [iOS FxA Keychain Rollout Enrollment](https://sql.telemetry.mozilla.org/dashboard/ios-credit-cards-key-regeneration-metrics) @@ -405,3 +442,42 @@ fn sync_legacy_dashboard_panel() -> Panel { } .into() } + +fn engine_where_clause(config: &TeamConfig) -> String { + let mut engines: Vec<_> = config + .components + .iter() + .flat_map(|c| c.sync_engines()) + .map(|e| format!("'{e}'")) + .collect(); + engines.sort_unstable(); + engines.dedup(); + format!("engine_name IN ({})", engines.join(", ")) +} + +/// Subquery that combines the desktop and mobile ETL tables +const COMBINED_SUBQUERY: &str = "\ +SELECT + submission_date, + channel, + application, + engine_name, + success_rate, + avg_sync_time, + count_total +FROM + moz-fx-data-shared-prod.sync_derived.desktop_v1 + +UNION ALL + +SELECT + submission_date, + channel, + application, + engine_name, + success_rate, + 0 as avg_sync_time, -- TODO: make this work on Mobile + count_total +FROM + moz-fx-data-shared-prod.sync_derived.mobile_v1 +"; diff --git a/tools/generate-rust-dashboards/src/schema.rs b/tools/generate-rust-dashboards/src/schema.rs index 42b4df538f2..62403d40bff 100644 --- a/tools/generate-rust-dashboards/src/schema.rs +++ b/tools/generate-rust-dashboards/src/schema.rs @@ -128,6 +128,8 @@ pub enum Unit { Microseconds, #[serde(rename = "ns")] Nanoseconds, + #[serde(rename = "sishort")] + SiShort, } #[derive(Default, Serialize)] diff --git a/tools/generate-rust-dashboards/src/team_config.rs b/tools/generate-rust-dashboards/src/team_config.rs index 8fea31c10ec..e3b0a184291 100644 --- a/tools/generate-rust-dashboards/src/team_config.rs +++ b/tools/generate-rust-dashboards/src/team_config.rs @@ -43,7 +43,15 @@ pub fn all_dashboards() -> Vec { "key_regenerated_corrupt", "key_regenerated_other", ], - applications: vec![Android], + applications: vec![Android, Ios], + } + .into(), + EventsMetric { + display_name: "Credit Card key regeneration", + ping: "metrics", + category: "credit_card_key_regeneration", + metrics: vec!["keychain_data_lost", "lost", "corrupt", "other"], + applications: vec![Ios], } .into(), ],