diff --git a/tools/generate-rust-dashboards/src/component_config.rs b/tools/generate-rust-dashboards/src/component_config.rs index f9dc2a0b29..3391f939ee 100644 --- a/tools/generate-rust-dashboards/src/component_config.rs +++ b/tools/generate-rust-dashboards/src/component_config.rs @@ -69,7 +69,7 @@ impl Component { match self { Self::Autofill => &["addresses", "creditcards"], Self::Fxa => &[], - Self::Logins => &["logins"], + Self::Logins => &["logins", "rust-logins"], Self::Places => &["bookmarks", "history"], Self::RemoteSettings => &[], Self::Suggest => &[], diff --git a/tools/generate-rust-dashboards/src/main.rs b/tools/generate-rust-dashboards/src/main.rs index 254aaf4766..6bb9371ad5 100644 --- a/tools/generate-rust-dashboards/src/main.rs +++ b/tools/generate-rust-dashboards/src/main.rs @@ -41,16 +41,16 @@ fn main() -> Result<()> { let mut main_dashboard_builder = main_dashboard::start_dashboard(&config); let mut extra_dashboards: Vec = vec![]; - if config.component_errors { - metrics::rust_component_errors::add_to_dashboard(&mut main_dashboard_builder, &config)?; - extra_dashboards.push(metrics::rust_component_errors::extra_dashboard(&config)?); - } - if config.sync_metrics { metrics::sync::add_to_main_dashboard(&mut main_dashboard_builder, &config)?; extra_dashboards.push(metrics::sync::extra_dashboard(&config)?); } + if config.component_errors { + metrics::rust_component_errors::add_to_dashboard(&mut main_dashboard_builder, &config)?; + extra_dashboards.push(metrics::rust_component_errors::extra_dashboard(&config)?); + } + for metric in config.main_dashboard_metrics.iter() { metric.add_to_dashboard(&mut main_dashboard_builder, &config)?; } diff --git a/tools/generate-rust-dashboards/src/metrics/sync.rs b/tools/generate-rust-dashboards/src/metrics/sync.rs index ebc66eefc0..76faf67d9a 100644 --- a/tools/generate-rust-dashboards/src/metrics/sync.rs +++ b/tools/generate-rust-dashboards/src/metrics/sync.rs @@ -3,11 +3,11 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use crate::{ - config::{Application, ReleaseChannel, TeamConfig}, + config::{Application, ReleaseChannel, TeamConfig, Unit}, schema::{ - CalculateFieldOptions, CustomVariable, Dashboard, DashboardBuilder, DataLink, Datasource, - FieldConfig, FieldConfigCustom, FieldConfigDefaults, GridPos, LogOptions, LogPanel, Panel, - Target, TextPanel, TimeSeriesPanel, Transformation, WindowFunctionWindow, + CustomVariable, Dashboard, DashboardBuilder, DataLink, Datasource, FieldConfig, + FieldConfigCustom, FieldConfigDefaults, GridPos, LogOptions, LogPanel, Panel, + ScaleDistribution, Target, TextPanel, TimeSeriesPanel, Transformation, }, sql::Query, util::{Join, UrlBuilder}, @@ -15,14 +15,27 @@ use crate::{ }; pub fn add_to_main_dashboard(builder: &mut DashboardBuilder, config: &TeamConfig) -> Result<()> { - builder.add_panel_title("Sync"); + add_overview_panels( + builder, + "Sync success rate", + config, + SyncMetric::SuccessRate, + ); + add_overview_panels( + builder, + "Sync: total counts", + config, + SyncMetric::TotalCounts, + ); + add_overview_panels( + builder, + "Sync: average time", + config, + SyncMetric::AverageTime, + ); - for app in config.applications().iter() { - builder.add_panel_third(overview_count_panel(config, *app, ReleaseChannel::Nightly)); - builder.add_panel_third(overview_count_panel(config, *app, ReleaseChannel::Beta)); - builder.add_panel_third(overview_count_panel(config, *app, ReleaseChannel::Release)); - } if config.team_name == "SYNC" { + builder.add_panel_title("Legacy dashboards"); builder.add_panel_full(sync_legacy_dashboard_panel()); } @@ -32,7 +45,7 @@ pub fn add_to_main_dashboard(builder: &mut DashboardBuilder, config: &TeamConfig pub fn extra_dashboard(config: &TeamConfig) -> Result { let mut builder = DashboardBuilder::new( format!("{} - Sync Details", config.team_name), - format!("{}-sync-details", config.team_slug()), + format!("{}-sync-extra", config.team_slug()), ); builder.add_application_variable(config)?; builder.add_channel_variable(); @@ -51,13 +64,15 @@ pub fn extra_dashboard(config: &TeamConfig) -> Result { builder.add_panel_title("Metrics"); builder.add_panel_full(details_dash_count_panel( "Success Rate", - "success_rate", - false, + SyncMetric::SuccessRate, )); builder.add_panel_full(details_dash_count_panel( "Total counts (7 day moving average)", - "count_total", - true, + SyncMetric::TotalCounts, + )); + builder.add_panel_full(details_dash_count_panel( + "Average sync time (7 day moving average)", + SyncMetric::AverageTime, )); builder.add_panel_title("Errors"); builder.add_panel_full(details_dash_error_count_panel(config)); @@ -66,10 +81,99 @@ pub fn extra_dashboard(config: &TeamConfig) -> Result { Ok(builder.dashboard) } -fn overview_count_panel( +#[derive(Clone, Copy)] +enum SyncMetric { + SuccessRate, + TotalCounts, + AverageTime, +} + +impl SyncMetric { + fn column_name(&self) -> &'static str { + match &self { + SyncMetric::SuccessRate => "success_rate", + SyncMetric::TotalCounts => "count_total", + SyncMetric::AverageTime => "avg_sync_time", + } + } + + fn moving_average(&self) -> bool { + matches!(self, SyncMetric::TotalCounts | SyncMetric::AverageTime) + } + + fn field_config_custom(&self) -> FieldConfigCustom { + match self { + SyncMetric::SuccessRate => FieldConfigCustom { + axis_label: "success rate".into(), + axis_soft_min: 99, + axis_soft_max: 100, + ..FieldConfigCustom::default() + }, + SyncMetric::TotalCounts => FieldConfigCustom { + scale_distribution: ScaleDistribution { + type_: "log".into(), + log: Some(10), + }, + ..FieldConfigCustom::default() + }, + SyncMetric::AverageTime => FieldConfigCustom::default(), + } + } + + fn unit(&self) -> Option { + match self { + SyncMetric::SuccessRate => None, + SyncMetric::TotalCounts => Some(Unit::SiShort), + SyncMetric::AverageTime => Some(Unit::Seconds), + } + } + + fn column_expr(&self) -> String { + let column_name = self.column_name(); + if !self.moving_average() { + column_name.into() + } else { + format!( + "AVG({column_name}) OVER ( + PARTITION BY engine_name + ORDER BY submission_date + ROWS BETWEEN 6 PRECEDING AND CURRENT ROW + ) AS {column_name}" + ) + } + } +} + +fn add_overview_panels( + builder: &mut DashboardBuilder, + title: &str, + config: &TeamConfig, + metric: SyncMetric, +) { + builder.add_panel_title(title); + + for app in config.applications().iter() { + builder.add_panel_third(overview_panel( + config, + *app, + ReleaseChannel::Nightly, + metric, + )); + builder.add_panel_third(overview_panel(config, *app, ReleaseChannel::Beta, metric)); + builder.add_panel_third(overview_panel( + config, + *app, + ReleaseChannel::Release, + metric, + )); + } +} + +fn overview_panel( config: &TeamConfig, application: Application, channel: ReleaseChannel, + metric: SyncMetric, ) -> Panel { if application == Application::Ios && channel == ReleaseChannel::Nightly { return TextPanel { @@ -80,7 +184,28 @@ fn overview_count_panel( .into(); } - let query = count_query(config, application, format!("'{channel}'")); + let column_name = metric.column_name(); + let query = Query { + select: vec![ + "TIMESTAMP(submission_date) as time".into(), + "engine_name".into(), + metric.column_expr(), + ], + from: format!("({COMBINED_SUBQUERY})"), + where_: vec![ + "$__timeFilter(TIMESTAMP(submission_date))".into(), + format!("channel = '{channel}'"), + match application { + Application::Desktop => "application = 'desktop'", + Application::Ios => "application = 'firefox-ios'", + Application::Android => "application = 'firefox-android'", + } + .into(), + engine_where_clause(config), + ], + order_by: Some("time".into()), + ..Query::default() + }; Panel::from(TimeSeriesPanel { title: application.display_name(channel), @@ -88,11 +213,11 @@ fn overview_count_panel( datasource: Datasource::bigquery(), // needs to be fairly large since the total sync count can be low on mobile/nightly interval: "1d".into(), - targets: vec![Target::table(query)], + targets: vec![Target::table(query.sql())], field_config: FieldConfig { defaults: FieldConfigDefaults { links: vec![DataLink { - url: UrlBuilder::new_dashboard(format!("{}-sync-details", config.team_slug())) + url: UrlBuilder::new_dashboard(format!("{}-sync-extra", config.team_slug())) .with_time_range_param() .with_param("var-application", application.slug()) .with_param("var-channel", channel.to_string()) @@ -102,13 +227,8 @@ fn overview_count_panel( one_click: true, title: "Errors".into(), }], - custom: FieldConfigCustom { - axis_label: "success rate".into(), - axis_soft_min: 99, - axis_soft_max: 100, - ..FieldConfigCustom::default() - }, - unit: None, + custom: metric.field_config_custom(), + unit: metric.unit(), }, }, transformations: vec![ @@ -118,7 +238,7 @@ fn overview_count_panel( }, // Fixup the field names for better legend labels Transformation::RenameByRegex { - regex: "success_rate (.*)".into(), + regex: format!("{column_name} (.*)"), rename_pattern: "$1".into(), }, ], @@ -126,71 +246,28 @@ fn overview_count_panel( }) } -/// Query to fetch sync success rates -fn count_query(config: &TeamConfig, application: Application, channel_expr: String) -> String { - let table_name = if application == Application::Desktop { - "desktop_v1" - } else { - "mobile_v1" - }; - let application_where = match application { - Application::Desktop => "application = 'desktop'", - Application::Ios => "application = 'firefox-ios'", - Application::Android => "application = 'firefox-android'", - }; - - let mut engines: Vec<_> = config - .components - .iter() - .flat_map(|c| c.sync_engines()) - .map(|e| format!("'{e}'")) - .collect(); - engines.sort_unstable(); - engines.dedup(); - let engines_where = format!("engine_name IN ({})", engines.join(", ")); - - format!( - "\ -SELECT - TIMESTAMP(submission_date) as time, - engine_name, - success_rate -FROM - moz-fx-data-shared-prod.sync_derived.{table_name} -WHERE - channel = {channel_expr} - AND $__timeFilter(TIMESTAMP(submission_date)) - AND {application_where} - AND {engines_where} -ORDER BY time" - ) -} - -fn details_dash_count_panel(title: &str, column_name: &str, moving_average: bool) -> Panel { +fn details_dash_count_panel(title: &str, metric: SyncMetric) -> Panel { let query = Query { - select: vec!["time".into(), column_name.into()], - from: format!("(\n{}\n)", details_dash_count_query()), - group_by: Some("1, 2".into()), + select: vec![ + "TIMESTAMP(submission_date) as time".into(), + metric.column_expr(), + ], + from: format!("(\n{COMBINED_SUBQUERY}\n)"), + where_: vec![ + "$__timeFilter(TIMESTAMP(submission_date))".into(), + "channel = '${channel}'".into(), + "application=CASE '${application}' + WHEN 'firefox_desktop' THEN 'desktop' + WHEN 'firefox_android' THEN 'firefox-android' + WHEN 'firefox_ios' THEN 'firefox-ios' + ELSE '${application}' + END" + .into(), + "engine_name = '${engine}'".into(), + ], ..Query::default() }; - let transformations = if moving_average { - vec![Transformation::CalculateField( - CalculateFieldOptions::WindowFunctions { - replace_fields: true, - window: WindowFunctionWindow { - field: "count_total".into(), - reducer: "mean".into(), - window_alignment: "centered".into(), - window_size: 7.0, - window_size_mode: "fixed".into(), - }, - }, - )] - } else { - vec![] - }; - TimeSeriesPanel { title: title.into(), grid_pos: GridPos::height(10), @@ -198,54 +275,19 @@ fn details_dash_count_panel(title: &str, column_name: &str, moving_average: bool // needs to be fairly large since the total sync count can be low on mobile/nightly interval: "1d".into(), targets: vec![Target::table(query.sql())], - transformations, + transformations: vec![], + field_config: FieldConfig { + defaults: FieldConfigDefaults { + links: vec![], + custom: metric.field_config_custom(), + unit: metric.unit(), + }, + }, ..TimeSeriesPanel::default() } .into() } -/// Query to count metrics for the details dashboard -fn details_dash_count_query() -> String { - "\ -SELECT - TIMESTAMP(submission_date) as time, - success_rate, - count_total -FROM - moz-fx-data-shared-prod.sync_derived.desktop_v1 -WHERE - $__timeFilter(TIMESTAMP(submission_date)) - AND channel = '${channel}' - AND application=CASE '${application}' - WHEN 'firefox_desktop' THEN 'desktop' - WHEN 'firefox_android' THEN 'firefox-android' - WHEN 'firefox_ios' THEN 'firefox-ios' - ELSE '${application}' - END - AND engine_name = '${engine}' - -UNION ALL - -SELECT - TIMESTAMP(submission_date) as time, - success_rate, - count_total -FROM - moz-fx-data-shared-prod.sync_derived.mobile_v1 -WHERE - $__timeFilter(TIMESTAMP(submission_date)) - AND channel = '${channel}' - AND application=CASE '${application}' - WHEN 'firefox_desktop' THEN 'desktop' - WHEN 'firefox_android' THEN 'firefox-android' - WHEN 'firefox_ios' THEN 'firefox-ios' - ELSE '${application}' - END - AND engine_name = '${engine}' -ORDER BY time" - .to_string() -} - fn details_dash_error_count_panel(config: &TeamConfig) -> Panel { let query = Query { select: vec![ @@ -255,7 +297,7 @@ fn details_dash_error_count_panel(config: &TeamConfig) -> Panel { ], where_: vec![ "application='${application}'".into(), - "engine_name='${engine}'".into(), + "engine_name = '${engine}'".into(), "normalized_channel = '${channel}'".into(), "$__timeFilter(submission_timestamp)".into(), ], @@ -295,7 +337,7 @@ fn details_dash_error_log_panel(config: &TeamConfig) -> Panel { ], from: format!("(\n{}\n)", error_subquery(config)), where_: vec![ - "engine_name='${engine}'".into(), + "engine_name = '${engine}'".into(), "normalized_channel = '${channel}'".into(), "application='${application}'".into(), "$__timeFilter(submission_timestamp)".into(), @@ -351,6 +393,8 @@ WHERE .components .iter() .flat_map(|c| c.sync_engines()) + // filter out desktop-only engines + .filter(|c| **c != "rust-logins") .flat_map(|engine_name| { [ format!( @@ -387,13 +431,6 @@ WHERE fn sync_legacy_dashboard_panel() -> Panel { let content = "\ # Legacy Sync dashboards -* [Sync: engine performance](https://sql.telemetry.mozilla.org/dashboard/sync-engine-performance?p_w73231_Days=7&p_w73233_Days=7&p_w73234_Days=7&p_w73237_Days=7&p_w73238_Days=60&p_w73239_Days=60&p_w73248_Days=60&p_w73249_Days=60&p_w73250_Days=60&p_w73251_Days=60&p_w73255_Days=7&p_w73256_Days=7&p_w73257_Days=60) -* [Desktop Sync Failures](https://sql.telemetry.mozilla.org/dashboard/sync-desktop?p_Days=60&p_engine_name=all-engines&p_w63728_engine_name=all-engines&p_w64027_engine_name=all-engines&p_w64028_engine_name=all-engines&p_w64029_engine_name=all-engines&p_w65780_channel=beta&p_w65780_days=30&p_w65780_engine_name=all-engines) -* [Android Sync Failures](https://sql.telemetry.mozilla.org/dashboard/android-sync-failures?p_channel=org_mozilla_fenix&p_engine_name=credit-cards&p_w64121_Months=24&p_w64121_engine_name=all-engines&p_w64122_Months=24&p_w64122_engine_name=all-engines&p_w64123_Months=24&p_w64123_engine_name=all-engines&p_w73261_Months=1&p_w73261_engine=bookmarks&p_w73261_minimum_error_count=0&p_w73262_Months=1&p_w73262_engine=bookmarks&p_w73262_minimum_error_count=0&p_w73263_Months=1&p_w73263_engine=bookmarks&p_w73263_minimum_error_count=0) -* [iOS Sync failures](https://sql.telemetry.mozilla.org/dashboard/ios-sync-failures?p_Days=60&p_Months=1&p_engine_name=all-engines&p_w67318_Months=1&p_w67318_engine=bookmarks&p_w67318_minimum%20error%20count=0&p_w67320_Months=1&p_w67320_engine=bookmarks&p_w67320_minimum%20error%20count=0) -* [Android Logins key regeneration errors](https://sql.telemetry.mozilla.org/queries/83554#207048) -* [iOS Logins Key Regeneration Metrics](https://sql.telemetry.mozilla.org/dashboard/ios-logins-key-regeneration-metrics) -* [iOS Credit Cards Key Regeneration Metrics](https://sql.telemetry.mozilla.org/dashboard/ios-credit-cards-key-regeneration-metrics) * [iOS Credit Cards Verification Usage](https://sql.telemetry.mozilla.org/dashboard/ios-credit-cards-verification-usage) * [Mobile Logins Verification Usage](https://sql.telemetry.mozilla.org/dashboard/mobile-logins-verification-usage?p_channel=org_mozilla_ios_firefox) * [iOS FxA Keychain Rollout Enrollment](https://sql.telemetry.mozilla.org/dashboard/ios-credit-cards-key-regeneration-metrics) @@ -405,3 +442,42 @@ fn sync_legacy_dashboard_panel() -> Panel { } .into() } + +fn engine_where_clause(config: &TeamConfig) -> String { + let mut engines: Vec<_> = config + .components + .iter() + .flat_map(|c| c.sync_engines()) + .map(|e| format!("'{e}'")) + .collect(); + engines.sort_unstable(); + engines.dedup(); + format!("engine_name IN ({})", engines.join(", ")) +} + +/// Subquery that combines the desktop and mobile ETL tables +const COMBINED_SUBQUERY: &str = "\ +SELECT + submission_date, + channel, + application, + engine_name, + success_rate, + avg_sync_time, + count_total +FROM + moz-fx-data-shared-prod.sync_derived.desktop_v1 + +UNION ALL + +SELECT + submission_date, + channel, + application, + engine_name, + success_rate, + 0 as avg_sync_time, -- TODO: make this work on Mobile + count_total +FROM + moz-fx-data-shared-prod.sync_derived.mobile_v1 +"; diff --git a/tools/generate-rust-dashboards/src/schema.rs b/tools/generate-rust-dashboards/src/schema.rs index 42b4df538f..62403d40bf 100644 --- a/tools/generate-rust-dashboards/src/schema.rs +++ b/tools/generate-rust-dashboards/src/schema.rs @@ -128,6 +128,8 @@ pub enum Unit { Microseconds, #[serde(rename = "ns")] Nanoseconds, + #[serde(rename = "sishort")] + SiShort, } #[derive(Default, Serialize)] diff --git a/tools/generate-rust-dashboards/src/team_config.rs b/tools/generate-rust-dashboards/src/team_config.rs index 8fea31c10e..e3b0a18429 100644 --- a/tools/generate-rust-dashboards/src/team_config.rs +++ b/tools/generate-rust-dashboards/src/team_config.rs @@ -43,7 +43,15 @@ pub fn all_dashboards() -> Vec { "key_regenerated_corrupt", "key_regenerated_other", ], - applications: vec![Android], + applications: vec![Android, Ios], + } + .into(), + EventsMetric { + display_name: "Credit Card key regeneration", + ping: "metrics", + category: "credit_card_key_regeneration", + metrics: vec!["keychain_data_lost", "lost", "corrupt", "other"], + applications: vec![Ios], } .into(), ],