From 0d3d922512e8e2525fe4f5898fa36383f7a0f68c Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 6 Aug 2026 16:46:51 +0200 Subject: [PATCH] test(android): Add SentryAndroid.init timing to macrobenchmark (JAVA-685) Read the SentryAndroid.init android.os.Trace section via a TraceSectionMetric so the cold-start macrobenchmark reports SDK-init duration on its own, separate from timeToInitialDisplay. This resolves init-cost changes that the whole-start metric would bury in cold-start noise. Co-Authored-By: Claude Opus 4.8 --- .../README.md | 24 +++++++++++-------- .../macrobenchmark/SentryStartupBenchmark.kt | 18 ++++++++++---- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md index eae36ac178f..4b89b7b6105 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md @@ -5,14 +5,17 @@ performance changes on a real device in a **stable, reproducible** way. Not run ## What it measures -`SentryStartupBenchmark` runs a cold start and reports **`timeToInitialDisplay`** -(`StartupTimingMetric`) per iteration — the whole app cold start, taken from framework trace -events. No trace markers are required in the SDK or the app. - -The flip side of marker-free measurement: an SDK change has to be large enough (roughly tens of -milliseconds) to show above cold-start noise. Sub-millisecond changes are not resolvable with -`timeToInitialDisplay` alone; for those, capture a perfetto trace and inspect the relevant slices -directly (each iteration's trace is saved under +`SentryStartupBenchmark` runs a cold start and reports two metrics per iteration: + +- **`timeToInitialDisplay`** (`StartupTimingMetric`) — the whole app cold start, taken from + framework trace events. Because it captures the entire start, an SDK change has to be large enough + (roughly tens of milliseconds) to show above cold-start noise. +- **`SentryAndroid.init`** (`TraceSectionMetric`) — the duration of the `SentryAndroid.init` + `android.os.Trace` section the SDK emits, which isolates SDK-init cost from the rest of the start + and resolves changes that `timeToInitialDisplay` would lose in the noise. + +For even finer detail (sub-millisecond changes, or cost inside init), capture a perfetto trace and +inspect the relevant slices directly (each iteration's trace is saved under `build/outputs/connected_android_test_additional_output/`). `CompilationMode.Full()` pins ART AOT so dexopt state can't drift between runs. `StartupMode.COLD` @@ -49,5 +52,6 @@ Results print to the console and are written to Macrobenchmark measures one build per run, so compare separate runs — but **interleave them**: running all of variant A followed by all of variant B lets thermal drift systematically penalize whichever variant runs second. Instead, alternate A/B rounds (build variant A, run, build variant -B, run, repeat 2–3 times), keep each round's `*-benchmarkData.json`, and compare the -`timeToInitialDisplay` values pooled per variant. +B, run, repeat 2–3 times), keep each round's `*-benchmarkData.json`, and compare the values pooled +per variant. Prefer the `SentryAndroid.init` metric for SDK-init changes — it isolates init cost, so +it moves on changes that `timeToInitialDisplay` would bury in cold-start noise. diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt index 24b8707ba5d..ee49fe8beff 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt @@ -1,8 +1,10 @@ package io.sentry.uitest.android.macrobenchmark import androidx.benchmark.macro.CompilationMode +import androidx.benchmark.macro.ExperimentalMetricApi import androidx.benchmark.macro.StartupMode import androidx.benchmark.macro.StartupTimingMetric +import androidx.benchmark.macro.TraceSectionMetric import androidx.benchmark.macro.junit4.MacrobenchmarkRule import androidx.test.ext.junit.runners.AndroidJUnit4 import org.junit.Rule @@ -13,10 +15,12 @@ import org.junit.runner.RunWith * Cold-start benchmark for the sentry-samples-android app, used to evaluate SDK-init changes on a * real device in a stable, repeatable way. * - * Reports timeToInitialDisplay ([StartupTimingMetric]) per iteration. This measures the whole app - * cold start from framework trace events, with no trace markers in the SDK or the app — which also - * means SDK changes need to be large enough (roughly tens of milliseconds) to show above cold-start - * noise. + * Reports two metrics per iteration: + * - timeToInitialDisplay ([StartupTimingMetric]) — the whole app cold start from framework trace + * events. Because it captures the entire start, an SDK change has to be large enough (roughly + * tens of milliseconds) to show above cold-start noise. + * - SentryAndroid.init ([TraceSectionMetric]) — the duration of the `SentryAndroid.init` + * [android.os.Trace] section the SDK emits, isolating SDK-init cost from the rest of the start. * * [CompilationMode.Full] pins ART AOT compilation so dexopt state does not drift between runs. * Iterations are capped at 12: on an unthrottled Pixel 3, back-to-back cold starts hit thermal @@ -24,6 +28,7 @@ import org.junit.runner.RunWith * it requires a connected device. To A/B an SDK change, see README.md (build the app twice, once * per SDK variant, in interleaved rounds). */ +@OptIn(ExperimentalMetricApi::class) @RunWith(AndroidJUnit4::class) class SentryStartupBenchmark { @@ -33,7 +38,7 @@ class SentryStartupBenchmark { fun startupFullCompilation() = benchmarkRule.measureRepeated( packageName = TARGET_PACKAGE, - metrics = listOf(StartupTimingMetric()), + metrics = listOf(StartupTimingMetric(), TraceSectionMetric(INIT_TRACE_SECTION)), compilationMode = CompilationMode.Full(), startupMode = StartupMode.COLD, iterations = 12, @@ -44,5 +49,8 @@ class SentryStartupBenchmark { private companion object { const val TARGET_PACKAGE = "io.sentry.samples.android" + + // Matches the android.os.Trace section name in SentryAndroid.init. + const val INIT_TRACE_SECTION = "SentryAndroid.init" } }