Rework instrumentation filtering

.circleci/config.yml

@@ -3,9 +3,15 @@ defaults: &defaults
   working_directory: ~/root/project
   resource_class: large
   docker:
-    - image: cimg/android:2025.12.1
+    - image: cimg/android:2026.08.1
       environment:
-        GRADLE_OPTS: -Xmx4096m -XX:+HeapDumpOnOutOfMemoryError -Dorg.gradle.daemon=false -Dorg.gradle.caching=true -Dorg.gradle.configureondemand=true -Dkotlin.compiler.execution.strategy=in-process -Dkotlin.incremental=false
+        GRADLE_OPTS: >-
+          -Xmx4096m
+          -XX:+HeapDumpOnOutOfMemoryError
+          -Dorg.gradle.caching=true
+          -Dorg.gradle.configureondemand=true
+          -Dorg.gradle.workers.max=2
+          -Dkotlin.compiler.execution.strategy=in-process
 
 cache_key: &cache_key
   key: jars-{{ checksum "build-logic/build.gradle.kts" }}-{{ checksum "plugin/build.gradle.kts" }}-{{ checksum "plugin/android-junit5/build.gradle.kts" }}-{{ checksum "plugin/gradle/wrapper/gradle-wrapper.properties" }}-{{ checksum "instrumentation/build.gradle.kts" }}-{{ checksum "instrumentation/core/build.gradle.kts" }}-{{ checksum "instrumentation/compose/build.gradle.kts" }}-{{ checksum "instrumentation/extensions/build.gradle.kts" }}-{{ checksum "instrumentation/runner/build.gradle.kts" }}-{{ checksum "instrumentation/sample/build.gradle.kts" }}-{{ checksum "instrumentation/gradle/wrapper/gradle-wrapper.properties" }}-{{ checksum "build-logic/src/main/kotlin/Environment.kt" }}-{{ checksum "build-logic/gradle/libs.versions.toml" }}
@@ -30,15 +36,6 @@ jobs:
       - run:
           name: (Plugin) Test
           command: cd plugin && ./gradlew :build-logic:test :android-junit5:check --stacktrace
-      - run:
-          name: (Instrumentation) Download Dependencies
-          command: >
-            cd instrumentation && ./gradlew \
-              :core:dependencies \
-              :compose:dependencies \
-              :extensions:dependencies \
-              :runner:dependencies \
-              :testutil:dependencies
       - run:
           name: (Instrumentation) Build
           command: cd instrumentation && ./gradlew assemble :core:assembleAndroidTest --stacktrace

build-logic/gradle/libs.versions.toml

@@ -70,7 +70,6 @@ junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api" }
 junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine" }
 junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" }
 junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher" }
-junit-platform-suiteapi = { module = "org.junit.platform:junit-platform-suite-api" }
 junit-vintage-api = { module = "junit:junit", version.ref = "junit4" }
 junit-vintage-engine = { module = "org.junit.vintage:junit-vintage-engine" }
 kgp = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" }

build-logic/src/main/kotlin/Environment.kt

@@ -38,8 +38,9 @@ enum class SupportedAgp(
     AGP_9_0("9.0.1", gradle = "9.1.0"),
     AGP_9_1("9.1.1", gradle = "9.3.1"),
     AGP_9_2("9.2.1", gradle = "9.4.1"),
-    AGP_9_3("9.3.0-rc01", gradle = "9.5.0"),
-    AGP_9_4("9.4.0-alpha01", gradle = "9.5.0"),
+    AGP_9_3("9.3.3", gradle = "9.5.0"),
+    AGP_9_4("9.4.0", gradle = "9.6.0"),
+    AGP_9_5("9.5.0-alpha06", gradle = "9.6.0"),
     ;
 
     companion object {

instrumentation/buildSrc/src/main/java/LocalProjectDependencyConfigurator.kt

@@ -2,14 +2,19 @@ import extensions.library
 import extensions.libs
 import org.gradle.api.Project
 
+private val applicableConfigurationNameRegex =
+    Regex("[a-z]+[A-Z][a-z]+((Android)|(Unit))Test[A-Z][a-z]+Classpath")
+
 private val instrumentationLibraryRegex =
     Regex("de\\.mannodermaus\\.junit5:android-test-([a-z0-9]+)(-.+)?:")
 
 fun Project.replaceAndroidTestLibsWithLocalProjectDependencies() {
     val self = this
 
     configurations.all {
-        if ("DebugAndroidTestRuntimeClasspath" !in name) {
+        val configuration = this
+
+        if (!name.matches(applicableConfigurationNameRegex)) {
             return@all
         }
 
@@ -32,8 +37,9 @@ fun Project.replaceAndroidTestLibsWithLocalProjectDependencies() {
             dependencySubstitution.all {
                 instrumentationLibraryRegex.find(requested.toString())?.let { result ->
                     val replacement = project(":${result.groupValues[1]}")
-                    println(
-                        "In $self, replace androidTest dependency '$requested' with $replacement"
+                    logger.lifecycle(
+                        "$self will replace dependency on '$requested' " +
+                                "with $replacement in $configuration"
                     )
                     useTarget(replacement, "Use $replacement to substitute dependency '$requested'")
                 }

instrumentation/core/build.gradle.kts

@@ -30,7 +30,6 @@ dependencies {
     // since it can't provide any JUnit 5 runtime libraries itself
     // due to fear of prematurely incrementing the minSdkVersion requirement.
     runtimeOnly(libs.junit.platform.launcher)
-    runtimeOnly(libs.junit.platform.suiteapi)
     runtimeOnly(libs.junit.jupiter.engine)
 
     // This transitive dependency of JUnit 5 is required to be on the runtime classpath,

instrumentation/runner/build.gradle.kts

@@ -33,7 +33,6 @@ dependencies {
     compileOnly(libs.junit.jupiter.api)
     compileOnly(libs.junit.jupiter.params)
     compileOnly(libs.junit.platform.launcher)
-    compileOnly(libs.junit.platform.suiteapi)
 
     testImplementation(project(":testutil"))
     testImplementation(libs.robolectric)

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/AndroidJUnitFrameworkBuilder.kt

@@ -58,9 +58,6 @@ public open class AndroidJUnitFrameworkBuilder internal constructor() : RunnerBu
 
     @Throws(Throwable::class)
     override fun runnerForClass(testClass: Class<*>): Runner? {
-        // Ignore a bunch of classes in internal packages
-        if (testClass.isInIgnorablePackage) return null
-
         try {
             return if (junitFrameworkAvailable) {
                 tryCreateJUnitFrameworkRunner(testClass, params)
@@ -82,14 +79,6 @@ public open class AndroidJUnitFrameworkBuilder internal constructor() : RunnerBu
 
     /* Private */
 
-    private val ignorablePackages =
-        setOf("java.", "javax.", "androidx.", "com.android.", "kotlin.", "kotlinx.")
-
-    private val Class<*>.isInIgnorablePackage: Boolean
-        get() {
-            return ignorablePackages.any { name.startsWith(it) }
-        }
-
     private fun JUnitFrameworkRunnerParams.registerEnvironmentVariables() {
         environmentVariables.forEach { (key, value) ->
             try {

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitFramework.kt

@@ -7,6 +7,7 @@ import de.mannodermaus.junit5.internal.runners.notification.ParallelRunNotifier
 import org.junit.platform.commons.JUnitException
 import org.junit.platform.engine.discovery.MethodSelector
 import org.junit.platform.launcher.core.LauncherFactory
+import org.junit.runner.Description
 import org.junit.runner.Runner
 import org.junit.runner.notification.RunNotifier
 
@@ -20,22 +21,33 @@ internal class AndroidJUnitFramework(
     private val testClass: Class<*>,
     params: JUnitFrameworkRunnerParams,
 ) : Runner() {
-    private val launcher = LauncherFactory.create()
+    private companion object {
+        private val launcher = LauncherFactory.create()
+
+        // Fallback for irrelevant classes passed to JUnit 4's RunnerBuilder
+        // (no test tree will be created for those, avoiding any potentially dangerous
+        // runtime lookups that can cause issues like `mannodermaus/android-junit-framework/413`)
+        private val emptyDescription = Description.createSuiteDescription("<empty>")
+    }
+
     private val testTree by lazy { generateTestTree(params) }
 
-    override fun getDescription() = testTree.suiteDescription
+    override fun getDescription(): Description = testTree?.suiteDescription ?: emptyDescription
 
     override fun run(notifier: RunNotifier) {
-        // Finally, launch the test plan on the JUnit Platform
-        launcher.execute(
-            testTree.testPlan,
-            AndroidJUnitPlatformRunnerListener(testTree, createNotifier(notifier)),
-        )
+        testTree?.let { tree ->
+            launcher.execute(
+                tree.testPlan,
+                AndroidJUnitPlatformRunnerListener(tree, tree.createNotifier(notifier)),
+            )
+        }
     }
 
     /* Private */
 
-    private fun generateTestTree(params: JUnitFrameworkRunnerParams): AndroidJUnitPlatformTestTree {
+    private fun generateTestTree(
+        params: JUnitFrameworkRunnerParams
+    ): AndroidJUnitPlatformTestTree? {
         val selectors = params.createSelectors(testClass)
         val isIsolatedMethodRun = selectors.size == 1 && selectors.first() is MethodSelector
         val isUsingOrchestrator = params.isUsingOrchestrator
@@ -67,16 +79,20 @@ internal class AndroidJUnitFramework(
                 EmptyTestPlan
             }
 
-        return AndroidJUnitPlatformTestTree(
-            testPlan = testPlan,
-            testClass = testClass,
-            needLegacyFormat = isIsolatedMethodRun || isUsingOrchestrator,
-            isParallelExecutionEnabled = params.isParallelExecutionEnabled,
-        )
+        return if (testPlan.containsTests()) {
+            AndroidJUnitPlatformTestTree(
+                testPlan = testPlan,
+                testClass = testClass,
+                needLegacyFormat = isIsolatedMethodRun || isUsingOrchestrator,
+                isParallelExecutionEnabled = params.isParallelExecutionEnabled,
+            )
+        } else {
+            null
+        }
     }
 
-    private fun createNotifier(nextNotifier: RunNotifier) =
-        if (testTree.isParallelExecutionEnabled) {
+    private fun AndroidJUnitPlatformTestTree.createNotifier(nextNotifier: RunNotifier) =
+        if (isParallelExecutionEnabled) {
             // Wrap the default notifier with a special handler for parallel test execution
             ParallelRunNotifier(nextNotifier)
         } else {

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitPlatformTestTree.kt

@@ -7,13 +7,11 @@ import de.mannodermaus.junit5.internal.extensions.format
 import de.mannodermaus.junit5.internal.extensions.isDynamicTest
 import java.util.Optional
 import java.util.function.Predicate
-import org.junit.platform.commons.util.AnnotationUtils
 import org.junit.platform.engine.UniqueId
 import org.junit.platform.engine.support.descriptor.ClassSource
 import org.junit.platform.engine.support.descriptor.MethodSource
 import org.junit.platform.launcher.TestIdentifier
 import org.junit.platform.launcher.TestPlan
-import org.junit.platform.suite.api.SuiteDisplayName
 import org.junit.runner.Description
 
 /**
@@ -82,17 +80,10 @@ internal class AndroidJUnitPlatformTestTree(
         return descriptions.getValue(identifier)
     }
 
-    private fun generateSuiteDescription(testPlan: TestPlan, testClass: Class<*>): Description {
-        return Description.createSuiteDescription(getSuiteDisplayName(testClass)).also {
+    private fun generateSuiteDescription(testPlan: TestPlan, testClass: Class<*>): Description =
+        Description.createSuiteDescription(testClass.name).also {
             buildDescriptionTree(it, testPlan)
         }
-    }
-
-    private fun getSuiteDisplayName(testClass: Class<*>): String =
-        AnnotationUtils.findAnnotation(testClass, SuiteDisplayName::class.java)
-            .map(SuiteDisplayName::value)
-            .filter(String::isNotBlank)
-            .orElse(testClass.name)
 
     private fun buildDescriptionTree(suiteDescription: Description, testPlan: TestPlan) {
         testPlan.roots.forEach { identifier ->

instrumentation/testutil/build.gradle.kts

@@ -24,5 +24,4 @@ dependencies {
     api(libs.junit.jupiter.api)
     api(libs.junit.jupiter.params)
     api(libs.junit.platform.launcher)
-    api(libs.junit.platform.suiteapi)
 }

plugin/android-junit5/src/test/kotlin/de/mannodermaus/gradle/plugins/junit5/FunctionalTests.kt

@@ -3,26 +3,16 @@ package de.mannodermaus.gradle.plugins.junit5
 import com.google.common.truth.Truth.assertWithMessage
 import de.mannodermaus.gradle.plugins.junit5.annotations.DisabledOnCI
 import de.mannodermaus.gradle.plugins.junit5.internal.extensions.capitalized
-import de.mannodermaus.gradle.plugins.junit5.util.BuildResultSubject
-import de.mannodermaus.gradle.plugins.junit5.util.TestEnvironment
-import de.mannodermaus.gradle.plugins.junit5.util.TestedAgp
-import de.mannodermaus.gradle.plugins.junit5.util.TestedJUnit
-import de.mannodermaus.gradle.plugins.junit5.util.prettyPrint
+import de.mannodermaus.gradle.plugins.junit5.util.*
 import de.mannodermaus.gradle.plugins.junit5.util.projects.FunctionalTestProjectCreator
-import de.mannodermaus.gradle.plugins.junit5.util.withPrunedPluginClasspath
 import java.io.File
 import org.gradle.testkit.runner.BuildResult
 import org.gradle.testkit.runner.GradleRunner
 import org.gradle.testkit.runner.TaskOutcome
-import org.junit.jupiter.api.AfterAll
-import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.*
 import org.junit.jupiter.api.DynamicContainer.dynamicContainer
-import org.junit.jupiter.api.DynamicNode
 import org.junit.jupiter.api.DynamicTest.dynamicTest
-import org.junit.jupiter.api.TestFactory
-import org.junit.jupiter.api.TestInstance
 import org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS
-import org.junit.jupiter.api.fail
 
 @TestInstance(PER_CLASS)
 @DisabledOnCI
@@ -78,7 +68,7 @@ class FunctionalTests {
                         "AGP ${agp.shortVersion}",
                         // Exercise each test project within the given environment
                         projectCreator.allSpecs.filterSpecs().map { spec ->
-                            dynamicTest("${spec.name} ($junit)") {
+                            dynamicTest("${spec.name} (${junit.fullVersion})") {
                                 // Required for visibility inside the IntelliJ logging console
                                 // (display names are still bugged in the IDE)
                                 println(