Skip to content

Restore junit4 test ordering on gradle 9.3+ #12098

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
daniel-mohedano wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Restore junit4 test ordering on gradle 9.3+ #12098

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8aacf2e
fix: restore junit4 test ordering on gradle 9.3+
daniel-mohedano Jul 29, 2026
4a7dc8f
Merge branch 'master' into daniel.mohedano/junit4-gradle-93-ordering-fix
daniel-mohedano Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions ...ce/instrumentation/gradle/junit4/AbstractJUnitTestDefinitionProcessorInstrumentation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package datadog.trace.instrumentation.gradle.junit4;

import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named;

import com.google.auto.service.AutoService;
import datadog.trace.agent.tooling.Instrumenter;
import datadog.trace.agent.tooling.InstrumenterModule;
import datadog.trace.api.Config;
import datadog.trace.api.civisibility.CIConstants;
import datadog.trace.instrumentation.junit4.JUnit4Instrumentation;
import net.bytebuddy.asm.Advice;
import org.gradle.api.internal.tasks.testing.ClassTestDefinition;
import org.gradle.api.internal.tasks.testing.TestDefinitionConsumer;

@AutoService(InstrumenterModule.class)
public class AbstractJUnitTestDefinitionProcessorInstrumentation
extends InstrumenterModule.CiVisibility
implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice {
public AbstractJUnitTestDefinitionProcessorInstrumentation() {
super("ci-visibility", "gradle", "junit4");
}

@Override
public boolean isEnabled() {
return super.isEnabled() && Config.get().getCiVisibilityTestOrder() != null;
}

@Override
public String instrumentedType() {
return "org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestDefinitionProcessor";
}

@Override
public String[] helperClassNames() {
return new String[] {
JUnit4Instrumentation.class.getPackage().getName() + ".SkippedByDatadog",
JUnit4Instrumentation.class.getPackage().getName() + ".JUnit4Utils",
JUnit4Instrumentation.class.getPackage().getName() + ".TestEventsHandlerHolder",
JUnit4Instrumentation.class.getPackage().getName() + ".TracingListener",
JUnit4Instrumentation.class.getPackage().getName() + ".order.JUnit4FailFastClassOrderer",
packageName + ".DDCollectAllTestDefinitionsExecutor",
};
}

@Override
public void methodAdvice(MethodTransformer transformer) {
transformer.applyAdvice(
named("stop"),
AbstractJUnitTestDefinitionProcessorInstrumentation.class.getName()
+ "$ProcessAllTestClassesAdvice");
}

@Override
public String muzzleDirective() {
return "skipMuzzle";
}

public static class ProcessAllTestClassesAdvice {
@SuppressWarnings("bytebuddy-exception-suppression")
@Advice.OnMethodEnter
public static void onStop(
@Advice.FieldValue(value = "executor")
final TestDefinitionConsumer<ClassTestDefinition> executor) {
String testOrder = Config.get().getCiVisibilityTestOrder();
if (!CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(testOrder)) {
throw new IllegalArgumentException("Unknown test order: " + testOrder);
}

if (executor instanceof DDCollectAllTestDefinitionsExecutor) {
((DDCollectAllTestDefinitionsExecutor) executor).processAllTestClasses();
}
}
}
}
59 changes: 59 additions & 0 deletions ...oovy/datadog/trace/instrumentation/gradle/junit4/DDCollectAllTestDefinitionsExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package datadog.trace.instrumentation.gradle.junit4;

import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
import datadog.trace.instrumentation.junit4.JUnit4Utils;
import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder;
import datadog.trace.instrumentation.junit4.order.JUnit4FailFastClassOrderer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.gradle.api.internal.tasks.testing.ClassTestDefinition;
import org.gradle.api.internal.tasks.testing.TestDefinitionConsumer;
import org.gradle.internal.UncheckedException;

public class DDCollectAllTestDefinitionsExecutor
implements TestDefinitionConsumer<ClassTestDefinition> {
private final List<Class<?>> testClasses = new ArrayList<>();
private final Map<String, ClassTestDefinition> testDefinitions = new HashMap<>();
private final TestDefinitionConsumer<ClassTestDefinition> delegate;
private final ClassLoader classLoader;

public DDCollectAllTestDefinitionsExecutor(
TestDefinitionConsumer<ClassTestDefinition> delegate, ClassLoader junitClassLoader) {
this.delegate = delegate;
this.classLoader = junitClassLoader;
}

@Override
public void accept(ClassTestDefinition testDefinition) {
Class<?> clazz = loadClass(testDefinition.getTestClassName());

TestFrameworkInstrumentation framework = JUnit4Utils.classToFramework(clazz);
if (framework == TestFrameworkInstrumentation.JUNIT4) {
TestEventsHandlerHolder.start(
TestFrameworkInstrumentation.JUNIT4, JUnit4Utils.capabilities(true));
}

testClasses.add(clazz);
testDefinitions.put(testDefinition.getTestClassName(), testDefinition);
}

public void processAllTestClasses() {
testClasses.sort(
new JUnit4FailFastClassOrderer(
TestEventsHandlerHolder.HANDLERS.get(TestFrameworkInstrumentation.JUNIT4)));

for (Class<?> clazz : testClasses) {
delegate.accept(testDefinitions.get(clazz.getName()));
}
}

private Class<?> loadClass(String testClassName) {
try {
return Class.forName(testClassName, false, classLoader);
} catch (ClassNotFoundException e) {
throw UncheckedException.throwAsUncheckedException(e);
}
}
}
75 changes: 75 additions & 0 deletions ...adog/trace/instrumentation/gradle/junit4/JUnitTestDefinitionProcessorInstrumentation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package datadog.trace.instrumentation.gradle.junit4;

import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.returns;
import static net.bytebuddy.matcher.ElementMatchers.takesArgument;

import com.google.auto.service.AutoService;
import datadog.trace.agent.tooling.Instrumenter;
import datadog.trace.agent.tooling.InstrumenterModule;
import datadog.trace.api.Config;
import datadog.trace.api.civisibility.CIConstants;
import datadog.trace.instrumentation.junit4.JUnit4Instrumentation;
import net.bytebuddy.asm.Advice;
import org.gradle.api.internal.tasks.testing.ClassTestDefinition;
import org.gradle.api.internal.tasks.testing.TestDefinitionConsumer;

@AutoService(InstrumenterModule.class)
public class JUnitTestDefinitionProcessorInstrumentation extends InstrumenterModule.CiVisibility
implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice {
public JUnitTestDefinitionProcessorInstrumentation() {
super("ci-visibility", "gradle", "junit4");
}

@Override
public boolean isEnabled() {
return super.isEnabled() && Config.get().getCiVisibilityTestOrder() != null;
}

@Override
public String instrumentedType() {
return "org.gradle.api.internal.tasks.testing.junit.JUnitTestDefinitionProcessor";
}

@Override
public String[] helperClassNames() {
return new String[] {
JUnit4Instrumentation.class.getPackage().getName() + ".JUnit4Utils",
JUnit4Instrumentation.class.getPackage().getName() + ".TestEventsHandlerHolder",
JUnit4Instrumentation.class.getPackage().getName() + ".SkippedByDatadog",
JUnit4Instrumentation.class.getPackage().getName() + ".TracingListener",
JUnit4Instrumentation.class.getPackage().getName() + ".order.JUnit4FailFastClassOrderer",
packageName + ".DDCollectAllTestDefinitionsExecutor",
};
}

@Override
public String muzzleDirective() {
return "skipMuzzle";
}

@Override
public void methodAdvice(MethodTransformer transformer) {
transformer.applyAdvice(
named("createTestExecutor")
.and(takesArgument(0, named("org.gradle.internal.actor.Actor")))
.and(returns(named("org.gradle.api.internal.tasks.testing.TestDefinitionConsumer"))),
JUnitTestDefinitionProcessorInstrumentation.class.getName() + "$TestExecutorAdvice");
}

public static class TestExecutorAdvice {
@SuppressWarnings("bytebuddy-exception-suppression")
@Advice.OnMethodExit
public static void onTestExecutorCreation(
@Advice.Return(readOnly = false) TestDefinitionConsumer<ClassTestDefinition> executor) {
String testOrder = Config.get().getCiVisibilityTestOrder();
if (!CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(testOrder)) {
throw new IllegalArgumentException("Unknown test order: " + testOrder);
}

executor =
new DDCollectAllTestDefinitionsExecutor(
executor, Thread.currentThread().getContextClassLoader());
}
}
}
10 changes: 6 additions & 4 deletions dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,11 +201,12 @@ void testInjectedDependencyVerification(String gradleVersion, boolean verificati
}
}

// TODO: add back LATEST_GRADLE_VERSION after fixing ordering on Gradle 9.3.0
@TableTest({
"scenario | gradleVersion | projectName | flakyTests | expectedOrder | eventsNumber",
"junit4-ordering-7.6.4 | 7.6.4 | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 ",
"junit4-ordering-9.2.1 | 9.2.1 | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 "
"scenario | gradleVersion | projectName | flakyTests | expectedOrder | eventsNumber",
"junit4-ordering-7.6.4 | 7.6.4 | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 ",
"junit4-ordering-9.2.1 | 9.2.1 | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 ",
"junit4-ordering-9.3.0 | 9.3.0 | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 ",
"junit4-ordering-latest | latest | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 "
})
@ParameterizedTest
void testJunit4ClassOrdering(
Expand All @@ -215,6 +216,7 @@ void testJunit4ClassOrdering(
List<TestFQN> expectedOrder,
int eventsNumber)
throws IOException {
gradleVersion = resolveVersion(gradleVersion);
givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion);
givenGradleProjectFiles(projectName);
givenGradleProjectProperties();
Expand Down