From ac97fb3052d5df8fe349fa917b69a4ac5972b6ac Mon Sep 17 00:00:00 2001 From: zhanglangning Date: Tue, 11 Aug 2026 00:18:21 +0800 Subject: [PATCH] Fix JUnit reporting for empty test cases Emit a testcase for executed test cases whose section tree would otherwise produce no JUnit testcase. Count the suite tests from emitted testcase nodes so the declared count matches the XML output. Fixes #2856 Assisted-by: OpenAI Codex --- src/catch2/reporters/catch_reporter_junit.cpp | 50 ++++++++++++++++--- src/catch2/reporters/catch_reporter_junit.hpp | 3 +- .../Baselines/junit.generator.approved.txt | 18 +++++++ .../Baselines/junit.section.approved.txt | 18 +++++++ .../Baselines/junit.skip.approved.txt | 17 +++++++ .../SelfTest/Baselines/junit.std.approved.txt | 12 +++++ .../SelfTest/Baselines/junit.sw.approved.txt | 2 +- .../Baselines/junit.sw.multi.approved.txt | 2 +- .../SelfTest/UsageTests/Generators.tests.cpp | 6 +++ tests/SelfTest/UsageTests/Misc.tests.cpp | 4 ++ tools/scripts/approvalTests.py | 10 ++++ 11 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 tests/SelfTest/Baselines/junit.generator.approved.txt create mode 100644 tests/SelfTest/Baselines/junit.section.approved.txt create mode 100644 tests/SelfTest/Baselines/junit.skip.approved.txt create mode 100644 tests/SelfTest/Baselines/junit.std.approved.txt diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index 00b4587610..11457d5e6b 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -81,6 +81,33 @@ namespace Catch { } } + bool shouldWriteSection( + CumulativeReporterBase::SectionNode const& sectionNode ) { + return sectionNode.stats.assertions.total() > 0 || + !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty(); + } + + std::size_t countTestCases( + CumulativeReporterBase::SectionNode const& sectionNode ) { + std::size_t count = shouldWriteSection( sectionNode ) ? 1 : 0; + for ( auto const& child : sectionNode.childSections ) { + count += countTestCases( *child ); + } + return count; + } + + std::size_t countTestCases( + CumulativeReporterBase::TestRunNode const& testRunNode ) { + std::size_t count = 0; + for ( auto const& testCase : testRunNode.children ) { + assert( testCase->children.size() == 1 ); + const auto sectionCount = + countTestCases( *testCase->children.front() ); + count += sectionCount > 0 ? sectionCount : 1; + } + return count; + } + } // anonymous namespace JunitReporter::JunitReporter( ReporterConfig&& _config ) @@ -136,7 +163,7 @@ namespace Catch { xml.writeAttribute( "errors"_sr, unexpectedExceptions ); xml.writeAttribute( "failures"_sr, stats.totals.assertions.failed-unexpectedExceptions ); xml.writeAttribute( "skipped"_sr, stats.totals.assertions.skipped ); - xml.writeAttribute( "tests"_sr, stats.totals.assertions.total() ); + xml.writeAttribute( "tests"_sr, countTestCases( testRunNode ) ); xml.writeAttribute( "hostname"_sr, "tbd"_sr ); // !TBD if( m_config->showDurations() == ShowDurations::Never ) xml.writeAttribute( "time"_sr, ""_sr ); @@ -188,20 +215,23 @@ namespace Catch { normalizeNamespaceMarkers(className); - writeSection( className, "", rootSection, stats.testInfo->okToFail() ); + writeSection( className, + "", + rootSection, + stats.testInfo->okToFail(), + countTestCases( rootSection ) == 0 ); } void JunitReporter::writeSection( std::string const& className, std::string const& rootName, SectionNode const& sectionNode, - bool testOkToFail) { + bool testOkToFail, + bool writeEmptyTestCase ) { std::string name = trim( sectionNode.stats.sectionInfo.name ); if( !rootName.empty() ) name = rootName + '/' + name; - if ( sectionNode.stats.assertions.total() > 0 - || !sectionNode.stdOut.empty() - || !sectionNode.stdErr.empty() ) { + if ( writeEmptyTestCase || shouldWriteSection( sectionNode ) ) { XmlWriter::ScopedElement e = xml.scopedElement( "testcase" ); if( className.empty() ) { xml.writeAttribute( "classname"_sr, name ); @@ -233,9 +263,13 @@ namespace Catch { } for( auto const& childNode : sectionNode.childSections ) if( className.empty() ) - writeSection( name, "", *childNode, testOkToFail ); + writeSection( name, "", *childNode, testOkToFail, false ); else - writeSection( className, name, *childNode, testOkToFail ); + writeSection( className, + name, + *childNode, + testOkToFail, + false ); } void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { diff --git a/src/catch2/reporters/catch_reporter_junit.hpp b/src/catch2/reporters/catch_reporter_junit.hpp index a265f78a82..6b9668e29d 100644 --- a/src/catch2/reporters/catch_reporter_junit.hpp +++ b/src/catch2/reporters/catch_reporter_junit.hpp @@ -38,7 +38,8 @@ namespace Catch { void writeSection( std::string const& className, std::string const& rootName, SectionNode const& sectionNode, - bool testOkToFail ); + bool testOkToFail, + bool writeEmptyTestCase ); void writeAssertions(SectionNode const& sectionNode); bool writeAssertion(AssertionStats const& stats); diff --git a/tests/SelfTest/Baselines/junit.generator.approved.txt b/tests/SelfTest/Baselines/junit.generator.approved.txt new file mode 100644 index 0000000000..ba9978c4e6 --- /dev/null +++ b/tests/SelfTest/Baselines/junit.generator.approved.txt @@ -0,0 +1,18 @@ + + + + + + + + + +12 + + + +12 + + + + diff --git a/tests/SelfTest/Baselines/junit.section.approved.txt b/tests/SelfTest/Baselines/junit.section.approved.txt new file mode 100644 index 0000000000..090c2ad433 --- /dev/null +++ b/tests/SelfTest/Baselines/junit.section.approved.txt @@ -0,0 +1,18 @@ + + + + + + + + + +section output + + + +section output + + + + diff --git a/tests/SelfTest/Baselines/junit.skip.approved.txt b/tests/SelfTest/Baselines/junit.skip.approved.txt new file mode 100644 index 0000000000..2f7e5e6f66 --- /dev/null +++ b/tests/SelfTest/Baselines/junit.skip.approved.txt @@ -0,0 +1,17 @@ + + + + + + + + + +SKIPPED +at Skip.tests.cpp: + + + + + + diff --git a/tests/SelfTest/Baselines/junit.std.approved.txt b/tests/SelfTest/Baselines/junit.std.approved.txt new file mode 100644 index 0000000000..16190d373e --- /dev/null +++ b/tests/SelfTest/Baselines/junit.std.approved.txt @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index f58412b65a..6325818a18 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index c80e5855ed..2c8a98962d 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + diff --git a/tests/SelfTest/UsageTests/Generators.tests.cpp b/tests/SelfTest/UsageTests/Generators.tests.cpp index 6eaf33a8b3..8b486e1c7d 100644 --- a/tests/SelfTest/UsageTests/Generators.tests.cpp +++ b/tests/SelfTest/UsageTests/Generators.tests.cpp @@ -13,8 +13,14 @@ #include #include +#include +TEST_CASE( "An assertion-free test with a generator", "[approvals]" ) { + const auto value = GENERATE( 1, 2 ); + DYNAMIC_SECTION( "value " << value ) { std::cout << value; } +} + // Generators and sections can be nested freely TEST_CASE("Generators -- simple", "[generators]") { auto i = GENERATE(1, 2, 3); diff --git a/tests/SelfTest/UsageTests/Misc.tests.cpp b/tests/SelfTest/UsageTests/Misc.tests.cpp index dec51ab8a7..1deec1aeda 100644 --- a/tests/SelfTest/UsageTests/Misc.tests.cpp +++ b/tests/SelfTest/UsageTests/Misc.tests.cpp @@ -248,6 +248,10 @@ TEST_CASE( "Factorials are computed", "[factorial]" ) { TEST_CASE( "An empty test with no assertions", "[empty]" ) {} +TEST_CASE( "An assertion-free test with a section", "[approvals]" ) { + SECTION( "output section" ) { std::cout << "section output"; } +} + TEST_CASE( "Nice descriptive name", "[tag1][tag2][tag3][.]" ) { WARN( "This one ran" ); } diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py index 72ff26cafa..d686938de8 100755 --- a/tools/scripts/approvalTests.py +++ b/tools/scripts/approvalTests.py @@ -212,6 +212,16 @@ def approve(baseName, args): # Standard console reporter approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals] *"] + base_args) +# Standard JUnit reporter with an executed test case without assertions +approve("junit.std", ["An empty test with no assertions"] + base_args + ["-r", "junit"]) + +# JUnit reporter with assertion-free sections and generators +approve("junit.section", ["An assertion-free test with a section"] + base_args + ["-r", "junit"]) +approve("junit.generator", ["An assertion-free test with a generator"] + base_args + ["-r", "junit"]) + +# JUnit reporter with a runtime-skipped test case +approve("junit.skip", ["tests can be skipped dynamically at runtime"] + base_args + ["-r", "junit"]) + # console reporter, include passes, warn about No Assertions, limit failures to first 4 approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-x", "4"] + base_args)