diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index 00b4587610..11457d5e6b 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -81,6 +81,33 @@ namespace Catch { } } + bool shouldWriteSection( + CumulativeReporterBase::SectionNode const& sectionNode ) { + return sectionNode.stats.assertions.total() > 0 || + !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty(); + } + + std::size_t countTestCases( + CumulativeReporterBase::SectionNode const& sectionNode ) { + std::size_t count = shouldWriteSection( sectionNode ) ? 1 : 0; + for ( auto const& child : sectionNode.childSections ) { + count += countTestCases( *child ); + } + return count; + } + + std::size_t countTestCases( + CumulativeReporterBase::TestRunNode const& testRunNode ) { + std::size_t count = 0; + for ( auto const& testCase : testRunNode.children ) { + assert( testCase->children.size() == 1 ); + const auto sectionCount = + countTestCases( *testCase->children.front() ); + count += sectionCount > 0 ? sectionCount : 1; + } + return count; + } + } // anonymous namespace JunitReporter::JunitReporter( ReporterConfig&& _config ) @@ -136,7 +163,7 @@ namespace Catch { xml.writeAttribute( "errors"_sr, unexpectedExceptions ); xml.writeAttribute( "failures"_sr, stats.totals.assertions.failed-unexpectedExceptions ); xml.writeAttribute( "skipped"_sr, stats.totals.assertions.skipped ); - xml.writeAttribute( "tests"_sr, stats.totals.assertions.total() ); + xml.writeAttribute( "tests"_sr, countTestCases( testRunNode ) ); xml.writeAttribute( "hostname"_sr, "tbd"_sr ); // !TBD if( m_config->showDurations() == ShowDurations::Never ) xml.writeAttribute( "time"_sr, ""_sr ); @@ -188,20 +215,23 @@ namespace Catch { normalizeNamespaceMarkers(className); - writeSection( className, "", rootSection, stats.testInfo->okToFail() ); + writeSection( className, + "", + rootSection, + stats.testInfo->okToFail(), + countTestCases( rootSection ) == 0 ); } void JunitReporter::writeSection( std::string const& className, std::string const& rootName, SectionNode const& sectionNode, - bool testOkToFail) { + bool testOkToFail, + bool writeEmptyTestCase ) { std::string name = trim( sectionNode.stats.sectionInfo.name ); if( !rootName.empty() ) name = rootName + '/' + name; - if ( sectionNode.stats.assertions.total() > 0 - || !sectionNode.stdOut.empty() - || !sectionNode.stdErr.empty() ) { + if ( writeEmptyTestCase || shouldWriteSection( sectionNode ) ) { XmlWriter::ScopedElement e = xml.scopedElement( "testcase" ); if( className.empty() ) { xml.writeAttribute( "classname"_sr, name ); @@ -233,9 +263,13 @@ namespace Catch { } for( auto const& childNode : sectionNode.childSections ) if( className.empty() ) - writeSection( name, "", *childNode, testOkToFail ); + writeSection( name, "", *childNode, testOkToFail, false ); else - writeSection( className, name, *childNode, testOkToFail ); + writeSection( className, + name, + *childNode, + testOkToFail, + false ); } void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { diff --git a/src/catch2/reporters/catch_reporter_junit.hpp b/src/catch2/reporters/catch_reporter_junit.hpp index a265f78a82..6b9668e29d 100644 --- a/src/catch2/reporters/catch_reporter_junit.hpp +++ b/src/catch2/reporters/catch_reporter_junit.hpp @@ -38,7 +38,8 @@ namespace Catch { void writeSection( std::string const& className, std::string const& rootName, SectionNode const& sectionNode, - bool testOkToFail ); + bool testOkToFail, + bool writeEmptyTestCase ); void writeAssertions(SectionNode const& sectionNode); bool writeAssertion(AssertionStats const& stats); diff --git a/tests/SelfTest/Baselines/junit.generator.approved.txt b/tests/SelfTest/Baselines/junit.generator.approved.txt new file mode 100644 index 0000000000..ba9978c4e6 --- /dev/null +++ b/tests/SelfTest/Baselines/junit.generator.approved.txt @@ -0,0 +1,18 @@ + + + + + + + + + +12 + + + +12 + + + + diff --git a/tests/SelfTest/Baselines/junit.section.approved.txt b/tests/SelfTest/Baselines/junit.section.approved.txt new file mode 100644 index 0000000000..090c2ad433 --- /dev/null +++ b/tests/SelfTest/Baselines/junit.section.approved.txt @@ -0,0 +1,18 @@ + + + + + + + + + +section output + + + +section output + + + + diff --git a/tests/SelfTest/Baselines/junit.skip.approved.txt b/tests/SelfTest/Baselines/junit.skip.approved.txt new file mode 100644 index 0000000000..2f7e5e6f66 --- /dev/null +++ b/tests/SelfTest/Baselines/junit.skip.approved.txt @@ -0,0 +1,17 @@ + + + + + + + + + +SKIPPED +at Skip.tests.cpp: + + + + + + diff --git a/tests/SelfTest/Baselines/junit.std.approved.txt b/tests/SelfTest/Baselines/junit.std.approved.txt new file mode 100644 index 0000000000..16190d373e --- /dev/null +++ b/tests/SelfTest/Baselines/junit.std.approved.txt @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index f58412b65a..6325818a18 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index c80e5855ed..2c8a98962d 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + diff --git a/tests/SelfTest/UsageTests/Generators.tests.cpp b/tests/SelfTest/UsageTests/Generators.tests.cpp index 6eaf33a8b3..8b486e1c7d 100644 --- a/tests/SelfTest/UsageTests/Generators.tests.cpp +++ b/tests/SelfTest/UsageTests/Generators.tests.cpp @@ -13,8 +13,14 @@ #include #include +#include +TEST_CASE( "An assertion-free test with a generator", "[approvals]" ) { + const auto value = GENERATE( 1, 2 ); + DYNAMIC_SECTION( "value " << value ) { std::cout << value; } +} + // Generators and sections can be nested freely TEST_CASE("Generators -- simple", "[generators]") { auto i = GENERATE(1, 2, 3); diff --git a/tests/SelfTest/UsageTests/Misc.tests.cpp b/tests/SelfTest/UsageTests/Misc.tests.cpp index dec51ab8a7..1deec1aeda 100644 --- a/tests/SelfTest/UsageTests/Misc.tests.cpp +++ b/tests/SelfTest/UsageTests/Misc.tests.cpp @@ -248,6 +248,10 @@ TEST_CASE( "Factorials are computed", "[factorial]" ) { TEST_CASE( "An empty test with no assertions", "[empty]" ) {} +TEST_CASE( "An assertion-free test with a section", "[approvals]" ) { + SECTION( "output section" ) { std::cout << "section output"; } +} + TEST_CASE( "Nice descriptive name", "[tag1][tag2][tag3][.]" ) { WARN( "This one ran" ); } diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py index 72ff26cafa..d686938de8 100755 --- a/tools/scripts/approvalTests.py +++ b/tools/scripts/approvalTests.py @@ -212,6 +212,16 @@ def approve(baseName, args): # Standard console reporter approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals] *"] + base_args) +# Standard JUnit reporter with an executed test case without assertions +approve("junit.std", ["An empty test with no assertions"] + base_args + ["-r", "junit"]) + +# JUnit reporter with assertion-free sections and generators +approve("junit.section", ["An assertion-free test with a section"] + base_args + ["-r", "junit"]) +approve("junit.generator", ["An assertion-free test with a generator"] + base_args + ["-r", "junit"]) + +# JUnit reporter with a runtime-skipped test case +approve("junit.skip", ["tests can be skipped dynamically at runtime"] + base_args + ["-r", "junit"]) + # console reporter, include passes, warn about No Assertions, limit failures to first 4 approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-x", "4"] + base_args)