Fix JUnit output for assertion-free test cases

src/catch2/reporters/catch_reporter_junit.cpp

@@ -81,6 +81,33 @@ namespace Catch {
             }
         }
 
+        bool shouldWriteSection(
+            CumulativeReporterBase::SectionNode const& sectionNode ) {
+            return sectionNode.stats.assertions.total() > 0 ||
+                   !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty();
+        }
+
+        std::size_t countTestCases(
+            CumulativeReporterBase::SectionNode const& sectionNode ) {
+            std::size_t count = shouldWriteSection( sectionNode ) ? 1 : 0;
+            for ( auto const& child : sectionNode.childSections ) {
+                count += countTestCases( *child );
+            }
+            return count;
+        }
+
+        std::size_t countTestCases(
+            CumulativeReporterBase::TestRunNode const& testRunNode ) {
+            std::size_t count = 0;
+            for ( auto const& testCase : testRunNode.children ) {
+                assert( testCase->children.size() == 1 );
+                const auto sectionCount =
+                    countTestCases( *testCase->children.front() );
+                count += sectionCount > 0 ? sectionCount : 1;
+            }
+            return count;
+        }
+
     } // anonymous namespace
 
     JunitReporter::JunitReporter( ReporterConfig&& _config )
@@ -136,7 +163,7 @@ namespace Catch {
         xml.writeAttribute( "errors"_sr, unexpectedExceptions );
         xml.writeAttribute( "failures"_sr, stats.totals.assertions.failed-unexpectedExceptions );
         xml.writeAttribute( "skipped"_sr, stats.totals.assertions.skipped );
-        xml.writeAttribute( "tests"_sr, stats.totals.assertions.total() );
+        xml.writeAttribute( "tests"_sr, countTestCases( testRunNode ) );
         xml.writeAttribute( "hostname"_sr, "tbd"_sr ); // !TBD
         if( m_config->showDurations() == ShowDurations::Never )
             xml.writeAttribute( "time"_sr, ""_sr );
@@ -188,20 +215,23 @@ namespace Catch {
 
         normalizeNamespaceMarkers(className);
 
-        writeSection( className, "", rootSection, stats.testInfo->okToFail() );
+        writeSection( className,
+                      "",
+                      rootSection,
+                      stats.testInfo->okToFail(),
+                      countTestCases( rootSection ) == 0 );
     }
 
     void JunitReporter::writeSection( std::string const& className,
                                       std::string const& rootName,
                                       SectionNode const& sectionNode,
-                                      bool testOkToFail) {
+                                      bool testOkToFail,
+                                      bool writeEmptyTestCase ) {
         std::string name = trim( sectionNode.stats.sectionInfo.name );
         if( !rootName.empty() )
             name = rootName + '/' + name;
 
-        if ( sectionNode.stats.assertions.total() > 0
-           || !sectionNode.stdOut.empty()
-           || !sectionNode.stdErr.empty() ) {
+        if ( writeEmptyTestCase || shouldWriteSection( sectionNode ) ) {
             XmlWriter::ScopedElement e = xml.scopedElement( "testcase" );
             if( className.empty() ) {
                 xml.writeAttribute( "classname"_sr, name );
@@ -233,9 +263,13 @@ namespace Catch {
         }
         for( auto const& childNode : sectionNode.childSections )
             if( className.empty() )
-                writeSection( name, "", *childNode, testOkToFail );
+                writeSection( name, "", *childNode, testOkToFail, false );
             else
-                writeSection( className, name, *childNode, testOkToFail );
+                writeSection( className,
+                              name,
+                              *childNode,
+                              testOkToFail,
+                              false );
     }
 
     void JunitReporter::writeAssertions( SectionNode const& sectionNode ) {

src/catch2/reporters/catch_reporter_junit.hpp

@@ -38,7 +38,8 @@ namespace Catch {
         void writeSection( std::string const& className,
                            std::string const& rootName,
                            SectionNode const& sectionNode,
-                           bool testOkToFail );
+                           bool testOkToFail,
+                           bool writeEmptyTestCase );
 
         void writeAssertions(SectionNode const& sectionNode);
         bool writeAssertion(AssertionStats const& stats);

tests/SelfTest/Baselines/junit.generator.approved.txt

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="<exe-name>" errors="0" failures="0" skipped="0" tests="1" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+    <properties>
+      <property name="random-seed" value="1"/>
+      <property name="filters" value="&quot;An assertion-free test with a generator&quot;"/>
+    </properties>
+    <testcase classname="<exe-name>.global" name="An assertion-free test with a generator/value 2" time="{duration}" status="run">
+      <system-out>
+12
+      </system-out>
+    </testcase>
+    <system-out>
+12
+    </system-out>
+    <system-err/>
+  </testsuite>
+</testsuites>

tests/SelfTest/Baselines/junit.section.approved.txt

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="<exe-name>" errors="0" failures="0" skipped="0" tests="1" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+    <properties>
+      <property name="random-seed" value="1"/>
+      <property name="filters" value="&quot;An assertion-free test with a section&quot;"/>
+    </properties>
+    <testcase classname="<exe-name>.global" name="An assertion-free test with a section/output section" time="{duration}" status="run">
+      <system-out>
+section output
+      </system-out>
+    </testcase>
+    <system-out>
+section output
+    </system-out>
+    <system-err/>
+  </testsuite>
+</testsuites>

tests/SelfTest/Baselines/junit.skip.approved.txt

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="<exe-name>" errors="0" failures="0" skipped="1" tests="1" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+    <properties>
+      <property name="random-seed" value="1"/>
+      <property name="filters" value="&quot;tests can be skipped dynamically at runtime&quot;"/>
+    </properties>
+    <testcase classname="<exe-name>.global" name="tests can be skipped dynamically at runtime" time="{duration}" status="run">
+      <skipped type="SKIP">
+SKIPPED
+at Skip.tests.cpp:<line number>
+      </skipped>
+    </testcase>
+    <system-out/>
+    <system-err/>
+  </testsuite>
+</testsuites>

tests/SelfTest/Baselines/junit.std.approved.txt

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="<exe-name>" errors="0" failures="0" skipped="0" tests="1" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+    <properties>
+      <property name="random-seed" value="1"/>
+      <property name="filters" value="&quot;An empty test with no assertions&quot;"/>
+    </properties>
+    <testcase classname="<exe-name>.global" name="An empty test with no assertions" time="{duration}" status="run"/>
+    <system-out/>
+    <system-err/>
+  </testsuite>
+</testsuites>

tests/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="141" skipped="12" tests="2435" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="141" skipped="12" tests="1038" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="random-seed" value="1"/>
       <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>

tests/SelfTest/Baselines/junit.sw.multi.approved.txt

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuites>
-  <testsuite name="<exe-name>" errors="17" failures="141" skipped="12" tests="2435" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="141" skipped="12" tests="1038" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="random-seed" value="1"/>
       <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>

tests/SelfTest/UsageTests/Generators.tests.cpp

@@ -13,8 +13,14 @@
 #include <catch2/generators/catch_generators_range.hpp>
 
 #include <cstring>
+#include <iostream>
 
 
+TEST_CASE( "An assertion-free test with a generator", "[approvals]" ) {
+    const auto value = GENERATE( 1, 2 );
+    DYNAMIC_SECTION( "value " << value ) { std::cout << value; }
+}
+
 // Generators and sections can be nested freely
 TEST_CASE("Generators -- simple", "[generators]") {
     auto i = GENERATE(1, 2, 3);

tests/SelfTest/UsageTests/Misc.tests.cpp

@@ -248,6 +248,10 @@ TEST_CASE( "Factorials are computed", "[factorial]" ) {
 
 TEST_CASE( "An empty test with no assertions", "[empty]" ) {}
 
+TEST_CASE( "An assertion-free test with a section", "[approvals]" ) {
+    SECTION( "output section" ) { std::cout << "section output"; }
+}
+
 TEST_CASE( "Nice descriptive name", "[tag1][tag2][tag3][.]" ) {
     WARN( "This one ran" );
 }

tools/scripts/approvalTests.py

@@ -212,6 +212,16 @@ def approve(baseName, args):
 # Standard console reporter
 approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals] *"] + base_args)
 
+# Standard JUnit reporter with an executed test case without assertions
+approve("junit.std", ["An empty test with no assertions"] + base_args + ["-r", "junit"])
+
+# JUnit reporter with assertion-free sections and generators
+approve("junit.section", ["An assertion-free test with a section"] + base_args + ["-r", "junit"])
+approve("junit.generator", ["An assertion-free test with a generator"] + base_args + ["-r", "junit"])
+
+# JUnit reporter with a runtime-skipped test case
+approve("junit.skip", ["tests can be skipped dynamically at runtime"] + base_args + ["-r", "junit"])
+
 # console reporter, include passes, warn about No Assertions, limit failures to first 4
 approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-x", "4"] + base_args)