Mark !mayfail tests as skipped in the JUnit reporter

Should fix #2116

include/reporters/catch_reporter_junit.cpp

@@ -182,12 +182,13 @@ namespace Catch {
         if ( !m_config->name().empty() )
             className = m_config->name() + "." + className;
 
-        writeSection( className, "", rootSection );
+        writeSection( className, "", rootSection, stats.testInfo.okToFail() );
     }
 
     void JunitReporter::writeSection( std::string const& className,
                                       std::string const& rootName,
-                        SectionNode const& sectionNode ) {
+                                      SectionNode const& sectionNode,
+                                      bool testOkToFail) {
         std::string name = trim( sectionNode.stats.sectionInfo.name );
         if( !rootName.empty() )
             name = rootName + '/' + name;
@@ -211,8 +212,14 @@ namespace Catch {
             // events and write those out appropriately.
             xml.writeAttribute( "status", "run" );
 
+            if (sectionNode.stats.assertions.failedButOk) {
+                xml.scopedElement("skipped")
+                    .writeAttribute("message", "TEST_CASE tagged with !mayfail");
+            }
+
             writeAssertions( sectionNode );
 
+
             if( !sectionNode.stdOut.empty() )
                 xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), XmlFormatting::Newline );
             if( !sectionNode.stdErr.empty() )
@@ -220,9 +227,9 @@ namespace Catch {
         }
         for( auto const& childNode : sectionNode.childSections )
             if( className.empty() )
-                writeSection( name, "", *childNode );
+                writeSection( name, "", *childNode, testOkToFail );
             else
-                writeSection( className, name, *childNode );
+                writeSection( className, name, *childNode, testOkToFail );
     }
 
     void JunitReporter::writeAssertions( SectionNode const& sectionNode ) {

include/reporters/catch_reporter_junit.h

@@ -43,7 +43,8 @@ namespace Catch {
 
         void writeSection( std::string const& className,
                            std::string const& rootName,
-                          SectionNode const& sectionNode);
+                           SectionNode const& sectionNode,
+                           bool testOkToFail );
 
         void writeAssertions(SectionNode const& sectionNode);
         void writeAssertion(AssertionStats const& stats);

projects/SelfTest/Baselines/compact.sw.approved.txt

@@ -846,6 +846,10 @@ Matchers.tests.cpp:<line number>: passed: testStringForMatching(), (Contains("st
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), (Contains("string") || Contains("different")) && Contains("random") for: "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" )
 Matchers.tests.cpp:<line number>: passed: testStringForMatching(), !Contains("different") for: "this string contains 'abc' as a substring" not contains: "different"
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), !Contains("substring") for: "this string contains 'abc' as a substring" not contains: "substring"
+Condition.tests.cpp:<line number>: failed: explicitly
+Condition.tests.cpp:<line number>: failed: explicitly
+Condition.tests.cpp:<line number>: failed: explicitly
+Condition.tests.cpp:<line number>: failed: explicitly
 Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
 Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail"
 Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6

projects/SelfTest/Baselines/console.std.approved.txt

@@ -694,6 +694,46 @@ Matchers.tests.cpp:<line number>: FAILED:
 with expansion:
   "this string contains 'abc' as a substring" not contains: "substring"
 
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
 -------------------------------------------------------------------------------
 Mismatching exception messages failing the test
 -------------------------------------------------------------------------------
@@ -1380,6 +1420,6 @@ due to unexpected exception with message:
   Why would you throw a std::string?
 
 ===============================================================================
-test cases:  322 |  248 passed |  70 failed |  4 failed as expected
+test cases:  323 |  248 passed |  70 failed |  5 failed as expected
-assertions: 1759 | 1607 passed | 131 failed | 21 failed as expected
+assertions: 1763 | 1607 passed | 131 failed | 25 failed as expected
 

projects/SelfTest/Baselines/console.sw.approved.txt

@@ -6492,6 +6492,46 @@ Matchers.tests.cpp:<line number>: FAILED:
 with expansion:
   "this string contains 'abc' as a substring" not contains: "substring"
 
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
 -------------------------------------------------------------------------------
 Mismatching exception messages failing the test
 -------------------------------------------------------------------------------
@@ -14138,6 +14178,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  322 |  232 passed |  86 failed |  4 failed as expected
+test cases:  323 |  232 passed |  86 failed |  5 failed as expected
-assertions: 1776 | 1607 passed | 148 failed | 21 failed as expected
+assertions: 1780 | 1607 passed | 148 failed | 25 failed as expected
 

projects/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="132" tests="1777" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="132" tests="1781" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
       <property name="random-seed" value="1"/>
@@ -48,6 +48,7 @@ Nor would this
     <testcase classname="<exe-name>.global" name="#1954 - 7 arg template test case sig compiles - 5, 1, 1, 1, 1, 0, 0" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="#1954 - 7 arg template test case sig compiles - 5, 3, 1, 1, 1, 0, 0" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
       <error type="TEST_CASE">
 FAILED:
 expected exception
@@ -56,6 +57,7 @@ Exception.tests.cpp:<line number>
       </error>
     </testcase>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/inside REQUIRE_NOTHROW" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
       <error message="thisThrows()" type="REQUIRE_NOTHROW">
 FAILED:
   REQUIRE_NOTHROW( thisThrows() )
@@ -68,6 +70,7 @@ Exception.tests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="#809" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="#833" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="#835 -- errno should not be touched by Catch" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
       <failure message="f() == 0" type="CHECK">
 FAILED:
   CHECK( f() == 0 )
@@ -443,6 +446,7 @@ Matchers.tests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="Enums in namespaces can quickly have stringification enabled using REGISTER_ENUM" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Epsilon only applies to Approx's value" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Equality checks that should fail" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
       <failure message="data.int_seven == 6" type="CHECK">
 FAILED:
   CHECK( data.int_seven == 6 )
@@ -736,6 +740,7 @@ Message.tests.cpp:<line number>
       </failure>
     </testcase>
     <testcase classname="<exe-name>.global" name="Inequality checks that should fail" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
       <failure message="data.int_seven != 7" type="CHECK">
 FAILED:
   CHECK( data.int_seven != 7 )
@@ -799,6 +804,34 @@ with expansion:
 Matchers.tests.cpp:<line number>
       </failure>
     </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/A" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/A" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/B" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/B" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
     <testcase classname="<exe-name>.global" name="Mismatching exception messages failing the test" time="{duration}" status="run">
       <failure message="thisThrows(), &quot;should fail&quot;" type="REQUIRE_THROWS_WITH">
 FAILED:

projects/SelfTest/Baselines/sonarqube.sw.approved.txt

@@ -575,6 +575,30 @@ Condition.tests.cpp:<line number>
       </skipped>
     </testCase>
     <testCase name="Inequality checks that should succeed" duration="{duration}"/>
+    <testCase name="Mayfail test case with nested sections/1/A" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
+    <testCase name="Mayfail test case with nested sections/2/A" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
+    <testCase name="Mayfail test case with nested sections/1/B" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
+    <testCase name="Mayfail test case with nested sections/2/B" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
     <testCase name="Ordering comparison checks that should fail" duration="{duration}">
       <failure message="CHECK(data.int_seven > 7)">
 FAILED:

projects/SelfTest/Baselines/xml.sw.approved.txt

@@ -7956,6 +7956,43 @@ Nor would this
       </Expression>
       <OverallResult success="false"/>
     </TestCase>
+    <TestCase name="Mayfail test case with nested sections" tags="[!mayfail]" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+      <Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Section name="1" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+          <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
+          <OverallResults successes="0" failures="0" expectedFailures="1"/>
+        </Section>
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Section name="2" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+          <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
+          <OverallResults successes="0" failures="0" expectedFailures="1"/>
+        </Section>
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <OverallResults successes="0" failures="0" expectedFailures="0"/>
+      </Section>
+      <Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Section name="1" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+          <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
+          <OverallResults successes="0" failures="0" expectedFailures="1"/>
+        </Section>
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Section name="2" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+          <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
+          <OverallResults successes="0" failures="0" expectedFailures="1"/>
+        </Section>
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <OverallResults successes="0" failures="0" expectedFailures="0"/>
+      </Section>
+      <OverallResult success="true"/>
+    </TestCase>
     <TestCase name="Mismatching exception messages failing the test" tags="[!hide][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
       <Expression success="true" type="REQUIRE_THROWS_WITH" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
         <Original>
@@ -16722,9 +16759,9 @@ loose text artifact
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="1607" failures="149" expectedFailures="21"/>
+    <OverallResults successes="1607" failures="149" expectedFailures="25"/>
-    <OverallResultsCases successes="232" failures="86" expectedFailures="4"/>
+    <OverallResultsCases successes="232" failures="86" expectedFailures="5"/>
   </Group>
-  <OverallResults successes="1607" failures="148" expectedFailures="21"/>
+  <OverallResults successes="1607" failures="148" expectedFailures="25"/>
-  <OverallResultsCases successes="232" failures="86" expectedFailures="4"/>
+  <OverallResultsCases successes="232" failures="86" expectedFailures="5"/>
 </Catch>

projects/SelfTest/UsageTests/Condition.tests.cpp

@@ -89,6 +89,19 @@ TEST_CASE( "Equality checks that should fail", "[.][failing][!mayfail]" )
     CHECK( x == Approx( 1.301 ) );
 }
 
+// Needed to test junit reporter's handling of mayfail test cases and sections
+TEST_CASE("Mayfail test case with nested sections", "[!mayfail]") {
+    SECTION("A") {
+        SECTION("1") { FAIL(); }
+        SECTION("2") { FAIL(); }
+    }
+    SECTION("B") {
+        SECTION("1") { FAIL(); }
+        SECTION("2") { FAIL(); }
+    }
+}
+
+
 TEST_CASE( "Inequality checks that should succeed" )
 {
     TestData data;