Mark !mayfail tests as skipped in the JUnit reporter

Should fix #2116
2025-08-01 21:05:39 +02:00 · 2021-06-07 19:53:34 +02:00
parent de67278e14
commit b406ad52a7
12 changed files with 231 additions and 15 deletions
--- a/tests/SelfTest/Baselines/automake.sw.approved.txt
+++ b/tests/SelfTest/Baselines/automake.sw.approved.txt
@@ -164,6 +164,7 @@ Nor would this
 :test-result: FAIL Matchers can be composed with both && and || - failing
 :test-result: PASS Matchers can be negated (Not) with the ! operator
 :test-result: FAIL Matchers can be negated (Not) with the ! operator - failing
+:test-result: XFAIL Mayfail test case with nested sections
 :test-result: FAIL Mismatching exception messages failing the test
 :test-result: PASS Nested generators and captured variables
 :test-result: FAIL Nice descriptive name
--- a/tests/SelfTest/Baselines/compact.sw.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -911,6 +911,10 @@ Matchers.tests.cpp:<line number>: passed: testStringForMatching(), ( ContainsSub
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), ( ContainsSubstring( "string" ) || ContainsSubstring( "different" ) ) && ContainsSubstring( "random" ) for: "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" )
 Matchers.tests.cpp:<line number>: passed: testStringForMatching(), !ContainsSubstring( "different" ) for: "this string contains 'abc' as a substring" not contains: "different"
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), !ContainsSubstring( "substring" ) for: "this string contains 'abc' as a substring" not contains: "substring"
+Condition.tests.cpp:<line number>: failed: explicitly
+Condition.tests.cpp:<line number>: failed: explicitly
+Condition.tests.cpp:<line number>: failed: explicitly
+Condition.tests.cpp:<line number>: failed: explicitly
 Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
 Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail"
 Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6
--- a/tests/SelfTest/Baselines/console.std.approved.txt
+++ b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -694,6 +694,46 @@ Matchers.tests.cpp:<line number>: FAILED:
 with expansion:
  "this string contains 'abc' as a substring" not contains: "substring"

+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
 -------------------------------------------------------------------------------
 Mismatching exception messages failing the test
 -------------------------------------------------------------------------------
@@ -1386,6 +1426,6 @@ due to unexpected exception with message:
  Why would you throw a std::string?

 ===============================================================================
-test cases:  371 |  295 passed |  70 failed |  6 failed as expected
-assertions: 2110 | 1958 passed | 129 failed | 23 failed as expected
+test cases:  372 |  295 passed |  70 failed |  7 failed as expected
+assertions: 2114 | 1958 passed | 129 failed | 27 failed as expected

--- a/tests/SelfTest/Baselines/console.sw.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -6962,6 +6962,46 @@ Matchers.tests.cpp:<line number>: FAILED:
 with expansion:
  "this string contains 'abc' as a substring" not contains: "substring"

+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  A
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  1
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+Mayfail test case with nested sections
+  B
+  2
+-------------------------------------------------------------------------------
+Condition.tests.cpp:<line number>
+...............................................................................
+
+Condition.tests.cpp:<line number>: FAILED:
+
 -------------------------------------------------------------------------------
 Mismatching exception messages failing the test
 -------------------------------------------------------------------------------
@@ -16990,6 +17030,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:

 ===============================================================================
-test cases:  371 |  279 passed |  86 failed |  6 failed as expected
-assertions: 2127 | 1958 passed | 146 failed | 23 failed as expected
+test cases:  372 |  279 passed |  86 failed |  7 failed as expected
+assertions: 2131 | 1958 passed | 146 failed | 27 failed as expected

--- a/tests/SelfTest/Baselines/junit.sw.approved.txt
+++ b/tests/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="129" tests="2127" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="129" tests="2131" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
    <properties>
      <property name="random-seed" value="1"/>
      <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
@@ -50,6 +50,7 @@ Nor would this
    <testcase classname="<exe-name>.global" name="#2152 - ULP checks between differently signed values were wrong - double" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="#2152 - ULP checks between differently signed values were wrong - float" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
      <error type="TEST_CASE">
 FAILED:
 expected exception
@@ -58,6 +59,7 @@ Exception.tests.cpp:<line number>
      </error>
    </testcase>
    <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/inside REQUIRE_NOTHROW" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
      <error message="thisThrows()" type="REQUIRE_NOTHROW">
 FAILED:
  REQUIRE_NOTHROW( thisThrows() )
@@ -70,6 +72,7 @@ Exception.tests.cpp:<line number>
    <testcase classname="<exe-name>.global" name="#809" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="#833" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="#835 -- errno should not be touched by Catch" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
      <failure message="f() == 0" type="CHECK">
 FAILED:
  CHECK( f() == 0 )
@@ -463,6 +466,7 @@ Matchers.tests.cpp:<line number>
    <testcase classname="<exe-name>.global" name="Enums in namespaces can quickly have stringification enabled using REGISTER_ENUM" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Epsilon only applies to Approx's value" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Equality checks that should fail" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
      <failure message="data.int_seven == 6" type="CHECK">
 FAILED:
  CHECK( data.int_seven == 6 )
@@ -756,6 +760,7 @@ Message.tests.cpp:<line number>
      </failure>
    </testcase>
    <testcase classname="<exe-name>.global" name="Inequality checks that should fail" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
      <failure message="data.int_seven != 7" type="CHECK">
 FAILED:
  CHECK( data.int_seven != 7 )
@@ -819,6 +824,34 @@ with expansion:
 Matchers.tests.cpp:<line number>
      </failure>
    </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/A" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/A" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/B" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/B" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
+      <failure type="FAIL">
+FAILED:
+Condition.tests.cpp:<line number>
+      </failure>
+    </testcase>
    <testcase classname="<exe-name>.global" name="Mismatching exception messages failing the test" time="{duration}" status="run">
      <failure message="thisThrows(), &quot;should fail&quot;" type="REQUIRE_THROWS_WITH">
 FAILED:
@@ -1285,12 +1318,14 @@ Misc.tests.cpp:<line number>
    <testcase classname="<exe-name>.global" name="Test with special, characters &quot;in name" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Testing checked-if" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Testing checked-if 2" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
      <failure type="FAIL">
 FAILED:
 Misc.tests.cpp:<line number>
      </failure>
    </testcase>
    <testcase classname="<exe-name>.global" name="Testing checked-if 3" time="{duration}" status="run">
+      <skipped message="TEST_CASE tagged with !mayfail"/>
      <failure type="FAIL">
 FAILED:
 Misc.tests.cpp:<line number>
--- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -660,6 +660,30 @@ Condition.tests.cpp:<line number>
      </skipped>
    </testCase>
    <testCase name="Inequality checks that should succeed" duration="{duration}"/>
+    <testCase name="Mayfail test case with nested sections/1/A" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
+    <testCase name="Mayfail test case with nested sections/2/A" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
+    <testCase name="Mayfail test case with nested sections/1/B" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
+    <testCase name="Mayfail test case with nested sections/2/B" duration="{duration}">
+      <skipped message="FAIL()">
+FAILED:
+Condition.tests.cpp:<line number>
+      </skipped>
+    </testCase>
    <testCase name="Ordering comparison checks that should fail" duration="{duration}">
      <failure message="CHECK(data.int_seven > 7)">
 FAILED:
--- a/tests/SelfTest/Baselines/tap.sw.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.approved.txt
@@ -1804,6 +1804,14 @@ not ok {test-number} - testStringForMatching(), ( ContainsSubstring( "string" )
 ok {test-number} - testStringForMatching(), !ContainsSubstring( "different" ) for: "this string contains 'abc' as a substring" not contains: "different"
 # Matchers can be negated (Not) with the ! operator - failing
 not ok {test-number} - testStringForMatching(), !ContainsSubstring( "substring" ) for: "this string contains 'abc' as a substring" not contains: "substring"
+# Mayfail test case with nested sections
+not ok {test-number} - explicitly
+# Mayfail test case with nested sections
+not ok {test-number} - explicitly
+# Mayfail test case with nested sections
+not ok {test-number} - explicitly
+# Mayfail test case with nested sections
+not ok {test-number} - explicitly
 # Mismatching exception messages failing the test
 ok {test-number} - thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
 # Mismatching exception messages failing the test
@@ -4256,5 +4264,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
 ok {test-number} -
 # xmlentitycheck
 ok {test-number} -
-1..2127
+1..2131

--- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt
+++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt
@@ -406,6 +406,12 @@ Matchers.tests.cpp:<line number>|nexpression failed|n  CHECK_THAT( testStringFor
 ##teamcity[testStarted name='Matchers can be negated (Not) with the ! operator - failing']
 Matchers.tests.cpp:<line number>|nexpression failed|n  CHECK_THAT( testStringForMatching(), !ContainsSubstring( "substring" ) )|nwith expansion:|n  "this string contains |'abc|' as a substring" not contains: "substring"|n']
 ##teamcity[testFinished name='Matchers can be negated (Not) with the ! operator - failing' duration="{duration}"]
+##teamcity[testStarted name='Mayfail test case with nested sections']
+Condition.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
+Condition.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
+Condition.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
+Condition.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
+##teamcity[testFinished name='Mayfail test case with nested sections' duration="{duration}"]
 ##teamcity[testStarted name='Mismatching exception messages failing the test']
 Exception.tests.cpp:<line number>|nexpression failed|n  REQUIRE_THROWS_WITH( thisThrows(), "should fail" )|nwith expansion:|n  "expected exception" equals: "should fail"|n']
 ##teamcity[testFinished name='Mismatching exception messages failing the test' duration="{duration}"]
--- a/tests/SelfTest/Baselines/xml.sw.approved.txt
+++ b/tests/SelfTest/Baselines/xml.sw.approved.txt
@@ -8426,6 +8426,43 @@ Nor would this
    </Expression>
    <OverallResult success="false"/>
  </TestCase>
+  <TestCase name="Mayfail test case with nested sections" tags="[!mayfail]" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+    <Section name="A" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+      <Section name="1" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Failure filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" />
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <OverallResults successes="0" failures="0" expectedFailures="1"/>
+    </Section>
+    <Section name="A" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+      <Section name="2" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Failure filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" />
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <OverallResults successes="0" failures="0" expectedFailures="1"/>
+    </Section>
+    <Section name="A" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+      <OverallResults successes="0" failures="0" expectedFailures="0"/>
+    </Section>
+    <Section name="B" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+      <Section name="1" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Failure filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" />
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <OverallResults successes="0" failures="0" expectedFailures="1"/>
+    </Section>
+    <Section name="B" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+      <Section name="2" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+        <Failure filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" />
+        <OverallResults successes="0" failures="0" expectedFailures="1"/>
+      </Section>
+      <OverallResults successes="0" failures="0" expectedFailures="1"/>
+    </Section>
+    <Section name="B" filename="tests/<exe-name>/UsageTests/Condition.tests.cpp" >
+      <OverallResults successes="0" failures="0" expectedFailures="0"/>
+    </Section>
+    <OverallResult success="true"/>
+  </TestCase>
  <TestCase name="Mismatching exception messages failing the test" tags="[!throws][.][failing]" filename="tests/<exe-name>/UsageTests/Exception.tests.cpp" >
    <Expression success="true" type="REQUIRE_THROWS_WITH" filename="tests/<exe-name>/UsageTests/Exception.tests.cpp" >
      <Original>
@@ -19990,6 +20027,6 @@ loose text artifact
    </Section>
    <OverallResult success="true"/>
  </TestCase>
-  <OverallResults successes="1958" failures="146" expectedFailures="23"/>
-  <OverallResultsCases successes="279" failures="86" expectedFailures="6"/>
+  <OverallResults successes="1958" failures="146" expectedFailures="27"/>
+  <OverallResultsCases successes="279" failures="86" expectedFailures="7"/>
 </Catch2TestRun>
--- a/tests/SelfTest/UsageTests/Condition.tests.cpp
+++ b/tests/SelfTest/UsageTests/Condition.tests.cpp
@@ -86,6 +86,19 @@ TEST_CASE( "Equality checks that should fail", "[.][failing][!mayfail]" )
    CHECK( x == Approx( 1.301 ) );
 }

+// Needed to test junit reporter's handling of mayfail test cases and sections
+TEST_CASE("Mayfail test case with nested sections", "[!mayfail]") {
+    SECTION("A") {
+        SECTION("1") { FAIL(); }
+        SECTION("2") { FAIL(); }
+    }
+    SECTION("B") {
+        SECTION("1") { FAIL(); }
+        SECTION("2") { FAIL(); }
+    }
+}
+
+
 TEST_CASE( "Inequality checks that should succeed" )
 {
    TestData data;