diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index 71a93051..00b00b47 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -168,12 +168,13 @@ namespace Catch { if ( !m_config->name().empty() ) className = static_cast(m_config->name()) + '.' + className; - writeSection( className, "", rootSection ); + writeSection( className, "", rootSection, stats.testInfo->okToFail() ); } - void JunitReporter::writeSection( std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode ) { + void JunitReporter::writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail) { std::string name = trim( sectionNode.stats.sectionInfo.name ); if( !rootName.empty() ) name = rootName + '/' + name; @@ -197,8 +198,14 @@ namespace Catch { // events and write those out appropriately. xml.writeAttribute( "status"_sr, "run"_sr ); + if (sectionNode.stats.assertions.failedButOk) { + xml.scopedElement("skipped") + .writeAttribute("message", "TEST_CASE tagged with !mayfail"); + } + writeAssertions( sectionNode ); + if( !sectionNode.stdOut.empty() ) xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), XmlFormatting::Newline ); if( !sectionNode.stdErr.empty() ) @@ -206,9 +213,9 @@ namespace Catch { } for( auto const& childNode : sectionNode.childSections ) if( className.empty() ) - writeSection( name, "", *childNode ); + writeSection( name, "", *childNode, testOkToFail ); else - writeSection( className, name, *childNode ); + writeSection( className, name, *childNode, testOkToFail ); } void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { diff --git a/src/catch2/reporters/catch_reporter_junit.hpp b/src/catch2/reporters/catch_reporter_junit.hpp index 6665663c..6991c224 100644 --- a/src/catch2/reporters/catch_reporter_junit.hpp +++ b/src/catch2/reporters/catch_reporter_junit.hpp @@ -39,7 +39,8 @@ namespace Catch { void writeSection( std::string const& className, std::string const& rootName, - SectionNode const& sectionNode ); + SectionNode const& sectionNode, + bool testOkToFail ); void writeAssertions(SectionNode const& sectionNode); void writeAssertion(AssertionStats const& stats); diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 3e9c35f0..12a47a7d 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -164,6 +164,7 @@ Nor would this :test-result: FAIL Matchers can be composed with both && and || - failing :test-result: PASS Matchers can be negated (Not) with the ! operator :test-result: FAIL Matchers can be negated (Not) with the ! operator - failing +:test-result: XFAIL Mayfail test case with nested sections :test-result: FAIL Mismatching exception messages failing the test :test-result: PASS Nested generators and captured variables :test-result: FAIL Nice descriptive name diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 6c932221..34b6272c 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -911,6 +911,10 @@ Matchers.tests.cpp:: passed: testStringForMatching(), ( ContainsSub Matchers.tests.cpp:: failed: testStringForMatching(), ( ContainsSubstring( "string" ) || ContainsSubstring( "different" ) ) && ContainsSubstring( "random" ) for: "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" ) Matchers.tests.cpp:: passed: testStringForMatching(), !ContainsSubstring( "different" ) for: "this string contains 'abc' as a substring" not contains: "different" Matchers.tests.cpp:: failed: testStringForMatching(), !ContainsSubstring( "substring" ) for: "this string contains 'abc' as a substring" not contains: "substring" +Condition.tests.cpp:: failed: explicitly +Condition.tests.cpp:: failed: explicitly +Condition.tests.cpp:: failed: explicitly +Condition.tests.cpp:: failed: explicitly Exception.tests.cpp:: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" Exception.tests.cpp:: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail" Generators.tests.cpp:: passed: values > -6 for: 3 > -6 diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index d7639e4d..24ef16c4 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -694,6 +694,46 @@ Matchers.tests.cpp:: FAILED: with expansion: "this string contains 'abc' as a substring" not contains: "substring" +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- Mismatching exception messages failing the test ------------------------------------------------------------------------------- @@ -1386,6 +1426,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 371 | 295 passed | 70 failed | 6 failed as expected -assertions: 2110 | 1958 passed | 129 failed | 23 failed as expected +test cases: 372 | 295 passed | 70 failed | 7 failed as expected +assertions: 2114 | 1958 passed | 129 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index f7ab5b3c..754fbff6 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -6962,6 +6962,46 @@ Matchers.tests.cpp:: FAILED: with expansion: "this string contains 'abc' as a substring" not contains: "substring" +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- Mismatching exception messages failing the test ------------------------------------------------------------------------------- @@ -16990,6 +17030,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 371 | 279 passed | 86 failed | 6 failed as expected -assertions: 2127 | 1958 passed | 146 failed | 23 failed as expected +test cases: 372 | 279 passed | 86 failed | 7 failed as expected +assertions: 2131 | 1958 passed | 146 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index d99f25e8..70a0c253 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -50,6 +50,7 @@ Nor would this + FAILED: expected exception @@ -58,6 +59,7 @@ Exception.tests.cpp: + FAILED: REQUIRE_NOTHROW( thisThrows() ) @@ -70,6 +72,7 @@ Exception.tests.cpp: + FAILED: CHECK( f() == 0 ) @@ -463,6 +466,7 @@ Matchers.tests.cpp: + FAILED: CHECK( data.int_seven == 6 ) @@ -756,6 +760,7 @@ Message.tests.cpp: + FAILED: CHECK( data.int_seven != 7 ) @@ -819,6 +824,34 @@ with expansion: Matchers.tests.cpp: + + + +FAILED: +Condition.tests.cpp: + + + + + +FAILED: +Condition.tests.cpp: + + + + + +FAILED: +Condition.tests.cpp: + + + + + +FAILED: +Condition.tests.cpp: + + FAILED: @@ -1285,12 +1318,14 @@ Misc.tests.cpp: + FAILED: Misc.tests.cpp: + FAILED: Misc.tests.cpp: diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index cb210515..f9f8c338 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -660,6 +660,30 @@ Condition.tests.cpp: + + +FAILED: +Condition.tests.cpp: + + + + +FAILED: +Condition.tests.cpp: + + + + +FAILED: +Condition.tests.cpp: + + + + +FAILED: +Condition.tests.cpp: + + FAILED: diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index d26511c0..e7aa9eb5 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -1804,6 +1804,14 @@ not ok {test-number} - testStringForMatching(), ( ContainsSubstring( "string" ) ok {test-number} - testStringForMatching(), !ContainsSubstring( "different" ) for: "this string contains 'abc' as a substring" not contains: "different" # Matchers can be negated (Not) with the ! operator - failing not ok {test-number} - testStringForMatching(), !ContainsSubstring( "substring" ) for: "this string contains 'abc' as a substring" not contains: "substring" +# Mayfail test case with nested sections +not ok {test-number} - explicitly +# Mayfail test case with nested sections +not ok {test-number} - explicitly +# Mayfail test case with nested sections +not ok {test-number} - explicitly +# Mayfail test case with nested sections +not ok {test-number} - explicitly # Mismatching exception messages failing the test ok {test-number} - thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" # Mismatching exception messages failing the test @@ -4256,5 +4264,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2127 +1..2131 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index 7b12bd22..da0cbd08 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -406,6 +406,12 @@ Matchers.tests.cpp:|nexpression failed|n CHECK_THAT( testStringFor ##teamcity[testStarted name='Matchers can be negated (Not) with the ! operator - failing'] Matchers.tests.cpp:|nexpression failed|n CHECK_THAT( testStringForMatching(), !ContainsSubstring( "substring" ) )|nwith expansion:|n "this string contains |'abc|' as a substring" not contains: "substring"|n'] ##teamcity[testFinished name='Matchers can be negated (Not) with the ! operator - failing' duration="{duration}"] +##teamcity[testStarted name='Mayfail test case with nested sections'] +Condition.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +Condition.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +Condition.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +Condition.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='Mayfail test case with nested sections' duration="{duration}"] ##teamcity[testStarted name='Mismatching exception messages failing the test'] Exception.tests.cpp:|nexpression failed|n REQUIRE_THROWS_WITH( thisThrows(), "should fail" )|nwith expansion:|n "expected exception" equals: "should fail"|n'] ##teamcity[testFinished name='Mismatching exception messages failing the test' duration="{duration}"] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 2fee44e4..3f06d48b 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -8426,6 +8426,43 @@ Nor would this + +
+
+ + +
+ +
+
+
+ + +
+ +
+
+ +
+
+
+ + +
+ +
+
+
+ + +
+ +
+
+ +
+ + @@ -19990,6 +20027,6 @@ loose text artifact - - + + diff --git a/tests/SelfTest/UsageTests/Condition.tests.cpp b/tests/SelfTest/UsageTests/Condition.tests.cpp index 167752e1..916e98f1 100644 --- a/tests/SelfTest/UsageTests/Condition.tests.cpp +++ b/tests/SelfTest/UsageTests/Condition.tests.cpp @@ -86,6 +86,19 @@ TEST_CASE( "Equality checks that should fail", "[.][failing][!mayfail]" ) CHECK( x == Approx( 1.301 ) ); } +// Needed to test junit reporter's handling of mayfail test cases and sections +TEST_CASE("Mayfail test case with nested sections", "[!mayfail]") { + SECTION("A") { + SECTION("1") { FAIL(); } + SECTION("2") { FAIL(); } + } + SECTION("B") { + SECTION("1") { FAIL(); } + SECTION("2") { FAIL(); } + } +} + + TEST_CASE( "Inequality checks that should succeed" ) { TestData data;