From eea3e9a5b5b0a405369961eb6601a56512a87ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Mon, 7 Jun 2021 19:53:34 +0200 Subject: [PATCH] Mark !mayfail tests as skipped in the JUnit reporter Should fix #2116 --- include/reporters/catch_reporter_junit.cpp | 19 +++++--- include/reporters/catch_reporter_junit.h | 7 +-- .../Baselines/compact.sw.approved.txt | 4 ++ .../Baselines/console.std.approved.txt | 44 +++++++++++++++++- .../Baselines/console.sw.approved.txt | 44 +++++++++++++++++- .../SelfTest/Baselines/junit.sw.approved.txt | 35 ++++++++++++++- .../Baselines/sonarqube.sw.approved.txt | 24 ++++++++++ .../SelfTest/Baselines/xml.sw.approved.txt | 45 +++++++++++++++++-- .../SelfTest/UsageTests/Condition.tests.cpp | 15 ++++++- 9 files changed, 218 insertions(+), 19 deletions(-) diff --git a/include/reporters/catch_reporter_junit.cpp b/include/reporters/catch_reporter_junit.cpp index a50ead5f..ceb941be 100644 --- a/include/reporters/catch_reporter_junit.cpp +++ b/include/reporters/catch_reporter_junit.cpp @@ -182,12 +182,13 @@ namespace Catch { if ( !m_config->name().empty() ) className = m_config->name() + "." + className; - writeSection( className, "", rootSection ); + writeSection( className, "", rootSection, stats.testInfo.okToFail() ); } - void JunitReporter::writeSection( std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode ) { + void JunitReporter::writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail) { std::string name = trim( sectionNode.stats.sectionInfo.name ); if( !rootName.empty() ) name = rootName + '/' + name; @@ -211,8 +212,14 @@ namespace Catch { // events and write those out appropriately. xml.writeAttribute( "status", "run" ); + if (sectionNode.stats.assertions.failedButOk) { + xml.scopedElement("skipped") + .writeAttribute("message", "TEST_CASE tagged with !mayfail"); + } + writeAssertions( sectionNode ); + if( !sectionNode.stdOut.empty() ) xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), XmlFormatting::Newline ); if( !sectionNode.stdErr.empty() ) @@ -220,9 +227,9 @@ namespace Catch { } for( auto const& childNode : sectionNode.childSections ) if( className.empty() ) - writeSection( name, "", *childNode ); + writeSection( name, "", *childNode, testOkToFail ); else - writeSection( className, name, *childNode ); + writeSection( className, name, *childNode, testOkToFail ); } void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { diff --git a/include/reporters/catch_reporter_junit.h b/include/reporters/catch_reporter_junit.h index 5ee3a570..da5b5721 100644 --- a/include/reporters/catch_reporter_junit.h +++ b/include/reporters/catch_reporter_junit.h @@ -41,9 +41,10 @@ namespace Catch { void writeTestCase(TestCaseNode const& testCaseNode); - void writeSection(std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode); + void writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail ); void writeAssertions(SectionNode const& sectionNode); void writeAssertion(AssertionStats const& stats); diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt index efefa833..0d12ebb9 100644 --- a/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -846,6 +846,10 @@ Matchers.tests.cpp:: passed: testStringForMatching(), (Contains("st Matchers.tests.cpp:: failed: testStringForMatching(), (Contains("string") || Contains("different")) && Contains("random") for: "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" ) Matchers.tests.cpp:: passed: testStringForMatching(), !Contains("different") for: "this string contains 'abc' as a substring" not contains: "different" Matchers.tests.cpp:: failed: testStringForMatching(), !Contains("substring") for: "this string contains 'abc' as a substring" not contains: "substring" +Condition.tests.cpp:: failed: explicitly +Condition.tests.cpp:: failed: explicitly +Condition.tests.cpp:: failed: explicitly +Condition.tests.cpp:: failed: explicitly Exception.tests.cpp:: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" Exception.tests.cpp:: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail" Generators.tests.cpp:: passed: values > -6 for: 3 > -6 diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index cfcdc72e..9de49d9a 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -694,6 +694,46 @@ Matchers.tests.cpp:: FAILED: with expansion: "this string contains 'abc' as a substring" not contains: "substring" +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- Mismatching exception messages failing the test ------------------------------------------------------------------------------- @@ -1380,6 +1420,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 322 | 248 passed | 70 failed | 4 failed as expected -assertions: 1759 | 1607 passed | 131 failed | 21 failed as expected +test cases: 323 | 248 passed | 70 failed | 5 failed as expected +assertions: 1763 | 1607 passed | 131 failed | 25 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 3c09bf31..7f663129 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -6492,6 +6492,46 @@ Matchers.tests.cpp:: FAILED: with expansion: "this string contains 'abc' as a substring" not contains: "substring" +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp: +............................................................................... + +Condition.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- Mismatching exception messages failing the test ------------------------------------------------------------------------------- @@ -14138,6 +14178,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 322 | 232 passed | 86 failed | 4 failed as expected -assertions: 1776 | 1607 passed | 148 failed | 21 failed as expected +test cases: 323 | 232 passed | 86 failed | 5 failed as expected +assertions: 1780 | 1607 passed | 148 failed | 25 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 89a0b438..b7e55e04 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -48,6 +48,7 @@ Nor would this + FAILED: expected exception @@ -56,6 +57,7 @@ Exception.tests.cpp: + FAILED: REQUIRE_NOTHROW( thisThrows() ) @@ -68,6 +70,7 @@ Exception.tests.cpp: + FAILED: CHECK( f() == 0 ) @@ -443,6 +446,7 @@ Matchers.tests.cpp: + FAILED: CHECK( data.int_seven == 6 ) @@ -736,6 +740,7 @@ Message.tests.cpp: + FAILED: CHECK( data.int_seven != 7 ) @@ -799,6 +804,34 @@ with expansion: Matchers.tests.cpp: + + + +FAILED: +Condition.tests.cpp: + + + + + +FAILED: +Condition.tests.cpp: + + + + + +FAILED: +Condition.tests.cpp: + + + + + +FAILED: +Condition.tests.cpp: + + FAILED: diff --git a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt index 12a4e5be..1b4248d3 100644 --- a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -575,6 +575,30 @@ Condition.tests.cpp: + + +FAILED: +Condition.tests.cpp: + + + + +FAILED: +Condition.tests.cpp: + + + + +FAILED: +Condition.tests.cpp: + + + + +FAILED: +Condition.tests.cpp: + + FAILED: diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 2b5da8d3..0a2d3377 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -7956,6 +7956,43 @@ Nor would this + +
+
+ + +
+ +
+
+
+ + +
+ +
+
+ +
+
+
+ + +
+ +
+
+
+ + +
+ +
+
+ +
+ + @@ -16722,9 +16759,9 @@ loose text artifact - - + + - - + + diff --git a/projects/SelfTest/UsageTests/Condition.tests.cpp b/projects/SelfTest/UsageTests/Condition.tests.cpp index 9aed5e2c..fa533095 100644 --- a/projects/SelfTest/UsageTests/Condition.tests.cpp +++ b/projects/SelfTest/UsageTests/Condition.tests.cpp @@ -11,7 +11,7 @@ // Wdouble-promotion is not supported until 3.8 # if (__clang_major__ > 3) || (__clang_major__ == 3 && __clang_minor__ > 7) # pragma clang diagnostic ignored "-Wdouble-promotion" -# endif +# endif #endif #include "catch.hpp" @@ -89,6 +89,19 @@ TEST_CASE( "Equality checks that should fail", "[.][failing][!mayfail]" ) CHECK( x == Approx( 1.301 ) ); } +// Needed to test junit reporter's handling of mayfail test cases and sections +TEST_CASE("Mayfail test case with nested sections", "[!mayfail]") { + SECTION("A") { + SECTION("1") { FAIL(); } + SECTION("2") { FAIL(); } + } + SECTION("B") { + SECTION("1") { FAIL(); } + SECTION("2") { FAIL(); } + } +} + + TEST_CASE( "Inequality checks that should succeed" ) { TestData data;