3rd-party
/
catch2
mirror of https://github.com/catchorg/Catch2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mark !mayfail tests as skipped in the JUnit reporter 

Should fix #2116
This commit is contained in:
Martin Hořeňovský 2021-06-07 19:53:34 +02:00
parent 7727c15290
commit eea3e9a5b5
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
9 changed files with 218 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
include/reporters/catch_reporter_junit.cpp
View File

@ -182,12 +182,13 @@ namespace Catch {
if ( !m_config->name().empty() )
className = m_config->name() + "." + className;
writeSection( className, "", rootSection );
writeSection( className, "", rootSection, stats.testInfo.okToFail() );
}
void JunitReporter::writeSection( std::string const& className,
std::string const& rootName,
SectionNode const& sectionNode ) {
void JunitReporter::writeSection( std::string const& className,
std::string const& rootName,
SectionNode const& sectionNode,
bool testOkToFail) {
std::string name = trim( sectionNode.stats.sectionInfo.name );
if( !rootName.empty() )
name = rootName + '/' + name;
@ -211,8 +212,14 @@ namespace Catch {
// events and write those out appropriately.
xml.writeAttribute( "status", "run" );
if (sectionNode.stats.assertions.failedButOk) {
xml.scopedElement("skipped")
.writeAttribute("message", "TEST_CASE tagged with !mayfail");
}
writeAssertions( sectionNode );
if( !sectionNode.stdOut.empty() )
xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), XmlFormatting::Newline );
if( !sectionNode.stdErr.empty() )
@ -220,9 +227,9 @@ namespace Catch {
}
for( auto const& childNode : sectionNode.childSections )
if( className.empty() )
writeSection( name, "", *childNode );
writeSection( name, "", *childNode, testOkToFail );
else
writeSection( className, name, *childNode );
writeSection( className, name, *childNode, testOkToFail );
}
void JunitReporter::writeAssertions( SectionNode const& sectionNode ) {

7
include/reporters/catch_reporter_junit.h
View File

@ -41,9 +41,10 @@ namespace Catch {
void writeTestCase(TestCaseNode const& testCaseNode);
void writeSection(std::string const& className,
std::string const& rootName,
SectionNode const& sectionNode);
void writeSection( std::string const& className,
std::string const& rootName,
SectionNode const& sectionNode,
bool testOkToFail );
void writeAssertions(SectionNode const& sectionNode);
void writeAssertion(AssertionStats const& stats);

4
projects/SelfTest/Baselines/compact.sw.approved.txt
View File

@ -846,6 +846,10 @@ Matchers.tests.cpp:<line number>: passed: testStringForMatching(), (Contains("st
Matchers.tests.cpp:<line number>: failed: testStringForMatching(), (Contains("string") || Contains("different")) && Contains("random") for: "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" )
Matchers.tests.cpp:<line number>: passed: testStringForMatching(), !Contains("different") for: "this string contains 'abc' as a substring" not contains: "different"
Matchers.tests.cpp:<line number>: failed: testStringForMatching(), !Contains("substring") for: "this string contains 'abc' as a substring" not contains: "substring"
Condition.tests.cpp:<line number>: failed: explicitly
Condition.tests.cpp:<line number>: failed: explicitly
Condition.tests.cpp:<line number>: failed: explicitly
Condition.tests.cpp:<line number>: failed: explicitly
Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail"
Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6

44
projects/SelfTest/Baselines/console.std.approved.txt
View File

@ -694,6 +694,46 @@ Matchers.tests.cpp:<line number>: FAILED:
with expansion:
"this string contains 'abc' as a substring" not contains: "substring"
-------------------------------------------------------------------------------
Mayfail test case with nested sections
A
1
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mayfail test case with nested sections
A
2
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mayfail test case with nested sections
B
1
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mayfail test case with nested sections
B
2
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mismatching exception messages failing the test
-------------------------------------------------------------------------------
@ -1380,6 +1420,6 @@ due to unexpected exception with message:
Why would you throw a std::string?
===============================================================================
test cases: 322 | 248 passed | 70 failed | 4 failed as expected
assertions: 1759 | 1607 passed | 131 failed | 21 failed as expected
test cases: 323 | 248 passed | 70 failed | 5 failed as expected
assertions: 1763 | 1607 passed | 131 failed | 25 failed as expected

44
projects/SelfTest/Baselines/console.sw.approved.txt
View File

@ -6492,6 +6492,46 @@ Matchers.tests.cpp:<line number>: FAILED:
with expansion:
"this string contains 'abc' as a substring" not contains: "substring"
-------------------------------------------------------------------------------
Mayfail test case with nested sections
A
1
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mayfail test case with nested sections
A
2
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mayfail test case with nested sections
B
1
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mayfail test case with nested sections
B
2
-------------------------------------------------------------------------------
Condition.tests.cpp:<line number>
...............................................................................
Condition.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
Mismatching exception messages failing the test
-------------------------------------------------------------------------------
@ -14138,6 +14178,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 322 | 232 passed | 86 failed | 4 failed as expected
assertions: 1776 | 1607 passed | 148 failed | 21 failed as expected
test cases: 323 | 232 passed | 86 failed | 5 failed as expected
assertions: 1780 | 1607 passed | 148 failed | 25 failed as expected

35
projects/SelfTest/Baselines/junit.sw.approved.txt
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1777" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="132" tests="1781" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
<property name="random-seed" value="1"/>
@ -48,6 +48,7 @@ Nor would this
<testcase classname="<exe-name>.global" name="#1954 - 7 arg template test case sig compiles - 5, 1, 1, 1, 1, 0, 0" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1954 - 7 arg template test case sig compiles - 5, 3, 1, 1, 1, 0, 0" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<error type="TEST_CASE">
FAILED:
expected exception
@ -56,6 +57,7 @@ Exception.tests.cpp:<line number>
</error>
</testcase>
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/inside REQUIRE_NOTHROW" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<error message="thisThrows()" type="REQUIRE_NOTHROW">
FAILED:
REQUIRE_NOTHROW( thisThrows() )
@ -68,6 +70,7 @@ Exception.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="#809" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#833" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#835 -- errno should not be touched by Catch" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure message="f() == 0" type="CHECK">
FAILED:
CHECK( f() == 0 )
@ -443,6 +446,7 @@ Matchers.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="Enums in namespaces can quickly have stringification enabled using REGISTER_ENUM" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Epsilon only applies to Approx's value" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Equality checks that should fail" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure message="data.int_seven == 6" type="CHECK">
FAILED:
CHECK( data.int_seven == 6 )
@ -736,6 +740,7 @@ Message.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Inequality checks that should fail" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure message="data.int_seven != 7" type="CHECK">
FAILED:
CHECK( data.int_seven != 7 )
@ -799,6 +804,34 @@ with expansion:
Matchers.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/A" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure type="FAIL">
FAILED:
Condition.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/A" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure type="FAIL">
FAILED:
Condition.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/B" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure type="FAIL">
FAILED:
Condition.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/B" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure type="FAIL">
FAILED:
Condition.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Mismatching exception messages failing the test" time="{duration}" status="run">
<failure message="thisThrows(), &quot;should fail&quot;" type="REQUIRE_THROWS_WITH">
FAILED:

24
projects/SelfTest/Baselines/sonarqube.sw.approved.txt
View File

@ -575,6 +575,30 @@ Condition.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="Inequality checks that should succeed" duration="{duration}"/>
<testCase name="Mayfail test case with nested sections/1/A" duration="{duration}">
<skipped message="FAIL()">
FAILED:
Condition.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="Mayfail test case with nested sections/2/A" duration="{duration}">
<skipped message="FAIL()">
FAILED:
Condition.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="Mayfail test case with nested sections/1/B" duration="{duration}">
<skipped message="FAIL()">
FAILED:
Condition.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="Mayfail test case with nested sections/2/B" duration="{duration}">
<skipped message="FAIL()">
FAILED:
Condition.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="Ordering comparison checks that should fail" duration="{duration}">
<failure message="CHECK(data.int_seven > 7)">
FAILED:

45
projects/SelfTest/Baselines/xml.sw.approved.txt
View File

@ -7956,6 +7956,43 @@ Nor would this
</Expression>
<OverallResult success="false"/>
</TestCase>
<TestCase name="Mayfail test case with nested sections" tags="[!mayfail]" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Section name="1" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Section name="2" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<OverallResults successes="0" failures="0" expectedFailures="0"/>
</Section>
<Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Section name="1" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Section name="2" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" />
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<OverallResults successes="0" failures="0" expectedFailures="1"/>
</Section>
<Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" >
<OverallResults successes="0" failures="0" expectedFailures="0"/>
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Mismatching exception messages failing the test" tags="[!hide][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Expression success="true" type="REQUIRE_THROWS_WITH" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Original>
@ -16722,9 +16759,9 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="1607" failures="149" expectedFailures="21"/>
<OverallResultsCases successes="232" failures="86" expectedFailures="4"/>
<OverallResults successes="1607" failures="149" expectedFailures="25"/>
<OverallResultsCases successes="232" failures="86" expectedFailures="5"/>
</Group>
<OverallResults successes="1607" failures="148" expectedFailures="21"/>
<OverallResultsCases successes="232" failures="86" expectedFailures="4"/>
<OverallResults successes="1607" failures="148" expectedFailures="25"/>
<OverallResultsCases successes="232" failures="86" expectedFailures="5"/>
</Catch>

15
projects/SelfTest/UsageTests/Condition.tests.cpp
View File

@ -11,7 +11,7 @@
// Wdouble-promotion is not supported until 3.8
# if (__clang_major__ > 3) || (__clang_major__ == 3 && __clang_minor__ > 7)
# pragma clang diagnostic ignored "-Wdouble-promotion"
# endif
# endif
#endif
#include "catch.hpp"
@ -89,6 +89,19 @@ TEST_CASE( "Equality checks that should fail", "[.][failing][!mayfail]" )
CHECK( x == Approx( 1.301 ) );
}
// Needed to test junit reporter's handling of mayfail test cases and sections
TEST_CASE("Mayfail test case with nested sections", "[!mayfail]") {
SECTION("A") {
SECTION("1") { FAIL(); }
SECTION("2") { FAIL(); }
}
SECTION("B") {
SECTION("1") { FAIL(); }
SECTION("2") { FAIL(); }
}
}
TEST_CASE( "Inequality checks that should succeed" )
{
TestData data;