3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2024-11-22 13:26:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Don’t include tests that rely on varying two-phase-lookup behaviour in approval tests

Browse Source
This commit is contained in:
Phil Nash 2017-01-23 17:47:58 +00:00
parent 31c23b9489
commit 225e90d8ba
6 changed files with 19 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
projects/SelfTest/Baselines/console.std.approved.txt
View File

@ -830,6 +830,6 @@ with expansion:
"first" == "second" "first" == "second"
=============================================================================== ===============================================================================
test cases: 159 | 115 passed | 42 failed | 2 failed as expected test cases: 157 | 113 passed | 42 failed | 2 failed as expected
assertions: 915 | 819 passed | 78 failed | 18 failed as expected assertions: 913 | 817 passed | 78 failed | 18 failed as expected

30
projects/SelfTest/Baselines/console.sw.approved.txt
View File

@ -8757,32 +8757,6 @@ with expansion:
== ==
"{ StringMaker<has_maker> }" "{ StringMaker<has_maker> }"
-------------------------------------------------------------------------------
toString( vectors<has_maker_and_toString )
-------------------------------------------------------------------------------
ToStringWhich.cpp:<line number>
...............................................................................
ToStringWhich.cpp:<line number>:
PASSED:
REQUIRE( Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" )
with expansion:
"{ StringMaker<has_maker_and_toString> }"
==
"{ StringMaker<has_maker_and_toString> }"
-------------------------------------------------------------------------------
toString( vectors<has_toString )
-------------------------------------------------------------------------------
ToStringWhich.cpp:<line number>
...............................................................................
ToStringWhich.cpp:<line number>:
PASSED:
REQUIRE( Catch::toString( v ) == "{ {?} }" )
with expansion:
"{ {?} }" == "{ {?} }"
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
toString(enum w/operator<<) toString(enum w/operator<<)
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
@ -9052,6 +9026,6 @@ MiscTests.cpp:<line number>:
PASSED: PASSED:
=============================================================================== ===============================================================================
test cases: 159 | 114 passed | 43 failed | 2 failed as expected test cases: 157 | 112 passed | 43 failed | 2 failed as expected
assertions: 917 | 819 passed | 80 failed | 18 failed as expected assertions: 915 | 817 passed | 80 failed | 18 failed as expected

4
projects/SelfTest/Baselines/junit.sw.approved.txt
View File

@ -1,6 +1,6 @@
<?xml version="1.1" encoding="UTF-8"?> <?xml version="1.1" encoding="UTF-8"?>
<testsuites> <testsuites>
<testsuite name="<exe-name>" errors="13" failures="68" tests="918" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <testsuite name="<exe-name>" errors="13" failures="68" tests="916" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testcase classname="global" name="# A test name that starts with a #" time="{duration}"/> <testcase classname="global" name="# A test name that starts with a #" time="{duration}"/>
<testcase classname="global" name="'Not' checks that should fail" time="{duration}"> <testcase classname="global" name="'Not' checks that should fail" time="{duration}">
<failure message="false != false" type="CHECK"> <failure message="false != false" type="CHECK">
@ -654,8 +654,6 @@ TrickyTests.cpp:<line number>
<testcase classname="global" name="toString( has_maker_and_toString )" time="{duration}"/> <testcase classname="global" name="toString( has_maker_and_toString )" time="{duration}"/>
<testcase classname="global" name="toString( has_toString )" time="{duration}"/> <testcase classname="global" name="toString( has_toString )" time="{duration}"/>
<testcase classname="global" name="toString( vectors&lt;has_maker )" time="{duration}"/> <testcase classname="global" name="toString( vectors&lt;has_maker )" time="{duration}"/>
<testcase classname="global" name="toString( vectors&lt;has_maker_and_toString )" time="{duration}"/>
<testcase classname="global" name="toString( vectors&lt;has_toString )" time="{duration}"/>
<testcase classname="global" name="toString(enum w/operator&lt;&lt;)" time="{duration}"/> <testcase classname="global" name="toString(enum w/operator&lt;&lt;)" time="{duration}"/>
<testcase classname="global" name="toString(enum)" time="{duration}"/> <testcase classname="global" name="toString(enum)" time="{duration}"/>
<testcase classname="global" name="vector&lt;int> -> toString" time="{duration}"/> <testcase classname="global" name="vector&lt;int> -> toString" time="{duration}"/>

28
projects/SelfTest/Baselines/xml.sw.approved.txt
View File

@ -9246,30 +9246,6 @@ there"
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<TestCase name="toString( vectors&lt;has_maker_and_toString )">
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ToStringWhich.cpp" >
<Original>
Catch::toString( v ) == "{ StringMaker&lt;has_maker_and_toString> }"
</Original>
<Expanded>
"{ StringMaker&lt;has_maker_and_toString> }"
==
"{ StringMaker&lt;has_maker_and_toString> }"
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="toString( vectors&lt;has_toString )">
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ToStringWhich.cpp" >
<Original>
Catch::toString( v ) == "{ {?} }"
</Original>
<Expanded>
"{ {?} }" == "{ {?} }"
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="toString(enum w/operator&lt;&lt;)"> <TestCase name="toString(enum w/operator&lt;&lt;)">
<Expression success="true" type="CHECK" filename="projects/<exe-name>/EnumToString.cpp" > <Expression success="true" type="CHECK" filename="projects/<exe-name>/EnumToString.cpp" >
<Original> <Original>
@ -9527,7 +9503,7 @@ there"
</Section> </Section>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<OverallResults successes="819" failures="81" expectedFailures="18"/> <OverallResults successes="817" failures="81" expectedFailures="18"/>
</Group> </Group>
<OverallResults successes="819" failures="80" expectedFailures="18"/> <OverallResults successes="817" failures="80" expectedFailures="18"/>
</Catch> </Catch>

11
projects/SelfTest/ToStringWhich.cpp
View File

@ -37,19 +37,21 @@ TEST_CASE( "toString( has_toString )", "[toString]" ) {
} }
// Call the overload // Call the overload
TEST_CASE( "toString( has_maker )", "[toString]" ) { TEST_CASE( "toString( has_maker )", "toString]" ) {
has_maker item; has_maker item;
REQUIRE( Catch::toString( item ) == "StringMaker<has_maker>" ); REQUIRE( Catch::toString( item ) == "StringMaker<has_maker>" );
} }
// Call the overload // Call the overload
TEST_CASE( "toString( has_maker_and_toString )", "[toString]" ) { TEST_CASE( "toString( has_maker_and_toString )", "[.][toString]" ) {
has_maker_and_toString item; has_maker_and_toString item;
REQUIRE( Catch::toString( item ) == "toString( has_maker_and_toString )" ); REQUIRE( Catch::toString( item ) == "toString( has_maker_and_toString )" );
} }
// Vectors... // Vectors...
TEST_CASE( "toString( vectors<has_toString )", "[toString]" ) {
// Don't run this in approval tests as it is sensitive to two phase lookup differences
TEST_CASE( "toString( vectors<has_toString )", "[.][toString][!nonportable]" ) {
std::vector<has_toString> v(1); std::vector<has_toString> v(1);
// This invokes template<T> toString which actually gives us '{ ? }' // This invokes template<T> toString which actually gives us '{ ? }'
REQUIRE( Catch::toString( v ) == "{ {?} }" ); REQUIRE( Catch::toString( v ) == "{ {?} }" );
@ -61,7 +63,8 @@ TEST_CASE( "toString( vectors<has_maker )", "[toString]" ) {
} }
TEST_CASE( "toString( vectors<has_maker_and_toString )", "[toString]" ) { // Don't run this in approval tests as it is sensitive to two phase lookup differences
TEST_CASE( "toString( vectors<has_maker_and_toString )", "[.][toString][!nonportable]" ) {
std::vector<has_maker_and_toString> v(1); std::vector<has_maker_and_toString> v(1);
// Note: This invokes the template<T> toString -> StringMaker // Note: This invokes the template<T> toString -> StringMaker
REQUIRE( Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" ); REQUIRE( Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" );

10
scripts/approvalTests.py
View File

@ -137,15 +137,15 @@ print("Running approvals against executable:")
print(" " + cmdPath) print(" " + cmdPath)
# Standard console reporter # Standard console reporter
approve("console.std", ["~[c++11]", "--order", "lex"]) approve("console.std", ["~[c++11]~[!nonportable]", "--order", "lex"])
# console reporter, include passes, warn about No Assertions # console reporter, include passes, warn about No Assertions
approve("console.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "--order", "lex"]) approve("console.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"])
# console reporter, include passes, warn about No Assertions, limit failures to first 4 # console reporter, include passes, warn about No Assertions, limit failures to first 4
approve("console.swa4", ["~[c++11]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) approve("console.swa4", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
# junit reporter, include passes, warn about No Assertions # junit reporter, include passes, warn about No Assertions
approve("junit.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) approve("junit.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
# xml reporter, include passes, warn about No Assertions # xml reporter, include passes, warn about No Assertions
approve("xml.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) approve("xml.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
if overallResult != 0: if overallResult != 0:
print("If these differenecs are expected run approve.py to approve new baselines") print("If these differenecs are expected run approve.py to approve new baselines")