3rd-party
/
catch2
mirror of https://github.com/catchorg/Catch2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix/disable failing approval tests

This commit is contained in:
Martin Hořeňovský 2017-09-01 20:28:49 +02:00
parent b3a923133d
commit ab5d176195
9 changed files with 37 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -317,6 +317,7 @@ if (NOT NO_SELFTEST)
add_test(NAME ListTags COMMAND SelfTest --list-tags) add_test(NAME ListTags COMMAND SelfTest --list-tags)
set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags") set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
# AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>) add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)
set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed") set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed")
endif() # !NO_SELFTEST endif() # !NO_SELFTEST

18
projects/SelfTest/Baselines/console.std.approved.txt
View File

@ -644,20 +644,6 @@ MessageTests.cpp:<line number>: FAILED:
explicitly with message: explicitly with message:
Message from section two Message from section two
-------------------------------------------------------------------------------
Pointers can be converted to strings
-------------------------------------------------------------------------------
MessageTests.cpp:<line number>
...............................................................................
MessageTests.cpp:<line number>:
warning:
actual address of p: 0x<hex digits>
MessageTests.cpp:<line number>:
warning:
toString(p): 0x<hex digits>
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Reconstruction should be based on stringification: #914 Reconstruction should be based on stringification: #914
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
@ -1020,6 +1006,6 @@ with expansion:
"{?}" == "1" "{?}" == "1"
=============================================================================== ===============================================================================
test cases: 182 | 131 passed | 47 failed | 4 failed as expected test cases: 179 | 128 passed | 47 failed | 4 failed as expected
assertions: 881 | 764 passed | 96 failed | 21 failed as expected assertions: 879 | 762 passed | 96 failed | 21 failed as expected

50
projects/SelfTest/Baselines/console.sw.approved.txt
View File

@ -955,20 +955,6 @@ PASSED:
with expansion: with expansion:
0x<hex digits> == 0x<hex digits> 0x<hex digits> == 0x<hex digits>
-------------------------------------------------------------------------------
Comparing member function pointers
-------------------------------------------------------------------------------
TrickyTests.cpp:<line number>
...............................................................................
TrickyTests.cpp:<line number>:
PASSED:
CHECK( m == &S::f )
with expansion:
0x<hex digits>
==
0x<hex digits>
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Comparison with explicitly convertible types Comparison with explicitly convertible types
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
@ -1182,11 +1168,9 @@ with expansion:
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul ) REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul )
with expansion: with expansion:
18446744073709551615 (0x<hex digits>) 4294967295 (0x<hex digits>) > 4
>
4
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Contains string matcher Contains string matcher
@ -2096,18 +2080,6 @@ PASSED:
with expansion: with expansion:
!{?} !{?}
-------------------------------------------------------------------------------
Operators at different namespace levels not hijacked by Koenig lookup
-------------------------------------------------------------------------------
TrickyTests.cpp:<line number>
...............................................................................
TrickyTests.cpp:<line number>:
PASSED:
REQUIRE( 0x<hex digits> == o )
with expansion:
3221225472 (0x<hex digits>) == {?}
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Ordering comparison checks that should fail Ordering comparison checks that should fail
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
@ -3432,20 +3404,6 @@ PASSED:
with expansion: with expansion:
0 != 0x<hex digits> 0 != 0x<hex digits>
-------------------------------------------------------------------------------
Pointers can be converted to strings
-------------------------------------------------------------------------------
MessageTests.cpp:<line number>
...............................................................................
MessageTests.cpp:<line number>:
warning:
actual address of p: 0x<hex digits>
MessageTests.cpp:<line number>:
warning:
toString(p): 0x<hex digits>
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Process can be configured on command line Process can be configured on command line
empty args don't cause a crash empty args don't cause a crash
@ -7521,6 +7479,6 @@ MiscTests.cpp:<line number>:
PASSED: PASSED:
=============================================================================== ===============================================================================
test cases: 182 | 128 passed | 50 failed | 4 failed as expected test cases: 179 | 125 passed | 50 failed | 4 failed as expected
assertions: 883 | 760 passed | 102 failed | 21 failed as expected assertions: 881 | 758 passed | 102 failed | 21 failed as expected

5
projects/SelfTest/Baselines/junit.sw.approved.txt
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact <testsuitesloose text artifact
> >
<testsuite name="<exe-name>" errors="15" failures="88" tests="884" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <testsuite name="<exe-name>" errors="15" failures="88" tests="882" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/> <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/> <testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}"> <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
@ -116,7 +116,6 @@ ExceptionTests.cpp:<line number>
<testcase classname="<exe-name>.global" name="Character pretty printing/Low ASCII" time="{duration}"/> <testcase classname="<exe-name>.global" name="Character pretty printing/Low ASCII" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Commas in various macros are allowed" time="{duration}"/> <testcase classname="<exe-name>.global" name="Commas in various macros are allowed" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Comparing function pointers" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparing function pointers" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Comparing member function pointers" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Comparison with explicitly convertible types" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparison with explicitly convertible types" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Comparisons between ints where one side is computed" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparisons between ints where one side is computed" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Comparisons between unsigned ints and negative signed ints match c++ standard behaviour" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparisons between unsigned ints and negative signed ints match c++ standard behaviour" time="{duration}"/>
@ -335,7 +334,6 @@ ExceptionTests.cpp:<line number>
</error> </error>
</testcase> </testcase>
<testcase classname="<exe-name>.global" name="Objects that evaluated in boolean contexts can be checked" time="{duration}"/> <testcase classname="<exe-name>.global" name="Objects that evaluated in boolean contexts can be checked" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Operators at different namespace levels not hijacked by Koenig lookup" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Ordering comparison checks that should fail" time="{duration}"> <testcase classname="<exe-name>.global" name="Ordering comparison checks that should fail" time="{duration}">
<failure message="7 > 7" type="CHECK"> <failure message="7 > 7" type="CHECK">
ConditionTests.cpp:<line number> ConditionTests.cpp:<line number>
@ -441,7 +439,6 @@ MessageTests.cpp:<line number>
<testcase classname="<exe-name>.global" name="Parse test names and tags/quoted string followed by tag exclusion" time="{duration}"/> <testcase classname="<exe-name>.global" name="Parse test names and tags/quoted string followed by tag exclusion" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Parsing a std::pair" time="{duration}"/> <testcase classname="<exe-name>.global" name="Parsing a std::pair" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Pointers can be compared to null" time="{duration}"/> <testcase classname="<exe-name>.global" name="Pointers can be compared to null" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Pointers can be converted to strings" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/empty args don't cause a crash" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/empty args don't cause a crash" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/default - no arguments" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/default - no arguments" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/test lists/1 test" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/test lists/1 test" time="{duration}"/>

45
projects/SelfTest/Baselines/xml.sw.approved.txt
View File

@ -1023,19 +1023,6 @@
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<TestCase name="Comparing member function pointers" tags="[Tricky][member function pointer]" filename="projects/<exe-name>/TrickyTests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/TrickyTests.cpp" >
<Original>
m == &amp;S::f
</Original>
<Expanded>
0x<hex digits>
==
0x<hex digits>
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Comparison with explicitly convertible types" tags="[Approx]" filename="projects/<exe-name>/ApproxTests.cpp" > <TestCase name="Comparison with explicitly convertible types" tags="[Approx]" filename="projects/<exe-name>/ApproxTests.cpp" >
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ApproxTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ApproxTests.cpp" >
<Original> <Original>
@ -1296,12 +1283,10 @@
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ConditionTests.cpp" >
<Original> <Original>
(std::numeric_limits&lt;unsigned long>::max)() > ul (std::numeric_limits&lt;uint32_t>::max)() > ul
</Original> </Original>
<Expanded> <Expanded>
18446744073709551615 (0x<hex digits>) 4294967295 (0x<hex digits>) > 4
>
4
</Expanded> </Expanded>
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
@ -2388,17 +2373,6 @@
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<TestCase name="Operators at different namespace levels not hijacked by Koenig lookup" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" >
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" >
<Original>
0x<hex digits> == o
</Original>
<Expanded>
3221225472 (0x<hex digits>) == {?}
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Ordering comparison checks that should fail" tags="[.][failing]" filename="projects/<exe-name>/ConditionTests.cpp" > <TestCase name="Ordering comparison checks that should fail" tags="[.][failing]" filename="projects/<exe-name>/ConditionTests.cpp" >
<Expression success="false" type="CHECK" filename="projects/<exe-name>/ConditionTests.cpp" > <Expression success="false" type="CHECK" filename="projects/<exe-name>/ConditionTests.cpp" >
<Original> <Original>
@ -3962,15 +3936,6 @@
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<TestCase name="Pointers can be converted to strings" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
<Warning>
actual address of p: 0x<hex digits>
</Warning>
<Warning>
toString(p): 0x<hex digits>
</Warning>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Process can be configured on command line" tags="[command-line][config]" filename="projects/<exe-name>/TestMain.cpp" > <TestCase name="Process can be configured on command line" tags="[command-line][config]" filename="projects/<exe-name>/TestMain.cpp" >
<Section name="empty args don't cause a crash" filename="projects/<exe-name>/TestMain.cpp" > <Section name="empty args don't cause a crash" filename="projects/<exe-name>/TestMain.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/TestMain.cpp" > <Expression success="true" type="CHECK" filename="projects/<exe-name>/TestMain.cpp" >
@ -4824,7 +4789,7 @@ A string sent directly to stderr
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<TestCase name="Standard error is reported and redirected" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" > <TestCase name="Standard error is reported and redirected" tags="[.][.approvals][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
<Section name="std::cerr" filename="projects/<exe-name>/MessageTests.cpp" > <Section name="std::cerr" filename="projects/<exe-name>/MessageTests.cpp" >
<OverallResults successes="0" failures="1" expectedFailures="0"/> <OverallResults successes="0" failures="1" expectedFailures="0"/>
</Section> </Section>
@ -8276,7 +8241,7 @@ loose text artifact
</Section> </Section>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<OverallResults successes="760" failures="103" expectedFailures="21"/> <OverallResults successes="758" failures="103" expectedFailures="21"/>
</Group> </Group>
<OverallResults successes="760" failures="102" expectedFailures="21"/> <OverallResults successes="758" failures="102" expectedFailures="21"/>
</Catch> </Catch>

3
projects/SelfTest/ConditionTests.cpp
View File

@ -13,6 +13,7 @@
#include <string> #include <string>
#include <limits> #include <limits>
#include <cstdint>
struct TestData { struct TestData {
int int_seven = 7; int int_seven = 7;
@ -184,7 +185,7 @@ TEST_CASE( "Comparisons with int literals don't warn when mixing signed/ unsigne
REQUIRE( 5 == c ); REQUIRE( 5 == c );
REQUIRE( 6 == uc ); REQUIRE( 6 == uc );
REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul ); REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul );
} }
// Disable warnings about sign conversions for the next two tests // Disable warnings about sign conversions for the next two tests

6
projects/SelfTest/MessageTests.cpp
View File

@ -85,7 +85,7 @@ TEST_CASE( "Standard output from all sections is reported", "[messages][.]" ) {
} }
} }
TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) { TEST_CASE( "Standard error is reported and redirected", "[messages][.][approvals]" ) {
SECTION( "std::cerr" ) { SECTION( "std::cerr" ) {
std::cerr << "Write to std::cerr" << std::endl; std::cerr << "Write to std::cerr" << std::endl;
} }
@ -98,7 +98,7 @@ TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) {
std::cerr << ' '; std::cerr << ' ';
std::clog << "writes"; std::clog << "writes";
std::cerr << " to error"; std::cerr << " to error";
std::clog << " streams\n"; std::clog << " streams" << std::endl;
} }
} }
@ -130,7 +130,7 @@ TEST_CASE( "sends information to INFO", "[.][failing]" ) {
REQUIRE( false ); REQUIRE( false );
} }
TEST_CASE( "Pointers can be converted to strings", "[messages][.]" ) { TEST_CASE( "Pointers can be converted to strings", "[messages][.][approvals]" ) {
int p; int p;
WARN( "actual address of p: " << &p ); WARN( "actual address of p: " << &p );
WARN( "toString(p): " << ::Catch::Detail::stringify( &p ) ); WARN( "toString(p): " << ::Catch::Detail::stringify( &p ) );

17
projects/SelfTest/TrickyTests.cpp
View File

@ -175,7 +175,7 @@ namespace ObjectWithConversions
TEST_CASE TEST_CASE
( (
"Operators at different namespace levels not hijacked by Koenig lookup", "Operators at different namespace levels not hijacked by Koenig lookup",
"[Tricky]" "[Tricky][approvals]"
) )
{ {
Object o; Object o;
@ -211,9 +211,16 @@ namespace ObjectWithNonConstEqualityOperator
namespace EnumBitFieldTests namespace EnumBitFieldTests
{ {
enum Bits {bit0 = 0x0001, bit1 = 0x0002, bit2 = 0x0004, bit3 = 0x0008, bit1and2 = 0x0006, enum Bits : uint32_t {
bit30 = 0x40000000, bit31 = 0x80000000, bit0 = 0x0001,
bit30and31 = 0xc0000000}; bit1 = 0x0002,
bit2 = 0x0004,
bit3 = 0x0008,
bit1and2 = bit1 | bit2,
bit30 = 0x40000000,
bit31 = 0x80000000,
bit30and31 = bit30 | bit31
};
TEST_CASE( "Test enum bit values", "[Tricky]" ) TEST_CASE( "Test enum bit values", "[Tricky]" )
{ {
@ -365,7 +372,7 @@ struct S
}; };
TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer]" ) TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer][approvals]" )
{ {
typedef void (S::*MF)(); typedef void (S::*MF)();
MF m = &S::f; MF m = &S::f;

12
scripts/approvalTests.py
View File

@ -41,6 +41,8 @@ errnoParser = re.compile(r'''
\(\*__errno_location\ \(\)\) \(\*__errno_location\ \(\)\)
| |
\(\*__error\(\)\) \(\*__error\(\)\)
|
\(\*_errno\(\)\)
''', re.VERBOSE) ''', re.VERBOSE)
if len(sys.argv) == 2: if len(sys.argv) == 2:
@ -145,15 +147,15 @@ print("Running approvals against executable:")
print(" " + cmdPath) print(" " + cmdPath)
# Standard console reporter # Standard console reporter
approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"]) approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex"])
# console reporter, include passes, warn about No Assertions # console reporter, include passes, warn about No Assertions
approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"]) approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex"])
# console reporter, include passes, warn about No Assertions, limit failures to first 4 # console reporter, include passes, warn about No Assertions, limit failures to first 4
approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
# junit reporter, include passes, warn about No Assertions # junit reporter, include passes, warn about No Assertions
approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
# xml reporter, include passes, warn about No Assertions # xml reporter, include passes, warn about No Assertions
approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
if overallResult != 0: if overallResult != 0:
print("If these differences are expected, run approve.py to approve new baselines.") print("If these differences are expected, run approve.py to approve new baselines.")