mirror of https://github.com/catchorg/Catch2.git
Fix/disable failing approval tests
This commit is contained in:
Martin Hořeňovský
parent
b3a923133d
commit
ab5d176195
|
|
@ -317,6 +317,7 @@ if (NOT NO_SELFTEST)
|
|||
add_test(NAME ListTags COMMAND SelfTest --list-tags)
|
||||
set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
|
||||
|
||||
# AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
|
||||
add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)
|
||||
set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed")
|
||||
endif() # !NO_SELFTEST
|
||||
|
|
|
|||
|
|
@ -644,20 +644,6 @@ MessageTests.cpp:<line number>: FAILED:
|
|||
explicitly with message:
|
||||
Message from section two
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Pointers can be converted to strings
|
||||
-------------------------------------------------------------------------------
|
||||
MessageTests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
MessageTests.cpp:<line number>:
|
||||
warning:
|
||||
actual address of p: 0x<hex digits>
|
||||
|
||||
MessageTests.cpp:<line number>:
|
||||
warning:
|
||||
toString(p): 0x<hex digits>
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Reconstruction should be based on stringification: #914
|
||||
-------------------------------------------------------------------------------
|
||||
|
|
@ -1020,6 +1006,6 @@ with expansion:
|
|||
"{?}" == "1"
|
||||
|
||||
===============================================================================
|
||||
test cases: 182 | 131 passed | 47 failed | 4 failed as expected
|
||||
assertions: 881 | 764 passed | 96 failed | 21 failed as expected
|
||||
test cases: 179 | 128 passed | 47 failed | 4 failed as expected
|
||||
assertions: 879 | 762 passed | 96 failed | 21 failed as expected
|
||||
|
||||
|
|
|
|||
|
|
@ -955,20 +955,6 @@ PASSED:
|
|||
with expansion:
|
||||
0x<hex digits> == 0x<hex digits>
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Comparing member function pointers
|
||||
-------------------------------------------------------------------------------
|
||||
TrickyTests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
TrickyTests.cpp:<line number>:
|
||||
PASSED:
|
||||
CHECK( m == &S::f )
|
||||
with expansion:
|
||||
0x<hex digits>
|
||||
==
|
||||
0x<hex digits>
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Comparison with explicitly convertible types
|
||||
-------------------------------------------------------------------------------
|
||||
|
|
@ -1182,11 +1168,9 @@ with expansion:
|
|||
|
||||
ConditionTests.cpp:<line number>:
|
||||
PASSED:
|
||||
REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul )
|
||||
REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul )
|
||||
with expansion:
|
||||
18446744073709551615 (0x<hex digits>)
|
||||
>
|
||||
4
|
||||
4294967295 (0x<hex digits>) > 4
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Contains string matcher
|
||||
|
|
@ -2096,18 +2080,6 @@ PASSED:
|
|||
with expansion:
|
||||
!{?}
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Operators at different namespace levels not hijacked by Koenig lookup
|
||||
-------------------------------------------------------------------------------
|
||||
TrickyTests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
TrickyTests.cpp:<line number>:
|
||||
PASSED:
|
||||
REQUIRE( 0x<hex digits> == o )
|
||||
with expansion:
|
||||
3221225472 (0x<hex digits>) == {?}
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Ordering comparison checks that should fail
|
||||
-------------------------------------------------------------------------------
|
||||
|
|
@ -3432,20 +3404,6 @@ PASSED:
|
|||
with expansion:
|
||||
0 != 0x<hex digits>
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Pointers can be converted to strings
|
||||
-------------------------------------------------------------------------------
|
||||
MessageTests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
MessageTests.cpp:<line number>:
|
||||
warning:
|
||||
actual address of p: 0x<hex digits>
|
||||
|
||||
MessageTests.cpp:<line number>:
|
||||
warning:
|
||||
toString(p): 0x<hex digits>
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
empty args don't cause a crash
|
||||
|
|
@ -7521,6 +7479,6 @@ MiscTests.cpp:<line number>:
|
|||
PASSED:
|
||||
|
||||
===============================================================================
|
||||
test cases: 182 | 128 passed | 50 failed | 4 failed as expected
|
||||
assertions: 883 | 760 passed | 102 failed | 21 failed as expected
|
||||
test cases: 179 | 125 passed | 50 failed | 4 failed as expected
|
||||
assertions: 881 | 758 passed | 102 failed | 21 failed as expected
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuitesloose text artifact
|
||||
>
|
||||
<testsuite name="<exe-name>" errors="15" failures="88" tests="884" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
|
||||
<testsuite name="<exe-name>" errors="15" failures="88" tests="882" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
|
||||
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
|
||||
|
|
@ -116,7 +116,6 @@ ExceptionTests.cpp:<line number>
|
|||
<testcase classname="<exe-name>.global" name="Character pretty printing/Low ASCII" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Commas in various macros are allowed" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Comparing function pointers" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Comparing member function pointers" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Comparison with explicitly convertible types" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Comparisons between ints where one side is computed" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Comparisons between unsigned ints and negative signed ints match c++ standard behaviour" time="{duration}"/>
|
||||
|
|
@ -335,7 +334,6 @@ ExceptionTests.cpp:<line number>
|
|||
</error>
|
||||
</testcase>
|
||||
<testcase classname="<exe-name>.global" name="Objects that evaluated in boolean contexts can be checked" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Operators at different namespace levels not hijacked by Koenig lookup" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Ordering comparison checks that should fail" time="{duration}">
|
||||
<failure message="7 > 7" type="CHECK">
|
||||
ConditionTests.cpp:<line number>
|
||||
|
|
@ -441,7 +439,6 @@ MessageTests.cpp:<line number>
|
|||
<testcase classname="<exe-name>.global" name="Parse test names and tags/quoted string followed by tag exclusion" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Parsing a std::pair" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Pointers can be compared to null" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Pointers can be converted to strings" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Process can be configured on command line/empty args don't cause a crash" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Process can be configured on command line/default - no arguments" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="Process can be configured on command line/test lists/1 test" time="{duration}"/>
|
||||
|
|
|
|||
|
|
@ -1023,19 +1023,6 @@
|
|||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="Comparing member function pointers" tags="[Tricky][member function pointer]" filename="projects/<exe-name>/TrickyTests.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="projects/<exe-name>/TrickyTests.cpp" >
|
||||
<Original>
|
||||
m == &S::f
|
||||
</Original>
|
||||
<Expanded>
|
||||
0x<hex digits>
|
||||
==
|
||||
0x<hex digits>
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="Comparison with explicitly convertible types" tags="[Approx]" filename="projects/<exe-name>/ApproxTests.cpp" >
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ApproxTests.cpp" >
|
||||
<Original>
|
||||
|
|
@ -1296,12 +1283,10 @@
|
|||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ConditionTests.cpp" >
|
||||
<Original>
|
||||
(std::numeric_limits<unsigned long>::max)() > ul
|
||||
(std::numeric_limits<uint32_t>::max)() > ul
|
||||
</Original>
|
||||
<Expanded>
|
||||
18446744073709551615 (0x<hex digits>)
|
||||
>
|
||||
4
|
||||
4294967295 (0x<hex digits>) > 4
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
|
|
@ -2388,17 +2373,6 @@
|
|||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="Operators at different namespace levels not hijacked by Koenig lookup" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" >
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" >
|
||||
<Original>
|
||||
0x<hex digits> == o
|
||||
</Original>
|
||||
<Expanded>
|
||||
3221225472 (0x<hex digits>) == {?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="Ordering comparison checks that should fail" tags="[.][failing]" filename="projects/<exe-name>/ConditionTests.cpp" >
|
||||
<Expression success="false" type="CHECK" filename="projects/<exe-name>/ConditionTests.cpp" >
|
||||
<Original>
|
||||
|
|
@ -3962,15 +3936,6 @@
|
|||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="Pointers can be converted to strings" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
|
||||
<Warning>
|
||||
actual address of p: 0x<hex digits>
|
||||
</Warning>
|
||||
<Warning>
|
||||
toString(p): 0x<hex digits>
|
||||
</Warning>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="Process can be configured on command line" tags="[command-line][config]" filename="projects/<exe-name>/TestMain.cpp" >
|
||||
<Section name="empty args don't cause a crash" filename="projects/<exe-name>/TestMain.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="projects/<exe-name>/TestMain.cpp" >
|
||||
|
|
@ -4824,7 +4789,7 @@ A string sent directly to stderr
|
|||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="Standard error is reported and redirected" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
|
||||
<TestCase name="Standard error is reported and redirected" tags="[.][.approvals][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
|
||||
<Section name="std::cerr" filename="projects/<exe-name>/MessageTests.cpp" >
|
||||
<OverallResults successes="0" failures="1" expectedFailures="0"/>
|
||||
</Section>
|
||||
|
|
@ -8276,7 +8241,7 @@ loose text artifact
|
|||
</Section>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<OverallResults successes="760" failures="103" expectedFailures="21"/>
|
||||
<OverallResults successes="758" failures="103" expectedFailures="21"/>
|
||||
</Group>
|
||||
<OverallResults successes="760" failures="102" expectedFailures="21"/>
|
||||
<OverallResults successes="758" failures="102" expectedFailures="21"/>
|
||||
</Catch>
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
|
||||
#include <string>
|
||||
#include <limits>
|
||||
#include <cstdint>
|
||||
|
||||
struct TestData {
|
||||
int int_seven = 7;
|
||||
|
|
@ -184,7 +185,7 @@ TEST_CASE( "Comparisons with int literals don't warn when mixing signed/ unsigne
|
|||
REQUIRE( 5 == c );
|
||||
REQUIRE( 6 == uc );
|
||||
|
||||
REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul );
|
||||
REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul );
|
||||
}
|
||||
|
||||
// Disable warnings about sign conversions for the next two tests
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ TEST_CASE( "Standard output from all sections is reported", "[messages][.]" ) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) {
|
||||
TEST_CASE( "Standard error is reported and redirected", "[messages][.][approvals]" ) {
|
||||
SECTION( "std::cerr" ) {
|
||||
std::cerr << "Write to std::cerr" << std::endl;
|
||||
}
|
||||
|
|
@ -98,7 +98,7 @@ TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) {
|
|||
std::cerr << ' ';
|
||||
std::clog << "writes";
|
||||
std::cerr << " to error";
|
||||
std::clog << " streams\n";
|
||||
std::clog << " streams" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -130,7 +130,7 @@ TEST_CASE( "sends information to INFO", "[.][failing]" ) {
|
|||
REQUIRE( false );
|
||||
}
|
||||
|
||||
TEST_CASE( "Pointers can be converted to strings", "[messages][.]" ) {
|
||||
TEST_CASE( "Pointers can be converted to strings", "[messages][.][approvals]" ) {
|
||||
int p;
|
||||
WARN( "actual address of p: " << &p );
|
||||
WARN( "toString(p): " << ::Catch::Detail::stringify( &p ) );
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ namespace ObjectWithConversions
|
|||
TEST_CASE
|
||||
(
|
||||
"Operators at different namespace levels not hijacked by Koenig lookup",
|
||||
"[Tricky]"
|
||||
"[Tricky][approvals]"
|
||||
)
|
||||
{
|
||||
Object o;
|
||||
|
|
@ -211,9 +211,16 @@ namespace ObjectWithNonConstEqualityOperator
|
|||
|
||||
namespace EnumBitFieldTests
|
||||
{
|
||||
enum Bits {bit0 = 0x0001, bit1 = 0x0002, bit2 = 0x0004, bit3 = 0x0008, bit1and2 = 0x0006,
|
||||
bit30 = 0x40000000, bit31 = 0x80000000,
|
||||
bit30and31 = 0xc0000000};
|
||||
enum Bits : uint32_t {
|
||||
bit0 = 0x0001,
|
||||
bit1 = 0x0002,
|
||||
bit2 = 0x0004,
|
||||
bit3 = 0x0008,
|
||||
bit1and2 = bit1 | bit2,
|
||||
bit30 = 0x40000000,
|
||||
bit31 = 0x80000000,
|
||||
bit30and31 = bit30 | bit31
|
||||
};
|
||||
|
||||
TEST_CASE( "Test enum bit values", "[Tricky]" )
|
||||
{
|
||||
|
|
@ -365,7 +372,7 @@ struct S
|
|||
};
|
||||
|
||||
|
||||
TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer]" )
|
||||
TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer][approvals]" )
|
||||
{
|
||||
typedef void (S::*MF)();
|
||||
MF m = &S::f;
|
||||
|
|
|
|||
|
|
@ -41,6 +41,8 @@ errnoParser = re.compile(r'''
|
|||
\(\*__errno_location\ \(\)\)
|
||||
|
|
||||
\(\*__error\(\)\)
|
||||
|
|
||||
\(\*_errno\(\)\)
|
||||
''', re.VERBOSE)
|
||||
|
||||
if len(sys.argv) == 2:
|
||||
|
|
@ -145,15 +147,15 @@ print("Running approvals against executable:")
|
|||
print(" " + cmdPath)
|
||||
|
||||
# Standard console reporter
|
||||
approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"])
|
||||
approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex"])
|
||||
# console reporter, include passes, warn about No Assertions
|
||||
approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"])
|
||||
approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex"])
|
||||
# console reporter, include passes, warn about No Assertions, limit failures to first 4
|
||||
approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
|
||||
approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
|
||||
# junit reporter, include passes, warn about No Assertions
|
||||
approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
|
||||
approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
|
||||
# xml reporter, include passes, warn about No Assertions
|
||||
approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
|
||||
approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
|
||||
|
||||
if overallResult != 0:
|
||||
print("If these differences are expected, run approve.py to approve new baselines.")
|
||||
|
|
|
|||
Loading…
Reference in New Issue