3rd-party
/
catch2
mirror of https://github.com/catchorg/Catch2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Approval tests can use Catch path fixed in env. var, and convert nullptr and __null to 0 for comparison

This commit is contained in:
Phil Nash 2017-01-09 14:12:12 +00:00
parent 37e1e24309
commit 976a655496
5 changed files with 158 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
projects/SelfTest/Baselines/console.sw.approved.txt
View File

@ -923,7 +923,7 @@ MiscTests.cpp:<line number>
MiscTests.cpp:<line number>:
PASSED:
REQUIRE_THAT( "", Equals(nullptr) )
REQUIRE_THAT( "", Equals(0) )
with expansion:
"" equals: ""
@ -5446,9 +5446,9 @@ ConditionTests.cpp:<line number>
ConditionTests.cpp:<line number>:
PASSED:
REQUIRE( p == nullptr )
REQUIRE( p == 0 )
with expansion:
NULL == nullptr
NULL == 0
ConditionTests.cpp:<line number>:
PASSED:
@ -5458,39 +5458,39 @@ with expansion:
ConditionTests.cpp:<line number>:
PASSED:
REQUIRE( p != nullptr )
REQUIRE( p != 0 )
with expansion:
0x<hex digits> != nullptr
0x<hex digits> != 0
ConditionTests.cpp:<line number>:
PASSED:
REQUIRE( cp != nullptr )
REQUIRE( cp != 0 )
with expansion:
0x<hex digits> != nullptr
0x<hex digits> != 0
ConditionTests.cpp:<line number>:
PASSED:
REQUIRE( cpc != nullptr )
REQUIRE( cpc != 0 )
with expansion:
0x<hex digits> != nullptr
0x<hex digits> != 0
ConditionTests.cpp:<line number>:
PASSED:
REQUIRE( returnsNull() == nullptr )
REQUIRE( returnsNull() == 0 )
with expansion:
{null string} == nullptr
{null string} == 0
ConditionTests.cpp:<line number>:
PASSED:
REQUIRE( returnsConstNull() == nullptr )
REQUIRE( returnsConstNull() == 0 )
with expansion:
{null string} == nullptr
{null string} == 0
ConditionTests.cpp:<line number>:
PASSED:
REQUIRE( nullptr != p )
REQUIRE( 0 != p )
with expansion:
nullptr != 0x<hex digits>
0 != 0x<hex digits>
-------------------------------------------------------------------------------
Pointers can be converted to strings
@ -7886,9 +7886,9 @@ TrickyTests.cpp:<line number>
TrickyTests.cpp:<line number>:
PASSED:
REQUIRE( obj.prop != nullptr )
REQUIRE( obj.prop != 0 )
with expansion:
0x<hex digits> != nullptr
0x<hex digits> != 0
-------------------------------------------------------------------------------
checkedElse
@ -8253,13 +8253,13 @@ MiscTests.cpp:<line number>
MiscTests.cpp:<line number>:
PASSED:
REQUIRE( makeString( false ) != static_cast<char*>(nullptr) )
REQUIRE( makeString( false ) != static_cast<char*>(0) )
with expansion:
"valid string" != {null string}
MiscTests.cpp:<line number>:
PASSED:
REQUIRE( makeString( true ) == static_cast<char*>(nullptr) )
REQUIRE( makeString( true ) == static_cast<char*>(0) )
with expansion:
{null string} == {null string}
@ -8271,9 +8271,9 @@ TrickyTests.cpp:<line number>
TrickyTests.cpp:<line number>:
PASSED:
REQUIRE( ptr.get() == nullptr )
REQUIRE( ptr.get() == 0 )
with expansion:
NULL == nullptr
NULL == 0
-------------------------------------------------------------------------------
pair<pair<int,const char *,pair<std::string,int> > -> toString
@ -8797,18 +8797,18 @@ with expansion:
"{ 0 }" == "{ 0 }"
-------------------------------------------------------------------------------
tuple<nullptr,int,const char *>
tuple<0,int,const char *>
-------------------------------------------------------------------------------
ToStringTuple.cpp:<line number>
...............................................................................
ToStringTuple.cpp:<line number>:
PASSED:
CHECK( "{ nullptr, 42, \"Catch me\" }" == Catch::toString(value) )
CHECK( "{ 0, 42, \"Catch me\" }" == Catch::toString(value) )
with expansion:
"{ nullptr, 42, "Catch me" }"
"{ 0, 42, "Catch me" }"
==
"{ nullptr, 42, "Catch me" }"
"{ 0, 42, "Catch me" }"
-------------------------------------------------------------------------------
tuple<string,string>

2
projects/SelfTest/Baselines/junit.sw.approved.txt
View File

@ -661,7 +661,7 @@ TrickyTests.cpp:<line number>
<testcase classname="global" name="tuple&lt;>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;float,int>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;int>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;nullptr,int,const char *>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;0,int,const char *>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;string,string>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;tuple&lt;int>,tuple&lt;>,float>" time="{duration}"/>
<testcase classname="global" name="vec&lt;vec&lt;string,alloc>> -> toString" time="{duration}"/>

50
projects/SelfTest/Baselines/xml.sw.approved.txt
View File

@ -1015,7 +1015,7 @@
<TestCase name="Equals string matcher, with NULL">
<Expression success="true" type="REQUIRE_THAT" filename="projects/SelfTest/MiscTests.cpp" >
<Original>
"", Equals(nullptr)
"", Equals(0)
</Original>
<Expanded>
"" equals: ""
@ -5709,10 +5709,10 @@ four"
<TestCase name="Pointers can be compared to null">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original>
p == nullptr
p == 0
</Original>
<Expanded>
NULL == nullptr
NULL == 0
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
@ -5725,50 +5725,50 @@ four"
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original>
p != nullptr
p != 0
</Original>
<Expanded>
0x<hex digits> != nullptr
0x<hex digits> != 0
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original>
cp != nullptr
cp != 0
</Original>
<Expanded>
0x<hex digits> != nullptr
0x<hex digits> != 0
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original>
cpc != nullptr
cpc != 0
</Original>
<Expanded>
0x<hex digits> != nullptr
0x<hex digits> != 0
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original>
returnsNull() == nullptr
returnsNull() == 0
</Original>
<Expanded>
{null string} == nullptr
{null string} == 0
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original>
returnsConstNull() == nullptr
returnsConstNull() == 0
</Original>
<Expanded>
{null string} == nullptr
{null string} == 0
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original>
nullptr != p
0 != p
</Original>
<Expanded>
nullptr != 0x<hex digits>
0 != 0x<hex digits>
</Expanded>
</Expression>
<OverallResult success="true"/>
@ -8318,10 +8318,10 @@ there"
<TestCase name="boolean member">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" >
<Original>
obj.prop != nullptr
obj.prop != 0
</Original>
<Expanded>
0x<hex digits> != nullptr
0x<hex digits> != 0
</Expanded>
</Expression>
<OverallResult success="true"/>
@ -8709,7 +8709,7 @@ there"
<TestCase name="null strings">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" >
<Original>
makeString( false ) != static_cast&lt;char*>(nullptr)
makeString( false ) != static_cast&lt;char*>(0)
</Original>
<Expanded>
"valid string" != {null string}
@ -8717,7 +8717,7 @@ there"
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" >
<Original>
makeString( true ) == static_cast&lt;char*>(nullptr)
makeString( true ) == static_cast&lt;char*>(0)
</Original>
<Expanded>
{null string} == {null string}
@ -8728,10 +8728,10 @@ there"
<TestCase name="null_ptr">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" >
<Original>
ptr.get() == nullptr
ptr.get() == 0
</Original>
<Expanded>
NULL == nullptr
NULL == 0
</Expanded>
</Expression>
<OverallResult success="true"/>
@ -9264,15 +9264,15 @@ there"
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="tuple&lt;nullptr,int,const char *>">
<TestCase name="tuple&lt;0,int,const char *>">
<Expression success="true" type="CHECK" filename="projects/SelfTest/ToStringTuple.cpp" >
<Original>
"{ nullptr, 42, \"Catch me\" }" == Catch::toString(value)
"{ 0, 42, \"Catch me\" }" == Catch::toString(value)
</Original>
<Expanded>
"{ nullptr, 42, "Catch me" }"
"{ 0, 42, "Catch me" }"
==
"{ nullptr, 42, "Catch me" }"
"{ 0, 42, "Catch me" }"
</Expanded>
</Expression>
<OverallResult success="true"/>

189
scripts/approvalTests.py
View File

@ -1,116 +1,129 @@
from __future__ import print_function
from __future__ import print_function
import os
import sys
import subprocess
import re
import scriptCommon
from scriptCommon import catchPath
rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' )
rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' )
filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' )
pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' )
lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' )
hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' )
durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' )
versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' )
devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' )
filenameParser = re.compile(r'(.*)/(.*\..pp:)(.*)')
filelineParser = re.compile(r'(.*\..pp:)([0-9]*)(.*)')
pathParser = re.compile(r'(.*?)/(.*\..pp)(.*)')
lineNumberParser = re.compile(r'(.*)line="[0-9]*"(.*)')
hexParser = re.compile(r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)')
durationsParser = re.compile(r'(.*)time="[0-9]*\.[0-9]*"(.*)')
versionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)')
devVersionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)')
nullParser = re.compile(r'(.*?)\b(__null|nullptr)\b(.*)')
if len(sys.argv) == 2:
cmdPath = sys.argv[1]
cmdPath = sys.argv[1]
else:
cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
# cmdPath = os.path.join( catchPath, 'projects/CMake/cmake-build-debug/SelfTest' )
cmdPath = scriptCommon.getBuildExecutable()
if not cmdPath.startswith("/"):
cmdPath = os.path.join(catchPath, cmdPath)
overallResult = 0
def filterLine( line ):
m = filenameParser.match( line )
if m:
line = m.group(2) + m.group(3)
m2 = filelineParser.match( line )
if m2:
line = m2.group(1) + "<line number>" + m2.group(3)
else:
m2 = lineNumberParser.match( line )
if m2:
line = m2.group(1) + m2.group(2)
m = pathParser.match( line )
if m:
path = "/" + m.group(2)
if path.startswith( catchPath ):
path = path[1+len(catchPath):]
line = m.group(1) + path + m.group(3)
m = devVersionParser.match( line )
if m:
line = m.group(1) + "<version>" + m.group(2)
else:
m = versionParser.match( line )
if m:
line = m.group(1) + "<version>" + m.group(2)
def filterLine(line):
m = filenameParser.match(line)
if m:
line = m.group(2) + m.group(3)
m2 = filelineParser.match(line)
if m2:
line = m2.group(1) + "<line number>" + m2.group(3)
else:
m2 = lineNumberParser.match(line)
if m2:
line = m2.group(1) + m2.group(2)
m = pathParser.match(line)
if m:
path = "/" + m.group(2)
if path.startswith(catchPath):
path = path[1 + len(catchPath):]
line = m.group(1) + path + m.group(3)
m = devVersionParser.match(line)
if m:
line = m.group(1) + "<version>" + m.group(2)
else:
m = versionParser.match(line)
if m:
line = m.group(1) + "<version>" + m.group(2)
while True:
m = hexParser.match( line )
if m:
line = m.group(1) + "0x<hex digits>" + m.group(3)
else:
break
m = durationsParser.match( line )
if m:
line = m.group(1) + 'time="{duration}"' + m.group(2)
return line
m = nullParser.match(line)
if m:
line = m.group(1) + "0" + m.group(3)
def approve( baseName, args ):
global overallResult
args[0:0] = [cmdPath]
if not os.path.exists( cmdPath ):
raise Exception( "Executable doesn't exist at " + cmdPath )
baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) )
rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) )
filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) )
while True:
m = hexParser.match(line)
if m:
line = m.group(1) + "0x<hex digits>" + m.group(3)
else:
break
m = durationsParser.match(line)
if m:
line = m.group(1) + 'time="{duration}"' + m.group(2)
return line
f = open( rawResultsPath, 'w' )
subprocess.call( args, stdout=f, stderr=f )
f.close()
rawFile = open( rawResultsPath, 'r' )
filteredFile = open( filteredResultsPath, 'w' )
for line in rawFile:
filteredFile.write( filterLine( line ).rstrip() + "\n" )
filteredFile.close()
rawFile.close()
def approve(baseName, args):
global overallResult
args[0:0] = [cmdPath]
if not os.path.exists(cmdPath):
raise Exception("Executable doesn't exist at " + cmdPath)
baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
os.remove( rawResultsPath )
print()
print( baseName + ":" )
if os.path.exists( baselinesPath ):
diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] )
if diffResult == 0:
os.remove( filteredResultsPath )
print( " \033[92mResults matched" )
else:
print( " \n****************************\n \033[91mResults differed" )
if diffResult > overallResult:
overallResult = diffResult
print( "\033[0m" )
else:
print( " first approval" )
if overallResult == 0:
overallResult = 1
f = open(rawResultsPath, 'w')
subprocess.call(args, stdout=f, stderr=f)
f.close()
rawFile = open(rawResultsPath, 'r')
filteredFile = open(filteredResultsPath, 'w')
for line in rawFile:
filteredFile.write(filterLine(line).rstrip() + "\n")
filteredFile.close()
rawFile.close()
os.remove(rawResultsPath)
print()
print(baseName + ":")
if os.path.exists(baselinesPath):
diffResult = subprocess.call(["diff", baselinesPath, filteredResultsPath])
if diffResult == 0:
os.remove(filteredResultsPath)
print(" \033[92mResults matched")
else:
print(" \n****************************\n \033[91mResults differed")
if diffResult > overallResult:
overallResult = diffResult
print("\033[0m")
else:
print(" first approval")
if overallResult == 0:
overallResult = 1
print("Running approvals against executable:")
print(" " + cmdPath)
# Standard console reporter
approve( "console.std", ["~_", "--order", "lex"] )
approve("console.std", ["~_", "--order", "lex"])
# console reporter, include passes, warn about No Assertions
approve( "console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"] )
approve("console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"])
# console reporter, include passes, warn about No Assertions, limit failures to first 4
approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"] )
approve("console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
# junit reporter, include passes, warn about No Assertions
approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"] )
approve("junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
# xml reporter, include passes, warn about No Assertions
approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"] )
approve("xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
if overallResult != 0:
print( "run approve.py to approve new baselines" )
exit( overallResult)
print("If these differenecs are expected run approve.py to approve new baselines")
exit(overallResult)

5
scripts/scriptCommon.py
View File

@ -2,8 +2,13 @@ import os
import sys
import subprocess
catchPath = os.path.dirname(os.path.realpath( os.path.dirname(sys.argv[0])))
def getBuildExecutable():
dir = os.environ.get('CATCH_DEV_OUT_DIR', "cmake-build-debug/SelfTest")
return dir
def runAndCapture( args ):
child = subprocess.Popen(" ".join( args ), shell=True, stdout=subprocess.PIPE)
lines = []