3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2024-11-22 13:26:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Approval tests can use Catch path fixed in env. var, and convert nullptr and __null to 0 for comparison

Browse Source
This commit is contained in:
Phil Nash 2017-01-09 14:12:12 +00:00
parent 37e1e24309
commit 976a655496
5 changed files with 158 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
projects/SelfTest/Baselines/console.sw.approved.txt
View File

@ -923,7 +923,7 @@ MiscTests.cpp:<line number>
MiscTests.cpp:<line number>: MiscTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE_THAT( "", Equals(nullptr) ) REQUIRE_THAT( "", Equals(0) )
with expansion: with expansion:
"" equals: "" "" equals: ""
@ -5446,9 +5446,9 @@ ConditionTests.cpp:<line number>
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( p == nullptr ) REQUIRE( p == 0 )
with expansion: with expansion:
NULL == nullptr NULL == 0
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
@ -5458,39 +5458,39 @@ with expansion:
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( p != nullptr ) REQUIRE( p != 0 )
with expansion: with expansion:
0x<hex digits> != nullptr 0x<hex digits> != 0
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( cp != nullptr ) REQUIRE( cp != 0 )
with expansion: with expansion:
0x<hex digits> != nullptr 0x<hex digits> != 0
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( cpc != nullptr ) REQUIRE( cpc != 0 )
with expansion: with expansion:
0x<hex digits> != nullptr 0x<hex digits> != 0
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( returnsNull() == nullptr ) REQUIRE( returnsNull() == 0 )
with expansion: with expansion:
{null string} == nullptr {null string} == 0
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( returnsConstNull() == nullptr ) REQUIRE( returnsConstNull() == 0 )
with expansion: with expansion:
{null string} == nullptr {null string} == 0
ConditionTests.cpp:<line number>: ConditionTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( nullptr != p ) REQUIRE( 0 != p )
with expansion: with expansion:
nullptr != 0x<hex digits> 0 != 0x<hex digits>
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Pointers can be converted to strings Pointers can be converted to strings
@ -7886,9 +7886,9 @@ TrickyTests.cpp:<line number>
TrickyTests.cpp:<line number>: TrickyTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( obj.prop != nullptr ) REQUIRE( obj.prop != 0 )
with expansion: with expansion:
0x<hex digits> != nullptr 0x<hex digits> != 0
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
checkedElse checkedElse
@ -8253,13 +8253,13 @@ MiscTests.cpp:<line number>
MiscTests.cpp:<line number>: MiscTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( makeString( false ) != static_cast<char*>(nullptr) ) REQUIRE( makeString( false ) != static_cast<char*>(0) )
with expansion: with expansion:
"valid string" != {null string} "valid string" != {null string}
MiscTests.cpp:<line number>: MiscTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( makeString( true ) == static_cast<char*>(nullptr) ) REQUIRE( makeString( true ) == static_cast<char*>(0) )
with expansion: with expansion:
{null string} == {null string} {null string} == {null string}
@ -8271,9 +8271,9 @@ TrickyTests.cpp:<line number>
TrickyTests.cpp:<line number>: TrickyTests.cpp:<line number>:
PASSED: PASSED:
REQUIRE( ptr.get() == nullptr ) REQUIRE( ptr.get() == 0 )
with expansion: with expansion:
NULL == nullptr NULL == 0
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
pair<pair<int,const char *,pair<std::string,int> > -> toString pair<pair<int,const char *,pair<std::string,int> > -> toString
@ -8797,18 +8797,18 @@ with expansion:
"{ 0 }" == "{ 0 }" "{ 0 }" == "{ 0 }"
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
tuple<nullptr,int,const char *> tuple<0,int,const char *>
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
ToStringTuple.cpp:<line number> ToStringTuple.cpp:<line number>
............................................................................... ...............................................................................
ToStringTuple.cpp:<line number>: ToStringTuple.cpp:<line number>:
PASSED: PASSED:
CHECK( "{ nullptr, 42, \"Catch me\" }" == Catch::toString(value) ) CHECK( "{ 0, 42, \"Catch me\" }" == Catch::toString(value) )
with expansion: with expansion:
"{ nullptr, 42, "Catch me" }" "{ 0, 42, "Catch me" }"
== ==
"{ nullptr, 42, "Catch me" }" "{ 0, 42, "Catch me" }"
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
tuple<string,string> tuple<string,string>

2
projects/SelfTest/Baselines/junit.sw.approved.txt
View File

@ -661,7 +661,7 @@ TrickyTests.cpp:<line number>
<testcase classname="global" name="tuple&lt;>" time="{duration}"/> <testcase classname="global" name="tuple&lt;>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;float,int>" time="{duration}"/> <testcase classname="global" name="tuple&lt;float,int>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;int>" time="{duration}"/> <testcase classname="global" name="tuple&lt;int>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;nullptr,int,const char *>" time="{duration}"/> <testcase classname="global" name="tuple&lt;0,int,const char *>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;string,string>" time="{duration}"/> <testcase classname="global" name="tuple&lt;string,string>" time="{duration}"/>
<testcase classname="global" name="tuple&lt;tuple&lt;int>,tuple&lt;>,float>" time="{duration}"/> <testcase classname="global" name="tuple&lt;tuple&lt;int>,tuple&lt;>,float>" time="{duration}"/>
<testcase classname="global" name="vec&lt;vec&lt;string,alloc>> -> toString" time="{duration}"/> <testcase classname="global" name="vec&lt;vec&lt;string,alloc>> -> toString" time="{duration}"/>

50
projects/SelfTest/Baselines/xml.sw.approved.txt
View File

@ -1015,7 +1015,7 @@
<TestCase name="Equals string matcher, with NULL"> <TestCase name="Equals string matcher, with NULL">
<Expression success="true" type="REQUIRE_THAT" filename="projects/SelfTest/MiscTests.cpp" > <Expression success="true" type="REQUIRE_THAT" filename="projects/SelfTest/MiscTests.cpp" >
<Original> <Original>
"", Equals(nullptr) "", Equals(0)
</Original> </Original>
<Expanded> <Expanded>
"" equals: "" "" equals: ""
@ -5709,10 +5709,10 @@ four"
<TestCase name="Pointers can be compared to null"> <TestCase name="Pointers can be compared to null">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original> <Original>
p == nullptr p == 0
</Original> </Original>
<Expanded> <Expanded>
NULL == nullptr NULL == 0
</Expanded> </Expanded>
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
@ -5725,50 +5725,50 @@ four"
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original> <Original>
p != nullptr p != 0
</Original> </Original>
<Expanded> <Expanded>
0x<hex digits> != nullptr 0x<hex digits> != 0
</Expanded> </Expanded>
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original> <Original>
cp != nullptr cp != 0
</Original> </Original>
<Expanded> <Expanded>
0x<hex digits> != nullptr 0x<hex digits> != 0
</Expanded> </Expanded>
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original> <Original>
cpc != nullptr cpc != 0
</Original> </Original>
<Expanded> <Expanded>
0x<hex digits> != nullptr 0x<hex digits> != 0
</Expanded> </Expanded>
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original> <Original>
returnsNull() == nullptr returnsNull() == 0
</Original> </Original>
<Expanded> <Expanded>
{null string} == nullptr {null string} == 0
</Expanded> </Expanded>
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original> <Original>
returnsConstNull() == nullptr returnsConstNull() == 0
</Original> </Original>
<Expanded> <Expanded>
{null string} == nullptr {null string} == 0
</Expanded> </Expanded>
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
<Original> <Original>
nullptr != p 0 != p
</Original> </Original>
<Expanded> <Expanded>
nullptr != 0x<hex digits> 0 != 0x<hex digits>
</Expanded> </Expanded>
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
@ -8318,10 +8318,10 @@ there"
<TestCase name="boolean member"> <TestCase name="boolean member">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" >
<Original> <Original>
obj.prop != nullptr obj.prop != 0
</Original> </Original>
<Expanded> <Expanded>
0x<hex digits> != nullptr 0x<hex digits> != 0
</Expanded> </Expanded>
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
@ -8709,7 +8709,7 @@ there"
<TestCase name="null strings"> <TestCase name="null strings">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" >
<Original> <Original>
makeString( false ) != static_cast&lt;char*>(nullptr) makeString( false ) != static_cast&lt;char*>(0)
</Original> </Original>
<Expanded> <Expanded>
"valid string" != {null string} "valid string" != {null string}
@ -8717,7 +8717,7 @@ there"
</Expression> </Expression>
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" >
<Original> <Original>
makeString( true ) == static_cast&lt;char*>(nullptr) makeString( true ) == static_cast&lt;char*>(0)
</Original> </Original>
<Expanded> <Expanded>
{null string} == {null string} {null string} == {null string}
@ -8728,10 +8728,10 @@ there"
<TestCase name="null_ptr"> <TestCase name="null_ptr">
<Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" >
<Original> <Original>
ptr.get() == nullptr ptr.get() == 0
</Original> </Original>
<Expanded> <Expanded>
NULL == nullptr NULL == 0
</Expanded> </Expanded>
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
@ -9264,15 +9264,15 @@ there"
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>
</TestCase> </TestCase>
<TestCase name="tuple&lt;nullptr,int,const char *>"> <TestCase name="tuple&lt;0,int,const char *>">
<Expression success="true" type="CHECK" filename="projects/SelfTest/ToStringTuple.cpp" > <Expression success="true" type="CHECK" filename="projects/SelfTest/ToStringTuple.cpp" >
<Original> <Original>
"{ nullptr, 42, \"Catch me\" }" == Catch::toString(value) "{ 0, 42, \"Catch me\" }" == Catch::toString(value)
</Original> </Original>
<Expanded> <Expanded>
"{ nullptr, 42, "Catch me" }" "{ 0, 42, "Catch me" }"
== ==
"{ nullptr, 42, "Catch me" }" "{ 0, 42, "Catch me" }"
</Expanded> </Expanded>
</Expression> </Expression>
<OverallResult success="true"/> <OverallResult success="true"/>

189
scripts/approvalTests.py
View File

@ -1,116 +1,129 @@
from __future__ import print_function from __future__ import print_function
import os import os
import sys import sys
import subprocess import subprocess
import re import re
import scriptCommon
from scriptCommon import catchPath from scriptCommon import catchPath
rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' ) rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' ) filenameParser = re.compile(r'(.*)/(.*\..pp:)(.*)')
filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' ) filelineParser = re.compile(r'(.*\..pp:)([0-9]*)(.*)')
pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' ) pathParser = re.compile(r'(.*?)/(.*\..pp)(.*)')
lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' ) lineNumberParser = re.compile(r'(.*)line="[0-9]*"(.*)')
hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' ) hexParser = re.compile(r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)')
durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' ) durationsParser = re.compile(r'(.*)time="[0-9]*\.[0-9]*"(.*)')
versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' ) versionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)')
devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' ) devVersionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)')
nullParser = re.compile(r'(.*?)\b(__null|nullptr)\b(.*)')
if len(sys.argv) == 2: if len(sys.argv) == 2:
cmdPath = sys.argv[1] cmdPath = sys.argv[1]
else: else:
cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' ) cmdPath = scriptCommon.getBuildExecutable()
# cmdPath = os.path.join( catchPath, 'projects/CMake/cmake-build-debug/SelfTest' )
if not cmdPath.startswith("/"):
cmdPath = os.path.join(catchPath, cmdPath)
overallResult = 0 overallResult = 0
def filterLine( line ): def filterLine(line):
m = filenameParser.match( line ) m = filenameParser.match(line)
if m: if m:
line = m.group(2) + m.group(3) line = m.group(2) + m.group(3)
m2 = filelineParser.match( line ) m2 = filelineParser.match(line)
if m2: if m2:
line = m2.group(1) + "<line number>" + m2.group(3) line = m2.group(1) + "<line number>" + m2.group(3)
else: else:
m2 = lineNumberParser.match( line ) m2 = lineNumberParser.match(line)
if m2: if m2:
line = m2.group(1) + m2.group(2) line = m2.group(1) + m2.group(2)
m = pathParser.match( line ) m = pathParser.match(line)
if m: if m:
path = "/" + m.group(2) path = "/" + m.group(2)
if path.startswith( catchPath ): if path.startswith(catchPath):
path = path[1+len(catchPath):] path = path[1 + len(catchPath):]
line = m.group(1) + path + m.group(3) line = m.group(1) + path + m.group(3)
m = devVersionParser.match( line ) m = devVersionParser.match(line)
if m: if m:
line = m.group(1) + "<version>" + m.group(2) line = m.group(1) + "<version>" + m.group(2)
else: else:
m = versionParser.match( line ) m = versionParser.match(line)
if m: if m:
line = m.group(1) + "<version>" + m.group(2) line = m.group(1) + "<version>" + m.group(2)
while True: m = nullParser.match(line)
m = hexParser.match( line ) if m:
if m: line = m.group(1) + "0" + m.group(3)
line = m.group(1) + "0x<hex digits>" + m.group(3)
else:
break
m = durationsParser.match( line )
if m:
line = m.group(1) + 'time="{duration}"' + m.group(2)
return line
def approve( baseName, args ): while True:
global overallResult m = hexParser.match(line)
args[0:0] = [cmdPath] if m:
if not os.path.exists( cmdPath ): line = m.group(1) + "0x<hex digits>" + m.group(3)
raise Exception( "Executable doesn't exist at " + cmdPath ) else:
baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) ) break
rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) m = durationsParser.match(line)
filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) ) if m:
line = m.group(1) + 'time="{duration}"' + m.group(2)
return line
f = open( rawResultsPath, 'w' )
subprocess.call( args, stdout=f, stderr=f )
f.close()
rawFile = open( rawResultsPath, 'r' ) def approve(baseName, args):
filteredFile = open( filteredResultsPath, 'w' ) global overallResult
for line in rawFile: args[0:0] = [cmdPath]
filteredFile.write( filterLine( line ).rstrip() + "\n" ) if not os.path.exists(cmdPath):
filteredFile.close() raise Exception("Executable doesn't exist at " + cmdPath)
rawFile.close() baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
os.remove( rawResultsPath ) f = open(rawResultsPath, 'w')
print() subprocess.call(args, stdout=f, stderr=f)
print( baseName + ":" ) f.close()
if os.path.exists( baselinesPath ):
diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] ) rawFile = open(rawResultsPath, 'r')
if diffResult == 0: filteredFile = open(filteredResultsPath, 'w')
os.remove( filteredResultsPath ) for line in rawFile:
print( " \033[92mResults matched" ) filteredFile.write(filterLine(line).rstrip() + "\n")
else: filteredFile.close()
print( " \n****************************\n \033[91mResults differed" ) rawFile.close()
if diffResult > overallResult:
overallResult = diffResult os.remove(rawResultsPath)
print( "\033[0m" ) print()
else: print(baseName + ":")
print( " first approval" ) if os.path.exists(baselinesPath):
if overallResult == 0: diffResult = subprocess.call(["diff", baselinesPath, filteredResultsPath])
overallResult = 1 if diffResult == 0:
os.remove(filteredResultsPath)
print(" \033[92mResults matched")
else:
print(" \n****************************\n \033[91mResults differed")
if diffResult > overallResult:
overallResult = diffResult
print("\033[0m")
else:
print(" first approval")
if overallResult == 0:
overallResult = 1
print("Running approvals against executable:")
print(" " + cmdPath)
# Standard console reporter # Standard console reporter
approve( "console.std", ["~_", "--order", "lex"] ) approve("console.std", ["~_", "--order", "lex"])
# console reporter, include passes, warn about No Assertions # console reporter, include passes, warn about No Assertions
approve( "console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"] ) approve("console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"])
# console reporter, include passes, warn about No Assertions, limit failures to first 4 # console reporter, include passes, warn about No Assertions, limit failures to first 4
approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"] ) approve("console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
# junit reporter, include passes, warn about No Assertions # junit reporter, include passes, warn about No Assertions
approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"] ) approve("junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
# xml reporter, include passes, warn about No Assertions # xml reporter, include passes, warn about No Assertions
approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"] ) approve("xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
if overallResult != 0: if overallResult != 0:
print( "run approve.py to approve new baselines" ) print("If these differenecs are expected run approve.py to approve new baselines")
exit( overallResult) exit(overallResult)

7
scripts/scriptCommon.py
View File

@ -2,8 +2,13 @@ import os
import sys import sys
import subprocess import subprocess
catchPath = os.path.dirname(os.path.realpath( os.path.dirname(sys.argv[0]))) catchPath = os.path.dirname(os.path.realpath( os.path.dirname(sys.argv[0])))
def getBuildExecutable():
dir = os.environ.get('CATCH_DEV_OUT_DIR', "cmake-build-debug/SelfTest")
return dir
def runAndCapture( args ): def runAndCapture( args ):
child = subprocess.Popen(" ".join( args ), shell=True, stdout=subprocess.PIPE) child = subprocess.Popen(" ".join( args ), shell=True, stdout=subprocess.PIPE)
lines = [] lines = []
@ -18,4 +23,4 @@ def runAndCapture( args ):
line = "" line = ""
else: else:
line = line + out line = line + out
return lines return lines