mirror of
https://github.com/catchorg/Catch2.git
synced 2025-10-31 12:17:11 +01:00
Approval tests can use Catch path fixed in env. var, and convert nullptr and __null to 0 for comparison
This commit is contained in:
Phil Nash
@@ -1,116 +1,129 @@
|
||||
from __future__ import print_function
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import re
|
||||
|
||||
import scriptCommon
|
||||
from scriptCommon import catchPath
|
||||
|
||||
rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' )
|
||||
rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
|
||||
|
||||
filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' )
|
||||
filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' )
|
||||
pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' )
|
||||
lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' )
|
||||
hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' )
|
||||
durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' )
|
||||
versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' )
|
||||
devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' )
|
||||
filenameParser = re.compile(r'(.*)/(.*\..pp:)(.*)')
|
||||
filelineParser = re.compile(r'(.*\..pp:)([0-9]*)(.*)')
|
||||
pathParser = re.compile(r'(.*?)/(.*\..pp)(.*)')
|
||||
lineNumberParser = re.compile(r'(.*)line="[0-9]*"(.*)')
|
||||
hexParser = re.compile(r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)')
|
||||
durationsParser = re.compile(r'(.*)time="[0-9]*\.[0-9]*"(.*)')
|
||||
versionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)')
|
||||
devVersionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)')
|
||||
nullParser = re.compile(r'(.*?)\b(__null|nullptr)\b(.*)')
|
||||
|
||||
if len(sys.argv) == 2:
|
||||
cmdPath = sys.argv[1]
|
||||
cmdPath = sys.argv[1]
|
||||
else:
|
||||
cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
|
||||
# cmdPath = os.path.join( catchPath, 'projects/CMake/cmake-build-debug/SelfTest' )
|
||||
cmdPath = scriptCommon.getBuildExecutable()
|
||||
|
||||
if not cmdPath.startswith("/"):
|
||||
cmdPath = os.path.join(catchPath, cmdPath)
|
||||
|
||||
overallResult = 0
|
||||
|
||||
def filterLine( line ):
|
||||
m = filenameParser.match( line )
|
||||
if m:
|
||||
line = m.group(2) + m.group(3)
|
||||
m2 = filelineParser.match( line )
|
||||
if m2:
|
||||
line = m2.group(1) + "<line number>" + m2.group(3)
|
||||
else:
|
||||
m2 = lineNumberParser.match( line )
|
||||
if m2:
|
||||
line = m2.group(1) + m2.group(2)
|
||||
m = pathParser.match( line )
|
||||
if m:
|
||||
path = "/" + m.group(2)
|
||||
if path.startswith( catchPath ):
|
||||
path = path[1+len(catchPath):]
|
||||
line = m.group(1) + path + m.group(3)
|
||||
m = devVersionParser.match( line )
|
||||
if m:
|
||||
line = m.group(1) + "<version>" + m.group(2)
|
||||
else:
|
||||
m = versionParser.match( line )
|
||||
if m:
|
||||
line = m.group(1) + "<version>" + m.group(2)
|
||||
def filterLine(line):
|
||||
m = filenameParser.match(line)
|
||||
if m:
|
||||
line = m.group(2) + m.group(3)
|
||||
m2 = filelineParser.match(line)
|
||||
if m2:
|
||||
line = m2.group(1) + "<line number>" + m2.group(3)
|
||||
else:
|
||||
m2 = lineNumberParser.match(line)
|
||||
if m2:
|
||||
line = m2.group(1) + m2.group(2)
|
||||
m = pathParser.match(line)
|
||||
if m:
|
||||
path = "/" + m.group(2)
|
||||
if path.startswith(catchPath):
|
||||
path = path[1 + len(catchPath):]
|
||||
line = m.group(1) + path + m.group(3)
|
||||
m = devVersionParser.match(line)
|
||||
if m:
|
||||
line = m.group(1) + "<version>" + m.group(2)
|
||||
else:
|
||||
m = versionParser.match(line)
|
||||
if m:
|
||||
line = m.group(1) + "<version>" + m.group(2)
|
||||
|
||||
while True:
|
||||
m = hexParser.match( line )
|
||||
if m:
|
||||
line = m.group(1) + "0x<hex digits>" + m.group(3)
|
||||
else:
|
||||
break
|
||||
m = durationsParser.match( line )
|
||||
if m:
|
||||
line = m.group(1) + 'time="{duration}"' + m.group(2)
|
||||
return line
|
||||
m = nullParser.match(line)
|
||||
if m:
|
||||
line = m.group(1) + "0" + m.group(3)
|
||||
|
||||
def approve( baseName, args ):
|
||||
global overallResult
|
||||
args[0:0] = [cmdPath]
|
||||
if not os.path.exists( cmdPath ):
|
||||
raise Exception( "Executable doesn't exist at " + cmdPath )
|
||||
baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) )
|
||||
rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) )
|
||||
filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) )
|
||||
while True:
|
||||
m = hexParser.match(line)
|
||||
if m:
|
||||
line = m.group(1) + "0x<hex digits>" + m.group(3)
|
||||
else:
|
||||
break
|
||||
m = durationsParser.match(line)
|
||||
if m:
|
||||
line = m.group(1) + 'time="{duration}"' + m.group(2)
|
||||
return line
|
||||
|
||||
f = open( rawResultsPath, 'w' )
|
||||
subprocess.call( args, stdout=f, stderr=f )
|
||||
f.close()
|
||||
|
||||
rawFile = open( rawResultsPath, 'r' )
|
||||
filteredFile = open( filteredResultsPath, 'w' )
|
||||
for line in rawFile:
|
||||
filteredFile.write( filterLine( line ).rstrip() + "\n" )
|
||||
filteredFile.close()
|
||||
rawFile.close()
|
||||
def approve(baseName, args):
|
||||
global overallResult
|
||||
args[0:0] = [cmdPath]
|
||||
if not os.path.exists(cmdPath):
|
||||
raise Exception("Executable doesn't exist at " + cmdPath)
|
||||
baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
|
||||
rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
|
||||
filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
|
||||
|
||||
os.remove( rawResultsPath )
|
||||
print()
|
||||
print( baseName + ":" )
|
||||
if os.path.exists( baselinesPath ):
|
||||
diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] )
|
||||
if diffResult == 0:
|
||||
os.remove( filteredResultsPath )
|
||||
print( " \033[92mResults matched" )
|
||||
else:
|
||||
print( " \n****************************\n \033[91mResults differed" )
|
||||
if diffResult > overallResult:
|
||||
overallResult = diffResult
|
||||
print( "\033[0m" )
|
||||
else:
|
||||
print( " first approval" )
|
||||
if overallResult == 0:
|
||||
overallResult = 1
|
||||
f = open(rawResultsPath, 'w')
|
||||
subprocess.call(args, stdout=f, stderr=f)
|
||||
f.close()
|
||||
|
||||
rawFile = open(rawResultsPath, 'r')
|
||||
filteredFile = open(filteredResultsPath, 'w')
|
||||
for line in rawFile:
|
||||
filteredFile.write(filterLine(line).rstrip() + "\n")
|
||||
filteredFile.close()
|
||||
rawFile.close()
|
||||
|
||||
os.remove(rawResultsPath)
|
||||
print()
|
||||
print(baseName + ":")
|
||||
if os.path.exists(baselinesPath):
|
||||
diffResult = subprocess.call(["diff", baselinesPath, filteredResultsPath])
|
||||
if diffResult == 0:
|
||||
os.remove(filteredResultsPath)
|
||||
print(" \033[92mResults matched")
|
||||
else:
|
||||
print(" \n****************************\n \033[91mResults differed")
|
||||
if diffResult > overallResult:
|
||||
overallResult = diffResult
|
||||
print("\033[0m")
|
||||
else:
|
||||
print(" first approval")
|
||||
if overallResult == 0:
|
||||
overallResult = 1
|
||||
|
||||
|
||||
print("Running approvals against executable:")
|
||||
print(" " + cmdPath)
|
||||
|
||||
# Standard console reporter
|
||||
approve( "console.std", ["~_", "--order", "lex"] )
|
||||
approve("console.std", ["~_", "--order", "lex"])
|
||||
# console reporter, include passes, warn about No Assertions
|
||||
approve( "console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"] )
|
||||
approve("console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"])
|
||||
# console reporter, include passes, warn about No Assertions, limit failures to first 4
|
||||
approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"] )
|
||||
approve("console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
|
||||
# junit reporter, include passes, warn about No Assertions
|
||||
approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"] )
|
||||
approve("junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
|
||||
# xml reporter, include passes, warn about No Assertions
|
||||
approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"] )
|
||||
approve("xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
|
||||
|
||||
if overallResult != 0:
|
||||
print( "run approve.py to approve new baselines" )
|
||||
exit( overallResult)
|
||||
print("If these differenecs are expected run approve.py to approve new baselines")
|
||||
exit(overallResult)
|
||||
|
||||
@@ -2,8 +2,13 @@ import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
|
||||
catchPath = os.path.dirname(os.path.realpath( os.path.dirname(sys.argv[0])))
|
||||
|
||||
def getBuildExecutable():
|
||||
dir = os.environ.get('CATCH_DEV_OUT_DIR', "cmake-build-debug/SelfTest")
|
||||
return dir
|
||||
|
||||
def runAndCapture( args ):
|
||||
child = subprocess.Popen(" ".join( args ), shell=True, stdout=subprocess.PIPE)
|
||||
lines = []
|
||||
@@ -18,4 +23,4 @@ def runAndCapture( args ):
|
||||
line = ""
|
||||
else:
|
||||
line = line + out
|
||||
return lines
|
||||
return lines
|
||||
|
||||
Reference in New Issue
Block a user