diff --git a/projects/VS2010/ManagedTestCatch/ManagedTestCatch.sln b/projects/VS2010/ManagedTestCatch/ManagedTestCatch.sln index 8d265189..7cfd3e6c 100644 --- a/projects/VS2010/ManagedTestCatch/ManagedTestCatch.sln +++ b/projects/VS2010/ManagedTestCatch/ManagedTestCatch.sln @@ -16,13 +16,19 @@ Global EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 + DebugMbcs|Win32 = DebugMbcs|Win32 Release|Win32 = Release|Win32 + ReleaseMbcs|Win32 = ReleaseMbcs|Win32 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.Debug|Win32.ActiveCfg = Debug|Win32 {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.Debug|Win32.Build.0 = Debug|Win32 + {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.DebugMbcs|Win32.ActiveCfg = DebugMbcs|Win32 + {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.DebugMbcs|Win32.Build.0 = DebugMbcs|Win32 {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.Release|Win32.ActiveCfg = Release|Win32 {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.Release|Win32.Build.0 = Release|Win32 + {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.ReleaseMbcs|Win32.ActiveCfg = ReleaseMbcs|Win32 + {7CC06A6B-763E-42B3-AF6C-8F1E340372A1}.ReleaseMbcs|Win32.Build.0 = ReleaseMbcs|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj b/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj index b14d3a5f..84f8baf6 100644 --- a/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj +++ b/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj @@ -1,17 +1,25 @@  + + DebugMbcs + Win32 + Debug Win32 + + ReleaseMbcs + Win32 + Release Win32 - DefaultTest + $(ProjectName) {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942} {7CC06A6B-763E-42B3-AF6C-8F1E340372A1} ManagedCProj @@ -23,24 +31,42 @@ true Unicode + + DynamicLibrary + true + MultiByte + DynamicLibrary Safe Unicode + + DynamicLibrary + Safe + MultiByte + + + + + + + if exist app.config copy app.config "$(OutDir)app.config" true + true false + false @@ -58,6 +84,22 @@ + + + Level4 + Disabled + WIN32;_DEBUG;%(PreprocessorDefinitions) + NotUsing + ..\..\..\include + true + MultiThreadedDebugDLL + + + true + + + + Level3 @@ -72,6 +114,20 @@ + + + Level3 + WIN32;NDEBUG;%(PreprocessorDefinitions) + NotUsing + true + ../../../include + + + true + + + + @@ -81,6 +137,7 @@ + @@ -90,13 +147,18 @@ + + + Create + Create Create + Create diff --git a/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj.filters b/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj.filters index 0f1c9922..fe8b5077 100644 --- a/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj.filters +++ b/projects/VS2010/ManagedTestCatch/ManagedTestCatch.vcxproj.filters @@ -63,6 +63,18 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + diff --git a/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj b/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj index d9105496..d7b89498 100644 --- a/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj +++ b/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj @@ -94,8 +94,6 @@ - - diff --git a/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj.filters b/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj.filters index b358c13d..c74faac3 100644 --- a/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj.filters +++ b/projects/VS2010/TestCatch/TestCatch/TestCatch.vcxproj.filters @@ -28,12 +28,6 @@ Sources - - Sources - - - Sources - Sources diff --git a/projects/VS2012/ManagedTestCatch/ManagedTestCatch.sln b/projects/VS2012/ManagedTestCatch/ManagedTestCatch.sln index b41b17a4..a0be7336 100644 --- a/projects/VS2012/ManagedTestCatch/ManagedTestCatch.sln +++ b/projects/VS2012/ManagedTestCatch/ManagedTestCatch.sln @@ -6,13 +6,19 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 + DebugMbcs|Win32 = DebugMbcs|Win32 Release|Win32 = Release|Win32 + ReleaseMbcs|Win32 = ReleaseMbcs|Win32 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {9757CB21-B840-49A6-B057-9F322E543DD6}.Debug|Win32.ActiveCfg = Debug|Win32 {9757CB21-B840-49A6-B057-9F322E543DD6}.Debug|Win32.Build.0 = Debug|Win32 + {9757CB21-B840-49A6-B057-9F322E543DD6}.DebugMbcs|Win32.ActiveCfg = DebugMbcs|Win32 + {9757CB21-B840-49A6-B057-9F322E543DD6}.DebugMbcs|Win32.Build.0 = DebugMbcs|Win32 {9757CB21-B840-49A6-B057-9F322E543DD6}.Release|Win32.ActiveCfg = Release|Win32 {9757CB21-B840-49A6-B057-9F322E543DD6}.Release|Win32.Build.0 = Release|Win32 + {9757CB21-B840-49A6-B057-9F322E543DD6}.ReleaseMbcs|Win32.ActiveCfg = ReleaseMbcs|Win32 + {9757CB21-B840-49A6-B057-9F322E543DD6}.ReleaseMbcs|Win32.Build.0 = ReleaseMbcs|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj b/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj index e5bcc201..d96ad74a 100644 --- a/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj +++ b/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj @@ -1,17 +1,25 @@  + + DebugMbcs + Win32 + Debug Win32 + + ReleaseMbcs + Win32 + Release Win32 - DefaultTest + $(ProjectName) {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942} {9757CB21-B840-49A6-B057-9F322E543DD6} ManagedCProj @@ -24,30 +32,54 @@ Safe Unicode - + DynamicLibrary v110 Safe + MultiByte + + + DynamicLibrary + v110 + true Unicode + + DynamicLibrary + v110 + true + MultiByte + + + + + + + if exist app.config copy app.config "$(OutDir)app.config" true + + true + false + + false + Level3 @@ -66,6 +98,25 @@ taskkill /F /IM vstest.executionengine.x86.exe /FI "MEMUSAGE gt 1" + + + Level3 + Disabled + WIN32;_DEBUG;%(PreprocessorDefinitions) + NotUsing + ../../../include + true + MultiThreadedDebugDLL + + + true + + + + + taskkill /F /IM vstest.executionengine.x86.exe /FI "MEMUSAGE gt 1" + + Level3 @@ -79,6 +130,20 @@ + + + Level3 + WIN32;NDEBUG;%(PreprocessorDefinitions) + NotUsing + ../../../include + true + + + true + + + + ..\..\..\..\..\Program Files (x86)\Microsoft Visual Studio 11.0\Common7\IDE\PublicAssemblies\Microsoft.VisualStudio.QualityTools.UnitTestFramework.dll @@ -91,6 +156,7 @@ + @@ -100,13 +166,18 @@ + + + Create + Create Create + Create diff --git a/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj.filters b/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj.filters index f32e36cb..54f6763d 100644 --- a/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj.filters +++ b/projects/VS2012/ManagedTestCatch/ManagedTestCatch.vcxproj.filters @@ -63,6 +63,18 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + diff --git a/projects/VS2012/NativeTestCatch/NativeTestCatch.sln b/projects/VS2012/NativeTestCatch/NativeTestCatch.sln index 18ec7dc0..9e250bd9 100644 --- a/projects/VS2012/NativeTestCatch/NativeTestCatch.sln +++ b/projects/VS2012/NativeTestCatch/NativeTestCatch.sln @@ -6,13 +6,19 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 + DebugMbcs|Win32 = DebugMbcs|Win32 Release|Win32 = Release|Win32 + ReleaseMbcs|Win32 = ReleaseMbcs|Win32 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {977CE524-3FC7-4281-9C1B-77C210F24A9B}.Debug|Win32.ActiveCfg = Debug|Win32 {977CE524-3FC7-4281-9C1B-77C210F24A9B}.Debug|Win32.Build.0 = Debug|Win32 + {977CE524-3FC7-4281-9C1B-77C210F24A9B}.DebugMbcs|Win32.ActiveCfg = DebugMbcs|Win32 + {977CE524-3FC7-4281-9C1B-77C210F24A9B}.DebugMbcs|Win32.Build.0 = DebugMbcs|Win32 {977CE524-3FC7-4281-9C1B-77C210F24A9B}.Release|Win32.ActiveCfg = Release|Win32 {977CE524-3FC7-4281-9C1B-77C210F24A9B}.Release|Win32.Build.0 = Release|Win32 + {977CE524-3FC7-4281-9C1B-77C210F24A9B}.ReleaseMbcs|Win32.ActiveCfg = ReleaseMbcs|Win32 + {977CE524-3FC7-4281-9C1B-77C210F24A9B}.ReleaseMbcs|Win32.Build.0 = ReleaseMbcs|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj b/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj index 87bdb6d4..eb1ff325 100644 --- a/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj +++ b/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj @@ -1,10 +1,18 @@  + + DebugMbcs + Win32 + Debug Win32 + + ReleaseMbcs + Win32 + Release Win32 @@ -23,6 +31,20 @@ Unicode false + + DynamicLibrary + true + v110 + MultiByte + false + + + DynamicLibrary + true + v110 + MultiByte + false + DynamicLibrary false @@ -37,6 +59,12 @@ + + + + + + @@ -44,6 +72,12 @@ true + + true + + + true + true @@ -62,6 +96,36 @@ $(VCInstallDir)UnitTest\lib;%(AdditionalLibraryDirectories) + + + NotUsing + Level3 + Disabled + ../../../include;$(VCInstallDir)UnitTest\include;%(AdditionalIncludeDirectories) + WIN32;_DEBUG;%(PreprocessorDefinitions) + true + + + Windows + true + $(VCInstallDir)UnitTest\lib;%(AdditionalLibraryDirectories) + + + + + NotUsing + Level3 + Disabled + ../../../include;$(VCInstallDir)UnitTest\include;%(AdditionalIncludeDirectories) + WIN32;_DEBUG;%(PreprocessorDefinitions) + true + + + Windows + true + $(VCInstallDir)UnitTest\lib;%(AdditionalLibraryDirectories) + + Level3 @@ -88,6 +152,7 @@ + @@ -97,11 +162,16 @@ + + + Create + Create + Create Create diff --git a/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj.filters b/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj.filters index 24538c37..a95b17b3 100644 --- a/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj.filters +++ b/projects/VS2012/NativeTestCatch/NativeTestCatch.vcxproj.filters @@ -68,5 +68,17 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + \ No newline at end of file diff --git a/scripts/approvalVSTests.py b/scripts/approvalVSTests.py new file mode 100644 index 00000000..bef1cb79 --- /dev/null +++ b/scripts/approvalVSTests.py @@ -0,0 +1,716 @@ +#!/c/Python27/python + +import os +import sys +import subprocess +import re + +import xml.etree.cElementTree as etree + +from scriptCommon import catchPath +#from rawfile import writeRawFile +#from rawfile import parseRawFileIntoTree +from catch_test_run import TestRunApprovedHandler +from catch_test_run import TestRunData +from catch_test_run import TestRunResultHandler +from catch_test_case import TestCaseResultParser +from catch_test_case import TestCaseData + +rootPath = os.path.join(os.path.join(os.path.join( catchPath, 'projects'), 'SelfTest'), 'Baselines' ) + +if len(sys.argv) == 2: + cmdPath = sys.argv[1] +else: + if sys.platform == 'win32': + cmdPath = os.path.join( catchPath, 'projects\\VS2010\\TestCatch\\Release\\TestCatch.exe' ) + dllPath = os.path.join( catchPath, 'projects\\VS2010\\ManagedTestCatch\\Release\\ManagedTestCatch.dll' ) + else: + cmdPath = os.path.join( catchPath, 'projects/XCode4/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' ) + +print cmdPath + +overallResult = 0 + +def approve( baseName, args ): + global overallResult + args[0:0] = [cmdPath] + baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) ) + baselinesSortedPath = os.path.join( rootPath, '{0}.sorted.approved.txt'.format( baseName ) ) + rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) + if os.path.exists( baselinesPath ): + approvedFileHandler = TestRunApprovedHandler(baselinesPath) + baselinesPathNew = os.path.join( rootPath, '{0}.approved.new.txt'.format( baseName ) ) + approvedFileHandler.writeRawFile(baselinesPathNew) + approvedFileHandler.writeSortedRawFile(baselinesSortedPath) + else: + raise Exception("Base file does not exist: '" + baselinesPath + "'") + + if not(os.path.exists( args[0] )): + raise Exception("Executable does not exist: '" + args[0] + "'") + + f = open( rawResultsPath, 'w' ) + subprocess.call( args, stdout=f, stderr=f ) + f.close() + + if os.path.exists( rawResultsPath ): + resultFileHandler = TestRunResultHandler(rawResultsPath) + rawPathNew = os.path.join( rootPath, '{0}.rewrite.txt'.format( baseName ) ) + #print "F:",rawPathNew,",",approvedFileHandler.current.outputLine + resultFileHandler.writeRawFile(rawPathNew) + rawPathNewSorted = os.path.join( rootPath, '{0}.sorted.unapproved.txt'.format( baseName ) ) + resultFileHandler.writeSortedUnapprovedFile(rawPathNewSorted, approvedFileHandler.current.outputLine) + else: + raise Exception("Results file does not exist: '" + rawResultsPath + "'") + + #os.remove( rawResultsPath ) + print + print baseName + ":" + if os.path.exists( baselinesSortedPath ) and os.path.exists( rawPathNewSorted ): + diffResult = subprocess.call([ "diff", "--ignore-all-space", baselinesSortedPath, rawPathNewSorted ] ) + if diffResult == 0: + #os.remove( filteredResultsPath ) + if not(sys.platform == 'win32'): + print " \033[92mResults matched" + else: + print " Results matched" + else: + if not(sys.platform == 'win32'): + print " \n****************************\n \033[91mResults differed" + else: + print " \n****************************\n Results differed" + if diffResult > overallResult: + overallResult = diffResult + if not(sys.platform == 'win32'): + print "\033[0m" + +def approveJunit( baseName, args ): + global overallResult + args[0:0] = [cmdPath] + baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) ) + baselinesSortedPath = os.path.join( rootPath, '{0}.sorted.approved.txt'.format( baseName ) ) + #baselinesFixedPath = os.path.join( rootPath, '{0}.rewrite.approved.txt'.format( baseName ) ) + rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) + if os.path.exists( baselinesPath ): + xml = "" + f = open( baselinesPath, 'r' ) + for line in f: + xml += line + xml = xml.replace("", "<line number>") + xml = xml.replace("", "<hex digits>") + #f2 = open( baselinesFixedPath, 'wb' ) + #f2.write(xml) + #f2.close() + + # ClassTests.cpp: + otherApprovedTestParser = re.compile( r'(.*\..pp).*:<(.*).*>' ) + testRun = TestRunData() + testcase = None + root = etree.fromstring(xml) + for testsuites in root: + if testsuites.tag == "testsuite": + testRun = TestRunData() + testRun.appname = testsuites.get("name") + testRun.errors = testsuites.get("errors") + testRun.failures = testsuites.get("failures") + testRun.tests = testsuites.get("tests") + for tc in testsuites: + if tc.tag == "testcase": + cls = tc.get("classname") + #print "C:",cls,tc + if len(cls): + testcase = testRun.addClassTestCase(cls, tc.get("name")) + else: + testcase = testRun.addTestCase(tc.get("name")) + for prop in tc: + if prop.tag == "failure": + text = prop.text.strip() + lines = text.splitlines() + filename = "" + lineNumber = "" + output = [] + for l in lines: + m = otherApprovedTestParser.match(l) + if m: + filename = m.group(1) + lineNumber = m.group(2) + else: + output.append(l) + testcase.addFailure(filename, lineNumber, output, prop.get("message"), prop.get("type")) + elif prop.tag == "error": + text = prop.text.strip() + lines = text.splitlines() + filename = "" + lineNumber = "" + output = [] + for l in lines: + m = otherApprovedTestParser.match(l) + if m: + filename = m.group(1) + lineNumber = m.group(2) + else: + output.append(l) + testcase.addError(filename, lineNumber, output, prop.get("message"), prop.get("type")) + elif prop.tag == "system-out": + text = prop.text.strip() + lines = text.splitlines() + testcase.addSysout(lines) + elif prop.tag == "system-err": + text = prop.text.strip() + lines = text.splitlines() + testcase.addSyserr(lines) + elif tc.tag == "system-out": + text = tc.text.strip() + lines = text.splitlines() + testRun.addSysout(lines) + elif tc.tag == "system-err": + text = tc.text.strip() + lines = text.splitlines() + testRun.addSyserr(lines) + else: + print tc.tag + + lines = testRun.generateSortedUnapprovedJunit() + + rawWriteFile = open( baselinesSortedPath, 'wb' ) + for line in lines: + #print "L:",line + rawWriteFile.write(line + "\n") + rawWriteFile.close() + + if not(os.path.exists( args[0] )): + raise Exception("Executable does not exist: '" + args[0] + "'") + + f = open( rawResultsPath, 'w' ) + subprocess.call( args, stdout=f, stderr=f ) + f.close() + + rawSortedPath = os.path.join( rootPath, '{0}.sorted.unapproved.txt'.format( baseName ) ) + if os.path.exists( rawResultsPath ): + xml = "" + f = open( rawResultsPath, 'r' ) + for line in f: + xml += line + #xml = xml.replace("", "<line number>") + #xml = xml.replace("", "<hex digits>") + + # ClassTests.cpp: + otherResultsTestParser = re.compile( r'(.*\\)(.*\..pp).*\((.*).*\)' ) + testRun = TestRunData() + testcase = None + root = etree.fromstring(xml) + for testsuites in root: + if testsuites.tag == "testsuite": + testRun = TestRunData() + testRun.appname = testsuites.get("name") + testRun.errors = testsuites.get("errors") + testRun.failures = testsuites.get("failures") + testRun.tests = testsuites.get("tests") + for tc in testsuites: + if tc.tag == "testcase": + cls = tc.get("classname") + #print "C:",cls,tc + if len(cls): + if cls.startswith("::"): + cls = cls[2:] + testcase = testRun.addClassTestCase(cls, tc.get("name")) + else: + testcase = testRun.addTestCase(tc.get("name")) + for prop in tc: + if prop.tag == "failure": + text = prop.text.strip() + lines = text.splitlines() + filename = "" + lineNumber = "" + output = [] + for l in lines: + m = otherResultsTestParser.match(l) + if m: + filename = m.group(2) + lineNumber = "line number" + else: + output.append(l) + testcase.addFailure(filename, lineNumber, output, prop.get("message"), prop.get("type")) + elif prop.tag == "error": + text = prop.text.strip() + lines = text.splitlines() + filename = "" + lineNumber = "" + output = [] + for l in lines: + m = otherResultsTestParser.match(l) + if m: + filename = m.group(2) + lineNumber = "line number" + else: + output.append(l) + testcase.addError(filename, lineNumber, output, prop.get("message"), prop.get("type")) + elif prop.tag == "system-out": + text = prop.text.strip() + lines = text.splitlines() + testcase.addSysout(lines) + elif prop.tag == "system-err": + text = prop.text.strip() + lines = text.splitlines() + testcase.addSyserr(lines) + elif tc.tag == "system-out": + text = tc.text.strip() + lines = text.splitlines() + testRun.addSysout(lines) + elif tc.tag == "system-err": + text = tc.text.strip() + lines = text.splitlines() + testRun.addSyserr(lines) + else: + print tc.tag + + lines = testRun.generateSortedUnapprovedJunit() + + rawWriteFile = open( rawSortedPath, 'wb' ) + for line in lines: + #print "L:",line + rawWriteFile.write(line + "\n") + rawWriteFile.close() + +def addSubSection(testcase, section, exp): + r = exp.find("OverallResults") + if r != None: + ores = [] + ores.append(r.get("successes")) + ores.append(r.get("failures")) + if section == None: + section = testcase.addSection(exp.get("name"), exp.get("description"), ores) + else: + section = testcase.addSubSection(section, exp.get("name"), exp.get("description"), ores) + e1 = False + for tmp in exp: + if tmp.tag == "OverallResults": + pass + elif tmp.tag == "Exception": + filename = tmp.get("filename") + text = tmp.text + ls = text.splitlines() + testcase.addSubException(section, filename, ls) + elif tmp.tag == "Section": + addSubSection(testcase, section, tmp) + elif tmp.tag == "Failure": + text = tmp.text + ls = text.splitlines() + testcase.addSubFailure(section, ls) + elif tmp.tag == "Expression": + #print "Exp:",tmp + e1 = True + result = tmp.get("success") + filename = tmp.get("filename") + subSection = testcase.addSubExpression(section,result,filename) + subExp = [] + for cond in tmp: + if cond.tag == "Original": + text = cond.text + ls = text.splitlines() + subExp.append(ls) + elif cond.tag == "Expanded" and len(subExp) == 1: + text = cond.text + ls = text.splitlines() + subExp.append(ls) + elif cond.tag == "Exception" and len(subExp) == 2: + subExp.append(cond.get("filename")) + text = cond.text + ls = text.splitlines() + subExp.append(ls) + else: + print "SX:",cond.tag + if len(subExp) >= 2: + testcase.addExpressionDetails(subSection, subExp) + else: + print "Z:",tmp.tag + #if e1: + # print "Section:",section + +def addResultsSubSection(otherResultsTestParser, testcase, section, exp): + r = exp.find("OverallResults") + if r != None: + ores = [] + ores.append(r.get("successes")) + ores.append(r.get("failures")) + if section == None: + section = testcase.addSection(exp.get("name"), exp.get("description"), ores) + else: + section = testcase.addSubSection(section, exp.get("name"), exp.get("description"), ores) + e1 = False + for tmp in exp: + if tmp.tag == "OverallResults": + pass + elif tmp.tag == "Exception": + filename = tmp.get("filename") + m = otherResultsTestParser.match(filename) + if m: + filename = "/Users/philnash/Dev/OSS/Catch/projects/SelfTest/" + m.group(2) + text = tmp.text + ls = text.splitlines() + testcase.addSubException(section, filename, ls) + elif tmp.tag == "Section": + addResultsSubSection(otherResultsTestParser, testcase, section, tmp) + elif tmp.tag == "Failure": + text = tmp.text + ls = text.splitlines() + testcase.addSubFailure(section, ls) + elif tmp.tag == "Expression": + #print "Exp:",tmp + e1 = True + result = tmp.get("success") + filename = tmp.get("filename") + m = otherResultsTestParser.match(filename) + if m: + filename = "/Users/philnash/Dev/OSS/Catch/projects/SelfTest/" + m.group(2) + subSection = testcase.addSubExpression(section,result,filename) + subExp = [] + for cond in tmp: + if cond.tag == "Original": + text = cond.text + ls = text.splitlines() + subExp.append(ls) + elif cond.tag == "Expanded" and len(subExp) == 1: + text = cond.text + ls = text.splitlines() + subExp.append(ls) + elif cond.tag == "Exception" and len(subExp) == 2: + filename = cond.get("filename") + m = otherResultsTestParser.match(filename) + if m: + filename = "/Users/philnash/Dev/OSS/Catch/projects/SelfTest/" + m.group(2) + subExp.append(filename) + text = cond.text + ls = text.splitlines() + subExp.append(ls) + else: + print "SX:",cond.tag + if len(subExp) >= 2: + testcase.addExpressionDetails(subSection, subExp) + else: + print "Z:",tmp.tag + #if e1: + # print "Section:",section + +def approveXml( baseName, args ): + global overallResult + args[0:0] = [cmdPath] + baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) ) + baselinesSortedPath = os.path.join( rootPath, '{0}.sorted.approved.txt'.format( baseName ) ) + #baselinesFixedPath = os.path.join( rootPath, '{0}.rewrite.approved.txt'.format( baseName ) ) + rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) + if os.path.exists( baselinesPath ): + xml = "" + f = open( baselinesPath, 'r' ) + for line in f: + xml += line + xml = xml.replace("", "<hex digits>") + + #otherApprovedTestParser = re.compile( r'(.*\..pp).*:<(.*).*>' ) + testRun = TestRunData() + testcase = None + root = etree.fromstring(xml) + testRun.appname = root.get("name") + for ts in root: + #print ts.tag + for tc in ts: + if tc.tag == "TestCase": + testcase = testRun.addTestCase(tc.get("name")) + for exp in tc: + if exp.tag == "Expression": + result = exp.get("success") + filename = exp.get("filename") + section = testcase.addExpression(result,filename) + subExp = [] + for cond in exp: + if cond.tag == "Original": + text = cond.text + ls = text.splitlines() + subExp.append(ls) + elif cond.tag == "Expanded" and len(subExp) == 1: + text = cond.text + ls = text.splitlines() + subExp.append(ls) + elif cond.tag == "Exception" and len(subExp) == 2: + subExp.append(cond.get("filename")) + text = cond.text + ls = text.splitlines() + subExp.append(ls) + else: + print "X:",cond.tag + if len(subExp) >= 2: + testcase.addExpressionDetails(section, subExp) + elif exp.tag == "Exception": + filename = exp.get("filename") + text = exp.text + ls = text.splitlines() + section = testcase.addException(filename,ls) + elif exp.tag == "Section": + addSubSection(testcase, None, exp) + elif exp.tag == "Info": + text = exp.text + ls = text.splitlines() + section = testcase.addInfo(ls) + elif exp.tag == "Warning": + text = exp.text + ls = text.splitlines() + section = testcase.addWarning(ls) + elif exp.tag == "Failure": + text = exp.text + ls = text.splitlines() + section = testcase.addSimpleFailure(ls) + elif exp.tag == "OverallResult": + testcase.addOverallResult(exp.get("success")) + else: + print "E:",exp.tag + elif tc.tag == "OverallResults": + testRun.tests = tc.get("successes") + testRun.failures = tc.get("failures") + else: + print "U:",tc.tag + + lines = testRun.generateSortedUnapprovedXml() + + rawWriteFile = open( baselinesSortedPath, 'wb' ) + for line in lines: + #print "L:",line + rawWriteFile.write(line + "\n") + rawWriteFile.close() + + if not(os.path.exists( args[0] )): + raise Exception("Executable does not exist: '" + args[0] + "'") + + f = open( rawResultsPath, 'w' ) + subprocess.call( args, stdout=f, stderr=f ) + f.close() + + rawSortedPath = os.path.join( rootPath, '{0}.sorted.unapproved.txt'.format( baseName ) ) + if os.path.exists( rawResultsPath ): + xml = "" + f = open( rawResultsPath, 'r' ) + for line in f: + xml += line + #xml = xml.replace("", "<hex digits>") + + otherResultsTestParser = re.compile( r'(.*\\)(.*\..pp)' ) + hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' ) + testRun = TestRunData() + testcase = None + root = etree.fromstring(xml) + testRun.appname = root.get("name") + if testRun.appname == "TestCatch.exe": + testRun.appname = "CatchSelfTest" + for ts in root: + #print ts.tag + for tc in ts: + if tc.tag == "TestCase": + testcase = testRun.addTestCase(tc.get("name")) + for exp in tc: + if exp.tag == "Expression": + result = exp.get("success") + filename = exp.get("filename") + m = otherResultsTestParser.match(filename) + if m: + filename = "/Users/philnash/Dev/OSS/Catch/projects/SelfTest/" + m.group(2) + section = testcase.addExpression(result,filename) + subExp = [] + for cond in exp: + if cond.tag == "Original": + text = cond.text + tmp = text.splitlines() + ls = [] + for li in tmp: + m = hexParser.match(li) + if m: + while m: + #print li, m.group(1), m.group(3) + li = m.group(1) + "0x" + m.group(3) + m = hexParser.match(li) + ls.append(li) + subExp.append(ls) + elif cond.tag == "Expanded" and len(subExp) == 1: + text = cond.text + tmp = text.splitlines() + ls = [] + for li in tmp: + m = hexParser.match(li) + if m: + while m: + #print li, m.group(1), m.group(3) + li = m.group(1) + "0x" + m.group(3) + m = hexParser.match(li) + ls.append(li) + subExp.append(ls) + elif cond.tag == "Exception" and len(subExp) == 2: + filename = cond.get("filename") + m = otherResultsTestParser.match(filename) + if m: + filename = "/Users/philnash/Dev/OSS/Catch/projects/SelfTest/" + m.group(2) + subExp.append(filename) + text = cond.text + ls = text.splitlines() + subExp.append(ls) + else: + print "X:",cond.tag + if len(subExp) >= 2: + testcase.addExpressionDetails(section, subExp) + elif exp.tag == "Exception": + filename = exp.get("filename") + m = otherResultsTestParser.match(filename) + if m: + filename = "/Users/philnash/Dev/OSS/Catch/projects/SelfTest/" + m.group(2) + text = exp.text + ls = text.splitlines() + section = testcase.addException(filename,ls) + elif exp.tag == "Section": + addResultsSubSection(otherResultsTestParser, testcase, None, exp) + elif exp.tag == "Info": + text = exp.text + ls = text.splitlines() + section = testcase.addInfo(ls) + elif exp.tag == "Warning": + text = exp.text + ls = text.splitlines() + section = testcase.addWarning(ls) + elif exp.tag == "Failure": + text = exp.text + ls = text.splitlines() + section = testcase.addSimpleFailure(ls) + elif exp.tag == "OverallResult": + testcase.addOverallResult(exp.get("success")) + else: + print "E:",exp.tag + elif tc.tag == "OverallResults": + testRun.tests = tc.get("successes") + testRun.failures = tc.get("failures") + else: + print "U:",tc.tag + + lines = testRun.generateSortedUnapprovedXml() + + rawWriteFile = open( rawSortedPath, 'wb' ) + for line in lines: + #print "L:",line + rawWriteFile.write(line + "\n") + rawWriteFile.close() + +def parseTrxFile(trxFile): + print "TRX file:" ,trxFile + if os.path.exists( trxFile ): + xml = "" + f = open( trxFile, 'r' ) + for line in f: + xml += line + + #otherResultsTestParser = re.compile( r'(.*\\)(.*\..pp)' ) + #hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' ) + testRun = TestRunData() + testRun.appname = "CatchSelfTest" + root = etree.fromstring(xml) + if testRun.appname == "TestCatch.exe": + testRun.appname = "CatchSelfTest" + qname=re.compile("{(?P.*)}(?P.*)") + ids = [] + for ts in root: + m = qname.match(ts.tag) + if m: + tag = m.group(2) + print tag + if tag != None: + if tag == "TestDefinitions": + for tc in ts: + m = qname.match(tc.tag) + if m: + tag = m.group(2) + if tag != None and tag == "UnitTest": + name = tc.get("name") + id = tc.get("id") + for item in tc: + m = qname.match(item.tag) + if m: + tag = m.group(2) + if tag != None and tag == "Description": + desc = item.text + #print desc, id + ids.append([id,desc]) + elif tag == "Results": + #print ids + ids = dict(ids) + #print ids["87ec526a-e414-1a3f-ba0f-e210b204bb42"] + resultParser = TestCaseResultParser() + for tc in ts: + m = qname.match(tc.tag) + if m: + tag = m.group(2) + if tag != None and tag == "UnitTestResult": + outcome = tc.get("outcome") + id = tc.get("testId") + if len(id) > 0: + for item in tc: + m = qname.match(item.tag) + if m: + tag = m.group(2) + if tag != None and tag == "Output": + for sub in item: + m = qname.match(sub.tag) + if m: + tag = m.group(2) + if tag != None and tag == "StdOut": + desc = sub.text + lines = desc.splitlines() + if (len(lines) > 2 and + lines[0].startswith("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") and + lines[1].startswith("Using Catch v") ): + lines = lines[2:-1] + #print "*******",desc + #print lines + for line in lines: + testcase = resultParser.parseResultLine(line) + if isinstance(testcase, TestCaseData): + testRun.testcases.append(testcase) + lines = testRun.generateSortedUnapprovedLines(0) + + rawSortedPath = os.path.join( rootPath, 'mstest.trx.sorted.unapproved.txt' ) + rawWriteFile = open( rawSortedPath, 'wb' ) + for line in lines: + #print "L:",line + rawWriteFile.write(line + "\n") + rawWriteFile.close() + + +def approveMsTest( baseName ): + rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) + if not(os.path.exists( dllPath )): + raise Exception("Managed DLL does not exist: '" + dllPath + "'") + + args = [] + args.append("MSTest.exe") + args.append("/testcontainer:" + dllPath) + #f = open( rawResultsPath, 'w' ) + #subprocess.call( args, stdout=f, stderr=f ) + #f.close() + + #if os.path.exists( rawResultsPath ): + # f = open( rawResultsPath, 'r' ) + # for line in f: + line = "Results file: c:\Projects\Catch\TestResults\NoyesMa_SACHDEW7 2013-12-09 11_43_57.trx" + + if line.startswith("Results file:"): + trxFile = line[13:].strip() + parseTrxFile(trxFile) + +# Standard console reporter +#approve( "console.std", ["~_"] ) +# console reporter, include passes, warn about No Assertions +#approve( "console.sw", ["~_", "-s", "-w", "NoAssertions"] ) +# console reporter, include passes, warn about No Assertions, limit failures to first 4 +#approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4"] ) +# junit reporter, include passes, warn about No Assertions +#approveJunit( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit"] ) +# xml reporter, include passes, warn about No Assertions +#approveXml( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] ) +#mstest runner, xml output +approveMsTest( "mstest.sw") + +if overallResult <> 0: + print "run approve.py to approve new baselines" +exit( overallResult) diff --git a/scripts/catch_conditions.py b/scripts/catch_conditions.py new file mode 100644 index 00000000..e7cc87b1 --- /dev/null +++ b/scripts/catch_conditions.py @@ -0,0 +1,398 @@ +import re + +class RandomOutput: + + def __init__(self): + output = [] + +class TestConditionData: + NONE = 0 + CONDITION = 1 + EXPANSION = 2 + MESSAGES = 3 + + hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' ) + + def __init__(self): + self.state = self.NONE + self.filenamePrefix = "" + self.filename = "" + self.lineNumber = "" + self.reason = "" + self.condition = "" + self.expansionPrefix = "" + self.expansion = [] + self.messagePrefix = "" + self.messages = [] + self.output = [] + self.noAssertions = "" + + def __eq__(self, other): + return (self.filenamePrefix == other.filenamePrefix and + self.filename == other.filename and self.lineNumber == other.lineNumber and + self.reason == other.reason and self.condition == other.condition and + self.expansion == other.expansion and self.messagePrefix == other.messagePrefix and + self.output == other.output and self.noAssertions == other.noAssertions) + + + def empty(self): + if self.state == self.NONE: + return True + return False + + def __repr__(self): + result = ("[" + self.reason + ", " + self.filename + ", " + self.lineNumber + ", " + self.condition + + ", " + self.expansionPrefix + ", [ ") + suffix = "" + for msg in self.expansion: + result += suffix + result += repr(msg) + suffix = ", " + result += "], " + self.messagePrefix + ", [ " + suffix = "" + for msg in self.messages: + result += suffix + result += repr(msg) + suffix = ", " + result += " ], [ " + suffix = "" + for msg in self.output: + result += suffix + result += repr(msg) + suffix = ", " + result += " ] ]" + return result + + def parseCondition(self,line): + if len(line): + if self.state == self.CONDITION and line.startswith(" "): + self.condition = line.strip() + elif self.state == self.CONDITION: + if len(self.reason) == 0 and line.startswith("PASSED:"): + self.reason = line.strip() + elif line.startswith("warning:") or line.startswith("with message"): + self.messagePrefix = line.strip() + self.state = self.MESSAGES + elif not(line.startswith(" ")): + self.expansionPrefix = line.strip() + self.state = self.EXPANSION + else: + raise Exception("Unknown condition parse line: '" + line + "'") + elif self.state == self.EXPANSION: + if line.startswith("with message"): + self.messagePrefix = line.strip() + self.state = self.MESSAGES + elif line.startswith(" "): + #print "***LINE: ", line + self.expansion.append(line[2:].rstrip()) + elif line.startswith("... message truncated due"): + #print "***MSG: ", line + self.messagePrefix = line.strip() + else: + #print "***OUTPUT: ", line + self.output.append(line.strip()) + elif self.state == self.MESSAGES: + if line.startswith(" "): + #print "***MESSAGE: ", line + self.messages.append(line.strip()) + else: + #print "***OUTPUT: ", line + self.output.append(line.strip()) + else: + raise Exception("Unknown parse line: '" + line + "'") + + if len(self.output) == 10: + if (self.output[0] == "Message from section one" and self.output[1] == "Message from section two" and + self.output[2] == "Some information" and self.output[3] == "An error" and + self.output[4] == "Message from section one" and self.output[5] == "Message from section two" and + self.output[6] == "Some information" and self.output[7] == "An error" and + self.output[8] == "hello" and self.output[9] == "hello" ): + obj = RandomOutput() + obj.output = self.output + self.output = [] + raise obj + + elif len(self.output) == 2: + if (self.output[0] == "hello" and self.output[1] == "hello" ): + obj = RandomOutput() + obj.output = self.output + self.output = [] + raise obj + + def generateApprovedLines(self): + if self.empty(): + raise Exception("Empty condition..." + repr(self)) + lines = [] + extraLine = False + if len(self.filename): + line = self.filename + ":<" + self.lineNumber + ">:" + reasonOnSameLine = False + if self.reason == "FAILED": + line += " " + "FAILED:" + reasonOnSameLine = True + lines.append(line) + if not(reasonOnSameLine) and len(self.reason): + lines.append(self.reason) + if len(self.condition): + lines.append(" " + self.condition) + if len(self.expansionPrefix): + lines.append(self.expansionPrefix) + extraLine = True + if len(self.expansion): + for line in self.expansion: + m = self.hexParser.match(line) + if m: + while m: + line = m.group(1) + "0x" + m.group(3) + m = self.hexParser.match(line) + lines.append(" " + line) + if len(self.messagePrefix): + lines.append(self.messagePrefix) + extraLine = True + if len(self.messages): + for msg in self.messages: + lines.append(" " + msg) + lines.append("") + if len(self.noAssertions) > 0: + if extraLine: + lines.append("") + lines.append(self.noAssertions) + lines.append("") + if len(self.output): + for msg in self.output: + lines.append(msg) + return lines + + def generateResultLines(self): + if self.empty(): + raise Exception("Empty condition..." + repr(self)) + lines = [] + extraLine = False + if len(self.filename): + if len(self.filenamePrefix): + line = self.filenamePrefix + self.filename + "(" + self.lineNumber + "):" + else: + line = self.filename + "(" + self.lineNumber + "):" + reasonOnSameLine = False + if self.reason == "FAILED": + line += " " + "FAILED:" + reasonOnSameLine = True + if (len(self.messagePrefix) > 0 and self.messagePrefix == "warning:" or + self.reason == "PASSED:" or self.expansionPrefix.startswith("FAILED - but was ok") ): + line += " " + lines.append(line) + if not(reasonOnSameLine) and len(self.reason): + lines.append(self.reason) + if len(self.condition): + lines.append(" " + self.condition) + if len(self.expansionPrefix): + lines.append(self.expansionPrefix) + extraLine = True + if len(self.expansion): + for line in self.expansion: + lines.append(" " + line) + if len(self.messagePrefix): + lines.append(self.messagePrefix) + extraLine = True + if len(self.messages): + for msg in self.messages: + lines.append(" " + msg) + lines.append("") + if len(self.noAssertions) > 0: + if extraLine: + lines.append("") + lines.append(self.noAssertions) + lines.append("") + if len(self.output): + for msg in self.output: + lines.append(msg) + return lines + + def generateUnapprovedLines(self): + if self.empty(): + raise Exception("Empty condition..." + repr(self)) + lines = [] + extraLine = False + if len(self.filename): + line = self.filename + ":<" + "line number" + ">:" + reasonOnSameLine = False + if self.reason == "FAILED": + line += " " + "FAILED:" + reasonOnSameLine = True + lines.append(line) + if not(reasonOnSameLine) and len(self.reason): + lines.append(self.reason) + if len(self.condition): + lines.append(" " + self.condition) + if len(self.expansionPrefix): + lines.append(self.expansionPrefix) + extraLine = True + if len(self.expansion): + for line in self.expansion: + m = self.hexParser.match(line) + if m: + while m: + line = m.group(1) + "0x" + m.group(3) + m = self.hexParser.match(line) + lines.append(" " + line) + if len(self.messagePrefix): + lines.append(self.messagePrefix) + extraLine = True + if len(self.messages): + for msg in self.messages: + lines.append(" " + msg) + lines.append("") + if len(self.noAssertions) > 0: + if extraLine: + lines.append("") + lines.append(self.noAssertions) + lines.append("") + if len(self.output): + for msg in self.output: + lines.append(msg) + return lines + + def addFailure(self, filename, lineNumber, output, message, type): + self.reason = "failure" + self.filename = filename + self.lineNumber = lineNumber + self.condition = type + if message != None: + self.expansion.append(message) + self.output = output + + def addError(self, filename, lineNumber, output, message, type): + self.reason = "error" + self.filename = filename + self.lineNumber = lineNumber + self.condition = type + if message != None: + self.expansion.append(message) + self.output = output + + def generateUnapprovedJunit(self): + lines = [] + msg = "" + for m in self.expansion: + msg += m + msg = msg.replace("\"", """) + msg = msg.replace("<", "<") + msg = msg.replace("<hex digits>", "") + #msg = msg.replace(">", ">") + + #print "R:",self.reason,msg,self.condition + if len(self.reason) > 0: + l = " <" + self.reason + if len(msg) > 0: + m = self.hexParser.match(msg) + if m: + while m: + msg = m.group(1) + "0x" + m.group(3) + m = self.hexParser.match(msg) + l += " message=\"" + msg + "\"" + if self.condition != None: + l += " type=\"" + self.condition + "\"" + l += ">" + lines.append(l) + if len(self.output) > 0: + for o in self.output: + lines.append(o) + if len(self.filename) > 0: + lines.append(self.filename + ":<" + self.lineNumber + ">") + lines.append(" ") + return lines + +class TestConditionApprovedParser: + failedApprovedTestParser = re.compile( r'(.*\..pp).*:<(.*).*>:(.*FAILED)' ) + otherApprovedTestParser = re.compile( r'(.*\..pp).*:<(.*).*>:' ) + + def __init__(self): + self.current = TestConditionData() + + def parseApprovedLine(self,line): + result = None + if line.startswith("==============================================================================="): + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + + elif line.startswith("-------------------------------------------------------------------------------"): + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + + else: + if line.startswith("No assertions in"): + self.current.noAssertions = line.strip() + self.current.state = self.current.MESSAGES + else: + m = self.failedApprovedTestParser.match(line) + if m: + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + self.current.filename = m.group(1).strip() + self.current.lineNumber = m.group(2).strip() + self.current.reason = m.group(3).strip() + self.current.state = self.current.CONDITION + else: + m = self.otherApprovedTestParser.match(line) + if m: + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + self.current.filename = m.group(1).strip() + self.current.lineNumber = m.group(2).strip() + self.current.state = self.current.CONDITION + elif not(self.current.empty()): + self.current.parseCondition(line) + return result + +class TestConditionResultParser: + failedResultsTestParser = re.compile( r'(.*\\)(.*\..pp).*\((.*).*\):(.*FAILED)' ) + otherResultsTestParser = re.compile( r'(.*\\)(.*\..pp).*\((.*).*\):' ) + + def __init__(self): + self.current = TestConditionData() + + def parseResultLine(self,line): + result = None + if line.startswith("==============================================================================="): + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + + elif line.startswith("-------------------------------------------------------------------------------"): + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + + else: + if line.startswith("No assertions in"): + self.current.noAssertions = line.strip() + self.current.state = self.current.MESSAGES + else: + m = self.failedResultsTestParser.match(line) + if m: + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + self.current.filenamePrefix = m.group(1).strip() + self.current.filename = m.group(2).strip() + self.current.lineNumber = m.group(3).strip() + self.current.reason = m.group(4).strip() + self.current.state = self.current.CONDITION + else: + m = self.otherResultsTestParser.match(line) + if m: + if not(self.current.empty()): + result = self.current + self.current = TestConditionData() + self.current.filenamePrefix = m.group(1).strip() + self.current.filename = m.group(2).strip() + self.current.lineNumber = m.group(3).strip() + self.current.state = self.current.CONDITION + elif not(self.current.empty()): + self.current.parseCondition(line) + return result + diff --git a/scripts/catch_test_case.py b/scripts/catch_test_case.py new file mode 100644 index 00000000..b685c9e8 --- /dev/null +++ b/scripts/catch_test_case.py @@ -0,0 +1,584 @@ +import re + +from catch_conditions import TestConditionData +from catch_conditions import TestConditionApprovedParser +from catch_conditions import TestConditionResultParser + +class EndOfClassName: + @staticmethod + def parseRawLine(line): + if line.startswith("..............................................................................."): + return ParseResult.END_OF_CLASS_NAME + return ParseResult.NONE + +class TestCaseData: + + def __init__(self): + self.name = "" + self.nameParts = [] + self.classname = "global" + self.sections = [] + self.filenamePrefix = "" + self.filename = "" + self.lineNumber = "" + self.conditions = [] + self.sysout = [] + self.syserr = [] + self.result = "" + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __repr__(self): + result = "[" + self.name + ", [ " + suffix = "" + for section in self.sections: + result += suffix + result += repr(section) + suffix = ", " + result + " ] " + result += self.filename + ", " + self.lineNumber + " [ " + suffix = "" + for cond in self.conditions: + result += suffix + result += repr(cond) + suffix = ", " + result + " ] ]" + return result + + def empty(self): + if len(self.name): + return False + return True + + def generateApprovedLines(self): + if self.empty(): + raise Exception("Empty test case..." + repr(self)) + lines = [] + if len(self.name): + lines.append("-------------------------------------------------------------------------------") + for n in self.nameParts: + lines.append(n) + if len(self.sections) > 0: + for section in self.sections: + lines.append(" " + section) + lines.append("-------------------------------------------------------------------------------") + if len(self.filename): + lines.append(self.filename + ":<" + self.lineNumber + ">") + lines.append("...............................................................................") + lines.append("") + for cond in self.conditions: + lines += cond.generateApprovedLines() + return lines + + def generateResultLines(self): + if self.empty(): + raise Exception("Empty test case..." + repr(self)) + lines = [] + if len(self.name): + lines.append("-------------------------------------------------------------------------------") + for n in self.nameParts: + lines.append(n) + if len(self.sections) > 0: + for section in self.sections: + lines.append(" " + section) + lines.append("-------------------------------------------------------------------------------") + if len(self.filename): + lines.append(self.filenamePrefix + self.filename + "(" + self.lineNumber + ")") + lines.append("...............................................................................") + lines.append("") + for cond in self.conditions: + lines += cond.generateResultLines() + return lines + + def generateUnapprovedLines(self): + if self.empty(): + raise Exception("Empty test case..." + repr(self)) + lines = [] + if len(self.name): + lines.append("-------------------------------------------------------------------------------") + for n in self.nameParts: + lines.append(n) + if len(self.sections) > 0: + for section in self.sections: + lines.append(" " + section) + lines.append("-------------------------------------------------------------------------------") + if len(self.filename): + lines.append(self.filename + ":<" + "line number" + ">") + lines.append("...............................................................................") + lines.append("") + for cond in self.conditions: + lines += cond.generateUnapprovedLines() + return lines + + def generateUnapprovedJunit(self): + lines = [] + + condLines = [] + for cond in self.conditions: + condLines += cond.generateUnapprovedJunit() + + if len(self.name): + l = " 0 or len(self.sysout) > 0 or len(self.syserr) > 0: + l += ">" + else: + l += "/>" + lines.append(l) + #if len(self.sections) > 0: + # for section in self.sections: + # lines.append(" " + section) + #if len(self.filename): + # lines.append(self.filename + ":<" + "line number" + ">") + # lines.append("...............................................................................") + lines += condLines + if len(self.sysout) > 0: + lines.append(" ") + for l in self.sysout: + lines.append(l) + lines.append(" ") + if len(self.syserr) > 0: + lines.append(" ") + for l in self.syserr: + lines.append(l) + lines.append(" ") + if len(condLines) > 0 or len(self.sysout) > 0 or len(self.syserr) > 0: + lines.append(" ") + return lines + + def generateRecursiveSection(self, prefix, section): + lines = [] + #print "S:",section + if section[0] == "S": + l = " " + prefix + "
","") + li = li.replace("\"",""") + l += " description=\"" + li + "\"" + l += ">" + lines.append(l) + if len(section) > 4: + index = 4 + while index < len(section): + tmp = section[index] + if len(tmp) > 0: + if tmp[0] == "E": + l = " " + prefix + " 3: + cond = tmp[3] + if len(cond[0]) > 0: + lines.append(" " + prefix + "") + for li in cond[0]: + if len(li.strip()) > 0: + li = li.replace("<","<") + li = li.replace("<hex digits>","") + li = li.replace("\"",""") + lines.append(li) + lines.append(" " + prefix + "") + if len(cond[1]) > 0: + lines.append(" " + prefix + "") + for li in cond[1]: + if len(li.strip()) > 0: + li = li.replace("<","<") + li = li.replace("<hex digits>","") + li = li.replace("\"",""") + lines.append(li) + lines.append(" " + prefix + "") + if len(cond) > 2: + filename = cond[2] + lines.append(" " + prefix + "") + if len(cond) > 3: + tmp = cond[3] + for li in tmp: + if len(li.strip()) > 0: + lines.append(li) + lines.append(" " + prefix + "") + elif len(tmp) > 4: + print "RE:",tmp[4] + + l = " " + prefix + "" + lines.append(l) + elif tmp[0] == "X": + l = " " + prefix + "" + lines.append(l) + for li in tmp[2]: + if len(li.strip()) > 0: + lines.append(li) + l = " " + prefix + "" + lines.append(l) + elif tmp[0] == "F": + l = " " + prefix + "" + lines.append(l) + for li in tmp[1]: + if len(li.strip()) > 0: + lines.append(li) + l = " " + prefix + "" + lines.append(l) + elif tmp[0] == "S": + lines += self.generateRecursiveSection(prefix + " ", tmp) + else: + print "RS2:",tmp[0] + else: + print "RS:",section[index] + index += 1 + + lines.append(" " + prefix + "") + l = " " + prefix + "
" + lines.append(l) + return lines + + def generateSection(self, prefix, sections): + lines = [] + for section in sections: + #print "S:",section + if section[0] == "S": + lines += self.generateRecursiveSection(prefix, section) + elif section[0] == "E": + l = " " + prefix + " 3: + cond = section[3] + if len(cond[0]) > 0: + lines.append(" " + prefix + "") + for li in cond[0]: + if len(li.strip()) > 0: + li = li.replace("&","&") + li = li.replace("<","<") + li = li.replace("<hex digits>","") + li = li.replace("\"",""") + lines.append(li) + lines.append(" " + prefix + "") + if len(cond[1]) > 0: + lines.append(" " + prefix + "") + for li in cond[1]: + if len(li.strip()) > 0: + li = li.replace("<","<") + li = li.replace("<hex digits>","") + li = li.replace("\"",""") + lines.append(li) + lines.append(" " + prefix + "") + if len(cond) > 2: + filename = cond[2] + lines.append(" " + prefix + "") + if len(cond) > 3: + tmp = cond[3] + for li in tmp: + if len(li.strip()) > 0: + lines.append(li) + lines.append(" " + prefix + "") + elif len(section) > 4: + print "RE:",section[4] + + l = " " + prefix + "" + lines.append(l) + elif section[0] == "X": + l = " " + prefix + "" + lines.append(l) + for li in section[2]: + if len(li.strip()) > 0: + lines.append(li) + l = " " + prefix + "" + lines.append(l) + elif section[0] == "I": + l = " " + prefix + "" + lines.append(l) + for li in section[1]: + if len(li.strip()) > 0: + lines.append(li) + l = " " + prefix + "" + lines.append(l) + elif section[0] == "W": + l = " " + prefix + "" + lines.append(l) + for li in section[1]: + if len(li.strip()) > 0: + lines.append(li) + l = " " + prefix + "" + lines.append(l) + elif section[0] == "F": + l = " " + prefix + "" + lines.append(l) + for li in section[1]: + if len(li.strip()) > 0: + lines.append(li) + l = " " + prefix + "" + lines.append(l) + return lines + + def generateUnapprovedXml(self): + lines = [] + + if len(self.name): + l = " " + lines.append(l) + + if len(self.sections) > 0: + prefix = " " + lines += self.generateSection(prefix, self.sections) + + if len(self.result) > 0: + lines.append(" ") + + if len(self.name): + l = " " + lines.append(l) + return lines + + def addFailure(self, filename, lineNumber, output, message, type): + self.filename = filename + self.lineNumber = lineNumber + condition = TestConditionData() + condition.addFailure(filename, lineNumber, output, message, type) + self.conditions.append(condition) + + def addError(self, filename, lineNumber, output, message, type): + self.filename = filename + self.lineNumber = lineNumber + condition = TestConditionData() + condition.addError(filename, lineNumber, output, message, type) + self.conditions.append(condition) + + def addSysout(self, output): + self.sysout = output + + def addSyserr(self, output): + self.syserr = output + + def addOverallResult(self,r): + self.result = r + + def addSection(self,name,desc, results): + section = [] + section.append("S") + section.append(name) + section.append(desc) + section.append(results) + self.sections.append(section) + return section + + def addExpression(self,result,filename): + section = [] + section.append("E") + section.append(result) + section.append(filename) + self.sections.append(section) + return section + + def addException(self,filename,text): + section = [] + section.append("X") + section.append(filename) + section.append(text) + #print section + self.sections.append(section) + return section + + def addInfo(self,text): + section = [] + section.append("I") + section.append(text) + self.sections.append(section) + return section + + def addWarning(self,text): + section = [] + section.append("W") + section.append(text) + self.sections.append(section) + return section + + def addSimpleFailure(self,text): + section = [] + section.append("F") + section.append(text) + self.sections.append(section) + return section + + def addExpressionDetails(self, section, subExp): + section.append(subExp) + + def addSubException(self, section, filename, text): + tmp = [] + tmp.append("X") + tmp.append(filename) + tmp.append(text) + section.append(tmp) + + def addSubFailure(self, section, text): + tmp = [] + tmp.append("F") + tmp.append(text) + section.append(tmp) + + def addSubExpression(self,section,result,filename): + tmp = [] + tmp.append("E") + tmp.append(result) + tmp.append(filename) + section.append(tmp) + #print "Section:",section + return tmp + + def addSubSection(self,section,name,desc,results): + tmp = [] + tmp.append("S") + tmp.append(name) + tmp.append(desc) + tmp.append(results) + section.append(tmp) + #print "Section:",section + return tmp + +class TestCaseApprovedParser: + NONE = 0 + TEST_CASE_NAME_EXPECTED = 1 + TEST_CASE_NAME = 2 + TEST_CLASS_EXPECTED = 3 + END_OF_CLASS_NAME_EXPECTED = 4 + TEST_CONDITION_EXPECTED = 5 + + testFilenameParser = re.compile( r'(.*\..pp).*:<(.*).*>' ) + + def __init__(self): + self.state = self.NONE + self.current = TestCaseData() + self.conditionParser = TestConditionApprovedParser() + + def parseApprovedLine(self,line): + result = None + if self.state == self.NONE: + if line.startswith("-------------------------------------------------------------------------------"): + self.state = self.TEST_CASE_NAME_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CASE_NAME_EXPECTED: + if len(line): + self.current.name = line.strip() + self.current.nameParts.append(line.strip()) + self.state = self.TEST_CASE_NAME + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CASE_NAME: + if line.startswith("-------------------------------------------------------------------------------"): + self.state = self.TEST_CLASS_EXPECTED + elif line.startswith(" "): + #print "***SECTION: ",line + self.current.sections.append(line[2:]) + elif len(line): + if len(self.current.name) > 0: + self.current.name += line.strip() + else: + self.current.name = line.strip() + self.current.nameParts.append(line.strip()) + elif self.state == self.TEST_CLASS_EXPECTED: + m = self.testFilenameParser.match(line) + if m: + self.current.filename = m.group(1).strip() + self.current.lineNumber = m.group(2).strip() + self.state = self.END_OF_CLASS_NAME_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.END_OF_CLASS_NAME_EXPECTED: + if line.startswith("..............................................................................."): + self.state = self.TEST_CONDITION_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CONDITION_EXPECTED: + #print "**** LINE " + line + condition = self.conditionParser.parseApprovedLine(line) + if isinstance(condition, TestConditionData): + #print "**** CASE " + repr(condition) + self.current.conditions.append(condition) + if line.startswith("-------------------------------------------------------------------------------"): + result = self.current + self.current = TestCaseData() + self.state = self.TEST_CASE_NAME_EXPECTED + elif line.startswith("==============================================================================="): + result = self.current + self.current = TestCaseData() + self.state = self.NONE + + return result + +class TestCaseResultParser: + NONE = 0 + TEST_CASE_NAME_EXPECTED = 1 + TEST_CASE_NAME = 2 + TEST_CLASS_EXPECTED = 3 + END_OF_CLASS_NAME_EXPECTED = 4 + TEST_CONDITION_EXPECTED = 5 + + testFilenameParser = re.compile( r'(.*\\)(.*\..pp).*\((.*).*\)' ) + + def __init__(self): + self.state = self.NONE + self.current = TestCaseData() + self.conditionParser = TestConditionResultParser() + + def parseResultLine(self,line): + result = None + if self.state == self.NONE: + if line.startswith("-------------------------------------------------------------------------------"): + self.state = self.TEST_CASE_NAME_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CASE_NAME_EXPECTED: + if len(line): + self.current.name = line.strip() + self.current.nameParts.append(line.strip()) + self.state = self.TEST_CASE_NAME + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CASE_NAME: + if line.startswith("-------------------------------------------------------------------------------"): + self.state = self.TEST_CLASS_EXPECTED + elif line.startswith(" "): + #print "***SECTION: ",line + self.current.sections.append(line[2:]) + elif len(line): + if len(self.current.name) > 0: + self.current.name += line.strip() + else: + self.current.name = line.strip() + self.current.nameParts.append(line.strip()) + elif self.state == self.TEST_CLASS_EXPECTED: + m = self.testFilenameParser.match(line) + if m: + self.current.filenamePrefix = m.group(1).strip() + self.current.filename = m.group(2).strip() + self.current.lineNumber = m.group(3).strip() + self.state = self.END_OF_CLASS_NAME_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.END_OF_CLASS_NAME_EXPECTED: + if line.startswith("..............................................................................."): + self.state = self.TEST_CONDITION_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CONDITION_EXPECTED: + #print "**** LINE " + line + condition = self.conditionParser.parseResultLine(line) + if isinstance(condition, TestConditionData): + #print "**** CASE " + repr(condition) + self.current.conditions.append(condition) + if line.startswith("-------------------------------------------------------------------------------"): + result = self.current + self.current = TestCaseData() + self.state = self.TEST_CASE_NAME_EXPECTED + elif line.startswith("==============================================================================="): + result = self.current + self.current = TestCaseData() + self.state = self.NONE + + return result + diff --git a/scripts/catch_test_run.py b/scripts/catch_test_run.py new file mode 100644 index 00000000..833d68a0 --- /dev/null +++ b/scripts/catch_test_run.py @@ -0,0 +1,430 @@ +import re +import os + +from catch_test_case import TestCaseApprovedParser +from catch_test_case import TestCaseResultParser +from catch_test_case import TestCaseData +from catch_conditions import RandomOutput + +class TestRunData: + + def __init__(self): + self.appname = "" + self.version = "" + self.testcases = [] + self.results = "" + self.output = [] + self.outputLine = 0 + self.writtenOutput = False + self.sysout = [] + self.syserr = [] + self.errors = "" + self.failures = "" + self.tests = "" + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __repr__(self): + result = "[" + self.appname + ", " + self.version + " [ " + suffix = "" + for case in self.testcases: + result += suffix + result += repr(case) + suffix = ", " + result += " ]" + result += self.results + result += " ]" + return result + + def empty(self): + if len(self.appname): + return False + return True + + def generateApprovedLines(self): + if self.empty(): + raise Exception("Empty test run..." + repr(self)) + lines = [] + self.writtenOutput = False + if not(self.writtenOutput) and len(self.output) > 0 and self.outputLine == 0: + lines += self.output + self.writtenOutput = True + if len(self.appname): + lines.append("") + lines.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + lines.append(self.appname + " is a " + self.version + " host application.") + lines.append("Run with -? for options") + lines.append("") + for case in self.testcases: + lines += case.generateApprovedLines() + if not(self.writtenOutput) and len(self.output) > 0 and len(lines) >= self.outputLine: + lines += self.output + self.writtenOutput = True + lines.append("===============================================================================") + lines.append(self.results) + lines.append("") + + return lines + + def generateSortedApprovedLines(self): + if self.empty(): + raise Exception("Empty test run..." + repr(self)) + lines = [] + self.writtenOutput = False + if not(self.writtenOutput) and len(self.output) > 0 and self.outputLine == 0: + lines += self.output + self.writtenOutput = True + if len(self.appname): + lines.append("") + lines.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + lines.append(self.appname + " is a " + self.version + " host application.") + lines.append("Run with -? for options") + lines.append("") + sortedTestcases = sorted(self.testcases, key=lambda x: x.name, reverse=False) + for case in sortedTestcases: + lines += case.generateApprovedLines() + if not(self.writtenOutput) and len(self.output) > 0 and len(lines) >= self.outputLine: + lines += self.output + self.writtenOutput = True + lines.append("===============================================================================") + lines.append(self.results) + lines.append("") + + return lines + + def generateResultLines(self): + if self.empty(): + raise Exception("Empty test run..." + repr(self)) + lines = [] + self.writtenOutput = False + if not(self.writtenOutput) and len(self.output) > 0 and self.outputLine == 0: + lines += self.output + self.writtenOutput = True + if len(self.appname): + lines.append("") + lines.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + lines.append(self.appname + " is a " + self.version + " host application.") + lines.append("Run with -? for options") + lines.append("") + for case in self.testcases: + lines += case.generateResultLines() + if not(self.writtenOutput) and len(self.output) > 0 and len(lines) >= self.outputLine: + lines += self.output + self.writtenOutput = True + lines.append("===============================================================================") + lines.append(self.results) + lines.append("") + + return lines + + def generateUnapprovedLines(self, outputLine): + if self.empty(): + raise Exception("Empty test run..." + repr(self)) + lines = [] + self.writtenOutput = False + #print "U:",outputLine,",",self.output + if not(self.writtenOutput) and len(self.output) > 0 and outputLine == 0: + lines += self.output + self.writtenOutput = True + if len(self.appname): + lines.append("") + lines.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + lines.append("CatchSelfTest" + " is a " + "" + " host application.") + lines.append("Run with -? for options") + lines.append("") + for case in self.testcases: + lines += case.generateUnapprovedLines() + if not(self.writtenOutput) and len(self.output) > 0 and len(lines) >= outputLine: + lines += self.output + self.writtenOutput = True + lines.append("===============================================================================") + lines.append(self.results) + lines.append("") + + return lines + + def generateSortedUnapprovedLines(self, outputLine): + if self.empty(): + raise Exception("Empty test run..." + repr(self)) + lines = [] + self.writtenOutput = False + #print "U:",outputLine,",",self.output + if not(self.writtenOutput) and len(self.output) > 0 and outputLine == 0: + lines += self.output + self.writtenOutput = True + if len(self.appname): + lines.append("") + lines.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + lines.append("CatchSelfTest" + " is a " + "" + " host application.") + lines.append("Run with -? for options") + lines.append("") + sortedTestcases = sorted(self.testcases, key=lambda x: x.name, reverse=False) + for case in sortedTestcases: + lines += case.generateUnapprovedLines() + if not(self.writtenOutput) and len(self.output) > 0 and len(lines) >= outputLine: + lines += self.output + self.writtenOutput = True + lines.append("===============================================================================") + lines.append(self.results) + lines.append("") + + return lines + + def generateSortedUnapprovedJunit(self): + lines = [] + #print "U:",outputLine,",",self.output + lines.append("") + l = " " + lines.append(l) + sortedTestcases = sorted(self.testcases, key=lambda x: x.classname, reverse=False) + sortedTestcases = sorted(sortedTestcases, key=lambda x: x.name, reverse=False) + #sortedTestcases = self.testcases + for case in sortedTestcases: + lines += case.generateUnapprovedJunit() + + if len(self.sysout) > 0: + lines.append(" ") + for l in self.sysout: + lines.append(l) + lines.append(" ") + if len(self.syserr) > 0: + lines.append(" ") + for l in self.syserr: + lines.append(l) + lines.append(" ") + + lines.append(" ") + lines.append("") + return lines + + def generateSortedUnapprovedXml(self): + lines = [] + #print "U:",outputLine,",",self.output + lines.append("") + lines.append(" ") + + sortedTestcases = sorted(self.testcases, key=lambda x: x.classname, reverse=False) + sortedTestcases = sorted(sortedTestcases, key=lambda x: x.name, reverse=False) + #sortedTestcases = self.testcases + for case in sortedTestcases: + lines += case.generateUnapprovedXml() + + l = "" + lines.append(" " + l) + lines.append(" ") + lines.append(" " + l) + lines.append("") + return lines + + def addTestCase(self, name): + testcase = TestCaseData() + testcase.name = name + testcase.nameParts.append(name) + self.testcases.append(testcase) + return testcase + + def addClassTestCase(self, cls, name): + testcase = TestCaseData() + testcase.classname = cls + testcase.name = name + testcase.nameParts.append(name) + self.testcases.append(testcase) + return testcase + + def addSysout(self, output): + self.sysout = output + + def addSyserr(self, output): + self.syserr = output + +class TestRunApprovedParser: + NONE = 0 + VERSION_EXPECTED = 1 + TEST_CASE_EXPECTED = 2 + END_RUN_INFO = 3 + + versionParser = re.compile( r'(.*)is a (*).*' ) + + def __init__(self): + self.state = self.NONE + self.current = TestRunData() + self.testcaseParser = TestCaseApprovedParser() + self.lineNumber = 0 + + def parseApprovedLine(self,line): + result = None + if self.state == self.NONE: + if line.startswith("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"): + self.state = self.VERSION_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.VERSION_EXPECTED: + m = self.versionParser.match(line) + if m: + self.current.appname = m.group(1).strip() + self.current.version = m.group(2).strip() + self.state = self.TEST_CASE_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CASE_EXPECTED: + if line == "Run with -? for options": + pass + else: + testcase = None + try: + testcase = self.testcaseParser.parseApprovedLine(line) + except RandomOutput as e: + #print "E:", self.lineNumber, ", ",e.output + self.current.output = e.output + self.current.outputLine = self.lineNumber - 10 + + if isinstance(testcase, TestCaseData): + self.current.testcases.append(testcase) + if line.startswith("==============================================================================="): + self.state = self.END_RUN_INFO + elif self.state == self.END_RUN_INFO: + if len(line): + self.current.results = line.strip() + result = self.current + + self.lineNumber += 1 + return result + +class TestRunApprovedHandler: + + def __init__(self, filePath): + rawFile = open( filePath, 'r' ) + parser = TestRunApprovedParser() + lineNumber = 0 + self.current = None + for line in rawFile: + line = line.rstrip() + #print "L:", lineNumber, "'",line,"'" + result = parser.parseApprovedLine(line) + if isinstance(result, TestRunData): + self.current = result + lineNumber += 1 + if not(isinstance(self.current, TestRunData) ): + raise Exception("File could not be parsed: '" + filePath + "'") + + def writeRawFile(self,filePath): + rawWriteFile = open( filePath, 'wb' ) + lines = self.current.generateApprovedLines() + for line in lines: + rawWriteFile.write(line + "\n") + + def writeSortedRawFile(self,filePath): + rawWriteFile = open( filePath, 'wb' ) + lines = self.current.generateSortedApprovedLines() + for line in lines: + rawWriteFile.write(line + "\n") + +class TestRunResultParser: + NONE = 0 + VERSION_EXPECTED = 1 + TEST_CASE_EXPECTED = 2 + END_RUN_INFO = 3 + + versionParser = re.compile( r'(.*)is a (Catch v[0-9]*.[0-9]* b[0-9]*).*' ) + + def __init__(self): + self.state = self.NONE + self.current = TestRunData() + self.testcaseParser = TestCaseResultParser() + self.lineNumber = 0 + + def parseResultLine(self,line): + result = None + if self.state == self.NONE: + if line.startswith("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"): + self.state = self.VERSION_EXPECTED + elif len(line): + self.current.output.append(line.strip()) + if len(self.current.output) == 10: + if (self.current.output[0] == "Message from section one" and self.current.output[1] == "Message from section two" and + self.current.output[2] == "Some information" and self.current.output[3] == "An error" and + self.current.output[4] == "Message from section one" and self.current.output[5] == "Message from section two" and + self.current.output[6] == "Some information" and self.current.output[7] == "An error" and + self.current.output[8] == "hello" and self.current.output[9] == "hello" ): + + self.current.outputLine = self.lineNumber - 9 + + elif self.state == self.VERSION_EXPECTED: + m = self.versionParser.match(line) + if m: + self.current.appname = m.group(1).strip() + self.current.version = m.group(2).strip() + self.state = self.TEST_CASE_EXPECTED + elif len(line): + raise Exception("Unknown parse line: '" + line + "'") + elif self.state == self.TEST_CASE_EXPECTED: + if line == "Run with -? for options": + pass + else: + testcase = None + try: + testcase = self.testcaseParser.parseResultLine(line) + except RandomOutput as e: + #print "E:", self.lineNumber, ", ",e.output + self.current.output = e.output + self.current.outputLine = self.lineNumber - 10 + + if isinstance(testcase, TestCaseData): + self.current.testcases.append(testcase) + if line.startswith("==============================================================================="): + self.state = self.END_RUN_INFO + elif self.state == self.END_RUN_INFO: + if len(line): + self.current.results = line.strip() + result = self.current + + self.lineNumber += 1 + return result + +class TestRunResultHandler: + + def __init__(self, filePath): + rawFile = open( filePath, 'r' ) + parser = TestRunResultParser() + lineNumber = 0 + self.current = None + for line in rawFile: + line = line.rstrip() + #print "L:", lineNumber, "'",line,"'" + result = parser.parseResultLine(line) + if isinstance(result, TestRunData): + self.current = result + lineNumber += 1 + if not(isinstance(self.current, TestRunData) ): + raise Exception("File could not be parsed: '" + filePath + "'") + + def writeRawFile(self,filePath): + rawWriteFile = open( filePath, 'wb' ) + lines = self.current.generateResultLines() + for line in lines: + rawWriteFile.write(line + os.linesep) + + def writeUnapprovedFile(self,filePath,outputLine): + rawWriteFile = open( filePath, 'wb' ) + lines = self.current.generateUnapprovedLines(outputLine) + for line in lines: + rawWriteFile.write(line + "\n") + def writeSortedUnapprovedFile(self,filePath,outputLine): + rawWriteFile = open( filePath, 'wb' ) + lines = self.current.generateSortedUnapprovedLines(outputLine) + for line in lines: + rawWriteFile.write(line + "\n") diff --git a/scripts/test_conditions.py b/scripts/test_conditions.py new file mode 100644 index 00000000..2b11bf7b --- /dev/null +++ b/scripts/test_conditions.py @@ -0,0 +1,698 @@ +import unittest +import catch_conditions + +from catch_conditions import TestConditionApprovedParser +from catch_conditions import TestConditionResultParser +from catch_conditions import TestConditionData +from catch_conditions import RandomOutput + +class ConditionTest(unittest.TestCase): + + def testConditionEquality(self): + c1 = TestConditionData() + c2 = TestConditionData() + c1.state = TestConditionData.CONDITION + c2.state = TestConditionData.EXPANSION + c1.filenamePrefix = "..\\..\\Test" + c2.filenamePrefix = "..\\..\\Test" + self.assertTrue(c1 == c2) + c2.filenamePrefix = "..\\..\\Junk" + self.assertFalse(c1 == c2) + self.assertTrue(c1 != c2) + + def testEndOfTestRunIsFound(self): + obj = TestConditionApprovedParser() + line = "===============================================================================" + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue(obj.current.empty()) + + def testEndOfTestCaseIsFound(self): + obj = TestConditionApprovedParser() + line = "-------------------------------------------------------------------------------" + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue(obj.current.empty()) + + def testFailedConditionIsFound(self): + obj = TestConditionApprovedParser() + line = "ClassTests.cpp:: FAILED:" + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue( not(obj.current.empty()) ) + + def testOtherConditionIsFound(self): + obj = TestConditionApprovedParser() + line = "ClassTests.cpp::" + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue( not(obj.current.empty()) ) + + def testFailedConditionSetsReason(self): + obj = TestConditionApprovedParser() + line = "ClassTests.cpp:: FAILED:" + result = obj.parseApprovedLine(line) + self.assertTrue(obj.current.reason == "FAILED") + self.assertTrue(obj.current.filename == "ClassTests.cpp") + self.assertTrue(obj.current.lineNumber == "line number") + + def testOtherConditionSetsFileNameAndLine(self): + obj = TestConditionApprovedParser() + line = "MessageTests.cpp::" + result = obj.parseApprovedLine(line) + self.assertTrue(obj.current.filename == "MessageTests.cpp") + self.assertTrue(obj.current.lineNumber == "line number") + + def testFailedConditionSetsCondition(self): + obj = TestConditionApprovedParser() + lines = ["ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + ""] + for line in lines: + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + + self.assertTrue(obj.current.condition == "REQUIRE( s == \"world\" )") + newLines = obj.current.generateApprovedLines() + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testExpansionConditionReturnsExpansion(self): + obj = TestConditionApprovedParser() + lines = ["ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:" ] + for line in lines: + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + + def testExpansionSetsExpansion(self): + obj = TestConditionApprovedParser() + lines = [ "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " 1 == 2", + "", + "-------------------------------------------------------------------------------" + ] + for line in lines: + result = obj.parseApprovedLine(line) + #print lines + self.assertTrue(isinstance(result, TestConditionData)) + self.assertTrue(len(result.expansion) == 1) + self.assertTrue(result.expansion[0] == "1 == 2") + newLines = result.generateApprovedLines() + newLines.append("-------------------------------------------------------------------------------") + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testTwoConditions(self): + obj = TestConditionApprovedParser() + lines = [ "ConditionTests.cpp:: FAILED:", + " CHECK( data.int_seven == 6 )", + "with expansion:", + " 7 == 6", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.int_seven == 8 )", + "with expansion:", + " 7 == 8", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testSuccessConditions(self): + obj = TestConditionApprovedParser() + lines = [ "ApproxTests.cpp::", + "PASSED:", + " REQUIRE( d == Approx( 1.23 ) )", + "with expansion:", + " 1.23 == Approx( 1.23 )", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print result + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testConditionsWithoutExpansion(self): + obj = TestConditionApprovedParser() + lines = [ "ConditionTests.cpp:: FAILED:", + " CHECK( false != false )", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( true != true )", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( !true )", + "with expansion:", + " false", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testExceptionsExplicit(self): + obj = TestConditionApprovedParser() + lines = [ "ExceptionTests.cpp:: FAILED:", + " CHECK_THROWS_AS( thisThrows() )", + "due to unexpected exception with message:", + " expected exception", + "", + "ExceptionTests.cpp:: FAILED:", + " CHECK_THROWS_AS( thisDoesntThrow() )", + "because no exception was thrown where one was expected:", + "", + "ExceptionTests.cpp:: FAILED:", + " CHECK_NOTHROW( thisThrows() )", + "due to unexpected exception with message:", + " expected exception", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testExceptionsImplicit(self): + obj = TestConditionApprovedParser() + lines = [ "ExceptionTests.cpp:: FAILED:", + "due to unexpected exception with message:", + " unexpected exception", + "", + "ExceptionTests.cpp:: FAILED:", + " {Unknown expression after the reported line}", + "due to unexpected exception with message:", + " unexpected exception", + "", + "ExceptionTests.cpp:: FAILED:", + "due to unexpected exception with message:", + " 3.14", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testWarning(self): + obj = TestConditionApprovedParser() + lines = [ "MessageTests.cpp::", + "warning:", + " this is a warning", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMessages1(self): + obj = TestConditionApprovedParser() + lines = [ "MessageTests.cpp:: FAILED:", + " REQUIRE( a == 1 )", + "with expansion:", + " 2 == 1", + "with messages:", + " this message should be logged", + " so should this", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMessagesExplicitFail(self): + obj = TestConditionApprovedParser() + lines = [ "MessageTests.cpp:: FAILED:", + "explicitly with message:", + " This is a failure", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMessagesOutput(self): + obj = TestConditionApprovedParser() + lines = [ "MessageTests.cpp:: FAILED:", + "explicitly with message:", + " Message from section two", + "", + "Message from section one", + "Message from section two", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMultiMessages(self): + obj = TestConditionApprovedParser() + lines = [ "MessageTests.cpp:: FAILED:", + " REQUIRE( i < 10 )", + "with expansion:", + " 10 < 10", + "with messages:", + " current counter 10", + " i := 10", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMiscMessages(self): + obj = TestConditionApprovedParser() + lines = [ "MiscTests.cpp:: FAILED:", + " CHECK( ( fib[i] % 2 ) == 0 )", + "with expansion:", + " 1 == 0", + "with message:", + " Testing if fib[0] (1) is even", + "", + "MiscTests.cpp:: FAILED:", + " CHECK( ( fib[i] % 2 ) == 0 )", + "with expansion:", + " 1 == 0", + "with message:", + " Testing if fib[1] (1) is even", + "", + "MiscTests.cpp:: FAILED:", + " CHECK( ( fib[i] % 2 ) == 0 )", + "with expansion:", + " 1 == 0", + "with message:", + " Testing if fib[3] (3) is even", + "", + "MiscTests.cpp:: FAILED:", + " CHECK( ( fib[i] % 2 ) == 0 )", + "with expansion:", + " 1 == 0", + "with message:", + " Testing if fib[4] (5) is even", + "", + "MiscTests.cpp:: FAILED:", + " CHECK( ( fib[i] % 2 ) == 0 )", + "with expansion:", + " 1 == 0", + "with message:", + " Testing if fib[6] (13) is even", + "", + "MiscTests.cpp:: FAILED:", + " CHECK( ( fib[i] % 2 ) == 0 )", + "with expansion:", + " 1 == 0", + "with message:", + " Testing if fib[7] (21) is even", + "", + "Some information", + "An error", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testRandomOutput(self): + obj = TestConditionApprovedParser() + lines = [ "MiscTests.cpp:: FAILED:", + "explicitly with message:", + " to infinity and beyond", + "", + "Message from section one", + "Message from section two", + "Some information", + "An error", + "Message from section one", + "Message from section two", + "Some information", + "An error", + "hello", + "hello", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + try: + result = obj.parseApprovedLine(line) + except RandomOutput as e: + randomOutput = e.output + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + self.assertTrue( len(randomOutput) == 10) + newLines += randomOutput + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMultiLineWarning(self): + obj = TestConditionApprovedParser() + lines = [ "TrickyTests.cpp::", + "warning:", + " Uncomment the code in this test to check that it gives a sensible compiler", + " error", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMultiMessagesAfterCondition(self): + obj = TestConditionApprovedParser() + lines = [ "MiscTests.cpp:: FAILED:", + " REQUIRE( false )", + "with messages:", + " hi", + " i := 7", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + randomOutput = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testNoAssertions(self): + obj = TestConditionApprovedParser() + lines = [ "", + "No assertions in test case './succeeding/exceptions/implicit'", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + randomOutput = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testNoAssertionsWithOutput(self): + obj = TestConditionApprovedParser() + lines = [ "", + "No assertions in section 'one'", + "", + "Message from section two", + "-------------------------------------------------------------------------------" + ] + newLines = [] + randomOutput = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testFailedButOk(self): + obj = TestConditionApprovedParser() + lines = [ "MessageTests.cpp::", + "FAILED - but was ok:", + " CHECK_NOFAIL( 1 == 2 )", + "", + "", + "No assertions in test case './succeeding/nofail'", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + randomOutput = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMultiLineExpansion(self): + obj = TestConditionApprovedParser() + lines = [ "MiscTests.cpp::", + "PASSED:", + " CHECK_THAT( testStringForMatching() AllOf( Catch::Contains( \"string\" ), Catch::Contains( \"abc\" ) ) )", + "with expansion:", + " \"this string contains 'abc' as a substring\" ( contains: \"string\" and", + " contains: \"abc\" )", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + randomOutput = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMultiLineExpansionWithWrap(self): + obj = TestConditionApprovedParser() + lines = [ "TestMain.cpp::", + "PASSED:", + " CHECK( text.toString() == \" one two\n three\n four\" )", + "with expansion:", + " \" one two", + " three", + " four\"", + " ==", + " \" one two", + " three", + " four\"", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + randomOutput = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testMultiLineExpansionWithTruncation(self): + obj = TestConditionApprovedParser() + lines = [ "TestMain.cpp::", + "PASSED:", + " CHECK_THAT( t.toString() EndsWith( \"... message truncated due to excessive size\" ) )", + "with expansion:", + " \"***************************************************************************-", + " ***-", + " ****************************************************************************-", + " **-", + "... message truncated due to excessive size", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateApprovedLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testBasicResultsParser(self): + obj = TestConditionResultParser() + lines = [ "..\..\..\SelfTest\ClassTests.cpp(28): FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseResultLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateResultLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testBasicResultsWarning(self): + obj = TestConditionResultParser() + lines = [ "..\..\..\SelfTest\MessageTests.cpp(17): ", + "warning:", + " this is a warning", + "", + "-------------------------------------------------------------------------------" + ] + newLines = [] + for line in lines: + result = obj.parseResultLine(line) + if isinstance(result, TestConditionData): + #print result + newLines += result.generateResultLines() + + newLines.append("-------------------------------------------------------------------------------") + #print lines + #print newLines + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/scripts/test_test_case.py b/scripts/test_test_case.py new file mode 100644 index 00000000..6976453c --- /dev/null +++ b/scripts/test_test_case.py @@ -0,0 +1,341 @@ +import unittest + +from catch_test_case import TestCaseApprovedParser +from catch_test_case import TestCaseResultParser +from catch_test_case import TestCaseData +from catch_conditions import TestConditionData + +class TestCaseTest(unittest.TestCase): + + def testTestCaseEquality(self): + c1 = TestConditionData() + c2 = TestConditionData() + c1.filenamePrefix = "..\\..\\Test" + c2.filenamePrefix = "..\\..\\Junk" + t1 = TestCaseData() + t2 = TestCaseData() + t1.name = "Test 1" + t2.name = "Test 1" + t1.conditions.append(c1) + t1.conditions.append(c2) + t2.conditions.append(c1) + t2.conditions.append(c2) + self.assertTrue(t1 == t2) + + c3 = TestConditionData() + c3.filenamePrefix = "..\\..\\Fail" + t2.conditions.append(c3) + self.assertFalse(t1 == t2) + + t1.conditions.append(c3) + self.assertTrue(t1 == t2) + + t2.name = "Test 2" + self.assertFalse(t1 == t2) + + def testEndOfTestCaseIsFound(self): + obj = TestCaseApprovedParser() + line = "-------------------------------------------------------------------------------" + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue(obj.current.empty()) + + def testTestCaseNameIsFound(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------" ] + for line in lines: + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue(obj.current.name == "./failing/TestClass/failingCase") + self.assertTrue( not(obj.current.empty()) ) + + def testTestCaseClassIsFound(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "..............................................................................." ] + for line in lines: + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue(obj.current.filename == "ClassTests.cpp") + self.assertTrue(obj.current.lineNumber == "line number") + + def testPartialConditionRequiresMoreData(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "ClassTests.cpp:: FAILED:" ] + for line in lines: + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue(len(obj.current.conditions) == 0) + + def testTestCaseConditionIsFound(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\""] + for line in lines: + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + self.assertTrue(not(obj.conditionParser.current.empty())) + self.assertTrue(obj.conditionParser.current.reason == "FAILED") + + def testTestCaseConditionIsInsertedIntoList(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "-------------------------------------------------------------------------------"] + for line in lines: + result = obj.parseApprovedLine(line) + + self.assertTrue(isinstance(result, TestCaseData)) + self.assertTrue(len(result.conditions) > 0) + self.assertTrue(result.conditions[0].filename == "ClassTests.cpp" ) + self.assertTrue(result.conditions[0].lineNumber == "line number" ) + self.assertTrue(result.conditions[0].reason == "FAILED" ) + newLines = result.generateApprovedLines() + newLines.append("-------------------------------------------------------------------------------") + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testTwoTestCases(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "-------------------------------------------------------------------------------", + "./failing/Fixture/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( m_a == 2 )", + "with expansion:", + " 1 == 2", + "", + "===============================================================================" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestCaseData): + newLines += result.generateApprovedLines() + + newLines.append("===============================================================================") + #for line in newLines: + # print line + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testTestCaseMultiConditionMatches(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/conditions/equality", + "-------------------------------------------------------------------------------", + "ConditionTests.cpp:", + "...............................................................................", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.int_seven == 6 )", + "with expansion:", + " 7 == 6", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.int_seven == 8 )", + "with expansion:", + " 7 == 8", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.int_seven == 0 )", + "with expansion:", + " 7 == 0", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.float_nine_point_one == Approx( 9.11f ) )", + "with expansion:", + " 9.1 == Approx( 9.11 )", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.float_nine_point_one == Approx( 9.0f ) )", + "with expansion:", + " 9.1 == Approx( 9 )", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.float_nine_point_one == Approx( 1 ) )", + "with expansion:", + " 9.1 == Approx( 1 )", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.float_nine_point_one == Approx( 0 ) )", + "with expansion:", + " 9.1 == Approx( 0 )", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.double_pi == Approx( 3.1415 ) )", + "with expansion:", + " 3.1415926535 == Approx( 3.1415 )", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.str_hello == \"goodbye\" )", + "with expansion:", + " \"hello\" == \"goodbye\"", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.str_hello == \"hell\" )", + "with expansion:", + " \"hello\" == \"hell\"", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.str_hello == \"hello1\" )", + "with expansion:", + " \"hello\" == \"hello1\"", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( data.str_hello.size() == 6 )", + "with expansion:", + " 5 == 6", + "", + "ConditionTests.cpp:: FAILED:", + " CHECK( x == Approx( 1.301 ) )", + "with expansion:", + " 1.3 == Approx( 1.301 )", + "", + "===============================================================================" + ] + + for line in lines: + result = obj.parseApprovedLine(line) + + self.assertTrue(isinstance(result, TestCaseData)) + newLines = result.generateApprovedLines() + newLines.append("===============================================================================") + #for line in newLines: + # print line + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testOneSection(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/message/sections", + " one", + "-------------------------------------------------------------------------------", + "MessageTests.cpp:", + "...............................................................................", + "", + "MessageTests.cpp:: FAILED:", + "explicitly with message:", + " Message from section one", + "", + "===============================================================================" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestCaseData): + newLines += result.generateApprovedLines() + + newLines.append("===============================================================================") + #for line in newLines: + # print line + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testOneSection(self): + obj = TestCaseApprovedParser() + lines = [ "-------------------------------------------------------------------------------", + "Comparisons between unsigned ints and negative signed ints match c++ standard", + "behaviour", + "-------------------------------------------------------------------------------", + "ConditionTests.cpp:", + "...............................................................................", + "", + "ConditionTests.cpp::", + "PASSED:", + " CHECK( ( -1 > 2u ) )", + "with expansion:", + " true", + "", + "ConditionTests.cpp::", + "PASSED:", + " CHECK( -1 > 2u )", + "with expansion:", + " -1 > 2", + "", + "===============================================================================" + ] + newLines = [] + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestCaseData): + newLines += result.generateApprovedLines() + + newLines.append("===============================================================================") + #for line in newLines: + # print line + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testCaseBasicResults(self): + obj = TestCaseResultParser() + lines = [ "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "..\..\..\SelfTest\ClassTests.cpp(34)", + "...............................................................................", + "", + "..\..\..\SelfTest\ClassTests.cpp(28): FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "===============================================================================" + ] + + newLines = [] + for line in lines: + result = obj.parseResultLine(line) + if isinstance(result, TestCaseData): + newLines += result.generateResultLines() + + newLines.append("===============================================================================") + #for line in newLines: + # print line + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/scripts/test_test_run.py b/scripts/test_test_run.py new file mode 100644 index 00000000..c75c35e4 --- /dev/null +++ b/scripts/test_test_run.py @@ -0,0 +1,334 @@ +import unittest +import catch_test_case + +from catch_test_run import TestRunData +from catch_test_run import TestRunApprovedParser +from catch_test_run import TestRunResultParser +from catch_test_case import TestCaseData +from catch_conditions import TestConditionData + +class TestCaseTest(unittest.TestCase): + + def testTestRunEquality(self): + c1 = TestConditionData() + c2 = TestConditionData() + c1.filenamePrefix = "..\\..\\Test" + c2.filenamePrefix = "..\\..\\Junk" + t1 = TestCaseData() + t2 = TestCaseData() + t1.name = "Test 1" + t2.name = "Test 1" + t1.conditions.append(c1) + t1.conditions.append(c2) + t2.conditions.append(c1) + t2.conditions.append(c2) + r1 = TestRunData() + r2 = TestRunData() + r1.appname = "One" + r2.appname = "One" + self.assertTrue(r1 == r2) + + r1.testcases.append(t1) + self.assertFalse(r1 == r2) + + r2.testcases.append(t2) + self.assertTrue(r1 == r2) + + c3 = TestConditionData() + c3.filenamePrefix = "..\\..\\Fail" + t2.conditions.append(c3) + self.assertFalse(r1 == r2) + + def testStartOfTestRunIsFound(self): + obj = TestRunApprovedParser() + line = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + result = obj.parseApprovedLine(line) + self.assertTrue(result == None) + + def testTestRunVersionIsSet(self): + obj = TestRunApprovedParser() + lines = [ "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "CatchSelfTest is a host application.", + "Run with -? for options" ] + for line in lines: + result = obj.parseApprovedLine(line) + #print obj + self.assertTrue(result == None) + self.assertTrue(obj.current.appname == "CatchSelfTest") + self.assertTrue(obj.current.version == "") + + def testTestRunParsesTestCase(self): + obj = TestRunApprovedParser() + lines = [ "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "CatchSelfTest is a host application.", + "Run with -? for options", + "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:" + " \"hello\" == \"world\""] + for line in lines: + result = obj.parseApprovedLine(line) + #print obj + self.assertTrue(obj.testcaseParser.conditionParser.current.filename == "ClassTests.cpp" ) + self.assertTrue(obj.testcaseParser.conditionParser.current.lineNumber == "line number" ) + self.assertTrue(obj.testcaseParser.conditionParser.current.reason == "FAILED" ) + + def testTestRunAddsTestCase(self): + obj = TestRunApprovedParser() + lines = [ "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "CatchSelfTest is a host application.", + "Run with -? for options", + "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:" + " \"hello\" == \"world\"", + "", + "-------------------------------------------------------------------------------" + ] + for line in lines: + result = obj.parseApprovedLine(line) + #print obj + self.assertTrue( result == None ) + self.assertTrue( len(obj.current.testcases) == 1 ) + self.assertTrue(obj.current.testcases[0].filename == "ClassTests.cpp" ) + self.assertTrue(obj.current.testcases[0].lineNumber == "line number" ) + self.assertTrue( len(obj.current.testcases[0].conditions) == 1 ) + self.assertTrue(obj.current.testcases[0].conditions[0].filename == "ClassTests.cpp" ) + self.assertTrue(obj.current.testcases[0].conditions[0].lineNumber == "line number" ) + self.assertTrue(obj.current.testcases[0].conditions[0].reason == "FAILED" ) + + def testTestRunParsesTwoTestCases(self): + obj = TestRunApprovedParser() + lines = [ "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "CatchSelfTest is a host application.", + "Run with -? for options", + "", + "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "-------------------------------------------------------------------------------", + "./failing/Fixture/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( m_a == 2 )", + "with expansion:", + " 1 == 2", + "", + "===============================================================================", + "122 test cases - 35 failed (753 assertions - 90 failed)", + "" + ] + testRun = None + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestRunData): + testRun = result + + self.assertTrue( isinstance(testRun, TestRunData) ) + newLines = testRun.generateApprovedLines() + #for line in newLines: + # print "L:",line + #print len(lines),",",len(newLines) + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testTestRunWithRandomOutput(self): + obj = TestRunApprovedParser() + lines = [ "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "CatchSelfTest is a host application.", + "Run with -? for options", + "", + "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "MiscTests.cpp:: FAILED:", + "explicitly with message:", + " to infinity and beyond", + "", + "Message from section one", + "Message from section two", + "Some information", + "An error", + "Message from section one", + "Message from section two", + "Some information", + "An error", + "hello", + "hello", + "===============================================================================", + "122 test cases - 35 failed (753 assertions - 90 failed)", + "" + ] + testRun = None + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestRunData): + testRun = result + + self.assertTrue( isinstance(testRun, TestRunData) ) + #print "O:",result.outputLine + self.assertTrue( testRun.outputLine == 14 ) + newLines = testRun.generateApprovedLines() + #for line in newLines: + # print line + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testRunBasicResult(self): + obj = TestRunResultParser() + lines = [ "Message from section one", + "Message from section two", + "Some information", + "An error", + "Message from section one", + "Message from section two", + "Some information", + "An error", + "hello", + "hello", + "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "TestCatch.exe is a Catch v1.0 b13 host application.", + "Run with -? for options", + "", + "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "..\..\..\SelfTest\ClassTests.cpp(34)", + "...............................................................................", + "", + "..\..\..\SelfTest\ClassTests.cpp(28): FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "===============================================================================", + "122 test cases - 35 failed (753 assertions - 90 failed)", + "" + ] + + testRun = None + for line in lines: + result = obj.parseResultLine(line) + if isinstance(result, TestRunData): + testRun = result + + self.assertTrue( isinstance(testRun, TestRunData) ) + #print "O:",testRun.outputLine + self.assertTrue( testRun.outputLine == 0 ) + newLines = testRun.generateResultLines() + #for line in newLines: + # print line + self.assertTrue( len(lines) == len(newLines) ) + self.assertTrue( lines == newLines ) + + def testTestRunSorted(self): + obj = TestRunApprovedParser() + lines = [ "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "CatchSelfTest is a host application.", + "Run with -? for options", + "", + "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "-------------------------------------------------------------------------------", + "./failing/Fixture/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( m_a == 2 )", + "with expansion:", + " 1 == 2", + "", + "===============================================================================", + "122 test cases - 35 failed (753 assertions - 90 failed)", + "" + ] + testRun = None + for line in lines: + result = obj.parseApprovedLine(line) + if isinstance(result, TestRunData): + testRun = result + + self.assertTrue( isinstance(testRun, TestRunData) ) + newLines = testRun.generateSortedApprovedLines() + #for line in newLines: + # print "L:",line + #print len(lines),",",len(newLines) + expectedLines = [ "", + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", + "CatchSelfTest is a host application.", + "Run with -? for options", + "", + "-------------------------------------------------------------------------------", + "./failing/Fixture/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( m_a == 2 )", + "with expansion:", + " 1 == 2", + "", + "-------------------------------------------------------------------------------", + "./failing/TestClass/failingCase", + "-------------------------------------------------------------------------------", + "ClassTests.cpp:", + "...............................................................................", + "", + "ClassTests.cpp:: FAILED:", + " REQUIRE( s == \"world\" )", + "with expansion:", + " \"hello\" == \"world\"", + "", + "===============================================================================", + "122 test cases - 35 failed (753 assertions - 90 failed)", + "" + ] + self.assertTrue( len(expectedLines) == len(newLines) ) + self.assertTrue( expectedLines == newLines ) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file