mirror of
https://github.com/catchorg/Catch2.git
synced 2024-11-04 05:09:53 +01:00
ac93f19437
Instead of redoing the whole line where path was found, only the directory part of the path is removed, instead of removing all of the line before the path starts. This results in slight change in how junit and sonarqube approvals come out, and significant change in how TeamCity reporter approvals come out. This latter difference is the reason for the change, as now the lines with `testFailed` and `testIgnored` messages are not completely butchered.
244 lines
8.6 KiB
Python
Executable File
244 lines
8.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import io
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import re
|
|
import difflib
|
|
import shutil
|
|
|
|
import scriptCommon
|
|
from scriptCommon import catchPath
|
|
|
|
if os.name == 'nt':
|
|
# Enable console colours on windows
|
|
os.system('')
|
|
|
|
rootPath = os.path.join(catchPath, 'tests/SelfTest/Baselines')
|
|
# Init so it is guaranteed to fail loudly if the scoping gets messed up
|
|
outputDirPath = None
|
|
|
|
if len(sys.argv) == 3:
|
|
cmdPath = sys.argv[1]
|
|
outputDirBasePath = sys.argv[2]
|
|
outputDirPath = os.path.join(outputDirBasePath, 'ApprovalTests')
|
|
if not os.path.isdir(outputDirPath):
|
|
os.mkdir(outputDirPath)
|
|
else:
|
|
print('Usage: {} path-to-SelfTest-executable path-to-temp-output-dir'.format(sys.argv[0]))
|
|
exit(1)
|
|
|
|
|
|
|
|
def get_rawResultsPath(baseName):
|
|
return os.path.join(outputDirPath, '_{0}.tmp'.format(baseName))
|
|
|
|
def get_baselinesPath(baseName):
|
|
return os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
|
|
|
|
def _get_unapprovedPath(path, baseName):
|
|
return os.path.join(path, '{0}.unapproved.txt'.format(baseName))
|
|
|
|
def get_filteredResultsPath(baseName):
|
|
return _get_unapprovedPath(outputDirPath, baseName)
|
|
|
|
def get_unapprovedResultsPath(baseName):
|
|
return _get_unapprovedPath(rootPath, baseName)
|
|
|
|
langFilenameParser = re.compile(r'(.+\.[ch]pp)')
|
|
filelocParser = re.compile(r'''
|
|
(?P<path_prefix>tests/SelfTest/(?:\w+/)*) # We separate prefix and fname, so that
|
|
(?P<filename>\w+\.tests\.[ch]pp) # we can keep only filename
|
|
(?::|\() # Linux has : as separator between fname and line number, Windows uses (
|
|
(\d*) # line number
|
|
\)? # Windows also uses an ending separator, )
|
|
''', re.VERBOSE)
|
|
lineNumberParser = re.compile(r' line="[0-9]*"')
|
|
hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b')
|
|
# Note: junit must serialize time with 3 (or or less) decimal places
|
|
# before generalizing this parser, make sure that this is checked
|
|
# in other places too.
|
|
junitDurationsParser = re.compile(r' time="[0-9]+\.[0-9]{3}"')
|
|
durationParser = re.compile(r''' duration=['"][0-9]+['"]''')
|
|
timestampsParser = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z')
|
|
versionParser = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+(-\w*\.[0-9]+)?')
|
|
nullParser = re.compile(r'\b(__null|nullptr)\b')
|
|
exeNameParser = re.compile(r'''
|
|
\b
|
|
SelfTest # Expected executable name
|
|
(?:.exe)? # Executable name contains .exe on Windows.
|
|
\b
|
|
''', re.VERBOSE)
|
|
# This is a hack until something more reasonable is figured out
|
|
specialCaseParser = re.compile(r'file\((\d+)\)')
|
|
|
|
sinceEpochParser = re.compile(r'\d+ .+ since epoch')
|
|
|
|
# The weird OR is there to always have at least empty string for group 1
|
|
tapTestNumParser = re.compile(r'^((?:not ok)|(?:ok)|(?:warning)|(?:info)) (\d+) -')
|
|
|
|
overallResult = 0
|
|
|
|
def diffFiles(fileA, fileB):
|
|
with io.open(fileA, 'r', encoding='utf-8', errors='surrogateescape') as file:
|
|
aLines = [line.rstrip() for line in file.readlines()]
|
|
with io.open(fileB, 'r', encoding='utf-8', errors='surrogateescape') as file:
|
|
bLines = [line.rstrip() for line in file.readlines()]
|
|
|
|
shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1]
|
|
shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1]
|
|
|
|
diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0)
|
|
return [line for line in diff if line[0] in ('+', '-')]
|
|
|
|
|
|
def normalizeFilepath(line):
|
|
# Sometimes the path separators used by compiler and Python can differ,
|
|
# so we try to match the path with both forward and backward path
|
|
# separators, to make the paths relative to Catch2 repo root.
|
|
forwardSlashPath = catchPath.replace('\\', '/')
|
|
if forwardSlashPath in line:
|
|
line = line.replace(forwardSlashPath + '/', '')
|
|
backwardSlashPath = catchPath.replace('/', '\\')
|
|
if backwardSlashPath in line:
|
|
line = line.replace(backwardSlashPath + '\\', '')
|
|
|
|
m = langFilenameParser.match(line)
|
|
if m:
|
|
filepath = m.group(0)
|
|
# go from \ in windows paths to /
|
|
filepath = filepath.replace('\\', '/')
|
|
# remove start of relative path
|
|
filepath = filepath.replace('../', '')
|
|
line = line[:m.start()] + filepath + line[m.end():]
|
|
|
|
return line
|
|
|
|
def filterLine(line, isCompact):
|
|
line = normalizeFilepath(line)
|
|
|
|
# strip source line numbers
|
|
# Note that this parser assumes an already normalized filepath from above,
|
|
# and might break terribly if it is moved around before the normalization.
|
|
line = filelocParser.sub('\g<filename>:<line number>', line)
|
|
|
|
line = lineNumberParser.sub(" ", line)
|
|
|
|
if isCompact:
|
|
line = line.replace(': FAILED', ': failed')
|
|
line = line.replace(': PASSED', ': passed')
|
|
|
|
# strip out the test order number in TAP to avoid massive diffs for every change
|
|
line = tapTestNumParser.sub("\g<1> {test-number} -", line)
|
|
|
|
# strip Catch2 version number
|
|
line = versionParser.sub("<version>", line)
|
|
|
|
# replace *null* with 0
|
|
line = nullParser.sub("0", line)
|
|
|
|
# strip executable name
|
|
line = exeNameParser.sub("<exe-name>", line)
|
|
|
|
# strip hexadecimal numbers (presumably pointer values)
|
|
line = hexParser.sub("0x<hex digits>", line)
|
|
|
|
# strip durations and timestamps
|
|
line = junitDurationsParser.sub(' time="{duration}"', line)
|
|
line = durationParser.sub(' duration="{duration}"', line)
|
|
line = timestampsParser.sub('{iso8601-timestamp}', line)
|
|
line = specialCaseParser.sub('file:\g<1>', line)
|
|
line = sinceEpochParser.sub('{since-epoch-report}', line)
|
|
return line
|
|
|
|
|
|
def run_test(baseName, args):
|
|
args[0:0] = [cmdPath]
|
|
if not os.path.exists(cmdPath):
|
|
raise Exception("Executable doesn't exist at " + cmdPath)
|
|
|
|
print(args)
|
|
rawResultsPath = get_rawResultsPath(baseName)
|
|
f = open(rawResultsPath, 'w')
|
|
subprocess.call(args, stdout=f, stderr=f)
|
|
f.close()
|
|
|
|
|
|
def check_outputs(baseName):
|
|
global overallResult
|
|
rawResultsPath = get_rawResultsPath(baseName)
|
|
baselinesPath = get_baselinesPath(baseName)
|
|
filteredResultsPath = get_filteredResultsPath(baseName)
|
|
|
|
rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape')
|
|
filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape')
|
|
for line in rawFile:
|
|
filteredFile.write(filterLine(line, 'compact' in baseName).rstrip() + "\n")
|
|
filteredFile.close()
|
|
rawFile.close()
|
|
|
|
os.remove(rawResultsPath)
|
|
print()
|
|
print(baseName + ":")
|
|
if not os.path.exists(baselinesPath):
|
|
print( 'first approval')
|
|
overallResult += 1
|
|
return
|
|
|
|
diffResult = diffFiles(baselinesPath, filteredResultsPath)
|
|
if diffResult:
|
|
print('\n'.join(diffResult))
|
|
print(" \n****************************\n \033[91mResults differed\033[0m")
|
|
overallResult += 1
|
|
shutil.move(filteredResultsPath, get_unapprovedResultsPath(baseName))
|
|
else:
|
|
os.remove(filteredResultsPath)
|
|
print(" \033[92mResults matched\033[0m")
|
|
|
|
|
|
def approve(baseName, args):
|
|
run_test(baseName, args)
|
|
check_outputs(baseName)
|
|
|
|
|
|
print("Running approvals against executable:")
|
|
print(" " + cmdPath)
|
|
|
|
|
|
base_args = ["--order", "lex", "--rng-seed", "1", "--colour-mode", "none"]
|
|
|
|
## special cases first:
|
|
# Standard console reporter
|
|
approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals] *"] + base_args)
|
|
|
|
# console reporter, include passes, warn about No Assertions, limit failures to first 4
|
|
approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-x", "4"] + base_args)
|
|
|
|
## Common reporter checks: include passes, warn about No Assertions
|
|
reporters = ('console', 'junit', 'xml', 'compact', 'sonarqube', 'tap', 'teamcity', 'automake')
|
|
for reporter in reporters:
|
|
filename = '{}.sw'.format(reporter)
|
|
common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions"] + base_args
|
|
reporter_args = ['-r', reporter]
|
|
approve(filename, common_args + reporter_args)
|
|
|
|
|
|
## All reporters at the same time
|
|
common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions"] + base_args
|
|
filenames = ['{}.sw.multi'.format(reporter) for reporter in reporters]
|
|
reporter_args = []
|
|
for reporter, filename in zip(reporters, filenames):
|
|
reporter_args += ['-r', '{}::out={}'.format(reporter, get_rawResultsPath(filename))]
|
|
|
|
run_test("default.sw.multi", common_args + reporter_args)
|
|
|
|
check_outputs("default.sw.multi")
|
|
for reporter, filename in zip(reporters, filenames):
|
|
check_outputs(filename)
|
|
|
|
|
|
if overallResult != 0:
|
|
print("If these differences are expected, run approve.py to approve new baselines.")
|
|
exit(2)
|