mirror of
https://github.com/catchorg/Catch2.git
synced 2025-08-01 21:05:39 +02:00
Add tests for CMake configure toggles passing to Catch2 build
This commit is contained in:
Martin Hořeňovský
@@ -471,6 +471,24 @@ set_tests_properties("Reporters::DashAsLocationInReporterSpecSendsOutputToStdout
|
||||
PASS_REGULAR_EXPRESSION "All tests passed \\(5 assertions in 1 test case\\)"
|
||||
)
|
||||
|
||||
if (CATCH_ENABLE_CONFIGURE_TESTS)
|
||||
add_test(NAME "CMakeConfig::DefaultReporter"
|
||||
COMMAND
|
||||
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDefaultReporter.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
)
|
||||
add_test(NAME "CMakeConfig::Disable"
|
||||
COMMAND
|
||||
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisable.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
)
|
||||
add_test(NAME "CMakeConfig::DisableStringification"
|
||||
COMMAND
|
||||
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisableStringification.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
)
|
||||
add_test(NAME "CMakeConfig::ExperimentalRedirect"
|
||||
COMMAND
|
||||
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureExperimentalRedirect.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
)
|
||||
endif()
|
||||
|
||||
if (CATCH_USE_VALGRIND)
|
||||
add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>)
|
||||
|
||||
75
tests/TestScripts/ConfigureTestsCommon.py
Normal file
75
tests/TestScripts/ConfigureTestsCommon.py
Normal file
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Catch2 Authors
|
||||
# Distributed under the Boost Software License, Version 1.0.
|
||||
# (See accompanying file LICENSE_1_0.txt or copy at
|
||||
# https://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
# SPDX-License-Identifier: BSL-1.0
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]):
|
||||
base_configure_cmd = ['cmake',
|
||||
'-B{}'.format(project_path),
|
||||
'-H{}'.format(source_path),
|
||||
'-DCMAKE_BUILD_TYPE=Debug',
|
||||
'-DCATCH_DEVELOPMENT_BUILD=ON']
|
||||
for option, value in options:
|
||||
base_configure_cmd.append('-D{}={}'.format(option, value))
|
||||
try:
|
||||
subprocess.run(base_configure_cmd,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
check = True)
|
||||
except subprocess.SubprocessError as ex:
|
||||
print("Could not configure build to '{}' from '{}'".format(project_path, source_path))
|
||||
print("Return code: {}".format(ex.returncode))
|
||||
print("output: {}".format(ex.output))
|
||||
raise
|
||||
print('Configuring {} finished'.format(project_path))
|
||||
|
||||
build_cmd = ['cmake',
|
||||
'--build', '{}'.format(project_path),
|
||||
# For now we assume that we only need Debug config
|
||||
'--config', 'Debug']
|
||||
try:
|
||||
subprocess.run(build_cmd,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
check = True)
|
||||
except subprocess.SubprocessError as ex:
|
||||
print("Could not build project in '{}'".format(project_path))
|
||||
print("Return code: {}".format(ex.returncode))
|
||||
print("output: {}".format(ex.output))
|
||||
raise
|
||||
print('Building {} finished'.format(project_path))
|
||||
|
||||
def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]:
|
||||
# For now we assume that Windows builds are done using MSBuild under
|
||||
# Debug configuration. This means that we need to add "Debug" folder
|
||||
# to the path when constructing it. On Linux, we don't add anything.
|
||||
config_path = "Debug" if os.name == 'nt' else ""
|
||||
full_path = os.path.join(base_path, config_path, binary_name)
|
||||
|
||||
base_cmd = [full_path]
|
||||
base_cmd.extend(other_options)
|
||||
|
||||
try:
|
||||
ret = subprocess.run(base_cmd,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.PIPE,
|
||||
check = True,
|
||||
universal_newlines = True)
|
||||
except subprocess.SubprocessError as ex:
|
||||
print('Could not run "{}"'.format(base_cmd))
|
||||
print('Args: "{}"'.format(other_options))
|
||||
print('Return code: {}'.format(ex.returncode))
|
||||
print('stdout: {}'.format(ex.stdout))
|
||||
print('stderr: {}'.format(ex.stdout))
|
||||
raise
|
||||
|
||||
return (ret.stdout, ret.stderr)
|
||||
44
tests/TestScripts/testConfigureDefaultReporter.py
Normal file
44
tests/TestScripts/testConfigureDefaultReporter.py
Normal file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Catch2 Authors
|
||||
# Distributed under the Boost Software License, Version 1.0.
|
||||
# (See accompanying file LICENSE_1_0.txt or copy at
|
||||
# https://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
# SPDX-License-Identifier: BSL-1.0
|
||||
|
||||
from ConfigureTestsCommon import configure_and_build, run_and_return_output
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
"""
|
||||
Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER
|
||||
|
||||
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
|
||||
exists, and path to where the output files should be stored.
|
||||
"""
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print('Wrong number of arguments: {}'.format(len(sys.argv)))
|
||||
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
|
||||
exit(1)
|
||||
|
||||
catch2_source_path = os.path.abspath(sys.argv[1])
|
||||
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter')
|
||||
|
||||
configure_and_build(catch2_source_path,
|
||||
build_dir_path,
|
||||
[("CATCH_CONFIG_DEFAULT_REPORTER", "compact")])
|
||||
|
||||
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]'])
|
||||
|
||||
|
||||
# This matches the summary line made by compact reporter, console reporter's
|
||||
# summary line does not match the regex.
|
||||
summary_regex = 'Passed \d+ test case with \d+ assertions.'
|
||||
if not re.match(summary_regex, stdout):
|
||||
print("Could not find '{}' in the stdout".format(summary_regex))
|
||||
print('stdout: "{}"'.format(stdout))
|
||||
exit(2)
|
||||
48
tests/TestScripts/testConfigureDisable.py
Normal file
48
tests/TestScripts/testConfigureDisable.py
Normal file
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Catch2 Authors
|
||||
# Distributed under the Boost Software License, Version 1.0.
|
||||
# (See accompanying file LICENSE_1_0.txt or copy at
|
||||
# https://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
# SPDX-License-Identifier: BSL-1.0
|
||||
|
||||
from ConfigureTestsCommon import configure_and_build, run_and_return_output
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
"""
|
||||
Tests the CMake configure option for CATCH_CONFIG_DISABLE
|
||||
|
||||
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
|
||||
exists, and path to where the output files should be stored.
|
||||
"""
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print('Wrong number of arguments: {}'.format(len(sys.argv)))
|
||||
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
|
||||
exit(1)
|
||||
|
||||
catch2_source_path = os.path.abspath(sys.argv[1])
|
||||
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'Disable')
|
||||
|
||||
configure_and_build(catch2_source_path,
|
||||
build_dir_path,
|
||||
[("CATCH_CONFIG_DISABLE", "ON"),
|
||||
# We need to turn off WERROR, because the compilers
|
||||
# can see that the various variables inside test cases
|
||||
# are set but unused.
|
||||
("CATCH_ENABLE_WERROR", "OFF")])
|
||||
|
||||
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
|
||||
'SelfTest',
|
||||
['--allow-running-no-tests'])
|
||||
|
||||
|
||||
summary_line = 'No tests ran'
|
||||
if not summary_line in stdout:
|
||||
print("Could not find '{}' in the stdout".format(summary_line))
|
||||
print('stdout: "{}"'.format(stdout))
|
||||
exit(2)
|
||||
44
tests/TestScripts/testConfigureDisableStringification.py
Normal file
44
tests/TestScripts/testConfigureDisableStringification.py
Normal file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Catch2 Authors
|
||||
# Distributed under the Boost Software License, Version 1.0.
|
||||
# (See accompanying file LICENSE_1_0.txt or copy at
|
||||
# https://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
# SPDX-License-Identifier: BSL-1.0
|
||||
|
||||
from ConfigureTestsCommon import configure_and_build, run_and_return_output
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
"""
|
||||
Tests the CMake configure option for CATCH_CONFIG_DISABLE_STRINGIFICATION
|
||||
|
||||
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
|
||||
exists, and path to where the output files should be stored.
|
||||
"""
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print('Wrong number of arguments: {}'.format(len(sys.argv)))
|
||||
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
|
||||
exit(1)
|
||||
|
||||
catch2_source_path = os.path.abspath(sys.argv[1])
|
||||
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DisableStringification')
|
||||
|
||||
configure_and_build(catch2_source_path,
|
||||
build_dir_path,
|
||||
[("CATCH_CONFIG_DISABLE_STRINGIFICATION", "ON")])
|
||||
|
||||
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
|
||||
'SelfTest',
|
||||
['-s', '[approx][custom]'])
|
||||
|
||||
|
||||
required_output = 'Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION'
|
||||
if not required_output in stdout:
|
||||
print("Could not find '{}' in the stdout".format(required_output))
|
||||
print('stdout: "{}"'.format(stdout))
|
||||
exit(2)
|
||||
49
tests/TestScripts/testConfigureExperimentalRedirect.py
Normal file
49
tests/TestScripts/testConfigureExperimentalRedirect.py
Normal file
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Catch2 Authors
|
||||
# Distributed under the Boost Software License, Version 1.0.
|
||||
# (See accompanying file LICENSE_1_0.txt or copy at
|
||||
# https://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
# SPDX-License-Identifier: BSL-1.0
|
||||
|
||||
from ConfigureTestsCommon import configure_and_build, run_and_return_output
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
"""
|
||||
Tests the CMake configure option for CATCH_CONFIG_EXPERIMENTAL_REDIRECT
|
||||
|
||||
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
|
||||
exists, and path to where the output files should be stored.
|
||||
"""
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print('Wrong number of arguments: {}'.format(len(sys.argv)))
|
||||
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
|
||||
exit(1)
|
||||
|
||||
catch2_source_path = os.path.abspath(sys.argv[1])
|
||||
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'ExperimentalRedirect')
|
||||
|
||||
configure_and_build(catch2_source_path,
|
||||
build_dir_path,
|
||||
[("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")])
|
||||
|
||||
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
|
||||
'SelfTest',
|
||||
['-r', 'xml', '"has printf"'])
|
||||
|
||||
|
||||
# The print from printf must be within the XML's reporter stdout tag.
|
||||
required_output = '''\
|
||||
<StdOut>
|
||||
loose text artifact
|
||||
</StdOut>
|
||||
'''
|
||||
if not required_output in stdout:
|
||||
print("Could not find '{}' in the stdout".format(required_output))
|
||||
print('stdout: "{}"'.format(stdout))
|
||||
exit(2)
|
||||
Reference in New Issue
Block a user