3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-10-31 04:07:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add tests for CMake configure toggles passing to Catch2 build

Browse Source
This commit is contained in:
Martin Hořeňovský
2022-02-19 21:27:53 +01:00
parent 4396a9119f
commit 72f0372664
13 changed files with 308 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/linux-other-builds.yml vendored
View File

@@ -51,6 +51,14 @@ jobs:
other_pkgs: clang-10
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
# Configure tests with Clang-10
- cxx: clang++-10
build_description: CMake configuration tests
build_type: Debug
std: 14
other_pks: clang-10
cmake_configurations: -DCATCH_ENABLE_CONFIGURE_TESTS=ON
steps:
- uses: actions/checkout@v2

1
CMakeLists.txt
View File

@@ -20,6 +20,7 @@ cmake_dependent_option(CATCH_BUILD_FUZZERS "Build fuzzers" OFF "CATCH_DEVELOPMEN
cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF)
cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
# Catch2's build breaks if done in-tree. You probably should not build

3
appveyor.yml
View File

@@ -51,9 +51,10 @@ test_script:
# build explicitly.
environment:
matrix:
- FLAVOR: VS 2019 x64 Debug Surrogates
- FLAVOR: VS 2019 x64 Debug Surrogates Configure Tests
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
surrogates: 1
configure_tests: 1
platform: x64
configuration: Debug

25
docs/contributing.md
View File

@@ -62,18 +62,23 @@ test using an external check script. Catch2 integration tests are written
using CTest, either as a direct command invocation + pass/fail regex,
or by delegating the check to a Python script.
There are also two more kinds of tests, examples and "ExtraTests".
Catch2 is slowly gaining more and more types of tests, currently Catch2
project also has buildable examples, "ExtraTests", and CMake config tests.
Examples present a small and self-contained snippets of code that
use Catch2's facilities for specific purpose. Currently they are assumed
passing if they compile. ExtraTests then are expensive tests, that we
do not want to run all the time. This can be either because they take
a long time to run, or because they take a long time to compile, e.g.
because they test compile time configuration and require separate
compilation.
passing if they compile.
Examples and ExtraTests are not compiled by default. To compile them,
add `-DCATCH_BUILD_EXAMPLES=ON` and `-DCATCH_BUILD_EXTRA_TESTS=ON` to
the invocation of CMake configuration step.
ExtraTests then are expensive tests, that we do not want to run all the
time. This can be either because they take a long time to run, or because
they take a long time to compile, e.g. because they test compile time
configuration and require separate compilation.
Finally, CMake config tests test that you set Catch2's compile-time
configuration options through CMake, using CMake options of the same name.
None of these tests are enabled by default. To enable them, add
`-DCATCH_BUILD_EXAMPLES=ON`, `-DCATCH_BUILD_EXTRA_TESTS=ON`, and
`-DCATCH_ENABLE_CONFIGURE_TESTS=ON` when configuration the CMake build.
Bringing this all together, the steps below should configure, build,
and run all tests in the `Debug` compilation.
@@ -85,7 +90,7 @@ and run all tests in the `Debug` compilation.
./tools/scripts/generateAmalgamatedFiles.py
# 2. Configure the full test build
cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON
cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON
# 3. Run the actual build
cmake --build debug-build

18
tests/CMakeLists.txt
View File

@@ -471,6 +471,24 @@ set_tests_properties("Reporters::DashAsLocationInReporterSpecSendsOutputToStdout
PASS_REGULAR_EXPRESSION "All tests passed \\(5 assertions in 1 test case\\)"
)
if (CATCH_ENABLE_CONFIGURE_TESTS)
add_test(NAME "CMakeConfig::DefaultReporter"
COMMAND
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDefaultReporter.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
)
add_test(NAME "CMakeConfig::Disable"
COMMAND
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisable.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
)
add_test(NAME "CMakeConfig::DisableStringification"
COMMAND
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisableStringification.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
)
add_test(NAME "CMakeConfig::ExperimentalRedirect"
COMMAND
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureExperimentalRedirect.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
)
endif()
if (CATCH_USE_VALGRIND)
add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>)

75
tests/TestScripts/ConfigureTestsCommon.py Normal file
View File

@@ -0,0 +1,75 @@
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
from typing import List, Tuple
import os
import subprocess
def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]):
base_configure_cmd = ['cmake',
'-B{}'.format(project_path),
'-H{}'.format(source_path),
'-DCMAKE_BUILD_TYPE=Debug',
'-DCATCH_DEVELOPMENT_BUILD=ON']
for option, value in options:
base_configure_cmd.append('-D{}={}'.format(option, value))
try:
subprocess.run(base_configure_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
check = True)
except subprocess.SubprocessError as ex:
print("Could not configure build to '{}' from '{}'".format(project_path, source_path))
print("Return code: {}".format(ex.returncode))
print("output: {}".format(ex.output))
raise
print('Configuring {} finished'.format(project_path))
build_cmd = ['cmake',
'--build', '{}'.format(project_path),
# For now we assume that we only need Debug config
'--config', 'Debug']
try:
subprocess.run(build_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
check = True)
except subprocess.SubprocessError as ex:
print("Could not build project in '{}'".format(project_path))
print("Return code: {}".format(ex.returncode))
print("output: {}".format(ex.output))
raise
print('Building {} finished'.format(project_path))
def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]:
# For now we assume that Windows builds are done using MSBuild under
# Debug configuration. This means that we need to add "Debug" folder
# to the path when constructing it. On Linux, we don't add anything.
config_path = "Debug" if os.name == 'nt' else ""
full_path = os.path.join(base_path, config_path, binary_name)
base_cmd = [full_path]
base_cmd.extend(other_options)
try:
ret = subprocess.run(base_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
check = True,
universal_newlines = True)
except subprocess.SubprocessError as ex:
print('Could not run "{}"'.format(base_cmd))
print('Args: "{}"'.format(other_options))
print('Return code: {}'.format(ex.returncode))
print('stdout: {}'.format(ex.stdout))
print('stderr: {}'.format(ex.stdout))
raise
return (ret.stdout, ret.stderr)

44
tests/TestScripts/testConfigureDefaultReporter.py Normal file
View File

@@ -0,0 +1,44 @@
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
from ConfigureTestsCommon import configure_and_build, run_and_return_output
import os
import re
import sys
"""
Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
exists, and path to where the output files should be stored.
"""
if len(sys.argv) != 3:
print('Wrong number of arguments: {}'.format(len(sys.argv)))
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
exit(1)
catch2_source_path = os.path.abspath(sys.argv[1])
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter')
configure_and_build(catch2_source_path,
build_dir_path,
[("CATCH_CONFIG_DEFAULT_REPORTER", "compact")])
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]'])
# This matches the summary line made by compact reporter, console reporter's
# summary line does not match the regex.
summary_regex = 'Passed \d+ test case with \d+ assertions.'
if not re.match(summary_regex, stdout):
print("Could not find '{}' in the stdout".format(summary_regex))
print('stdout: "{}"'.format(stdout))
exit(2)

48
tests/TestScripts/testConfigureDisable.py Normal file
View File

@@ -0,0 +1,48 @@
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
from ConfigureTestsCommon import configure_and_build, run_and_return_output
import os
import re
import sys
"""
Tests the CMake configure option for CATCH_CONFIG_DISABLE
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
exists, and path to where the output files should be stored.
"""
if len(sys.argv) != 3:
print('Wrong number of arguments: {}'.format(len(sys.argv)))
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
exit(1)
catch2_source_path = os.path.abspath(sys.argv[1])
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'Disable')
configure_and_build(catch2_source_path,
build_dir_path,
[("CATCH_CONFIG_DISABLE", "ON"),
# We need to turn off WERROR, because the compilers
# can see that the various variables inside test cases
# are set but unused.
("CATCH_ENABLE_WERROR", "OFF")])
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
'SelfTest',
['--allow-running-no-tests'])
summary_line = 'No tests ran'
if not summary_line in stdout:
print("Could not find '{}' in the stdout".format(summary_line))
print('stdout: "{}"'.format(stdout))
exit(2)

44
tests/TestScripts/testConfigureDisableStringification.py Normal file
View File

@@ -0,0 +1,44 @@
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
from ConfigureTestsCommon import configure_and_build, run_and_return_output
import os
import re
import sys
"""
Tests the CMake configure option for CATCH_CONFIG_DISABLE_STRINGIFICATION
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
exists, and path to where the output files should be stored.
"""
if len(sys.argv) != 3:
print('Wrong number of arguments: {}'.format(len(sys.argv)))
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
exit(1)
catch2_source_path = os.path.abspath(sys.argv[1])
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DisableStringification')
configure_and_build(catch2_source_path,
build_dir_path,
[("CATCH_CONFIG_DISABLE_STRINGIFICATION", "ON")])
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
'SelfTest',
['-s', '[approx][custom]'])
required_output = 'Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION'
if not required_output in stdout:
print("Could not find '{}' in the stdout".format(required_output))
print('stdout: "{}"'.format(stdout))
exit(2)

49
tests/TestScripts/testConfigureExperimentalRedirect.py Normal file
View File

@@ -0,0 +1,49 @@
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
from ConfigureTestsCommon import configure_and_build, run_and_return_output
import os
import re
import sys
"""
Tests the CMake configure option for CATCH_CONFIG_EXPERIMENTAL_REDIRECT
Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
exists, and path to where the output files should be stored.
"""
if len(sys.argv) != 3:
print('Wrong number of arguments: {}'.format(len(sys.argv)))
print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
exit(1)
catch2_source_path = os.path.abspath(sys.argv[1])
build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'ExperimentalRedirect')
configure_and_build(catch2_source_path,
build_dir_path,
[("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")])
stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
'SelfTest',
['-r', 'xml', '"has printf"'])
# The print from printf must be within the XML's reporter stdout tag.
required_output = '''\
<StdOut>
loose text artifact
</StdOut>
'''
if not required_output in stdout:
print("Could not find '{}' in the stdout".format(required_output))
print('stdout: "{}"'.format(stdout))
exit(2)

4
tools/misc/appveyorBuildConfigurationScript.bat
View File

@@ -10,10 +10,10 @@ if "%CONFIGURATION%"=="Debug" (
@REM # coverage needs to build the special helper as well as the main
cmake -Htools/misc -Bbuild-misc -A%PLATFORM% || exit /b !ERRORLEVEL!
cmake --build build-misc || exit /b !ERRORLEVEL!
cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL!
cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_ENABLE_CONFIGURE_TESTS=%configure_tests% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL!
) else (
@REM # We know that coverage is 0
cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_BUILD_SURROGATES=%surrogates% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL!
cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_BUILD_SURROGATES=%surrogates% -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_ENABLE_CONFIGURE_TESTS=%configure_tests% || exit /b !ERRORLEVEL!
)
)
if "%CONFIGURATION%"=="Release" (

2
tools/scripts/buildAndTest.cmd
View File

@@ -6,7 +6,7 @@ rem 1. Regenerate the amalgamated distribution
python tools\scripts\generateAmalgamatedFiles.py
rem 2. Configure the full test build
cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON
cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_ENABLE_CONFIGURE_TESTS=ON
rem 3. Run the actual build
cmake --build debug-build

2
tools/scripts/buildAndTest.sh
View File

@@ -8,7 +8,7 @@
./tools/scripts/generateAmalgamatedFiles.py
# 2. Configure the full test build
cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON
cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_ENABLE_CONFIGURE_TESTS=ON
# 3. Run the actual build
cmake --build debug-build