diff --git a/.github/workflows/linux-other-builds.yml b/.github/workflows/linux-other-builds.yml index 61ebf3d7..1863871f 100644 --- a/.github/workflows/linux-other-builds.yml +++ b/.github/workflows/linux-other-builds.yml @@ -51,6 +51,14 @@ jobs: other_pkgs: clang-10 cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + # Configure tests with Clang-10 + - cxx: clang++-10 + build_description: CMake configuration tests + build_type: Debug + std: 14 + other_pks: clang-10 + cmake_configurations: -DCATCH_ENABLE_CONFIGURE_TESTS=ON + steps: - uses: actions/checkout@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 296ede56..e6f880ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ cmake_dependent_option(CATCH_BUILD_FUZZERS "Build fuzzers" OFF "CATCH_DEVELOPMEN cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io" OFF "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF) +cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF) # Catch2's build breaks if done in-tree. You probably should not build diff --git a/appveyor.yml b/appveyor.yml index d84d775c..a8f77df0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,9 +51,10 @@ test_script: # build explicitly. environment: matrix: - - FLAVOR: VS 2019 x64 Debug Surrogates + - FLAVOR: VS 2019 x64 Debug Surrogates Configure Tests APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 surrogates: 1 + configure_tests: 1 platform: x64 configuration: Debug diff --git a/docs/contributing.md b/docs/contributing.md index f7d0f73a..1b84f095 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -62,18 +62,23 @@ test using an external check script. Catch2 integration tests are written using CTest, either as a direct command invocation + pass/fail regex, or by delegating the check to a Python script. -There are also two more kinds of tests, examples and "ExtraTests". +Catch2 is slowly gaining more and more types of tests, currently Catch2 +project also has buildable examples, "ExtraTests", and CMake config tests. Examples present a small and self-contained snippets of code that use Catch2's facilities for specific purpose. Currently they are assumed -passing if they compile. ExtraTests then are expensive tests, that we -do not want to run all the time. This can be either because they take -a long time to run, or because they take a long time to compile, e.g. -because they test compile time configuration and require separate -compilation. +passing if they compile. -Examples and ExtraTests are not compiled by default. To compile them, -add `-DCATCH_BUILD_EXAMPLES=ON` and `-DCATCH_BUILD_EXTRA_TESTS=ON` to -the invocation of CMake configuration step. +ExtraTests then are expensive tests, that we do not want to run all the +time. This can be either because they take a long time to run, or because +they take a long time to compile, e.g. because they test compile time +configuration and require separate compilation. + +Finally, CMake config tests test that you set Catch2's compile-time +configuration options through CMake, using CMake options of the same name. + +None of these tests are enabled by default. To enable them, add +`-DCATCH_BUILD_EXAMPLES=ON`, `-DCATCH_BUILD_EXTRA_TESTS=ON`, and +`-DCATCH_ENABLE_CONFIGURE_TESTS=ON` when configuration the CMake build. Bringing this all together, the steps below should configure, build, and run all tests in the `Debug` compilation. @@ -85,7 +90,7 @@ and run all tests in the `Debug` compilation. ./tools/scripts/generateAmalgamatedFiles.py # 2. Configure the full test build -cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON +cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON # 3. Run the actual build cmake --build debug-build diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0f25bfe5..2115a690 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -471,6 +471,24 @@ set_tests_properties("Reporters::DashAsLocationInReporterSpecSendsOutputToStdout PASS_REGULAR_EXPRESSION "All tests passed \\(5 assertions in 1 test case\\)" ) +if (CATCH_ENABLE_CONFIGURE_TESTS) + add_test(NAME "CMakeConfig::DefaultReporter" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDefaultReporter.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + add_test(NAME "CMakeConfig::Disable" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisable.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + add_test(NAME "CMakeConfig::DisableStringification" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisableStringification.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + add_test(NAME "CMakeConfig::ExperimentalRedirect" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureExperimentalRedirect.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) +endif() if (CATCH_USE_VALGRIND) add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $) diff --git a/tests/TestScripts/ConfigureTestsCommon.py b/tests/TestScripts/ConfigureTestsCommon.py new file mode 100644 index 00000000..ad5b358b --- /dev/null +++ b/tests/TestScripts/ConfigureTestsCommon.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from typing import List, Tuple + +import os +import subprocess + +def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]): + base_configure_cmd = ['cmake', + '-B{}'.format(project_path), + '-H{}'.format(source_path), + '-DCMAKE_BUILD_TYPE=Debug', + '-DCATCH_DEVELOPMENT_BUILD=ON'] + for option, value in options: + base_configure_cmd.append('-D{}={}'.format(option, value)) + try: + subprocess.run(base_configure_cmd, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + check = True) + except subprocess.SubprocessError as ex: + print("Could not configure build to '{}' from '{}'".format(project_path, source_path)) + print("Return code: {}".format(ex.returncode)) + print("output: {}".format(ex.output)) + raise + print('Configuring {} finished'.format(project_path)) + + build_cmd = ['cmake', + '--build', '{}'.format(project_path), + # For now we assume that we only need Debug config + '--config', 'Debug'] + try: + subprocess.run(build_cmd, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + check = True) + except subprocess.SubprocessError as ex: + print("Could not build project in '{}'".format(project_path)) + print("Return code: {}".format(ex.returncode)) + print("output: {}".format(ex.output)) + raise + print('Building {} finished'.format(project_path)) + +def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]: + # For now we assume that Windows builds are done using MSBuild under + # Debug configuration. This means that we need to add "Debug" folder + # to the path when constructing it. On Linux, we don't add anything. + config_path = "Debug" if os.name == 'nt' else "" + full_path = os.path.join(base_path, config_path, binary_name) + + base_cmd = [full_path] + base_cmd.extend(other_options) + + try: + ret = subprocess.run(base_cmd, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + check = True, + universal_newlines = True) + except subprocess.SubprocessError as ex: + print('Could not run "{}"'.format(base_cmd)) + print('Args: "{}"'.format(other_options)) + print('Return code: {}'.format(ex.returncode)) + print('stdout: {}'.format(ex.stdout)) + print('stderr: {}'.format(ex.stdout)) + raise + + return (ret.stdout, ret.stderr) diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py new file mode 100644 index 00000000..24ec8228 --- /dev/null +++ b/tests/TestScripts/testConfigureDefaultReporter.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_DEFAULT_REPORTER", "compact")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]']) + + +# This matches the summary line made by compact reporter, console reporter's +# summary line does not match the regex. +summary_regex = 'Passed \d+ test case with \d+ assertions.' +if not re.match(summary_regex, stdout): + print("Could not find '{}' in the stdout".format(summary_regex)) + print('stdout: "{}"'.format(stdout)) + exit(2) diff --git a/tests/TestScripts/testConfigureDisable.py b/tests/TestScripts/testConfigureDisable.py new file mode 100644 index 00000000..92946d8d --- /dev/null +++ b/tests/TestScripts/testConfigureDisable.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_DISABLE + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'Disable') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_DISABLE", "ON"), + # We need to turn off WERROR, because the compilers + # can see that the various variables inside test cases + # are set but unused. + ("CATCH_ENABLE_WERROR", "OFF")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), + 'SelfTest', + ['--allow-running-no-tests']) + + +summary_line = 'No tests ran' +if not summary_line in stdout: + print("Could not find '{}' in the stdout".format(summary_line)) + print('stdout: "{}"'.format(stdout)) + exit(2) diff --git a/tests/TestScripts/testConfigureDisableStringification.py b/tests/TestScripts/testConfigureDisableStringification.py new file mode 100644 index 00000000..a8a53e4f --- /dev/null +++ b/tests/TestScripts/testConfigureDisableStringification.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_DISABLE_STRINGIFICATION + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DisableStringification') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_DISABLE_STRINGIFICATION", "ON")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), + 'SelfTest', + ['-s', '[approx][custom]']) + + +required_output = 'Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION' +if not required_output in stdout: + print("Could not find '{}' in the stdout".format(required_output)) + print('stdout: "{}"'.format(stdout)) + exit(2) diff --git a/tests/TestScripts/testConfigureExperimentalRedirect.py b/tests/TestScripts/testConfigureExperimentalRedirect.py new file mode 100644 index 00000000..b5313fe9 --- /dev/null +++ b/tests/TestScripts/testConfigureExperimentalRedirect.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_EXPERIMENTAL_REDIRECT + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'ExperimentalRedirect') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), + 'SelfTest', + ['-r', 'xml', '"has printf"']) + + +# The print from printf must be within the XML's reporter stdout tag. +required_output = '''\ + +loose text artifact + +''' +if not required_output in stdout: + print("Could not find '{}' in the stdout".format(required_output)) + print('stdout: "{}"'.format(stdout)) + exit(2) diff --git a/tools/misc/appveyorBuildConfigurationScript.bat b/tools/misc/appveyorBuildConfigurationScript.bat index d9be52eb..727f829b 100644 --- a/tools/misc/appveyorBuildConfigurationScript.bat +++ b/tools/misc/appveyorBuildConfigurationScript.bat @@ -10,10 +10,10 @@ if "%CONFIGURATION%"=="Debug" ( @REM # coverage needs to build the special helper as well as the main cmake -Htools/misc -Bbuild-misc -A%PLATFORM% || exit /b !ERRORLEVEL! cmake --build build-misc || exit /b !ERRORLEVEL! - cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL! + cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_ENABLE_CONFIGURE_TESTS=%configure_tests% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL! ) else ( @REM # We know that coverage is 0 - cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_BUILD_SURROGATES=%surrogates% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL! + cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_BUILD_SURROGATES=%surrogates% -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_ENABLE_CONFIGURE_TESTS=%configure_tests% || exit /b !ERRORLEVEL! ) ) if "%CONFIGURATION%"=="Release" ( diff --git a/tools/scripts/buildAndTest.cmd b/tools/scripts/buildAndTest.cmd index bd753a3b..e5222a01 100644 --- a/tools/scripts/buildAndTest.cmd +++ b/tools/scripts/buildAndTest.cmd @@ -6,7 +6,7 @@ rem 1. Regenerate the amalgamated distribution python tools\scripts\generateAmalgamatedFiles.py rem 2. Configure the full test build -cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON +cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_ENABLE_CONFIGURE_TESTS=ON rem 3. Run the actual build cmake --build debug-build diff --git a/tools/scripts/buildAndTest.sh b/tools/scripts/buildAndTest.sh index b343acad..4a741598 100755 --- a/tools/scripts/buildAndTest.sh +++ b/tools/scripts/buildAndTest.sh @@ -8,7 +8,7 @@ ./tools/scripts/generateAmalgamatedFiles.py # 2. Configure the full test build -cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON +cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_ENABLE_CONFIGURE_TESTS=ON # 3. Run the actual build cmake --build debug-build