Add tests for CMake configure toggles passing to Catch2 build

.github/workflows/linux-other-builds.yml

@@ -51,6 +51,14 @@ jobs:
             other_pkgs: clang-10
             cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
 
+          # Configure tests with Clang-10
+          - cxx: clang++-10
+            build_description: CMake configuration tests
+            build_type: Debug
+            std: 14
+            other_pks: clang-10
+            cmake_configurations: -DCATCH_ENABLE_CONFIGURE_TESTS=ON
+
 
     steps:
     - uses: actions/checkout@v2

CMakeLists.txt

@@ -20,6 +20,7 @@ cmake_dependent_option(CATCH_BUILD_FUZZERS "Build fuzzers" OFF "CATCH_DEVELOPMEN
 cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
+cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 
 
 # Catch2's build breaks if done in-tree. You probably should not build

appveyor.yml

@@ -51,9 +51,10 @@ test_script:
 # build explicitly.
 environment:
   matrix:
-    - FLAVOR: VS 2019 x64 Debug Surrogates
+    - FLAVOR: VS 2019 x64 Debug Surrogates Configure Tests
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
       surrogates: 1
+      configure_tests: 1
       platform: x64
       configuration: Debug
     

docs/contributing.md

@@ -62,18 +62,23 @@ test using an external check script. Catch2 integration tests are written
 using CTest, either as a direct command invocation + pass/fail regex,
 or by delegating the check to a Python script.
 
-There are also two more kinds of tests, examples and "ExtraTests".
+Catch2 is slowly gaining more and more types of tests, currently Catch2
+project also has buildable examples, "ExtraTests", and CMake config tests.
 Examples present a small and self-contained snippets of code that
 use Catch2's facilities for specific purpose. Currently they are assumed
-passing if they compile. ExtraTests then are expensive tests, that we
-do not want to run all the time. This can be either because they take
-a long time to run, or because they take a long time to compile, e.g.
-because they test compile time configuration and require separate
-compilation.
+passing if they compile.
 
-Examples and ExtraTests are not compiled by default. To compile them,
-add `-DCATCH_BUILD_EXAMPLES=ON` and `-DCATCH_BUILD_EXTRA_TESTS=ON` to
-the invocation of CMake configuration step.
+ExtraTests then are expensive tests, that we do not want to run all the
+time. This can be either because they take a long time to run, or because
+they take a long time to compile, e.g. because they test compile time
+configuration and require separate compilation.
+
+Finally, CMake config tests test that you set Catch2's compile-time
+configuration options through CMake, using CMake options of the same name.
+
+None of these tests are enabled by default. To enable them, add
+`-DCATCH_BUILD_EXAMPLES=ON`, `-DCATCH_BUILD_EXTRA_TESTS=ON`, and
+`-DCATCH_ENABLE_CONFIGURE_TESTS=ON` when configuration the CMake build.
 
 Bringing this all together, the steps below should configure, build,
 and run all tests in the `Debug` compilation.
@@ -85,7 +90,7 @@ and run all tests in the `Debug` compilation.
 ./tools/scripts/generateAmalgamatedFiles.py
 
 # 2. Configure the full test build
-cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON
+cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON
 
 # 3. Run the actual build
 cmake --build debug-build

tests/CMakeLists.txt

@@ -471,6 +471,24 @@ set_tests_properties("Reporters::DashAsLocationInReporterSpecSendsOutputToStdout
     PASS_REGULAR_EXPRESSION "All tests passed \\(5 assertions in 1 test case\\)"
 )
 
+if (CATCH_ENABLE_CONFIGURE_TESTS)
+    add_test(NAME "CMakeConfig::DefaultReporter"
+      COMMAND
+        "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDefaultReporter.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
+    )
+    add_test(NAME "CMakeConfig::Disable"
+      COMMAND
+        "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisable.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
+    )
+    add_test(NAME "CMakeConfig::DisableStringification"
+      COMMAND
+        "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisableStringification.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
+    )
+    add_test(NAME "CMakeConfig::ExperimentalRedirect"
+      COMMAND
+        "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureExperimentalRedirect.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
+    )
+endif()
 
 if (CATCH_USE_VALGRIND)
     add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>)

tests/TestScripts/ConfigureTestsCommon.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from typing import List, Tuple
+
+import os
+import subprocess
+
+def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]):
+    base_configure_cmd = ['cmake',
+                          '-B{}'.format(project_path),
+                          '-H{}'.format(source_path),
+                          '-DCMAKE_BUILD_TYPE=Debug',
+                          '-DCATCH_DEVELOPMENT_BUILD=ON']
+    for option, value in options:
+        base_configure_cmd.append('-D{}={}'.format(option, value))
+    try:
+        subprocess.run(base_configure_cmd,
+                       stdout = subprocess.PIPE,
+                       stderr = subprocess.STDOUT,
+                       check = True)
+    except subprocess.SubprocessError as ex:
+        print("Could not configure build to '{}' from '{}'".format(project_path, source_path))
+        print("Return code: {}".format(ex.returncode))
+        print("output: {}".format(ex.output))
+        raise
+    print('Configuring {} finished'.format(project_path))
+
+    build_cmd = ['cmake',
+                 '--build', '{}'.format(project_path),
+                 # For now we assume that we only need Debug config
+                 '--config', 'Debug']
+    try:
+        subprocess.run(build_cmd,
+                       stdout = subprocess.PIPE,
+                       stderr = subprocess.STDOUT,
+                       check = True)
+    except subprocess.SubprocessError as ex:
+        print("Could not build project in '{}'".format(project_path))
+        print("Return code: {}".format(ex.returncode))
+        print("output: {}".format(ex.output))
+        raise
+    print('Building {} finished'.format(project_path))
+
+def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]:
+    # For now we assume that Windows builds are done using MSBuild under
+    # Debug configuration. This means that we need to add "Debug" folder
+    # to the path when constructing it. On Linux, we don't add anything.
+    config_path = "Debug" if os.name == 'nt' else ""
+    full_path = os.path.join(base_path, config_path, binary_name)
+
+    base_cmd = [full_path]
+    base_cmd.extend(other_options)
+
+    try:
+        ret = subprocess.run(base_cmd,
+                             stdout = subprocess.PIPE,
+                             stderr = subprocess.PIPE,
+                             check = True,
+                             universal_newlines = True)
+    except subprocess.SubprocessError as ex:
+        print('Could not run "{}"'.format(base_cmd))
+        print('Args: "{}"'.format(other_options))
+        print('Return code: {}'.format(ex.returncode))
+        print('stdout: {}'.format(ex.stdout))
+        print('stderr: {}'.format(ex.stdout))
+        raise
+
+    return (ret.stdout, ret.stderr)

tests/TestScripts/testConfigureDefaultReporter.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_DEFAULT_REPORTER", "compact")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]'])
+
+
+# This matches the summary line made by compact reporter, console reporter's
+# summary line does not match the regex.
+summary_regex = 'Passed \d+ test case with \d+ assertions.'
+if not re.match(summary_regex, stdout):
+    print("Could not find '{}' in the stdout".format(summary_regex))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tests/TestScripts/testConfigureDisable.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DISABLE
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'Disable')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_DISABLE", "ON"),
+                     # We need to turn off WERROR, because the compilers
+                     # can see that the various variables inside test cases
+                     # are set but unused.
+                     ("CATCH_ENABLE_WERROR", "OFF")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                                  'SelfTest',
+                                  ['--allow-running-no-tests'])
+
+
+summary_line = 'No tests ran'
+if not summary_line in stdout:
+    print("Could not find '{}' in the stdout".format(summary_line))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tests/TestScripts/testConfigureDisableStringification.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DISABLE_STRINGIFICATION
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DisableStringification')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_DISABLE_STRINGIFICATION", "ON")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                                  'SelfTest',
+                                  ['-s', '[approx][custom]'])
+
+
+required_output = 'Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION'
+if not required_output in stdout:
+    print("Could not find '{}' in the stdout".format(required_output))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tests/TestScripts/testConfigureExperimentalRedirect.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_EXPERIMENTAL_REDIRECT
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'ExperimentalRedirect')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                                  'SelfTest',
+                                  ['-r', 'xml', '"has printf"'])
+
+
+# The print from printf must be within the XML's reporter stdout tag.
+required_output = '''\
+      <StdOut>
+loose text artifact
+      </StdOut>
+'''
+if not required_output in stdout:
+    print("Could not find '{}' in the stdout".format(required_output))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tools/misc/appveyorBuildConfigurationScript.bat

@@ -10,10 +10,10 @@ if "%CONFIGURATION%"=="Debug" (
     @REM # coverage needs to build the special helper as well as the main
     cmake -Htools/misc -Bbuild-misc -A%PLATFORM% || exit /b !ERRORLEVEL!
     cmake --build build-misc || exit /b !ERRORLEVEL!
-    cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL!
+    cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_ENABLE_CONFIGURE_TESTS=%configure_tests% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL!
   ) else (
     @REM # We know that coverage is 0
-    cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_BUILD_SURROGATES=%surrogates% -DCATCH_DEVELOPMENT_BUILD=ON || exit /b !ERRORLEVEL!
+    cmake -H. -BBuild -A%PLATFORM% -DCATCH_TEST_USE_WMAIN=%wmain% -DCATCH_BUILD_EXAMPLES=%examples% -DCATCH_BUILD_EXTRA_TESTS=%examples% -DCATCH_BUILD_SURROGATES=%surrogates% -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_ENABLE_CONFIGURE_TESTS=%configure_tests% || exit /b !ERRORLEVEL!
   )
 )
 if "%CONFIGURATION%"=="Release" (

tools/scripts/buildAndTest.cmd

@@ -6,7 +6,7 @@ rem 1. Regenerate the amalgamated distribution
 python tools\scripts\generateAmalgamatedFiles.py
 
 rem 2. Configure the full test build
-cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON
+cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_ENABLE_CONFIGURE_TESTS=ON
 
 rem 3. Run the actual build
 cmake --build debug-build

tools/scripts/buildAndTest.sh

@@ -8,7 +8,7 @@
 ./tools/scripts/generateAmalgamatedFiles.py
 
 # 2. Configure the full test build
-cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_DEVELOPMENT_BUILD=ON
+cmake -Bdebug-build -H. -DCMAKE_BUILD_TYPE=Debug -DCATCH_DEVELOPMENT_BUILD=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_ENABLE_CONFIGURE_TESTS=ON
 
 # 3. Run the actual build
 cmake --build debug-build