Add test scaffolding for catch_discover_tests

.github/workflows/linux-other-builds.yml

@@ -29,13 +29,13 @@ jobs:
             build_type: Debug
             std: 14
             other_pkgs: g++-7
-            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
+            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
           - cxx: g++-7
             build_description: Extras + Examples
             build_type: Release
             std: 14
             other_pkgs: g++-7
-            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
+            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
 
           # Extras and examples with Clang-10
           - cxx: clang++-10
@@ -43,13 +43,13 @@ jobs:
             build_type: Debug
             std: 17
             other_pkgs: clang-10
-            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
+            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
           - cxx: clang++-10
             build_description: Extras + Examples
             build_type: Release
             std: 17
             other_pkgs: clang-10
-            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
+            cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
 
           # Configure tests with Clang-10
           - cxx: clang++-10

CMakeLists.txt

@@ -21,6 +21,7 @@ cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io"
 cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
+cmake_dependent_option(CATCH_ENABLE_CMAKE_HELPER_TESTS "Enable CMake helper tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 
 
 # Catch2's build breaks if done in-tree. You probably should not build

CMakePresets.json

@@ -18,7 +18,8 @@
                 "CATCH_BUILD_EXAMPLES": "ON",
                 "CATCH_BUILD_EXTRA_TESTS": "ON",
                 "CATCH_BUILD_SURROGATES": "ON",
-                "CATCH_ENABLE_CONFIGURE_TESTS": "ON"
+                "CATCH_ENABLE_CONFIGURE_TESTS": "ON",
+                "CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON"
             }
         }
     ]   

tests/CMakeLists.txt

@@ -622,6 +622,18 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
     endforeach()
 endif()
 
+if (CATCH_ENABLE_CMAKE_HELPER_TESTS)
+    add_test(NAME "CMakeHelper::DiscoverTests"
+      COMMAND
+        "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/DiscoverTests/VerifyRegistration.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
+    )
+    set_tests_properties("CMakeHelper::DiscoverTests"
+      PROPERTIES
+        COST 240
+        LABELS "uses-python"
+    )
+endif()
+
 foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
                       "compact"
                       "console"

tests/TestScripts/DiscoverTests/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(discover-tests-test
+  LANGUAGES CXX
+)
+
+add_executable(tests
+  register-tests.cpp
+)
+
+add_subdirectory(${CATCH2_PATH} catch2-build)
+target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
+
+include(CTest)
+include(Catch)
+catch_discover_tests(tests)

tests/TestScripts/DiscoverTests/VerifyRegistration.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+import os
+import subprocess
+import sys
+
+
+def build_project(sources_dir, output_base_path, catch2_path):
+    build_dir = os.path.join(output_base_path, 'ctest-registration-test')
+    config_cmd = ['cmake',
+                  '-B', build_dir,
+                  '-S', sources_dir,
+                  f'-DCATCH2_PATH={catch2_path}',
+                  '-DCMAKE_BUILD_TYPE=Debug']
+                  
+    build_cmd = ['cmake',
+                 '--build', build_dir,
+                 '--config', 'Debug']
+    
+    try:
+        subprocess.run(config_cmd,
+                       capture_output = True,
+                       check = True,
+                       text = True)
+        subprocess.run(build_cmd,
+                       capture_output = True,
+                       check = True,
+                       text = True)
+    except subprocess.CalledProcessError as err:
+        print('Error when building the test project')
+        print(f'cmd: {err.cmd}')
+        print(f'stderr: {err.stderr}')
+        print(f'stdout: {err.stdout}')
+        exit(3)
+    
+    return build_dir
+    
+
+
+def get_test_names(build_path):
+    # For now we assume that Windows builds are done using MSBuild under
+    # Debug configuration. This means that we need to add "Debug" folder
+    # to the path when constructing it. On Linux, we don't add anything.
+    config_path = "Debug" if os.name == 'nt' else ""
+    full_path = os.path.join(build_path, config_path, 'tests')
+
+
+    cmd = [full_path, '--reporter', 'xml', '--list-tests']
+    result = subprocess.run(cmd,
+                            capture_output = True,
+                            check = True,
+                            text = True)
+
+    import xml.etree.ElementTree as ET
+    root = ET.fromstring(result.stdout)
+    return [tc.text for tc in root.findall('TestCase/Name')]
+
+
+def list_ctest_tests(build_path):
+    old_path = os.getcwd()
+    os.chdir(build_path)
+
+    cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
+    result = subprocess.run(cmd,
+                            capture_output = True,
+                            check = True,
+                            text = True)
+    os.chdir(old_path)
+
+    import json
+
+    ctest_response = json.loads(result.stdout)
+    tests = ctest_response['tests']
+    test_names = []
+    for test in tests:
+        test_command = test['command']
+        # First part of the command is the binary, second is the filter.
+        # If there are less, registration has failed. If there are more,
+        # registration has changed and the script needs updating.
+        assert len(test_command) == 2
+        test_names.append(test_command[1])
+        test_name = test_command[1]
+
+    return test_names
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
+        exit(2)
+    catch2_path = sys.argv[1]
+    output_base_path = sys.argv[2]
+    sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
+
+    build_path = build_project(sources_dir, output_base_path, catch2_path)
+
+    catch_test_names = get_test_names(build_path)
+    ctest_test_names = list_ctest_tests(build_path)
+
+    if len(catch_test_names) != len(ctest_test_names):
+        print("Mismatch between catch test names and ctest test names!")
+        for catch_test in catch_test_names:
+            if catch_test not in ctest_test_names:
+                print(f"Catch2 test '{catch_test}' not found in CTest")
+        for ctest_test in ctest_test_names:
+            if ctest_test not in catch_test_names:
+                print(f"CTest test '{ctest_test}' not found in Catch2")
+
+        exit(1)
+

tests/TestScripts/DiscoverTests/register-tests.cpp

@@ -0,0 +1,11 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/catch_test_macros.hpp>
+
+TEST_CASE("Some test") {}