From c8363143e7caad34520042158468a69aeae59a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Wed, 14 Jun 2023 16:30:56 +0200 Subject: [PATCH] Add test scaffolding for catch_discover_tests --- .github/workflows/linux-other-builds.yml | 8 +- CMakeLists.txt | 1 + CMakePresets.json | 3 +- tests/CMakeLists.txt | 12 ++ .../TestScripts/DiscoverTests/CMakeLists.txt | 16 +++ .../DiscoverTests/VerifyRegistration.py | 117 ++++++++++++++++++ .../DiscoverTests/register-tests.cpp | 11 ++ 7 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 tests/TestScripts/DiscoverTests/CMakeLists.txt create mode 100644 tests/TestScripts/DiscoverTests/VerifyRegistration.py create mode 100644 tests/TestScripts/DiscoverTests/register-tests.cpp diff --git a/.github/workflows/linux-other-builds.yml b/.github/workflows/linux-other-builds.yml index cf4e2c06..da24104d 100644 --- a/.github/workflows/linux-other-builds.yml +++ b/.github/workflows/linux-other-builds.yml @@ -29,13 +29,13 @@ jobs: build_type: Debug std: 14 other_pkgs: g++-7 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON - cxx: g++-7 build_description: Extras + Examples build_type: Release std: 14 other_pkgs: g++-7 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON # Extras and examples with Clang-10 - cxx: clang++-10 @@ -43,13 +43,13 @@ jobs: build_type: Debug std: 17 other_pkgs: clang-10 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON - cxx: clang++-10 build_description: Extras + Examples build_type: Release std: 17 other_pkgs: clang-10 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON # Configure tests with Clang-10 - cxx: clang++-10 diff --git a/CMakeLists.txt b/CMakeLists.txt index b3e81153..5ef06de7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io" cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF) +cmake_dependent_option(CATCH_ENABLE_CMAKE_HELPER_TESTS "Enable CMake helper tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF) # Catch2's build breaks if done in-tree. You probably should not build diff --git a/CMakePresets.json b/CMakePresets.json index 00f3a6d3..88541285 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -18,7 +18,8 @@ "CATCH_BUILD_EXAMPLES": "ON", "CATCH_BUILD_EXTRA_TESTS": "ON", "CATCH_BUILD_SURROGATES": "ON", - "CATCH_ENABLE_CONFIGURE_TESTS": "ON" + "CATCH_ENABLE_CONFIGURE_TESTS": "ON", + "CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON" } } ] diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7be57abe..14a68e34 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -622,6 +622,18 @@ if (CATCH_ENABLE_CONFIGURE_TESTS) endforeach() endif() +if (CATCH_ENABLE_CMAKE_HELPER_TESTS) + add_test(NAME "CMakeHelper::DiscoverTests" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/DiscoverTests/VerifyRegistration.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + set_tests_properties("CMakeHelper::DiscoverTests" + PROPERTIES + COST 240 + LABELS "uses-python" + ) +endif() + foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata "compact" "console" diff --git a/tests/TestScripts/DiscoverTests/CMakeLists.txt b/tests/TestScripts/DiscoverTests/CMakeLists.txt new file mode 100644 index 00000000..d19f2f88 --- /dev/null +++ b/tests/TestScripts/DiscoverTests/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.10) + +project(discover-tests-test + LANGUAGES CXX +) + +add_executable(tests + register-tests.cpp +) + +add_subdirectory(${CATCH2_PATH} catch2-build) +target_link_libraries(tests PRIVATE Catch2::Catch2WithMain) + +include(CTest) +include(Catch) +catch_discover_tests(tests) diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py new file mode 100644 index 00000000..058499e1 --- /dev/null +++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +import os +import subprocess +import sys + + +def build_project(sources_dir, output_base_path, catch2_path): + build_dir = os.path.join(output_base_path, 'ctest-registration-test') + config_cmd = ['cmake', + '-B', build_dir, + '-S', sources_dir, + f'-DCATCH2_PATH={catch2_path}', + '-DCMAKE_BUILD_TYPE=Debug'] + + build_cmd = ['cmake', + '--build', build_dir, + '--config', 'Debug'] + + try: + subprocess.run(config_cmd, + capture_output = True, + check = True, + text = True) + subprocess.run(build_cmd, + capture_output = True, + check = True, + text = True) + except subprocess.CalledProcessError as err: + print('Error when building the test project') + print(f'cmd: {err.cmd}') + print(f'stderr: {err.stderr}') + print(f'stdout: {err.stdout}') + exit(3) + + return build_dir + + + +def get_test_names(build_path): + # For now we assume that Windows builds are done using MSBuild under + # Debug configuration. This means that we need to add "Debug" folder + # to the path when constructing it. On Linux, we don't add anything. + config_path = "Debug" if os.name == 'nt' else "" + full_path = os.path.join(build_path, config_path, 'tests') + + + cmd = [full_path, '--reporter', 'xml', '--list-tests'] + result = subprocess.run(cmd, + capture_output = True, + check = True, + text = True) + + import xml.etree.ElementTree as ET + root = ET.fromstring(result.stdout) + return [tc.text for tc in root.findall('TestCase/Name')] + + +def list_ctest_tests(build_path): + old_path = os.getcwd() + os.chdir(build_path) + + cmd = ['ctest', '-C', 'debug', '--show-only=json-v1'] + result = subprocess.run(cmd, + capture_output = True, + check = True, + text = True) + os.chdir(old_path) + + import json + + ctest_response = json.loads(result.stdout) + tests = ctest_response['tests'] + test_names = [] + for test in tests: + test_command = test['command'] + # First part of the command is the binary, second is the filter. + # If there are less, registration has failed. If there are more, + # registration has changed and the script needs updating. + assert len(test_command) == 2 + test_names.append(test_command[1]) + test_name = test_command[1] + + return test_names + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path') + exit(2) + catch2_path = sys.argv[1] + output_base_path = sys.argv[2] + sources_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + + build_path = build_project(sources_dir, output_base_path, catch2_path) + + catch_test_names = get_test_names(build_path) + ctest_test_names = list_ctest_tests(build_path) + + if len(catch_test_names) != len(ctest_test_names): + print("Mismatch between catch test names and ctest test names!") + for catch_test in catch_test_names: + if catch_test not in ctest_test_names: + print(f"Catch2 test '{catch_test}' not found in CTest") + for ctest_test in ctest_test_names: + if ctest_test not in catch_test_names: + print(f"CTest test '{ctest_test}' not found in Catch2") + + exit(1) + diff --git a/tests/TestScripts/DiscoverTests/register-tests.cpp b/tests/TestScripts/DiscoverTests/register-tests.cpp new file mode 100644 index 00000000..91382eaa --- /dev/null +++ b/tests/TestScripts/DiscoverTests/register-tests.cpp @@ -0,0 +1,11 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include + +TEST_CASE("Some test") {}