Add test scaffolding for catch_discover_tests

This commit is contained in:
Martin Hořeňovský 2023-06-14 16:30:56 +02:00
parent 7a52dfa77b
commit c8363143e7
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
7 changed files with 163 additions and 5 deletions

View File

@ -29,13 +29,13 @@ jobs:
build_type: Debug build_type: Debug
std: 14 std: 14
other_pkgs: g++-7 other_pkgs: g++-7
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
- cxx: g++-7 - cxx: g++-7
build_description: Extras + Examples build_description: Extras + Examples
build_type: Release build_type: Release
std: 14 std: 14
other_pkgs: g++-7 other_pkgs: g++-7
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
# Extras and examples with Clang-10 # Extras and examples with Clang-10
- cxx: clang++-10 - cxx: clang++-10
@ -43,13 +43,13 @@ jobs:
build_type: Debug build_type: Debug
std: 17 std: 17
other_pkgs: clang-10 other_pkgs: clang-10
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
- cxx: clang++-10 - cxx: clang++-10
build_description: Extras + Examples build_description: Extras + Examples
build_type: Release build_type: Release
std: 17 std: 17
other_pkgs: clang-10 other_pkgs: clang-10
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
# Configure tests with Clang-10 # Configure tests with Clang-10
- cxx: clang++-10 - cxx: clang++-10

View File

@ -21,6 +21,7 @@ cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io"
cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF)
cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
cmake_dependent_option(CATCH_ENABLE_CMAKE_HELPER_TESTS "Enable CMake helper tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
# Catch2's build breaks if done in-tree. You probably should not build # Catch2's build breaks if done in-tree. You probably should not build

View File

@ -18,7 +18,8 @@
"CATCH_BUILD_EXAMPLES": "ON", "CATCH_BUILD_EXAMPLES": "ON",
"CATCH_BUILD_EXTRA_TESTS": "ON", "CATCH_BUILD_EXTRA_TESTS": "ON",
"CATCH_BUILD_SURROGATES": "ON", "CATCH_BUILD_SURROGATES": "ON",
"CATCH_ENABLE_CONFIGURE_TESTS": "ON" "CATCH_ENABLE_CONFIGURE_TESTS": "ON",
"CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON"
} }
} }
] ]

View File

@ -622,6 +622,18 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
endforeach() endforeach()
endif() endif()
if (CATCH_ENABLE_CMAKE_HELPER_TESTS)
add_test(NAME "CMakeHelper::DiscoverTests"
COMMAND
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/DiscoverTests/VerifyRegistration.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
)
set_tests_properties("CMakeHelper::DiscoverTests"
PROPERTIES
COST 240
LABELS "uses-python"
)
endif()
foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
"compact" "compact"
"console" "console"

View File

@ -0,0 +1,16 @@
cmake_minimum_required(VERSION 3.10)
project(discover-tests-test
LANGUAGES CXX
)
add_executable(tests
register-tests.cpp
)
add_subdirectory(${CATCH2_PATH} catch2-build)
target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
include(CTest)
include(Catch)
catch_discover_tests(tests)

View File

@ -0,0 +1,117 @@
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
import os
import subprocess
import sys
def build_project(sources_dir, output_base_path, catch2_path):
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
config_cmd = ['cmake',
'-B', build_dir,
'-S', sources_dir,
f'-DCATCH2_PATH={catch2_path}',
'-DCMAKE_BUILD_TYPE=Debug']
build_cmd = ['cmake',
'--build', build_dir,
'--config', 'Debug']
try:
subprocess.run(config_cmd,
capture_output = True,
check = True,
text = True)
subprocess.run(build_cmd,
capture_output = True,
check = True,
text = True)
except subprocess.CalledProcessError as err:
print('Error when building the test project')
print(f'cmd: {err.cmd}')
print(f'stderr: {err.stderr}')
print(f'stdout: {err.stdout}')
exit(3)
return build_dir
def get_test_names(build_path):
# For now we assume that Windows builds are done using MSBuild under
# Debug configuration. This means that we need to add "Debug" folder
# to the path when constructing it. On Linux, we don't add anything.
config_path = "Debug" if os.name == 'nt' else ""
full_path = os.path.join(build_path, config_path, 'tests')
cmd = [full_path, '--reporter', 'xml', '--list-tests']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
import xml.etree.ElementTree as ET
root = ET.fromstring(result.stdout)
return [tc.text for tc in root.findall('TestCase/Name')]
def list_ctest_tests(build_path):
old_path = os.getcwd()
os.chdir(build_path)
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
os.chdir(old_path)
import json
ctest_response = json.loads(result.stdout)
tests = ctest_response['tests']
test_names = []
for test in tests:
test_command = test['command']
# First part of the command is the binary, second is the filter.
# If there are less, registration has failed. If there are more,
# registration has changed and the script needs updating.
assert len(test_command) == 2
test_names.append(test_command[1])
test_name = test_command[1]
return test_names
if __name__ == '__main__':
if len(sys.argv) != 3:
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
exit(2)
catch2_path = sys.argv[1]
output_base_path = sys.argv[2]
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
build_path = build_project(sources_dir, output_base_path, catch2_path)
catch_test_names = get_test_names(build_path)
ctest_test_names = list_ctest_tests(build_path)
if len(catch_test_names) != len(ctest_test_names):
print("Mismatch between catch test names and ctest test names!")
for catch_test in catch_test_names:
if catch_test not in ctest_test_names:
print(f"Catch2 test '{catch_test}' not found in CTest")
for ctest_test in ctest_test_names:
if ctest_test not in catch_test_names:
print(f"CTest test '{ctest_test}' not found in Catch2")
exit(1)

View File

@ -0,0 +1,11 @@
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#include <catch2/catch_test_macros.hpp>
TEST_CASE("Some test") {}