mirror of https://github.com/catchorg/Catch2.git
Add test scaffolding for catch_discover_tests
This commit is contained in:
Martin Hořeňovský
parent
7a52dfa77b
commit
c8363143e7
|
|
@ -29,13 +29,13 @@ jobs:
|
|||
build_type: Debug
|
||||
std: 14
|
||||
other_pkgs: g++-7
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
|
||||
- cxx: g++-7
|
||||
build_description: Extras + Examples
|
||||
build_type: Release
|
||||
std: 14
|
||||
other_pkgs: g++-7
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
|
||||
|
||||
# Extras and examples with Clang-10
|
||||
- cxx: clang++-10
|
||||
|
|
@ -43,13 +43,13 @@ jobs:
|
|||
build_type: Debug
|
||||
std: 17
|
||||
other_pkgs: clang-10
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
|
||||
- cxx: clang++-10
|
||||
build_description: Extras + Examples
|
||||
build_type: Release
|
||||
std: 17
|
||||
other_pkgs: clang-10
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON
|
||||
cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON
|
||||
|
||||
# Configure tests with Clang-10
|
||||
- cxx: clang++-10
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io"
|
|||
cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF)
|
||||
cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
|
||||
cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
|
||||
cmake_dependent_option(CATCH_ENABLE_CMAKE_HELPER_TESTS "Enable CMake helper tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
|
||||
|
||||
|
||||
# Catch2's build breaks if done in-tree. You probably should not build
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@
|
|||
"CATCH_BUILD_EXAMPLES": "ON",
|
||||
"CATCH_BUILD_EXTRA_TESTS": "ON",
|
||||
"CATCH_BUILD_SURROGATES": "ON",
|
||||
"CATCH_ENABLE_CONFIGURE_TESTS": "ON"
|
||||
"CATCH_ENABLE_CONFIGURE_TESTS": "ON",
|
||||
"CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -622,6 +622,18 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
|
|||
endforeach()
|
||||
endif()
|
||||
|
||||
if (CATCH_ENABLE_CMAKE_HELPER_TESTS)
|
||||
add_test(NAME "CMakeHelper::DiscoverTests"
|
||||
COMMAND
|
||||
"${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/DiscoverTests/VerifyRegistration.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
)
|
||||
set_tests_properties("CMakeHelper::DiscoverTests"
|
||||
PROPERTIES
|
||||
COST 240
|
||||
LABELS "uses-python"
|
||||
)
|
||||
endif()
|
||||
|
||||
foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
|
||||
"compact"
|
||||
"console"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
cmake_minimum_required(VERSION 3.10)
|
||||
|
||||
project(discover-tests-test
|
||||
LANGUAGES CXX
|
||||
)
|
||||
|
||||
add_executable(tests
|
||||
register-tests.cpp
|
||||
)
|
||||
|
||||
add_subdirectory(${CATCH2_PATH} catch2-build)
|
||||
target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
|
||||
|
||||
include(CTest)
|
||||
include(Catch)
|
||||
catch_discover_tests(tests)
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright Catch2 Authors
|
||||
# Distributed under the Boost Software License, Version 1.0.
|
||||
# (See accompanying file LICENSE.txt or copy at
|
||||
# https://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
# SPDX-License-Identifier: BSL-1.0
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def build_project(sources_dir, output_base_path, catch2_path):
|
||||
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
|
||||
config_cmd = ['cmake',
|
||||
'-B', build_dir,
|
||||
'-S', sources_dir,
|
||||
f'-DCATCH2_PATH={catch2_path}',
|
||||
'-DCMAKE_BUILD_TYPE=Debug']
|
||||
|
||||
build_cmd = ['cmake',
|
||||
'--build', build_dir,
|
||||
'--config', 'Debug']
|
||||
|
||||
try:
|
||||
subprocess.run(config_cmd,
|
||||
capture_output = True,
|
||||
check = True,
|
||||
text = True)
|
||||
subprocess.run(build_cmd,
|
||||
capture_output = True,
|
||||
check = True,
|
||||
text = True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
print('Error when building the test project')
|
||||
print(f'cmd: {err.cmd}')
|
||||
print(f'stderr: {err.stderr}')
|
||||
print(f'stdout: {err.stdout}')
|
||||
exit(3)
|
||||
|
||||
return build_dir
|
||||
|
||||
|
||||
|
||||
def get_test_names(build_path):
|
||||
# For now we assume that Windows builds are done using MSBuild under
|
||||
# Debug configuration. This means that we need to add "Debug" folder
|
||||
# to the path when constructing it. On Linux, we don't add anything.
|
||||
config_path = "Debug" if os.name == 'nt' else ""
|
||||
full_path = os.path.join(build_path, config_path, 'tests')
|
||||
|
||||
|
||||
cmd = [full_path, '--reporter', 'xml', '--list-tests']
|
||||
result = subprocess.run(cmd,
|
||||
capture_output = True,
|
||||
check = True,
|
||||
text = True)
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.fromstring(result.stdout)
|
||||
return [tc.text for tc in root.findall('TestCase/Name')]
|
||||
|
||||
|
||||
def list_ctest_tests(build_path):
|
||||
old_path = os.getcwd()
|
||||
os.chdir(build_path)
|
||||
|
||||
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
|
||||
result = subprocess.run(cmd,
|
||||
capture_output = True,
|
||||
check = True,
|
||||
text = True)
|
||||
os.chdir(old_path)
|
||||
|
||||
import json
|
||||
|
||||
ctest_response = json.loads(result.stdout)
|
||||
tests = ctest_response['tests']
|
||||
test_names = []
|
||||
for test in tests:
|
||||
test_command = test['command']
|
||||
# First part of the command is the binary, second is the filter.
|
||||
# If there are less, registration has failed. If there are more,
|
||||
# registration has changed and the script needs updating.
|
||||
assert len(test_command) == 2
|
||||
test_names.append(test_command[1])
|
||||
test_name = test_command[1]
|
||||
|
||||
return test_names
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 3:
|
||||
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
|
||||
exit(2)
|
||||
catch2_path = sys.argv[1]
|
||||
output_base_path = sys.argv[2]
|
||||
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
|
||||
|
||||
build_path = build_project(sources_dir, output_base_path, catch2_path)
|
||||
|
||||
catch_test_names = get_test_names(build_path)
|
||||
ctest_test_names = list_ctest_tests(build_path)
|
||||
|
||||
if len(catch_test_names) != len(ctest_test_names):
|
||||
print("Mismatch between catch test names and ctest test names!")
|
||||
for catch_test in catch_test_names:
|
||||
if catch_test not in ctest_test_names:
|
||||
print(f"Catch2 test '{catch_test}' not found in CTest")
|
||||
for ctest_test in ctest_test_names:
|
||||
if ctest_test not in catch_test_names:
|
||||
print(f"CTest test '{ctest_test}' not found in Catch2")
|
||||
|
||||
exit(1)
|
||||
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
|
||||
// Copyright Catch2 Authors
|
||||
// Distributed under the Boost Software License, Version 1.0.
|
||||
// (See accompanying file LICENSE.txt or copy at
|
||||
// https://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
// SPDX-License-Identifier: BSL-1.0
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
TEST_CASE("Some test") {}
|
||||
Loading…
Reference in New Issue