3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-01-10 11:53:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Support adding test tags as CTest labels in catch_discover_tests

Browse Source 
We also bump the minimum CMake version to 3.20 as per #2943
This commit is contained in:
Martin Hořeňovský 2025-01-03 10:30:47 +01:00
parent b0d0aa43e6
commit 7d7b2f89f2
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
11 changed files with 148 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.conan/test_package/CMakeLists.txt
View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.15)
cmake_minimum_required(VERSION 3.20)
project(PackageTest CXX)
find_package(Catch2 CONFIG REQUIRED)

12
CMakeLists.txt
View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.10)
cmake_minimum_required(VERSION 3.20)
# detect if Catch is being bundled,
# disable testsuite in that case
@ -37,9 +37,7 @@ endif()
project(Catch2
VERSION 3.7.1 # CML version placeholder, don't delete
LANGUAGES CXX
# HOMEPAGE_URL is not supported until CMake version 3.12, which
# we do not target yet.
# HOMEPAGE_URL "https://github.com/catchorg/Catch2"
HOMEPAGE_URL "https://github.com/catchorg/Catch2"
DESCRIPTION "A modern, C++-native, unit test framework."
)
@ -192,12 +190,6 @@ if (NOT_SUBPROJECT)
${PKGCONFIG_INSTALL_DIR}
)
# CPack/CMake started taking the package version from project version 3.12
# So we need to set the version manually for older CMake versions
if(${CMAKE_VERSION} VERSION_LESS "3.12.0")
set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION})
endif()
set(CPACK_PACKAGE_CONTACT "https://github.com/catchorg/Catch2/")

18
docs/cmake-integration.md
View File

@ -81,12 +81,11 @@ to your CMake module path.
`Catch.cmake` provides function `catch_discover_tests` to get tests from
a target. This function works by running the resulting executable with
`--list-test-names-only` flag, and then parsing the output to find all
existing tests.
`--list-test` flag, and then parsing the output to find all existing tests.
#### Usage
```cmake
cmake_minimum_required(VERSION 3.5)
cmake_minimum_required(VERSION 3.20)
project(baz LANGUAGES CXX VERSION 0.0.1)
@ -128,6 +127,8 @@ catch_discover_tests(target
[OUTPUT_PREFIX prefix]
[OUTPUT_SUFFIX suffix]
[DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
[SKIP_IS_FAILURE]
[ADD_TAGS_AS_LABELS]
)
```
@ -211,6 +212,15 @@ execution (useful e.g. in cross-compilation environments).
calling ``catch_discover_tests``. This provides a mechanism for globally
selecting a preferred test discovery behavior.
* `SKIP_IS_FAILURE`
Skipped tests will be marked as failed instead.
* `ADD_TAGS_AS_LABELS`
Add the tags from tests as labels to CTest.
### `ParseAndAddCatchTests.cmake`
⚠ This script is [deprecated](https://github.com/catchorg/Catch2/pull/2120)
@ -229,7 +239,7 @@ parsed are *silently ignored*.
#### Usage
```cmake
cmake_minimum_required(VERSION 3.5)
cmake_minimum_required(VERSION 3.20)
project(baz LANGUAGES CXX VERSION 0.0.1)

2
examples/CMakeLists.txt
View File

@ -1,4 +1,4 @@
cmake_minimum_required( VERSION 3.10 )
cmake_minimum_required( VERSION 3.20 )
project( Catch2Examples LANGUAGES CXX )

32
extras/Catch.cmake
View File

@ -39,6 +39,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
[OUTPUT_SUFFIX suffix]
[DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
[SKIP_IS_FAILURE]
[ADD_TAGS_AS_LABELS]
)
``catch_discover_tests`` sets up a post-build command on the test executable
@ -148,6 +149,9 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
``SKIP_IS_FAILURE``
Disables skipped test detection.
``ADD_TAGS_AS_LABELS``
Adds all test tags as CTest labels.
#]=======================================================================]
#------------------------------------------------------------------------------
@ -155,12 +159,16 @@ function(catch_discover_tests TARGET)
cmake_parse_arguments(
""
"SKIP_IS_FAILURE"
"SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS"
"TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE"
"TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS"
${ARGN}
)
if (${CMAKE_VERSION} VERSION_LESS "3.19")
message(FATAL_ERROR "This script requires JSON support from CMake version 3.19 or greater.")
endif()
if(NOT _WORKING_DIRECTORY)
set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
endif()
@ -222,6 +230,7 @@ function(catch_discover_tests TARGET)
-D "TEST_DL_PATHS=${_DL_PATHS}"
-D "TEST_DL_FRAMEWORK_PATHS=${_DL_FRAMEWORK_PATHS}"
-D "CTEST_FILE=${ctest_tests_file}"
-D "ADD_TAGS_AS_LABELS=${_ADD_TAGS_AS_LABELS}"
-P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
VERBATIM
)
@ -267,6 +276,7 @@ function(catch_discover_tests TARGET)
" CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n"
" TEST_DL_PATHS" " [==[" "${_DL_PATHS}" "]==]" "\n"
" TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}" "]==]" "\n"
" ADD_TAGS_AS_LABELS" " [==[" "${_ADD_TAGS_AS_LABELS}" "]==]" "\n"
" )" "\n"
" endif()" "\n"
" include(\"${ctest_tests_file}\")" "\n"
@ -293,22 +303,10 @@ function(catch_discover_tests TARGET)
endif()
endif()
if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0")
# Add discovered tests to directory TEST_INCLUDE_FILES
set_property(DIRECTORY
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
)
else()
# Add discovered tests as directory TEST_INCLUDE_FILE if possible
get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET)
if (NOT ${test_include_file_set})
set_property(DIRECTORY
PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}"
)
else()
message(FATAL_ERROR "Cannot set more than one TEST_INCLUDE_FILE")
endif()
endif()
# Add discovered tests to directory TEST_INCLUDE_FILES
set_property(DIRECTORY
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
)
endfunction()

86
extras/CatchAddTests.cmake
View File

@ -22,10 +22,11 @@ function(catch_discover_tests_impl)
""
""
"TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
"TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS"
"TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
${ARGN}
)
set(add_tags "${_ADD_TAGS_AS_LABELS}")
set(prefix "${_TEST_PREFIX}")
set(suffix "${_TEST_SUFFIX}")
set(spec ${_TEST_SPEC})
@ -72,25 +73,19 @@ function(catch_discover_tests_impl)
endif()
execute_process(
COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --verbosity quiet
OUTPUT_VARIABLE output
COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --reporter json
OUTPUT_VARIABLE listing_output
RESULT_VARIABLE result
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
)
if(NOT ${result} EQUAL 0)
message(FATAL_ERROR
"Error running test executable '${_TEST_EXECUTABLE}':\n"
"Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
" Result: ${result}\n"
" Output: ${output}\n"
" Output: ${listing_output}\n"
)
endif()
# Make sure to escape ; (semicolons) in test names first, because
# that'd break the foreach loop for "Parse output" later and create
# wrongly splitted and thus failing test cases (false positives)
string(REPLACE ";" "\;" output "${output}")
string(REPLACE "\n" ";" output "${output}")
# Prepare reporter
if(reporter)
set(reporter_arg "--reporter ${reporter}")
@ -110,7 +105,7 @@ function(catch_discover_tests_impl)
)
elseif(NOT ${reporter_check_result} EQUAL 0)
message(FATAL_ERROR
"Error running test executable '${_TEST_EXECUTABLE}':\n"
"Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n"
" Result: ${reporter_check_result}\n"
" Output: ${reporter_check_output}\n"
)
@ -139,46 +134,86 @@ function(catch_discover_tests_impl)
endforeach()
endif()
# Parse output
foreach(line ${output})
set(test "${line}")
# Parse JSON output for list of tests/class names/tags
string(JSON version GET "${listing_output}" "version")
if (NOT version STREQUAL "1")
message(FATAL_ERROR "Unsupported catch output version: '${version}'")
endif()
# Speed-up reparsing by cutting away unneeded parts of JSON.
string(JSON test_listing GET "${listing_output}" "listings" "tests")
string(JSON num_tests LENGTH "${test_listing}")
# CMake's foreach-RANGE is inclusive, so we have to subtract 1
math(EXPR num_tests "${num_tests} - 1")
foreach(idx RANGE ${num_tests})
string(JSON single_test GET ${test_listing} ${idx})
string(JSON test_tags GET "${single_test}" "tags")
string(JSON plain_name GET "${single_test}" "name")
# Escape characters in test case names that would be parsed by Catch2
# Note that the \ escaping must happen FIRST! Do not change the order.
set(test_name "${test}")
foreach(char \\ , [ ])
string(REPLACE ${char} "\\${char}" test_name "${test_name}")
set(escaped_name "${plain_name}")
foreach(char \\ , [ ] ;)
string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
endforeach(char)
# ...add output dir
if(output_dir)
string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean "${test_name}")
set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}")
string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
endif()
# ...and add to script
add_command(add_test
"${prefix}${test}${suffix}"
"${prefix}${plain_name}${suffix}"
${_TEST_EXECUTOR}
"${_TEST_EXECUTABLE}"
"${test_name}"
"${escaped_name}"
${extra_args}
"${reporter_arg}"
"${output_dir_arg}"
)
add_command(set_tests_properties
"${prefix}${test}${suffix}"
"${prefix}${plain_name}${suffix}"
PROPERTIES
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
${properties}
)
if (add_tags)
string(JSON num_tags LENGTH "${test_tags}")
math(EXPR num_tags "${num_tags} - 1")
set(tag_list "")
if (num_tags GREATER_EQUAL "0")
foreach(tag_idx RANGE ${num_tags})
string(JSON a_tag GET "${test_tags}" "${tag_idx}")
# Catch2's tags can contain semicolons, which are list element separators
# in CMake, so we have to escape them. Ideally we could use the [=[...]=]
# syntax for this, but CTest currently keeps the square quotes in the label
# name. So we add 2 backslashes to escape it instead.
# **IMPORTANT**: The number of backslashes depends on how many layers
# of CMake the tag goes. If this script is changed, the
# number of backslashes to escape may change as well.
string(REPLACE ";" "\\;" a_tag "${a_tag}")
list(APPEND tag_list "${a_tag}")
endforeach()
add_command(set_tests_properties
"${prefix}${plain_name}${suffix}"
PROPERTIES
LABELS "${tag_list}"
)
endif()
endif(add_tags)
if(environment_modifications)
add_command(set_tests_properties
"${prefix}${test}${suffix}"
"${prefix}${plain_name}${suffix}"
PROPERTIES
ENVIRONMENT_MODIFICATION "${environment_modifications}")
endif()
list(APPEND tests "${prefix}${test}${suffix}")
list(APPEND tests "${prefix}${plain_name}${suffix}")
endforeach()
# Create a list of all discovered tests, which users may use to e.g. set
@ -207,5 +242,6 @@ if(CMAKE_SCRIPT_MODE_FILE)
TEST_DL_PATHS ${TEST_DL_PATHS}
TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
CTEST_FILE ${CTEST_FILE}
ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
)
endif()

2
tests/ExtraTests/CMakeLists.txt
View File

@ -2,7 +2,7 @@
# Build extra tests.
#
cmake_minimum_required( VERSION 3.10 )
cmake_minimum_required( VERSION 3.20 )
project( Catch2ExtraTests LANGUAGES CXX )

11
tests/TestScripts/DiscoverTests/CMakeLists.txt
View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.10)
cmake_minimum_required(VERSION 3.20)
project(discover-tests-test
LANGUAGES CXX
@ -19,4 +19,11 @@ if (CMAKE_VERSION GREATER_EQUAL 3.27)
DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.."
)
endif ()
catch_discover_tests(tests ${extra_args})
catch_discover_tests(
tests
ADD_TAGS_AS_LABELS
DISCOVERY_MODE PRE_TEST
${extra_args}
)
# DISCOVERY_MODE <POST_BUILD|PRE_TEST>

50
tests/TestScripts/DiscoverTests/VerifyRegistration.py
View File

@ -12,6 +12,10 @@ import subprocess
import sys
import re
import json
from collections import namedtuple
from typing import List
TestInfo = namedtuple('TestInfo', ['name', 'tags'])
cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)')
@ -61,7 +65,7 @@ def build_project(sources_dir, output_base_path, catch2_path):
def get_test_names(build_path):
def get_test_names(build_path: str) -> List[TestInfo]:
# For now we assume that Windows builds are done using MSBuild under
# Debug configuration. This means that we need to add "Debug" folder
# to the path when constructing it. On Linux, we don't add anything.
@ -69,15 +73,23 @@ def get_test_names(build_path):
full_path = os.path.join(build_path, config_path, 'tests')
cmd = [full_path, '--reporter', 'xml', '--list-tests']
cmd = [full_path, '--reporter', 'json', '--list-tests']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
import xml.etree.ElementTree as ET
root = ET.fromstring(result.stdout)
return [tc.text for tc in root.findall('TestCase/Name')]
test_listing = json.loads(result.stdout)
assert test_listing['version'] == 1
tests = []
for test in test_listing['listings']['tests']:
test_name = test['name']
tags = test['tags']
tests.append(TestInfo(test_name, tags))
return tests
def get_ctest_listing(build_path):
old_path = os.getcwd()
@ -91,20 +103,25 @@ def get_ctest_listing(build_path):
os.chdir(old_path)
return result.stdout
def extract_tests_from_ctest(ctest_output):
def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
ctest_response = json.loads(ctest_output)
tests = ctest_response['tests']
test_names = []
test_infos = []
for test in tests:
test_command = test['command']
# First part of the command is the binary, second is the filter.
# If there are less, registration has failed. If there are more,
# registration has changed and the script needs updating.
assert len(test_command) == 2
test_names.append(test_command[1])
test_name = test_command[1]
labels = []
for prop in test['properties']:
if prop['name'] == 'LABELS':
labels = prop['value']
return test_names
test_infos.append(TestInfo(test_name, labels))
return test_infos
def check_DL_PATHS(ctest_output):
ctest_response = json.loads(ctest_output)
@ -115,10 +132,14 @@ def check_DL_PATHS(ctest_output):
if property['name'] == 'ENVIRONMENT_MODIFICATION':
assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
def escape_catch2_test_name(name):
for char in ('\\', ',', '[', ']'):
name = name.replace(char, f"\\{char}")
return name
def escape_catch2_test_names(infos: List[TestInfo]):
escaped = []
for info in infos:
name = info.name
for char in ('\\', ',', '[', ']'):
name = name.replace(char, f"\\{char}")
escaped.append(TestInfo(name, info.tags))
return escaped
if __name__ == '__main__':
if len(sys.argv) != 3:
@ -130,7 +151,7 @@ if __name__ == '__main__':
build_path = build_project(sources_dir, output_base_path, catch2_path)
catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)]
catch_test_names = escape_catch2_test_names(get_test_names(build_path))
ctest_output = get_ctest_listing(build_path)
ctest_test_names = extract_tests_from_ctest(ctest_output)
@ -147,6 +168,7 @@ if __name__ == '__main__':
if mismatched:
print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
exit(1)
print(f"{len(catch_test_names)} tests matched")
cmake_version = get_cmake_version()
if cmake_version >= (3, 27):

7
tests/TestScripts/DiscoverTests/register-tests.cpp
View File

@ -14,3 +14,10 @@ TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. "
"Really looooooooooooong. Even longer than that. Multiple lines "
"worth of test name. Yep, like this." ) {}
TEST_CASE( "And now a test case with weird tags.", "[tl;dr][tl;dw][foo,bar]" ) {}
// Also check that we handle tests on class, which have name in output as 'class-name', not 'name'.
class TestCaseFixture {
public:
int m_a;
};
TEST_CASE_METHOD(TestCaseFixture, "A test case as method", "[tagstagstags]") {}

2
tools/misc/CMakeLists.txt
View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.0)
cmake_minimum_required(VERSION 3.20)
project(CatchCoverageHelper)