Support adding test tags as CTest labels in catch_discover_tests

We also bump the minimum CMake version to 3.20 as per #2943

.conan/test_package/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.15)
+cmake_minimum_required(VERSION 3.20)
 project(PackageTest CXX)
 
 find_package(Catch2 CONFIG REQUIRED)

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.20)
 
 # detect if Catch is being bundled,
 # disable testsuite in that case
@@ -37,9 +37,7 @@ endif()
 project(Catch2
   VERSION 3.7.1 # CML version placeholder, don't delete
   LANGUAGES CXX
-  # HOMEPAGE_URL is not supported until CMake version 3.12, which
+  HOMEPAGE_URL "https://github.com/catchorg/Catch2"
-  # we do not target yet.
-  # HOMEPAGE_URL "https://github.com/catchorg/Catch2"
   DESCRIPTION "A modern, C++-native, unit test framework."
 )
 
@@ -192,12 +190,6 @@ if (NOT_SUBPROJECT)
         ${PKGCONFIG_INSTALL_DIR}
     )
 
-    # CPack/CMake started taking the package version from project version 3.12
-    # So we need to set the version manually for older CMake versions
-    if(${CMAKE_VERSION} VERSION_LESS "3.12.0")
-        set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION})
-    endif()
-
     set(CPACK_PACKAGE_CONTACT "https://github.com/catchorg/Catch2/")
 
 

docs/cmake-integration.md

@@ -81,12 +81,11 @@ to your CMake module path.
 
 `Catch.cmake` provides function `catch_discover_tests` to get tests from
 a target. This function works by running the resulting executable with
-`--list-test-names-only` flag, and then parsing the output to find all
+`--list-test` flag, and then parsing the output to find all existing tests.
-existing tests.
 
 #### Usage
 ```cmake
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.20)
 
 project(baz LANGUAGES CXX VERSION 0.0.1)
 
@@ -128,6 +127,8 @@ catch_discover_tests(target
                      [OUTPUT_PREFIX prefix]
                      [OUTPUT_SUFFIX suffix]
                      [DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
+                     [SKIP_IS_FAILURE]
+                     [ADD_TAGS_AS_LABELS]
 )
 ```
 
@@ -211,6 +212,15 @@ execution (useful e.g. in cross-compilation environments).
 calling ``catch_discover_tests``. This provides a mechanism for globally
 selecting a preferred test discovery behavior.
 
+* `SKIP_IS_FAILURE`
+
+Skipped tests will be marked as failed instead.
+
+* `ADD_TAGS_AS_LABELS`
+
+Add the tags from tests as labels to CTest.
+
+
 ### `ParseAndAddCatchTests.cmake`
 
 ⚠ This script is [deprecated](https://github.com/catchorg/Catch2/pull/2120)
@@ -229,7 +239,7 @@ parsed are *silently ignored*.
 #### Usage
 
 ```cmake
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.20)
 
 project(baz LANGUAGES CXX VERSION 0.0.1)
 

examples/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 3.10 )
+cmake_minimum_required( VERSION 3.20 )
 
 project( Catch2Examples LANGUAGES CXX )
 

extras/Catch.cmake

@@ -39,6 +39,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
                          [OUTPUT_SUFFIX suffix]
                          [DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
                          [SKIP_IS_FAILURE]
+                         [ADD_TAGS_AS_LABELS]
     )
 
   ``catch_discover_tests`` sets up a post-build command on the test executable
@@ -148,6 +149,9 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
   ``SKIP_IS_FAILURE``
     Disables skipped test detection.
 
+  ``ADD_TAGS_AS_LABELS``
+    Adds all test tags as CTest labels.
+
 #]=======================================================================]
 
 #------------------------------------------------------------------------------
@@ -155,12 +159,16 @@ function(catch_discover_tests TARGET)
 
   cmake_parse_arguments(
     ""
-    "SKIP_IS_FAILURE"
+    "SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS"
     "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE"
     "TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS"
     ${ARGN}
   )
 
+  if (${CMAKE_VERSION} VERSION_LESS "3.19")
+    message(FATAL_ERROR "This script requires JSON support from CMake version 3.19 or greater.")
+  endif()
+
   if(NOT _WORKING_DIRECTORY)
     set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
   endif()
@@ -222,6 +230,7 @@ function(catch_discover_tests TARGET)
               -D "TEST_DL_PATHS=${_DL_PATHS}"
               -D "TEST_DL_FRAMEWORK_PATHS=${_DL_FRAMEWORK_PATHS}"
               -D "CTEST_FILE=${ctest_tests_file}"
+              -D "ADD_TAGS_AS_LABELS=${_ADD_TAGS_AS_LABELS}"
               -P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
       VERBATIM
     )
@@ -267,6 +276,7 @@ function(catch_discover_tests TARGET)
       "      CTEST_FILE"             " [==[" "${ctest_tests_file}"        "]==]"   "\n"
       "      TEST_DL_PATHS"          " [==[" "${_DL_PATHS}"               "]==]"   "\n"
       "      TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}"     "]==]"   "\n"
+      "      ADD_TAGS_AS_LABELS"     " [==[" "${_ADD_TAGS_AS_LABELS}"     "]==]"   "\n"
       "    )"                                                                      "\n"
       "  endif()"                                                                  "\n"
       "  include(\"${ctest_tests_file}\")"                                         "\n"
@@ -293,22 +303,10 @@ function(catch_discover_tests TARGET)
     endif()
   endif()
 
-  if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0")
+  # Add discovered tests to directory TEST_INCLUDE_FILES
-    # Add discovered tests to directory TEST_INCLUDE_FILES
+  set_property(DIRECTORY
-    set_property(DIRECTORY
+    APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
-      APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
+  )
-    )
-  else()
-    # Add discovered tests as directory TEST_INCLUDE_FILE if possible
-    get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET)
-    if (NOT ${test_include_file_set})
-      set_property(DIRECTORY
-        PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}"
-      )
-    else()
-      message(FATAL_ERROR "Cannot set more than one TEST_INCLUDE_FILE")
-    endif()
-  endif()
 
 endfunction()
 

extras/CatchAddTests.cmake

@@ -22,10 +22,11 @@ function(catch_discover_tests_impl)
     ""
     ""
     "TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
-    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS"
+    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
     ${ARGN}
   )
 
+  set(add_tags "${_ADD_TAGS_AS_LABELS}")
   set(prefix "${_TEST_PREFIX}")
   set(suffix "${_TEST_SUFFIX}")
   set(spec ${_TEST_SPEC})
@@ -72,25 +73,19 @@ function(catch_discover_tests_impl)
   endif()
 
   execute_process(
-    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --verbosity quiet
+    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --reporter json
-    OUTPUT_VARIABLE output
+    OUTPUT_VARIABLE listing_output
     RESULT_VARIABLE result
     WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
   )
   if(NOT ${result} EQUAL 0)
     message(FATAL_ERROR
-      "Error running test executable '${_TEST_EXECUTABLE}':\n"
+      "Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
       "  Result: ${result}\n"
-      "  Output: ${output}\n"
+      "  Output: ${listing_output}\n"
     )
   endif()
 
-  # Make sure to escape ; (semicolons) in test names first, because
-  # that'd break the foreach loop for "Parse output" later and create
-  # wrongly splitted and thus failing test cases (false positives)
-  string(REPLACE ";" "\;" output "${output}")
-  string(REPLACE "\n" ";" output "${output}")
-
   # Prepare reporter
   if(reporter)
     set(reporter_arg "--reporter ${reporter}")
@@ -110,7 +105,7 @@ function(catch_discover_tests_impl)
       )
     elseif(NOT ${reporter_check_result} EQUAL 0)
       message(FATAL_ERROR
-        "Error running test executable '${_TEST_EXECUTABLE}':\n"
+        "Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n"
         "  Result: ${reporter_check_result}\n"
         "  Output: ${reporter_check_output}\n"
       )
@@ -139,46 +134,86 @@ function(catch_discover_tests_impl)
     endforeach()
   endif()
 
-  # Parse output
+  # Parse JSON output for list of tests/class names/tags
-  foreach(line ${output})
+  string(JSON version GET "${listing_output}" "version")
-    set(test "${line}")
+  if (NOT version STREQUAL "1")
+    message(FATAL_ERROR "Unsupported catch output version: '${version}'")
+  endif()
+
+  # Speed-up reparsing by cutting away unneeded parts of JSON.
+  string(JSON test_listing GET "${listing_output}" "listings" "tests")
+  string(JSON num_tests LENGTH "${test_listing}")
+  # CMake's foreach-RANGE is inclusive, so we have to subtract 1
+  math(EXPR num_tests "${num_tests} - 1")
+
+  foreach(idx RANGE ${num_tests})
+    string(JSON single_test GET ${test_listing} ${idx})
+    string(JSON test_tags GET "${single_test}" "tags")
+    string(JSON plain_name GET "${single_test}" "name")
+
     # Escape characters in test case names that would be parsed by Catch2
     # Note that the \ escaping must happen FIRST! Do not change the order.
-    set(test_name "${test}")
+    set(escaped_name "${plain_name}")
-    foreach(char \\ , [ ])
+    foreach(char \\ , [ ] ;)
-      string(REPLACE ${char} "\\${char}" test_name "${test_name}")
+      string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
     endforeach(char)
     # ...add output dir
     if(output_dir)
-      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean "${test_name}")
+      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
-      set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}")
+      set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
     endif()
 
     # ...and add to script
     add_command(add_test
-      "${prefix}${test}${suffix}"
+      "${prefix}${plain_name}${suffix}"
       ${_TEST_EXECUTOR}
       "${_TEST_EXECUTABLE}"
-      "${test_name}"
+      "${escaped_name}"
       ${extra_args}
       "${reporter_arg}"
       "${output_dir_arg}"
     )
     add_command(set_tests_properties
-      "${prefix}${test}${suffix}"
+      "${prefix}${plain_name}${suffix}"
       PROPERTIES
       WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
       ${properties}
     )
 
+    if (add_tags)
+      string(JSON num_tags LENGTH "${test_tags}")
+      math(EXPR num_tags "${num_tags} - 1")
+      set(tag_list "")
+      if (num_tags GREATER_EQUAL "0")
+        foreach(tag_idx RANGE ${num_tags})
+          string(JSON a_tag GET "${test_tags}" "${tag_idx}")
+          # Catch2's tags can contain semicolons, which are list element separators
+          # in CMake, so we have to escape them. Ideally we could use the [=[...]=]
+          # syntax for this, but CTest currently keeps the square quotes in the label
+          # name. So we add 2 backslashes to escape it instead.
+          # **IMPORTANT**: The number of backslashes depends on how many layers
+          #                of CMake the tag goes. If this script is changed, the
+          #                number of backslashes to escape may change as well.
+          string(REPLACE ";" "\\;" a_tag "${a_tag}")
+          list(APPEND tag_list "${a_tag}")
+        endforeach()
+
+        add_command(set_tests_properties
+          "${prefix}${plain_name}${suffix}"
+          PROPERTIES
+          LABELS "${tag_list}"
+        )
+      endif()
+    endif(add_tags)
+
     if(environment_modifications)
       add_command(set_tests_properties
-        "${prefix}${test}${suffix}"
+        "${prefix}${plain_name}${suffix}"
         PROPERTIES
         ENVIRONMENT_MODIFICATION "${environment_modifications}")
     endif()
 
-    list(APPEND tests "${prefix}${test}${suffix}")
+    list(APPEND tests "${prefix}${plain_name}${suffix}")
   endforeach()
 
   # Create a list of all discovered tests, which users may use to e.g. set
@@ -207,5 +242,6 @@ if(CMAKE_SCRIPT_MODE_FILE)
     TEST_DL_PATHS ${TEST_DL_PATHS}
     TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
     CTEST_FILE ${CTEST_FILE}
+    ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
   )
 endif()

tests/ExtraTests/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Build extra tests.
 #
 
-cmake_minimum_required( VERSION 3.10 )
+cmake_minimum_required( VERSION 3.20 )
 
 project( Catch2ExtraTests LANGUAGES CXX )
 

tests/TestScripts/DiscoverTests/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.20)
 
 project(discover-tests-test
   LANGUAGES CXX
@@ -19,4 +19,11 @@ if (CMAKE_VERSION GREATER_EQUAL 3.27)
             DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.."
     )
 endif ()
-catch_discover_tests(tests ${extra_args})
+catch_discover_tests(
+  tests
+  ADD_TAGS_AS_LABELS
+  DISCOVERY_MODE PRE_TEST
+  ${extra_args}
+)
+
+# DISCOVERY_MODE <POST_BUILD|PRE_TEST>

tests/TestScripts/DiscoverTests/VerifyRegistration.py

@@ -12,6 +12,10 @@ import subprocess
 import sys
 import re
 import json
+from collections import namedtuple
+from typing import List
+
+TestInfo = namedtuple('TestInfo', ['name', 'tags'])
 
 cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)')
 
@@ -61,7 +65,7 @@ def build_project(sources_dir, output_base_path, catch2_path):
 
 
 
-def get_test_names(build_path):
+def get_test_names(build_path: str) -> List[TestInfo]:
     # For now we assume that Windows builds are done using MSBuild under
     # Debug configuration. This means that we need to add "Debug" folder
     # to the path when constructing it. On Linux, we don't add anything.
@@ -69,15 +73,23 @@ def get_test_names(build_path):
     full_path = os.path.join(build_path, config_path, 'tests')
 
 
-    cmd = [full_path, '--reporter', 'xml', '--list-tests']
+    cmd = [full_path, '--reporter', 'json', '--list-tests']
     result = subprocess.run(cmd,
                             capture_output = True,
                             check = True,
                             text = True)
 
-    import xml.etree.ElementTree as ET
+    test_listing = json.loads(result.stdout)
-    root = ET.fromstring(result.stdout)
+
-    return [tc.text for tc in root.findall('TestCase/Name')]
+    assert test_listing['version'] == 1
+
+    tests = []
+    for test in test_listing['listings']['tests']:
+        test_name = test['name']
+        tags = test['tags']
+        tests.append(TestInfo(test_name, tags))
+
+    return tests
 
 def get_ctest_listing(build_path):
     old_path = os.getcwd()
@@ -91,20 +103,25 @@ def get_ctest_listing(build_path):
     os.chdir(old_path)
     return result.stdout
 
-def extract_tests_from_ctest(ctest_output):
+def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
     ctest_response = json.loads(ctest_output)
     tests = ctest_response['tests']
-    test_names = []
+    test_infos = []
     for test in tests:
         test_command = test['command']
         # First part of the command is the binary, second is the filter.
         # If there are less, registration has failed. If there are more,
         # registration has changed and the script needs updating.
         assert len(test_command) == 2
-        test_names.append(test_command[1])
         test_name = test_command[1]
+        labels = []
+        for prop in test['properties']:
+            if prop['name'] == 'LABELS':
+                labels = prop['value']
 
-    return test_names
+        test_infos.append(TestInfo(test_name, labels))
+
+    return test_infos
 
 def check_DL_PATHS(ctest_output):
     ctest_response = json.loads(ctest_output)
@@ -115,10 +132,14 @@ def check_DL_PATHS(ctest_output):
             if property['name'] == 'ENVIRONMENT_MODIFICATION':
                 assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
 
-def escape_catch2_test_name(name):
+def escape_catch2_test_names(infos: List[TestInfo]):
-    for char in ('\\', ',', '[', ']'):
+    escaped = []
-        name = name.replace(char, f"\\{char}")
+    for info in infos:
-    return name
+      name = info.name
+      for char in ('\\', ',', '[', ']'):
+          name = name.replace(char, f"\\{char}")
+      escaped.append(TestInfo(name, info.tags))
+    return escaped
 
 if __name__ == '__main__':
     if len(sys.argv) != 3:
@@ -130,7 +151,7 @@ if __name__ == '__main__':
 
     build_path = build_project(sources_dir, output_base_path, catch2_path)
 
-    catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)]
+    catch_test_names = escape_catch2_test_names(get_test_names(build_path))
     ctest_output = get_ctest_listing(build_path)
     ctest_test_names = extract_tests_from_ctest(ctest_output)
 
@@ -147,6 +168,7 @@ if __name__ == '__main__':
     if mismatched:
         print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
         exit(1)
+    print(f"{len(catch_test_names)} tests matched")
 
     cmake_version = get_cmake_version()
     if cmake_version >= (3, 27):

tests/TestScripts/DiscoverTests/register-tests.cpp

@@ -14,3 +14,10 @@ TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. "
            "Really looooooooooooong. Even longer than that. Multiple lines "
            "worth of test name. Yep, like this." ) {}
 TEST_CASE( "And now a test case with weird tags.", "[tl;dr][tl;dw][foo,bar]" ) {}
+// Also check that we handle tests on class, which have name in output as 'class-name', not 'name'.
+class TestCaseFixture {
+public:
+    int m_a;
+};
+
+TEST_CASE_METHOD(TestCaseFixture, "A test case as method", "[tagstagstags]") {}

tools/misc/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.20)
 
 project(CatchCoverageHelper)