diff --git a/.github/workflows/linux-other-builds.yml b/.github/workflows/linux-other-builds.yml index cf4e2c06..da24104d 100644 --- a/.github/workflows/linux-other-builds.yml +++ b/.github/workflows/linux-other-builds.yml @@ -29,13 +29,13 @@ jobs: build_type: Debug std: 14 other_pkgs: g++-7 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON - cxx: g++-7 build_description: Extras + Examples build_type: Release std: 14 other_pkgs: g++-7 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON # Extras and examples with Clang-10 - cxx: clang++-10 @@ -43,13 +43,13 @@ jobs: build_type: Debug std: 17 other_pkgs: clang-10 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON - cxx: clang++-10 build_description: Extras + Examples build_type: Release std: 17 other_pkgs: clang-10 - cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON + cmake_configurations: -DCATCH_BUILD_EXTRA_TESTS=ON -DCATCH_BUILD_EXAMPLES=ON -DCATCH_ENABLE_CMAKE_HELPER_TESTS=ON # Configure tests with Clang-10 - cxx: clang++-10 diff --git a/BUILD.bazel b/BUILD.bazel index 3125e7c5..02ec9226 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -43,12 +43,14 @@ expand_template( "#cmakedefine CATCH_CONFIG_NO_GLOBAL_NEXTAFTER": "", "#cmakedefine CATCH_CONFIG_NO_POSIX_SIGNALS": "", "#cmakedefine CATCH_CONFIG_NO_USE_ASYNC": "", + "#cmakedefine CATCH_CONFIG_NO_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT": "", "#cmakedefine CATCH_CONFIG_NO_WCHAR": "", "#cmakedefine CATCH_CONFIG_NO_WINDOWS_SEH": "", "#cmakedefine CATCH_CONFIG_NOSTDOUT": "", "#cmakedefine CATCH_CONFIG_POSIX_SIGNALS": "", "#cmakedefine CATCH_CONFIG_PREFIX_ALL": "", "#cmakedefine CATCH_CONFIG_SHARED_LIBRARY": "", + "#cmakedefine CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT": "", "#cmakedefine CATCH_CONFIG_USE_ASYNC": "", "#cmakedefine CATCH_CONFIG_WCHAR": "", "#cmakedefine CATCH_CONFIG_WINDOWS_CRTDBG": "", diff --git a/CMake/CatchConfigOptions.cmake b/CMake/CatchConfigOptions.cmake index e59f3a1e..067739dc 100644 --- a/CMake/CatchConfigOptions.cmake +++ b/CMake/CatchConfigOptions.cmake @@ -41,6 +41,7 @@ set(_OverridableOptions "WCHAR" "WINDOWS_SEH" "GETENV" + "EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT" ) foreach(OptionName ${_OverridableOptions}) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3e81153..5ef06de7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io" cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF) cmake_dependent_option(CATCH_ENABLE_CONFIGURE_TESTS "Enable CMake configuration tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF) +cmake_dependent_option(CATCH_ENABLE_CMAKE_HELPER_TESTS "Enable CMake helper tests. WARNING: VERY EXPENSIVE" OFF "CATCH_DEVELOPMENT_BUILD" OFF) # Catch2's build breaks if done in-tree. You probably should not build diff --git a/CMakePresets.json b/CMakePresets.json index 00f3a6d3..88541285 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -18,7 +18,8 @@ "CATCH_BUILD_EXAMPLES": "ON", "CATCH_BUILD_EXTRA_TESTS": "ON", "CATCH_BUILD_SURROGATES": "ON", - "CATCH_ENABLE_CONFIGURE_TESTS": "ON" + "CATCH_ENABLE_CONFIGURE_TESTS": "ON", + "CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON" } } ] diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index d962a995..a5c6182d 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -4,10 +4,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "bazel_skylib", - sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7", + sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.2/bazel-skylib-1.4.2.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.2/bazel-skylib-1.4.2.tar.gz", ], ) diff --git a/docs/configuration.md b/docs/configuration.md index 18e8b5f7..b62034dd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -15,6 +15,7 @@ [Enabling stringification](#enabling-stringification)
[Disabling exceptions](#disabling-exceptions)
[Overriding Catch's debug break (`-b`)](#overriding-catchs-debug-break--b)
+[Static analysis support](#static-analysis-support)
Catch2 is designed to "just work" as much as possible, and most of the configuration options below are changed automatically during compilation, diff --git a/docs/generators.md b/docs/generators.md index 37d7424a..09799752 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -221,3 +221,21 @@ For full example of implementing your own generator, look into Catch2's examples, specifically [Generators: Create your own generator](../examples/300-Gen-OwnGenerator.cpp). + +### Handling empty generators + +The generator interface assumes that a generator always has at least one +element. This is not always true, e.g. if the generator depends on an external +datafile, the file might be missing. + +There are two ways to handle this, depending on whether you want this +to be an error or not. + + * If empty generator **is** an error, throw an exception in constructor. + * If empty generator **is not** an error, use the [`SKIP`](skipping-passing-failing.md#skipping-test-cases-at-runtime) in constructor. + + + +--- + +[Home](Readme.md#top) diff --git a/docs/reporters.md b/docs/reporters.md index 2c28f256..ef5863f2 100644 --- a/docs/reporters.md +++ b/docs/reporters.md @@ -52,7 +52,7 @@ its machine-readable XML output to file `result-junit.xml`, and the uses ANSI colour codes for colouring the output. Using multiple reporters (or one reporter and one-or-more [event -listeners](event-listener.md#top)) can have surprisingly complex semantics +listeners](event-listeners.md#top)) can have surprisingly complex semantics when using customization points provided to reporters by Catch2, namely capturing stdout/stderr from test cases. diff --git a/docs/skipping-passing-failing.md b/docs/skipping-passing-failing.md index 7c8240e2..6922bd49 100644 --- a/docs/skipping-passing-failing.md +++ b/docs/skipping-passing-failing.md @@ -84,6 +84,12 @@ exit code, same as it does if no test cases have run. This behaviour can be overridden using the [--allow-running-no-tests](command-line.md#no-tests-override) flag. +### `SKIP` inside generators + +You can also use the `SKIP` macro inside generator's constructor to handle +cases where the generator is empty, but you do not want to fail the test +case. + ## Passing and failing test cases diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake index 91f79f3c..46d7a2a9 100644 --- a/extras/CatchAddTests.cmake +++ b/extras/CatchAddTests.cmake @@ -74,6 +74,10 @@ function(catch_discover_tests_impl) ) endif() + # Make sure to escape ; (semicolons) in test names first, because + # that'd break the foreach loop for "Parse output" later and create + # wrongly splitted and thus failing test cases (false positives) + string(REPLACE ";" "\;" output "${output}") string(REPLACE "\n" ";" output "${output}") # Prepare reporter @@ -119,15 +123,16 @@ function(catch_discover_tests_impl) # Parse output foreach(line ${output}) - set(test ${line}) + set(test "${line}") # Escape characters in test case names that would be parsed by Catch2 - set(test_name ${test}) - foreach(char , [ ]) - string(REPLACE ${char} "\\${char}" test_name ${test_name}) + # Note that the \ escaping must happen FIRST! Do not change the order. + set(test_name "${test}") + foreach(char \\ , [ ]) + string(REPLACE ${char} "\\${char}" test_name "${test_name}") endforeach(char) # ...add output dir if(output_dir) - string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean ${test_name}) + string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean "${test_name}") set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}") endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1e0a09f1..0fdf931e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -73,6 +73,7 @@ set(IMPL_HEADERS ${SOURCES_DIR}/internal/catch_compiler_capabilities.hpp ${SOURCES_DIR}/internal/catch_config_android_logwrite.hpp ${SOURCES_DIR}/internal/catch_config_counter.hpp + ${SOURCES_DIR}/internal/catch_config_static_analysis_support.hpp ${SOURCES_DIR}/internal/catch_config_uncaught_exceptions.hpp ${SOURCES_DIR}/internal/catch_config_wchar.hpp ${SOURCES_DIR}/internal/catch_console_colour.hpp diff --git a/src/catch2/catch_all.hpp b/src/catch2/catch_all.hpp index 72de8de9..70ec402d 100644 --- a/src/catch2/catch_all.hpp +++ b/src/catch2/catch_all.hpp @@ -54,6 +54,7 @@ #include #include #include +#include #include #include #include diff --git a/src/catch2/catch_user_config.hpp.in b/src/catch2/catch_user_config.hpp.in index f7973af1..11ab5a6d 100644 --- a/src/catch2/catch_user_config.hpp.in +++ b/src/catch2/catch_user_config.hpp.in @@ -169,6 +169,15 @@ #endif +#cmakedefine CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT +#cmakedefine CATCH_CONFIG_NO_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT + +#if defined( CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT ) && \ + defined( CATCH_CONFIG_NO_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT ) +# error Cannot force STATIC_ANALYSIS_SUPPORT to both ON and OFF +#endif + + // ------ // Simple toggle defines // their value is never used and they cannot be overridden diff --git a/src/catch2/internal/catch_config_static_analysis_support.hpp b/src/catch2/internal/catch_config_static_analysis_support.hpp new file mode 100644 index 00000000..81bdf39f --- /dev/null +++ b/src/catch2/internal/catch_config_static_analysis_support.hpp @@ -0,0 +1,34 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +/** \file + * Wrapper for the STATIC_ANALYSIS_SUPPORT configuration option + * + * Some of Catch2's macros can be defined differently to work better with + * static analysis tools, like clang-tidy or coverity. + * Currently the main use case is to show that `SECTION`s are executed + * exclusively, and not all in one run of a `TEST_CASE`. + */ + +#ifndef CATCH_CONFIG_STATIC_ANALYSIS_SUPPORT_HPP_INCLUDED +#define CATCH_CONFIG_STATIC_ANALYSIS_SUPPORT_HPP_INCLUDED + +#include + +#if defined(__clang_analyzer__) || defined(__COVERITY__) + #define CATCH_INTERNAL_CONFIG_STATIC_ANALYSIS_SUPPORT +#endif + +#if defined( CATCH_INTERNAL_CONFIG_STATIC_ANALYSIS_SUPPORT ) && \ + !defined( CATCH_CONFIG_NO_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT ) && \ + !defined( CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT ) +# define CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT +#endif + + +#endif // CATCH_CONFIG_STATIC_ANALYSIS_SUPPORT_HPP_INCLUDED diff --git a/src/catch2/internal/catch_section.hpp b/src/catch2/internal/catch_section.hpp index 8c1a882b..bd92bdf4 100644 --- a/src/catch2/internal/catch_section.hpp +++ b/src/catch2/internal/catch_section.hpp @@ -9,6 +9,7 @@ #define CATCH_SECTION_HPP_INCLUDED #include +#include #include #include #include @@ -38,16 +39,62 @@ namespace Catch { } // end namespace Catch -#define INTERNAL_CATCH_SECTION( ... ) \ - CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ - CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ - if( Catch::Section const& INTERNAL_CATCH_UNIQUE_NAME( catch_internal_Section ) = Catch::Section( CATCH_INTERNAL_LINEINFO, __VA_ARGS__ ) ) \ - CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION +#if !defined(CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT) +# define INTERNAL_CATCH_SECTION( ... ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ + if ( Catch::Section const& INTERNAL_CATCH_UNIQUE_NAME( \ + catch_internal_Section ) = \ + Catch::Section( CATCH_INTERNAL_LINEINFO, __VA_ARGS__ ) ) \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION + +# define INTERNAL_CATCH_DYNAMIC_SECTION( ... ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ + if ( Catch::Section const& INTERNAL_CATCH_UNIQUE_NAME( \ + catch_internal_Section ) = \ + Catch::SectionInfo( \ + CATCH_INTERNAL_LINEINFO, \ + ( Catch::ReusableStringStream() << __VA_ARGS__ ) \ + .str() ) ) \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION + +#else + +// These section definitions imply that at most one section at one level +// will be intered (because only one section's __LINE__ can be equal to +// the dummy `catchInternalSectionHint` variable from `TEST_CASE`). + +namespace Catch { + namespace Detail { + // Intentionally without linkage, as it should only be used as a dummy + // symbol for static analysis. + int GetNewSectionHint(); + } // namespace Detail +} // namespace Catch + + +# define INTERNAL_CATCH_SECTION( ... ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ + CATCH_INTERNAL_SUPPRESS_SHADOW_WARNINGS \ + if ( [[maybe_unused]] int catchInternalPreviousSectionHint = \ + catchInternalSectionHint, \ + catchInternalSectionHint = Catch::Detail::GetNewSectionHint(); \ + catchInternalPreviousSectionHint == __LINE__ ) \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION + +# define INTERNAL_CATCH_DYNAMIC_SECTION( ... ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ + CATCH_INTERNAL_SUPPRESS_SHADOW_WARNINGS \ + if ( [[maybe_unused]] int catchInternalPreviousSectionHint = \ + catchInternalSectionHint, \ + catchInternalSectionHint = Catch::Detail::GetNewSectionHint(); \ + catchInternalPreviousSectionHint == __LINE__ ) \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION + +#endif -#define INTERNAL_CATCH_DYNAMIC_SECTION( ... ) \ - CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ - CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ - if( Catch::Section const& INTERNAL_CATCH_UNIQUE_NAME( catch_internal_Section ) = Catch::SectionInfo( CATCH_INTERNAL_LINEINFO, (Catch::ReusableStringStream() << __VA_ARGS__).str() ) ) \ - CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION #endif // CATCH_SECTION_HPP_INCLUDED diff --git a/src/catch2/internal/catch_test_registry.hpp b/src/catch2/internal/catch_test_registry.hpp index 5cb73067..d248d3cf 100644 --- a/src/catch2/internal/catch_test_registry.hpp +++ b/src/catch2/internal/catch_test_registry.hpp @@ -8,6 +8,7 @@ #ifndef CATCH_TEST_REGISTRY_HPP_INCLUDED #define CATCH_TEST_REGISTRY_HPP_INCLUDED +#include #include #include #include @@ -72,6 +73,9 @@ struct AutoReg : Detail::NonCopyable { void TestName::test() #endif + +#if !defined(CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT) + /////////////////////////////////////////////////////////////////////////////// #define INTERNAL_CATCH_TESTCASE2( TestName, ... ) \ static void TestName(); \ @@ -84,19 +88,40 @@ struct AutoReg : Detail::NonCopyable { #define INTERNAL_CATCH_TESTCASE( ... ) \ INTERNAL_CATCH_TESTCASE2( INTERNAL_CATCH_UNIQUE_NAME( CATCH2_INTERNAL_TEST_ ), __VA_ARGS__ ) - /////////////////////////////////////////////////////////////////////////////// - #define INTERNAL_CATCH_METHOD_AS_TEST_CASE( QualifiedMethod, ... ) \ - CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ - CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ - CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ - namespace { \ - const Catch::AutoReg INTERNAL_CATCH_UNIQUE_NAME( autoRegistrar )( \ - Catch::makeTestInvoker( &QualifiedMethod ), \ - CATCH_INTERNAL_LINEINFO, \ - "&" #QualifiedMethod##_catch_sr, \ - Catch::NameAndTags{ __VA_ARGS__ } ); \ - } /* NOLINT */ \ - CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION +#else // ^^ !CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT | vv CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT + + +// Dummy registrator for the dumy test case macros +namespace Catch { + namespace Detail { + struct DummyUse { + DummyUse( void ( * )( int ) ); + }; + } // namespace Detail +} // namespace Catch + +// Note that both the presence of the argument and its exact name are +// necessary for the section support. + +// We provide a shadowed variable so that a `SECTION` inside non-`TEST_CASE` +// tests can compile. The redefined `TEST_CASE` shadows this with param. +static int catchInternalSectionHint = 0; + +# define INTERNAL_CATCH_TESTCASE2( fname ) \ + static void fname( int ); \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ + static const Catch::Detail::DummyUse INTERNAL_CATCH_UNIQUE_NAME( \ + dummyUser )( &fname ); \ + CATCH_INTERNAL_SUPPRESS_SHADOW_WARNINGS \ + static void fname( [[maybe_unused]] int catchInternalSectionHint ) \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION +# define INTERNAL_CATCH_TESTCASE( ... ) \ + INTERNAL_CATCH_TESTCASE2( INTERNAL_CATCH_UNIQUE_NAME( dummyFunction ) ) + + +#endif // CATCH_CONFIG_EXPERIMENTAL_STATIC_ANALYSIS_SUPPORT /////////////////////////////////////////////////////////////////////////////// #define INTERNAL_CATCH_TEST_CASE_METHOD2( TestName, ClassName, ... )\ @@ -118,6 +143,22 @@ struct AutoReg : Detail::NonCopyable { #define INTERNAL_CATCH_TEST_CASE_METHOD( ClassName, ... ) \ INTERNAL_CATCH_TEST_CASE_METHOD2( INTERNAL_CATCH_UNIQUE_NAME( CATCH2_INTERNAL_TEST_ ), ClassName, __VA_ARGS__ ) + + /////////////////////////////////////////////////////////////////////////////// + #define INTERNAL_CATCH_METHOD_AS_TEST_CASE( QualifiedMethod, ... ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + CATCH_INTERNAL_SUPPRESS_UNUSED_VARIABLE_WARNINGS \ + namespace { \ + const Catch::AutoReg INTERNAL_CATCH_UNIQUE_NAME( autoRegistrar )( \ + Catch::makeTestInvoker( &QualifiedMethod ), \ + CATCH_INTERNAL_LINEINFO, \ + "&" #QualifiedMethod##_catch_sr, \ + Catch::NameAndTags{ __VA_ARGS__ } ); \ + } /* NOLINT */ \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION + + /////////////////////////////////////////////////////////////////////////////// #define INTERNAL_CATCH_REGISTER_TESTCASE( Function, ... ) \ do { \ diff --git a/src/catch2/meson.build b/src/catch2/meson.build index 89e96e23..2e9469d8 100644 --- a/src/catch2/meson.build +++ b/src/catch2/meson.build @@ -78,6 +78,7 @@ internal_headers = [ 'internal/catch_compiler_capabilities.hpp', 'internal/catch_config_android_logwrite.hpp', 'internal/catch_config_counter.hpp', + 'internal/catch_config_static_analysis_support.hpp', 'internal/catch_config_uncaught_exceptions.hpp', 'internal/catch_config_wchar.hpp', 'internal/catch_console_colour.hpp', diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7be57abe..14a68e34 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -622,6 +622,18 @@ if (CATCH_ENABLE_CONFIGURE_TESTS) endforeach() endif() +if (CATCH_ENABLE_CMAKE_HELPER_TESTS) + add_test(NAME "CMakeHelper::DiscoverTests" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/DiscoverTests/VerifyRegistration.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + set_tests_properties("CMakeHelper::DiscoverTests" + PROPERTIES + COST 240 + LABELS "uses-python" + ) +endif() + foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata "compact" "console" diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 7c2836e8..6b5938a6 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -130,6 +130,7 @@ Nor would this :test-result: FAIL Custom std-exceptions can be custom translated :test-result: PASS Default scale is invisible to comparison :test-result: PASS Directly creating an EnumInfo +:test-result: SKIP Empty generators can SKIP in constructor :test-result: PASS Empty stream name opens cout stream :test-result: FAIL EndsWith string matcher :test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 75569816..cd56e648 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -128,6 +128,7 @@ :test-result: FAIL Custom std-exceptions can be custom translated :test-result: PASS Default scale is invisible to comparison :test-result: PASS Directly creating an EnumInfo +:test-result: SKIP Empty generators can SKIP in constructor :test-result: PASS Empty stream name opens cout stream :test-result: FAIL EndsWith string matcher :test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index c72f140d..be7a4120 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -520,6 +520,7 @@ ToString.tests.cpp:: passed: enumInfo->lookup(1) == "Value2" for: V ToString.tests.cpp:: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +Skip.tests.cpp:: skipped: 'This generator is empty' Stream.tests.cpp:: passed: Catch::makeStream( "" )->isConsole() for: true Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring" Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive) @@ -2537,7 +2538,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 7970ca44..6c48ab91 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -518,6 +518,7 @@ ToString.tests.cpp:: passed: enumInfo->lookup(1) == "Value2" for: V ToString.tests.cpp:: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +Skip.tests.cpp:: skipped: 'This generator is empty' Stream.tests.cpp:: passed: Catch::makeStream( "" )->isConsole() for: true Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring" Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive) @@ -2526,7 +2527,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 31ca04f0..0945f0df 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -383,6 +383,16 @@ Exception.tests.cpp:: FAILED: due to unexpected exception with message: custom std exception +------------------------------------------------------------------------------- +Empty generators can SKIP in constructor +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + This generator is empty + ------------------------------------------------------------------------------- EndsWith string matcher ------------------------------------------------------------------------------- @@ -1533,6 +1543,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 408 | 322 passed | 69 failed | 6 skipped | 11 failed as expected +test cases: 409 | 322 passed | 69 failed | 7 skipped | 11 failed as expected assertions: 2208 | 2048 passed | 128 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 68bda6cd..150980e8 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -3956,6 +3956,16 @@ with expansion: == "{** unexpected enum value **}" +------------------------------------------------------------------------------- +Empty generators can SKIP in constructor +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + This generator is empty + ------------------------------------------------------------------------------- Empty stream name opens cout stream ------------------------------------------------------------------------------- @@ -18222,6 +18232,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 3f5e91d1..4cc942dd 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -3954,6 +3954,16 @@ with expansion: == "{** unexpected enum value **}" +------------------------------------------------------------------------------- +Empty generators can SKIP in constructor +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + This generator is empty + ------------------------------------------------------------------------------- Empty stream name opens cout stream ------------------------------------------------------------------------------- @@ -18211,6 +18221,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 74e08986..c992154c 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -462,6 +462,13 @@ at Exception.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 73f37422..79c32365 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -461,6 +461,13 @@ at Exception.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index eeb8d17b..592887f9 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1870,6 +1870,13 @@ at Misc.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + SKIPPED diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index a804cb6b..3509287f 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1869,6 +1869,13 @@ at Misc.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + SKIPPED diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index ba7d2dfa..acd0a1c1 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -984,6 +984,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1" ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2" # Directly creating an EnumInfo ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +# Empty generators can SKIP in constructor +ok {test-number} - # SKIP 'This generator is empty' # Empty stream name opens cout stream ok {test-number} - Catch::makeStream( "" )->isConsole() for: true # EndsWith string matcher @@ -4475,5 +4477,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2236 +1..2237 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 07014c2e..03329049 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -982,6 +982,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1" ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2" # Directly creating an EnumInfo ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +# Empty generators can SKIP in constructor +ok {test-number} - # SKIP 'This generator is empty' # Empty stream name opens cout stream ok {test-number} - Catch::makeStream( "" )->isConsole() for: true # EndsWith string matcher @@ -4464,5 +4466,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2236 +1..2237 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index aff95cf6..a298633a 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -299,6 +299,9 @@ ##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"] ##teamcity[testStarted name='Directly creating an EnumInfo'] ##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"] +##teamcity[testStarted name='Empty generators can SKIP in constructor'] +##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "This generator is empty"'] +##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"] ##teamcity[testStarted name='Empty stream name opens cout stream'] ##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"] ##teamcity[testStarted name='EndsWith string matcher'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index f16bc135..861d6471 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -299,6 +299,9 @@ ##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"] ##teamcity[testStarted name='Directly creating an EnumInfo'] ##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"] +##teamcity[testStarted name='Empty generators can SKIP in constructor'] +##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "This generator is empty"'] +##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"] ##teamcity[testStarted name='Empty stream name opens cout stream'] ##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"] ##teamcity[testStarted name='EndsWith string matcher'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 8509e370..bf9cf205 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -4364,6 +4364,12 @@ C + + + This generator is empty + + + @@ -21192,6 +21198,6 @@ b1! - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index adf83986..41dc8cb3 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -4364,6 +4364,12 @@ C + + + This generator is empty + + + @@ -21191,6 +21197,6 @@ b1! - - + + diff --git a/tests/SelfTest/IntrospectiveTests/Details.tests.cpp b/tests/SelfTest/IntrospectiveTests/Details.tests.cpp index 80eb647b..d7175756 100644 --- a/tests/SelfTest/IntrospectiveTests/Details.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/Details.tests.cpp @@ -95,8 +95,8 @@ namespace { MoveChecker() = default; MoveChecker( MoveChecker const& rhs ) = default; MoveChecker& operator=( MoveChecker const& rhs ) = default; - MoveChecker( MoveChecker&& rhs ) { rhs.has_moved = true; } - MoveChecker& operator=( MoveChecker&& rhs ) { + MoveChecker( MoveChecker&& rhs ) noexcept { rhs.has_moved = true; } + MoveChecker& operator=( MoveChecker&& rhs ) noexcept { rhs.has_moved = true; return *this; } diff --git a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp index b74580c6..1568c951 100644 --- a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp @@ -163,7 +163,7 @@ namespace { std::vector& recorder, Catch::IConfig const* config ): EventListenerBase( config ), - m_witness( witness ), + m_witness( CATCH_MOVE(witness) ), m_recorder( recorder ) {} @@ -181,7 +181,7 @@ namespace { std::vector& recorder, Catch::ReporterConfig&& config ): StreamingReporterBase( CATCH_MOVE(config) ), - m_witness( witness ), + m_witness( CATCH_MOVE(witness) ), m_recorder( recorder ) {} diff --git a/tests/SelfTest/UsageTests/Generators.tests.cpp b/tests/SelfTest/UsageTests/Generators.tests.cpp index 274c63b8..8e2c387a 100644 --- a/tests/SelfTest/UsageTests/Generators.tests.cpp +++ b/tests/SelfTest/UsageTests/Generators.tests.cpp @@ -261,6 +261,10 @@ TEST_CASE("Copy and then generate a range", "[generators]") { } } +#if defined( __clang__ ) +# pragma clang diagnostic pop +#endif + TEST_CASE("#1913 - GENERATE inside a for loop should not keep recreating the generator", "[regression][generators]") { static int counter = 0; for (int i = 0; i < 3; ++i) { @@ -305,9 +309,5 @@ TEST_CASE( "#2615 - Throwing in constructor generator fails test case but does n // this should fail the test case, but not abort the application auto sample = GENERATE( make_test_generator() ); // this assertion shouldn't trigger - REQUIRE( sample == 0U ); + REQUIRE( sample == 0 ); } - -#if defined( __clang__ ) -# pragma clang diagnostic pop -#endif diff --git a/tests/SelfTest/UsageTests/Matchers.tests.cpp b/tests/SelfTest/UsageTests/Matchers.tests.cpp index a40908d7..74bedf5e 100644 --- a/tests/SelfTest/UsageTests/Matchers.tests.cpp +++ b/tests/SelfTest/UsageTests/Matchers.tests.cpp @@ -890,7 +890,7 @@ struct MatcherA : Catch::Matchers::MatcherGenericBase { return "equals: (int) 1 or (string) \"1\""; } bool match( int i ) const { return i == 1; } - bool match( std::string s ) const { return s == "1"; } + bool match( std::string const& s ) const { return s == "1"; } }; struct MatcherB : Catch::Matchers::MatcherGenericBase { diff --git a/tests/SelfTest/UsageTests/Skip.tests.cpp b/tests/SelfTest/UsageTests/Skip.tests.cpp index 6bd4189b..661795e1 100644 --- a/tests/SelfTest/UsageTests/Skip.tests.cpp +++ b/tests/SelfTest/UsageTests/Skip.tests.cpp @@ -71,3 +71,30 @@ TEST_CASE( "failing for some generator values causes entire test case to fail", FAIL(); } } + +namespace { + class test_skip_generator : public Catch::Generators::IGenerator { + public: + explicit test_skip_generator() { SKIP( "This generator is empty" ); } + + auto get() const -> int const& override { + static constexpr int value = 1; + return value; + } + + auto next() -> bool override { return false; } + }; + + static auto make_test_skip_generator() + -> Catch::Generators::GeneratorWrapper { + return { new test_skip_generator() }; + } + +} // namespace + +TEST_CASE( "Empty generators can SKIP in constructor", "[skipping]" ) { + // The generator signals emptiness with `SKIP` + auto sample = GENERATE( make_test_skip_generator() ); + // This assertion would fail, but shouldn't trigger + REQUIRE( sample == 0 ); +} diff --git a/tests/TestScripts/DiscoverTests/CMakeLists.txt b/tests/TestScripts/DiscoverTests/CMakeLists.txt new file mode 100644 index 00000000..d19f2f88 --- /dev/null +++ b/tests/TestScripts/DiscoverTests/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.10) + +project(discover-tests-test + LANGUAGES CXX +) + +add_executable(tests + register-tests.cpp +) + +add_subdirectory(${CATCH2_PATH} catch2-build) +target_link_libraries(tests PRIVATE Catch2::Catch2WithMain) + +include(CTest) +include(Catch) +catch_discover_tests(tests) diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py new file mode 100644 index 00000000..9ec42f24 --- /dev/null +++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +import os +import subprocess +import sys + + +def build_project(sources_dir, output_base_path, catch2_path): + build_dir = os.path.join(output_base_path, 'ctest-registration-test') + config_cmd = ['cmake', + '-B', build_dir, + '-S', sources_dir, + f'-DCATCH2_PATH={catch2_path}', + '-DCMAKE_BUILD_TYPE=Debug'] + + build_cmd = ['cmake', + '--build', build_dir, + '--config', 'Debug'] + + try: + subprocess.run(config_cmd, + capture_output = True, + check = True, + text = True) + subprocess.run(build_cmd, + capture_output = True, + check = True, + text = True) + except subprocess.CalledProcessError as err: + print('Error when building the test project') + print(f'cmd: {err.cmd}') + print(f'stderr: {err.stderr}') + print(f'stdout: {err.stdout}') + exit(3) + + return build_dir + + + +def get_test_names(build_path): + # For now we assume that Windows builds are done using MSBuild under + # Debug configuration. This means that we need to add "Debug" folder + # to the path when constructing it. On Linux, we don't add anything. + config_path = "Debug" if os.name == 'nt' else "" + full_path = os.path.join(build_path, config_path, 'tests') + + + cmd = [full_path, '--reporter', 'xml', '--list-tests'] + result = subprocess.run(cmd, + capture_output = True, + check = True, + text = True) + + import xml.etree.ElementTree as ET + root = ET.fromstring(result.stdout) + return [tc.text for tc in root.findall('TestCase/Name')] + + +def list_ctest_tests(build_path): + old_path = os.getcwd() + os.chdir(build_path) + + cmd = ['ctest', '-C', 'debug', '--show-only=json-v1'] + result = subprocess.run(cmd, + capture_output = True, + check = True, + text = True) + os.chdir(old_path) + + import json + + ctest_response = json.loads(result.stdout) + tests = ctest_response['tests'] + test_names = [] + for test in tests: + test_command = test['command'] + # First part of the command is the binary, second is the filter. + # If there are less, registration has failed. If there are more, + # registration has changed and the script needs updating. + assert len(test_command) == 2 + test_names.append(test_command[1]) + test_name = test_command[1] + + return test_names + +def escape_catch2_test_name(name): + for char in ('\\', ',', '[', ']'): + name = name.replace(char, f"\\{char}") + return name + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path') + exit(2) + catch2_path = sys.argv[1] + output_base_path = sys.argv[2] + sources_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + + build_path = build_project(sources_dir, output_base_path, catch2_path) + + catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)] + ctest_test_names = list_ctest_tests(build_path) + + mismatched = 0 + for catch_test in catch_test_names: + if catch_test not in ctest_test_names: + print(f"Catch2 test '{catch_test}' not found in CTest") + mismatched += 1 + for ctest_test in ctest_test_names: + if ctest_test not in catch_test_names: + print(f"CTest test '{ctest_test}' not found in Catch2") + mismatched += 1 + + if mismatched: + print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!") + exit(1) diff --git a/tests/TestScripts/DiscoverTests/register-tests.cpp b/tests/TestScripts/DiscoverTests/register-tests.cpp new file mode 100644 index 00000000..21238d59 --- /dev/null +++ b/tests/TestScripts/DiscoverTests/register-tests.cpp @@ -0,0 +1,12 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include + +TEST_CASE("@Script[C:\\EPM1A]=x;\"SCALA_ZERO:\"", "[script regressions]"){} +TEST_CASE("Some test") {}