diff --git a/docs/cmake-integration.md b/docs/cmake-integration.md index 347cfd5a..9637be9b 100644 --- a/docs/cmake-integration.md +++ b/docs/cmake-integration.md @@ -63,17 +63,19 @@ target_link_libraries(tests PRIVATE Catch2::Catch2WithMain) ## Automatic test registration -Catch2's repository also contains two CMake scripts that help users +Catch2's repository also contains three CMake scripts that help users with automatically registering their `TEST_CASE`s with CTest. They can be found in the `extras` folder, and are 1) `Catch.cmake` (and its dependency `CatchAddTests.cmake`) 2) `ParseAndAddCatchTests.cmake` (deprecated) +3) `CatchShardTests.cmake` (and its dependency `CatchShardTestsImpl.cmake`) If Catch2 has been installed in system, both of these can be used after doing `find_package(Catch2 REQUIRED)`. Otherwise you need to add them to your CMake module path. + ### `Catch.cmake` and `CatchAddTests.cmake` `Catch.cmake` provides function `catch_discover_tests` to get tests from @@ -257,6 +259,49 @@ unset(OptionalCatchTestLauncher) ParseAndAddCatchTests(bar) ``` + +### `CatchShardTests.cmake` + +> `CatchShardTests.cmake` was introduced in Catch2 X.Y.Z. + +`CatchShardTests.cmake` provides a function +`catch_add_sharded_tests(TEST_BINARY)` that splits tests from `TEST_BINARY` +into multiple shards. The tests in each shard and their order is randomized, +and the seed changes every invocation of CTest. + +Currently there are 3 customization points for this script: + + * SHARD_COUNT - number of shards to split target's tests into + * REPORTER - reporter spec to use for tests + * TEST_SPEC - test spec used for filtering tests + +Example usage: + +``` +include(CatchShardTests) + +catch_add_sharded_tests(foo-tests + SHARD_COUNT 4 + REPORTER "xml::out=-" + TEST_SPEC "A" +) + +catch_add_sharded_tests(tests + SHARD_COUNT 8 + REPORTER "xml::out=-" + TEST_SPEC "B" +) +``` + +This registers total of 12 CTest tests (4 + 8 shards) to run shards +from `foo-tests` test binary, filtered by a test spec. + +_Note that this script is currently a proof-of-concept for reseeding +shards per CTest run, and thus does not support (nor does it currently +aim to support) all customization points from +[`catch_discover_tests`](#catch_discover_tests)._ + + ## CMake project options Catch2's CMake project also provides some options for other projects diff --git a/extras/CatchShardTests.cmake b/extras/CatchShardTests.cmake new file mode 100644 index 00000000..d3b5e6fc --- /dev/null +++ b/extras/CatchShardTests.cmake @@ -0,0 +1,66 @@ + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +# Supported optional args: +# * SHARD_COUNT - number of shards to split target's tests into +# * REPORTER - reporter spec to use for tests +# * TEST_SPEC - test spec used for filtering tests +function(catch_add_sharded_tests TARGET) + if (${CMAKE_VERSION} VERSION_LESS "3.10.0") + message(FATAL_ERROR "add_sharded_catch_tests only supports CMake versions 3.10.0 and up") + endif() + + cmake_parse_arguments( + "" + "" + "SHARD_COUNT;REPORTER;TEST_SPEC" + "" + ${ARGN} + ) + + if (NOT DEFINED _SHARD_COUNT) + set(_SHARD_COUNT 2) + endif() + + # Generate a unique name based on the extra arguments + string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX} ${_SHARD_COUNT}") + string(SUBSTRING ${args_hash} 0 7 args_hash) + + set(ctest_include_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-sharded-tests-include-${args_hash}.cmake") + set(ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-sharded-tests-impl-${args_hash}.cmake") + + file(WRITE "${ctest_include_file}" + "if(EXISTS \"${ctest_tests_file}\")\n" + " include(\"${ctest_tests_file}\")\n" + "else()\n" + " add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n" + "endif()\n" + ) + + set_property(DIRECTORY + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" + ) + + set(shard_impl_script_file "${CMAKE_CURRENT_LIST_DIR}/CatchShardTestsImpl.cmake") + + add_custom_command( + TARGET ${TARGET} POST_BUILD + BYPRODUCTS "${ctest_tests_file}" + COMMAND "${CMAKE_COMMAND}" + -D "TARGET_NAME=${TARGET}" + -D "TEST_BINARY=$" + -D "CTEST_FILE=${ctest_tests_file}" + -D "SHARD_COUNT=${_SHARD_COUNT}" + -D "REPORTER_SPEC=${_REPORTER}" + -D "TEST_SPEC=${_TEST_SPEC}" + -P "${shard_impl_script_file}" + VERBATIM + ) + + +endfunction() diff --git a/extras/CatchShardTestsImpl.cmake b/extras/CatchShardTestsImpl.cmake new file mode 100644 index 00000000..d18dddef --- /dev/null +++ b/extras/CatchShardTestsImpl.cmake @@ -0,0 +1,52 @@ + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +# Indirection for CatchShardTests that allows us to delay the script +# file generation until build time. + +# Expected args: +# * TEST_BINARY - full path to the test binary to run sharded +# * CTEST_FILE - full path to ctest script file to write to +# * TARGET_NAME - name of the target to shard (used for test names) +# * SHARD_COUNT - number of shards to split the binary into +# Optional args: +# * REPORTER_SPEC - reporter specs to be passed down to the binary +# * TEST_SPEC - test spec to pass down to the test binary + +if(NOT EXISTS "${TEST_BINARY}") + message(FATAL_ERROR + "Specified test binary '${TEST_BINARY}' does not exist" + ) +endif() + +set(other_args "") +if (TEST_SPEC) + set(other_args "${other_args} ${TEST_SPEC}") +endif() +if (REPORTER_SPEC) + set(other_args "${other_args} --reporter ${REPORTER_SPEC}") +endif() + +# foreach RANGE in cmake is inclusive of the end, so we have to adjust it +math(EXPR adjusted_shard_count "${SHARD_COUNT} - 1") + +file(WRITE "${CTEST_FILE}" + "string(RANDOM LENGTH 8 ALPHABET \"0123456789abcdef\" rng_seed)\n" + "\n" + "foreach(shard_idx RANGE ${adjusted_shard_count})\n" + " add_test(${TARGET_NAME}-shard-" [[${shard_idx}]] "/${adjusted_shard_count}\n" + " ${TEST_BINARY}" + " --shard-index " [[${shard_idx}]] + " --shard-count ${SHARD_COUNT}" + " --rng-seed " [[0x${rng_seed}]] + " --order rand" + "${other_args}" + "\n" + " )\n" + "endforeach()\n" +)