diff --git a/docs/command-line.md b/docs/command-line.md index f8e80270..0154f075 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -29,6 +29,7 @@ [Specify the section to run](#specify-the-section-to-run)
[Filenames as tags](#filenames-as-tags)
[Override output colouring](#override-output-colouring)
+[Test Sharding](#test-sharding)
Catch works quite nicely without any command line options at all - but for those times when you want greater control the following options are available. Click one of the following links to take you straight to that option - or scroll on to browse the available options. @@ -67,6 +68,8 @@ Click one of the following links to take you straight to that option - or scroll ` --benchmark-no-analysis`
` --benchmark-warmup-time`
` --use-colour`
+ ` --shard-count`
+ ` --shard-index`

@@ -425,6 +428,20 @@ processing of output. `--use-colour yes` forces coloured output, `--use-colour no` disables coloured output. The default behaviour is `--use-colour auto`. + +## Test Sharding +
--shard-count <#number of shards>, --shard-index <#shard index to run>
+ +> [Introduced](https://github.com/catchorg/Catch2/pull/2257) in Catch2 X.Y.Z. + +When `--shard-count <#number of shards>` is used, the tests to execute will be split evenly in to the given number of sets, +identified by indicies starting at 0. The tests in the set given by `--shard-index <#shard index to run>` will be executed. +The default shard count is `1`, and the default index to run is `0`. It is an error to specify a shard index greater than +the number of shards. + +This is useful when you want to split test execution across multiple processes, as is done with [Bazel test sharding](https://docs.bazel.build/versions/main/test-encyclopedia.html#test-sharding). + + --- [Home](Readme.md#top) diff --git a/src/catch2/catch_all.hpp b/src/catch2/catch_all.hpp index 27282de5..742d89a9 100644 --- a/src/catch2/catch_all.hpp +++ b/src/catch2/catch_all.hpp @@ -85,6 +85,7 @@ #include #include #include +#include #include #include #include diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index b79059ec..c9fc8eef 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -73,6 +73,8 @@ namespace Catch { double Config::minDuration() const { return m_data.minDuration; } TestRunOrder Config::runOrder() const { return m_data.runOrder; } uint32_t Config::rngSeed() const { return m_data.rngSeed; } + unsigned int Config::shardCount() const { return m_data.shardCount; } + unsigned int Config::shardIndex() const { return m_data.shardIndex; } UseColour Config::useColour() const { return m_data.useColour; } bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; } int Config::abortAfter() const { return m_data.abortAfter; } diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index 7b67ba76..6aa6c4d6 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -37,6 +37,9 @@ namespace Catch { int abortAfter = -1; uint32_t rngSeed = generateRandomSeed(GenerateFrom::Default); + unsigned int shardCount = 1; + unsigned int shardIndex = 0; + bool benchmarkNoAnalysis = false; unsigned int benchmarkSamples = 100; double benchmarkConfidenceInterval = 0.95; @@ -99,6 +102,8 @@ namespace Catch { double minDuration() const override; TestRunOrder runOrder() const override; uint32_t rngSeed() const override; + unsigned int shardCount() const override; + unsigned int shardIndex() const override; UseColour useColour() const override; bool shouldDebugBreak() const override; int abortAfter() const override; diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index e7878117..3b58adbd 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,8 @@ namespace Catch { for (auto const& match : m_matches) m_tests.insert(match.tests.begin(), match.tests.end()); } + + m_tests = createShard(m_tests, m_config->shardCount(), m_config->shardIndex()); } Totals execute() { @@ -171,6 +174,7 @@ namespace Catch { return 1; auto result = m_cli.parse( Clara::Args( argc, argv ) ); + if( !result ) { config(); getCurrentMutableContext().setConfig(m_config.get()); @@ -253,6 +257,12 @@ namespace Catch { if( m_startupExceptions ) return 1; + + if( m_configData.shardIndex >= m_configData.shardCount ) { + Catch::cerr() << "The shard count (" << m_configData.shardCount << ") must be greater than the shard index (" << m_configData.shardIndex << ")\n" << std::flush; + return 1; + } + if (m_configData.showHelp || m_configData.libIdentify) { return 0; } diff --git a/src/catch2/interfaces/catch_interfaces_config.hpp b/src/catch2/interfaces/catch_interfaces_config.hpp index e40cf702..bd0c8a40 100644 --- a/src/catch2/interfaces/catch_interfaces_config.hpp +++ b/src/catch2/interfaces/catch_interfaces_config.hpp @@ -74,6 +74,8 @@ namespace Catch { virtual std::vector const& getTestsOrTags() const = 0; virtual TestRunOrder runOrder() const = 0; virtual uint32_t rngSeed() const = 0; + virtual unsigned int shardCount() const = 0; + virtual unsigned int shardIndex() const = 0; virtual UseColour useColour() const = 0; virtual std::vector const& getSectionsToRun() const = 0; virtual Verbosity verbosity() const = 0; diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp index ad571cc5..5f8a44d5 100644 --- a/src/catch2/internal/catch_commandline.cpp +++ b/src/catch2/internal/catch_commandline.cpp @@ -149,6 +149,15 @@ namespace Catch { return ParserResult::runtimeError( "Unrecognized reporter, '" + reporter + "'. Check available with --list-reporters" ); return ParserResult::ok( ParseResultType::Matched ); }; + auto const setShardCount = [&]( std::string const& shardCount ) { + auto result = Clara::Detail::convertInto( shardCount, config.shardCount ); + + if (config.shardCount == 0) { + return ParserResult::runtimeError( "The shard count must be greater than 0" ); + } else { + return result; + } + }; auto cli = ExeName( config.processName ) @@ -240,6 +249,12 @@ namespace Catch { | Opt( config.benchmarkWarmupTime, "benchmarkWarmupTime" ) ["--benchmark-warmup-time"] ( "amount of time in milliseconds spent on warming up each test (default: 100)" ) + | Opt( setShardCount, "shard count" ) + ["--shard-count"] + ( "split the tests to execute into this many groups" ) + | Opt( config.shardIndex, "shard index" ) + ["--shard-index"] + ( "index of the group of tests to execute (see --shard-count)" ) | Arg( config.testsOrTags, "test name|pattern|tags" ) ( "which test or tests to use" ); diff --git a/src/catch2/internal/catch_sharding.hpp b/src/catch2/internal/catch_sharding.hpp new file mode 100644 index 00000000..0e8822ee --- /dev/null +++ b/src/catch2/internal/catch_sharding.hpp @@ -0,0 +1,41 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_SHARDING_HPP_INCLUDED +#define CATCH_SHARDING_HPP_INCLUDED + +#include + +#include + +namespace Catch { + + template + Container createShard(Container const& container, std::size_t const shardCount, std::size_t const shardIndex) { + assert(shardCount > shardIndex); + + if (shardCount == 1) { + return container; + } + + const std::size_t totalTestCount = container.size(); + + const std::size_t shardSize = totalTestCount / shardCount; + const std::size_t leftoverTests = totalTestCount % shardCount; + + const std::size_t startIndex = shardIndex * shardSize + (std::min)(shardIndex, leftoverTests); + const std::size_t endIndex = (shardIndex + 1) * shardSize + (std::min)(shardIndex + 1, leftoverTests); + + auto startIterator = std::next(container.begin(), startIndex); + auto endIterator = std::next(container.begin(), endIndex); + + return Container(startIterator, endIterator); + } + +} + +#endif // CATCH_SHARDING_HPP_INCLUDED diff --git a/src/catch2/internal/catch_test_case_registry_impl.cpp b/src/catch2/internal/catch_test_case_registry_impl.cpp index dd1744a7..3c1ab54b 100644 --- a/src/catch2/internal/catch_test_case_registry_impl.cpp +++ b/src/catch2/internal/catch_test_case_registry_impl.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -135,7 +136,7 @@ namespace { filtered.push_back(testCase); } } - return filtered; + return createShard(filtered, config.shardCount(), config.shardIndex()); } std::vector const& getAllTestCasesSorted( IConfig const& config ) { return getRegistryHub().getTestCaseRegistry().getAllTestsSorted( config ); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7f7c32fe..9a46f6e2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -87,6 +87,7 @@ set(TEST_SOURCES ${SELF_TEST_DIR}/IntrospectiveTests/RandomNumberGeneration.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Reporters.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Tag.tests.cpp + ${SELF_TEST_DIR}/IntrospectiveTests/Sharding.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/String.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/StringManip.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Xml.tests.cpp @@ -310,6 +311,7 @@ set_tests_properties(TagAlias PROPERTIES add_test(NAME RandomTestOrdering COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testRandomOrder.py $) + add_test(NAME CheckConvenienceHeaders COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/checkConvenienceHeaders.py diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index bce10df5..f2add6d6 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -8,6 +8,22 @@ project( Catch2ExtraTests LANGUAGES CXX ) message( STATUS "Extra tests included" ) + +add_test( + NAME TestShardingIntegration + COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testSharding.py $ +) + +add_test( + NAME TestSharding::OverlyLargeShardIndex + COMMAND $ --shard-index 5 --shard-count 5 +) +set_tests_properties( + TestSharding::OverlyLargeShardIndex + PROPERTIES + PASS_REGULAR_EXPRESSION "The shard count \\(5\\) must be greater than the shard index \\(5\\)" +) + # The MinDuration reporting tests do not need separate compilation, but # they have non-trivial execution time, so they are categorized as # extra tests, so that they are run less. diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index dfb08230..3f39422e 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1273,6 +1273,12 @@ CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-no-ana CmdLine.tests.cpp:: passed: config.benchmarkNoAnalysis for: true CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?} CmdLine.tests.cpp:: passed: config.benchmarkWarmupTime == 10 for: 10 == 10 +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--shard-count=8"}) for: {?} +CmdLine.tests.cpp:: passed: config.shardCount == 8 for: 8 == 8 +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--shard-index=2"}) for: {?} +CmdLine.tests.cpp:: passed: config.shardIndex == 2 for: 2 == 2 +CmdLine.tests.cpp:: passed: !result for: true +CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) for: "The shard count must be greater than 0" contains: "The shard count must be greater than 0" Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 3 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 2 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 1 >= 1 diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index f7320bd7..af7fb09c 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1427,5 +1427,5 @@ due to unexpected exception with message: =============================================================================== test cases: 373 | 296 passed | 70 failed | 7 failed as expected -assertions: 2115 | 1959 passed | 129 failed | 27 failed as expected +assertions: 2121 | 1965 passed | 129 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 695a30ef..189944cb 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -9390,6 +9390,61 @@ CmdLine.tests.cpp:: PASSED: with expansion: 10 == 10 +------------------------------------------------------------------------------- +Process can be configured on command line + Sharding options + shard-count +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--shard-count=8"}) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.shardCount == 8 ) +with expansion: + 8 == 8 + +------------------------------------------------------------------------------- +Process can be configured on command line + Sharding options + shard-index +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--shard-index=2"}) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.shardIndex == 2 ) +with expansion: + 2 == 2 + +------------------------------------------------------------------------------- +Process can be configured on command line + Sharding options + Zero shard-count +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( !result ) +with expansion: + true + +CmdLine.tests.cpp:: PASSED: + CHECK_THAT( result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) ) +with expansion: + "The shard count must be greater than 0" contains: "The shard count must be + greater than 0" + ------------------------------------------------------------------------------- Product with differing arities - std::tuple ------------------------------------------------------------------------------- @@ -17040,5 +17095,5 @@ Misc.tests.cpp:: PASSED: =============================================================================== test cases: 373 | 280 passed | 86 failed | 7 failed as expected -assertions: 2132 | 1959 passed | 146 failed | 27 failed as expected +assertions: 2138 | 1965 passed | 146 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index c2486391..ab55d93c 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1096,6 +1096,9 @@ Message.tests.cpp: + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 18bca507..4f5e0b12 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -75,6 +75,9 @@ + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index aad246f5..e5247b5f 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2468,6 +2468,18 @@ ok {test-number} - config.benchmarkNoAnalysis for: true ok {test-number} - cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?} # Process can be configured on command line ok {test-number} - config.benchmarkWarmupTime == 10 for: 10 == 10 +# Process can be configured on command line +ok {test-number} - cli.parse({ "test", "--shard-count=8"}) for: {?} +# Process can be configured on command line +ok {test-number} - config.shardCount == 8 for: 8 == 8 +# Process can be configured on command line +ok {test-number} - cli.parse({ "test", "--shard-index=2"}) for: {?} +# Process can be configured on command line +ok {test-number} - config.shardIndex == 2 for: 2 == 2 +# Process can be configured on command line +ok {test-number} - !result for: true +# Process can be configured on command line +ok {test-number} - result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) for: "The shard count must be greater than 0" contains: "The shard count must be greater than 0" # Product with differing arities - std::tuple ok {test-number} - std::tuple_size::value >= 1 for: 3 >= 1 # Product with differing arities - std::tuple @@ -4266,5 +4278,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2132 +1..2138 diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 419941de..256e43fd 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -11469,6 +11469,72 @@ Nor would this +
+
+ + + cli.parse({ "test", "--shard-count=8"}) + + + {?} + + + + + config.shardCount == 8 + + + 8 == 8 + + + +
+ +
+
+
+ + + cli.parse({ "test", "--shard-index=2"}) + + + {?} + + + + + config.shardIndex == 2 + + + 2 == 2 + + + +
+ +
+
+
+ + + !result + + + true + + + + + result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) + + + "The shard count must be greater than 0" contains: "The shard count must be greater than 0" + + + +
+ +
@@ -20030,6 +20096,6 @@ loose text artifact - + diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index c1fd6527..5b58bdec 100644 --- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -569,6 +569,27 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" REQUIRE(config.benchmarkWarmupTime == 10); } } + + SECTION("Sharding options") { + SECTION("shard-count") { + CHECK(cli.parse({ "test", "--shard-count=8"})); + + REQUIRE(config.shardCount == 8); + } + + SECTION("shard-index") { + CHECK(cli.parse({ "test", "--shard-index=2"})); + + REQUIRE(config.shardIndex == 2); + } + + SECTION("Zero shard-count") { + auto result = cli.parse({ "test", "--shard-count=0"}); + + CHECK( !result ); + CHECK_THAT( result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) ); + } + } } TEST_CASE("Test with special, characters \"in name", "[cli][regression]") { diff --git a/tests/SelfTest/IntrospectiveTests/Sharding.tests.cpp b/tests/SelfTest/IntrospectiveTests/Sharding.tests.cpp new file mode 100644 index 00000000..9ef80d23 --- /dev/null +++ b/tests/SelfTest/IntrospectiveTests/Sharding.tests.cpp @@ -0,0 +1,41 @@ +/* + * Distributed under the Boost Software License, Version 1.0. (See accompanying + * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + */ +#include +#include + +#include + +#include +#include + +TEST_CASE("Sharding Function", "[approvals]") { + std::vector testContainer = { 0, 1, 2, 3, 4, 5, 6 }; + std::unordered_map> expectedShardSizes = { + {1, {7}}, + {2, {4, 3}}, + {3, {3, 2, 2}}, + {4, {2, 2, 2, 1}}, + {5, {2, 2, 1, 1, 1}}, + {6, {2, 1, 1, 1, 1, 1}}, + {7, {1, 1, 1, 1, 1, 1, 1}}, + }; + + auto shardCount = GENERATE(range(1, 7)); + auto shardIndex = GENERATE_COPY(filter([=](int i) { return i < shardCount; }, range(0, 6))); + + std::vector result = Catch::createShard(testContainer, shardCount, shardIndex); + + auto& sizes = expectedShardSizes[shardCount]; + REQUIRE(result.size() == sizes[shardIndex]); + + std::size_t startIndex = 0; + for(int i = 0; i < shardIndex; i++) { + startIndex += sizes[i]; + } + + for(std::size_t i = 0; i < sizes[shardIndex]; i++) { + CHECK(result[i] == testContainer[i + startIndex]); + } +} \ No newline at end of file diff --git a/tests/TestScripts/testSharding.py b/tests/TestScripts/testSharding.py new file mode 100644 index 00000000..fcea03e2 --- /dev/null +++ b/tests/TestScripts/testSharding.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +""" +This test script verifies that sharding tests does change which tests are run. +This is done by running the binary multiple times, once to list all the tests, +once per shard to list the tests for that shard, and once again per shard to +execute the tests. The sharded lists are compared to the full list to ensure +none are skipped, duplicated, and that the order remains the same. This process +is repeated for multiple command line argument combinations to ensure sharding +works with different filters and test orderings. +""" + +import subprocess +import sys +import xml.etree.ElementTree as ET + +from collections import namedtuple + +def make_base_commandline(self_test_exe): + return [ + self_test_exe, + '--reporter', 'xml', + "--shard-count", "5", + "--shard-index", "2", + "[generators]~[benchmarks]~[.]" + ] + +def list_tests(self_test_exe): + cmd = make_base_commandline(self_test_exe) + ['--list-tests'] + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if stderr: + raise RuntimeError("Unexpected error output:\n" + process.stderr) + + root = ET.fromstring(stdout) + result = [elem.text for elem in root.findall('./TestCase/Name')] + + if len(result) < 2: + raise RuntimeError("Unexpectedly few tests listed (got {})".format( + len(result))) + + return result + + +def execute_tests(self_test_exe): + cmd = make_base_commandline(self_test_exe) + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if stderr: + raise RuntimeError("Unexpected error output:\n" + process.stderr) + + root = ET.fromstring(stdout) + result = [elem.attrib["name"] for elem in root.findall('./TestCase')] + + if len(result) < 2: + raise RuntimeError("Unexpectedly few tests listed (got {})".format( + len(result))) + return result + + +def check_listed_and_executed_tests_match(listed_tests, executed_tests): + listed_names = set(listed_tests) + executed_names = set(executed_tests) + + listed_string = "\n".join(listed_names) + exeucted_string = "\n".join(executed_names) + + assert listed_names == executed_names, ( + "Executed tests do not match the listed tests:\nExecuted:\n{}\n\nListed:\n{}".format(exeucted_string, listed_string) + ) + + +def test_sharding(self_test_exe): + listed_tests = list_tests(self_test_exe) + executed_tests = execute_tests(self_test_exe) + + check_listed_and_executed_tests_match(listed_tests, executed_tests) + + +def main(): + self_test_exe, = sys.argv[1:] + + test_sharding(self_test_exe) + +if __name__ == '__main__': + sys.exit(main())