diff --git a/docs/command-line.md b/docs/command-line.md
index f8e80270..0154f075 100644
--- a/docs/command-line.md
+++ b/docs/command-line.md
@@ -29,6 +29,7 @@
[Specify the section to run](#specify-the-section-to-run)
[Filenames as tags](#filenames-as-tags)
[Override output colouring](#override-output-colouring)
+[Test Sharding](#test-sharding)
Catch works quite nicely without any command line options at all - but for those times when you want greater control the following options are available.
Click one of the following links to take you straight to that option - or scroll on to browse the available options.
@@ -67,6 +68,8 @@ Click one of the following links to take you straight to that option - or scroll
` --benchmark-no-analysis`
` --benchmark-warmup-time`
` --use-colour`
+ ` --shard-count`
+ ` --shard-index`
@@ -425,6 +428,20 @@ processing of output.
`--use-colour yes` forces coloured output, `--use-colour no` disables coloured
output. The default behaviour is `--use-colour auto`.
+
+## Test Sharding
+
--shard-count <#number of shards>, --shard-index <#shard index to run>
+
+> [Introduced](https://github.com/catchorg/Catch2/pull/2257) in Catch2 X.Y.Z.
+
+When `--shard-count <#number of shards>` is used, the tests to execute will be split evenly in to the given number of sets,
+identified by indicies starting at 0. The tests in the set given by `--shard-index <#shard index to run>` will be executed.
+The default shard count is `1`, and the default index to run is `0`. It is an error to specify a shard index greater than
+the number of shards.
+
+This is useful when you want to split test execution across multiple processes, as is done with [Bazel test sharding](https://docs.bazel.build/versions/main/test-encyclopedia.html#test-sharding).
+
+
---
[Home](Readme.md#top)
diff --git a/src/catch2/catch_all.hpp b/src/catch2/catch_all.hpp
index 27282de5..742d89a9 100644
--- a/src/catch2/catch_all.hpp
+++ b/src/catch2/catch_all.hpp
@@ -85,6 +85,7 @@
#include
#include
#include
+#include
#include
#include
#include
diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp
index b79059ec..c9fc8eef 100644
--- a/src/catch2/catch_config.cpp
+++ b/src/catch2/catch_config.cpp
@@ -73,6 +73,8 @@ namespace Catch {
double Config::minDuration() const { return m_data.minDuration; }
TestRunOrder Config::runOrder() const { return m_data.runOrder; }
uint32_t Config::rngSeed() const { return m_data.rngSeed; }
+ unsigned int Config::shardCount() const { return m_data.shardCount; }
+ unsigned int Config::shardIndex() const { return m_data.shardIndex; }
UseColour Config::useColour() const { return m_data.useColour; }
bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; }
int Config::abortAfter() const { return m_data.abortAfter; }
diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp
index 7b67ba76..6aa6c4d6 100644
--- a/src/catch2/catch_config.hpp
+++ b/src/catch2/catch_config.hpp
@@ -37,6 +37,9 @@ namespace Catch {
int abortAfter = -1;
uint32_t rngSeed = generateRandomSeed(GenerateFrom::Default);
+ unsigned int shardCount = 1;
+ unsigned int shardIndex = 0;
+
bool benchmarkNoAnalysis = false;
unsigned int benchmarkSamples = 100;
double benchmarkConfidenceInterval = 0.95;
@@ -99,6 +102,8 @@ namespace Catch {
double minDuration() const override;
TestRunOrder runOrder() const override;
uint32_t rngSeed() const override;
+ unsigned int shardCount() const override;
+ unsigned int shardIndex() const override;
UseColour useColour() const override;
bool shouldDebugBreak() const override;
int abortAfter() const override;
diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp
index e7878117..3b58adbd 100644
--- a/src/catch2/catch_session.cpp
+++ b/src/catch2/catch_session.cpp
@@ -16,6 +16,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -72,6 +73,8 @@ namespace Catch {
for (auto const& match : m_matches)
m_tests.insert(match.tests.begin(), match.tests.end());
}
+
+ m_tests = createShard(m_tests, m_config->shardCount(), m_config->shardIndex());
}
Totals execute() {
@@ -171,6 +174,7 @@ namespace Catch {
return 1;
auto result = m_cli.parse( Clara::Args( argc, argv ) );
+
if( !result ) {
config();
getCurrentMutableContext().setConfig(m_config.get());
@@ -253,6 +257,12 @@ namespace Catch {
if( m_startupExceptions )
return 1;
+
+ if( m_configData.shardIndex >= m_configData.shardCount ) {
+ Catch::cerr() << "The shard count (" << m_configData.shardCount << ") must be greater than the shard index (" << m_configData.shardIndex << ")\n" << std::flush;
+ return 1;
+ }
+
if (m_configData.showHelp || m_configData.libIdentify) {
return 0;
}
diff --git a/src/catch2/interfaces/catch_interfaces_config.hpp b/src/catch2/interfaces/catch_interfaces_config.hpp
index e40cf702..bd0c8a40 100644
--- a/src/catch2/interfaces/catch_interfaces_config.hpp
+++ b/src/catch2/interfaces/catch_interfaces_config.hpp
@@ -74,6 +74,8 @@ namespace Catch {
virtual std::vector const& getTestsOrTags() const = 0;
virtual TestRunOrder runOrder() const = 0;
virtual uint32_t rngSeed() const = 0;
+ virtual unsigned int shardCount() const = 0;
+ virtual unsigned int shardIndex() const = 0;
virtual UseColour useColour() const = 0;
virtual std::vector const& getSectionsToRun() const = 0;
virtual Verbosity verbosity() const = 0;
diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp
index ad571cc5..5f8a44d5 100644
--- a/src/catch2/internal/catch_commandline.cpp
+++ b/src/catch2/internal/catch_commandline.cpp
@@ -149,6 +149,15 @@ namespace Catch {
return ParserResult::runtimeError( "Unrecognized reporter, '" + reporter + "'. Check available with --list-reporters" );
return ParserResult::ok( ParseResultType::Matched );
};
+ auto const setShardCount = [&]( std::string const& shardCount ) {
+ auto result = Clara::Detail::convertInto( shardCount, config.shardCount );
+
+ if (config.shardCount == 0) {
+ return ParserResult::runtimeError( "The shard count must be greater than 0" );
+ } else {
+ return result;
+ }
+ };
auto cli
= ExeName( config.processName )
@@ -240,6 +249,12 @@ namespace Catch {
| Opt( config.benchmarkWarmupTime, "benchmarkWarmupTime" )
["--benchmark-warmup-time"]
( "amount of time in milliseconds spent on warming up each test (default: 100)" )
+ | Opt( setShardCount, "shard count" )
+ ["--shard-count"]
+ ( "split the tests to execute into this many groups" )
+ | Opt( config.shardIndex, "shard index" )
+ ["--shard-index"]
+ ( "index of the group of tests to execute (see --shard-count)" )
| Arg( config.testsOrTags, "test name|pattern|tags" )
( "which test or tests to use" );
diff --git a/src/catch2/internal/catch_sharding.hpp b/src/catch2/internal/catch_sharding.hpp
new file mode 100644
index 00000000..0e8822ee
--- /dev/null
+++ b/src/catch2/internal/catch_sharding.hpp
@@ -0,0 +1,41 @@
+
+// Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE_1_0.txt or copy at
+// https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+#ifndef CATCH_SHARDING_HPP_INCLUDED
+#define CATCH_SHARDING_HPP_INCLUDED
+
+#include
+
+#include
+
+namespace Catch {
+
+ template
+ Container createShard(Container const& container, std::size_t const shardCount, std::size_t const shardIndex) {
+ assert(shardCount > shardIndex);
+
+ if (shardCount == 1) {
+ return container;
+ }
+
+ const std::size_t totalTestCount = container.size();
+
+ const std::size_t shardSize = totalTestCount / shardCount;
+ const std::size_t leftoverTests = totalTestCount % shardCount;
+
+ const std::size_t startIndex = shardIndex * shardSize + (std::min)(shardIndex, leftoverTests);
+ const std::size_t endIndex = (shardIndex + 1) * shardSize + (std::min)(shardIndex + 1, leftoverTests);
+
+ auto startIterator = std::next(container.begin(), startIndex);
+ auto endIterator = std::next(container.begin(), endIndex);
+
+ return Container(startIterator, endIterator);
+ }
+
+}
+
+#endif // CATCH_SHARDING_HPP_INCLUDED
diff --git a/src/catch2/internal/catch_test_case_registry_impl.cpp b/src/catch2/internal/catch_test_case_registry_impl.cpp
index dd1744a7..3c1ab54b 100644
--- a/src/catch2/internal/catch_test_case_registry_impl.cpp
+++ b/src/catch2/internal/catch_test_case_registry_impl.cpp
@@ -12,6 +12,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -135,7 +136,7 @@ namespace {
filtered.push_back(testCase);
}
}
- return filtered;
+ return createShard(filtered, config.shardCount(), config.shardIndex());
}
std::vector const& getAllTestCasesSorted( IConfig const& config ) {
return getRegistryHub().getTestCaseRegistry().getAllTestsSorted( config );
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 7f7c32fe..9a46f6e2 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -87,6 +87,7 @@ set(TEST_SOURCES
${SELF_TEST_DIR}/IntrospectiveTests/RandomNumberGeneration.tests.cpp
${SELF_TEST_DIR}/IntrospectiveTests/Reporters.tests.cpp
${SELF_TEST_DIR}/IntrospectiveTests/Tag.tests.cpp
+ ${SELF_TEST_DIR}/IntrospectiveTests/Sharding.tests.cpp
${SELF_TEST_DIR}/IntrospectiveTests/String.tests.cpp
${SELF_TEST_DIR}/IntrospectiveTests/StringManip.tests.cpp
${SELF_TEST_DIR}/IntrospectiveTests/Xml.tests.cpp
@@ -310,6 +311,7 @@ set_tests_properties(TagAlias PROPERTIES
add_test(NAME RandomTestOrdering COMMAND ${PYTHON_EXECUTABLE}
${CATCH_DIR}/tests/TestScripts/testRandomOrder.py $)
+
add_test(NAME CheckConvenienceHeaders
COMMAND
${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/checkConvenienceHeaders.py
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index bce10df5..f2add6d6 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -8,6 +8,22 @@ project( Catch2ExtraTests LANGUAGES CXX )
message( STATUS "Extra tests included" )
+
+add_test(
+ NAME TestShardingIntegration
+ COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testSharding.py $
+)
+
+add_test(
+ NAME TestSharding::OverlyLargeShardIndex
+ COMMAND $ --shard-index 5 --shard-count 5
+)
+set_tests_properties(
+ TestSharding::OverlyLargeShardIndex
+ PROPERTIES
+ PASS_REGULAR_EXPRESSION "The shard count \\(5\\) must be greater than the shard index \\(5\\)"
+)
+
# The MinDuration reporting tests do not need separate compilation, but
# they have non-trivial execution time, so they are categorized as
# extra tests, so that they are run less.
diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt
index dfb08230..3f39422e 100644
--- a/tests/SelfTest/Baselines/compact.sw.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -1273,6 +1273,12 @@ CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-no-ana
CmdLine.tests.cpp:: passed: config.benchmarkNoAnalysis for: true
CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?}
CmdLine.tests.cpp:: passed: config.benchmarkWarmupTime == 10 for: 10 == 10
+CmdLine.tests.cpp:: passed: cli.parse({ "test", "--shard-count=8"}) for: {?}
+CmdLine.tests.cpp:: passed: config.shardCount == 8 for: 8 == 8
+CmdLine.tests.cpp:: passed: cli.parse({ "test", "--shard-index=2"}) for: {?}
+CmdLine.tests.cpp:: passed: config.shardIndex == 2 for: 2 == 2
+CmdLine.tests.cpp:: passed: !result for: true
+CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) for: "The shard count must be greater than 0" contains: "The shard count must be greater than 0"
Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 3 >= 1
Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 2 >= 1
Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 1 >= 1
diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt
index f7320bd7..af7fb09c 100644
--- a/tests/SelfTest/Baselines/console.std.approved.txt
+++ b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -1427,5 +1427,5 @@ due to unexpected exception with message:
===============================================================================
test cases: 373 | 296 passed | 70 failed | 7 failed as expected
-assertions: 2115 | 1959 passed | 129 failed | 27 failed as expected
+assertions: 2121 | 1965 passed | 129 failed | 27 failed as expected
diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt
index 695a30ef..189944cb 100644
--- a/tests/SelfTest/Baselines/console.sw.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -9390,6 +9390,61 @@ CmdLine.tests.cpp:: PASSED:
with expansion:
10 == 10
+-------------------------------------------------------------------------------
+Process can be configured on command line
+ Sharding options
+ shard-count
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:
+...............................................................................
+
+CmdLine.tests.cpp:: PASSED:
+ CHECK( cli.parse({ "test", "--shard-count=8"}) )
+with expansion:
+ {?}
+
+CmdLine.tests.cpp:: PASSED:
+ REQUIRE( config.shardCount == 8 )
+with expansion:
+ 8 == 8
+
+-------------------------------------------------------------------------------
+Process can be configured on command line
+ Sharding options
+ shard-index
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:
+...............................................................................
+
+CmdLine.tests.cpp:: PASSED:
+ CHECK( cli.parse({ "test", "--shard-index=2"}) )
+with expansion:
+ {?}
+
+CmdLine.tests.cpp:: PASSED:
+ REQUIRE( config.shardIndex == 2 )
+with expansion:
+ 2 == 2
+
+-------------------------------------------------------------------------------
+Process can be configured on command line
+ Sharding options
+ Zero shard-count
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:
+...............................................................................
+
+CmdLine.tests.cpp:: PASSED:
+ CHECK( !result )
+with expansion:
+ true
+
+CmdLine.tests.cpp:: PASSED:
+ CHECK_THAT( result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) )
+with expansion:
+ "The shard count must be greater than 0" contains: "The shard count must be
+ greater than 0"
+
-------------------------------------------------------------------------------
Product with differing arities - std::tuple
-------------------------------------------------------------------------------
@@ -17040,5 +17095,5 @@ Misc.tests.cpp:: PASSED:
===============================================================================
test cases: 373 | 280 passed | 86 failed | 7 failed as expected
-assertions: 2132 | 1959 passed | 146 failed | 27 failed as expected
+assertions: 2138 | 1965 passed | 146 failed | 27 failed as expected
diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt
index c2486391..ab55d93c 100644
--- a/tests/SelfTest/Baselines/junit.sw.approved.txt
+++ b/tests/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
-
+
@@ -1096,6 +1096,9 @@ Message.tests.cpp:
+
+
+
diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
index 18bca507..4f5e0b12 100644
--- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -75,6 +75,9 @@
+
+
+
diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt
index aad246f5..e5247b5f 100644
--- a/tests/SelfTest/Baselines/tap.sw.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.approved.txt
@@ -2468,6 +2468,18 @@ ok {test-number} - config.benchmarkNoAnalysis for: true
ok {test-number} - cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?}
# Process can be configured on command line
ok {test-number} - config.benchmarkWarmupTime == 10 for: 10 == 10
+# Process can be configured on command line
+ok {test-number} - cli.parse({ "test", "--shard-count=8"}) for: {?}
+# Process can be configured on command line
+ok {test-number} - config.shardCount == 8 for: 8 == 8
+# Process can be configured on command line
+ok {test-number} - cli.parse({ "test", "--shard-index=2"}) for: {?}
+# Process can be configured on command line
+ok {test-number} - config.shardIndex == 2 for: 2 == 2
+# Process can be configured on command line
+ok {test-number} - !result for: true
+# Process can be configured on command line
+ok {test-number} - result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) for: "The shard count must be greater than 0" contains: "The shard count must be greater than 0"
# Product with differing arities - std::tuple
ok {test-number} - std::tuple_size::value >= 1 for: 3 >= 1
# Product with differing arities - std::tuple
@@ -4266,5 +4278,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
-1..2132
+1..2138
diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt
index 419941de..256e43fd 100644
--- a/tests/SelfTest/Baselines/xml.sw.approved.txt
+++ b/tests/SelfTest/Baselines/xml.sw.approved.txt
@@ -11469,6 +11469,72 @@ Nor would this
+
+
+
+
+ cli.parse({ "test", "--shard-count=8"})
+
+
+ {?}
+
+
+
+
+ config.shardCount == 8
+
+
+ 8 == 8
+
+
+
+
+
+
+
+
+
+
+ cli.parse({ "test", "--shard-index=2"})
+
+
+ {?}
+
+
+
+
+ config.shardIndex == 2
+
+
+ 2 == 2
+
+
+
+
+
+
+
+
+
+
+ !result
+
+
+ true
+
+
+
+
+ result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" )
+
+
+ "The shard count must be greater than 0" contains: "The shard count must be greater than 0"
+
+
+
+
+
+
@@ -20030,6 +20096,6 @@ loose text artifact
-
+
diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
index c1fd6527..5b58bdec 100644
--- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
+++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
@@ -569,6 +569,27 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
REQUIRE(config.benchmarkWarmupTime == 10);
}
}
+
+ SECTION("Sharding options") {
+ SECTION("shard-count") {
+ CHECK(cli.parse({ "test", "--shard-count=8"}));
+
+ REQUIRE(config.shardCount == 8);
+ }
+
+ SECTION("shard-index") {
+ CHECK(cli.parse({ "test", "--shard-index=2"}));
+
+ REQUIRE(config.shardIndex == 2);
+ }
+
+ SECTION("Zero shard-count") {
+ auto result = cli.parse({ "test", "--shard-count=0"});
+
+ CHECK( !result );
+ CHECK_THAT( result.errorMessage(), ContainsSubstring( "The shard count must be greater than 0" ) );
+ }
+ }
}
TEST_CASE("Test with special, characters \"in name", "[cli][regression]") {
diff --git a/tests/SelfTest/IntrospectiveTests/Sharding.tests.cpp b/tests/SelfTest/IntrospectiveTests/Sharding.tests.cpp
new file mode 100644
index 00000000..9ef80d23
--- /dev/null
+++ b/tests/SelfTest/IntrospectiveTests/Sharding.tests.cpp
@@ -0,0 +1,41 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0. (See accompanying
+ * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+ */
+#include
+#include
+
+#include
+
+#include
+#include
+
+TEST_CASE("Sharding Function", "[approvals]") {
+ std::vector testContainer = { 0, 1, 2, 3, 4, 5, 6 };
+ std::unordered_map> expectedShardSizes = {
+ {1, {7}},
+ {2, {4, 3}},
+ {3, {3, 2, 2}},
+ {4, {2, 2, 2, 1}},
+ {5, {2, 2, 1, 1, 1}},
+ {6, {2, 1, 1, 1, 1, 1}},
+ {7, {1, 1, 1, 1, 1, 1, 1}},
+ };
+
+ auto shardCount = GENERATE(range(1, 7));
+ auto shardIndex = GENERATE_COPY(filter([=](int i) { return i < shardCount; }, range(0, 6)));
+
+ std::vector result = Catch::createShard(testContainer, shardCount, shardIndex);
+
+ auto& sizes = expectedShardSizes[shardCount];
+ REQUIRE(result.size() == sizes[shardIndex]);
+
+ std::size_t startIndex = 0;
+ for(int i = 0; i < shardIndex; i++) {
+ startIndex += sizes[i];
+ }
+
+ for(std::size_t i = 0; i < sizes[shardIndex]; i++) {
+ CHECK(result[i] == testContainer[i + startIndex]);
+ }
+}
\ No newline at end of file
diff --git a/tests/TestScripts/testSharding.py b/tests/TestScripts/testSharding.py
new file mode 100644
index 00000000..fcea03e2
--- /dev/null
+++ b/tests/TestScripts/testSharding.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+"""
+This test script verifies that sharding tests does change which tests are run.
+This is done by running the binary multiple times, once to list all the tests,
+once per shard to list the tests for that shard, and once again per shard to
+execute the tests. The sharded lists are compared to the full list to ensure
+none are skipped, duplicated, and that the order remains the same. This process
+is repeated for multiple command line argument combinations to ensure sharding
+works with different filters and test orderings.
+"""
+
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+
+from collections import namedtuple
+
+def make_base_commandline(self_test_exe):
+ return [
+ self_test_exe,
+ '--reporter', 'xml',
+ "--shard-count", "5",
+ "--shard-index", "2",
+ "[generators]~[benchmarks]~[.]"
+ ]
+
+def list_tests(self_test_exe):
+ cmd = make_base_commandline(self_test_exe) + ['--list-tests']
+
+ process = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ if stderr:
+ raise RuntimeError("Unexpected error output:\n" + process.stderr)
+
+ root = ET.fromstring(stdout)
+ result = [elem.text for elem in root.findall('./TestCase/Name')]
+
+ if len(result) < 2:
+ raise RuntimeError("Unexpectedly few tests listed (got {})".format(
+ len(result)))
+
+ return result
+
+
+def execute_tests(self_test_exe):
+ cmd = make_base_commandline(self_test_exe)
+
+ process = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ if stderr:
+ raise RuntimeError("Unexpected error output:\n" + process.stderr)
+
+ root = ET.fromstring(stdout)
+ result = [elem.attrib["name"] for elem in root.findall('./TestCase')]
+
+ if len(result) < 2:
+ raise RuntimeError("Unexpectedly few tests listed (got {})".format(
+ len(result)))
+ return result
+
+
+def check_listed_and_executed_tests_match(listed_tests, executed_tests):
+ listed_names = set(listed_tests)
+ executed_names = set(executed_tests)
+
+ listed_string = "\n".join(listed_names)
+ exeucted_string = "\n".join(executed_names)
+
+ assert listed_names == executed_names, (
+ "Executed tests do not match the listed tests:\nExecuted:\n{}\n\nListed:\n{}".format(exeucted_string, listed_string)
+ )
+
+
+def test_sharding(self_test_exe):
+ listed_tests = list_tests(self_test_exe)
+ executed_tests = execute_tests(self_test_exe)
+
+ check_listed_and_executed_tests_match(listed_tests, executed_tests)
+
+
+def main():
+ self_test_exe, = sys.argv[1:]
+
+ test_sharding(self_test_exe)
+
+if __name__ == '__main__':
+ sys.exit(main())