mirror of
https://github.com/catchorg/Catch2.git
synced 2024-11-20 04:16:11 +01:00
Change random test shuffling technique (refactored)
Previously a random test ordering was obtained by applying std::shuffle to the tests in declaration order. This has two problems: - It depends on the declaration order, so the order in which the tests will be run will be platform-specific. - When trying to debug accidental inter-test dependencies, it is helpful to be able to find a minimal subset of tests which exhibits the issue. However, any change to the set of tests being run will completely change the test ordering, making it difficult or impossible to reduce the set of tests being run in any reasonably efficient manner. Therefore, change the randomization approach to resolve both these issues. Generate a random value based on the user-provided RNG seed. Convert every test case to an integer by hashing a combination of that value with the test name. Sort the test cases by this integer. The test names and RNG are platform-independent, so this should be consistent across platforms. Also, removing one test does not change the integer value associated with the remaining tests, so they remain in the same order. To hash, use the FNV-1a hash, except with the basis being our randomly selected value rather than the fixed basis set in the algorithm. Cannot use std::hash, because it is important that the result be platform-independent.
This commit is contained in:
parent
87a8b61d5a
commit
26b2c3e7e2
@ -16,27 +16,70 @@
|
|||||||
#include <catch2/catch_test_case_info.hpp>
|
#include <catch2/catch_test_case_info.hpp>
|
||||||
#include <catch2/catch_test_spec.hpp>
|
#include <catch2/catch_test_spec.hpp>
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <iterator>
|
||||||
|
#include <random>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
|
||||||
namespace Catch {
|
namespace Catch {
|
||||||
|
|
||||||
std::vector<TestCaseHandle> sortTests( IConfig const& config, std::vector<TestCaseHandle> const& unsortedTestCases ) {
|
namespace {
|
||||||
|
struct HashTest {
|
||||||
std::vector<TestCaseHandle> sorted = unsortedTestCases;
|
explicit HashTest(SimplePcg32& rng) {
|
||||||
|
basis = rng();
|
||||||
switch( config.runOrder() ) {
|
basis <<= 32;
|
||||||
case RunTests::InLexicographicalOrder:
|
basis |= rng();
|
||||||
std::sort( sorted.begin(), sorted.end() );
|
|
||||||
break;
|
|
||||||
case RunTests::InRandomOrder:
|
|
||||||
seedRng( config );
|
|
||||||
std::shuffle( sorted.begin(), sorted.end(), rng() );
|
|
||||||
break;
|
|
||||||
case RunTests::InDeclarationOrder:
|
|
||||||
// already in declaration order
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
return sorted;
|
|
||||||
|
uint64_t basis;
|
||||||
|
|
||||||
|
uint64_t operator()(TestCaseInfo const& t) const {
|
||||||
|
// Modified FNV-1a hash
|
||||||
|
static constexpr uint64_t prime = 1099511628211;
|
||||||
|
uint64_t hash = basis;
|
||||||
|
for (const char c : t.name) {
|
||||||
|
hash ^= c;
|
||||||
|
hash *= prime;
|
||||||
|
}
|
||||||
|
return hash;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} // end anonymous namespace
|
||||||
|
|
||||||
|
std::vector<TestCaseHandle> sortTests( IConfig const& config, std::vector<TestCaseHandle> const& unsortedTestCases ) {
|
||||||
|
switch (config.runOrder()) {
|
||||||
|
case RunTests::InDeclarationOrder:
|
||||||
|
return unsortedTestCases;
|
||||||
|
|
||||||
|
case RunTests::InLexicographicalOrder: {
|
||||||
|
std::vector<TestCaseHandle> sorted = unsortedTestCases;
|
||||||
|
std::sort(sorted.begin(), sorted.end());
|
||||||
|
return sorted;
|
||||||
|
}
|
||||||
|
case RunTests::InRandomOrder: {
|
||||||
|
seedRng(config);
|
||||||
|
HashTest h(rng());
|
||||||
|
std::vector<std::pair<uint64_t, TestCaseHandle>> indexed_tests;
|
||||||
|
indexed_tests.reserve(unsortedTestCases.size());
|
||||||
|
|
||||||
|
for (auto const& handle : unsortedTestCases) {
|
||||||
|
indexed_tests.emplace_back(h(handle.getTestCaseInfo()), handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::sort(indexed_tests.begin(), indexed_tests.end());
|
||||||
|
|
||||||
|
std::vector<TestCaseHandle> randomized;
|
||||||
|
randomized.reserve(indexed_tests.size());
|
||||||
|
|
||||||
|
for (auto const& indexed : indexed_tests) {
|
||||||
|
randomized.push_back(indexed.second);
|
||||||
|
}
|
||||||
|
|
||||||
|
return randomized;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CATCH_INTERNAL_ERROR("Unknown test order value!");
|
||||||
}
|
}
|
||||||
|
|
||||||
bool isThrowSafe( TestCaseHandle const& testCase, IConfig const& config ) {
|
bool isThrowSafe( TestCaseHandle const& testCase, IConfig const& config ) {
|
||||||
|
Loading…
Reference in New Issue
Block a user