2017-07-25 22:41:35 +02:00
|
|
|
/*
|
|
|
|
* Created by Martin on 25/07/2017
|
|
|
|
*
|
|
|
|
* Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
|
|
* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
*/
|
|
|
|
|
2017-09-07 12:24:33 +02:00
|
|
|
#include "catch_test_case_registry_impl.h"
|
2017-07-25 22:41:35 +02:00
|
|
|
|
|
|
|
#include "catch_context.h"
|
2017-08-01 18:46:33 +02:00
|
|
|
#include "catch_enforce.h"
|
2017-07-25 22:41:35 +02:00
|
|
|
#include "catch_interfaces_registry_hub.h"
|
2017-08-30 15:32:44 +02:00
|
|
|
#include "catch_random_number_generator.h"
|
2019-10-06 21:47:54 +02:00
|
|
|
#include "catch_run_context.h"
|
2017-07-25 22:41:35 +02:00
|
|
|
#include "catch_string_manip.h"
|
|
|
|
#include "catch_test_case_info.h"
|
|
|
|
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
#include <algorithm>
|
2017-07-25 22:41:35 +02:00
|
|
|
#include <sstream>
|
|
|
|
|
|
|
|
namespace Catch {
|
|
|
|
|
2020-04-14 16:39:45 +02:00
|
|
|
namespace {
|
|
|
|
struct TestHasher {
|
2020-11-17 10:16:36 +01:00
|
|
|
using hash_t = uint64_t;
|
2017-07-25 22:41:35 +02:00
|
|
|
|
2020-11-17 21:08:33 +01:00
|
|
|
explicit TestHasher( hash_t hashSuffix ):
|
|
|
|
m_hashSuffix{ hashSuffix } {}
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
|
2020-11-17 21:08:33 +01:00
|
|
|
uint32_t operator()( TestCase const& t ) const {
|
2020-11-17 10:16:36 +01:00
|
|
|
// FNV-1a hash with multiplication fold.
|
|
|
|
const hash_t prime = 1099511628211u;
|
|
|
|
hash_t hash = 14695981039346656037u;
|
|
|
|
for ( const char c : t.name ) {
|
2020-04-14 16:39:45 +02:00
|
|
|
hash ^= c;
|
|
|
|
hash *= prime;
|
|
|
|
}
|
2020-11-17 21:08:33 +01:00
|
|
|
hash ^= m_hashSuffix;
|
2020-11-17 10:16:36 +01:00
|
|
|
hash *= prime;
|
|
|
|
const uint32_t low{ static_cast<uint32_t>( hash ) };
|
|
|
|
const uint32_t high{ static_cast<uint32_t>( hash >> 32 ) };
|
2020-11-17 21:08:33 +01:00
|
|
|
return low * high;
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
}
|
2020-11-17 21:08:33 +01:00
|
|
|
|
2020-11-17 10:16:36 +01:00
|
|
|
private:
|
2020-11-17 21:08:33 +01:00
|
|
|
hash_t m_hashSuffix;
|
2020-04-14 16:39:45 +02:00
|
|
|
};
|
|
|
|
} // end unnamed namespace
|
|
|
|
|
2017-07-25 22:41:35 +02:00
|
|
|
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
std::vector<TestCase> sortTests( IConfig const& config, std::vector<TestCase> const& unsortedTestCases ) {
|
2017-07-25 22:41:35 +02:00
|
|
|
switch( config.runOrder() ) {
|
2020-04-14 16:39:45 +02:00
|
|
|
case RunTests::InDeclarationOrder:
|
|
|
|
// already in declaration order
|
|
|
|
break;
|
|
|
|
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
case RunTests::InLexicographicalOrder: {
|
|
|
|
std::vector<TestCase> sorted = unsortedTestCases;
|
2017-07-25 22:41:35 +02:00
|
|
|
std::sort( sorted.begin(), sorted.end() );
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
return sorted;
|
|
|
|
}
|
2020-04-14 16:39:45 +02:00
|
|
|
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
case RunTests::InRandomOrder: {
|
2017-07-25 22:41:35 +02:00
|
|
|
seedRng( config );
|
2020-11-17 10:16:36 +01:00
|
|
|
TestHasher h{ config.rngSeed() };
|
2020-04-14 16:39:45 +02:00
|
|
|
|
2020-11-17 10:16:36 +01:00
|
|
|
using hashedTest = std::pair<TestHasher::hash_t, TestCase const*>;
|
2020-04-14 16:39:45 +02:00
|
|
|
std::vector<hashedTest> indexed_tests;
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
indexed_tests.reserve( unsortedTestCases.size() );
|
2020-04-14 16:39:45 +02:00
|
|
|
|
|
|
|
for (auto const& testCase : unsortedTestCases) {
|
|
|
|
indexed_tests.emplace_back(h(testCase), &testCase);
|
|
|
|
}
|
|
|
|
|
|
|
|
std::sort(indexed_tests.begin(), indexed_tests.end(),
|
|
|
|
[](hashedTest const& lhs, hashedTest const& rhs) {
|
|
|
|
if (lhs.first == rhs.first) {
|
|
|
|
return lhs.second->name < rhs.second->name;
|
|
|
|
}
|
|
|
|
return lhs.first < rhs.first;
|
|
|
|
});
|
|
|
|
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
std::vector<TestCase> sorted;
|
2020-04-14 16:39:45 +02:00
|
|
|
sorted.reserve( indexed_tests.size() );
|
|
|
|
|
|
|
|
for (auto const& hashed : indexed_tests) {
|
|
|
|
sorted.emplace_back(*hashed.second);
|
|
|
|
}
|
|
|
|
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
return sorted;
|
|
|
|
}
|
2017-07-25 22:41:35 +02:00
|
|
|
}
|
Change random test shuffling technique
Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order. This has two problems:
- It depends on the declaration order, so the order in which the tests
will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
to be able to find a minimal subset of tests which exhibits the issue.
However, any change to the set of tests being run will completely
change the test ordering, making it difficult or impossible to reduce
the set of tests being run in any reasonably efficient manner.
Therefore, change the randomization approach to resolve both these
issues.
Generate a random value based on the user-provided RNG seed. Convert
every test case to an integer by hashing a combination of that value
with the test name. Sort the test cases by this integer.
The test names and RNG are platform-independent, so this should be
consistent across platforms. Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.
To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm. Cannot
use std::hash, because it is important that the result be
platform-independent.
2020-04-10 02:17:05 +02:00
|
|
|
return unsortedTestCases;
|
2017-07-25 22:41:35 +02:00
|
|
|
}
|
2019-08-06 20:51:19 +02:00
|
|
|
|
|
|
|
bool isThrowSafe( TestCase const& testCase, IConfig const& config ) {
|
|
|
|
return !testCase.throws() || config.allowThrows();
|
|
|
|
}
|
|
|
|
|
2017-07-25 22:41:35 +02:00
|
|
|
bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ) {
|
2019-08-06 20:51:19 +02:00
|
|
|
return testSpec.matches( testCase ) && isThrowSafe( testCase, config );
|
2017-07-25 22:41:35 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void enforceNoDuplicateTestCases( std::vector<TestCase> const& functions ) {
|
|
|
|
std::set<TestCase> seenFunctions;
|
|
|
|
for( auto const& function : functions ) {
|
|
|
|
auto prev = seenFunctions.insert( function );
|
|
|
|
CATCH_ENFORCE( prev.second,
|
|
|
|
"error: TEST_CASE( \"" << function.name << "\" ) already defined.\n"
|
|
|
|
<< "\tFirst seen at " << prev.first->getTestCaseInfo().lineInfo << "\n"
|
|
|
|
<< "\tRedefined at " << function.getTestCaseInfo().lineInfo );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<TestCase> filterTests( std::vector<TestCase> const& testCases, TestSpec const& testSpec, IConfig const& config ) {
|
|
|
|
std::vector<TestCase> filtered;
|
|
|
|
filtered.reserve( testCases.size() );
|
2019-04-09 11:50:59 +02:00
|
|
|
for (auto const& testCase : testCases) {
|
|
|
|
if ((!testSpec.hasFilters() && !testCase.isHidden()) ||
|
|
|
|
(testSpec.hasFilters() && matchTest(testCase, testSpec, config))) {
|
|
|
|
filtered.push_back(testCase);
|
|
|
|
}
|
|
|
|
}
|
2017-07-25 22:41:35 +02:00
|
|
|
return filtered;
|
|
|
|
}
|
|
|
|
std::vector<TestCase> const& getAllTestCasesSorted( IConfig const& config ) {
|
|
|
|
return getRegistryHub().getTestCaseRegistry().getAllTestsSorted( config );
|
|
|
|
}
|
|
|
|
|
|
|
|
void TestRegistry::registerTest( TestCase const& testCase ) {
|
|
|
|
std::string name = testCase.getTestCaseInfo().name;
|
|
|
|
if( name.empty() ) {
|
2017-11-07 19:01:10 +01:00
|
|
|
ReusableStringStream rss;
|
|
|
|
rss << "Anonymous test case " << ++m_unnamedCount;
|
|
|
|
return registerTest( testCase.withName( rss.str() ) );
|
2017-07-25 22:41:35 +02:00
|
|
|
}
|
|
|
|
m_functions.push_back( testCase );
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<TestCase> const& TestRegistry::getAllTests() const {
|
|
|
|
return m_functions;
|
|
|
|
}
|
|
|
|
std::vector<TestCase> const& TestRegistry::getAllTestsSorted( IConfig const& config ) const {
|
|
|
|
if( m_sortedFunctions.empty() )
|
|
|
|
enforceNoDuplicateTestCases( m_functions );
|
|
|
|
|
|
|
|
if( m_currentSortOrder != config.runOrder() || m_sortedFunctions.empty() ) {
|
|
|
|
m_sortedFunctions = sortTests( config, m_functions );
|
|
|
|
m_currentSortOrder = config.runOrder();
|
|
|
|
}
|
|
|
|
return m_sortedFunctions;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////
|
|
|
|
TestInvokerAsFunction::TestInvokerAsFunction( void(*testAsFunction)() ) noexcept : m_testAsFunction( testAsFunction ) {}
|
|
|
|
|
|
|
|
void TestInvokerAsFunction::invoke() const {
|
|
|
|
m_testAsFunction();
|
|
|
|
}
|
|
|
|
|
2018-03-02 16:22:18 +01:00
|
|
|
std::string extractClassName( StringRef const& classOrQualifiedMethodName ) {
|
2019-09-08 14:49:40 +02:00
|
|
|
std::string className(classOrQualifiedMethodName);
|
2017-07-25 22:41:35 +02:00
|
|
|
if( startsWith( className, '&' ) )
|
|
|
|
{
|
|
|
|
std::size_t lastColons = className.rfind( "::" );
|
|
|
|
std::size_t penultimateColons = className.rfind( "::", lastColons-1 );
|
|
|
|
if( penultimateColons == std::string::npos )
|
|
|
|
penultimateColons = 1;
|
|
|
|
className = className.substr( penultimateColons, lastColons-penultimateColons );
|
|
|
|
}
|
|
|
|
return className;
|
|
|
|
}
|
|
|
|
|
|
|
|
} // end namespace Catch
|