Provide random-device option for --rng-seed and make it default

This commit is contained in:
Martin Hořeňovský 2021-10-08 21:32:03 +02:00
parent 200a487cf2
commit ed1f343a41
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
7 changed files with 24 additions and 17 deletions

View File

@ -277,13 +277,20 @@ is that as long as the random seed is fixed, running only some tests
<a id="rng-seed"></a> <a id="rng-seed"></a>
## Specify a seed for the Random Number Generator ## Specify a seed for the Random Number Generator
<pre>--rng-seed &lt;'time'|number&gt;</pre> <pre>--rng-seed &lt;'time'|'random-device'|number&gt;</pre>
Sets a seed for the random number generator using ```std::srand()```. Sets the seed for random number generators used by Catch2. These are used
If a number is provided this is used directly as the seed so the random pattern is repeatable. e.g. to shuffle tests when user asks for tests to be in random order.
Alternatively if the keyword ```time``` is provided then the result of calling ```std::time(0)``` is used and so the pattern becomes unpredictable. In some cases, you might need to pass the keyword ```time``` in double quotes instead of single quotes.
Using `time` as the argument asks Catch2 generate the seed through call
to `std::time(nullptr)`. This provides very weak randomness and multiple
runs of the binary can generate the same seed if they are started close
to each other.
Using `random-device` asks for `std::random_device` to be used instead.
If your implementation provides working `std::random_device`, it should
be preferred to using `time`. Catch2 uses `std::random_device` by default.
In either case the actual value for the seed is printed as part of Catch's output so if an issue is discovered that is sensitive to test ordering the ordering can be reproduced - even if it was originally seeded from ```std::time(0)```.
<a id="libidentify"></a> <a id="libidentify"></a>
## Identify framework and version according to the libIdentify standard ## Identify framework and version according to the libIdentify standard

View File

@ -131,6 +131,7 @@ new design.
* With the exception of the XmlReporter, the outputs of first party reporters should remain the same * With the exception of the XmlReporter, the outputs of first party reporters should remain the same
* New pair of events were added * New pair of events were added
* One obsolete event was removed * One obsolete event was removed
* Catch2 generates a random seed if one hasn't been specified by the user
### Improvements ### Improvements
@ -161,6 +162,7 @@ new design.
* `--list-*` flags write their output to file specified by the `-o` flag (#2061, #2163) * `--list-*` flags write their output to file specified by the `-o` flag (#2061, #2163)
* `Approx::operator()` is now properly `const` * `Approx::operator()` is now properly `const`
* Catch2's internal helper variables no longer use reserved identifiers (#578) * Catch2's internal helper variables no longer use reserved identifiers (#578)
* `--rng-seed` now accepts string `"random-device"` to generate random seed using `std::random_device`
### Fixes ### Fixes

View File

@ -72,7 +72,7 @@ namespace Catch {
ShowDurations Config::showDurations() const { return m_data.showDurations; } ShowDurations Config::showDurations() const { return m_data.showDurations; }
double Config::minDuration() const { return m_data.minDuration; } double Config::minDuration() const { return m_data.minDuration; }
TestRunOrder Config::runOrder() const { return m_data.runOrder; } TestRunOrder Config::runOrder() const { return m_data.runOrder; }
unsigned int Config::rngSeed() const { return *m_data.rngSeed; } uint32_t Config::rngSeed() const { return m_data.rngSeed; }
UseColour Config::useColour() const { return m_data.useColour; } UseColour Config::useColour() const { return m_data.useColour; }
bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; } bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; }
int Config::abortAfter() const { return m_data.abortAfter; } int Config::abortAfter() const { return m_data.abortAfter; }

View File

@ -12,6 +12,7 @@
#include <catch2/interfaces/catch_interfaces_config.hpp> #include <catch2/interfaces/catch_interfaces_config.hpp>
#include <catch2/internal/catch_unique_ptr.hpp> #include <catch2/internal/catch_unique_ptr.hpp>
#include <catch2/internal/catch_optional.hpp> #include <catch2/internal/catch_optional.hpp>
#include <catch2/internal/catch_random_seed_generation.hpp>
#include <vector> #include <vector>
#include <string> #include <string>
@ -34,7 +35,7 @@ namespace Catch {
bool libIdentify = false; bool libIdentify = false;
int abortAfter = -1; int abortAfter = -1;
Optional<unsigned int> rngSeed; uint32_t rngSeed = generateRandomSeed(GenerateFrom::Default);
bool benchmarkNoAnalysis = false; bool benchmarkNoAnalysis = false;
unsigned int benchmarkSamples = 100; unsigned int benchmarkSamples = 100;
@ -97,7 +98,7 @@ namespace Catch {
ShowDurations showDurations() const override; ShowDurations showDurations() const override;
double minDuration() const override; double minDuration() const override;
TestRunOrder runOrder() const override; TestRunOrder runOrder() const override;
unsigned int rngSeed() const override; uint32_t rngSeed() const override;
UseColour useColour() const override; UseColour useColour() const override;
bool shouldDebugBreak() const override; bool shouldDebugBreak() const override;
int abortAfter() const override; int abortAfter() const override;

View File

@ -24,7 +24,6 @@
#include <catch2/internal/catch_move_and_forward.hpp> #include <catch2/internal/catch_move_and_forward.hpp>
#include <algorithm> #include <algorithm>
#include <ctime>
#include <iomanip> #include <iomanip>
#include <set> #include <set>
@ -171,10 +170,6 @@ namespace Catch {
if( m_startupExceptions ) if( m_startupExceptions )
return 1; return 1;
if (!m_configData.rngSeed) {
m_configData.rngSeed = static_cast<unsigned int>(std::time(nullptr));
}
auto result = m_cli.parse( Clara::Args( argc, argv ) ); auto result = m_cli.parse( Clara::Args( argc, argv ) );
if( !result ) { if( !result ) {
config(); config();

View File

@ -73,7 +73,7 @@ namespace Catch {
virtual bool hasTestFilters() const = 0; virtual bool hasTestFilters() const = 0;
virtual std::vector<std::string> const& getTestsOrTags() const = 0; virtual std::vector<std::string> const& getTestsOrTags() const = 0;
virtual TestRunOrder runOrder() const = 0; virtual TestRunOrder runOrder() const = 0;
virtual unsigned int rngSeed() const = 0; virtual uint32_t rngSeed() const = 0;
virtual UseColour useColour() const = 0; virtual UseColour useColour() const = 0;
virtual std::vector<std::string> const& getSectionsToRun() const = 0; virtual std::vector<std::string> const& getSectionsToRun() const = 0;
virtual Verbosity verbosity() const = 0; virtual Verbosity verbosity() const = 0;

View File

@ -15,7 +15,6 @@
#include <catch2/interfaces/catch_interfaces_reporter.hpp> #include <catch2/interfaces/catch_interfaces_reporter.hpp>
#include <fstream> #include <fstream>
#include <ctime>
#include <string> #include <string>
namespace Catch { namespace Catch {
@ -74,7 +73,10 @@ namespace Catch {
}; };
auto const setRngSeed = [&]( std::string const& seed ) { auto const setRngSeed = [&]( std::string const& seed ) {
if( seed == "time" ) { if( seed == "time" ) {
config.rngSeed = static_cast<unsigned int>(std::time(nullptr)); config.rngSeed = generateRandomSeed(GenerateFrom::Time);
return ParserResult::ok(ParseResultType::Matched);
} else if (seed == "random-device") {
config.rngSeed = generateRandomSeed(GenerateFrom::RandomDevice);
return ParserResult::ok(ParseResultType::Matched); return ParserResult::ok(ParseResultType::Matched);
} }
@ -211,7 +213,7 @@ namespace Catch {
| Opt( setTestOrder, "decl|lex|rand" ) | Opt( setTestOrder, "decl|lex|rand" )
["--order"] ["--order"]
( "test case order (defaults to decl)" ) ( "test case order (defaults to decl)" )
| Opt( setRngSeed, "'time'|number" ) | Opt( setRngSeed, "'time'|'random-device'|number" )
["--rng-seed"] ["--rng-seed"]
( "set a specific seed for random numbers" ) ( "set a specific seed for random numbers" )
| Opt( setColourUsage, "yes|no" ) | Opt( setColourUsage, "yes|no" )