diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp index 1d3d0329..3f036ec5 100644 --- a/src/catch2/reporters/catch_reporter_compact.cpp +++ b/src/catch2/reporters/catch_reporter_compact.cpp @@ -7,6 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 #include +#include #include #include #include @@ -253,6 +254,16 @@ private: m_stream << "No test cases matched '" << unmatchedSpec << "'\n"; } + void CompactReporter::testRunStarting( TestRunInfo const& ) { + if ( m_config->testSpec().hasFilters() ) { + m_stream << m_colour->guardColour( Colour::BrightYellow ) + << "Filters: " + << serializeFilters( m_config->getTestsOrTags() ) + << '\n'; + } + m_stream << "RNG seed: " << m_config->rngSeed() << '\n'; + } + void CompactReporter::assertionEnded( AssertionStats const& _assertionStats ) { AssertionResult const& result = _assertionStats.assertionResult; diff --git a/src/catch2/reporters/catch_reporter_compact.hpp b/src/catch2/reporters/catch_reporter_compact.hpp index 4232138c..6362fc29 100644 --- a/src/catch2/reporters/catch_reporter_compact.hpp +++ b/src/catch2/reporters/catch_reporter_compact.hpp @@ -24,6 +24,8 @@ namespace Catch { void noMatchingTestCases( StringRef unmatchedSpec ) override; + void testRunStarting( TestRunInfo const& _testInfo ) override; + void assertionEnded(AssertionStats const& _assertionStats) override; void sectionEnded(SectionStats const& _sectionStats) override; diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp index 6b564b15..bb755ed7 100644 --- a/src/catch2/reporters/catch_reporter_console.cpp +++ b/src/catch2/reporters/catch_reporter_console.cpp @@ -496,7 +496,11 @@ void ConsoleReporter::testRunEnded(TestRunStats const& _testRunStats) { } void ConsoleReporter::testRunStarting(TestRunInfo const& _testInfo) { StreamingReporterBase::testRunStarting(_testInfo); - printTestFilters(); + if ( m_config->testSpec().hasFilters() ) { + m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: " + << serializeFilters( m_config->getTestsOrTags() ) << '\n'; + } + m_stream << "Randomness seeded to: " << m_config->rngSeed() << '\n'; } void ConsoleReporter::lazyPrint() { @@ -521,8 +525,7 @@ void ConsoleReporter::lazyPrintRunInfo() { << m_colour->guardColour( Colour::SecondaryText ) << currentTestRunInfo.name << " is a Catch2 v" << libraryVersion() << " host application.\n" - << "Run with -? for options\n\n" - << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + << "Run with -? for options\n\n"; m_testRunInfoPrinted = true; } @@ -701,13 +704,6 @@ void ConsoleReporter::printSummaryDivider() { m_stream << lineOfChars('-') << '\n'; } -void ConsoleReporter::printTestFilters() { - if (m_config->testSpec().hasFilters()) { - m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: " - << serializeFilters( m_config->getTestsOrTags() ) << '\n'; - } -} - } // end namespace Catch #if defined(_MSC_VER) diff --git a/src/catch2/reporters/catch_reporter_console.hpp b/src/catch2/reporters/catch_reporter_console.hpp index 12bfc0e2..719dcc44 100644 --- a/src/catch2/reporters/catch_reporter_console.hpp +++ b/src/catch2/reporters/catch_reporter_console.hpp @@ -63,7 +63,6 @@ namespace Catch { void printTotalsDivider(Totals const& totals); void printSummaryDivider(); - void printTestFilters(); bool m_headerPrinted = false; bool m_testRunInfoPrinted = false; diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp index d45bbfe1..8119c3b8 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.cpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -9,13 +9,25 @@ #include #include +#include +#include #include namespace Catch { + namespace { + std::string createRngSeedString(uint32_t seed) { + ReusableStringStream sstr; + sstr << "rng-seed=" << seed; + return sstr.str(); + } + } + void SonarQubeReporter::testRunStarting(TestRunInfo const& testRunInfo) { CumulativeReporterBase::testRunStarting(testRunInfo); + + xml.writeComment( createRngSeedString( m_config->rngSeed() ) ); xml.startElement("testExecutions"); xml.writeAttribute("version"_sr, '1'); } diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp index 71249f87..8ecf3928 100644 --- a/src/catch2/reporters/catch_reporter_tap.cpp +++ b/src/catch2/reporters/catch_reporter_tap.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -189,6 +190,10 @@ namespace Catch { } // End anonymous namespace + void TAPReporter::testRunStarting( TestRunInfo const& ) { + m_stream << "# rng-seed: " << m_config->rngSeed() << '\n'; + } + void TAPReporter::noMatchingTestCases( StringRef unmatchedSpec ) { m_stream << "# No test cases matched '" << unmatchedSpec << "'\n"; } diff --git a/src/catch2/reporters/catch_reporter_tap.hpp b/src/catch2/reporters/catch_reporter_tap.hpp index b8c3208f..0c4b14b4 100644 --- a/src/catch2/reporters/catch_reporter_tap.hpp +++ b/src/catch2/reporters/catch_reporter_tap.hpp @@ -26,6 +26,8 @@ namespace Catch { return "Reports test results in TAP format, suitable for test harnesses"s; } + void testRunStarting( TestRunInfo const& testInfo ) override; + void noMatchingTestCases( StringRef unmatchedSpec ) override; void assertionEnded(AssertionStats const& _assertionStats) override; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5ea6b7cd..911f3857 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -523,5 +523,27 @@ if (CATCH_ENABLE_CONFIGURE_TESTS) endforeach() endif() +foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata + "compact" + "console" + "JUnit" + "SonarQube" + "TAP" + # "TeamCity" - does not seem to support test suite-level metadata/comments + "XML") + add_test(NAME "Reporters:RngSeed:${reporterName}" + COMMAND + $ "Factorials are computed" + --reporter ${reporterName} + --rng-seed 18181818 + ) + set_tests_properties("Reporters:RngSeed:${reporterName}" + PROPERTIES + PASS_REGULAR_EXPRESSION "18181818" + ) + +endforeach() + + list(APPEND CATCH_WARNING_TARGETS SelfTest) set(CATCH_WARNING_TARGETS ${CATCH_WARNING_TARGETS} PARENT_SCOPE) diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 03a07f53..bf9482c6 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1,3 +1,5 @@ +Filters: ~[!nonportable]~[!benchmark]~[approvals] * +RNG seed: 1 Misc.tests.cpp:: passed: with 1 message: 'yay' Compilation.tests.cpp:: passed: y.v == 0 for: 0 == 0 Compilation.tests.cpp:: passed: 0 == y.v for: 0 == 0 diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 105789c9..a94192eb 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -1,3 +1,5 @@ +Filters: ~[!nonportable]~[!benchmark]~[approvals] * +RNG seed: 1 Misc.tests.cpp:: passed: with 1 message: 'yay' Compilation.tests.cpp:: passed: y.v == 0 for: 0 == 0 Compilation.tests.cpp:: passed: 0 == y.v for: 0 == 0 diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index a6847cdf..c85a9203 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- #1455 - INFO and WARN can start with a linebreak ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 66706e13..8bb1c3db 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- # A test name that starts with a # ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 08b16228..a64c56a5 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- # A test name that starts with a # ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/console.swa4.approved.txt b/tests/SelfTest/Baselines/console.swa4.approved.txt index 8b4b4c13..c25719d8 100644 --- a/tests/SelfTest/Baselines/console.swa4.approved.txt +++ b/tests/SelfTest/Baselines/console.swa4.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- # A test name that starts with a # ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 83b5d3cb..f2ec6f49 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1,4 +1,5 @@ + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 2723afe8..98cb8b7d 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1,4 +1,5 @@ + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 2f375905..a2e69782 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -1,3 +1,4 @@ +# rng-seed: 1 # # A test name that starts with a # ok {test-number} - with 1 message: 'yay' # #1027: Bitfields can be captured diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 2c5790b4..69d85c7a 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -1,3 +1,4 @@ +# rng-seed: 1 # # A test name that starts with a # ok {test-number} - with 1 message: 'yay' # #1027: Bitfields can be captured diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py index 24ec8228..66c88da0 100644 --- a/tests/TestScripts/testConfigureDefaultReporter.py +++ b/tests/TestScripts/testConfigureDefaultReporter.py @@ -38,7 +38,7 @@ stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTe # This matches the summary line made by compact reporter, console reporter's # summary line does not match the regex. summary_regex = 'Passed \d+ test case with \d+ assertions.' -if not re.match(summary_regex, stdout): +if not re.search(summary_regex, stdout): print("Could not find '{}' in the stdout".format(summary_regex)) print('stdout: "{}"'.format(stdout)) exit(2)