From 17a04f88d90d3687bd561e89dab40e4ba6a55b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Tue, 19 Apr 2022 10:32:13 +0200 Subject: [PATCH] Always report rng seed from builtin reporters Not all reporters use a format that supports this, so TeamCity and Automake reporters still do not report it. The console reporter now reports it even on successful runs, where before it only reported the rng seed in the header, which was showed either for failed run, or for run with `-s`. CLoses #2065 --- .../reporters/catch_reporter_compact.cpp | 11 ++++++++++ .../reporters/catch_reporter_compact.hpp | 2 ++ .../reporters/catch_reporter_console.cpp | 16 +++++--------- .../reporters/catch_reporter_console.hpp | 1 - .../reporters/catch_reporter_sonarqube.cpp | 12 ++++++++++ src/catch2/reporters/catch_reporter_tap.cpp | 5 +++++ src/catch2/reporters/catch_reporter_tap.hpp | 2 ++ tests/CMakeLists.txt | 22 +++++++++++++++++++ .../Baselines/compact.sw.approved.txt | 2 ++ .../Baselines/compact.sw.multi.approved.txt | 2 ++ .../Baselines/console.std.approved.txt | 3 +-- .../Baselines/console.sw.approved.txt | 3 +-- .../Baselines/console.sw.multi.approved.txt | 3 +-- .../Baselines/console.swa4.approved.txt | 3 +-- .../Baselines/sonarqube.sw.approved.txt | 1 + .../Baselines/sonarqube.sw.multi.approved.txt | 1 + tests/SelfTest/Baselines/tap.sw.approved.txt | 1 + .../Baselines/tap.sw.multi.approved.txt | 1 + .../testConfigureDefaultReporter.py | 2 +- 19 files changed, 73 insertions(+), 20 deletions(-) diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp index 1d3d0329..3f036ec5 100644 --- a/src/catch2/reporters/catch_reporter_compact.cpp +++ b/src/catch2/reporters/catch_reporter_compact.cpp @@ -7,6 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 #include +#include #include #include #include @@ -253,6 +254,16 @@ private: m_stream << "No test cases matched '" << unmatchedSpec << "'\n"; } + void CompactReporter::testRunStarting( TestRunInfo const& ) { + if ( m_config->testSpec().hasFilters() ) { + m_stream << m_colour->guardColour( Colour::BrightYellow ) + << "Filters: " + << serializeFilters( m_config->getTestsOrTags() ) + << '\n'; + } + m_stream << "RNG seed: " << m_config->rngSeed() << '\n'; + } + void CompactReporter::assertionEnded( AssertionStats const& _assertionStats ) { AssertionResult const& result = _assertionStats.assertionResult; diff --git a/src/catch2/reporters/catch_reporter_compact.hpp b/src/catch2/reporters/catch_reporter_compact.hpp index 4232138c..6362fc29 100644 --- a/src/catch2/reporters/catch_reporter_compact.hpp +++ b/src/catch2/reporters/catch_reporter_compact.hpp @@ -24,6 +24,8 @@ namespace Catch { void noMatchingTestCases( StringRef unmatchedSpec ) override; + void testRunStarting( TestRunInfo const& _testInfo ) override; + void assertionEnded(AssertionStats const& _assertionStats) override; void sectionEnded(SectionStats const& _sectionStats) override; diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp index 6b564b15..bb755ed7 100644 --- a/src/catch2/reporters/catch_reporter_console.cpp +++ b/src/catch2/reporters/catch_reporter_console.cpp @@ -496,7 +496,11 @@ void ConsoleReporter::testRunEnded(TestRunStats const& _testRunStats) { } void ConsoleReporter::testRunStarting(TestRunInfo const& _testInfo) { StreamingReporterBase::testRunStarting(_testInfo); - printTestFilters(); + if ( m_config->testSpec().hasFilters() ) { + m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: " + << serializeFilters( m_config->getTestsOrTags() ) << '\n'; + } + m_stream << "Randomness seeded to: " << m_config->rngSeed() << '\n'; } void ConsoleReporter::lazyPrint() { @@ -521,8 +525,7 @@ void ConsoleReporter::lazyPrintRunInfo() { << m_colour->guardColour( Colour::SecondaryText ) << currentTestRunInfo.name << " is a Catch2 v" << libraryVersion() << " host application.\n" - << "Run with -? for options\n\n" - << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + << "Run with -? for options\n\n"; m_testRunInfoPrinted = true; } @@ -701,13 +704,6 @@ void ConsoleReporter::printSummaryDivider() { m_stream << lineOfChars('-') << '\n'; } -void ConsoleReporter::printTestFilters() { - if (m_config->testSpec().hasFilters()) { - m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: " - << serializeFilters( m_config->getTestsOrTags() ) << '\n'; - } -} - } // end namespace Catch #if defined(_MSC_VER) diff --git a/src/catch2/reporters/catch_reporter_console.hpp b/src/catch2/reporters/catch_reporter_console.hpp index 12bfc0e2..719dcc44 100644 --- a/src/catch2/reporters/catch_reporter_console.hpp +++ b/src/catch2/reporters/catch_reporter_console.hpp @@ -63,7 +63,6 @@ namespace Catch { void printTotalsDivider(Totals const& totals); void printSummaryDivider(); - void printTestFilters(); bool m_headerPrinted = false; bool m_testRunInfoPrinted = false; diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp index d45bbfe1..8119c3b8 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.cpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -9,13 +9,25 @@ #include #include +#include +#include #include namespace Catch { + namespace { + std::string createRngSeedString(uint32_t seed) { + ReusableStringStream sstr; + sstr << "rng-seed=" << seed; + return sstr.str(); + } + } + void SonarQubeReporter::testRunStarting(TestRunInfo const& testRunInfo) { CumulativeReporterBase::testRunStarting(testRunInfo); + + xml.writeComment( createRngSeedString( m_config->rngSeed() ) ); xml.startElement("testExecutions"); xml.writeAttribute("version"_sr, '1'); } diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp index 71249f87..8ecf3928 100644 --- a/src/catch2/reporters/catch_reporter_tap.cpp +++ b/src/catch2/reporters/catch_reporter_tap.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -189,6 +190,10 @@ namespace Catch { } // End anonymous namespace + void TAPReporter::testRunStarting( TestRunInfo const& ) { + m_stream << "# rng-seed: " << m_config->rngSeed() << '\n'; + } + void TAPReporter::noMatchingTestCases( StringRef unmatchedSpec ) { m_stream << "# No test cases matched '" << unmatchedSpec << "'\n"; } diff --git a/src/catch2/reporters/catch_reporter_tap.hpp b/src/catch2/reporters/catch_reporter_tap.hpp index b8c3208f..0c4b14b4 100644 --- a/src/catch2/reporters/catch_reporter_tap.hpp +++ b/src/catch2/reporters/catch_reporter_tap.hpp @@ -26,6 +26,8 @@ namespace Catch { return "Reports test results in TAP format, suitable for test harnesses"s; } + void testRunStarting( TestRunInfo const& testInfo ) override; + void noMatchingTestCases( StringRef unmatchedSpec ) override; void assertionEnded(AssertionStats const& _assertionStats) override; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5ea6b7cd..911f3857 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -523,5 +523,27 @@ if (CATCH_ENABLE_CONFIGURE_TESTS) endforeach() endif() +foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata + "compact" + "console" + "JUnit" + "SonarQube" + "TAP" + # "TeamCity" - does not seem to support test suite-level metadata/comments + "XML") + add_test(NAME "Reporters:RngSeed:${reporterName}" + COMMAND + $ "Factorials are computed" + --reporter ${reporterName} + --rng-seed 18181818 + ) + set_tests_properties("Reporters:RngSeed:${reporterName}" + PROPERTIES + PASS_REGULAR_EXPRESSION "18181818" + ) + +endforeach() + + list(APPEND CATCH_WARNING_TARGETS SelfTest) set(CATCH_WARNING_TARGETS ${CATCH_WARNING_TARGETS} PARENT_SCOPE) diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 03a07f53..bf9482c6 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1,3 +1,5 @@ +Filters: ~[!nonportable]~[!benchmark]~[approvals] * +RNG seed: 1 Misc.tests.cpp:: passed: with 1 message: 'yay' Compilation.tests.cpp:: passed: y.v == 0 for: 0 == 0 Compilation.tests.cpp:: passed: 0 == y.v for: 0 == 0 diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 105789c9..a94192eb 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -1,3 +1,5 @@ +Filters: ~[!nonportable]~[!benchmark]~[approvals] * +RNG seed: 1 Misc.tests.cpp:: passed: with 1 message: 'yay' Compilation.tests.cpp:: passed: y.v == 0 for: 0 == 0 Compilation.tests.cpp:: passed: 0 == y.v for: 0 == 0 diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index a6847cdf..c85a9203 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- #1455 - INFO and WARN can start with a linebreak ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 66706e13..8bb1c3db 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- # A test name that starts with a # ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 08b16228..a64c56a5 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- # A test name that starts with a # ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/console.swa4.approved.txt b/tests/SelfTest/Baselines/console.swa4.approved.txt index 8b4b4c13..c25719d8 100644 --- a/tests/SelfTest/Baselines/console.swa4.approved.txt +++ b/tests/SelfTest/Baselines/console.swa4.approved.txt @@ -1,11 +1,10 @@ Filters: ~[!nonportable]~[!benchmark]~[approvals] * +Randomness seeded to: 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ is a host application. Run with -? for options -Randomness seeded to: 1 - ------------------------------------------------------------------------------- # A test name that starts with a # ------------------------------------------------------------------------------- diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 83b5d3cb..f2ec6f49 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1,4 +1,5 @@ + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 2723afe8..98cb8b7d 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1,4 +1,5 @@ + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 2f375905..a2e69782 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -1,3 +1,4 @@ +# rng-seed: 1 # # A test name that starts with a # ok {test-number} - with 1 message: 'yay' # #1027: Bitfields can be captured diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 2c5790b4..69d85c7a 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -1,3 +1,4 @@ +# rng-seed: 1 # # A test name that starts with a # ok {test-number} - with 1 message: 'yay' # #1027: Bitfields can be captured diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py index 24ec8228..66c88da0 100644 --- a/tests/TestScripts/testConfigureDefaultReporter.py +++ b/tests/TestScripts/testConfigureDefaultReporter.py @@ -38,7 +38,7 @@ stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTe # This matches the summary line made by compact reporter, console reporter's # summary line does not match the regex. summary_regex = 'Passed \d+ test case with \d+ assertions.' -if not re.match(summary_regex, stdout): +if not re.search(summary_regex, stdout): print("Could not find '{}' in the stdout".format(summary_regex)) print('stdout: "{}"'.format(stdout)) exit(2)