Always report rng seed from builtin reporters

Not all reporters use a format that supports this, so TeamCity
and Automake reporters still do not report it. The console
reporter now reports it even on successful runs, where before
it only reported the rng seed in the header, which was showed
either for failed run, or for run with `-s`.

CLoses #2065

src/catch2/reporters/catch_reporter_compact.cpp

@@ -7,6 +7,7 @@
 // SPDX-License-Identifier: BSL-1.0
 #include <catch2/reporters/catch_reporter_compact.hpp>
 
+#include <catch2/catch_test_spec.hpp>
 #include <catch2/reporters/catch_reporter_helpers.hpp>
 #include <catch2/interfaces/catch_interfaces_config.hpp>
 #include <catch2/internal/catch_platform.hpp>
@@ -253,6 +254,16 @@ private:
             m_stream << "No test cases matched '" << unmatchedSpec << "'\n";
         }
 
+        void CompactReporter::testRunStarting( TestRunInfo const& ) {
+            if ( m_config->testSpec().hasFilters() ) {
+                m_stream << m_colour->guardColour( Colour::BrightYellow )
+                         << "Filters: "
+                         << serializeFilters( m_config->getTestsOrTags() )
+                         << '\n';
+            }
+            m_stream << "RNG seed: " << m_config->rngSeed() << '\n';
+        }
+
         void CompactReporter::assertionEnded( AssertionStats const& _assertionStats ) {
             AssertionResult const& result = _assertionStats.assertionResult;
 

src/catch2/reporters/catch_reporter_compact.hpp

@@ -24,6 +24,8 @@ namespace Catch {
 
         void noMatchingTestCases( StringRef unmatchedSpec ) override;
 
+        void testRunStarting( TestRunInfo const& _testInfo ) override;
+
         void assertionEnded(AssertionStats const& _assertionStats) override;
 
         void sectionEnded(SectionStats const& _sectionStats) override;

src/catch2/reporters/catch_reporter_console.cpp

@@ -496,7 +496,11 @@ void ConsoleReporter::testRunEnded(TestRunStats const& _testRunStats) {
 }
 void ConsoleReporter::testRunStarting(TestRunInfo const& _testInfo) {
     StreamingReporterBase::testRunStarting(_testInfo);
-    printTestFilters();
+    if ( m_config->testSpec().hasFilters() ) {
+        m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: "
+                 << serializeFilters( m_config->getTestsOrTags() ) << '\n';
+    }
+    m_stream << "Randomness seeded to: " << m_config->rngSeed() << '\n';
 }
 
 void ConsoleReporter::lazyPrint() {
@@ -521,8 +525,7 @@ void ConsoleReporter::lazyPrintRunInfo() {
              << m_colour->guardColour( Colour::SecondaryText )
              << currentTestRunInfo.name << " is a Catch2 v" << libraryVersion()
              << " host application.\n"
-             << "Run with -? for options\n\n"
+             << "Run with -? for options\n\n";
-             << "Randomness seeded to: " << m_config->rngSeed() << "\n\n";
 
     m_testRunInfoPrinted = true;
 }
@@ -701,13 +704,6 @@ void ConsoleReporter::printSummaryDivider() {
     m_stream << lineOfChars('-') << '\n';
 }
 
-void ConsoleReporter::printTestFilters() {
-    if (m_config->testSpec().hasFilters()) {
-        m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: "
-                 << serializeFilters( m_config->getTestsOrTags() ) << '\n';
-    }
-}
-
 } // end namespace Catch
 
 #if defined(_MSC_VER)

src/catch2/reporters/catch_reporter_console.hpp

@@ -63,7 +63,6 @@ namespace Catch {
 
         void printTotalsDivider(Totals const& totals);
         void printSummaryDivider();
-        void printTestFilters();
 
         bool m_headerPrinted = false;
         bool m_testRunInfoPrinted = false;

src/catch2/reporters/catch_reporter_sonarqube.cpp

@@ -9,13 +9,25 @@
 
 #include <catch2/internal/catch_string_manip.hpp>
 #include <catch2/catch_test_case_info.hpp>
+#include <catch2/internal/catch_reusable_string_stream.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
 
 #include <map>
 
 namespace Catch {
 
+    namespace {
+        std::string createRngSeedString(uint32_t seed) {
+            ReusableStringStream sstr;
+            sstr << "rng-seed=" << seed;
+            return sstr.str();
+        }
+    }
+
     void SonarQubeReporter::testRunStarting(TestRunInfo const& testRunInfo) {
         CumulativeReporterBase::testRunStarting(testRunInfo);
+
+        xml.writeComment( createRngSeedString( m_config->rngSeed() ) );
         xml.startElement("testExecutions");
         xml.writeAttribute("version"_sr, '1');
     }

src/catch2/reporters/catch_reporter_tap.cpp

@@ -9,6 +9,7 @@
 #include <catch2/internal/catch_console_colour.hpp>
 #include <catch2/internal/catch_string_manip.hpp>
 #include <catch2/catch_test_case_info.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
 
 #include <algorithm>
 #include <iterator>
@@ -189,6 +190,10 @@ namespace Catch {
 
     } // End anonymous namespace
 
+    void TAPReporter::testRunStarting( TestRunInfo const& ) {
+        m_stream << "# rng-seed: " << m_config->rngSeed() << '\n';
+    }
+
     void TAPReporter::noMatchingTestCases( StringRef unmatchedSpec ) {
         m_stream << "# No test cases matched '" << unmatchedSpec << "'\n";
     }

src/catch2/reporters/catch_reporter_tap.hpp

@@ -26,6 +26,8 @@ namespace Catch {
             return "Reports test results in TAP format, suitable for test harnesses"s;
         }
 
+        void testRunStarting( TestRunInfo const& testInfo ) override;
+
         void noMatchingTestCases( StringRef unmatchedSpec ) override;
 
         void assertionEnded(AssertionStats const& _assertionStats) override;

tests/CMakeLists.txt

@@ -523,5 +523,27 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
     endforeach()
 endif()
 
+foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
+                      "compact"
+                      "console"
+                      "JUnit"
+                      "SonarQube"
+                      "TAP"
+                      # "TeamCity" - does not seem to support test suite-level metadata/comments
+                      "XML")
+    add_test(NAME "Reporters:RngSeed:${reporterName}"
+      COMMAND
+        $<TARGET_FILE:SelfTest> "Factorials are computed"
+          --reporter ${reporterName}
+          --rng-seed 18181818
+    )
+    set_tests_properties("Reporters:RngSeed:${reporterName}"
+      PROPERTIES
+        PASS_REGULAR_EXPRESSION "18181818"
+    )
+
+endforeach()
+
+
 list(APPEND CATCH_WARNING_TARGETS SelfTest)
 set(CATCH_WARNING_TARGETS ${CATCH_WARNING_TARGETS} PARENT_SCOPE)

tests/SelfTest/Baselines/compact.sw.approved.txt

@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
 Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
 Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
 Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0

tests/SelfTest/Baselines/compact.sw.multi.approved.txt

@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
 Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
 Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
 Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0

tests/SelfTest/Baselines/console.std.approved.txt

@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 #1455 - INFO and WARN can start with a linebreak
 -------------------------------------------------------------------------------

tests/SelfTest/Baselines/console.sw.approved.txt

@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 # A test name that starts with a #
 -------------------------------------------------------------------------------

tests/SelfTest/Baselines/console.sw.multi.approved.txt

@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 # A test name that starts with a #
 -------------------------------------------------------------------------------

tests/SelfTest/Baselines/console.swa4.approved.txt

@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 # A test name that starts with a #
 -------------------------------------------------------------------------------

tests/SelfTest/Baselines/sonarqube.sw.approved.txt

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
 <testExecutions version="1"loose text artifact
 >
   <file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">

tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
 <testExecutions version="1">
   <file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">
     <testCase name="Clara::Arg supports single-arg parse the way Opt does" duration="{duration}"/>

tests/SelfTest/Baselines/tap.sw.approved.txt

@@ -1,3 +1,4 @@
+# rng-seed: 1
 # # A test name that starts with a #
 ok {test-number} - with 1 message: 'yay'
 # #1027: Bitfields can be captured

tests/SelfTest/Baselines/tap.sw.multi.approved.txt

@@ -1,3 +1,4 @@
+# rng-seed: 1
 # # A test name that starts with a #
 ok {test-number} - with 1 message: 'yay'
 # #1027: Bitfields can be captured

tests/TestScripts/testConfigureDefaultReporter.py

@@ -38,7 +38,7 @@ stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTe
 # This matches the summary line made by compact reporter, console reporter's
 # summary line does not match the regex.
 summary_regex = 'Passed \d+ test case with \d+ assertions.'
-if not re.match(summary_regex, stdout):
+if not re.search(summary_regex, stdout):
     print("Could not find '{}' in the stdout".format(summary_regex))
     print('stdout: "{}"'.format(stdout))
     exit(2)