3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-08-01 21:05:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

Always report rng seed from builtin reporters

Browse Source 
Not all reporters use a format that supports this, so TeamCity
and Automake reporters still do not report it. The console
reporter now reports it even on successful runs, where before
it only reported the rng seed in the header, which was showed
either for failed run, or for run with `-s`.

CLoses #2065
This commit is contained in:
Martin Hořeňovský
2022-04-19 10:32:13 +02:00
parent 90e6905050
commit 17a04f88d9
19 changed files with 73 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
tests/CMakeLists.txt
View File

@@ -523,5 +523,27 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
endforeach()
endif()
foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
"compact"
"console"
"JUnit"
"SonarQube"
"TAP"
# "TeamCity" - does not seem to support test suite-level metadata/comments
"XML")
add_test(NAME "Reporters:RngSeed:${reporterName}"
COMMAND
$<TARGET_FILE:SelfTest> "Factorials are computed"
--reporter ${reporterName}
--rng-seed 18181818
)
set_tests_properties("Reporters:RngSeed:${reporterName}"
PROPERTIES
PASS_REGULAR_EXPRESSION "18181818"
)
endforeach()
list(APPEND CATCH_WARNING_TARGETS SelfTest)
set(CATCH_WARNING_TARGETS ${CATCH_WARNING_TARGETS} PARENT_SCOPE)

2
tests/SelfTest/Baselines/compact.sw.approved.txt
View File

@@ -1,3 +1,5 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
RNG seed: 1
Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0

2
tests/SelfTest/Baselines/compact.sw.multi.approved.txt
View File

@@ -1,3 +1,5 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
RNG seed: 1
Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0

3
tests/SelfTest/Baselines/console.std.approved.txt
View File

@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
Randomness seeded to: 1
-------------------------------------------------------------------------------
#1455 - INFO and WARN can start with a linebreak
-------------------------------------------------------------------------------

3
tests/SelfTest/Baselines/console.sw.approved.txt
View File

@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
Randomness seeded to: 1
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------

3
tests/SelfTest/Baselines/console.sw.multi.approved.txt
View File

@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
Randomness seeded to: 1
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------

3
tests/SelfTest/Baselines/console.swa4.approved.txt
View File

@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
Randomness seeded to: 1
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------

1
tests/SelfTest/Baselines/sonarqube.sw.approved.txt
View File

@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- rng-seed=1 -->
<testExecutions version="1"loose text artifact
>
<file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">

1
tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
View File

@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- rng-seed=1 -->
<testExecutions version="1">
<file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">
<testCase name="Clara::Arg supports single-arg parse the way Opt does" duration="{duration}"/>

1
tests/SelfTest/Baselines/tap.sw.approved.txt
View File

@@ -1,3 +1,4 @@
# rng-seed: 1
# # A test name that starts with a #
ok {test-number} - with 1 message: 'yay'
# #1027: Bitfields can be captured

1
tests/SelfTest/Baselines/tap.sw.multi.approved.txt
View File

@@ -1,3 +1,4 @@
# rng-seed: 1
# # A test name that starts with a #
ok {test-number} - with 1 message: 'yay'
# #1027: Bitfields can be captured

2
tests/TestScripts/testConfigureDefaultReporter.py
View File

@@ -38,7 +38,7 @@ stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTe
# This matches the summary line made by compact reporter, console reporter's
# summary line does not match the regex.
summary_regex = 'Passed \d+ test case with \d+ assertions.'
if not re.match(summary_regex, stdout):
if not re.search(summary_regex, stdout):
print("Could not find '{}' in the stdout".format(summary_regex))
print('stdout: "{}"'.format(stdout))
exit(2)