From 4ff8b27bb6bed4b8b309e56cd269b4f1fbc24e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Wed, 26 Jan 2022 23:47:40 +0100 Subject: [PATCH] Cleanups output normalization in ApprovalTests Removed: * NaN normalization * INFINITY normalization * errno normalization * Completely unused duration regex Tests using these macros should be tagged `[approvals]` so they are not run as part of approval tests. Also simplified regex for the test's executable filename, and hidden some tests relying on nullptr normalization. --- .../Baselines/automake.sw.approved.txt | 4 +- .../Baselines/automake.sw.multi.approved.txt | 4 +- .../Baselines/compact.sw.approved.txt | 7 +--- .../Baselines/compact.sw.multi.approved.txt | 7 +--- .../Baselines/console.std.approved.txt | 6 +-- .../Baselines/console.sw.approved.txt | 38 ++---------------- .../Baselines/console.sw.multi.approved.txt | 38 ++---------------- .../Baselines/console.swa4.approved.txt | 25 ++---------- .../SelfTest/Baselines/junit.sw.approved.txt | 6 +-- .../Baselines/junit.sw.multi.approved.txt | 6 +-- .../Baselines/sonarqube.sw.approved.txt | 4 +- .../Baselines/sonarqube.sw.multi.approved.txt | 4 +- tests/SelfTest/Baselines/tap.sw.approved.txt | 14 ++----- .../Baselines/tap.sw.multi.approved.txt | 14 ++----- .../Baselines/teamcity.sw.approved.txt | 8 +--- .../Baselines/teamcity.sw.multi.approved.txt | 8 +--- tests/SelfTest/Baselines/xml.sw.approved.txt | 40 ++----------------- .../Baselines/xml.sw.multi.approved.txt | 40 ++----------------- .../UsageTests/Decomposition.tests.cpp | 2 +- tests/SelfTest/UsageTests/Misc.tests.cpp | 9 ++++- .../UsageTests/ToStringTuple.tests.cpp | 3 +- tools/scripts/approvalTests.py | 32 +-------------- 22 files changed, 55 insertions(+), 264 deletions(-) diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 1fff117c..c575637f 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -1,5 +1,4 @@ :test-result: PASS # A test name that starts with a # -:test-result: PASS #1005: Comparing pointer to int and long (NULL can be either on various systems) :test-result: PASS #1027: Bitfields can be captured :test-result: PASS #1147 :test-result: PASS #1175 - Hidden Test @@ -29,7 +28,7 @@ Nor would this :test-result: XFAIL #748 - captures with unexpected exceptions :test-result: PASS #809 :test-result: PASS #833 -:test-result: XFAIL #835 -- errno should not be touched by Catch +:test-result: XFAIL #835 -- errno should not be touched by Catch2 :test-result: PASS #872 :test-result: PASS #961 -- Dynamically created sections should all be reported :test-result: FAIL 'Not' checks that should fail @@ -375,7 +374,6 @@ loose text artifact :test-result: PASS tuple<> :test-result: PASS tuple :test-result: PASS tuple -:test-result: PASS tuple<0,int,const char *> :test-result: PASS tuple :test-result: PASS tuple,tuple<>,float> :test-result: PASS uniform samples diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 4b75ed64..31c0b106 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -1,5 +1,4 @@ :test-result: PASS # A test name that starts with a # -:test-result: PASS #1005: Comparing pointer to int and long (NULL can be either on various systems) :test-result: PASS #1027: Bitfields can be captured :test-result: PASS #1147 :test-result: PASS #1175 - Hidden Test @@ -27,7 +26,7 @@ :test-result: XFAIL #748 - captures with unexpected exceptions :test-result: PASS #809 :test-result: PASS #833 -:test-result: XFAIL #835 -- errno should not be touched by Catch +:test-result: XFAIL #835 -- errno should not be touched by Catch2 :test-result: PASS #872 :test-result: PASS #961 -- Dynamically created sections should all be reported :test-result: FAIL 'Not' checks that should fail @@ -367,7 +366,6 @@ :test-result: PASS tuple<> :test-result: PASS tuple :test-result: PASS tuple -:test-result: PASS tuple<0,int,const char *> :test-result: PASS tuple :test-result: PASS tuple,tuple<>,float> :test-result: PASS uniform samples diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index f921fc2d..20098061 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1,6 +1,4 @@ Misc.tests.cpp:: passed: with 1 message: 'yay' -Decomposition.tests.cpp:: passed: fptr == 0 for: 0 == 0 -Decomposition.tests.cpp:: passed: fptr == 0l for: 0 == 0 Compilation.tests.cpp:: passed: y.v == 0 for: 0 == 0 Compilation.tests.cpp:: passed: 0 == y.v for: 0 == 0 Compilation.tests.cpp:: passed: t1 == t2 for: {?} == {?} @@ -96,7 +94,7 @@ Compilation.tests.cpp:: passed: throws_int(false) Compilation.tests.cpp:: passed: "aaa", Catch::Matchers::EndsWith("aaa") for: "aaa" ends with: "aaa" Compilation.tests.cpp:: passed: templated_tests(3) for: true Misc.tests.cpp:: failed: f() == 0 for: 1 == 0 -Misc.tests.cpp:: passed: errno == 1 for: 1 == 1 +Misc.tests.cpp:: passed: errno_after == 1 for: 1 == 1 Compilation.tests.cpp:: passed: x == 4 for: {?} == 4 with 1 message: 'dummy := 0' Misc.tests.cpp:: passed: with 1 message: 'Everything is OK' Misc.tests.cpp:: passed: with 1 message: 'Everything is OK' @@ -2426,9 +2424,6 @@ ToStringTuple.tests.cpp:: passed: "{ }" == ::Catch::Detail::stringi ToStringTuple.tests.cpp:: passed: "1.2f" == ::Catch::Detail::stringify(float(1.2)) for: "1.2f" == "1.2f" ToStringTuple.tests.cpp:: passed: "{ 1.2f, 0 }" == ::Catch::Detail::stringify(type{1.2f,0}) for: "{ 1.2f, 0 }" == "{ 1.2f, 0 }" ToStringTuple.tests.cpp:: passed: "{ 0 }" == ::Catch::Detail::stringify(type{0}) for: "{ 0 }" == "{ 0 }" -ToStringTuple.tests.cpp:: passed: "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) for: "{ 0, 42, "Catch me" }" -== -"{ 0, 42, "Catch me" }" ToStringTuple.tests.cpp:: passed: "{ \"hello\", \"world\" }" == ::Catch::Detail::stringify(type{"hello","world"}) for: "{ "hello", "world" }" == "{ "hello", "world" }" diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index ffd0932a..993a7b7c 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -1,6 +1,4 @@ Misc.tests.cpp:: passed: with 1 message: 'yay' -Decomposition.tests.cpp:: passed: fptr == 0 for: 0 == 0 -Decomposition.tests.cpp:: passed: fptr == 0l for: 0 == 0 Compilation.tests.cpp:: passed: y.v == 0 for: 0 == 0 Compilation.tests.cpp:: passed: 0 == y.v for: 0 == 0 Compilation.tests.cpp:: passed: t1 == t2 for: {?} == {?} @@ -94,7 +92,7 @@ Compilation.tests.cpp:: passed: throws_int(false) Compilation.tests.cpp:: passed: "aaa", Catch::Matchers::EndsWith("aaa") for: "aaa" ends with: "aaa" Compilation.tests.cpp:: passed: templated_tests(3) for: true Misc.tests.cpp:: failed: f() == 0 for: 1 == 0 -Misc.tests.cpp:: passed: errno == 1 for: 1 == 1 +Misc.tests.cpp:: passed: errno_after == 1 for: 1 == 1 Compilation.tests.cpp:: passed: x == 4 for: {?} == 4 with 1 message: 'dummy := 0' Misc.tests.cpp:: passed: with 1 message: 'Everything is OK' Misc.tests.cpp:: passed: with 1 message: 'Everything is OK' @@ -2418,9 +2416,6 @@ ToStringTuple.tests.cpp:: passed: "{ }" == ::Catch::Detail::stringi ToStringTuple.tests.cpp:: passed: "1.2f" == ::Catch::Detail::stringify(float(1.2)) for: "1.2f" == "1.2f" ToStringTuple.tests.cpp:: passed: "{ 1.2f, 0 }" == ::Catch::Detail::stringify(type{1.2f,0}) for: "{ 1.2f, 0 }" == "{ 1.2f, 0 }" ToStringTuple.tests.cpp:: passed: "{ 0 }" == ::Catch::Detail::stringify(type{0}) for: "{ 0 }" == "{ 0 }" -ToStringTuple.tests.cpp:: passed: "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) for: "{ 0, 42, "Catch me" }" -== -"{ 0, 42, "Catch me" }" ToStringTuple.tests.cpp:: passed: "{ \"hello\", \"world\" }" == ::Catch::Detail::stringify(type{"hello","world"}) for: "{ "hello", "world" }" == "{ "hello", "world" }" diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index ca299535..f6f6b292 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -54,7 +54,7 @@ due to unexpected exception with messages: expected exception ------------------------------------------------------------------------------- -#835 -- errno should not be touched by Catch +#835 -- errno should not be touched by Catch2 ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... @@ -1395,6 +1395,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 383 | 307 passed | 69 failed | 7 failed as expected -assertions: 2212 | 2057 passed | 128 failed | 27 failed as expected +test cases: 381 | 305 passed | 69 failed | 7 failed as expected +assertions: 2209 | 2054 passed | 128 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 11ee162e..8d1a25b4 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -16,23 +16,6 @@ Misc.tests.cpp:: PASSED: with message: yay -------------------------------------------------------------------------------- -#1005: Comparing pointer to int and long (NULL can be either on various - systems) -------------------------------------------------------------------------------- -Decomposition.tests.cpp: -............................................................................... - -Decomposition.tests.cpp:: PASSED: - REQUIRE( fptr == 0 ) -with expansion: - 0 == 0 - -Decomposition.tests.cpp:: PASSED: - REQUIRE( fptr == 0l ) -with expansion: - 0 == 0 - ------------------------------------------------------------------------------- #1027: Bitfields can be captured ------------------------------------------------------------------------------- @@ -863,7 +846,7 @@ with expansion: true ------------------------------------------------------------------------------- -#835 -- errno should not be touched by Catch +#835 -- errno should not be touched by Catch2 ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... @@ -874,7 +857,7 @@ with expansion: 1 == 0 Misc.tests.cpp:: PASSED: - REQUIRE( errno == 1 ) + REQUIRE( errno_after == 1 ) with expansion: 1 == 1 @@ -17232,19 +17215,6 @@ ToStringTuple.tests.cpp:: PASSED: with expansion: "{ 0 }" == "{ 0 }" -------------------------------------------------------------------------------- -tuple<0,int,const char *> -------------------------------------------------------------------------------- -ToStringTuple.tests.cpp: -............................................................................... - -ToStringTuple.tests.cpp:: PASSED: - CHECK( "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) ) -with expansion: - "{ 0, 42, "Catch me" }" - == - "{ 0, 42, "Catch me" }" - ------------------------------------------------------------------------------- tuple ------------------------------------------------------------------------------- @@ -17784,6 +17754,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 383 | 293 passed | 83 failed | 7 failed as expected -assertions: 2227 | 2057 passed | 143 failed | 27 failed as expected +test cases: 381 | 291 passed | 83 failed | 7 failed as expected +assertions: 2224 | 2054 passed | 143 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 9155ad62..920a1c7a 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -16,23 +16,6 @@ Misc.tests.cpp:: PASSED: with message: yay -------------------------------------------------------------------------------- -#1005: Comparing pointer to int and long (NULL can be either on various - systems) -------------------------------------------------------------------------------- -Decomposition.tests.cpp: -............................................................................... - -Decomposition.tests.cpp:: PASSED: - REQUIRE( fptr == 0 ) -with expansion: - 0 == 0 - -Decomposition.tests.cpp:: PASSED: - REQUIRE( fptr == 0l ) -with expansion: - 0 == 0 - ------------------------------------------------------------------------------- #1027: Bitfields can be captured ------------------------------------------------------------------------------- @@ -861,7 +844,7 @@ with expansion: true ------------------------------------------------------------------------------- -#835 -- errno should not be touched by Catch +#835 -- errno should not be touched by Catch2 ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... @@ -872,7 +855,7 @@ with expansion: 1 == 0 Misc.tests.cpp:: PASSED: - REQUIRE( errno == 1 ) + REQUIRE( errno_after == 1 ) with expansion: 1 == 1 @@ -17224,19 +17207,6 @@ ToStringTuple.tests.cpp:: PASSED: with expansion: "{ 0 }" == "{ 0 }" -------------------------------------------------------------------------------- -tuple<0,int,const char *> -------------------------------------------------------------------------------- -ToStringTuple.tests.cpp: -............................................................................... - -ToStringTuple.tests.cpp:: PASSED: - CHECK( "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) ) -with expansion: - "{ 0, 42, "Catch me" }" - == - "{ 0, 42, "Catch me" }" - ------------------------------------------------------------------------------- tuple ------------------------------------------------------------------------------- @@ -17776,6 +17746,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 383 | 293 passed | 83 failed | 7 failed as expected -assertions: 2227 | 2057 passed | 143 failed | 27 failed as expected +test cases: 381 | 291 passed | 83 failed | 7 failed as expected +assertions: 2224 | 2054 passed | 143 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.swa4.approved.txt b/tests/SelfTest/Baselines/console.swa4.approved.txt index df003bd3..8b4b4c13 100644 --- a/tests/SelfTest/Baselines/console.swa4.approved.txt +++ b/tests/SelfTest/Baselines/console.swa4.approved.txt @@ -16,23 +16,6 @@ Misc.tests.cpp:: PASSED: with message: yay -------------------------------------------------------------------------------- -#1005: Comparing pointer to int and long (NULL can be either on various - systems) -------------------------------------------------------------------------------- -Decomposition.tests.cpp: -............................................................................... - -Decomposition.tests.cpp:: PASSED: - REQUIRE( fptr == 0 ) -with expansion: - 0 == 0 - -Decomposition.tests.cpp:: PASSED: - REQUIRE( fptr == 0l ) -with expansion: - 0 == 0 - ------------------------------------------------------------------------------- #1027: Bitfields can be captured ------------------------------------------------------------------------------- @@ -863,7 +846,7 @@ with expansion: true ------------------------------------------------------------------------------- -#835 -- errno should not be touched by Catch +#835 -- errno should not be touched by Catch2 ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... @@ -874,7 +857,7 @@ with expansion: 1 == 0 Misc.tests.cpp:: PASSED: - REQUIRE( errno == 1 ) + REQUIRE( errno_after == 1 ) with expansion: 1 == 1 @@ -959,6 +942,6 @@ Condition.tests.cpp:: FAILED: CHECK( true != true ) =============================================================================== -test cases: 33 | 28 passed | 3 failed | 2 failed as expected -assertions: 103 | 96 passed | 4 failed | 3 failed as expected +test cases: 32 | 27 passed | 3 failed | 2 failed as expected +assertions: 101 | 94 passed | 4 failed | 3 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 459b77ed..c020fa4b 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,13 +1,12 @@ - + - @@ -71,7 +70,7 @@ Exception.tests.cpp: - + FAILED: @@ -1866,7 +1865,6 @@ Exception.tests.cpp: - diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 8f8eccf2..91398339 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,12 +1,11 @@ - + - @@ -70,7 +69,7 @@ Exception.tests.cpp: - + FAILED: @@ -1865,7 +1864,6 @@ Exception.tests.cpp: - diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index e597f41a..cc901e29 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -865,7 +865,6 @@ Condition.tests.cpp: - FAILED: @@ -1553,7 +1552,7 @@ Message.tests.cpp: - + FAILED: CHECK( f() == 0 ) @@ -1847,7 +1846,6 @@ Misc.tests.cpp: - diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 9637e8a1..0359fa40 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -864,7 +864,6 @@ Condition.tests.cpp: - FAILED: @@ -1552,7 +1551,7 @@ Message.tests.cpp: - + FAILED: CHECK( f() == 0 ) @@ -1846,7 +1845,6 @@ Misc.tests.cpp: - diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 6306f579..bd0e9881 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -1,9 +1,5 @@ # # A test name that starts with a # ok {test-number} - with 1 message: 'yay' -# #1005: Comparing pointer to int and long (NULL can be either on various systems) -ok {test-number} - fptr == 0 for: 0 == 0 -# #1005: Comparing pointer to int and long (NULL can be either on various systems) -ok {test-number} - fptr == 0l for: 0 == 0 # #1027: Bitfields can be captured ok {test-number} - y.v == 0 for: 0 == 0 # #1027: Bitfields can be captured @@ -188,10 +184,10 @@ ok {test-number} - throws_int(false) ok {test-number} - "aaa", Catch::Matchers::EndsWith("aaa") for: "aaa" ends with: "aaa" # #833 ok {test-number} - templated_tests(3) for: true -# #835 -- errno should not be touched by Catch +# #835 -- errno should not be touched by Catch2 not ok {test-number} - f() == 0 for: 1 == 0 -# #835 -- errno should not be touched by Catch -ok {test-number} - errno == 1 for: 1 == 1 +# #835 -- errno should not be touched by Catch2 +ok {test-number} - errno_after == 1 for: 1 == 1 # #872 ok {test-number} - x == 4 for: {?} == 4 with 1 message: 'dummy := 0' # #961 -- Dynamically created sections should all be reported @@ -4320,8 +4316,6 @@ ok {test-number} - "1.2f" == ::Catch::Detail::stringify(float(1.2)) for: "1.2f" ok {test-number} - "{ 1.2f, 0 }" == ::Catch::Detail::stringify(type{1.2f,0}) for: "{ 1.2f, 0 }" == "{ 1.2f, 0 }" # tuple ok {test-number} - "{ 0 }" == ::Catch::Detail::stringify(type{0}) for: "{ 0 }" == "{ 0 }" -# tuple<0,int,const char *> -ok {test-number} - "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) for: "{ 0, 42, "Catch me" }" == "{ 0, 42, "Catch me" }" # tuple ok {test-number} - "{ \"hello\", \"world\" }" == ::Catch::Detail::stringify(type{"hello","world"}) for: "{ "hello", "world" }" == "{ "hello", "world" }" # tuple,tuple<>,float> @@ -4456,5 +4450,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2227 +1..2224 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index c6b4a8bd..59e8d1a8 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -1,9 +1,5 @@ # # A test name that starts with a # ok {test-number} - with 1 message: 'yay' -# #1005: Comparing pointer to int and long (NULL can be either on various systems) -ok {test-number} - fptr == 0 for: 0 == 0 -# #1005: Comparing pointer to int and long (NULL can be either on various systems) -ok {test-number} - fptr == 0l for: 0 == 0 # #1027: Bitfields can be captured ok {test-number} - y.v == 0 for: 0 == 0 # #1027: Bitfields can be captured @@ -186,10 +182,10 @@ ok {test-number} - throws_int(false) ok {test-number} - "aaa", Catch::Matchers::EndsWith("aaa") for: "aaa" ends with: "aaa" # #833 ok {test-number} - templated_tests(3) for: true -# #835 -- errno should not be touched by Catch +# #835 -- errno should not be touched by Catch2 not ok {test-number} - f() == 0 for: 1 == 0 -# #835 -- errno should not be touched by Catch -ok {test-number} - errno == 1 for: 1 == 1 +# #835 -- errno should not be touched by Catch2 +ok {test-number} - errno_after == 1 for: 1 == 1 # #872 ok {test-number} - x == 4 for: {?} == 4 with 1 message: 'dummy := 0' # #961 -- Dynamically created sections should all be reported @@ -4312,8 +4308,6 @@ ok {test-number} - "1.2f" == ::Catch::Detail::stringify(float(1.2)) for: "1.2f" ok {test-number} - "{ 1.2f, 0 }" == ::Catch::Detail::stringify(type{1.2f,0}) for: "{ 1.2f, 0 }" == "{ 1.2f, 0 }" # tuple ok {test-number} - "{ 0 }" == ::Catch::Detail::stringify(type{0}) for: "{ 0 }" == "{ 0 }" -# tuple<0,int,const char *> -ok {test-number} - "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) for: "{ 0, 42, "Catch me" }" == "{ 0, 42, "Catch me" }" # tuple ok {test-number} - "{ \"hello\", \"world\" }" == ::Catch::Detail::stringify(type{"hello","world"}) for: "{ "hello", "world" }" == "{ "hello", "world" }" # tuple,tuple<>,float> @@ -4448,5 +4442,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2227 +1..2224 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index 90e318f1..e7279bc4 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -1,8 +1,6 @@ ##teamcity[testSuiteStarted name=''] ##teamcity[testStarted name='# A test name that starts with a #'] ##teamcity[testFinished name='# A test name that starts with a #' duration="{duration}"] -##teamcity[testStarted name='#1005: Comparing pointer to int and long (NULL can be either on various systems)'] -##teamcity[testFinished name='#1005: Comparing pointer to int and long (NULL can be either on various systems)' duration="{duration}"] ##teamcity[testStarted name='#1027: Bitfields can be captured'] ##teamcity[testFinished name='#1027: Bitfields can be captured' duration="{duration}"] ##teamcity[testStarted name='#1147'] @@ -62,9 +60,9 @@ Exception.tests.cpp:|nunexpected exception with messages:|n "answe ##teamcity[testFinished name='#809' duration="{duration}"] ##teamcity[testStarted name='#833'] ##teamcity[testFinished name='#833' duration="{duration}"] -##teamcity[testStarted name='#835 -- errno should not be touched by Catch'] +##teamcity[testStarted name='#835 -- errno should not be touched by Catch2'] Misc.tests.cpp:|nexpression failed|n CHECK( f() == 0 )|nwith expansion:|n 1 == 0|n- failure ignore as test marked as |'ok to fail|'|n'] -##teamcity[testFinished name='#835 -- errno should not be touched by Catch' duration="{duration}"] +##teamcity[testFinished name='#835 -- errno should not be touched by Catch2' duration="{duration}"] ##teamcity[testStarted name='#872'] ##teamcity[testFinished name='#872' duration="{duration}"] ##teamcity[testStarted name='#961 -- Dynamically created sections should all be reported'] @@ -898,8 +896,6 @@ Exception.tests.cpp:|nunexpected exception with message:|n "Why wo ##teamcity[testFinished name='tuple' duration="{duration}"] ##teamcity[testStarted name='tuple'] ##teamcity[testFinished name='tuple' duration="{duration}"] -##teamcity[testStarted name='tuple<0,int,const char *>'] -##teamcity[testFinished name='tuple<0,int,const char *>' duration="{duration}"] ##teamcity[testStarted name='tuple'] ##teamcity[testFinished name='tuple' duration="{duration}"] ##teamcity[testStarted name='tuple,tuple<>,float>'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index dc266f2f..28912d91 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -1,8 +1,6 @@ ##teamcity[testSuiteStarted name=''] ##teamcity[testStarted name='# A test name that starts with a #'] ##teamcity[testFinished name='# A test name that starts with a #' duration="{duration}"] -##teamcity[testStarted name='#1005: Comparing pointer to int and long (NULL can be either on various systems)'] -##teamcity[testFinished name='#1005: Comparing pointer to int and long (NULL can be either on various systems)' duration="{duration}"] ##teamcity[testStarted name='#1027: Bitfields can be captured'] ##teamcity[testFinished name='#1027: Bitfields can be captured' duration="{duration}"] ##teamcity[testStarted name='#1147'] @@ -62,9 +60,9 @@ Exception.tests.cpp:|nunexpected exception with messages:|n "answe ##teamcity[testFinished name='#809' duration="{duration}"] ##teamcity[testStarted name='#833'] ##teamcity[testFinished name='#833' duration="{duration}"] -##teamcity[testStarted name='#835 -- errno should not be touched by Catch'] +##teamcity[testStarted name='#835 -- errno should not be touched by Catch2'] Misc.tests.cpp:|nexpression failed|n CHECK( f() == 0 )|nwith expansion:|n 1 == 0|n- failure ignore as test marked as |'ok to fail|'|n'] -##teamcity[testFinished name='#835 -- errno should not be touched by Catch' duration="{duration}"] +##teamcity[testFinished name='#835 -- errno should not be touched by Catch2' duration="{duration}"] ##teamcity[testStarted name='#872'] ##teamcity[testFinished name='#872' duration="{duration}"] ##teamcity[testStarted name='#961 -- Dynamically created sections should all be reported'] @@ -897,8 +895,6 @@ Exception.tests.cpp:|nunexpected exception with message:|n "Why wo ##teamcity[testFinished name='tuple' duration="{duration}"] ##teamcity[testStarted name='tuple'] ##teamcity[testFinished name='tuple' duration="{duration}"] -##teamcity[testStarted name='tuple<0,int,const char *>'] -##teamcity[testFinished name='tuple<0,int,const char *>' duration="{duration}"] ##teamcity[testStarted name='tuple'] ##teamcity[testFinished name='tuple' duration="{duration}"] ##teamcity[testStarted name='tuple,tuple<>,float>'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 1305af19..49b11484 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -3,25 +3,6 @@ - - - - fptr == 0 - - - 0 == 0 - - - - - fptr == 0l - - - 0 == 0 - - - - @@ -799,7 +780,7 @@ Nor would this - + f() == 0 @@ -810,7 +791,7 @@ Nor would this - errno == 1 + errno_after == 1 1 == 1 @@ -20267,19 +20248,6 @@ loose text artifact - - - - "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) - - - "{ 0, 42, "Catch me" }" -== -"{ 0, 42, "Catch me" }" - - - - @@ -20895,6 +20863,6 @@ loose text artifact - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index f03c7986..f73fe121 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -3,25 +3,6 @@ - - - - fptr == 0 - - - 0 == 0 - - - - - fptr == 0l - - - 0 == 0 - - - - @@ -799,7 +780,7 @@ Nor would this - + f() == 0 @@ -810,7 +791,7 @@ Nor would this - errno == 1 + errno_after == 1 1 == 1 @@ -20266,19 +20247,6 @@ There is no extra whitespace here - - - - "{ 0, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) - - - "{ 0, 42, "Catch me" }" -== -"{ 0, 42, "Catch me" }" - - - - @@ -20894,6 +20862,6 @@ There is no extra whitespace here - - + + diff --git a/tests/SelfTest/UsageTests/Decomposition.tests.cpp b/tests/SelfTest/UsageTests/Decomposition.tests.cpp index 15feefc2..1ad5b796 100644 --- a/tests/SelfTest/UsageTests/Decomposition.tests.cpp +++ b/tests/SelfTest/UsageTests/Decomposition.tests.cpp @@ -29,7 +29,7 @@ TEST_CASE( "Reconstruction should be based on stringification: #914" , "[Decompo CHECK(truthy(false)); } -TEST_CASE("#1005: Comparing pointer to int and long (NULL can be either on various systems)", "[Decomposition]") { +TEST_CASE("#1005: Comparing pointer to int and long (NULL can be either on various systems)", "[Decomposition][approvals]") { FILE* fptr = nullptr; REQUIRE(fptr == 0); REQUIRE(fptr == 0l); diff --git a/tests/SelfTest/UsageTests/Misc.tests.cpp b/tests/SelfTest/UsageTests/Misc.tests.cpp index e5409772..f1f7db03 100644 --- a/tests/SelfTest/UsageTests/Misc.tests.cpp +++ b/tests/SelfTest/UsageTests/Misc.tests.cpp @@ -472,10 +472,15 @@ TEST_CASE( "# A test name that starts with a #" ) { SUCCEED( "yay" ); } -TEST_CASE( "#835 -- errno should not be touched by Catch", "[.][failing][!shouldfail]" ) { +TEST_CASE( "#835 -- errno should not be touched by Catch2", "[.][failing][!shouldfail]" ) { errno = 1; + // Check that reporting failed test doesn't change errno. CHECK(f() == 0); - REQUIRE(errno == 1); // Check that f() doesn't touch errno. + // We want to avoid expanding `errno` macro in assertion, because + // we capture the expression after macro expansion, and would have + // to normalize the ways different platforms spell `errno`. + const auto errno_after = errno; + REQUIRE(errno_after == 1); } TEST_CASE( "#961 -- Dynamically created sections should all be reported", "[.]" ) { diff --git a/tests/SelfTest/UsageTests/ToStringTuple.tests.cpp b/tests/SelfTest/UsageTests/ToStringTuple.tests.cpp index 1d1a7ab0..3912285b 100644 --- a/tests/SelfTest/UsageTests/ToStringTuple.tests.cpp +++ b/tests/SelfTest/UsageTests/ToStringTuple.tests.cpp @@ -38,8 +38,7 @@ TEST_CASE( "tuple,tuple<>,float>", "[toString][tuple]" ) CHECK( "{ { 42 }, { }, 1.2f }" == ::Catch::Detail::stringify(value) ); } -TEST_CASE( "tuple", "[toString][tuple]" ) -{ +TEST_CASE( "tuple", "[approvals][toString][tuple]" ) { typedef std::tuple type; type value { nullptr, 42, "Catch me" }; CHECK( "{ nullptr, 42, \"Catch me\" }" == ::Catch::Detail::stringify(value) ); diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py index 59b00d01..36bbdd2f 100755 --- a/tools/scripts/approvalTests.py +++ b/tools/scripts/approvalTests.py @@ -28,7 +28,6 @@ filelocParser = re.compile(r''' ''', re.VERBOSE) lineNumberParser = re.compile(r' line="[0-9]*"') hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b') -durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"') # Note: junit must serialize time with 3 (or or less) decimal places # before generalizing this parser, make sure that this is checked # in other places too. @@ -39,40 +38,14 @@ versionParser = re.compile(r'Catch2 v[0-9]+\.[0-9]+\.[0-9]+(-\w*\.[0-9]+)?') nullParser = re.compile(r'\b(__null|nullptr)\b') exeNameParser = re.compile(r''' \b - (CatchSelfTest|SelfTest) # Expected executable name + SelfTest # Expected executable name (?:.exe)? # Executable name contains .exe on Windows. \b ''', re.VERBOSE) # This is a hack until something more reasonable is figured out specialCaseParser = re.compile(r'file\((\d+)\)') -# errno macro expands into various names depending on platform, so we need to fix them up as well -errnoParser = re.compile(r''' - \(\*__errno_location\s*\(\)\) - | - \(\*__error\(\)\) - | - \(\*_errno\(\)\) -''', re.VERBOSE) sinceEpochParser = re.compile(r'\d+ .+ since epoch') -infParser = re.compile(r''' - \(\(float\)\(1e\+300\ \*\ 1e\+300\)\) # MSVC INFINITY macro - | - \(__builtin_inff\(\)\) # Linux (ubuntu) INFINITY macro - | - \(__builtin_inff\ \(\)\) # Fedora INFINITY macro - | - __builtin_huge_valf\(\) # OSX macro -''', re.VERBOSE) -nanParser = re.compile(r''' - \(\(float\)\(\(\(float\)\(1e\+300\ \*\ 1e\+300\)\)\ \*\ 0\.0F\)\) # MSVC NAN macro - | - \(\(float\)\(INFINITY\ \*\ 0\.0F\)\) # Yet another MSVC NAN macro - | - \(__builtin_nanf\ \(""\)\) # Linux (ubuntu) NAN macro - | - __builtin_nanf\("0x"\) # The weird content of the brackets is there because a different parser has already ran before this one -''', re.VERBOSE) # The weird OR is there to always have at least empty string for group 1 tapTestNumParser = re.compile(r'^((?:not ok)|(?:ok)|(?:warning)|(?:info)) (\d+) -') @@ -157,10 +130,7 @@ def filterLine(line, isCompact): line = durationParser.sub(' duration="{duration}"', line) line = timestampsParser.sub('{iso8601-timestamp}', line) line = specialCaseParser.sub('file:\g<1>', line) - line = errnoParser.sub('errno', line) line = sinceEpochParser.sub('{since-epoch-report}', line) - line = infParser.sub('INFINITY', line) - line = nanParser.sub('NAN', line) return line