From a654e4b0382f3c41bdb911739b05276a11cf644a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Tue, 30 Apr 2024 19:21:27 +0200 Subject: [PATCH] Don't include numerically unstable tests in approvals --- .../Baselines/automake.sw.approved.txt | 1 - .../Baselines/automake.sw.multi.approved.txt | 1 - .../Baselines/compact.sw.approved.txt | 17 +----- .../Baselines/compact.sw.multi.approved.txt | 17 +----- .../Baselines/console.std.approved.txt | 4 +- .../Baselines/console.sw.approved.txt | 43 +-------------- .../Baselines/console.sw.multi.approved.txt | 43 +-------------- .../SelfTest/Baselines/junit.sw.approved.txt | 3 +- .../Baselines/junit.sw.multi.approved.txt | 3 +- .../Baselines/sonarqube.sw.approved.txt | 1 - .../Baselines/sonarqube.sw.multi.approved.txt | 1 - tests/SelfTest/Baselines/tap.sw.approved.txt | 12 +--- .../Baselines/tap.sw.multi.approved.txt | 12 +--- .../Baselines/teamcity.sw.approved.txt | 2 - .../Baselines/teamcity.sw.multi.approved.txt | 2 - tests/SelfTest/Baselines/xml.sw.approved.txt | 55 +------------------ .../Baselines/xml.sw.multi.approved.txt | 55 +------------------ .../InternalBenchmark.tests.cpp | 2 +- 18 files changed, 19 insertions(+), 255 deletions(-) diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 88c23e17..dbdf395c 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -354,7 +354,6 @@ b1! :test-result: FAIL nested sections can be skipped dynamically at runtime :test-result: PASS non streamable - with conv. op :test-result: PASS non-copyable objects -:test-result: PASS normal_cdf :test-result: PASS normal_quantile :test-result: PASS not allowed :test-result: FAIL not prints unscoped info from previous failures diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index a37b1a2b..4029ec65 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -343,7 +343,6 @@ :test-result: FAIL nested sections can be skipped dynamically at runtime :test-result: PASS non streamable - with conv. op :test-result: PASS non-copyable objects -:test-result: PASS normal_cdf :test-result: PASS normal_quantile :test-result: PASS not allowed :test-result: FAIL not prints unscoped info from previous failures diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 51b659eb..9363cb54 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -2582,19 +2582,6 @@ Skip.tests.cpp:: skipped: ! Tricky.tests.cpp:: passed: s == "7" for: "7" == "7" Tricky.tests.cpp:: passed: ti == typeid(int) for: {?} == {?} -InternalBenchmark.tests.cpp:: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293 -== -Approx( 0.84134474606854293 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707 -== -Approx( 0.15865525393145705 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409 -== -Approx( 0.99752083845315409 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035 -== -Approx( 0.08809652095066035 ) InternalBenchmark.tests.cpp:: passed: normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195 == Approx( 0.13015979861484198 ) @@ -2853,7 +2840,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected -assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected +test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected +assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 0281c24c..4007dfca 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -2571,19 +2571,6 @@ Misc.tests.cpp:: passed: a != b for: 1 != 2 Skip.tests.cpp:: skipped: Tricky.tests.cpp:: passed: s == "7" for: "7" == "7" Tricky.tests.cpp:: passed: ti == typeid(int) for: {?} == {?} -InternalBenchmark.tests.cpp:: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293 -== -Approx( 0.84134474606854293 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707 -== -Approx( 0.15865525393145705 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409 -== -Approx( 0.99752083845315409 ) -InternalBenchmark.tests.cpp:: passed: normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035 -== -Approx( 0.08809652095066035 ) InternalBenchmark.tests.cpp:: passed: normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195 == Approx( 0.13015979861484198 ) @@ -2842,7 +2829,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected -assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected +test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected +assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index e9dd2e45..9b398435 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1598,6 +1598,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 417 | 326 passed | 70 failed | 7 skipped | 14 failed as expected -assertions: 2243 | 2079 passed | 129 failed | 35 failed as expected +test cases: 416 | 325 passed | 70 failed | 7 skipped | 14 failed as expected +assertions: 2238 | 2074 passed | 129 failed | 35 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 41b11c2a..de89152e 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -17075,45 +17075,6 @@ Tricky.tests.cpp:: PASSED: with expansion: {?} == {?} -------------------------------------------------------------------------------- -normal_cdf -------------------------------------------------------------------------------- -InternalBenchmark.tests.cpp: -............................................................................... - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(0.000000) == Approx(0.50000000000000000) ) -with expansion: - 0.5 == Approx( 0.5 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(1.000000) == Approx(0.84134474606854293) ) -with expansion: - 0.84134474606854293 - == - Approx( 0.84134474606854293 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(-1.000000) == Approx(0.15865525393145705) ) -with expansion: - 0.15865525393145707 - == - Approx( 0.15865525393145705 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(2.809729) == Approx(0.99752083845315409) ) -with expansion: - 0.99752083845315409 - == - Approx( 0.99752083845315409 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(-1.352570) == Approx(0.08809652095066035) ) -with expansion: - 0.08809652095066035 - == - Approx( 0.08809652095066035 ) - ------------------------------------------------------------------------------- normal_quantile ------------------------------------------------------------------------------- @@ -18933,6 +18894,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected -assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected +test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected +assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 8a90805c..f70c0d6f 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -17064,45 +17064,6 @@ Tricky.tests.cpp:: PASSED: with expansion: {?} == {?} -------------------------------------------------------------------------------- -normal_cdf -------------------------------------------------------------------------------- -InternalBenchmark.tests.cpp: -............................................................................... - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(0.000000) == Approx(0.50000000000000000) ) -with expansion: - 0.5 == Approx( 0.5 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(1.000000) == Approx(0.84134474606854293) ) -with expansion: - 0.84134474606854293 - == - Approx( 0.84134474606854293 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(-1.000000) == Approx(0.15865525393145705) ) -with expansion: - 0.15865525393145707 - == - Approx( 0.15865525393145705 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(2.809729) == Approx(0.99752083845315409) ) -with expansion: - 0.99752083845315409 - == - Approx( 0.99752083845315409 ) - -InternalBenchmark.tests.cpp:: PASSED: - CHECK( normal_cdf(-1.352570) == Approx(0.08809652095066035) ) -with expansion: - 0.08809652095066035 - == - Approx( 0.08809652095066035 ) - ------------------------------------------------------------------------------- normal_quantile ------------------------------------------------------------------------------- @@ -18922,6 +18883,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected -assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected +test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected +assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 4f83b9f0..1888139a 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1932,7 +1932,6 @@ b1! - diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index f8938128..b594c8c0 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1931,7 +1931,6 @@ b1! - diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index a64503e3..537145cc 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -121,7 +121,6 @@ at AssertionHandler.tests.cpp: - diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 87e65b99..5965774d 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -120,7 +120,6 @@ at AssertionHandler.tests.cpp: - diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 39923e36..383bd4b0 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -4107,16 +4107,6 @@ ok {test-number} - # SKIP ok {test-number} - s == "7" for: "7" == "7" # non-copyable objects ok {test-number} - ti == typeid(int) for: {?} == {?} -# normal_cdf -ok {test-number} - normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) -# normal_cdf -ok {test-number} - normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293 == Approx( 0.84134474606854293 ) -# normal_cdf -ok {test-number} - normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707 == Approx( 0.15865525393145705 ) -# normal_cdf -ok {test-number} - normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409 == Approx( 0.99752083845315409 ) -# normal_cdf -ok {test-number} - normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035 == Approx( 0.08809652095066035 ) # normal_quantile ok {test-number} - normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195 == Approx( 0.13015979861484198 ) # normal_quantile @@ -4549,5 +4539,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2272 +1..2267 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index a112f551..6622a96a 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -4096,16 +4096,6 @@ ok {test-number} - # SKIP ok {test-number} - s == "7" for: "7" == "7" # non-copyable objects ok {test-number} - ti == typeid(int) for: {?} == {?} -# normal_cdf -ok {test-number} - normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) -# normal_cdf -ok {test-number} - normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293 == Approx( 0.84134474606854293 ) -# normal_cdf -ok {test-number} - normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707 == Approx( 0.15865525393145705 ) -# normal_cdf -ok {test-number} - normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409 == Approx( 0.99752083845315409 ) -# normal_cdf -ok {test-number} - normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035 == Approx( 0.08809652095066035 ) # normal_quantile ok {test-number} - normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195 == Approx( 0.13015979861484198 ) # normal_quantile @@ -4538,5 +4528,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2272 +1..2267 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index 8f645988..cec3a0ee 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -861,8 +861,6 @@ loose text artifact ##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"] ##teamcity[testStarted name='non-copyable objects'] ##teamcity[testFinished name='non-copyable objects' duration="{duration}"] -##teamcity[testStarted name='normal_cdf'] -##teamcity[testFinished name='normal_cdf' duration="{duration}"] ##teamcity[testStarted name='normal_quantile'] ##teamcity[testFinished name='normal_quantile' duration="{duration}"] ##teamcity[testStarted name='not allowed'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index b2bc2700..6f7d8f19 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -860,8 +860,6 @@ ##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"] ##teamcity[testStarted name='non-copyable objects'] ##teamcity[testFinished name='non-copyable objects' duration="{duration}"] -##teamcity[testStarted name='normal_cdf'] -##teamcity[testFinished name='normal_cdf' duration="{duration}"] ##teamcity[testStarted name='normal_quantile'] ##teamcity[testFinished name='normal_quantile' duration="{duration}"] ##teamcity[testStarted name='not allowed'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 0fec02bb..f8acf1d4 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -19733,57 +19733,6 @@ b1! - - - - normal_cdf(0.000000) == Approx(0.50000000000000000) - - - 0.5 == Approx( 0.5 ) - - - - - normal_cdf(1.000000) == Approx(0.84134474606854293) - - - 0.84134474606854293 -== -Approx( 0.84134474606854293 ) - - - - - normal_cdf(-1.000000) == Approx(0.15865525393145705) - - - 0.15865525393145707 -== -Approx( 0.15865525393145705 ) - - - - - normal_cdf(2.809729) == Approx(0.99752083845315409) - - - 0.99752083845315409 -== -Approx( 0.99752083845315409 ) - - - - - normal_cdf(-1.352570) == Approx(0.08809652095066035) - - - 0.08809652095066035 -== -Approx( 0.08809652095066035 ) - - - - @@ -21883,6 +21832,6 @@ Approx( -1.95996398454005449 ) - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 6231968c..5d815a51 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -19732,57 +19732,6 @@ b1! - - - - normal_cdf(0.000000) == Approx(0.50000000000000000) - - - 0.5 == Approx( 0.5 ) - - - - - normal_cdf(1.000000) == Approx(0.84134474606854293) - - - 0.84134474606854293 -== -Approx( 0.84134474606854293 ) - - - - - normal_cdf(-1.000000) == Approx(0.15865525393145705) - - - 0.15865525393145707 -== -Approx( 0.15865525393145705 ) - - - - - normal_cdf(2.809729) == Approx(0.99752083845315409) - - - 0.99752083845315409 -== -Approx( 0.99752083845315409 ) - - - - - normal_cdf(-1.352570) == Approx(0.08809652095066035) - - - 0.08809652095066035 -== -Approx( 0.08809652095066035 ) - - - - @@ -21882,6 +21831,6 @@ Approx( -1.95996398454005449 ) - - + + diff --git a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp index bc8d715b..69251d97 100644 --- a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp @@ -172,7 +172,7 @@ TEST_CASE("uniform samples", "[benchmark]") { } -TEST_CASE("normal_cdf", "[benchmark]") { +TEST_CASE("normal_cdf", "[benchmark][approvals]") { using Catch::Benchmark::Detail::normal_cdf; using Catch::Approx; CHECK(normal_cdf(0.000000) == Approx(0.50000000000000000));