Don't include numerically unstable tests in approvals

This commit is contained in:
Martin Hořeňovský 2024-04-30 19:21:27 +02:00
parent ef713582d2
commit a654e4b038
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
18 changed files with 19 additions and 255 deletions

View File

@ -354,7 +354,6 @@ b1!
:test-result: FAIL nested sections can be skipped dynamically at runtime
:test-result: PASS non streamable - with conv. op
:test-result: PASS non-copyable objects
:test-result: PASS normal_cdf
:test-result: PASS normal_quantile
:test-result: PASS not allowed
:test-result: FAIL not prints unscoped info from previous failures

View File

@ -343,7 +343,6 @@
:test-result: FAIL nested sections can be skipped dynamically at runtime
:test-result: PASS non streamable - with conv. op
:test-result: PASS non-copyable objects
:test-result: PASS normal_cdf
:test-result: PASS normal_quantile
:test-result: PASS not allowed
:test-result: FAIL not prints unscoped info from previous failures

View File

@ -2582,19 +2582,6 @@ Skip.tests.cpp:<line number>: skipped:
!
Tricky.tests.cpp:<line number>: passed: s == "7" for: "7" == "7"
Tricky.tests.cpp:<line number>: passed: ti == typeid(int) for: {?} == {?}
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293
==
Approx( 0.84134474606854293 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707
==
Approx( 0.15865525393145705 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409
==
Approx( 0.99752083845315409 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035
==
Approx( 0.08809652095066035 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195
==
Approx( 0.13015979861484198 )
@ -2853,7 +2840,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected
test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected

View File

@ -2571,19 +2571,6 @@ Misc.tests.cpp:<line number>: passed: a != b for: 1 != 2
Skip.tests.cpp:<line number>: skipped:
Tricky.tests.cpp:<line number>: passed: s == "7" for: "7" == "7"
Tricky.tests.cpp:<line number>: passed: ti == typeid(int) for: {?} == {?}
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293
==
Approx( 0.84134474606854293 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707
==
Approx( 0.15865525393145705 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409
==
Approx( 0.99752083845315409 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035
==
Approx( 0.08809652095066035 )
InternalBenchmark.tests.cpp:<line number>: passed: normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195
==
Approx( 0.13015979861484198 )
@ -2842,7 +2829,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected
test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected

View File

@ -1598,6 +1598,6 @@ due to unexpected exception with message:
Why would you throw a std::string?
===============================================================================
test cases: 417 | 326 passed | 70 failed | 7 skipped | 14 failed as expected
assertions: 2243 | 2079 passed | 129 failed | 35 failed as expected
test cases: 416 | 325 passed | 70 failed | 7 skipped | 14 failed as expected
assertions: 2238 | 2074 passed | 129 failed | 35 failed as expected

View File

@ -17075,45 +17075,6 @@ Tricky.tests.cpp:<line number>: PASSED:
with expansion:
{?} == {?}
-------------------------------------------------------------------------------
normal_cdf
-------------------------------------------------------------------------------
InternalBenchmark.tests.cpp:<line number>
...............................................................................
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(0.000000) == Approx(0.50000000000000000) )
with expansion:
0.5 == Approx( 0.5 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(1.000000) == Approx(0.84134474606854293) )
with expansion:
0.84134474606854293
==
Approx( 0.84134474606854293 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(-1.000000) == Approx(0.15865525393145705) )
with expansion:
0.15865525393145707
==
Approx( 0.15865525393145705 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(2.809729) == Approx(0.99752083845315409) )
with expansion:
0.99752083845315409
==
Approx( 0.99752083845315409 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(-1.352570) == Approx(0.08809652095066035) )
with expansion:
0.08809652095066035
==
Approx( 0.08809652095066035 )
-------------------------------------------------------------------------------
normal_quantile
-------------------------------------------------------------------------------
@ -18933,6 +18894,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected
test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected

View File

@ -17064,45 +17064,6 @@ Tricky.tests.cpp:<line number>: PASSED:
with expansion:
{?} == {?}
-------------------------------------------------------------------------------
normal_cdf
-------------------------------------------------------------------------------
InternalBenchmark.tests.cpp:<line number>
...............................................................................
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(0.000000) == Approx(0.50000000000000000) )
with expansion:
0.5 == Approx( 0.5 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(1.000000) == Approx(0.84134474606854293) )
with expansion:
0.84134474606854293
==
Approx( 0.84134474606854293 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(-1.000000) == Approx(0.15865525393145705) )
with expansion:
0.15865525393145707
==
Approx( 0.15865525393145705 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(2.809729) == Approx(0.99752083845315409) )
with expansion:
0.99752083845315409
==
Approx( 0.99752083845315409 )
InternalBenchmark.tests.cpp:<line number>: PASSED:
CHECK( normal_cdf(-1.352570) == Approx(0.08809652095066035) )
with expansion:
0.08809652095066035
==
Approx( 0.08809652095066035 )
-------------------------------------------------------------------------------
normal_quantile
-------------------------------------------------------------------------------
@ -18922,6 +18883,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 417 | 312 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2260 | 2079 passed | 146 failed | 35 failed as expected
test cases: 416 | 311 passed | 85 failed | 6 skipped | 14 failed as expected
assertions: 2255 | 2074 passed | 146 failed | 35 failed as expected

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="129" skipped="12" tests="2272" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="129" skipped="12" tests="2267" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@ -1932,7 +1932,6 @@ b1!
</testcase>
<testcase classname="<exe-name>.global" name="non streamable - with conv. op" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="non-copyable objects" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="normal_cdf" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="normal_quantile" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="not allowed" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="not prints unscoped info from previous failures" time="{duration}" status="run">

View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="<exe-name>" errors="17" failures="129" skipped="12" tests="2272" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="129" skipped="12" tests="2267" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@ -1931,7 +1931,6 @@ b1!
</testcase>
<testcase classname="<exe-name>.global" name="non streamable - with conv. op" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="non-copyable objects" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="normal_cdf" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="normal_quantile" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="not allowed" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="not prints unscoped info from previous failures" time="{duration}" status="run">

View File

@ -121,7 +121,6 @@ at AssertionHandler.tests.cpp:<line number>
<testCase name="estimate_clock_resolution" duration="{duration}"/>
<testCase name="mean" duration="{duration}"/>
<testCase name="measure" duration="{duration}"/>
<testCase name="normal_cdf" duration="{duration}"/>
<testCase name="normal_quantile" duration="{duration}"/>
<testCase name="resolution" duration="{duration}"/>
<testCase name="run_for_at_least, chronometer" duration="{duration}"/>

View File

@ -120,7 +120,6 @@ at AssertionHandler.tests.cpp:<line number>
<testCase name="estimate_clock_resolution" duration="{duration}"/>
<testCase name="mean" duration="{duration}"/>
<testCase name="measure" duration="{duration}"/>
<testCase name="normal_cdf" duration="{duration}"/>
<testCase name="normal_quantile" duration="{duration}"/>
<testCase name="resolution" duration="{duration}"/>
<testCase name="run_for_at_least, chronometer" duration="{duration}"/>

View File

@ -4107,16 +4107,6 @@ ok {test-number} - # SKIP
ok {test-number} - s == "7" for: "7" == "7"
# non-copyable objects
ok {test-number} - ti == typeid(int) for: {?} == {?}
# normal_cdf
ok {test-number} - normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 )
# normal_cdf
ok {test-number} - normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293 == Approx( 0.84134474606854293 )
# normal_cdf
ok {test-number} - normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707 == Approx( 0.15865525393145705 )
# normal_cdf
ok {test-number} - normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409 == Approx( 0.99752083845315409 )
# normal_cdf
ok {test-number} - normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035 == Approx( 0.08809652095066035 )
# normal_quantile
ok {test-number} - normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195 == Approx( 0.13015979861484198 )
# normal_quantile
@ -4549,5 +4539,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2272
1..2267

View File

@ -4096,16 +4096,6 @@ ok {test-number} - # SKIP
ok {test-number} - s == "7" for: "7" == "7"
# non-copyable objects
ok {test-number} - ti == typeid(int) for: {?} == {?}
# normal_cdf
ok {test-number} - normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 )
# normal_cdf
ok {test-number} - normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.84134474606854293 == Approx( 0.84134474606854293 )
# normal_cdf
ok {test-number} - normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.15865525393145707 == Approx( 0.15865525393145705 )
# normal_cdf
ok {test-number} - normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.99752083845315409 == Approx( 0.99752083845315409 )
# normal_cdf
ok {test-number} - normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.08809652095066035 == Approx( 0.08809652095066035 )
# normal_quantile
ok {test-number} - normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.13015979861484195 == Approx( 0.13015979861484198 )
# normal_quantile
@ -4538,5 +4528,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2272
1..2267

View File

@ -861,8 +861,6 @@ loose text artifact
##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"]
##teamcity[testStarted name='non-copyable objects']
##teamcity[testFinished name='non-copyable objects' duration="{duration}"]
##teamcity[testStarted name='normal_cdf']
##teamcity[testFinished name='normal_cdf' duration="{duration}"]
##teamcity[testStarted name='normal_quantile']
##teamcity[testFinished name='normal_quantile' duration="{duration}"]
##teamcity[testStarted name='not allowed']

View File

@ -860,8 +860,6 @@
##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"]
##teamcity[testStarted name='non-copyable objects']
##teamcity[testFinished name='non-copyable objects' duration="{duration}"]
##teamcity[testStarted name='normal_cdf']
##teamcity[testFinished name='normal_cdf' duration="{duration}"]
##teamcity[testStarted name='normal_quantile']
##teamcity[testFinished name='normal_quantile' duration="{duration}"]
##teamcity[testStarted name='not allowed']

View File

@ -19733,57 +19733,6 @@ b1!
</Expression>
<OverallResult success="true" skips="0"/>
</TestCase>
<TestCase name="normal_cdf" tags="[benchmark]" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(0.000000) == Approx(0.50000000000000000)
</Original>
<Expanded>
0.5 == Approx( 0.5 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(1.000000) == Approx(0.84134474606854293)
</Original>
<Expanded>
0.84134474606854293
==
Approx( 0.84134474606854293 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(-1.000000) == Approx(0.15865525393145705)
</Original>
<Expanded>
0.15865525393145707
==
Approx( 0.15865525393145705 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(2.809729) == Approx(0.99752083845315409)
</Original>
<Expanded>
0.99752083845315409
==
Approx( 0.99752083845315409 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(-1.352570) == Approx(0.08809652095066035)
</Original>
<Expanded>
0.08809652095066035
==
Approx( 0.08809652095066035 )
</Expanded>
</Expression>
<OverallResult success="true" skips="0"/>
</TestCase>
<TestCase name="normal_quantile" tags="[benchmark]" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
@ -21883,6 +21832,6 @@ Approx( -1.95996398454005449 )
</Section>
<OverallResult success="true" skips="0"/>
</TestCase>
<OverallResults successes="2079" failures="146" expectedFailures="35" skips="12"/>
<OverallResultsCases successes="312" failures="85" expectedFailures="14" skips="6"/>
<OverallResults successes="2074" failures="146" expectedFailures="35" skips="12"/>
<OverallResultsCases successes="311" failures="85" expectedFailures="14" skips="6"/>
</Catch2TestRun>

View File

@ -19732,57 +19732,6 @@ b1!
</Expression>
<OverallResult success="true" skips="0"/>
</TestCase>
<TestCase name="normal_cdf" tags="[benchmark]" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(0.000000) == Approx(0.50000000000000000)
</Original>
<Expanded>
0.5 == Approx( 0.5 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(1.000000) == Approx(0.84134474606854293)
</Original>
<Expanded>
0.84134474606854293
==
Approx( 0.84134474606854293 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(-1.000000) == Approx(0.15865525393145705)
</Original>
<Expanded>
0.15865525393145707
==
Approx( 0.15865525393145705 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(2.809729) == Approx(0.99752083845315409)
</Original>
<Expanded>
0.99752083845315409
==
Approx( 0.99752083845315409 )
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
normal_cdf(-1.352570) == Approx(0.08809652095066035)
</Original>
<Expanded>
0.08809652095066035
==
Approx( 0.08809652095066035 )
</Expanded>
</Expression>
<OverallResult success="true" skips="0"/>
</TestCase>
<TestCase name="normal_quantile" tags="[benchmark]" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/InternalBenchmark.tests.cpp" >
<Original>
@ -21882,6 +21831,6 @@ Approx( -1.95996398454005449 )
</Section>
<OverallResult success="true" skips="0"/>
</TestCase>
<OverallResults successes="2079" failures="146" expectedFailures="35" skips="12"/>
<OverallResultsCases successes="312" failures="85" expectedFailures="14" skips="6"/>
<OverallResults successes="2074" failures="146" expectedFailures="35" skips="12"/>
<OverallResultsCases successes="311" failures="85" expectedFailures="14" skips="6"/>
</Catch2TestRun>

View File

@ -172,7 +172,7 @@ TEST_CASE("uniform samples", "[benchmark]") {
}
TEST_CASE("normal_cdf", "[benchmark]") {
TEST_CASE("normal_cdf", "[benchmark][approvals]") {
using Catch::Benchmark::Detail::normal_cdf;
using Catch::Approx;
CHECK(normal_cdf(0.000000) == Approx(0.50000000000000000));