From 4421672fb869318d3445a80deb5de09761f8a119 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Sat, 5 Aug 2017 11:26:20 +0100 Subject: [PATCH] Removed benchmark tests from approval tests (for now) --- include/internal/catch_test_case_info.cpp | 2 + include/internal/catch_test_case_info.h | 3 +- .../Baselines/console.std.approved.txt | 20 +----- .../Baselines/console.sw.approved.txt | 65 +------------------ .../SelfTest/Baselines/junit.sw.approved.txt | 3 +- .../SelfTest/Baselines/xml.sw.approved.txt | 39 +---------- projects/SelfTest/Benchmark.tests.cpp | 2 +- scripts/approvalTests.py | 10 +-- 8 files changed, 17 insertions(+), 127 deletions(-) diff --git a/include/internal/catch_test_case_info.cpp b/include/internal/catch_test_case_info.cpp index 5a8d8c0f..9c707307 100644 --- a/include/internal/catch_test_case_info.cpp +++ b/include/internal/catch_test_case_info.cpp @@ -31,6 +31,8 @@ namespace Catch { return TestCaseInfo::MayFail; else if( tag == "!nonportable" ) return TestCaseInfo::NonPortable; + else if( tag == "!benchmark" ) + return static_cast( TestCaseInfo::Benchmark | TestCaseInfo::IsHidden ); else return TestCaseInfo::None; } diff --git a/include/internal/catch_test_case_info.h b/include/internal/catch_test_case_info.h index 626f95cb..2a911b06 100644 --- a/include/internal/catch_test_case_info.h +++ b/include/internal/catch_test_case_info.h @@ -30,7 +30,8 @@ namespace Catch { ShouldFail = 1 << 2, MayFail = 1 << 3, Throws = 1 << 4, - NonPortable = 1 << 5 + NonPortable = 1 << 5, + Benchmark = 1 << 6 }; TestCaseInfo( std::string const& _name, diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index acd58b94..9e41f7f5 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -850,22 +850,6 @@ warning: Uncomment the code in this test to check that it gives a sensible compiler error -------------------------------------------------------------------------------- -benchmarked -------------------------------------------------------------------------------- -String.tests.cpp: -............................................................................... - -+-----------------------------------------------------------------------------+ -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Load up a vector | 1 | 8459 | 8459 ns | -| Load up a map | 1 | 28774 | 28774 ns | -| Reserved vector | 1 | 2430 | 2430 ns | -| A fixed size array that should require | | | | -| no allocations | 10 | 2563 | 256 ns | -+-----------------------------------------------------------------------------+ ------------------------------------------------------------------------------- checkedElse, failing ------------------------------------------------------------------------------- @@ -1032,6 +1016,6 @@ with expansion: "{?}" == "1" =============================================================================== -test cases: 183 | 132 passed | 47 failed | 4 failed as expected -assertions: 900 | 783 passed | 96 failed | 21 failed as expected +test cases: 182 | 131 passed | 47 failed | 4 failed as expected +assertions: 896 | 779 passed | 96 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 7f4861d4..971c0e32 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -6381,67 +6381,6 @@ PASSED: with expansion: 0 == 0 -------------------------------------------------------------------------------- -benchmarked -------------------------------------------------------------------------------- -String.tests.cpp: -............................................................................... - -+-----------------------------------------------------------------------------+ -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Load up a vector | 1 | 6957 | 6957 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:: -PASSED: - REQUIRE( v.size() == size ) -with expansion: - 100 == 100 - - -+-----------------------------------------------------------------------------+ - -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Load up a map | 1 | 27868 | 27868 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:: -PASSED: - REQUIRE( m.size() == size ) -with expansion: - 100 == 100 - - -+-----------------------------------------------------------------------------+ - -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Reserved vector | 1 | 2711 | 2711 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:: -PASSED: - REQUIRE( v.size() == size ) -with expansion: - 100 == 100 - - -+-----------------------------------------------------------------------------+ - -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| A fixed size array that should require | | | | -| no allocations | 1 | 530 | 530 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:: -PASSED: - REQUIRE( sum > size ) -with expansion: - 4950 (0x) > 100 - ------------------------------------------------------------------------------- boolean member ------------------------------------------------------------------------------- @@ -7651,6 +7590,6 @@ MiscTests.cpp:: PASSED: =============================================================================== -test cases: 183 | 131 passed | 48 failed | 4 failed as expected -assertions: 902 | 783 passed | 98 failed | 21 failed as expected +test cases: 182 | 130 passed | 48 failed | 4 failed as expected +assertions: 898 | 779 passed | 98 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 31ad3c5a..58807c18 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,6 +1,6 @@ - + @@ -641,7 +641,6 @@ ExceptionTests.cpp: - diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 01ec013c..e70ad12e 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -7201,41 +7201,6 @@ Message from section two - - - - v.size() == size - - - 100 == 100 - - - - - m.size() == size - - - 100 == 100 - - - - - v.size() == size - - - 100 == 100 - - - - - sum > size - - - 4950 (0x) > 100 - - - - @@ -8476,7 +8441,7 @@ spanner - + - + diff --git a/projects/SelfTest/Benchmark.tests.cpp b/projects/SelfTest/Benchmark.tests.cpp index ff1ec479..ddf69504 100644 --- a/projects/SelfTest/Benchmark.tests.cpp +++ b/projects/SelfTest/Benchmark.tests.cpp @@ -2,7 +2,7 @@ #include -TEST_CASE( "benchmarked", "[.][benchmark]" ) { +TEST_CASE( "benchmarked", "[!benchmark]" ) { static const int size = 100; diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index 016997cc..49553349 100755 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -145,15 +145,15 @@ print("Running approvals against executable:") print(" " + cmdPath) # Standard console reporter -approve("console.std", ["~[!nonportable]", "--order", "lex"]) +approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"]) # console reporter, include passes, warn about No Assertions -approve("console.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"]) +approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"]) # console reporter, include passes, warn about No Assertions, limit failures to first 4 -approve("console.swa4", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) +approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) # junit reporter, include passes, warn about No Assertions -approve("junit.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) +approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) # xml reporter, include passes, warn about No Assertions -approve("xml.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) +approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) if overallResult != 0: print("If these differences are expected, run approve.py to approve new baselines.")