Removed benchmark tests from approval tests (for now)

include/internal/catch_test_case_info.cpp

@@ -31,6 +31,8 @@ namespace Catch {
             return TestCaseInfo::MayFail;
         else if( tag == "!nonportable" )
             return TestCaseInfo::NonPortable;
+        else if( tag == "!benchmark" )
+            return static_cast<TestCaseInfo::SpecialProperties>( TestCaseInfo::Benchmark | TestCaseInfo::IsHidden );
         else
             return TestCaseInfo::None;
     }

include/internal/catch_test_case_info.h

@@ -30,7 +30,8 @@ namespace Catch {
             ShouldFail = 1 << 2,
             MayFail = 1 << 3,
             Throws = 1 << 4,
-            NonPortable = 1 << 5
+            NonPortable = 1 << 5,
+            Benchmark = 1 << 6
         };
 
         TestCaseInfo(   std::string const& _name,

projects/SelfTest/Baselines/console.std.approved.txt

@@ -850,22 +850,6 @@ warning:
   Uncomment the code in this test to check that it gives a sensible compiler
   error
 
--------------------------------------------------------------------------------
-benchmarked
--------------------------------------------------------------------------------
-String.tests.cpp:<line number>
-...............................................................................
-
-+-----------------------------------------------------------------------------+
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Load up a vector                         |      1 |       8459 |    8459 ns |
-| Load up a map                            |      1 |      28774 |   28774 ns |
-| Reserved vector                          |      1 |       2430 |    2430 ns |
-| A fixed size array that should require   |        |            |            |
-| no allocations                           |     10 |       2563 |     256 ns |
-+-----------------------------------------------------------------------------+
 -------------------------------------------------------------------------------
 checkedElse, failing
 -------------------------------------------------------------------------------
@@ -1032,6 +1016,6 @@ with expansion:
   "{?}" == "1"
 
 ===============================================================================
-test cases: 183 | 132 passed | 47 failed |  4 failed as expected
+test cases: 182 | 131 passed | 47 failed |  4 failed as expected
-assertions: 900 | 783 passed | 96 failed | 21 failed as expected
+assertions: 896 | 779 passed | 96 failed | 21 failed as expected
 

projects/SelfTest/Baselines/console.sw.approved.txt

@@ -6381,67 +6381,6 @@ PASSED:
 with expansion:
   0 == 0
 
--------------------------------------------------------------------------------
-benchmarked
--------------------------------------------------------------------------------
-String.tests.cpp:<line number>
-...............................................................................
-
-+-----------------------------------------------------------------------------+
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Load up a vector                         |      1 |       6957 |    6957 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( v.size() == size )
-with expansion:
-  100 == 100
-
-
-+-----------------------------------------------------------------------------+
-
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Load up a map                            |      1 |      27868 |   27868 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( m.size() == size )
-with expansion:
-  100 == 100
-
-
-+-----------------------------------------------------------------------------+
-
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Reserved vector                          |      1 |       2711 |    2711 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( v.size() == size )
-with expansion:
-  100 == 100
-
-
-+-----------------------------------------------------------------------------+
-
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| A fixed size array that should require   |        |            |            |
-| no allocations                           |      1 |        530 |     530 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( sum > size )
-with expansion:
-  4950 (0x<hex digits>) > 100
-
 -------------------------------------------------------------------------------
 boolean member
 -------------------------------------------------------------------------------
@@ -7651,6 +7590,6 @@ MiscTests.cpp:<line number>:
 PASSED:
 
 ===============================================================================
-test cases: 183 | 131 passed | 48 failed |  4 failed as expected
+test cases: 182 | 130 passed | 48 failed |  4 failed as expected
-assertions: 902 | 783 passed | 98 failed | 21 failed as expected
+assertions: 898 | 779 passed | 98 failed | 21 failed as expected
 

projects/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesspanner>
-  <testsuite name="<exe-name>" errors="15" failures="84" tests="903" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="15" failures="84" tests="899" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
       <error type="TEST_CASE">
@@ -641,7 +641,6 @@ ExceptionTests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (1)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (x7F)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="atomic if" time="{duration}"/>
-    <testcase classname="<exe-name>.global" name="benchmarked" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="boolean member" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="checkedElse" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="checkedElse, failing" time="{duration}">

projects/SelfTest/Baselines/xml.sw.approved.txt

@@ -7201,41 +7201,6 @@ Message from section two
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="benchmarked" tags="[benchmark]" filename="projects/<exe-name>/String.tests.cpp" >
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          v.size() == size
-        </Original>
-        <Expanded>
-          100 == 100
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          m.size() == size
-        </Original>
-        <Expanded>
-          100 == 100
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          v.size() == size
-        </Original>
-        <Expanded>
-          100 == 100
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          sum > size
-        </Original>
-        <Expanded>
-          4950 (0x<hex digits>) > 100
-        </Expanded>
-      </Expression>
-      <OverallResult success="true"/>
-    </TestCase>
     <TestCase name="boolean member" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" >
       <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" >
         <Original>
@@ -8476,7 +8441,7 @@ spanner      <OverallResult success="true"/>
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="783" failures="99" expectedFailures="21"/>
+    <OverallResults successes="779" failures="99" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="783" failures="98" expectedFailures="21"/>
+  <OverallResults successes="779" failures="98" expectedFailures="21"/>
 </Catch>

projects/SelfTest/Benchmark.tests.cpp

@@ -2,7 +2,7 @@
 
 #include <map>
 
-TEST_CASE( "benchmarked", "[.][benchmark]" ) {
+TEST_CASE( "benchmarked", "[!benchmark]" ) {
 
     static const int size = 100;
 

scripts/approvalTests.py

@@ -145,15 +145,15 @@ print("Running approvals against executable:")
 print("  " + cmdPath)
 
 # Standard console reporter
-approve("console.std", ["~[!nonportable]", "--order", "lex"])
+approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions
-approve("console.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"])
+approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve("console.swa4", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
+approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
 # junit reporter, include passes, warn about No Assertions
-approve("junit.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
+approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
 # xml reporter, include passes, warn about No Assertions
-approve("xml.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
+approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
 
 if overallResult != 0:
     print("If these differences are expected, run approve.py to approve new baselines.")