mirror of
https://github.com/catchorg/Catch2.git
synced 2024-11-07 22:59:54 +01:00
Removed benchmark tests from approval tests (for now)
This commit is contained in:
parent
f45d35c980
commit
4421672fb8
@ -31,6 +31,8 @@ namespace Catch {
|
||||
return TestCaseInfo::MayFail;
|
||||
else if( tag == "!nonportable" )
|
||||
return TestCaseInfo::NonPortable;
|
||||
else if( tag == "!benchmark" )
|
||||
return static_cast<TestCaseInfo::SpecialProperties>( TestCaseInfo::Benchmark | TestCaseInfo::IsHidden );
|
||||
else
|
||||
return TestCaseInfo::None;
|
||||
}
|
||||
|
@ -30,7 +30,8 @@ namespace Catch {
|
||||
ShouldFail = 1 << 2,
|
||||
MayFail = 1 << 3,
|
||||
Throws = 1 << 4,
|
||||
NonPortable = 1 << 5
|
||||
NonPortable = 1 << 5,
|
||||
Benchmark = 1 << 6
|
||||
};
|
||||
|
||||
TestCaseInfo( std::string const& _name,
|
||||
|
@ -850,22 +850,6 @@ warning:
|
||||
Uncomment the code in this test to check that it gives a sensible compiler
|
||||
error
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
benchmarked
|
||||
-------------------------------------------------------------------------------
|
||||
String.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
+-----------------------------------------------------------------------------+
|
||||
| benchmark name | iters | elapsed ns | average |
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| Load up a vector | 1 | 8459 | 8459 ns |
|
||||
| Load up a map | 1 | 28774 | 28774 ns |
|
||||
| Reserved vector | 1 | 2430 | 2430 ns |
|
||||
| A fixed size array that should require | | | |
|
||||
| no allocations | 10 | 2563 | 256 ns |
|
||||
+-----------------------------------------------------------------------------+
|
||||
-------------------------------------------------------------------------------
|
||||
checkedElse, failing
|
||||
-------------------------------------------------------------------------------
|
||||
@ -1032,6 +1016,6 @@ with expansion:
|
||||
"{?}" == "1"
|
||||
|
||||
===============================================================================
|
||||
test cases: 183 | 132 passed | 47 failed | 4 failed as expected
|
||||
assertions: 900 | 783 passed | 96 failed | 21 failed as expected
|
||||
test cases: 182 | 131 passed | 47 failed | 4 failed as expected
|
||||
assertions: 896 | 779 passed | 96 failed | 21 failed as expected
|
||||
|
||||
|
@ -6381,67 +6381,6 @@ PASSED:
|
||||
with expansion:
|
||||
0 == 0
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
benchmarked
|
||||
-------------------------------------------------------------------------------
|
||||
String.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
+-----------------------------------------------------------------------------+
|
||||
| benchmark name | iters | elapsed ns | average |
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| Load up a vector | 1 | 6957 | 6957 ns |
|
||||
+-----------------------------------------------------------------------------+
|
||||
String.tests.cpp:<line number>:
|
||||
PASSED:
|
||||
REQUIRE( v.size() == size )
|
||||
with expansion:
|
||||
100 == 100
|
||||
|
||||
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| benchmark name | iters | elapsed ns | average |
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| Load up a map | 1 | 27868 | 27868 ns |
|
||||
+-----------------------------------------------------------------------------+
|
||||
String.tests.cpp:<line number>:
|
||||
PASSED:
|
||||
REQUIRE( m.size() == size )
|
||||
with expansion:
|
||||
100 == 100
|
||||
|
||||
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| benchmark name | iters | elapsed ns | average |
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| Reserved vector | 1 | 2711 | 2711 ns |
|
||||
+-----------------------------------------------------------------------------+
|
||||
String.tests.cpp:<line number>:
|
||||
PASSED:
|
||||
REQUIRE( v.size() == size )
|
||||
with expansion:
|
||||
100 == 100
|
||||
|
||||
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| benchmark name | iters | elapsed ns | average |
|
||||
+-----------------------------------------------------------------------------+
|
||||
|
||||
| A fixed size array that should require | | | |
|
||||
| no allocations | 1 | 530 | 530 ns |
|
||||
+-----------------------------------------------------------------------------+
|
||||
String.tests.cpp:<line number>:
|
||||
PASSED:
|
||||
REQUIRE( sum > size )
|
||||
with expansion:
|
||||
4950 (0x<hex digits>) > 100
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
boolean member
|
||||
-------------------------------------------------------------------------------
|
||||
@ -7651,6 +7590,6 @@ MiscTests.cpp:<line number>:
|
||||
PASSED:
|
||||
|
||||
===============================================================================
|
||||
test cases: 183 | 131 passed | 48 failed | 4 failed as expected
|
||||
assertions: 902 | 783 passed | 98 failed | 21 failed as expected
|
||||
test cases: 182 | 130 passed | 48 failed | 4 failed as expected
|
||||
assertions: 898 | 779 passed | 98 failed | 21 failed as expected
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuitesspanner>
|
||||
<testsuite name="<exe-name>" errors="15" failures="84" tests="903" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
|
||||
<testsuite name="<exe-name>" errors="15" failures="84" tests="899" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
|
||||
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
|
||||
<error type="TEST_CASE">
|
||||
@ -641,7 +641,6 @@ ExceptionTests.cpp:<line number>
|
||||
<testcase classname="<exe-name>.global" name="XmlEncode/string with control char (1)" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="XmlEncode/string with control char (x7F)" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="atomic if" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="benchmarked" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="boolean member" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="checkedElse" time="{duration}"/>
|
||||
<testcase classname="<exe-name>.global" name="checkedElse, failing" time="{duration}">
|
||||
|
@ -7201,41 +7201,6 @@ Message from section two
|
||||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="benchmarked" tags="[benchmark]" filename="projects/<exe-name>/String.tests.cpp" >
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
|
||||
<Original>
|
||||
v.size() == size
|
||||
</Original>
|
||||
<Expanded>
|
||||
100 == 100
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
|
||||
<Original>
|
||||
m.size() == size
|
||||
</Original>
|
||||
<Expanded>
|
||||
100 == 100
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
|
||||
<Original>
|
||||
v.size() == size
|
||||
</Original>
|
||||
<Expanded>
|
||||
100 == 100
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
|
||||
<Original>
|
||||
sum > size
|
||||
</Original>
|
||||
<Expanded>
|
||||
4950 (0x<hex digits>) > 100
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<TestCase name="boolean member" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" >
|
||||
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" >
|
||||
<Original>
|
||||
@ -8476,7 +8441,7 @@ spanner <OverallResult success="true"/>
|
||||
</Section>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<OverallResults successes="783" failures="99" expectedFailures="21"/>
|
||||
<OverallResults successes="779" failures="99" expectedFailures="21"/>
|
||||
</Group>
|
||||
<OverallResults successes="783" failures="98" expectedFailures="21"/>
|
||||
<OverallResults successes="779" failures="98" expectedFailures="21"/>
|
||||
</Catch>
|
||||
|
@ -2,7 +2,7 @@
|
||||
|
||||
#include <map>
|
||||
|
||||
TEST_CASE( "benchmarked", "[.][benchmark]" ) {
|
||||
TEST_CASE( "benchmarked", "[!benchmark]" ) {
|
||||
|
||||
static const int size = 100;
|
||||
|
||||
|
@ -145,15 +145,15 @@ print("Running approvals against executable:")
|
||||
print(" " + cmdPath)
|
||||
|
||||
# Standard console reporter
|
||||
approve("console.std", ["~[!nonportable]", "--order", "lex"])
|
||||
approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"])
|
||||
# console reporter, include passes, warn about No Assertions
|
||||
approve("console.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"])
|
||||
approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"])
|
||||
# console reporter, include passes, warn about No Assertions, limit failures to first 4
|
||||
approve("console.swa4", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
|
||||
approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
|
||||
# junit reporter, include passes, warn about No Assertions
|
||||
approve("junit.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
|
||||
approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
|
||||
# xml reporter, include passes, warn about No Assertions
|
||||
approve("xml.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
|
||||
approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
|
||||
|
||||
if overallResult != 0:
|
||||
print("If these differences are expected, run approve.py to approve new baselines.")
|
||||
|
Loading…
Reference in New Issue
Block a user