3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2024-11-24 14:26:10 +01:00
Code Issues Packages Projects Releases Wiki Activity
0e805512e5
catch2/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
Martin Hořeňovský e340ab8db6
Various improvements to the benchmarking support 
* Units from <ratio> are no longer redeclared in our own namespace
* The default clock is `steady_clock`, not `high_resolution_clock`,
because, as HH says "high_resolution_clock is useless. If you want
measure the passing of time, use steady_clock. If you want user
friendly time, use system_clock".
* Benchmarking support is opt-in, not opt-out, to avoid the large
(~10%) compile time penalty.
* Benchmarking-related options in CLI are always present, to decrease
the amount of code that is only compiled conditionally and making
the whole shebang more maintainble.
2019-06-06 21:28:56 +02:00

492 lines
17 KiB
C++
Raw Blame History

/*
* Created by Phil on 13/5/2013.
* Copyright 2014 Two Blue Cubes Ltd. All rights reserved.
*
* Distributed under the Boost Software License, Version 1.0. (See accompanying
* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*/
#include "catch.hpp"
#include "internal/catch_test_spec_parser.h"
#include "internal/catch_config.hpp"
#include "internal/catch_commandline.h"
#ifdef __clang__
# pragma clang diagnostic ignored "-Wc++98-compat"
#endif
inline Catch::TestCase fakeTestCase(const char* name, const char* desc = "") { return Catch::makeTestCase(nullptr, "", { name, desc }, CATCH_INTERNAL_LINEINFO); }
TEST_CASE( "Parse test names and tags" ) {
using Catch::parseTestSpec;
using Catch::TestSpec;
Catch::TestCase tcA = fakeTestCase( "a" );
Catch::TestCase tcB = fakeTestCase( "b", "[one][x]" );
Catch::TestCase tcC = fakeTestCase( "longer name with spaces", "[two][three][.][x]" );
Catch::TestCase tcD = fakeTestCase( "zlonger name with spacesz" );
SECTION( "Empty test spec should have no filters" ) {
TestSpec spec;
CHECK( spec.hasFilters() == false );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
}
SECTION( "Test spec from empty string should have no filters" ) {
TestSpec spec = parseTestSpec( "" );
CHECK( spec.hasFilters() == false );
CHECK( spec.matches(tcA ) == false );
CHECK( spec.matches( tcB ) == false );
}
SECTION( "Test spec from just a comma should have no filters" ) {
TestSpec spec = parseTestSpec( "," );
CHECK( spec.hasFilters() == false );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
}
SECTION( "Test spec from name should have one filter" ) {
TestSpec spec = parseTestSpec( "b" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == true );
}
SECTION( "Test spec from quoted name should have one filter" ) {
TestSpec spec = parseTestSpec( "\"b\"" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == true );
}
SECTION( "Test spec from name should have one filter" ) {
TestSpec spec = parseTestSpec( "b" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == false );
}
SECTION( "Wildcard at the start" ) {
TestSpec spec = parseTestSpec( "*spaces" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == false );
CHECK( parseTestSpec( "*a" ).matches( tcA ) == true );
}
SECTION( "Wildcard at the end" ) {
TestSpec spec = parseTestSpec( "long*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == false );
CHECK( parseTestSpec( "a*" ).matches( tcA ) == true );
}
SECTION( "Wildcard at both ends" ) {
TestSpec spec = parseTestSpec( "*name*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == true );
CHECK( parseTestSpec( "*a*" ).matches( tcA ) == true );
}
SECTION( "Redundant wildcard at the start" ) {
TestSpec spec = parseTestSpec( "*a" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
}
SECTION( "Redundant wildcard at the end" ) {
TestSpec spec = parseTestSpec( "a*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
}
SECTION( "Redundant wildcard at both ends" ) {
TestSpec spec = parseTestSpec( "*a*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
}
SECTION( "Wildcard at both ends, redundant at start" ) {
TestSpec spec = parseTestSpec( "*longer*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == true );
}
SECTION( "Just wildcard" ) {
TestSpec spec = parseTestSpec( "*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == true );
}
SECTION( "Single tag" ) {
TestSpec spec = parseTestSpec( "[one]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == false );
}
SECTION( "Single tag, two matches" ) {
TestSpec spec = parseTestSpec( "[x]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == true );
}
SECTION( "Two tags" ) {
TestSpec spec = parseTestSpec( "[two][x]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
}
SECTION( "Two tags, spare separated" ) {
TestSpec spec = parseTestSpec( "[two] [x]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
}
SECTION( "Wildcarded name and tag" ) {
TestSpec spec = parseTestSpec( "*name*[x]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == false );
}
SECTION( "Single tag exclusion" ) {
TestSpec spec = parseTestSpec( "~[one]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
}
SECTION( "One tag exclusion and one tag inclusion" ) {
TestSpec spec = parseTestSpec( "~[two][x]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == false );
}
SECTION( "One tag exclusion and one wldcarded name inclusion" ) {
TestSpec spec = parseTestSpec( "~[two]*name*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == true );
}
SECTION( "One tag exclusion, using exclude:, and one wldcarded name inclusion" ) {
TestSpec spec = parseTestSpec( "exclude:[two]*name*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == true );
}
SECTION( "name exclusion" ) {
TestSpec spec = parseTestSpec( "~b" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == true );
}
SECTION( "wildcarded name exclusion" ) {
TestSpec spec = parseTestSpec( "~*name*" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == false );
}
SECTION( "wildcarded name exclusion with tag inclusion" ) {
TestSpec spec = parseTestSpec( "~*name*,[three]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == false );
}
SECTION( "wildcarded name exclusion, using exclude:, with tag inclusion" ) {
TestSpec spec = parseTestSpec( "exclude:*name*,[three]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == true );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == false );
}
SECTION( "two wildcarded names" ) {
TestSpec spec = parseTestSpec( "\"longer*\"\"*spaces\"" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcD ) == false );
}
SECTION( "empty tag" ) {
TestSpec spec = parseTestSpec( "[]" );
CHECK( spec.hasFilters() == false );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == false );
}
SECTION( "empty quoted name" ) {
TestSpec spec = parseTestSpec( "\"\"" );
CHECK( spec.hasFilters() == false );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == false );
}
SECTION( "quoted string followed by tag exclusion" ) {
TestSpec spec = parseTestSpec( "\"*name*\"~[.]" );
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == false );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == true );
}
}
TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) {
#ifndef CATCH_CONFIG_DISABLE_MATCHERS
using namespace Catch::Matchers;
#endif
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser(config);
SECTION("empty args don't cause a crash") {
auto result = cli.parse({""});
CHECK(result);
CHECK(config.processName == "");
}
SECTION("default - no arguments") {
auto result = cli.parse({"test"});
CHECK(result);
CHECK(config.processName == "test");
CHECK(config.shouldDebugBreak == false);
CHECK(config.abortAfter == -1);
CHECK(config.noThrow == false);
CHECK(config.reporterName == "console");
Catch::Config cfg(config);
CHECK_FALSE(cfg.hasTestFilters());
}
SECTION("test lists") {
SECTION("Specify one test case using") {
auto result = cli.parse({"test", "test1"});
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")));
}
SECTION("Specify one test case exclusion using exclude:") {
auto result = cli.parse({"test", "exclude:test1"});
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
}
SECTION("Specify one test case exclusion using ~") {
auto result = cli.parse({"test", "~test1"});
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
}
}
SECTION("reporter") {
SECTION("-r/console") {
CHECK(cli.parse({"test", "-r", "console"}));
REQUIRE(config.reporterName == "console");
}
SECTION("-r/xml") {
CHECK(cli.parse({"test", "-r", "xml"}));
REQUIRE(config.reporterName == "xml");
}
SECTION("--reporter/junit") {
CHECK(cli.parse({"test", "--reporter", "junit"}));
REQUIRE(config.reporterName == "junit");
}
SECTION("Only one reporter is accepted") {
REQUIRE_FALSE(cli.parse({ "test", "-r", "xml", "-r", "junit" }));
}
SECTION("must match one of the available ones") {
auto result = cli.parse({"test", "--reporter", "unsupported"});
CHECK(!result);
#ifndef CATCH_CONFIG_DISABLE_MATCHERS
REQUIRE_THAT(result.errorMessage(), Contains("Unrecognized reporter"));
#endif
}
}
SECTION("debugger") {
SECTION("-b") {
CHECK(cli.parse({"test", "-b"}));
REQUIRE(config.shouldDebugBreak == true);
}
SECTION("--break") {
CHECK(cli.parse({"test", "--break"}));
REQUIRE(config.shouldDebugBreak);
}
}
SECTION("abort") {
SECTION("-a aborts after first failure") {
CHECK(cli.parse({"test", "-a"}));
REQUIRE(config.abortAfter == 1);
}
SECTION("-x 2 aborts after two failures") {
CHECK(cli.parse({"test", "-x", "2"}));
REQUIRE(config.abortAfter == 2);
}
SECTION("-x must be numeric") {
auto result = cli.parse({"test", "-x", "oops"});
CHECK(!result);
#ifndef CATCH_CONFIG_DISABLE_MATCHERS
REQUIRE_THAT(result.errorMessage(), Contains("convert") && Contains("oops"));
#endif
}
}
SECTION("nothrow") {
SECTION("-e") {
CHECK(cli.parse({"test", "-e"}));
REQUIRE(config.noThrow);
}
SECTION("--nothrow") {
CHECK(cli.parse({"test", "--nothrow"}));
REQUIRE(config.noThrow);
}
}
SECTION("output filename") {
SECTION("-o filename") {
CHECK(cli.parse({"test", "-o", "filename.ext"}));
REQUIRE(config.outputFilename == "filename.ext");
}
SECTION("--out") {
CHECK(cli.parse({"test", "--out", "filename.ext"}));
REQUIRE(config.outputFilename == "filename.ext");
}
}
SECTION("combinations") {
SECTION("Single character flags can be combined") {
CHECK(cli.parse({"test", "-abe"}));
CHECK(config.abortAfter == 1);
CHECK(config.shouldDebugBreak);
CHECK(config.noThrow == true);
}
}
SECTION( "use-colour") {
using Catch::UseColour;
SECTION( "without option" ) {
CHECK(cli.parse({"test"}));
REQUIRE( config.useColour == UseColour::Auto );
}
SECTION( "auto" ) {
CHECK(cli.parse({"test", "--use-colour", "auto"}));
REQUIRE( config.useColour == UseColour::Auto );
}
SECTION( "yes" ) {
CHECK(cli.parse({"test", "--use-colour", "yes"}));
REQUIRE( config.useColour == UseColour::Yes );
}
SECTION( "no" ) {
CHECK(cli.parse({"test", "--use-colour", "no"}));
REQUIRE( config.useColour == UseColour::No );
}
SECTION( "error" ) {
auto result = cli.parse({"test", "--use-colour", "wrong"});
CHECK( !result );
#ifndef CATCH_CONFIG_DISABLE_MATCHERS
CHECK_THAT( result.errorMessage(), Contains( "colour mode must be one of" ) );
#endif
}
}
SECTION("Benchmark options") {
SECTION("samples") {
CHECK(cli.parse({ "test", "--benchmark-samples=200" }));
REQUIRE(config.benchmarkSamples == 200);
}
SECTION("resamples") {
CHECK(cli.parse({ "test", "--benchmark-resamples=20000" }));
REQUIRE(config.benchmarkResamples == 20000);
}
SECTION("resamples") {
CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
REQUIRE(config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99));
}
SECTION("resamples") {
CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
REQUIRE(config.benchmarkNoAnalysis);
}
}
}
Reference in New Issue View Git Blame Copy Permalink