From d084162b2f41a319adb3d988f15a3aa3f127b936 Mon Sep 17 00:00:00 2001 From: Lotte Steenbrink Date: Mon, 7 Jan 2019 16:40:23 +0100 Subject: [PATCH 1/2] Avoid superfluous re-run when specific section is selected Fixes #1394 --- examples/300-FilteredSection.cpp | 35 ++++++++++++++++++++ examples/CMakeLists.txt | 1 + examples/runFilteredSection.sh | 8 +++++ include/internal/catch_test_case_tracker.cpp | 11 ++++++ include/internal/catch_test_case_tracker.h | 2 ++ 5 files changed, 57 insertions(+) create mode 100644 examples/300-FilteredSection.cpp create mode 100755 examples/runFilteredSection.sh diff --git a/examples/300-FilteredSection.cpp b/examples/300-FilteredSection.cpp new file mode 100644 index 00000000..7057ed10 --- /dev/null +++ b/examples/300-FilteredSection.cpp @@ -0,0 +1,35 @@ +// Provide a basis to check whether section selection via the `-c` command line +// option works as intended. + +#define CATCH_CONFIG_MAIN +#include + +bool previouslyRun = false; +bool previouslyRunNested = false; + +TEST_CASE( "#1394" ) { + // -- Don't re-run after specified section is done + REQUIRE(previouslyRun == false); + + SECTION( "RunSection" ) { + previouslyRun = true; + } + SECTION( "SkipSection" ) { + // cause an error if this section is called because it shouldn't be + REQUIRE(1 == 0); + } +} + +TEST_CASE( "#1394 nested" ) { + REQUIRE(previouslyRunNested == false); + + SECTION( "NestedRunSection" ) { + SECTION( "s1" ) { + previouslyRunNested = true; + } + } + SECTION( "NestedSkipSection" ) { + // cause an error if this section is called because it shouldn't be + REQUIRE(1 == 0); + } +} diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2551e77c..c889b087 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -21,6 +21,7 @@ set( REPORTER_HEADER_DIR ${CATCH_DIR}/include/reporters ) set( SOURCES_SINGLE_FILE 010-TestCase.cpp 231-Cfg-OutputStreams.cpp + 300-FilteredSection.cpp ) # multiple-file modules: diff --git a/examples/runFilteredSection.sh b/examples/runFilteredSection.sh new file mode 100755 index 00000000..2bd88392 --- /dev/null +++ b/examples/runFilteredSection.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# make sure single-file header was regenerated +../scripts/generateSingleHeader.py + +# run Test case with specified section to trigger bug #1394 +../cmake-build-debug/examples/300-FilteredSection \#1394 -c RunSection +../cmake-build-debug/examples/300-FilteredSection \#1394\ nested -c NestedRunSection -c s1 diff --git a/include/internal/catch_test_case_tracker.cpp b/include/internal/catch_test_case_tracker.cpp index 7839a71b..8c188457 100644 --- a/include/internal/catch_test_case_tracker.cpp +++ b/include/internal/catch_test_case_tracker.cpp @@ -190,6 +190,17 @@ namespace TestCaseTracking { } } + bool SectionTracker::isComplete() const { + bool complete = true; + + if ((m_filters.empty() || m_filters[0] == "") || + std::find(m_filters.begin(), m_filters.end(), + m_nameAndLocation.name) != m_filters.end()) + complete = TrackerBase::isComplete(); + return complete; + + } + bool SectionTracker::isSectionTracker() const { return true; } SectionTracker& SectionTracker::acquire( TrackerContext& ctx, NameAndLocation const& nameAndLocation ) { diff --git a/include/internal/catch_test_case_tracker.h b/include/internal/catch_test_case_tracker.h index 72b506a9..485118ed 100644 --- a/include/internal/catch_test_case_tracker.h +++ b/include/internal/catch_test_case_tracker.h @@ -140,6 +140,8 @@ namespace TestCaseTracking { bool isSectionTracker() const override; + bool isComplete() const override; + static SectionTracker& acquire( TrackerContext& ctx, NameAndLocation const& nameAndLocation ); void tryOpen(); From 8989c9b560f14a4ddbadbf564499d732bed8b264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Tue, 15 Jan 2019 23:13:16 +0100 Subject: [PATCH 2/2] Integrate tests for #1394 into our test suite --- examples/300-FilteredSection.cpp | 35 ----------------- examples/CMakeLists.txt | 1 - examples/runFilteredSection.sh | 8 ---- misc/coverage-helper.cpp | 39 +++++++++++++++++-- projects/CMakeLists.txt | 6 +++ .../IntrospectiveTests/PartTracker.tests.cpp | 30 ++++++++++++++ 6 files changed, 72 insertions(+), 47 deletions(-) delete mode 100644 examples/300-FilteredSection.cpp delete mode 100755 examples/runFilteredSection.sh diff --git a/examples/300-FilteredSection.cpp b/examples/300-FilteredSection.cpp deleted file mode 100644 index 7057ed10..00000000 --- a/examples/300-FilteredSection.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Provide a basis to check whether section selection via the `-c` command line -// option works as intended. - -#define CATCH_CONFIG_MAIN -#include - -bool previouslyRun = false; -bool previouslyRunNested = false; - -TEST_CASE( "#1394" ) { - // -- Don't re-run after specified section is done - REQUIRE(previouslyRun == false); - - SECTION( "RunSection" ) { - previouslyRun = true; - } - SECTION( "SkipSection" ) { - // cause an error if this section is called because it shouldn't be - REQUIRE(1 == 0); - } -} - -TEST_CASE( "#1394 nested" ) { - REQUIRE(previouslyRunNested == false); - - SECTION( "NestedRunSection" ) { - SECTION( "s1" ) { - previouslyRunNested = true; - } - } - SECTION( "NestedSkipSection" ) { - // cause an error if this section is called because it shouldn't be - REQUIRE(1 == 0); - } -} diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index c889b087..2551e77c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -21,7 +21,6 @@ set( REPORTER_HEADER_DIR ${CATCH_DIR}/include/reporters ) set( SOURCES_SINGLE_FILE 010-TestCase.cpp 231-Cfg-OutputStreams.cpp - 300-FilteredSection.cpp ) # multiple-file modules: diff --git a/examples/runFilteredSection.sh b/examples/runFilteredSection.sh deleted file mode 100755 index 2bd88392..00000000 --- a/examples/runFilteredSection.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# make sure single-file header was regenerated -../scripts/generateSingleHeader.py - -# run Test case with specified section to trigger bug #1394 -../cmake-build-debug/examples/300-FilteredSection \#1394 -c RunSection -../cmake-build-debug/examples/300-FilteredSection \#1394\ nested -c NestedRunSection -c s1 diff --git a/misc/coverage-helper.cpp b/misc/coverage-helper.cpp index 17218905..a6643838 100644 --- a/misc/coverage-helper.cpp +++ b/misc/coverage-helper.cpp @@ -9,6 +9,38 @@ #include #include +std::string escape_arg(const std::string& arg) { + if (arg.empty() == false && + arg.find_first_of(" \t\n\v\"") == arg.npos) { + return arg; + } + + std::string escaped; + escaped.push_back('"'); + for (auto it = arg.begin(); ; ++it) { + int num_backslashes = 0; + + while (it != arg.end() && *it == '\\') { + ++it; + ++num_backslashes; + } + + if (it == arg.end()) { + escaped.append(num_backslashes * 2, '\\'); + break; + } else if (*it == '"') { + escaped.append(num_backslashes * 2 + 1, '\\'); + escaped.push_back(*it); + } else { + escaped.append(num_backslashes, '\\'); + escaped.push_back(*it); + } + } + escaped.push_back('"'); + + return escaped; +} + void create_empty_file(std::string const& path) { std::ofstream ofs(path); @@ -60,8 +92,9 @@ std::string windowsify_path(std::string path) { void exec_cmd(std::string const& cmd, int log_num, std::string const& path) { std::array buffer; #if defined(_WIN32) + // cmd has already been escaped outside this function. auto real_cmd = "OpenCppCoverage --export_type binary:cov-report" + std::to_string(log_num) - + ".bin --quiet " + "--sources " + path + " --cover_children -- " + cmd; + + ".bin --quiet " + "--sources " + escape_arg(path) + " --cover_children -- " + cmd; std::cout << "=== Marker ===: Cmd: " << real_cmd << '\n'; std::shared_ptr pipe(_popen(real_cmd.c_str(), "r"), _pclose); #else // Just for testing, in the real world we will always work under WIN32 @@ -91,9 +124,9 @@ int main(int argc, char** argv) { assert(sep - begin(args) == 2 && "Structure differs from expected!"); auto num = parse_log_file_arg(args[1]); - + auto cmdline = std::accumulate(++sep, end(args), std::string{}, [] (const std::string& lhs, const std::string& rhs) { - return lhs + ' ' + rhs; + return lhs + ' ' + escape_arg(rhs); }); try { diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt index fbd3fd2e..62fa7167 100644 --- a/projects/CMakeLists.txt +++ b/projects/CMakeLists.txt @@ -345,10 +345,16 @@ set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No asserti add_test(NAME NoTest COMMAND $ -w NoTests "___nonexistent_test___") set_tests_properties(NoTest PROPERTIES PASS_REGULAR_EXPRESSION "No test cases matched") +add_test(NAME FilteredSection-1 COMMAND $ \#1394 -c RunSection) +set_tests_properties(FilteredSection-1 PROPERTIES FAIL_REGULAR_EXPRESSION "No tests ran") +add_test(NAME FilteredSection-2 COMMAND $ \#1394\ nested -c NestedRunSection -c s1) +set_tests_properties(FilteredSection-2 PROPERTIES FAIL_REGULAR_EXPRESSION "No tests ran") + # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable add_test(NAME ApprovalTests COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/scripts/approvalTests.py $) set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed") + if (CATCH_USE_VALGRIND) add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $) add_test(NAME ValgrindListTests COMMAND valgrind --leak-check=full --error-exitcode=1 $ --list-tests --verbosity high) diff --git a/projects/SelfTest/IntrospectiveTests/PartTracker.tests.cpp b/projects/SelfTest/IntrospectiveTests/PartTracker.tests.cpp index 7e7f14b4..12dcca91 100644 --- a/projects/SelfTest/IntrospectiveTests/PartTracker.tests.cpp +++ b/projects/SelfTest/IntrospectiveTests/PartTracker.tests.cpp @@ -324,3 +324,33 @@ TEST_CASE( "Tracker" ) { // two sections within a generator } } + +static bool previouslyRun = false; +static bool previouslyRunNested = false; + +TEST_CASE( "#1394", "[.][approvals][tracker]" ) { + // -- Don't re-run after specified section is done + REQUIRE(previouslyRun == false); + + SECTION( "RunSection" ) { + previouslyRun = true; + } + SECTION( "SkipSection" ) { + // cause an error if this section is called because it shouldn't be + REQUIRE(1 == 0); + } +} + +TEST_CASE( "#1394 nested", "[.][approvals][tracker]" ) { + REQUIRE(previouslyRunNested == false); + + SECTION( "NestedRunSection" ) { + SECTION( "s1" ) { + previouslyRunNested = true; + } + } + SECTION( "NestedSkipSection" ) { + // cause an error if this section is called because it shouldn't be + REQUIRE(1 == 0); + } +}