Add more tests for test spec parser

Originally the tests were from #1912, but as it turned out, the issue was somewhere else. Still, the inputs provided were interesting, so they are now part of our test suite.
2025-11-01 04:37:10 +01:00 · 2020-04-17 21:16:18 +02:00
parent 38f897c887
commit 37cbf4a4fe
8 changed files with 144 additions and 9 deletions
--- a/projects/SelfTest/Baselines/compact.sw.approved.txt
+++ b/projects/SelfTest/Baselines/compact.sw.approved.txt
@@ -26,6 +26,10 @@ Compilation.tests.cpp:<line number>: passed: std::is_same<TypeList<int>, TypeLis
 CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase("spec . char")) for: true
 CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase("spec , char")) for: true
 CmdLine.tests.cpp:<line number>: passed: !(spec.matches(fakeTestCase(R"(spec \, char)"))) for: !false
+CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase(R"(spec {a} char)")) for: true
+CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase(R"(spec [a] char)")) for: true
+CmdLine.tests.cpp:<line number>: passed: !(spec.matches(fakeTestCase("differs but has similar tag", "[a]"))) for: !false
+CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase(R"(spec \ char)")) for: true
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'
--- a/projects/SelfTest/Baselines/console.std.approved.txt
+++ b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -1380,6 +1380,6 @@ due to unexpected exception with message:
  Why would you throw a std::string?

 ===============================================================================
-test cases:  308 |  234 passed |  70 failed |  4 failed as expected
-assertions: 1688 | 1536 passed | 131 failed | 21 failed as expected
+test cases:  309 |  235 passed |  70 failed |  4 failed as expected
+assertions: 1692 | 1540 passed | 131 failed | 21 failed as expected

--- a/projects/SelfTest/Baselines/console.sw.approved.txt
+++ b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -209,6 +209,40 @@ CmdLine.tests.cpp:<line number>: PASSED:
 with expansion:
  !false

+-------------------------------------------------------------------------------
+#1912 -- test spec parser handles escaping
+  Various parentheses
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:<line number>
+...............................................................................
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE( spec.matches(fakeTestCase(R"(spec {a} char)")) )
+with expansion:
+  true
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE( spec.matches(fakeTestCase(R"(spec [a] char)")) )
+with expansion:
+  true
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE_FALSE( spec.matches(fakeTestCase("differs but has similar tag", "[a]")) )
+with expansion:
+  !false
+
+-------------------------------------------------------------------------------
+#1912 -- test spec parser handles escaping
+  backslash in test name
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:<line number>
+...............................................................................
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE( spec.matches(fakeTestCase(R"(spec \ char)")) )
+with expansion:
+  true
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
  outside assertions
@@ -13488,6 +13522,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:

 ===============================================================================
-test cases:  308 |  218 passed |  86 failed |  4 failed as expected
-assertions: 1705 | 1536 passed | 148 failed | 21 failed as expected
+test cases:  309 |  219 passed |  86 failed |  4 failed as expected
+assertions: 1709 | 1540 passed | 148 failed | 21 failed as expected

--- a/projects/SelfTest/Baselines/console.swa4.approved.txt
+++ b/projects/SelfTest/Baselines/console.swa4.approved.txt
@@ -209,6 +209,40 @@ CmdLine.tests.cpp:<line number>: PASSED:
 with expansion:
  !false

+-------------------------------------------------------------------------------
+#1912 -- test spec parser handles escaping
+  Various parentheses
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:<line number>
+...............................................................................
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE( spec.matches(fakeTestCase(R"(spec {a} char)")) )
+with expansion:
+  true
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE( spec.matches(fakeTestCase(R"(spec [a] char)")) )
+with expansion:
+  true
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE_FALSE( spec.matches(fakeTestCase("differs but has similar tag", "[a]")) )
+with expansion:
+  !false
+
+-------------------------------------------------------------------------------
+#1912 -- test spec parser handles escaping
+  backslash in test name
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:<line number>
+...............................................................................
+
+CmdLine.tests.cpp:<line number>: PASSED:
+  REQUIRE( spec.matches(fakeTestCase(R"(spec \ char)")) )
+with expansion:
+  true
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
  outside assertions
@@ -389,6 +423,6 @@ Condition.tests.cpp:<line number>: FAILED:
  CHECK( true != true )

 ===============================================================================
-test cases: 20 | 15 passed | 3 failed | 2 failed as expected
-assertions: 45 | 38 passed | 4 failed | 3 failed as expected
+test cases: 21 | 16 passed | 3 failed | 2 failed as expected
+assertions: 49 | 42 passed | 4 failed | 3 failed as expected

--- a/projects/SelfTest/Baselines/junit.sw.approved.txt
+++ b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="132" tests="1706" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="132" tests="1710" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
    <properties>
      <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
      <property name="random-seed" value="1"/>
@@ -31,6 +31,8 @@ Nor would this
    </testcase>
    <testcase classname="<exe-name>.global" name="#1548" time="{duration}"/>
    <testcase classname="<exe-name>.global" name="#1905 -- test spec parser properly clears internal state between compound tests" time="{duration}"/>
+    <testcase classname="<exe-name>.global" name="#1912 -- test spec parser handles escaping/Various parentheses" time="{duration}"/>
+    <testcase classname="<exe-name>.global" name="#1912 -- test spec parser handles escaping/backslash in test name" time="{duration}"/>
    <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
      <error type="TEST_CASE">
 FAILED:
--- a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt
+++ b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -3,6 +3,8 @@
 >
  <file path="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp">
    <testCase name="#1905 -- test spec parser properly clears internal state between compound tests" duration="{duration}"/>
+    <testCase name="#1912 -- test spec parser handles escaping/Various parentheses" duration="{duration}"/>
+    <testCase name="#1912 -- test spec parser handles escaping/backslash in test name" duration="{duration}"/>
    <testCase name="Parse test names and tags/Empty test spec should have no filters" duration="{duration}"/>
    <testCase name="Parse test names and tags/Test spec from empty string should have no filters" duration="{duration}"/>
    <testCase name="Parse test names and tags/Test spec from just a comma should have no filters" duration="{duration}"/>
--- a/projects/SelfTest/Baselines/xml.sw.approved.txt
+++ b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -223,6 +223,47 @@ Nor would this
      </Expression>
      <OverallResult success="true"/>
    </TestCase>
+    <TestCase name="#1912 -- test spec parser handles escaping" tags="[command-line][test-spec]" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+      <Section name="Various parentheses" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+        <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+          <Original>
+            spec.matches(fakeTestCase(R"(spec {a} char)"))
+          </Original>
+          <Expanded>
+            true
+          </Expanded>
+        </Expression>
+        <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+          <Original>
+            spec.matches(fakeTestCase(R"(spec [a] char)"))
+          </Original>
+          <Expanded>
+            true
+          </Expanded>
+        </Expression>
+        <Expression success="true" type="REQUIRE_FALSE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+          <Original>
+            !(spec.matches(fakeTestCase("differs but has similar tag", "[a]")))
+          </Original>
+          <Expanded>
+            !false
+          </Expanded>
+        </Expression>
+        <OverallResults successes="3" failures="0" expectedFailures="0"/>
+      </Section>
+      <Section name="backslash in test name" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+        <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+          <Original>
+            spec.matches(fakeTestCase(R"(spec \ char)"))
+          </Original>
+          <Expanded>
+            true
+          </Expanded>
+        </Expression>
+        <OverallResults successes="1" failures="0" expectedFailures="0"/>
+      </Section>
+      <OverallResult success="true"/>
+    </TestCase>
    <TestCase name="#748 - captures with unexpected exceptions" tags="[!hide][!shouldfail][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
      <Section name="outside assertions" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
        <Info>
@@ -16139,7 +16180,7 @@ loose text artifact
      </Section>
      <OverallResult success="true"/>
    </TestCase>
-    <OverallResults successes="1536" failures="149" expectedFailures="21"/>
+    <OverallResults successes="1540" failures="149" expectedFailures="21"/>
  </Group>
-  <OverallResults successes="1536" failures="148" expectedFailures="21"/>
+  <OverallResults successes="1540" failures="148" expectedFailures="21"/>
 </Catch>
--- a/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
+++ b/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
@@ -303,6 +303,24 @@ TEST_CASE("#1905 -- test spec parser properly clears internal state between comp
    REQUIRE_FALSE(spec.matches(fakeTestCase(R"(spec \, char)")));
 }

+TEST_CASE("#1912 -- test spec parser handles escaping", "[command-line][test-spec]") {
+    using Catch::parseTestSpec;
+    using Catch::TestSpec;
+
+    SECTION("Various parentheses") {
+        TestSpec spec = parseTestSpec(R"(spec {a} char,spec \[a] char)");
+
+        REQUIRE(spec.matches(fakeTestCase(R"(spec {a} char)")));
+        REQUIRE(spec.matches(fakeTestCase(R"(spec [a] char)")));
+        REQUIRE_FALSE(spec.matches(fakeTestCase("differs but has similar tag", "[a]")));
+    }
+    SECTION("backslash in test name") {
+        TestSpec spec = parseTestSpec(R"(spec \\ char)");
+
+        REQUIRE(spec.matches(fakeTestCase(R"(spec \ char)")));
+    }
+}
+
 TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) {

 #ifndef CATCH_CONFIG_DISABLE_MATCHERS