Improve travis.yml

- Added new compilers and OS X images
- Option to run SelfTest under Valgrind
- Merge "Debug" and "Release" configurations into one run
-- This saves apt setup and cmake download step per compiler, 60-90s
- Fix C++14 compilation under Clang 3.8 and up

.travis.yml

@@ -1,180 +1,188 @@
 language: cpp
 sudo: false
 
+common_sources: &all_sources
+  - ubuntu-toolchain-r-test
+  - llvm-toolchain-trusty
+  - llvm-toolchain-trusty-3.9
+  - llvm-toolchain-trusty-4.0
+  - llvm-toolchain-trusty-5.0
+
 matrix:
   include:
 
     # 1/ Linux Clang Builds
     - os: linux
       compiler: clang
-      addons: &clang35
+      addons:
         apt:
-          sources: ['llvm-toolchain-precise-3.5', 'ubuntu-toolchain-r-test']
+          sources: *all_sources
-          packages: ['clang-3.5']
+          packages: ['valgrind', 'clang-3.5']
-      env: COMPILER='clang++-3.5' BUILD_TYPE='Release'
+      env: COMPILER='clang++-3.5' VALGRIND=1
 
     - os: linux
       compiler: clang
-      addons: *clang35
+      addons:
-      env: COMPILER='clang++-3.5' BUILD_TYPE='Debug'
-
-
-    - os: linux
-      compiler: clang
-      addons: &clang36
         apt:
-          sources: ['llvm-toolchain-precise-3.6', 'ubuntu-toolchain-r-test']
+          sources: *all_sources
-          packages: ['clang-3.6']
+          packages: ['valgrind', 'clang-3.6']
-      env: COMPILER='clang++-3.6' BUILD_TYPE='Release'
+      env: COMPILER='clang++-3.6' VALGRIND=1
+
+# Travis's containers do not seem to have Clang 3.7 in apt, no matter what sources I add. 
+#    - os: linux
+#      compiler: clang
+#      addons:
+#        apt:
+#          sources: *all_sources
+#          packages: ['valgrind', 'clang-3.7']
+#      env: COMPILER='clang++-3.7' VALGRIND=1
 
     - os: linux
       compiler: clang
-      addons: *clang36
+      addons:
-      env: COMPILER='clang++-3.6' BUILD_TYPE='Debug'
-
-
-    - os: linux
-      compiler: clang
-      addons: &clang37
         apt:
-          sources: ['llvm-toolchain-precise-3.7', 'ubuntu-toolchain-r-test']
+          sources: *all_sources
-          packages: ['clang-3.7']
+          packages: ['valgrind', 'clang-3.8']
-      env: COMPILER='clang++-3.7' BUILD_TYPE='Release'
+      env: COMPILER='clang++-3.8' VALGRIND=1
 
     - os: linux
       compiler: clang
-      addons: *clang37
+      addons:
-      env: COMPILER='clang++-3.7' BUILD_TYPE='Debug'
-
-
-    - os: linux
-      compiler: clang
-      addons: &clang38
           apt:
-          sources: ['llvm-toolchain-precise-3.8', 'ubuntu-toolchain-r-test']
+              sources: *all_sources
-          packages: ['clang-3.8']
+              packages: ['clang-3.9', 'valgrind']
-      env: COMPILER='clang++-3.8' BUILD_TYPE='Release'
+      env: COMPILER='clang++-3.9' VALGRIND=1
 
     - os: linux
       compiler: clang
-      addons: *clang38
+      addons:
-      env: COMPILER='clang++-3.8' BUILD_TYPE='Debug'
+          apt:
+              sources: *all_sources
+              packages: ['clang-4.0', 'valgrind']
+      env: COMPILER='clang++-4.0' VALGRIND=1
 
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-5.0', 'valgrind']
+      env: COMPILER='clang++-5.0' VALGRIND=1
 
     # 2/ Linux GCC Builds
     - os: linux
       compiler: gcc
-      addons: &gcc48
+      addons:
         apt:
          sources: ['ubuntu-toolchain-r-test']
-         packages: ['g++-4.8']
+         packages: ['valgrind', 'g++-4.8']
-      env: COMPILER='g++-4.8' BUILD_TYPE='Release'
+      env: COMPILER='g++-4.8' VALGRIND=1
 
     - os: linux
       compiler: gcc
-      addons: *gcc48
+      addons:
-      env: COMPILER='g++-4.8' BUILD_TYPE='Debug'
-
-
-    - os: linux
-      compiler: gcc
-      addons: &gcc49
         apt:
-          sources: ['ubuntu-toolchain-r-test']
+          sources: *all_sources
-          packages: ['g++-4.9']
+          packages: ['valgrind', 'g++-4.9']
-      env: COMPILER='g++-4.9' BUILD_TYPE='Release'
+      env: COMPILER='g++-4.9' VALGRIND=1
 
     - os: linux
       compiler: gcc
-      addons: *gcc49
+      addons:
-      env: COMPILER='g++-4.9' BUILD_TYPE='Debug'
-
-
-    - os: linux
-      compiler: gcc
-      addons: &gcc5
         apt:
-          sources: ['ubuntu-toolchain-r-test']
+          sources: *all_sources
-          packages: ['g++-5']
+          packages: ['valgrind', 'g++-5']
-      env: COMPILER='g++-5' BUILD_TYPE='Release'
+      env: COMPILER='g++-5' VALGRIND=1
-
-    - os: linux
-      compiler: gcc
-      addons: *gcc5
-      env: COMPILER='g++-5' BUILD_TYPE='Debug'
-
 
     - os: linux
       compiler: gcc
       addons: &gcc6
         apt:
-          sources: ['ubuntu-toolchain-r-test']
+          sources: *all_sources
-          packages: ['g++-6']
+          packages: ['valgrind', 'g++-6']
-      env: COMPILER='g++-6' BUILD_TYPE='Release'
+      env: COMPILER='g++-6' VALGRIND=1
 
     - os: linux
       compiler: gcc
-      addons: *gcc6
+      addons: &gcc7
-      env: COMPILER='g++-6' BUILD_TYPE='Debug'
+        apt:
+          sources: *all_sources
+          packages: ['valgrind', 'g++-7']
+      env: COMPILER='g++-7' VALGRIND=1
+
 
     # 3b/ Linux C++14 Clang builds
     - os: linux
       compiler: clang
-      addons: &clang38cpp14
+      addons:
           apt:
-              packages: ["clang-3.8", "libstdc++-6-dev"]
+              packages: ['clang-3.8', 'valgrind', 'libstdc++-6-dev']
               sources:
                   - ubuntu-toolchain-r-test
                   - llvm-toolchain-trusty
-      env: COMPILER='clang++-3.8' BUILD_TYPE='Release' CPP14=1
+      env: COMPILER='clang++-3.8' CPP14=1 VALGRIND=1
 
     - os: linux
       compiler: clang
-      addons: *clang38cpp14
+      addons:
-      env: COMPILER='clang++-3.8' BUILD_TYPE='Debug' CPP14=1
+          apt:
+              sources: *all_sources
+              packages: ['clang-3.9', 'valgrind', 'libstdc++-6-dev']
+      env: COMPILER='clang++-3.9' CPP14=1 VALGRIND=1
+
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-4.0', 'valgrind', 'libstdc++-6-dev']
+      env: COMPILER='clang++-4.0' CPP14=1 VALGRIND=1
+
+    - os: linux
+      compiler: clang
+      addons:
+          apt:
+              sources: *all_sources
+              packages: ['clang-5.0', 'valgrind', 'libstdc++-6-dev']
+      env: COMPILER='clang++-5.0' CPP14=1 VALGRIND=1
+
 
     # 4a/ Linux C++14 GCC builds
     - os: linux
       compiler: gcc
       addons: *gcc6
-      env: COMPILER='g++-6' BUILD_TYPE='Release' CPP14=1
+      env: COMPILER='g++-6' CPP14=1 VALGRIND=1
 
     - os: linux
       compiler: gcc
-      addons: *gcc6
+      addons: *gcc7
-      env: COMPILER='g++-6' BUILD_TYPE='Debug' CPP14=1
+      env: COMPILER='g++-7' CPP14=1 VALGRIND=1
 
     # 5/ OSX Clang Builds
     - os: osx
       osx_image: xcode7.3
       compiler: clang
-      env: COMPILER='clang++' BUILD_TYPE='Debug'
+      env: COMPILER='clang++'
-
-    - os: osx
-      osx_image: xcode7.3
-      compiler: clang
-      env: COMPILER='clang++' BUILD_TYPE='Release'
 
     - os: osx
       osx_image: xcode8
       compiler: clang
-      env: COMPILER='clang++' BUILD_TYPE='Debug'
+      env: COMPILER='clang++'
 
     - os: osx
-      osx_image: xcode8
+      osx_image: xcode9
       compiler: clang
-      env: COMPILER='clang++' BUILD_TYPE='Release'
+      env: COMPILER='clang++'
 
     - os: osx
-      osx_image: xcode8
+      osx_image: xcode9.1
       compiler: clang
-      env: COMPILER='clang++' BUILD_TYPE='Debug' USE_CPP14=1
+      env: COMPILER='clang++'
 
     - os: osx
-      osx_image: xcode8
+      osx_image: xcode9.1
       compiler: clang
-      env: COMPILER='clang++' BUILD_TYPE='Release' USE_CPP14=1
+      env: COMPILER='clang++' USE_CPP14=1
-
 
 install:
   - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps"
@@ -191,9 +199,14 @@ install:
 before_script:
   - export CXX=${COMPILER}
   - cd ${TRAVIS_BUILD_DIR}
-  - cmake -H. -BBuild -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -Wdev -DUSE_CPP14=${CPP14}
+  # Only run valgrind in debug build
-  - cd Build
+  - cmake -H. -BBuild-Debug -DCMAKE_BUILD_TYPE=Debug -Wdev -DUSE_CPP14=${CPP14} -DUSE_VALGRIND=${VALGRIND}
+  - cmake -H. -BBuild-Release -DCMAKE_BUILD_TYPE=Release -Wdev -DUSE_CPP14=${CPP14}
 
 script:
+  - cd Build-Debug
+  - make -j 2
+  - ctest -V -j 2
+  - cd ../Build-Release
   - make -j 2
   - ctest -V -j 2

CMakeLists.txt

@@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.0)
 
 project(CatchSelfTest)
 
+option(USE_VALGRIND "Perform SelfTests with Valgrind" OFF)
+
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 
 # define some folders
@@ -307,17 +309,26 @@ if (NOT NO_SELFTEST)
 
     # configure unit tests via CTest
     enable_testing()
-    add_test(NAME RunTests COMMAND SelfTest)
+    add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
 
-    add_test(NAME ListTests COMMAND SelfTest --list-tests)
+    add_test(NAME ListTests COMMAND $<TARGET_FILE:SelfTest> --list-tests)
     set_tests_properties(ListTests PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ test cases")
 
-    add_test(NAME ListTags COMMAND SelfTest --list-tags)
+    add_test(NAME ListTags COMMAND $<TARGET_FILE:SelfTest> --list-tags)
     set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
 
     # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
     add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)
     set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed")
+    
+    if (USE_VALGRIND)
+        add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>)
+        add_test(NAME ValgrindListTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest> --list-tests)
+        set_tests_properties(ValgrindListTests PROPERTIES PASS_REGULAR_EXPRESSION "definitely lost: 0 bytes in 0 blocks")
+        add_test(NAME ValgrindListTags COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest> --list-tags)
+        set_tests_properties(ValgrindListTags PROPERTIES PASS_REGULAR_EXPRESSION "definitely lost: 0 bytes in 0 blocks")
+    endif()
+    
 endif() # !NO_SELFTEST
 
 

appveyor.yml

@@ -1,6 +1,10 @@
 # version string format -- This will be overwritten later anyway
 version: "{build}"
 
+branches:
+  except:
+    - /dev-travis.+/
+
 os:
   - Visual Studio 2017
   - Visual Studio 2015