Add codecov.io coverage collection from AppVeyor

Also had to add new project to redirect CTest output, add
separate batch scripts for AppVeyor because it doesn't handle
multi-line batch scripts in yaml properly, and other helper
scripts.

CMakeLists.txt

@@ -325,7 +325,7 @@ if (NOT NO_SELFTEST)
 
 
     # configure unit tests via CTest
-    enable_testing()
+    include(CTest)
     add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
 
     add_test(NAME ListTests COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
@@ -341,6 +341,7 @@ if (NOT NO_SELFTEST)
     set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher")
 
 
+
     # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
     add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)
     set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed")
@@ -360,7 +361,6 @@ if(BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
 
-
 install(DIRECTORY "single_include/" DESTINATION "include/catch")
 
 ## Provide some pkg-config integration

appveyor.yml

@@ -30,8 +30,9 @@ init:
   # Set build version to git commit-hash
   - ps: Update-AppveyorBuild -Version "$($env:APPVEYOR_REPO_BRANCH) - $($env:APPVEYOR_REPO_COMMIT)"
 
-# fetch repository as zip archive
-shallow_clone: true
+install:
+  - ps: if (($env:CONFIGURATION) -eq "Debug" ) { python -m pip install codecov }
+  - ps: if (($env:CONFIGURATION) -eq "Debug" ) { .\misc\installOpenCppCoverage.ps1 }
 
 # Win32 and x64 are CMake-compatible solution platform names.
 # This allows us to pass %PLATFORM% to CMake -A.
@@ -46,9 +47,12 @@ configuration:
 
 #Cmake will autodetect the compiler, but we set the arch
 before_build:
-  - python scripts/generateSingleHeader.py
   - set CXXFLAGS=%additional_flags%
-  - cmake -H. -BBuild -A%PLATFORM% -DUSE_WMAIN=%wmain% -DBUILD_EXAMPLES=ON
+  # Indirection because appveyor doesn't handle multiline batch scripts properly
+  # https://stackoverflow.com/questions/37627248/how-to-split-a-command-over-multiple-lines-in-appveyor-yml/37647169#37647169
+  # https://help.appveyor.com/discussions/questions/3888-multi-line-cmd-or-powershell-warning-ignore
+  - cmd: .\misc\appveyorBuildConfigurationScript.bat
+
 
 # build with MSBuild
 build:
@@ -57,6 +61,5 @@ build:
   verbosity: normal                     # MSBuild verbosity level {quiet|minimal|normal|detailed}
 
 test_script:
-  - cd Build
   - set CTEST_OUTPUT_ON_FAILURE=1
-  - ctest -j 2 -C %CONFIGURATION%
+  - cmd: .\misc\appveyorTestRunScript.bat

codecov.yml

@@ -1,9 +1,5 @@
 codecov:
   branch: master
-  ci:
-    # Don't wait for AppVeyor build to finalize coverage information
-    # We don't have a way to generate coverage from it anyway
-    - !appveyor
 
 coverage:
   ignore:

misc/CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.0)
+
+project(CatchCoverageHelper)
+
+add_executable(CoverageHelper coverage-helper.cpp)
+set_property(TARGET CoverageHelper PROPERTY CXX_STANDARD 11)
+set_property(TARGET CoverageHelper PROPERTY CXX_STANDARD_REQUIRED ON)
+set_property(TARGET CoverageHelper PROPERTY CXX_EXTENSIONS OFF)
+if (MSVC)
+    target_compile_options( CoverageHelper PRIVATE /W4 /w44265 /WX /w44061 /w44062 )
+endif()

misc/appveyorBuildConfigurationScript.bat

@@ -0,0 +1,15 @@
+@REM  # In debug build, we want to
+@REM  # 1) Prebuild memcheck redirecter
+@REM  # 2) Regenerate single header include for examples
+@REM  # 3) Enable building examples
+if "%CONFIGURATION%"=="Debug" (
+  echo "buildConfiguration.bat thinks this is a Debug build"
+  python scripts\generateSingleHeader.py
+  cmake -Hmisc -Bbuild-misc -A%PLATFORM%
+  cmake --build build-misc
+  cmake -H. -BBuild -A%PLATFORM% -DUSE_WMAIN=%wmain% -DBUILD_EXAMPLES=ON -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind
+)
+if "%CONFIGURATION%"=="Release" (
+  echo "buildConfiguration.bat thinks this is a Release build"
+  cmake -H. -BBuild -A%PLATFORM% -DUSE_WMAIN=%wmain%
+)

misc/appveyorMergeCoverageScript.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python2
+
+import glob
+import subprocess
+
+if __name__ == '__main__':
+    cov_files = list(glob.glob('cov-report*.bin'))
+    base_cmd = ['OpenCppCoverage', '--quiet', '--export_type=cobertura:cobertura.xml'] + ['--input_coverage={}'.format(f) for f in cov_files]
+    subprocess.call(base_cmd)

misc/appveyorTestRunScript.bat

@@ -0,0 +1,11 @@
+cd Build
+if "%CONFIGURATION%"=="Debug" (
+  echo "appveyorTestScript.bat thinks this is a Debug build"
+  ctest -j 2 -C %CONFIGURATION% -D ExperimentalMemCheck
+  python ..\misc\appveyorMergeCoverageScript.py
+  codecov --root .. --no-color --disable gcov -f cobertura.xml -t %CODECOV_TOKEN%
+)
+if "%CONFIGURATION%"=="Release" (
+  echo "appveyorTestScript.bat thinks this is a Release build"
+  ctest -j 2 -C %CONFIGURATION%
+)

misc/coverage-helper.cpp

@@ -0,0 +1,100 @@
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <regex>
+#include <string>
+#include <vector>
+
+
+void create_empty_file(std::string const& path) {
+    std::ofstream ofs(path);
+    ofs << '\n';
+}
+
+const std::string separator = "--sep--";
+const std::string logfile_prefix = "--log-file=";
+
+bool starts_with(std::string const& str, std::string const& pref) {
+    return str.find(pref) == 0;
+}
+
+int parse_log_file_arg(std::string const& arg) {
+    assert(starts_with(arg, logfile_prefix) && "Attempting to parse incorrect arg!");
+    auto fname = arg.substr(logfile_prefix.size());
+    create_empty_file(fname);
+    std::regex regex("MemoryChecker\\.(\\d+)\\.log", std::regex::icase);
+    std::smatch match;
+    if (std::regex_search(fname, match, regex)) {
+        return std::stoi(match[1]);
+    } else {
+        throw std::domain_error("Couldn't find desired expression in string: " + fname);
+    }
+}
+
+std::string catch_path(std::string path) {
+    auto start = path.find("catch");
+    // try capitalized instead
+    if (start == std::string::npos) {
+        start = path.find("Catch");
+    }
+    if (start == std::string::npos) {
+        throw std::domain_error("Couldn't find Catch's base path");
+    }
+    auto end = path.find_first_of("\\/", start);
+    return path.substr(0, end);
+}
+
+std::string windowsify_path(std::string path) {
+    for (auto& c : path) {
+        if (c == '/') {
+            c = '\\';
+        }
+    }
+    return path;
+}
+
+void exec_cmd(std::string const& cmd, int log_num, std::string const& path) {
+    std::array<char, 128> buffer;
+#if defined(_WIN32)
+    auto real_cmd = "OpenCppCoverage --export_type binary:cov-report" + std::to_string(log_num)
+        + ".bin --quiet " + "--sources " + path + " --cover_children -- " + cmd;
+    std::cout << "=== Marker ===: Cmd: " << real_cmd << '\n';
+    std::shared_ptr<FILE> pipe(_popen(real_cmd.c_str(), "r"), _pclose);
+#else // Just for testing, in the real world we will always work under WIN32
+    (void)log_num; (void)path;
+    std::shared_ptr<FILE> pipe(popen(cmd.c_str(), "r"), pclose);
+#endif
+
+    if (!pipe) {
+        throw std::runtime_error("popen() failed!");
+    }
+    while (!feof(pipe.get())) {
+        if (fgets(buffer.data(), 128, pipe.get()) != nullptr) {
+            std::cout << buffer.data();
+        }
+    }
+}
+
+// argv should be:
+// [0]: our path
+// [1]: "--log-file=<path>"
+// [2]: "--sep--"
+// [3]+: the actual command
+
+int main(int argc, char** argv) {
+    std::vector<std::string> args(argv, argv + argc);
+    auto sep = std::find(begin(args), end(args), separator);
+    assert(sep - begin(args) == 2 && "Structure differs from expected!");
+
+    auto num = parse_log_file_arg(args[1]);
+
+    auto cmdline = std::accumulate(++sep, end(args), std::string{}, [] (const std::string& lhs, const std::string& rhs) {
+        return lhs + ' ' + rhs;
+    });
+
+    exec_cmd(cmdline, num, windowsify_path(catch_path(args[0])));
+}

misc/installOpenCppCoverage.ps1

@@ -0,0 +1,19 @@
+# Downloads are done from the oficial github release page links
+$downloadUrl = "https://github.com/OpenCppCoverage/OpenCppCoverage/releases/download/release-0.9.6.1/OpenCppCoverageSetup-x64-0.9.6.1.exe"
+$installerPath = [System.IO.Path]::Combine($Env:USERPROFILE, "Downloads", "OpenCppCoverageSetup.exe")
+
+if(-Not (Test-Path $installerPath)) {
+    Write-Host -ForegroundColor White ("Downloading OpenCppCoverage from: " + $downloadUrl)
+    Start-FileDownload $downloadUrl -FileName $installerPath
+}
+
+Write-Host -ForegroundColor White "About to install OpenCppCoverage..."
+
+$installProcess = (Start-Process $installerPath -ArgumentList '/VERYSILENT' -PassThru -Wait)
+if($installProcess.ExitCode -ne 0) {
+    throw [System.String]::Format("Failed to install OpenCppCoverage, ExitCode: {0}.", $installProcess.ExitCode)
+}
+
+# Assume standard, boring, installation path of ".../Program Files/OpenCppCoverage"
+$installPath = [System.IO.Path]::Combine(${Env:ProgramFiles}, "OpenCppCoverage")
+$env:Path="$env:Path;$installPath"