From e344984a1b0c1c4f267d3dbfaa73bb5874a86211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sat, 25 Nov 2017 16:58:29 +0100 Subject: [PATCH] Add codecov.io coverage tracking * Every Linux build tracks coverage when running Debug mode * OS X not supported yet (Future WIP) * Our own unit tests, non-default reporters and Clara are ignored --- .travis.yml | 69 +++++--- CMake/FindGcov.cmake | 157 ++++++++++++++++++ CMake/FindLcov.cmake | 354 ++++++++++++++++++++++++++++++++++++++++ CMake/Findcodecov.cmake | 258 +++++++++++++++++++++++++++++ CMake/llvm-cov-wrapper | 56 +++++++ CMakeLists.txt | 10 ++ codecov.yml | 14 ++ 7 files changed, 892 insertions(+), 26 deletions(-) create mode 100644 CMake/FindGcov.cmake create mode 100644 CMake/FindLcov.cmake create mode 100644 CMake/Findcodecov.cmake create mode 100755 CMake/llvm-cov-wrapper create mode 100644 codecov.yml diff --git a/.travis.yml b/.travis.yml index 1e8a3742..26f89e52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['valgrind', 'clang-3.5'] + packages: ['valgrind', 'lcov', 'clang-3.5'] env: COMPILER='clang++-3.5' VALGRIND=1 - os: linux @@ -25,7 +25,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['valgrind', 'clang-3.6'] + packages: ['valgrind', 'lcov', 'clang-3.6'] env: COMPILER='clang++-3.6' VALGRIND=1 # Travis's containers do not seem to have Clang 3.7 in apt, no matter what sources I add. @@ -42,7 +42,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['valgrind', 'clang-3.8'] + packages: ['valgrind', 'lcov', 'clang-3.8'] env: COMPILER='clang++-3.8' VALGRIND=1 - os: linux @@ -50,7 +50,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['clang-3.9', 'valgrind'] + packages: ['clang-3.9', 'valgrind', 'lcov'] env: COMPILER='clang++-3.9' VALGRIND=1 - os: linux @@ -58,7 +58,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['clang-4.0', 'valgrind'] + packages: ['clang-4.0', 'valgrind', 'lcov'] env: COMPILER='clang++-4.0' VALGRIND=1 - os: linux @@ -66,7 +66,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['clang-5.0', 'valgrind'] + packages: ['clang-5.0', 'valgrind', 'lcov'] env: COMPILER='clang++-5.0' VALGRIND=1 # 2/ Linux GCC Builds @@ -75,7 +75,7 @@ matrix: addons: apt: sources: ['ubuntu-toolchain-r-test'] - packages: ['valgrind', 'g++-4.8'] + packages: ['valgrind', 'lcov', 'g++-4.8'] env: COMPILER='g++-4.8' VALGRIND=1 - os: linux @@ -83,7 +83,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['valgrind', 'g++-4.9'] + packages: ['valgrind', 'lcov', 'g++-4.9'] env: COMPILER='g++-4.9' VALGRIND=1 - os: linux @@ -91,7 +91,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['valgrind', 'g++-5'] + packages: ['valgrind', 'lcov', 'g++-5'] env: COMPILER='g++-5' VALGRIND=1 - os: linux @@ -99,7 +99,7 @@ matrix: addons: &gcc6 apt: sources: *all_sources - packages: ['valgrind', 'g++-6'] + packages: ['valgrind', 'lcov', 'g++-6'] env: COMPILER='g++-6' VALGRIND=1 - os: linux @@ -107,7 +107,7 @@ matrix: addons: &gcc7 apt: sources: *all_sources - packages: ['valgrind', 'g++-7'] + packages: ['valgrind', 'lcov', 'g++-7'] env: COMPILER='g++-7' VALGRIND=1 # 3b/ Linux C++14 Clang builds @@ -115,7 +115,7 @@ matrix: compiler: clang addons: apt: - packages: ['clang-3.8', 'valgrind', 'libstdc++-6-dev'] + packages: ['clang-3.8', 'valgrind', 'lcov', 'libstdc++-6-dev'] sources: - ubuntu-toolchain-r-test - llvm-toolchain-trusty @@ -126,7 +126,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['clang-3.9', 'valgrind', 'libstdc++-6-dev'] + packages: ['clang-3.9', 'valgrind', 'lcov', 'libstdc++-6-dev'] env: COMPILER='clang++-3.9' CPP14=1 VALGRIND=1 - os: linux @@ -134,7 +134,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['clang-4.0', 'valgrind', 'libstdc++-6-dev'] + packages: ['clang-4.0', 'valgrind', 'lcov', 'libstdc++-6-dev'] env: COMPILER='clang++-4.0' CPP14=1 VALGRIND=1 - os: linux @@ -142,7 +142,7 @@ matrix: addons: apt: sources: *all_sources - packages: ['clang-5.0', 'valgrind', 'libstdc++-6-dev'] + packages: ['clang-5.0', 'valgrind', 'lcov', 'libstdc++-6-dev'] env: COMPILER='clang++-5.0' CPP14=1 VALGRIND=1 @@ -183,6 +183,7 @@ matrix: compiler: clang env: COMPILER='clang++' USE_CPP14=1 + install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR} @@ -192,7 +193,13 @@ install: mkdir cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C cmake export PATH=${DEPS_DIR}/cmake/bin:${PATH} elif [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then - which cmake || brew install cmake + for pkg in cmake lcov; do + if brew list -1 | grep -q "^${pkg}\$"; then + brew outdated $pkg || brew upgrade $pkg; + else + brew install $pkg; + fi + done fi before_script: @@ -201,15 +208,25 @@ before_script: # Regenerate single header file, so it is tested in the examples... - python scripts/generateSingleHeader.py - # Use Debug builds for running Valgrind and building examples - - cmake -H. -BBuild-Debug -DCMAKE_BUILD_TYPE=Debug -Wdev -DUSE_CPP14=${CPP14} -DUSE_VALGRIND=${VALGRIND} -DBUILD_EXAMPLES=ON - # Check that we don't miscompile with optimalizations... - - cmake -H. -BBuild-Release -DCMAKE_BUILD_TYPE=Release -Wdev -DUSE_CPP14=${CPP14} + - | + # Use Debug builds for running Valgrind and building examples + cmake -H. -BBuild-Debug -DCMAKE_BUILD_TYPE=Debug -Wdev -DUSE_CPP14=${CPP14} -DUSE_VALGRIND=${VALGRIND} -DBUILD_EXAMPLES=ON -DENABLE_COVERAGE=ON + # Don't bother with release build for coverage build + cmake -H. -BBuild-Release -DCMAKE_BUILD_TYPE=Release -Wdev -DUSE_CPP14=${CPP14} + script: - - cd Build-Debug - - make -j 2 - - CTEST_OUTPUT_ON_FAILURE=1 ctest -j 2 - - cd ../Build-Release - - make -j 2 - - CTEST_OUTPUT_ON_FAILURE=1 ctest -j 2 + - | + cd Build-Debug + make -j 2 + CTEST_OUTPUT_ON_FAILURE=1 ctest -j 2 + # Coverage collection does not work for OS X atm + if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then + make gcov + make lcov + bash <(curl -s https://codecov.io/bash) -X gcov || echo "Codecov did not collect coverage reports" + fi + # Go to release build + cd ../Build-Release + make -j 2 + CTEST_OUTPUT_ON_FAILURE=1 ctest -j 2 diff --git a/CMake/FindGcov.cmake b/CMake/FindGcov.cmake new file mode 100644 index 00000000..6ffd6eac --- /dev/null +++ b/CMake/FindGcov.cmake @@ -0,0 +1,157 @@ +# This file is part of CMake-codecov. +# +# Copyright (c) +# 2015-2017 RWTH Aachen University, Federal Republic of Germany +# +# See the LICENSE file in the package base directory for details +# +# Written by Alexander Haase, alexander.haase@rwth-aachen.de +# + + +# include required Modules +include(FindPackageHandleStandardArgs) + + +# Search for gcov binary. +set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET}) +set(CMAKE_REQUIRED_QUIET ${codecov_FIND_QUIETLY}) + +get_property(ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) +foreach (LANG ${ENABLED_LANGUAGES}) + # Gcov evaluation is dependend on the used compiler. Check gcov support for + # each compiler that is used. If gcov binary was already found for this + # compiler, do not try to find it again. + if (NOT GCOV_${CMAKE_${LANG}_COMPILER_ID}_BIN) + get_filename_component(COMPILER_PATH "${CMAKE_${LANG}_COMPILER}" PATH) + + if ("${CMAKE_${LANG}_COMPILER_ID}" STREQUAL "GNU") + # Some distributions like OSX (homebrew) ship gcov with the compiler + # version appended as gcov-x. To find this binary we'll build the + # suggested binary name with the compiler version. + string(REGEX MATCH "^[0-9]+" GCC_VERSION + "${CMAKE_${LANG}_COMPILER_VERSION}") + + find_program(GCOV_BIN NAMES gcov-${GCC_VERSION} gcov + HINTS ${COMPILER_PATH}) + + elseif ("${CMAKE_${LANG}_COMPILER_ID}" STREQUAL "Clang") + # Some distributions like Debian ship llvm-cov with the compiler + # version appended as llvm-cov-x.y. To find this binary we'll build + # the suggested binary name with the compiler version. + string(REGEX MATCH "^[0-9]+.[0-9]+" LLVM_VERSION + "${CMAKE_${LANG}_COMPILER_VERSION}") + + # llvm-cov prior version 3.5 seems to be not working with coverage + # evaluation tools, but these versions are compatible with the gcc + # gcov tool. + if(LLVM_VERSION VERSION_GREATER 3.4) + find_program(LLVM_COV_BIN NAMES "llvm-cov-${LLVM_VERSION}" + "llvm-cov" HINTS ${COMPILER_PATH}) + mark_as_advanced(LLVM_COV_BIN) + + if (LLVM_COV_BIN) + find_program(LLVM_COV_WRAPPER "llvm-cov-wrapper" PATHS + ${CMAKE_MODULE_PATH}) + if (LLVM_COV_WRAPPER) + set(GCOV_BIN "${LLVM_COV_WRAPPER}" CACHE FILEPATH "") + + # set additional parameters + set(GCOV_${CMAKE_${LANG}_COMPILER_ID}_ENV + "LLVM_COV_BIN=${LLVM_COV_BIN}" CACHE STRING + "Environment variables for llvm-cov-wrapper.") + mark_as_advanced(GCOV_${CMAKE_${LANG}_COMPILER_ID}_ENV) + endif () + endif () + endif () + + if (NOT GCOV_BIN) + # Fall back to gcov binary if llvm-cov was not found or is + # incompatible. This is the default on OSX, but may crash on + # recent Linux versions. + find_program(GCOV_BIN gcov HINTS ${COMPILER_PATH}) + endif () + endif () + + + if (GCOV_BIN) + set(GCOV_${CMAKE_${LANG}_COMPILER_ID}_BIN "${GCOV_BIN}" CACHE STRING + "${LANG} gcov binary.") + + if (NOT CMAKE_REQUIRED_QUIET) + message("-- Found gcov evaluation for " + "${CMAKE_${LANG}_COMPILER_ID}: ${GCOV_BIN}") + endif() + + unset(GCOV_BIN CACHE) + endif () + endif () +endforeach () + + + + +# Add a new global target for all gcov targets. This target could be used to +# generate the gcov files for the whole project instead of calling -gcov +# for each target. +if (NOT TARGET gcov) + add_custom_target(gcov) +endif (NOT TARGET gcov) + + + +# This function will add gcov evaluation for target . Only sources of +# this target will be evaluated and no dependencies will be added. It will call +# Gcov on any source file of once and store the gcov file in the same +# directory. +function (add_gcov_target TNAME) + set(TDIR ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${TNAME}.dir) + + # We don't have to check, if the target has support for coverage, thus this + # will be checked by add_coverage_target in Findcoverage.cmake. Instead we + # have to determine which gcov binary to use. + get_target_property(TSOURCES ${TNAME} SOURCES) + set(SOURCES "") + set(TCOMPILER "") + foreach (FILE ${TSOURCES}) + codecov_path_of_source(${FILE} FILE) + if (NOT "${FILE}" STREQUAL "") + codecov_lang_of_source(${FILE} LANG) + if (NOT "${LANG}" STREQUAL "") + list(APPEND SOURCES "${FILE}") + set(TCOMPILER ${CMAKE_${LANG}_COMPILER_ID}) + endif () + endif () + endforeach () + + # If no gcov binary was found, coverage data can't be evaluated. + if (NOT GCOV_${TCOMPILER}_BIN) + message(WARNING "No coverage evaluation binary found for ${TCOMPILER}.") + return() + endif () + + set(GCOV_BIN "${GCOV_${TCOMPILER}_BIN}") + set(GCOV_ENV "${GCOV_${TCOMPILER}_ENV}") + + + set(BUFFER "") + foreach(FILE ${SOURCES}) + get_filename_component(FILE_PATH "${TDIR}/${FILE}" PATH) + + # call gcov + add_custom_command(OUTPUT ${TDIR}/${FILE}.gcov + COMMAND ${GCOV_ENV} ${GCOV_BIN} ${TDIR}/${FILE}.gcno > /dev/null + DEPENDS ${TNAME} ${TDIR}/${FILE}.gcno + WORKING_DIRECTORY ${FILE_PATH} + ) + + list(APPEND BUFFER ${TDIR}/${FILE}.gcov) + endforeach() + + + # add target for gcov evaluation of + add_custom_target(${TNAME}-gcov DEPENDS ${BUFFER}) + + # add evaluation target to the global gcov target. + add_dependencies(gcov ${TNAME}-gcov) +endfunction (add_gcov_target) diff --git a/CMake/FindLcov.cmake b/CMake/FindLcov.cmake new file mode 100644 index 00000000..beb925ae --- /dev/null +++ b/CMake/FindLcov.cmake @@ -0,0 +1,354 @@ +# This file is part of CMake-codecov. +# +# Copyright (c) +# 2015-2017 RWTH Aachen University, Federal Republic of Germany +# +# See the LICENSE file in the package base directory for details +# +# Written by Alexander Haase, alexander.haase@rwth-aachen.de +# + + +# configuration +set(LCOV_DATA_PATH "${CMAKE_BINARY_DIR}/lcov/data") +set(LCOV_DATA_PATH_INIT "${LCOV_DATA_PATH}/init") +set(LCOV_DATA_PATH_CAPTURE "${LCOV_DATA_PATH}/capture") +set(LCOV_HTML_PATH "${CMAKE_BINARY_DIR}/lcov/html") + + + + +# Search for Gcov which is used by Lcov. +find_package(Gcov) + + + + +# This function will add lcov evaluation for target . Only sources of +# this target will be evaluated and no dependencies will be added. It will call +# geninfo on any source file of once and store the info file in the same +# directory. +# +# Note: This function is only a wrapper to define this function always, even if +# coverage is not supported by the compiler or disabled. This function must +# be defined here, because the module will be exited, if there is no coverage +# support by the compiler or it is disabled by the user. +function (add_lcov_target TNAME) + if (LCOV_FOUND) + # capture initial coverage data + lcov_capture_initial_tgt(${TNAME}) + + # capture coverage data after execution + lcov_capture_tgt(${TNAME}) + endif () +endfunction (add_lcov_target) + + + + +# include required Modules +include(FindPackageHandleStandardArgs) + +# Search for required lcov binaries. +find_program(LCOV_BIN lcov) +find_program(GENINFO_BIN geninfo) +find_program(GENHTML_BIN genhtml) +find_package_handle_standard_args(lcov + REQUIRED_VARS LCOV_BIN GENINFO_BIN GENHTML_BIN +) + +# enable genhtml C++ demangeling, if c++filt is found. +set(GENHTML_CPPFILT_FLAG "") +find_program(CPPFILT_BIN c++filt) +if (NOT CPPFILT_BIN STREQUAL "") + set(GENHTML_CPPFILT_FLAG "--demangle-cpp") +endif (NOT CPPFILT_BIN STREQUAL "") + +# enable no-external flag for lcov, if available. +if (GENINFO_BIN AND NOT DEFINED GENINFO_EXTERN_FLAG) + set(FLAG "") + execute_process(COMMAND ${GENINFO_BIN} --help OUTPUT_VARIABLE GENINFO_HELP) + string(REGEX MATCH "external" GENINFO_RES "${GENINFO_HELP}") + if (GENINFO_RES) + set(FLAG "--no-external") + endif () + + set(GENINFO_EXTERN_FLAG "${FLAG}" + CACHE STRING "Geninfo flag to exclude system sources.") +endif () + +# If Lcov was not found, exit module now. +if (NOT LCOV_FOUND) + return() +endif (NOT LCOV_FOUND) + + + + +# Create directories to be used. +file(MAKE_DIRECTORY ${LCOV_DATA_PATH_INIT}) +file(MAKE_DIRECTORY ${LCOV_DATA_PATH_CAPTURE}) + +set(LCOV_REMOVE_PATTERNS "") + +# This function will merge lcov files to a single target file. Additional lcov +# flags may be set with setting LCOV_EXTRA_FLAGS before calling this function. +function (lcov_merge_files OUTFILE ...) + # Remove ${OUTFILE} from ${ARGV} and generate lcov parameters with files. + list(REMOVE_AT ARGV 0) + + # Generate merged file. + string(REPLACE "${CMAKE_BINARY_DIR}/" "" FILE_REL "${OUTFILE}") + add_custom_command(OUTPUT "${OUTFILE}.raw" + COMMAND cat ${ARGV} > ${OUTFILE}.raw + DEPENDS ${ARGV} + COMMENT "Generating ${FILE_REL}" + ) + + add_custom_command(OUTPUT "${OUTFILE}" + COMMAND ${LCOV_BIN} --quiet -a ${OUTFILE}.raw --output-file ${OUTFILE} + --base-directory ${PROJECT_SOURCE_DIR} ${LCOV_EXTRA_FLAGS} + COMMAND ${LCOV_BIN} --quiet -r ${OUTFILE} ${LCOV_REMOVE_PATTERNS} + --output-file ${OUTFILE} ${LCOV_EXTRA_FLAGS} + DEPENDS ${OUTFILE}.raw + COMMENT "Post-processing ${FILE_REL}" + ) +endfunction () + + + + +# Add a new global target to generate initial coverage reports for all targets. +# This target will be used to generate the global initial info file, which is +# used to gather even empty report data. +if (NOT TARGET lcov-capture-init) + add_custom_target(lcov-capture-init) + set(LCOV_CAPTURE_INIT_FILES "" CACHE INTERNAL "") +endif (NOT TARGET lcov-capture-init) + + +# This function will add initial capture of coverage data for target , +# which is needed to get also data for objects, which were not loaded at +# execution time. It will call geninfo for every source file of once and +# store the info file in the same directory. +function (lcov_capture_initial_tgt TNAME) + # We don't have to check, if the target has support for coverage, thus this + # will be checked by add_coverage_target in Findcoverage.cmake. Instead we + # have to determine which gcov binary to use. + get_target_property(TSOURCES ${TNAME} SOURCES) + set(SOURCES "") + set(TCOMPILER "") + foreach (FILE ${TSOURCES}) + codecov_path_of_source(${FILE} FILE) + if (NOT "${FILE}" STREQUAL "") + codecov_lang_of_source(${FILE} LANG) + if (NOT "${LANG}" STREQUAL "") + list(APPEND SOURCES "${FILE}") + set(TCOMPILER ${CMAKE_${LANG}_COMPILER_ID}) + endif () + endif () + endforeach () + + # If no gcov binary was found, coverage data can't be evaluated. + if (NOT GCOV_${TCOMPILER}_BIN) + message(WARNING "No coverage evaluation binary found for ${TCOMPILER}.") + return() + endif () + + set(GCOV_BIN "${GCOV_${TCOMPILER}_BIN}") + set(GCOV_ENV "${GCOV_${TCOMPILER}_ENV}") + + + set(TDIR ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${TNAME}.dir) + set(GENINFO_FILES "") + foreach(FILE ${SOURCES}) + # generate empty coverage files + set(OUTFILE "${TDIR}/${FILE}.info.init") + list(APPEND GENINFO_FILES ${OUTFILE}) + + add_custom_command(OUTPUT ${OUTFILE} COMMAND ${GCOV_ENV} ${GENINFO_BIN} + --quiet --base-directory ${PROJECT_SOURCE_DIR} --initial + --gcov-tool ${GCOV_BIN} --output-filename ${OUTFILE} + ${GENINFO_EXTERN_FLAG} ${TDIR}/${FILE}.gcno + DEPENDS ${TNAME} + COMMENT "Capturing initial coverage data for ${FILE}" + ) + endforeach() + + # Concatenate all files generated by geninfo to a single file per target. + set(OUTFILE "${LCOV_DATA_PATH_INIT}/${TNAME}.info") + set(LCOV_EXTRA_FLAGS "--initial") + lcov_merge_files("${OUTFILE}" ${GENINFO_FILES}) + add_custom_target(${TNAME}-capture-init ALL DEPENDS ${OUTFILE}) + + # add geninfo file generation to global lcov-geninfo target + add_dependencies(lcov-capture-init ${TNAME}-capture-init) + set(LCOV_CAPTURE_INIT_FILES "${LCOV_CAPTURE_INIT_FILES}" + "${OUTFILE}" CACHE INTERNAL "" + ) +endfunction (lcov_capture_initial_tgt) + + +# This function will generate the global info file for all targets. It has to be +# called after all other CMake functions in the root CMakeLists.txt file, to get +# a full list of all targets that generate coverage data. +function (lcov_capture_initial) + # Skip this function (and do not create the following targets), if there are + # no input files. + if ("${LCOV_CAPTURE_INIT_FILES}" STREQUAL "") + return() + endif () + + # Add a new target to merge the files of all targets. + set(OUTFILE "${LCOV_DATA_PATH_INIT}/all_targets.info") + lcov_merge_files("${OUTFILE}" ${LCOV_CAPTURE_INIT_FILES}) + add_custom_target(lcov-geninfo-init ALL DEPENDS ${OUTFILE} + lcov-capture-init + ) +endfunction (lcov_capture_initial) + + + + +# Add a new global target to generate coverage reports for all targets. This +# target will be used to generate the global info file. +if (NOT TARGET lcov-capture) + add_custom_target(lcov-capture) + set(LCOV_CAPTURE_FILES "" CACHE INTERNAL "") +endif (NOT TARGET lcov-capture) + + +# This function will add capture of coverage data for target , which is +# needed to get also data for objects, which were not loaded at execution time. +# It will call geninfo for every source file of once and store the info +# file in the same directory. +function (lcov_capture_tgt TNAME) + # We don't have to check, if the target has support for coverage, thus this + # will be checked by add_coverage_target in Findcoverage.cmake. Instead we + # have to determine which gcov binary to use. + get_target_property(TSOURCES ${TNAME} SOURCES) + set(SOURCES "") + set(TCOMPILER "") + foreach (FILE ${TSOURCES}) + codecov_path_of_source(${FILE} FILE) + if (NOT "${FILE}" STREQUAL "") + codecov_lang_of_source(${FILE} LANG) + if (NOT "${LANG}" STREQUAL "") + list(APPEND SOURCES "${FILE}") + set(TCOMPILER ${CMAKE_${LANG}_COMPILER_ID}) + endif () + endif () + endforeach () + + # If no gcov binary was found, coverage data can't be evaluated. + if (NOT GCOV_${TCOMPILER}_BIN) + message(WARNING "No coverage evaluation binary found for ${TCOMPILER}.") + return() + endif () + + set(GCOV_BIN "${GCOV_${TCOMPILER}_BIN}") + set(GCOV_ENV "${GCOV_${TCOMPILER}_ENV}") + + + set(TDIR ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${TNAME}.dir) + set(GENINFO_FILES "") + foreach(FILE ${SOURCES}) + # Generate coverage files. If no .gcda file was generated during + # execution, the empty coverage file will be used instead. + set(OUTFILE "${TDIR}/${FILE}.info") + list(APPEND GENINFO_FILES ${OUTFILE}) + + add_custom_command(OUTPUT ${OUTFILE} + COMMAND test -f "${TDIR}/${FILE}.gcda" + && ${GCOV_ENV} ${GENINFO_BIN} --quiet --base-directory + ${PROJECT_SOURCE_DIR} --gcov-tool ${GCOV_BIN} + --output-filename ${OUTFILE} ${GENINFO_EXTERN_FLAG} + ${TDIR}/${FILE}.gcda + || cp ${OUTFILE}.init ${OUTFILE} + DEPENDS ${TNAME} ${TNAME}-capture-init + COMMENT "Capturing coverage data for ${FILE}" + ) + endforeach() + + # Concatenate all files generated by geninfo to a single file per target. + set(OUTFILE "${LCOV_DATA_PATH_CAPTURE}/${TNAME}.info") + lcov_merge_files("${OUTFILE}" ${GENINFO_FILES}) + add_custom_target(${TNAME}-geninfo DEPENDS ${OUTFILE}) + + # add geninfo file generation to global lcov-capture target + add_dependencies(lcov-capture ${TNAME}-geninfo) + set(LCOV_CAPTURE_FILES "${LCOV_CAPTURE_FILES}" "${OUTFILE}" CACHE INTERNAL + "" + ) + + # Add target for generating html output for this target only. + file(MAKE_DIRECTORY ${LCOV_HTML_PATH}/${TNAME}) + add_custom_target(${TNAME}-genhtml + COMMAND ${GENHTML_BIN} --quiet --sort --prefix ${PROJECT_SOURCE_DIR} + --baseline-file ${LCOV_DATA_PATH_INIT}/${TNAME}.info + --output-directory ${LCOV_HTML_PATH}/${TNAME} + --title "${CMAKE_PROJECT_NAME} - target ${TNAME}" + ${GENHTML_CPPFILT_FLAG} ${OUTFILE} + DEPENDS ${TNAME}-geninfo ${TNAME}-capture-init + ) +endfunction (lcov_capture_tgt) + + +# This function will generate the global info file for all targets. It has to be +# called after all other CMake functions in the root CMakeLists.txt file, to get +# a full list of all targets that generate coverage data. +function (lcov_capture) + # Skip this function (and do not create the following targets), if there are + # no input files. + if ("${LCOV_CAPTURE_FILES}" STREQUAL "") + return() + endif () + + # Add a new target to merge the files of all targets. + set(OUTFILE "${LCOV_DATA_PATH_CAPTURE}/all_targets.info") + lcov_merge_files("${OUTFILE}" ${LCOV_CAPTURE_FILES}) + add_custom_target(lcov-geninfo DEPENDS ${OUTFILE} lcov-capture) + + # Add a new global target for all lcov targets. This target could be used to + # generate the lcov html output for the whole project instead of calling + # -geninfo and -genhtml for each target. It will also be + # used to generate a html site for all project data together instead of one + # for each target. + if (NOT TARGET lcov) + file(MAKE_DIRECTORY ${LCOV_HTML_PATH}/all_targets) + add_custom_target(lcov + COMMAND ${GENHTML_BIN} --quiet --sort + --baseline-file ${LCOV_DATA_PATH_INIT}/all_targets.info + --output-directory ${LCOV_HTML_PATH}/all_targets + --title "${CMAKE_PROJECT_NAME}" --prefix "${PROJECT_SOURCE_DIR}" + ${GENHTML_CPPFILT_FLAG} ${OUTFILE} + DEPENDS lcov-geninfo-init lcov-geninfo + ) + endif () +endfunction (lcov_capture) + + + + +# Add a new global target to generate the lcov html report for the whole project +# instead of calling -genhtml for each target (to create an own report +# for each target). Instead of the lcov target it does not require geninfo for +# all targets, so you have to call -geninfo to generate the info files +# the targets you'd like to have in your report or lcov-geninfo for generating +# info files for all targets before calling lcov-genhtml. +file(MAKE_DIRECTORY ${LCOV_HTML_PATH}/selected_targets) +if (NOT TARGET lcov-genhtml) + add_custom_target(lcov-genhtml + COMMAND ${GENHTML_BIN} + --quiet + --output-directory ${LCOV_HTML_PATH}/selected_targets + --title \"${CMAKE_PROJECT_NAME} - targets `find + ${LCOV_DATA_PATH_CAPTURE} -name \"*.info\" ! -name + \"all_targets.info\" -exec basename {} .info \\\;`\" + --prefix ${PROJECT_SOURCE_DIR} + --sort + ${GENHTML_CPPFILT_FLAG} + `find ${LCOV_DATA_PATH_CAPTURE} -name \"*.info\" ! -name + \"all_targets.info\"` + ) +endif (NOT TARGET lcov-genhtml) diff --git a/CMake/Findcodecov.cmake b/CMake/Findcodecov.cmake new file mode 100644 index 00000000..fa135fa8 --- /dev/null +++ b/CMake/Findcodecov.cmake @@ -0,0 +1,258 @@ +# This file is part of CMake-codecov. +# +# Copyright (c) +# 2015-2017 RWTH Aachen University, Federal Republic of Germany +# +# See the LICENSE file in the package base directory for details +# +# Written by Alexander Haase, alexander.haase@rwth-aachen.de +# + + +# Add an option to choose, if coverage should be enabled or not. If enabled +# marked targets will be build with coverage support and appropriate targets +# will be added. If disabled coverage will be ignored for *ALL* targets. +option(ENABLE_COVERAGE "Enable coverage build." OFF) + +set(COVERAGE_FLAG_CANDIDATES + # gcc and clang + "-O0 -g -fprofile-arcs -ftest-coverage" + + # gcc and clang fallback + "-O0 -g --coverage" +) + + +# Add coverage support for target ${TNAME} and register target for coverage +# evaluation. If coverage is disabled or not supported, this function will +# simply do nothing. +# +# Note: This function is only a wrapper to define this function always, even if +# coverage is not supported by the compiler or disabled. This function must +# be defined here, because the module will be exited, if there is no coverage +# support by the compiler or it is disabled by the user. +function (add_coverage TNAME) + # only add coverage for target, if coverage is support and enabled. + if (ENABLE_COVERAGE) + foreach (TNAME ${ARGV}) + add_coverage_target(${TNAME}) + endforeach () + endif () +endfunction (add_coverage) + + +# Add global target to gather coverage information after all targets have been +# added. Other evaluation functions could be added here, after checks for the +# specific module have been passed. +# +# Note: This function is only a wrapper to define this function always, even if +# coverage is not supported by the compiler or disabled. This function must +# be defined here, because the module will be exited, if there is no coverage +# support by the compiler or it is disabled by the user. +function (coverage_evaluate) + # add lcov evaluation + if (LCOV_FOUND) + lcov_capture_initial() + lcov_capture() + endif (LCOV_FOUND) +endfunction () + + +# Exit this module, if coverage is disabled. add_coverage is defined before this +# return, so this module can be exited now safely without breaking any build- +# scripts. +if (NOT ENABLE_COVERAGE) + return() +endif () + + + + +# Find the reuired flags foreach language. +set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET}) +set(CMAKE_REQUIRED_QUIET ${codecov_FIND_QUIETLY}) + +get_property(ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) +foreach (LANG ${ENABLED_LANGUAGES}) + # Coverage flags are not dependend on language, but the used compiler. So + # instead of searching flags foreach language, search flags foreach compiler + # used. + set(COMPILER ${CMAKE_${LANG}_COMPILER_ID}) + if (NOT COVERAGE_${COMPILER}_FLAGS) + foreach (FLAG ${COVERAGE_FLAG_CANDIDATES}) + if(NOT CMAKE_REQUIRED_QUIET) + message(STATUS "Try ${COMPILER} code coverage flag = [${FLAG}]") + endif() + + set(CMAKE_REQUIRED_FLAGS "${FLAG}") + unset(COVERAGE_FLAG_DETECTED CACHE) + + if (${LANG} STREQUAL "C") + include(CheckCCompilerFlag) + check_c_compiler_flag("${FLAG}" COVERAGE_FLAG_DETECTED) + + elseif (${LANG} STREQUAL "CXX") + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("${FLAG}" COVERAGE_FLAG_DETECTED) + + elseif (${LANG} STREQUAL "Fortran") + # CheckFortranCompilerFlag was introduced in CMake 3.x. To be + # compatible with older Cmake versions, we will check if this + # module is present before we use it. Otherwise we will define + # Fortran coverage support as not available. + include(CheckFortranCompilerFlag OPTIONAL + RESULT_VARIABLE INCLUDED) + if (INCLUDED) + check_fortran_compiler_flag("${FLAG}" + COVERAGE_FLAG_DETECTED) + elseif (NOT CMAKE_REQUIRED_QUIET) + message("-- Performing Test COVERAGE_FLAG_DETECTED") + message("-- Performing Test COVERAGE_FLAG_DETECTED - Failed" + " (Check not supported)") + endif () + endif() + + if (COVERAGE_FLAG_DETECTED) + set(COVERAGE_${COMPILER}_FLAGS "${FLAG}" + CACHE STRING "${COMPILER} flags for code coverage.") + mark_as_advanced(COVERAGE_${COMPILER}_FLAGS) + break() + else () + message(WARNING "Code coverage is not available for ${COMPILER}" + " compiler. Targets using this compiler will be " + "compiled without it.") + endif () + endforeach () + endif () +endforeach () + +set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE}) + + + + +# Helper function to get the language of a source file. +function (codecov_lang_of_source FILE RETURN_VAR) + get_filename_component(FILE_EXT "${FILE}" EXT) + string(TOLOWER "${FILE_EXT}" FILE_EXT) + string(SUBSTRING "${FILE_EXT}" 1 -1 FILE_EXT) + + get_property(ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) + foreach (LANG ${ENABLED_LANGUAGES}) + list(FIND CMAKE_${LANG}_SOURCE_FILE_EXTENSIONS "${FILE_EXT}" TEMP) + if (NOT ${TEMP} EQUAL -1) + set(${RETURN_VAR} "${LANG}" PARENT_SCOPE) + return() + endif () + endforeach() + + set(${RETURN_VAR} "" PARENT_SCOPE) +endfunction () + + +# Helper function to get the relative path of the source file destination path. +# This path is needed by FindGcov and FindLcov cmake files to locate the +# captured data. +function (codecov_path_of_source FILE RETURN_VAR) + string(REGEX MATCH "TARGET_OBJECTS:([^ >]+)" _source ${FILE}) + + # If expression was found, SOURCEFILE is a generator-expression for an + # object library. Currently we found no way to call this function automatic + # for the referenced target, so it must be called in the directoryso of the + # object library definition. + if (NOT "${_source}" STREQUAL "") + set(${RETURN_VAR} "" PARENT_SCOPE) + return() + endif () + + + string(REPLACE "${CMAKE_CURRENT_BINARY_DIR}/" "" FILE "${FILE}") + if(IS_ABSOLUTE ${FILE}) + file(RELATIVE_PATH FILE ${CMAKE_CURRENT_SOURCE_DIR} ${FILE}) + endif() + + # get the right path for file + string(REPLACE ".." "__" PATH "${FILE}") + + set(${RETURN_VAR} "${PATH}" PARENT_SCOPE) +endfunction() + + + + +# Add coverage support for target ${TNAME} and register target for coverage +# evaluation. +function(add_coverage_target TNAME) + # Check if all sources for target use the same compiler. If a target uses + # e.g. C and Fortran mixed and uses different compilers (e.g. clang and + # gfortran) this can trigger huge problems, because different compilers may + # use different implementations for code coverage. + get_target_property(TSOURCES ${TNAME} SOURCES) + set(TARGET_COMPILER "") + set(ADDITIONAL_FILES "") + foreach (FILE ${TSOURCES}) + # If expression was found, FILE is a generator-expression for an object + # library. Object libraries will be ignored. + string(REGEX MATCH "TARGET_OBJECTS:([^ >]+)" _file ${FILE}) + if ("${_file}" STREQUAL "") + codecov_lang_of_source(${FILE} LANG) + if (LANG) + list(APPEND TARGET_COMPILER ${CMAKE_${LANG}_COMPILER_ID}) + + list(APPEND ADDITIONAL_FILES "${FILE}.gcno") + list(APPEND ADDITIONAL_FILES "${FILE}.gcda") + endif () + endif () + endforeach () + + list(REMOVE_DUPLICATES TARGET_COMPILER) + list(LENGTH TARGET_COMPILER NUM_COMPILERS) + + if (NUM_COMPILERS GREATER 1) + message(WARNING "Can't use code coverage for target ${TNAME}, because " + "it will be compiled by incompatible compilers. Target will be " + "compiled without code coverage.") + return() + + elseif (NUM_COMPILERS EQUAL 0) + message(WARNING "Can't use code coverage for target ${TNAME}, because " + "it uses an unknown compiler. Target will be compiled without " + "code coverage.") + return() + + elseif (NOT DEFINED "COVERAGE_${TARGET_COMPILER}_FLAGS") + # A warning has been printed before, so just return if flags for this + # compiler aren't available. + return() + endif() + + + # enable coverage for target + set_property(TARGET ${TNAME} APPEND_STRING + PROPERTY COMPILE_FLAGS " ${COVERAGE_${TARGET_COMPILER}_FLAGS}") + set_property(TARGET ${TNAME} APPEND_STRING + PROPERTY LINK_FLAGS " ${COVERAGE_${TARGET_COMPILER}_FLAGS}") + + + # Add gcov files generated by compiler to clean target. + set(CLEAN_FILES "") + foreach (FILE ${ADDITIONAL_FILES}) + codecov_path_of_source(${FILE} FILE) + list(APPEND CLEAN_FILES "CMakeFiles/${TNAME}.dir/${FILE}") + endforeach() + + set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES + "${CLEAN_FILES}") + + + add_gcov_target(${TNAME}) + add_lcov_target(${TNAME}) +endfunction(add_coverage_target) + + + + +# Include modules for parsing the collected data and output it in a readable +# format (like gcov and lcov). +find_package(Gcov) +find_package(Lcov) diff --git a/CMake/llvm-cov-wrapper b/CMake/llvm-cov-wrapper new file mode 100755 index 00000000..2ac33102 --- /dev/null +++ b/CMake/llvm-cov-wrapper @@ -0,0 +1,56 @@ +#!/bin/sh + +# This file is part of CMake-codecov. +# +# Copyright (c) +# 2015-2017 RWTH Aachen University, Federal Republic of Germany +# +# See the LICENSE file in the package base directory for details +# +# Written by Alexander Haase, alexander.haase@rwth-aachen.de +# + +if [ -z "$LLVM_COV_BIN" ] +then + echo "LLVM_COV_BIN not set!" >& 2 + exit 1 +fi + + +# Get LLVM version to find out. +LLVM_VERSION=$($LLVM_COV_BIN -version | grep -i "LLVM version" \ + | sed "s/^\([A-Za-z ]*\)\([0-9]\).\([0-9]\).*$/\2.\3/g") + +if [ "$1" = "-v" ] +then + echo "llvm-cov-wrapper $LLVM_VERSION" + exit 0 +fi + + +if [ -n "$LLVM_VERSION" ] +then + MAJOR=$(echo $LLVM_VERSION | cut -d'.' -f1) + MINOR=$(echo $LLVM_VERSION | cut -d'.' -f2) + + if [ $MAJOR -eq 3 ] && [ $MINOR -le 4 ] + then + if [ -f "$1" ] + then + filename=$(basename "$1") + extension="${filename##*.}" + + case "$extension" in + "gcno") exec $LLVM_COV_BIN --gcno="$1" ;; + "gcda") exec $LLVM_COV_BIN --gcda="$1" ;; + esac + fi + fi + + if [ $MAJOR -eq 3 ] && [ $MINOR -le 5 ] + then + exec $LLVM_COV_BIN $@ + fi +fi + +exec $LLVM_COV_BIN gcov $@ diff --git a/CMakeLists.txt b/CMakeLists.txt index d0f80aae..ef02c139 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ project(CatchSelfTest) option(USE_VALGRIND "Perform SelfTests with Valgrind" OFF) option(BUILD_EXAMPLES "Build documentation examples" OFF) +option(ENABLE_COVERAGE "Generate coverage for codecov.io" OFF) set_property(GLOBAL PROPERTY USE_FOLDERS ON) @@ -281,6 +282,10 @@ SOURCE_GROUP("Tests" FILES ${TEST_SOURCES}) SOURCE_GROUP("Surrogates" FILES ${SURROGATE_SOURCES}) # configure the executable +if (ENABLE_COVERAGE) + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMake") + find_package(codecov) +endif() # Projects consuming Catch via ExternalProject_Add might want to use install step # without building all of our selftests. @@ -299,6 +304,11 @@ if (NOT NO_SELFTEST) set_property(TARGET SelfTest PROPERTY CXX_STANDARD_REQUIRED ON) set_property(TARGET SelfTest PROPERTY CXX_EXTENSIONS OFF) + if (ENABLE_COVERAGE) + add_coverage(SelfTest) + list(APPEND LCOV_REMOVE_PATTERNS "/sys/") + coverage_evaluate() + endif() # Add desired warnings if ( CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang|GNU" ) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..18ff598e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,14 @@ +codecov: + branch: master + ci: + # Don't wait for AppVeyor build to finalize coverage information + # We don't have a way to generate coverage from it anyway + - !appveyor + +coverage: + ignore: + - "projects/SelfTest" + - "**/catch_reporter_tap.hpp" + - "**/catch_reporter_automake.hpp" + - "**/catch_reporter_teamcity.hpp" + - "**/external/clara.hpp"