Resolve "CI: Simplify coverage scripts"

Alberto Mirandarequested to merge
amiranda/252-ci-simplify-coverage-scripts into master

This MR simplifies coverage generation in the following ways:

  • Replaces the old scripts/ci/coverage.sh script with a new scripts/dev/coverage.py written in Python, making it more robust and simpler to modify if needed.
  • Adds specific CMake targets for generating coverage reports directly from CMake.
  • Adds specialized CMake presets to make it simpler to configure the different builds required to generate coverage information.
  • Replaces gcovr with lcov + lcov_cobertura, since gcovr exhibited some errors that were difficult to track down and lcov worked out of the box.
  • Adds HTML documentation about the coverage generation in docs/sphinx/devs/coverage.md

This MR also updates the v0.9.2 Docker images to update CMake and include coverage-related packages. The coverage image in particular is no longer needed and has been removed.

Closes #252 (closed) #256 (closed) #257 (closed) #258 (closed)

Edited by Alberto Miranda

Merge request reports

CMake/gkfs-code-coverage.cmake

+135 −69
Changes for CMake/gkfs-code-coverage.cmake: 135 added lines, 69 removed lines.
Original line number Diff line number Diff line
@@ -26,85 +26,151 @@ 
# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



# Variables

option(GKFS_ENABLE_CODE_COVERAGE

  "Builds GekkoFS targets with code coverage instrumentation."

  OFF

  )



# Common initialization/checks

if(GKFS_ENABLE_CODE_COVERAGE AND NOT GKFS_CODE_COVERAGE_ADDED)

option(GKFS_GENERATE_COVERAGE_REPORTS "Generate coverage reports" ON)



  set(GKFS_CODE_COVERAGE_ADDED ON)

macro(gkfs_enable_coverage_reports)



  if(CMAKE_C_COMPILER_ID MATCHES "(Apple)?[Cc]lang"

     OR CMAKE_CXX_COMPILER_ID MATCHES "(Apple)?[Cc]lang")

  set(OPTIONS)

  set(SINGLE_VALUE)

  set(MULTI_VALUE EXCLUDE_DIRECTORIES)

  cmake_parse_arguments(

    ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}

  )



    message(STATUS "[gekkofs] Building with LLVM Code Coverage Tools")

  find_program(COVERAGE_PY

    coverage.py

    PATHS ${CMAKE_SOURCE_DIR}/scripts/dev

    REQUIRED)



  elseif(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES

                                              "GNU")

  if(NOT COVERAGE_OUTPUT_DIR)

    set(COVERAGE_OUTPUT_DIR "${CMAKE_BINARY_DIR}/coverage")

  endif()



    message(STATUS "[gekkofs] Building with GCC Code Coverage Tools")

  file(MAKE_DIRECTORY ${COVERAGE_OUTPUT_DIR})



    if(CMAKE_BUILD_TYPE)

      string(TOUPPER ${CMAKE_BUILD_TYPE} upper_build_type)

      if(NOT ${upper_build_type} STREQUAL "DEBUG")

        message(

          WARNING

            "Code coverage results with an optimized (non-Debug) build may be misleading"

        )

      endif()

    else()

      message(

        WARNING

          "Code coverage results with an optimized (non-Debug) build may be misleading"

      )

    endif()

  else()

    message(FATAL_ERROR "Code coverage requires Clang or GCC. Aborting.")

  if(NOT COVERAGE_ZEROCOUNT_TRACEFILE)

    set(COVERAGE_ZEROCOUNT_TRACEFILE "${COVERAGE_OUTPUT_DIR}/zerocount.info")

  endif()



  if(NOT COVERAGE_CAPTURE_TRACEFILE)

    set(COVERAGE_CAPTURE_TRACEFILE "${COVERAGE_OUTPUT_DIR}/capture.info")

  endif()



# Adds code coverage instrumentation to libraries and executable targets.

# ~~~

# Required:

# TARGET_NAME - Name of the target to generate code coverage for.

# Optional:

# PUBLIC   - Sets the visibility for added compile options to targets to PUBLIC

#            instead of the default of PRIVATE.

# PRIVATE - Sets the visibility for added compile options to targets to

#           INTERFACE instead of the default of PRIVATE.

# ~~~

function(target_code_coverage TARGET_NAME)

  # Argument parsing

  set(options PUBLIC INTERFACE)

  cmake_parse_arguments(target_code_coverage "${options}" "" "" ${ARGN})



  # Set the visibility of target functions to PUBLIC, INTERFACE or default to

  # PRIVATE.

  if(target_code_coverage_PUBLIC)

    set(TARGET_VISIBILITY PUBLIC)

  elseif(target_code_coverage_INTERFACE)

    set(TARGET_VISIBILITY INTERFACE)

  else()

    set(TARGET_VISIBILITY PRIVATE)

  if(NOT COVERAGE_UNIFIED_TRACEFILE)

    set(COVERAGE_UNIFIED_TRACEFILE "${COVERAGE_OUTPUT_DIR}/coverage.info")

  endif()



  if(GKFS_ENABLE_CODE_COVERAGE)



    # Add code coverage instrumentation to the target's linker command

    if(CMAKE_C_COMPILER_ID MATCHES "(Apple)?[Cc]lang"

       OR CMAKE_CXX_COMPILER_ID MATCHES "(Apple)?[Cc]lang")

      target_compile_options(${TARGET_NAME} ${TARGET_VISIBILITY}

                             -fprofile-instr-generate -fcoverage-mapping)

      target_link_options(${TARGET_NAME} ${TARGET_VISIBILITY}

                          -fprofile-instr-generate -fcoverage-mapping)

    elseif(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES

                                                "GNU")

      target_compile_options(${TARGET_NAME} ${TARGET_VISIBILITY} -fprofile-arcs

        -ftest-coverage)

      target_link_libraries(${TARGET_NAME} ${TARGET_VISIBILITY} gcov)

  if(NOT COVERAGE_HTML_REPORT_DIRECTORY)

    set(COVERAGE_HTML_REPORT_DIRECTORY "${COVERAGE_OUTPUT_DIR}/coverage_html")

  endif()



  if(NOT COVERAGE_XML_REPORT)

    set(COVERAGE_XML_REPORT "${COVERAGE_OUTPUT_DIR}/coverage-cobertura.xml")

  endif()

endfunction()
 


  # add a `coverage-zerocount` target for the initial baseline gathering of

  # coverage information

  add_custom_command(

    OUTPUT "${COVERAGE_ZEROCOUNT_TRACEFILE}"

    COMMAND

      ${COVERAGE_PY}

        capture

        --initial

        --root-directory "${CMAKE_BINARY_DIR}"

        --output-file "${COVERAGE_ZEROCOUNT_TRACEFILE}"

        --sources-directory "${CMAKE_SOURCE_DIR}"

        "$<$<BOOL:${ARGS_EXCLUDE_DIRECTORIES}>:--exclude;$<JOIN:${ARGS_EXCLUDE_DIRECTORIES},;--exclude;>>"

    COMMAND_EXPAND_LISTS VERBATIM

    COMMENT "Generating zerocount coverage tracefile"

  )



  add_custom_target(coverage-zerocount

    DEPENDS ${COVERAGE_ZEROCOUNT_TRACEFILE})



  # add a `coverage-capture` target to capture coverage data from any

  # of the existing .gcda files

  add_custom_command(

    OUTPUT "${COVERAGE_CAPTURE_TRACEFILE}"

    COMMAND

    ${COVERAGE_PY}

    capture

    --root-directory "${CMAKE_BINARY_DIR}"

    --output-file "${COVERAGE_CAPTURE_TRACEFILE}"

    --sources-directory "${CMAKE_SOURCE_DIR}"

    "$<$<BOOL:${ARGS_EXCLUDE_DIRECTORIES}>:--exclude;$<JOIN:${ARGS_EXCLUDE_DIRECTORIES},;--exclude;>>"

    COMMAND_EXPAND_LISTS VERBATIM

    COMMENT "Generating capture coverage tracefile"

  )



  add_custom_target(coverage-capture DEPENDS ${COVERAGE_CAPTURE_TRACEFILE})



  # add a `coverage-unified` target to merge all coverage data available in

  # ${COVERAGE_OUTPUT_DIR} into a unified coverage trace

  add_custom_command(

    OUTPUT ${COVERAGE_UNIFIED_TRACEFILE}

    DEPENDS ${COVERAGE_CAPTURE_TRACEFILE}

    COMMAND

    ${COVERAGE_PY}

    merge

    --search-pattern "${COVERAGE_OUTPUT_DIR}/*.info"

    --output-file "${COVERAGE_UNIFIED_TRACEFILE}"

    VERBATIM

    COMMENT "Generating unified coverage tracefile"

  )



  add_custom_target(

    coverage-unified

    DEPENDS "${COVERAGE_UNIFIED_TRACEFILE}"

  )



  # add a `coverage-summary` target to print a summary of coverage data

  add_custom_target(

    coverage-summary

    COMMAND

    ${COVERAGE_PY}

    summary

    --input-tracefile ${COVERAGE_UNIFIED_TRACEFILE}

    DEPENDS ${COVERAGE_UNIFIED_TRACEFILE}

    COMMENT "Gathering coverage information"

  )



  # add a `coverage-html` target to generate a coverage HTML report

  add_custom_command(OUTPUT

    "${COVERAGE_HTML_REPORT_DIRECTORY}"

    COMMAND

    ${COVERAGE_PY}

    html_report

    --input-tracefile "${COVERAGE_UNIFIED_TRACEFILE}"

    --prefix "${CMAKE_SOURCE_DIR}"

    --output-directory "${COVERAGE_HTML_REPORT_DIRECTORY}"

    DEPENDS ${COVERAGE_UNIFIED_TRACEFILE}

    VERBATIM

    COMMENT "Generating HTML report"

    )



  add_custom_target(

    coverage-html

    DEPENDS "${COVERAGE_HTML_REPORT_DIRECTORY}")



  # add a `coverage-cobertura` target to generate a Cobertura XML report

  add_custom_command(OUTPUT

    "${COVERAGE_XML_REPORT}"

    COMMAND

    ${COVERAGE_PY}

    cobertura

    --input-tracefile "${COVERAGE_UNIFIED_TRACEFILE}"

    --base-dir "${CMAKE_SOURCE_DIR}"

    --output-file "${COVERAGE_XML_REPORT}"

    DEPENDS ${COVERAGE_UNIFIED_TRACEFILE}

    VERBATIM

    COMMENT "Generating Cobertura report"

    )



  add_custom_target(

    coverage-cobertura

    DEPENDS "${COVERAGE_XML_REPORT}"

  )

  set_target_properties(

    coverage-cobertura PROPERTIES ADDITIONAL_CLEAN_FILES ${COVERAGE_XML_REPORT}

  )

endmacro()

CMake/GkfsPythonTesting.cmake→CMake/gkfs-python-testing.cmake

+0 −0

File moved.

View file

CMake/gkfs-testing.cmake

0 → 100644
+30 −0
Changes for CMake/gkfs-testing.cmake: 30 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



include(gkfs-code-coverage)

include(gkfs-python-testing)

docker/0.9.2/core/Dockerfile

+6 −5
Changes for docker/0.9.2/core/Dockerfile: 6 added lines, 5 removed lines.
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ RUN apt-get update && \ 
		automake \

		gcc \

		g++ \

		ninja-build \

		procps \

		# AGIOS dependencies

		libconfig-dev \
@@ -28,12 +29,12 @@ RUN apt-get update && \ 
		# GekkoFS dependencies

		libboost-program-options-dev \

		uuid-dev && \

    # install cmake 3.14 since it's needed for some dependencies

    curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \

    chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \

    ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \

    # install cmake 3.14+ since it's needed for some dependencies

    curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \

    chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \

    ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \

    # Clean apt cache to reduce image layer size

    rm -rf /var/lib/apt/lists/* && \

    # Clean apt caches of packages

    apt-get clean && apt-get autoclean && \

    rm ./cmake-3.14.5-Linux-x86_64.sh
 
    rm ./cmake-3.25.2-Linux-x86_64.sh

docker/0.9.2/coverage/Dockerfile

deleted100644 → 0
+0 −20
Changes for docker/0.9.2/coverage/Dockerfile: 0 added lines, 20 removed lines.
Original line number Diff line number Diff line 
FROM debian:bullseye-slim



LABEL Description="Environment to generate coverage reports in GekkoFS"



RUN apt-get update && \

    apt-get install -y --no-install-recommends \

        wget \

        git \

        cmake \

        gcc \

        g++ \

        lcov \

        python3 \

        python3-pip \

        python3-setuptools && \

    rm -rf /var/lib/apt/lists/* && \

    apt-get clean && \

    apt-get autoclean && \

    python3 -m pip install --upgrade pip && \

    pip3 install gcovr