Commit b74dc17a authored by Ramon Nou's avatar Ramon Nou
Browse files

Resolve "Optimize CI pipeline" 

This MR optimizes the CI pipelines by doing the following:

- [x] Enable out-of-order execution of jobs using the `needs` clause
- [x] Split tests for better parallelization
  - [x] Evaluate the `parallel` keyword
  - [x] Evaluate the `interruptible` keyword
- [x] Reduce artifact size by removing unneeded object files between stages
- [x] Refactor Docker images and make them versioned
  - [x] Create a `gekkofs/core` image for system dependencies
  - [x] Create a `gekkofs/deps` image for downloaded dependencies
  - [x] Create a `gekkofs/linter` image for validation of formatting
  - [x] Create a `gekkofs/testing` image for running tests
  - [x] Create a `gekkofs/coverage` image to generate coverage reports
  - [x] Add configuration profiles to `dl_dep.sh` and `compile_dep.sh`

It also improves the reporting of tests by integrating it with the GitLab UI. This required the following changes:

- [x] Enable JUnit reporting of tests
  - [x] Enable JUnit reporting for `integration` tests
    - [x] Fix reported filename for GitLab integration
    - [x] ~~Integrate daemon and client logs into JUnit XML~~
  - [x] Enable JUnit reporting for `unit` tests

Closes #165 #157 #166 #167

See merge request !103
parents fa3eea49 4e960645

.gitlab-ci.yml

+141 −110
Original line number Diff line number Diff line 
stages:

  - check format

  - download deps

  - build deps

  - lint

  - build

  - test

  - report



variables:

  DEPS_SRC_PATH:                "${CI_PROJECT_DIR}/deps/src"

  DEPS_INSTALL_PATH:            "${CI_PROJECT_DIR}/deps/install"

  DEPS_COMMIT:                  "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit"

  SCRIPTS_DIR:                  "${CI_PROJECT_DIR}/scripts"

  CI_SCRIPTS_DIR:               "${CI_PROJECT_DIR}/scripts/ci"

  BUILD_PATH:                   "${CI_PROJECT_DIR}/gkfs/build"

  INSTALL_PATH:                 "${CI_PROJECT_DIR}/gkfs/install"

  INTEGRATION_TESTS_BIN_PATH:   "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration"

  INTEGRATION_TESTS_RUN_PATH:   "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run"

  TESTS_BUILD_PATH:             "${CI_PROJECT_DIR}/test/build"

  COVERAGE_PATH:                "${CI_PROJECT_DIR}/gkfs/build/.coverage"

  PYTEST:                       "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test"

  LOG_PATH:                     "${CI_PROJECT_DIR}/logs"

  LD_LIBRARY_PATH:              "${CI_PROJECT_DIR}/deps/install/lib:${CI_PROJECT_DIR}/deps/install/lib64"

  # Configuration variables

  GKFS_LOG_LEVEL:               "100"
@@ -25,36 +20,55 @@ variables: 
  LIBGKFS_LOG_OUTPUT:           "${CI_PROJECT_DIR}/logs/gkfs_client.log"

  GIT_SUBMODULE_STRATEGY:       recursive



image: gekkofs/gekkofs:build_env-0.8.0

# base image

image: gekkofs/core:0.8.0



################################################################################

## Validating

################################################################################

check format:

  stage: check format

  stage: lint

  image: gekkofs/linter:0.8.0

  needs: []

  script:

    - ${CI_PROJECT_DIR}/scripts/check_format.sh -s "${CI_PROJECT_DIR}/src" -i "${CI_PROJECT_DIR}/include" -v

    - ${SCRIPTS_DIR}/check_format.sh

      -s "${CI_PROJECT_DIR}/src"

      -i "${CI_PROJECT_DIR}/include"

      -v



compile dependencies:

  stage: build deps

  cache:

    key: deps-cache

    paths:

     - ${DEPS_INSTALL_PATH}/



################################################################################

## Building

################################################################################

gkfs:

  stage: build

  image: gekkofs/deps:0.8.0

  interruptible: true

  needs: []

  script:

   # Folder of built dependencies is cached and marked with the ID of the commit from which have been built.

   # If the download and compile script have been modified the cache gets invalidated and dependencies will be built again.

   - ( [ -f "${DEPS_COMMIT}" ] && git diff --quiet "`cat ${DEPS_COMMIT}`" -- scripts/dl_dep.sh scripts/compile_dep.sh ) || (

           rm -f ${DEPS_COMMIT} &&

           scripts/dl_dep.sh ${DEPS_SRC_PATH} -n ofi -c ci &&

           scripts/compile_dep.sh -n ofi -c ci ${DEPS_SRC_PATH} ${DEPS_INSTALL_PATH} &&

           echo "${CI_COMMIT_SHA}" > "${DEPS_COMMIT}"

     )

    - mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}

    - cmake

      -Wdev

      -Wdeprecate

      -DCMAKE_BUILD_TYPE=Debug

      -DGKFS_ENABLE_CODE_COVERAGE:BOOL=ON

      -DGKFS_BUILD_TESTS:BOOL=ON

      -DGKFS_INSTALL_TESTS:BOOL=ON

      -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}

      ${CI_PROJECT_DIR}

    - make -j$(nproc) install

    # reduce artifacts size

    - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}

  artifacts:

    paths:

     - ${DEPS_INSTALL_PATH}

      - ${BUILD_PATH}

      - ${INSTALL_PATH}



compile GekkoFS:

gkfwd:

  stage: build

  dependencies:

    - "compile dependencies"

  image: gekkofs/deps:0.8.0

  interruptible: true

  needs: []

  script:

    - mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}

    - cmake
@@ -66,132 +80,149 @@ compile GekkoFS: 
      -DGKFS_INSTALL_TESTS:BOOL=ON

      -DGKFS_ENABLE_FORWARDING:BOOL=ON

      -DGKFS_ENABLE_AGIOS:BOOL=ON

      -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH}

      -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}

      ${CI_PROJECT_DIR}

    - make -j$(nproc) install

    - find ${BUILD_PATH} -name "*.o" -delete

    # cleanup nlohmann_json: it includes around 500MiB of test data

    - rm -rf ${BUILD_PATH}/_deps/nlohmann_json-src/test/data

    - rm -rf ${BUILD_PATH}/_deps/nlohmann_json-src/benchmarks/data

    - rm -rf ${BUILD_PATH}/_deps/nlohmann_json-src/.git

    # reduce artifacts size

    - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}

  artifacts:

    paths:

      - ${BUILD_PATH}

      - ${INSTALL_PATH}

    exclude:

      # remove object files

      - ${BUILD_PATH}/**/*.o



compile tests:

  stage: build

  dependencies:

    - "compile dependencies"

  script:

    - mkdir -p ${TESTS_BUILD_PATH} && cd ${TESTS_BUILD_PATH}

    - cmake -DCMAKE_BUILD_TYPE=Debug ..

    - make -j$(nproc)

  artifacts:

    paths:

     - ${TESTS_BUILD_PATH}



integration tests:

################################################################################

## Testing

################################################################################



## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

  image: gekkofs/testing:0.8.0

  interruptible: true

  needs: ['gkfs']

  parallel:

    matrix:

      - SUBTEST: [ data, directories, operations, position, shell, status ]



  script:

    ## run tests

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - unbuffer ${PYTEST} -v | tee session.log

    - ${PYTEST} -v -n $(nproc)

          ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}

          --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}

          --junit-xml=report.xml



    ## capture coverage information

    - cd ${BUILD_PATH}

    - ${CI_PROJECT_DIR}/scripts/ci/coverage.sh

          --capture integration

    - ${CI_SCRIPTS_DIR}/coverage.sh

          --verbose

          --capture integration_${SUBTEST}

          --root-dir ${CI_PROJECT_DIR}

          --build-dir ${BUILD_PATH}

          --exclusions "${CI_PROJECT_DIR}/scripts/ci/.coverage-exclusions"

          --exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"

          --log-file "${COVERAGE_PATH}/partial/integration_${SUBTEST}/capture.log"



  # fix relative paths so that GitLab can find the correct files

  after_script:

    - perl -i.orig

          -pe 's%file="(.*?)"%file="tests/integration/$1"%;'

          -pe 's%(../)+install/share/gkfs/%%g;'

          ${BUILD_PATH}/tests/integration/report.xml



  artifacts:

    #    when: on_failure

    expire_in: 1 week

    when: always

    paths:

      #      - "${INTEGRATION_TESTS_RUN_PATH}"

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/integration/report.xml





unit:

## == integration tests for gkfwd ==========

gkfwd:integration:

  stage: test

  image: gekkofs/testing:0.8.0

  interruptible: true

  needs: ['gkfwd']

  parallel:

    matrix:

      - SUBTEST: [ forwarding ]



  script:

    ## run actual tests

    - cd ${BUILD_PATH}

    - ctest -j $(nproc) -L unit::all

    ## run tests

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

          ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}

          --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}

          --junit-xml=report.xml



    ## capture coverage information

    - ${CI_PROJECT_DIR}/scripts/ci/coverage.sh

          --capture unit

    - cd ${BUILD_PATH}

    - ${CI_SCRIPTS_DIR}/coverage.sh

          --verbose

          --capture integration_${SUBTEST}

          --root-dir ${CI_PROJECT_DIR}

          --build-dir ${BUILD_PATH}

          --exclusions "${CI_PROJECT_DIR}/scripts/ci/.coverage-exclusions"

  artifacts:

    #    when: on_failure

    paths:

      - ${BUILD_PATH}

        #      - Testing

          --exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"

          --log-file "${COVERAGE_PATH}/partial/integration_${SUBTEST}/capture.log"



test wr:

  stage: test

  script:

    - mkdir -p "${LOG_PATH}"

    - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &

    - sleep 4

    - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_wr

  artifacts:

    when: on_failure

    paths:

     - "${LOG_PATH}"

  # fix relative paths so that GitLab can find the correct files

  after_script:

    - perl -i.orig

          -pe 's%file="(.*?)"%file="tests/integration/$1"%;'

          -pe 's%(../)+install/share/gkfs/%%g;'

          ${BUILD_PATH}/tests/integration/report.xml



test directories:

  stage: test

  script:

    - mkdir -p "${LOG_PATH}"

    - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &

    - sleep 4

    - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_dir

  artifacts:

    when: on_failure

    expire_in: 1 week

    when: always

    paths:

     - "${LOG_PATH}"

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/integration/report.xml



test truncate:

  stage: test

  script:

    - mkdir -p "${LOG_PATH}"

    - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &

    - sleep 4

    - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_truncate

  artifacts:

    when: on_failure

    paths:

     - "${LOG_PATH}"



test lseek:

## == unit tests for gkfs ==================

gkfs:unit:

  stage: test

  image: gekkofs/testing:0.8.0

  needs: ['gkfs']

  script:

    - mkdir -p "${LOG_PATH}"

    - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &

    - sleep 4

    - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_lseek

    ## run actual tests

    - cd ${BUILD_PATH}/tests/unit

    - ctest -j $(nproc) -L unit::all --output-junit report.xml



    ## capture coverage information

    - ${CI_SCRIPTS_DIR}/coverage.sh

          --verbose

          --capture unit

          --root-dir ${CI_PROJECT_DIR}

          --build-dir ${BUILD_PATH}

          --exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"

          --log-file "${COVERAGE_PATH}/partial/unit/capture.log"

  artifacts:

    when: on_failure

    expire_in: 1 week

    paths:

     - "${LOG_PATH}"

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/unit/report.xml





################################################################################

## Generation of code coverage reports

################################################################################



coverage:

  stage: report

  image: gekkofs/coverage

  image: gekkofs/coverage:0.8.0

  needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]

  script:

    - cd ${BUILD_PATH}

    ## merge the partial coverage files from each test in the pipeline

    - ${CI_PROJECT_DIR}/scripts/ci/coverage.sh

    - ${CI_SCRIPTS_DIR}/coverage.sh

          --verbose

          --merge

          --root-dir ${CI_PROJECT_DIR}

          --build-dir ${BUILD_PATH}

CMakeLists.txt

+1 −1
Original line number Diff line number Diff line
@@ -280,7 +280,7 @@ if (GKFS_BUILD_TESTS) 
    message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")



    message(STATUS "[gekkofs] Check for forwarding tests...")

    if (ENABLE_FORWARDING)

    if (GKFS_ENABLE_FORWARDING)

        set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")

    else ()

        set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")

docker/debian_build_env.docker→docker/0.8.0/core/Dockerfile

+37 −0
Original line number Diff line number Diff line 
FROM debian:stable-slim



LABEL Description="This is a debian based environment to build GekkoFS"



ENV GKFS_PATH	/opt/gkfs



ENV SCRIPTS_PATH	${GKFS_PATH}/scripts

ENV DEPS_SRC_PATH	${GKFS_PATH}/deps_src

ENV INSTALL_PATH	${GKFS_PATH}/build_deps

LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"



RUN apt-get update && \

    apt-get install -y --no-install-recommends \
@@ -25,7 +19,7 @@ RUN apt-get update && \ 
		# Mercury dependencies

		libltdl-dev \

		lbzip2 \

		# RocksDB

		# RocksDB dependencies

		libsnappy-dev \

		liblz4-dev \

		libzstd-dev \
@@ -33,41 +27,11 @@ RUN apt-get update && \ 
		zlib1g-dev \

		# syscall_intercept dependencies

		libcapstone-dev \

		# GekkoFS

		# GekkoFS dependencies

		libboost-filesystem-dev \

		libboost-program-options-dev \

		valgrind \

		uuid-dev \

		python3 \

		python3-pip \

		python3-dev \

		python3-venv \

		python3-setuptools \

		expect \

		# clang 10 deps

		lsb-release \

		wget \

		software-properties-common \

		gnupg2 \

    # add clang-10 repos

    wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 10 && \

    # install clang-format

    apt-get update && apt-get install -y --no-install-recommends clang-format-10 && \

    # install gcovr

    # (required for partial coverage reports in parallel runs)

    python3 -m pip install --upgrade pip && \

    pip3 install gcovr && \

		uuid-dev && \

    # Clean apt cache to reduce image layer size

    rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \

    rm -rf /var/lib/apt/lists/* && \

    # Clean apt caches of packages

    apt-get clean && apt-get autoclean
 


## COPY scripts/dl_dep.sh		$SCRIPTS_PATH/

## COPY scripts/compile_dep.sh $SCRIPTS_PATH/

## COPY scripts/patches        $SCRIPTS_PATH/patches

## 

## # Download dependencies source

## RUN /bin/bash $SCRIPTS_PATH/dl_dep.sh $DEPS_SRC_PATH all

## 

## # Compile dependencies

## RUN /bin/bash $SCRIPTS_PATH/compile_dep.sh $DEPS_SRC_PATH $INSTALL_PATH

docker/0.8.0/core/Makefile

0 → 100644
+4 −0
Original line number Diff line number Diff line 
.PHONY: all



all:

	docker build -t gekkofs/core:0.8.0 .

docker/0.8.0/coverage/Dockerfile

0 → 100644
+20 −0
Original line number Diff line number Diff line 
FROM debian:stable-slim



LABEL Description="Environment to generate coverage reports in GekkoFS"



RUN apt-get update && \

    apt-get install -y --no-install-recommends \

        wget \

        git \

        cmake \

        gcc \

        g++ \

        lcov \

        python3 \

        python3-pip \

        python3-setuptools && \

    rm -rf /var/lib/apt/lists/* && \

    apt-get clean && \

    apt-get autoclean && \

    python3 -m pip install --upgrade pip && \

    pip3 install gcovr