Commit 5aba378d authored by Alberto Miranda's avatar Alberto Miranda ♨️
Browse files

Merge branch 'amiranda/6-automatically-run-tests-in-pipelines' into 'main' 

Resolve "Automatically run tests in  pipelines"

This MR enables automatic testing using `ctest`, and also 
updates the CI pipelines so that they run `build` and `test`
jobs exercising the code. We also add a `Dockerfile` that builds
an appropriate CI environment to build, test, and run Cargo.

Closes #6

Closes #6

See merge request !8
parents a3de90e3 30128df1

.gitlab-ci.yml

+65 −9
Original line number Original line Diff line number Diff line 
# You can override the included template(s) by including variable overrides

image: bscstorage/cargo:0.2.0-wip

# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings



# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings

# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings

# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings

# Note that environment variables can be set in several places

# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence

stages:

stages:

  - build

  - test

  - test



variables:

  PREFIX:

    /usr/local

  LD_LIBRARY_PATH: >-

    /usr/lib/:

    /usr/lib64:

    /usr/local/lib:

    /usr/local/lib64

  PKG_CONFIG_PATH: >-

    /usr/lib/pkgconfig:

    /usr/lib64/pkgconfig:

    /usr/local/lib/pkgconfig:

    /usr/local/lib64/pkgconfig:

    /usr/lib64/openmpi/lib/pkgconfig



before_script:

  - source /etc/profile.d/modules.sh

  - module load mpi



release:

  stage: build

  script:

    - cmake --preset ci-release

    - cmake --build builds/ci-release -j$(nproc) --target install



debug:

  stage: build

  script:

    - cmake --preset ci-debug

    - cmake --build builds/ci-debug -j$(nproc) --target install

    - cd builds/ci-debug

    # cleanup intermediate files to save on artifact space

    - grep "^rule.*\(_COMPILER_\|_STATIC_LIBRARY_\)"

        $(find . -name rules.ninja) |

          cut -d ' ' -f2 |

          xargs -n1 ninja -t clean -r

  artifacts:

    expire_in: 2 days

    paths:

      - builds/ci-debug



integration:

  stage: test

  needs: [ debug ]

  variables:

    OMPI_ALLOW_RUN_AS_ROOT: "1"

    OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1"

  script:

    - ctest --test-dir builds/ci-debug -j$(nproc) --output-junit report.xml



  artifacts:

    expire_in: 1 week

    paths:

      - builds/ci-debug/Testing/Temporary

    reports:

      junit: builds/ci-debug/report.xml



sast:

sast:

  stage: test

  stage: test

  before_script: []

  needs: []

include:

include:

  - template: Security/SAST.gitlab-ci.yml
 
  - template: Security/SAST.gitlab-ci.yml

CMakeLists.txt

+3 −2
Original line number Original line Diff line number Diff line
@@ -141,9 +141,9 @@ message(STATUS "[${PROJECT_NAME}] server bind address: ${CARGO_BIND_ADDRESS}") 




### server bind port

### server bind port

set(CARGO_BIND_PORT

set(CARGO_BIND_PORT

  "52000"

  "62000"

  CACHE STRING

  CACHE STRING

  "Define the bind port for the ${PROJECT_NAME} server (default: 52000)"

  "Define the bind port for the ${PROJECT_NAME} server (default: 62000)"

  )

  )

message(STATUS "[${PROJECT_NAME}] server bind port: ${CARGO_BIND_PORT}")

message(STATUS "[${PROJECT_NAME}] server bind port: ${CARGO_BIND_PORT}")
@@ -322,6 +322,7 @@ endif () 
add_subdirectory(etc)

add_subdirectory(etc)

add_subdirectory(lib)

add_subdirectory(lib)

add_subdirectory(src)

add_subdirectory(src)

add_subdirectory(util)





if(CARGO_BUILD_TESTS)

if(CARGO_BUILD_TESTS)

  add_subdirectory(tests)

  add_subdirectory(tests)

CMakePresets.json

0 → 100644
+203 −0
Original line number Original line Diff line number Diff line 
{

  "version": 2,

  "cmakeMinimumRequired": {

    "major": 3,

    "minor": 19,

    "patch": 0

  },

  "configurePresets": [

    {

      "name": "base",

      "displayName": "GCC",

      "description": "Sets prefix, build, and install directories as well as common options",

      "hidden": true,

      "generator": "Ninja",

      "binaryDir": "${sourceDir}/builds/${presetName}",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER_LAUNCHER": "/usr/bin/ccache",

        "CMAKE_C_COMPILER_LAUNCHER": "/usr/bin/ccache",

        "CMAKE_PREFIX_PATH": "${sourceParentDir}/prefix",

        "CMAKE_INSTALL_PREFIX": "${sourceParentDir}/prefix",

        "CARGO_BUILD_TESTS": true

      }

    },

    {

      "name": "debug",

      "displayName": "Debug",

      "description": "Build options for Debug",

      "hidden": true,

      "cacheVariables": {

        "CMAKE_BUILD_TYPE": "Debug",

        "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always --pedantic"

      }

    },

    {

      "name": "release",

      "displayName": "Releae",

      "description": "Build options for Release",

      "hidden": true,

      "cacheVariables": {

        "CMAKE_BUILD_TYPE": "Release"

      }

    },

    {

      "name": "gcc",

      "displayName": "GCC (system default)",

      "description": "Build options for GCC (system default)",

      "inherits": "base",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER": "/usr/bin/g++",

        "CMAKE_C_COMPILER": "/usr/bin/gcc",

        "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always",

        "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always"

      }

    },

    {

      "name": "gcc-debug",

      "displayName": "GCC (system default, debug)",

      "description": "Build options for GCC (system default)",

      "inherits": "gcc",

      "cacheVariables": {

        "CMAKE_BUILD_TYPE": "Debug",

        "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always -O0"

      }

    },

    {

      "name": "gcc-10",

      "displayName": "GCC 10",

      "description": "Build options for GCC 10",

      "inherits": "base",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER": "/usr/bin/g++-10",

        "CMAKE_C_COMPILER": "/usr/bin/gcc-10",

        "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always",

        "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always"

      }

    },

    {

      "name": "gcc-11",

      "displayName": "GCC 11",

      "description": "Build options for GCC 11",

      "inherits": "base",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER": "/usr/bin/g++-11",

        "CMAKE_C_COMPILER": "/usr/bin/gcc-11",

        "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always",

        "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always"

      }

    },

    {

      "name": "clang",

      "displayName": "Clang (system default)",

      "description": "Build options for Clang (system default)",

      "inherits": "base",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER": "/usr/bin/clang++",

        "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always",

        "CMAKE_C_COMPILER": "/usr/bin/clang",

        "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always"

      }

    },

    {

      "name": "clang-10",

      "displayName": "Clang 10",

      "description": "Build options for Clang 10",

      "inherits": "base",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER": "/usr/bin/clang++-10",

        "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always",

        "CMAKE_C_COMPILER": "/usr/bin/clang-10",

        "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always"

      }

    },

    {

      "name": "clang-11",

      "displayName": "Clang 11",

      "description": "Build options for Clang 11",

      "inherits": "base",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER": "/usr/bin/clang++-11",

        "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always",

        "CMAKE_C_COMPILER": "/usr/bin/clang-11",

        "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always"

      }

    },

    {

      "name": "clang-12",

      "displayName": "Clang 12",

      "description": "Build options for Clang 12",

      "inherits": "base",

      "cacheVariables": {

        "CMAKE_CXX_COMPILER": "/usr/bin/clang++-12",

        "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always",

        "CMAKE_C_COMPILER": "/usr/bin/clang-12",

        "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always"

      }

    },

    {

      "name": "ci",

      "displayName": "CI",

      "description": "Build options for CI",

      "inherits": "base",

      "hidden": true,

      "cacheVariables": {

        "CMAKE_CXX_COMPILER_LAUNCHER": null,

        "CMAKE_C_COMPILER_LAUNCHER": null,

        "CMAKE_C_COMPILER": "/usr/bin/gcc",

        "CMAKE_CXX_COMPILER": "/usr/bin/g++",

        "CMAKE_INSTALL_PREFIX": "/usr/local",

        "CMAKE_PREFIX_PATH": "/usr/local",

        "Boost_LIBRARY_DIR": "/usr/lib;/usr/lib64;/usr/lib64/openmpi/lib",

        "CARGO_TRANSPORT_LIBRARY": "libfabric",

        "CARGO_TRANSPORT_PROTOCOL": "ofi+tcp",

        "CARGO_BIND_ADDRESS": "127.0.0.1",

        "CARGO_BIND_PORT": "62000",

        "CARGO_BUILD_TESTS": true

      }

    },

    {

      "name": "ci-debug",

        "displayName": "CI (debug)",

        "description": "Build options for CI (debug)",

        "inherits": ["ci", "debug"]

    },

    {

      "name": "ci-release",

      "displayName": "CI (debug)",

      "description": "Build options for CI (debug)",

      "inherits": ["ci", "release"]

    }

  ],

  "buildPresets": [

    {

      "name": "core-build",

      "description": "Inherits environment from base configurePreset",

      "configurePreset": "base",

      "hidden": true

    },

    {

      "name": "gcc",

      "description": "Build with default GCC",

      "configurePreset": "gcc",

      "inherits": "core-build"

    },

    {

      "name": "gcc-11",

      "description": "Build with GCC 11",

      "configurePreset": "gcc-11",

      "inherits": "core-build"

    },

    {

      "name": "clang",

      "description": "Build with default Clang",

      "configurePreset": "clang",

      "inherits": "core-build"

    },

    {

      "name": "clang-10",

      "description": "Build with Clang 10",

      "configurePreset": "clang-10",

      "inherits": "core-build"

    }

  ]

}

README.md

+32 −4
Original line number Original line Diff line number Diff line 
# Cargo

<div align="center">

<h1> Cargo </h1>



[![build status)](https://img.shields.io/gitlab/pipeline-status/hpc/cargo?gitlab_url=https%3A%2F%2Fstorage.bsc.es%2Fgitlab%2F&logo=gitlab)](https://storage.bsc.es/gitlab/hpc/cargo/-/pipelines)

[![latest release](https://storage.bsc.es/gitlab/hpc/cargo/-/badges/release.svg)](https://storage.bsc.es/gitlab/hpc/cargo/-/releases)

[![GitLab (self-managed)](https://img.shields.io/gitlab/license/hpc/cargo?gitlab_url=https%3A%2F%2Fstorage.bsc.es%2Fgitlab)](https://storage.bsc.es/gitlab/hpc/cargo/-/blob/main/COPYING)

[![Language](https://img.shields.io/static/v1?label=language&message=C99%20%2F%20C%2B%2B20&color=red)](https://en.wikipedia.org/wiki/C%2B%2B20)



<p><b>A parallel data staging service for HPC clusters</b></p>



</div>





Cargo is a HPC data staging service that runs alongside applications helping 

Cargo is a HPC data staging service that runs alongside applications helping 

them to transfer data in parallel between local and shared storage tiers.

them to transfer data in parallel between local and shared storage tiers.
@@ -137,8 +147,26 @@ library (`${INSTALL_DIR}/lib/libcargo.so`) and its headers 




## Testing

## Testing





Tests can be run automatically with CTest:



```shell

cd build

ctest -VV --output-on-failure --stop-on-failure -j 8

```



When this happens, a Cargo server with 3 workers is automatically started

(via `mpirun`/`mpiexec`) and stopped (via RPC) so that tests can progress.



Alternatively, during development one may desire to run the Cargo server

manually and then the tests. In this case, the following commands can be used:



```shell

```shell

cd build/tests/

# start the Cargo server with 3 workers. The server will be listening on

mpirun -np 4 ${INSTALL_DIR}/bin/cargo -C

# port 62000 and will communicate with workers via MPI messages. The server can

./tests -S ofi+tcp://127.0.0.1:52000

# be stopped with Ctrl+C, `kill -TERM <pid>` or `cargo_shutdown <address>`.)

mpirun -np 4 ${INSTALL_DIR}/bin/cargo -l ofi+tcp://127.0.0.1:62000



# run the tests

cd build

RUNNER_SKIP_START=1 ctest -VV --output-on-failure --stop-on-failure -j 8

```
 
```

docker/0.2.0-wip/Dockerfile

0 → 100644
+133 −0
Original line number Original line Diff line number Diff line 
FROM rockylinux:9.2



RUN set -ex \

    && yum makecache \

    && yum -y update \

    && yum -y install dnf-plugins-core \

    && yum config-manager --set-enabled crb \

    && yum -y install \

           gcc \

           gcc-c++\

           gdb \

           git \

           gnupg \

           make \

           automake \

           libtool \

           file \

           ninja-build \

           json-c-devel \

           libibverbs-devel \

           boost-devel \

           boost-openmpi-devel \

           json-c-devel \

           openmpi-devel \

           libconfig-devel \

    # install cmake 3.21+ since we need to produce JUnit XML files

    && curl -OL https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-Linux-x86_64.sh \

    && chmod u+x ./cmake-3.27.6-Linux-x86_64.sh \

    && ./cmake-3.27.6-Linux-x86_64.sh --skip-license --prefix=/usr \

    # cleanup

    && yum clean all \

    && rm -rf /var/cache/yum \

    && rm ./cmake-3.27.6-Linux-x86_64.sh



# Download and install dependencies

RUN set -ex \

    && export LD_LIBRARY_PATH=${DEPS_INSTALL_PATH}/lib:${DEPS_INSTALL_PATH}/lib64 \

    && export PKG_CONFIG_PATH=${DEPS_INSTALL_PATH}/lib/pkgconfig:${DEPS_INSTALL_PATH}/lib64/pkgconfig \

    && cd  \

    && mkdir deps  \

    && cd deps \

    && git clone https://github.com/ofiwg/libfabric --recurse-submodules \

    && git clone https://github.com/pmodels/argobots --recurse-submodules \

    && git clone https://github.com/mercury-hpc/mercury --recurse-submodules \

    && git clone https://github.com/mochi-hpc/mochi-margo --recurse-submodules \

    && git clone https://github.com/USCiLab/cereal --recurse-submodules \

    && git clone https://github.com/mochi-hpc/mochi-thallium --recurse-submodules \

    \

    && cd \

    ### argobots

    && cd deps/argobots \

    && ./autogen.sh \

    && mkdir build  \

    && cd build \

    && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \

    && make install -j \

    && cd ..  \

    && rm -rf build  \

    && cd \

    \

    ### libfabric

    && cd deps/libfabric \

    && git checkout v1.14.0rc3 \

    && ./autogen.sh \

    && mkdir build  \

    && cd build \

    && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \

    && make install -j \

    && cd ..  \

    && rm -rf build  \

    && cd \

    \

    ### mercury

    && cd deps/mercury  \

    && mkdir build && cd build \

    && cmake  \

          -DMERCURY_USE_SELF_FORWARD:BOOL=ON \

          -DBUILD_TESTING:BOOL=ON \

          -DMERCURY_USE_BOOST_PP:BOOL=ON \

          -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH}  \

          -DBUILD_SHARED_LIBS:BOOL=ON \

          -DNA_USE_OFI:BOOL=ON  \

          -DCMAKE_POSITION_INDEPENDENT_CODE=ON \

          -DCMAKE_BUILD_TYPE:STRING=Debug  \

          ..  \

    && make install -j \

    && cd ..  \

    && rm -rf build  \

    && cd \

    \

    ### mochi-margo

    && cd deps/mochi-margo \

    && ./prepare.sh \

    && mkdir build  \

    && cd build \

    && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \

    && make -j install \

    && cd ..  \

    && rm -rf build  \

    && cd \

    \

    ### cereal

    && cd deps/cereal \

    && mkdir build  \

    && cd build  \

    \

    && cmake  \

          -DCMAKE_BUILD_TYPE:STRING=Debug  \

          -DBUILD_DOC:BOOL=OFF \

          -DBUILD_SANDBOX:BOOL=OFF  \

          -DBUILD_TESTS:BOOL=OFF \

          -DSKIP_PERFORMANCE_COMPARISON:BOOL=ON \

          -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          .. \

    && make -j install \

    && cd ..  \

    && rm -rf build  \

    && cd \

    \

    ### mochi-thallium

    && cd deps/mochi-thallium \

    && mkdir build  \

    && cd build  \

    && cmake  \

          -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH}  \

          -DCMAKE_BUILD_TYPE:STRING=Debug  \

          .. \

    && make -j install  \

    && cd ..  \

    && rm -rf build  \

    && cd  \

    \

    && rm -rf deps