diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2384ecb8b9c0de52e702b306098b3e3260107d12 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,144 @@ +# NORNS +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) +[![Build Status](https://travis-ci.com/bsc-ssrg/NORNS.svg?branch=master)](https://travis-ci.com/bsc-ssrg/NORNS) +[![Coverage Status](https://codecov.io/gh/bsc-ssrg/NORNS/branch/master/graph/badge.svg)](https://codecov.io/gh/bsc-ssrg/NORNS) + +NORNS is an open-source data scheduling service that orchestrates asynchronous +data transfers between different storage backends in an HPC cluster. Through +its API, NORNS provides three different functions. First, it allows system +administrators to expose the storage architecture of an HPC cluster by creating +**dataspaces** associated to different storage backends such as node-local NVMs +and POSIX file systems, or system-wide parallel file systems and object stores, +thus making them available to applications, services, and users. +Second, it provides a framework for submitting and monitoring the asynchronous +transfers of **data resources** between the (local and remote) dataspaces +available to a user, such as process buffers, POSIX files and directories or +objects. Third, it arbitrates requests by managing a queue of pending work and +evaluating requests to maximize dataspace throughput while minimizing the +interferences with normal application I/O. + +NORNS has currently been tested only under GNU/Linux. + +## Building and installing from source + +Distribution tarballs are available from the +[releases](https://github.com/bsc-ssrg/NORNS/releases) tab. If you are building +NORNS from a developer Git clone, you must first run the `bootstrap.sh` script, +which will invoke the GNU Autotools to bootstrap NORNS' configuration and build +mechanisms. If you are building NORNS from an official distribiution tarball, +there is no need to run the `bootstrap.sh` script, since all distribution +tarballs are already boostrapped. + +### Dependencies + +Compiling and running NORNS requires up-to-date versions of the following +software packages (note that, though it may compile and run, using excessively +old versions of these packages can cause indirect errors that are very +difficult to track down): + +- A standard **C++11** conforming compiler (the code is routinely tested + against GCC 4.9/Clang 3.3 and higher). +- Autotools (autoconf 2.69 or higher, automake 1.14.1 or higher, and libtool + 2.4.2 or higher) and CMake (3.10.0 or higher). +- The following Boost libraries (1.53 or higher): `system`, `filesystem`, + `program_options`, and `thread`. Optionally, the `regex` library may also be + required if self tests are enabled with the `--enable-tests` option. +- Google's Protocol Buffers for [C](https://github.com/protobuf-c/protobuf-c) + (1.0.2 or higher) and for [C++](https://github.com/protocolbuffers/protobuf) + (2.5.0 or higher). +- [LibYAML](https://github.com/yaml/libyaml) (0.1.4 or higher) and + [yaml-cpp](https://github.com/jbeder/yaml-cpp) (0.5.1 or higher). +- [Mercury](https://github.com/mercury-hpc/mercury) 1.0 or higher + (**IMPORTANT** Mercury may require additional dependencies such as libfabric + depending on the desired transport protocol). +- [Hermes](https://github.com/bsc-ssrg/hermes) (our own C++ wrapper for + Mercury. It should be automatically downloaded when cloning with the + `--recursive` option). +- [libarchive](https://github.com/libarchive/libarchive) (3.1.2 or higher). + +#### Installation in CentOS + +```bash +# Installing dependencies available through package manager +$ yum install -y boost-devel protobuf protobuf-devel protobuf-c protobuf-c-devel \ + yaml-cpp yaml-cpp-devel libyaml-devel libarchive + +# Building and installing libfabric (required for Mercury's OFI/libfabric plugin) +$ git clone https://github.com/ofiwg/libfabric.git && cd libfabric +$ ./autogen.sh +$ mkdir build && cd build +$ ../configure && make && make install + +# Building and installing Mercury with OFI/libfabric plugin +$ git clone https://github.com/mercury-hpc/mercury.git && cd mercury +$ mkdir build && cd build +$ cmake -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_TESTING:BOOL=OFF \ + -DMERCURY_USE_SM_ROUTING:BOOL=OFF -DMERCURY_USE_SELF_FORWARD:BOOL=OFF \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DMERCURY_USE_EAGER_BULK:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON \ + -DNA_USE_OFI:BOOL=ON \ + .. +$ make && make install + +# Building, testing and installing NORNS under '/usr/local/', with configuration +# files under '/etc/norns/' and temporary files under '/var/run/norns/' +$ git clone --recursive https://github.com/bsc-ssrg/NORNS.git && cd norns +$ ./bootstrap.sh +$ mkdir build && cd build +$ ../configure \ + --enable-tests \ + --prefix=/usr/local \ + --sysconfdir=/etc/norns \ + --localstatedir=/var/run/norns +$ make && make check && make install + +``` + +#### Installation in Ubuntu + +```bash +# Installing dependencies available through package manager +$ apt-get install -y libboost-system-dev libboost-filesystem-dev \ + libboost-program-options-dev libboost-thread-dev \ + libboost-regex-dev libprotobuf-dev protobuf-compiler \ + libprotobuf-c-dev protobuf-c-compiler \ + libyaml-cpp-dev libyaml-dev libarchive-dev + +# Building and installing libfabric (required for Mercury's OFI/libfabric plugin) +$ git clone https://github.com/ofiwg/libfabric.git && cd libfabric +$ ./autogen.sh +$ mkdir build && cd build +$ ../configure && make && make install + +# Building and installing Mercury with OFI/libfabric plugin +$ git clone https://github.com/mercury-hpc/mercury.git && cd mercury +$ mkdir build && cd build +$ cmake -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_TESTING:BOOL=OFF \ + -DMERCURY_USE_SM_ROUTING:BOOL=OFF -DMERCURY_USE_SELF_FORWARD:BOOL=OFF \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DMERCURY_USE_EAGER_BULK:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON \ + -DNA_USE_OFI:BOOL=ON \ + .. +$ make && make install + +# Building, testing and installing NORNS under '/usr/local/', with configuration +# files under '/etc/norns/' and temporary files under '/var/run/norns/' +$ git clone --recursive https://github.com/bsc-ssrg/NORNS.git && cd norns +$ ./bootstrap.sh +$ mkdir build && cd build +$ ../configure \ + --enable-tests \ + --prefix=/usr/local \ + --sysconfdir=/etc/norns \ + --localstatedir=/var/run/norns +$ make && make check && make install +``` + + +#### Configuration + +```bash +# Optional: providing file system permission override capabilities to NORNS +# control daemon +$ setcap cap_sys_ptrace,cap_chown=+ep /usr/local/bin/urd +``` diff --git a/.gitmodules b/.gitmodules index f1ecde20ccf4b5df7bfa3fbb8c23e8d6e30f18eb..4f6f7d00da1468e7ca593e4f9873c8797480d4c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/externals/hermes"] path = src/externals/hermes - url = ../hermes.git + url = https://github.com/bsc-ssrg/hermes.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000000000000000000000000000000000..0780be5b76e78afd774d8a54c4b3a5914e9ad4e8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,120 @@ +dist: xenial +sudo: true +language: + - cpp + +env: + global: + - CXXFLAGS="-ggdb3 -O0 --coverage" + - CFLAGS="-ggdb3 -O0 --coverage" + - LDFLAGS="--coverage" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE="1" + +matrix: + include: + - os : linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + # - g++-8 + - libboost-system-dev + - libboost-filesystem-dev + - libboost-program-options-dev + - libboost-thread-dev + - libboost-regex-dev + - libprotobuf-dev + - protobuf-compiler + - libprotobuf-c-dev + - protobuf-c-compiler + - libyaml-cpp-dev + - libyaml-dev + - libarchive-dev + - lcov + +branches: + only: + - master + +before_install: + - env |sort + - | + ( + set -Eeuo pipefail + set -x + # if [[ "$CC" == "gcc" && `command -v gcc-8` ]]; then export CC=gcc-8; export CXX=g++-8; export AR=gcc-ar-8; export NM=gcc-nm-8; export RANLIB=gcc-ranlib-8; GCOV=`which gcov-8`; sudo ln -sf $GCOV /usr/bin/gcov; fi + $CC --version + gcov --version + ) + +before_script: + - | + ( + pushd . + git clone https://github.com/ofiwg/libfabric.git + cd libfabric + ./autogen.sh + mkdir build + cd build + ../configure + make -j 2 + sudo make install + popd + sudo ldconfig + ) + - | + ( + set -Eeuo pipefail + set -x + pushd . + git clone https://github.com/mercury-hpc/mercury.git + cd mercury + mkdir build + cd build + cmake \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=OFF \ + -DMERCURY_USE_SM_ROUTING:BOOL=OFF \ + -DMERCURY_USE_SELF_FORWARD:BOOL=OFF \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DMERCURY_USE_EAGER_BULK:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DNA_USE_OFI:BOOL=ON \ + .. + make -j2 + sudo make install + popd + sudo ldconfig + ) + +script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build + cd build + ../configure --enable-tests #CC=/usr/bin/gcc-8 CXX=/usr/bin/g++-8 + make -j2 + cd tests + make -j2 core + make -j2 api + ./core + sudo setcap cap_sys_ptrace,cap_chown=+ep ./api + ./api + ) + +after_success: + - | + ( + pwd + ls -l + pushd . + cd build + ../gencov.sh + bash <(curl -s https://codecov.io/bash) -f norns.info || echo "Codecov did not collect coverage reports" + popd + ) diff --git a/gencov.sh b/gencov.sh index 7fde5a00f73fd560bd51a292f3b35a524574da49..d1ab976d8c74956128a865bc9bd4f6bca2d7bf8f 100755 --- a/gencov.sh +++ b/gencov.sh @@ -39,7 +39,7 @@ GIT_ROOTDIR=`git rev-parse --show-toplevel` LCOV=`which lcov` ${LCOV} \ - `find ${GIT_ROOTDIR} -name "*.gcda" 2>/dev/null | xargs -I{} dirname {} | uniq | xargs -I {} echo -n " --directory "{}` \ + `find ${GIT_ROOTDIR} -not \( -path *mercury* -prune \) -not \( -path *libfabric* -prune \) -and -name "*.gcda" 2>/dev/null | xargs -I{} dirname {} | uniq | xargs -I {} echo -n " --directory "{}` \ --capture \ --output-file gcov.info @@ -49,7 +49,7 @@ ${LCOV} \ '/usr/local/include/*' \ '*/externals/*' \ '*/spdlog/*' \ - '*/build/*' \ + '$PWD/build/*' \ '*/tests/*' \ -o norns.info diff --git a/src/externals/hermes b/src/externals/hermes index 43280bd68daacebcd303061694ed083277dbd016..9779ad922b3e1a0e657516294721a5386f52d89d 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit 43280bd68daacebcd303061694ed083277dbd016 +Subproject commit 9779ad922b3e1a0e657516294721a5386f52d89d