diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a77a63cf038c549f4098f2a845757e2fd89bf347..61005dd6eca5e9e69d43c815e62f9fdc00005d67 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,22 +1,17 @@
stages:
- - check format
- - download deps
- - build deps
+ - lint
- build
- test
- report
variables:
- DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src"
- DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install"
- DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit"
+ SCRIPTS_DIR: "${CI_PROJECT_DIR}/scripts"
+ CI_SCRIPTS_DIR: "${CI_PROJECT_DIR}/scripts/ci"
BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build"
INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install"
INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration"
- INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run"
- TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build"
+ COVERAGE_PATH: "${CI_PROJECT_DIR}/gkfs/build/.coverage"
PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test"
- LOG_PATH: "${CI_PROJECT_DIR}/logs"
LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib:${CI_PROJECT_DIR}/deps/install/lib64"
# Configuration variables
GKFS_LOG_LEVEL: "100"
@@ -25,36 +20,55 @@ variables:
LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log"
GIT_SUBMODULE_STRATEGY: recursive
-image: gekkofs/gekkofs:build_env-0.8.0
+# base image
+image: gekkofs/core:0.8.0
+################################################################################
+## Validating
+################################################################################
check format:
- stage: check format
+ stage: lint
+ image: gekkofs/linter:0.8.0
+ needs: []
script:
- - ${CI_PROJECT_DIR}/scripts/check_format.sh -s "${CI_PROJECT_DIR}/src" -i "${CI_PROJECT_DIR}/include" -v
+ - ${SCRIPTS_DIR}/check_format.sh
+ -s "${CI_PROJECT_DIR}/src"
+ -i "${CI_PROJECT_DIR}/include"
+ -v
-compile dependencies:
- stage: build deps
- cache:
- key: deps-cache
- paths:
- - ${DEPS_INSTALL_PATH}/
+
+################################################################################
+## Building
+################################################################################
+gkfs:
+ stage: build
+ image: gekkofs/deps:0.8.0
+ interruptible: true
+ needs: []
script:
- # Folder of built dependencies is cached and marked with the ID of the commit from which have been built.
- # If the download and compile script have been modified the cache gets invalidated and dependencies will be built again.
- - ( [ -f "${DEPS_COMMIT}" ] && git diff --quiet "`cat ${DEPS_COMMIT}`" -- scripts/dl_dep.sh scripts/compile_dep.sh ) || (
- rm -f ${DEPS_COMMIT} &&
- scripts/dl_dep.sh ${DEPS_SRC_PATH} -n ofi -c ci &&
- scripts/compile_dep.sh -n ofi -c ci ${DEPS_SRC_PATH} ${DEPS_INSTALL_PATH} &&
- echo "${CI_COMMIT_SHA}" > "${DEPS_COMMIT}"
- )
+ - mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}
+ - cmake
+ -Wdev
+ -Wdeprecate
+ -DCMAKE_BUILD_TYPE=Debug
+ -DGKFS_ENABLE_CODE_COVERAGE:BOOL=ON
+ -DGKFS_BUILD_TESTS:BOOL=ON
+ -DGKFS_INSTALL_TESTS:BOOL=ON
+ -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
+ ${CI_PROJECT_DIR}
+ - make -j$(nproc) install
+ # reduce artifacts size
+ - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}
artifacts:
paths:
- - ${DEPS_INSTALL_PATH}
+ - ${BUILD_PATH}
+ - ${INSTALL_PATH}
-compile GekkoFS:
+gkfwd:
stage: build
- dependencies:
- - "compile dependencies"
+ image: gekkofs/deps:0.8.0
+ interruptible: true
+ needs: []
script:
- mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}
- cmake
@@ -66,132 +80,149 @@ compile GekkoFS:
-DGKFS_INSTALL_TESTS:BOOL=ON
-DGKFS_ENABLE_FORWARDING:BOOL=ON
-DGKFS_ENABLE_AGIOS:BOOL=ON
- -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH}
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
${CI_PROJECT_DIR}
- make -j$(nproc) install
- - find ${BUILD_PATH} -name "*.o" -delete
- # cleanup nlohmann_json: it includes around 500MiB of test data
- - rm -rf ${BUILD_PATH}/_deps/nlohmann_json-src/test/data
- - rm -rf ${BUILD_PATH}/_deps/nlohmann_json-src/benchmarks/data
- - rm -rf ${BUILD_PATH}/_deps/nlohmann_json-src/.git
+ # reduce artifacts size
+ - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}
artifacts:
paths:
- ${BUILD_PATH}
- ${INSTALL_PATH}
- exclude:
- # remove object files
- - ${BUILD_PATH}/**/*.o
-compile tests:
- stage: build
- dependencies:
- - "compile dependencies"
- script:
- - mkdir -p ${TESTS_BUILD_PATH} && cd ${TESTS_BUILD_PATH}
- - cmake -DCMAKE_BUILD_TYPE=Debug ..
- - make -j$(nproc)
- artifacts:
- paths:
- - ${TESTS_BUILD_PATH}
-integration tests:
+################################################################################
+## Testing
+################################################################################
+
+## == integration tests for gkfs ===========
+gkfs:integration:
stage: test
+ image: gekkofs/testing:0.8.0
+ interruptible: true
+ needs: ['gkfs']
+ parallel:
+ matrix:
+ - SUBTEST: [ data, directories, operations, position, shell, status ]
+
script:
## run tests
+ - mkdir -p ${BUILD_PATH}/tests/run
- cd ${BUILD_PATH}/tests/integration
- - unbuffer ${PYTEST} -v | tee session.log
+ - ${PYTEST} -v -n $(nproc)
+ ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}
+ --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}
+ --junit-xml=report.xml
+
## capture coverage information
- cd ${BUILD_PATH}
- - ${CI_PROJECT_DIR}/scripts/ci/coverage.sh
- --capture integration
+ - ${CI_SCRIPTS_DIR}/coverage.sh
+ --verbose
+ --capture integration_${SUBTEST}
--root-dir ${CI_PROJECT_DIR}
--build-dir ${BUILD_PATH}
- --exclusions "${CI_PROJECT_DIR}/scripts/ci/.coverage-exclusions"
+ --exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"
+ --log-file "${COVERAGE_PATH}/partial/integration_${SUBTEST}/capture.log"
+
+ # fix relative paths so that GitLab can find the correct files
+ after_script:
+ - perl -i.orig
+ -pe 's%file="(.*?)"%file="tests/integration/$1"%;'
+ -pe 's%(../)+install/share/gkfs/%%g;'
+ ${BUILD_PATH}/tests/integration/report.xml
+
artifacts:
- # when: on_failure
+ expire_in: 1 week
+ when: always
paths:
- # - "${INTEGRATION_TESTS_RUN_PATH}"
- ${BUILD_PATH}
+ reports:
+ junit: ${BUILD_PATH}/tests/integration/report.xml
+
-unit:
+## == integration tests for gkfwd ==========
+gkfwd:integration:
stage: test
+ image: gekkofs/testing:0.8.0
+ interruptible: true
+ needs: ['gkfwd']
+ parallel:
+ matrix:
+ - SUBTEST: [ forwarding ]
+
script:
- ## run actual tests
- - cd ${BUILD_PATH}
- - ctest -j $(nproc) -L unit::all
+ ## run tests
+ - mkdir -p ${BUILD_PATH}/tests/run
+ - cd ${BUILD_PATH}/tests/integration
+ - ${PYTEST} -v -n $(nproc)
+ ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}
+ --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}
+ --junit-xml=report.xml
+
## capture coverage information
- - ${CI_PROJECT_DIR}/scripts/ci/coverage.sh
- --capture unit
+ - cd ${BUILD_PATH}
+ - ${CI_SCRIPTS_DIR}/coverage.sh
+ --verbose
+ --capture integration_${SUBTEST}
--root-dir ${CI_PROJECT_DIR}
--build-dir ${BUILD_PATH}
- --exclusions "${CI_PROJECT_DIR}/scripts/ci/.coverage-exclusions"
- artifacts:
- # when: on_failure
- paths:
- - ${BUILD_PATH}
- # - Testing
+ --exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"
+ --log-file "${COVERAGE_PATH}/partial/integration_${SUBTEST}/capture.log"
-test wr:
- stage: test
- script:
- - mkdir -p "${LOG_PATH}"
- - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &
- - sleep 4
- - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_wr
- artifacts:
- when: on_failure
- paths:
- - "${LOG_PATH}"
+ # fix relative paths so that GitLab can find the correct files
+ after_script:
+ - perl -i.orig
+ -pe 's%file="(.*?)"%file="tests/integration/$1"%;'
+ -pe 's%(../)+install/share/gkfs/%%g;'
+ ${BUILD_PATH}/tests/integration/report.xml
-test directories:
- stage: test
- script:
- - mkdir -p "${LOG_PATH}"
- - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &
- - sleep 4
- - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_dir
artifacts:
- when: on_failure
+ expire_in: 1 week
+ when: always
paths:
- - "${LOG_PATH}"
+ - ${BUILD_PATH}
+ reports:
+ junit: ${BUILD_PATH}/tests/integration/report.xml
-test truncate:
- stage: test
- script:
- - mkdir -p "${LOG_PATH}"
- - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &
- - sleep 4
- - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_truncate
- artifacts:
- when: on_failure
- paths:
- - "${LOG_PATH}"
-test lseek:
+## == unit tests for gkfs ==================
+gkfs:unit:
stage: test
+ image: gekkofs/testing:0.8.0
+ needs: ['gkfs']
script:
- - mkdir -p "${LOG_PATH}"
- - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root &
- - sleep 4
- - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_lseek
+ ## run actual tests
+ - cd ${BUILD_PATH}/tests/unit
+ - ctest -j $(nproc) -L unit::all --output-junit report.xml
+
+ ## capture coverage information
+ - ${CI_SCRIPTS_DIR}/coverage.sh
+ --verbose
+ --capture unit
+ --root-dir ${CI_PROJECT_DIR}
+ --build-dir ${BUILD_PATH}
+ --exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"
+ --log-file "${COVERAGE_PATH}/partial/unit/capture.log"
artifacts:
- when: on_failure
+ expire_in: 1 week
paths:
- - "${LOG_PATH}"
+ - ${BUILD_PATH}
+ reports:
+ junit: ${BUILD_PATH}/tests/unit/report.xml
################################################################################
## Generation of code coverage reports
################################################################################
-
coverage:
stage: report
- image: gekkofs/coverage
+ image: gekkofs/coverage:0.8.0
+ needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]
script:
- cd ${BUILD_PATH}
## merge the partial coverage files from each test in the pipeline
- - ${CI_PROJECT_DIR}/scripts/ci/coverage.sh
+ - ${CI_SCRIPTS_DIR}/coverage.sh
+ --verbose
--merge
--root-dir ${CI_PROJECT_DIR}
--build-dir ${BUILD_PATH}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3061cc5b217536e339d6b22b41749f8adaed18a4..3388b9015ad7cdfc7e4e6dc85d796e44c0c0d5bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -280,7 +280,7 @@ if (GKFS_BUILD_TESTS)
message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")
message(STATUS "[gekkofs] Check for forwarding tests...")
- if (ENABLE_FORWARDING)
+ if (GKFS_ENABLE_FORWARDING)
set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
else ()
set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
diff --git a/docker/0.8.0/core/Dockerfile b/docker/0.8.0/core/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..82180a23d2287b99ca4fb127fabfeed9127bb0c5
--- /dev/null
+++ b/docker/0.8.0/core/Dockerfile
@@ -0,0 +1,37 @@
+FROM debian:stable-slim
+
+LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ git \
+ curl \
+ ca-certificates \
+ libtool \
+ pkg-config \
+ make \
+ automake \
+ cmake \
+ gcc \
+ g++ \
+ # AGIOS dependencies
+ libconfig-dev \
+ # Mercury dependencies
+ libltdl-dev \
+ lbzip2 \
+ # RocksDB dependencies
+ libsnappy-dev \
+ liblz4-dev \
+ libzstd-dev \
+ libbz2-dev \
+ zlib1g-dev \
+ # syscall_intercept dependencies
+ libcapstone-dev \
+ # GekkoFS dependencies
+ libboost-filesystem-dev \
+ libboost-program-options-dev \
+ uuid-dev && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.8.0/core/Makefile b/docker/0.8.0/core/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..0fb130661b8d47c800800b3618e73895ac173b2b
--- /dev/null
+++ b/docker/0.8.0/core/Makefile
@@ -0,0 +1,4 @@
+.PHONY: all
+
+all:
+ docker build -t gekkofs/core:0.8.0 .
diff --git a/docker/0.8.0/coverage/Dockerfile b/docker/0.8.0/coverage/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..487bd89316dea156417212da7b15e8f9265b7161
--- /dev/null
+++ b/docker/0.8.0/coverage/Dockerfile
@@ -0,0 +1,20 @@
+FROM debian:stable-slim
+
+LABEL Description="Environment to generate coverage reports in GekkoFS"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ wget \
+ git \
+ cmake \
+ gcc \
+ g++ \
+ lcov \
+ python3 \
+ python3-pip \
+ python3-setuptools && \
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && \
+ apt-get autoclean && \
+ python3 -m pip install --upgrade pip && \
+ pip3 install gcovr
diff --git a/docker/0.8.0/coverage/Makefile b/docker/0.8.0/coverage/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..cc8d4af223296b7044c915feb394d14073747f8b
--- /dev/null
+++ b/docker/0.8.0/coverage/Makefile
@@ -0,0 +1,4 @@
+.PHONY: all
+
+all:
+ docker build -t gekkofs/coverage:0.8.0 .
diff --git a/docker/0.8.0/deps/Dockerfile b/docker/0.8.0/deps/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..62860a864cc4eb34673b96e8e298076ee8a13d44
--- /dev/null
+++ b/docker/0.8.0/deps/Dockerfile
@@ -0,0 +1,35 @@
+FROM gekkofs/core:0.8.0
+
+LABEL Description="Debian-based environment to build GekkoFS"
+
+ENV GKFS_PATH /opt/gkfs
+ENV GKFS_VERSION 0.8.0
+
+ENV SCRIPTS_PATH ${GKFS_PATH}/scripts
+ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src
+ENV INSTALL_PATH /usr/local
+
+COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/patches ${SCRIPTS_PATH}/patches
+COPY scripts/profiles ${SCRIPTS_PATH}/profiles
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ python3 \
+ python3-pip \
+ python3-dev \
+ python3-venv \
+ python3-setuptools && \
+ python3 -m pip install --upgrade pip && \
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && apt-get autoclean
+
+# Download and build dependencies
+RUN cd ${SCRIPTS_PATH} && \
+ /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \
+ /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \
+ rm -rf ${DEPS_SRC_PATH} && \
+ rm -rf ${SCRIPTS_PATH} && \
+ rmdir ${GKFS_PATH} && \
+ ldconfig
diff --git a/docker/0.8.0/deps/Makefile b/docker/0.8.0/deps/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..0bbe7e93265aa012677e11bf6d1b950719c40e11
--- /dev/null
+++ b/docker/0.8.0/deps/Makefile
@@ -0,0 +1,17 @@
+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts umount-scripts
+
+mount-scripts:
+ mkdir -p $(CWD)/scripts
+ mount --bind $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: mount-scripts
+ docker build -t gekkofs/deps:0.8.0 .
+
+umount-scripts:
+ - umount $(CWD)/scripts
+ - rmdir $(CWD)/scripts
diff --git a/docker/0.8.0/linter/Dockerfile b/docker/0.8.0/linter/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..4f93de54d6254364433c55d937dde21166a47f18
--- /dev/null
+++ b/docker/0.8.0/linter/Dockerfile
@@ -0,0 +1,19 @@
+FROM gekkofs/core:0.8.0
+
+LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # clang 10 deps for clang-format
+ lsb-release \
+ wget \
+ software-properties-common \
+ gnupg2 && \
+ # add clang-10 repos
+ wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 10 && \
+ # install clang-format
+ apt-get update && apt-get install -y --no-install-recommends clang-format-10 && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.8.0/linter/Makefile b/docker/0.8.0/linter/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..f6752471b03631595821a13838c7bb13b19dbbb4
--- /dev/null
+++ b/docker/0.8.0/linter/Makefile
@@ -0,0 +1,4 @@
+.PHONY: all
+
+all:
+ docker build -t gekkofs/linter:0.8.0 .
diff --git a/docker/0.8.0/testing/Dockerfile b/docker/0.8.0/testing/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..f0fd39fe8e07ef407c0b1cab820f68a8c3fd318b
--- /dev/null
+++ b/docker/0.8.0/testing/Dockerfile
@@ -0,0 +1,14 @@
+FROM gekkofs/deps:0.8.0
+
+LABEL Description="Debian-based environment to test GekkoFS"
+
+RUN \
+ # install cmake 3.21 since we need to produce JUnit XML files
+ curl -OL https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.sh && \
+ chmod u+x ./cmake-3.21.1-linux-x86_64.sh && \
+ ./cmake-3.21.1-linux-x86_64.sh --skip-license --prefix=/usr && \
+ # install gcovr
+ # (required for partial coverage reports in parallel runs)
+ pip3 install gcovr && \
+ # cleanup
+ rm ./cmake-3.21.1-linux-x86_64.sh
diff --git a/docker/0.8.0/testing/Makefile b/docker/0.8.0/testing/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..f0d8290400a31e1d0ec20801ec75a8702b551e93
--- /dev/null
+++ b/docker/0.8.0/testing/Makefile
@@ -0,0 +1,4 @@
+.PHONY: all
+
+all:
+ docker build -t gekkofs/testing:0.8.0 .
diff --git a/docker/debian_build_env.docker b/docker/debian_build_env.docker
deleted file mode 100644
index b616be8c67b383748751e25c7702e7b35532f1cd..0000000000000000000000000000000000000000
--- a/docker/debian_build_env.docker
+++ /dev/null
@@ -1,73 +0,0 @@
-FROM debian:stable-slim
-
-LABEL Description="This is a debian based environment to build GekkoFS"
-
-ENV GKFS_PATH /opt/gkfs
-
-ENV SCRIPTS_PATH ${GKFS_PATH}/scripts
-ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src
-ENV INSTALL_PATH ${GKFS_PATH}/build_deps
-
-RUN apt-get update && \
- apt-get install -y --no-install-recommends \
- git \
- curl \
- ca-certificates \
- libtool \
- pkg-config \
- make \
- automake \
- cmake \
- gcc \
- g++ \
- # AGIOS dependencies
- libconfig-dev \
- # Mercury dependencies
- libltdl-dev \
- lbzip2 \
- # RocksDB
- libsnappy-dev \
- liblz4-dev \
- libzstd-dev \
- libbz2-dev \
- zlib1g-dev \
- # syscall_intercept dependencies
- libcapstone-dev \
- # GekkoFS
- libboost-filesystem-dev \
- libboost-program-options-dev \
- valgrind \
- uuid-dev \
- python3 \
- python3-pip \
- python3-dev \
- python3-venv \
- python3-setuptools \
- expect \
- # clang 10 deps
- lsb-release \
- wget \
- software-properties-common \
- gnupg2 \
- # add clang-10 repos
- wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 10 && \
- # install clang-format
- apt-get update && apt-get install -y --no-install-recommends clang-format-10 && \
- # install gcovr
- # (required for partial coverage reports in parallel runs)
- python3 -m pip install --upgrade pip && \
- pip3 install gcovr && \
- # Clean apt cache to reduce image layer size
- rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
- # Clean apt caches of packages
- apt-get clean && apt-get autoclean
-
-## COPY scripts/dl_dep.sh $SCRIPTS_PATH/
-## COPY scripts/compile_dep.sh $SCRIPTS_PATH/
-## COPY scripts/patches $SCRIPTS_PATH/patches
-##
-## # Download dependencies source
-## RUN /bin/bash $SCRIPTS_PATH/dl_dep.sh $DEPS_SRC_PATH all
-##
-## # Compile dependencies
-## RUN /bin/bash $SCRIPTS_PATH/compile_dep.sh $DEPS_SRC_PATH $INSTALL_PATH
diff --git a/docker/latest b/docker/latest
new file mode 120000
index 0000000000000000000000000000000000000000..8adc70fdd9d6a086cc636d251adf0c7a7bca8319
--- /dev/null
+++ b/docker/latest
@@ -0,0 +1 @@
+0.8.0
\ No newline at end of file
diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh
index d980d0958309bb57577899125402f8370b14b517..28979d5a24f082c1abd56a718dfba725f711f1c6 100755
--- a/scripts/ci/coverage.sh
+++ b/scripts/ci/coverage.sh
@@ -33,6 +33,8 @@ export CCOV_BUILD_DIR="${PWD}"
export CCOV_MODE=""
export CCOV_CAPTURE_NAME=""
export CCOV_EXCLUSIONS_FILE=".coverage-exclusions"
+export CCOV_LOG_FILE="/dev/stdout"
+export CCOV_VERBOSE=false
export GCOVR_EXTRA_OPTIONS=()
usage() {
@@ -64,6 +66,10 @@ Options:
and/or [[BUILD_DIR]] tags that will be expanded with the
appropriate values.
Defaults to .coverage-exclusions.
+ -l, --log-file LOG_FILE
+ Redirect output to LOG_FILE.
+ -v, --verbose
+ Increase verbosity.
EOF
# EOF is found above and hence cat command stops reading. This is equivalent to
# echo but much neater when printing out.
@@ -76,8 +82,8 @@ parse_args() {
# -l is for long options with double dash like --version
# the comma separates different long options
options=$(getopt -l \
- "capture:,merge,help,root-dir:,build-dir:,exclusions:,output:" \
- -o "cmhr:b:e:o:" -- "$@")
+ "capture:,merge,help,root-dir:,build-dir:,exclusions:,output:,log-file:,verbose" \
+ -o "cmhr:b:e:o:l:v" -- "$@")
# set --:
# If no arguments follow this option, then the positional parameters are
@@ -162,6 +168,26 @@ parse_args() {
export CCOV_EXCLUSIONS_FILE=$1
;;
+ -l|--log-file)
+ shift
+
+ if [[ -z $1 ]]; then
+ echo "Missing mandatory argument for '${opt}'."
+ exit 1
+ fi
+
+ if [[ $1 =~ ^--.* ]]; then
+ echo "Invalid argument '${1}' for '${opt}'."
+ exit 1
+ fi
+
+ export CCOV_LOG_FILE=$1
+ ;;
+
+ -v|--verbose)
+ CCOV_VERBOSE=true
+ ;;
+
--)
shift
GCOVR_EXTRA_OPTIONS="$@"
@@ -206,19 +232,24 @@ capture() {
! [[ -d "${COVERAGE_OUTPUT_DIR}" ]] && mkdir -p "${COVERAGE_OUTPUT_DIR}"
- exclude_args=()
-
- for p in "${CCOV_EXCLUSIONS[@]}";
- do
- exclude_args+=( "--exclude '${p}'" )
- done
+ if [ "$CCOV_VERBOSE" = true ]; then
+ printf "Executing capture command:"
+ printf " gcovr"
+ printf " --root ${CCOV_ROOT_DIR}"
+ printf " %s\n" "${CCOV_EXCLUSIONS[@]/#/--exclude=}"
+ printf " --json"
+ printf " --output ${COVERAGE_OUTPUT_DIR}/coverage.json"
+ printf " --verbose"
+ printf " %s\n" "${GCOVR_EXTRA_OPTIONS[@]}"
+ fi
gcovr \
--root "${CCOV_ROOT_DIR}" \
- "${CCOV_EXCLUSIONS[@]/#/--exclude=/}" \
+ "${CCOV_EXCLUSIONS[@]/#/--exclude=}" \
--json \
--output "${COVERAGE_OUTPUT_DIR}/coverage.json" \
- ${GCOVR_EXTRA_OPTIONS[@]}
+ --verbose \
+ ${GCOVR_EXTRA_OPTIONS[@]} > "${CCOV_LOG_FILE}" 2>&1
echo "Coverage report written to ${COVERAGE_OUTPUT_DIR}/coverage.json"
}
@@ -234,6 +265,20 @@ merge() {
mapfile -d $'\0' tracefiles < \
<(find "${PWD}/.coverage/partial" -name coverage.json -print0)
+
+ if [ "$CCOV_VERBOSE" = true ]; then
+ printf "Executing merge command:"
+ printf " gcovr"
+ printf " --root ${CCOV_ROOT_DIR}"
+ printf " %s\n" "${tracefiles[@]/#/--add-tracefile=}"
+ printf " --html-details ${COVERAGE_OUTPUT_DIR}/coverage.html"
+ printf " --xml"
+ printf " --output ${COVERAGE_OUTPUT_DIR}/coverage-cobertura.xml"
+ printf " --print-summary"
+ printf " --verbose"
+ printf " %s\n" "${GCOVR_EXTRA_OPTIONS[@]}"
+ fi
+
gcovr \
--root "${CCOV_ROOT_DIR}" \
"${tracefiles[@]/#/--add-tracefile=}" \
@@ -241,7 +286,8 @@ merge() {
--xml \
--output "${COVERAGE_OUTPUT_DIR}/coverage-cobertura.xml" \
--print-summary \
- ${GCOVR_EXTRA_OPTIONS[@]}
+ --verbose \
+ ${GCOVR_EXTRA_OPTIONS[@]} > "${CCOV_LOG_FILE}" 2>&1
echo "Cobertura XML report written to ${COVERAGE_OUTPUT_DIR}/coverage-cobertura.xml"
echo "HTML report written to ${COVERAGE_OUTPUT_DIR}/coverage.html"
diff --git a/scripts/ci/trim_build_artifacts.sh b/scripts/ci/trim_build_artifacts.sh
new file mode 100755
index 0000000000000000000000000000000000000000..eb2fa239447731cfdec221f685b0e5fd639cbfac
--- /dev/null
+++ b/scripts/ci/trim_build_artifacts.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+## Reduce the size of CI artifacts
+BUILDDIR=$1
+
+usage() {
+ echo -e "ERROR: Missing build directory\n"
+ echo " Usage: $(basename $0) BUILD_DIR"
+ exit 1
+}
+
+if [[ -z "${BUILDDIR}" ]]; then
+ usage
+fi
+
+echo "Cleaning up ${BUILDDIR}:"
+
+echo " * Removing object files"
+
+find ${BUILDDIR} \
+ \( \
+ -name "*.a" -or \
+ -name "*.o" -or \
+ -name "*.pdb" \
+ \) \
+ -delete
+
+echo " * Removing already-installed binaries:"
+
+find ${BUILDDIR} \
+ \( \
+ -name "libgkfs_intercept.so" -or \
+ -name "gkfs_daemon" \
+ \) \
+ -delete
+
+echo " * Removing sources automatically-downloaded by CMake"
+
+find ${BUILDDIR}/_deps -type d -name "*-src" -prune -exec rm -rf {} \;
diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh
index d187b8115d5e24a7b35a14a424a146a5a51160f5..798ea6ecf5f5248c4a78dbd8218ddcda498377d2 100755
--- a/scripts/compile_dep.sh
+++ b/scripts/compile_dep.sh
@@ -30,46 +30,29 @@
PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PATCH_DIR="${PATCH_DIR}/patches"
DEPENDENCY=""
-NA_LAYER=""
CORES=""
-SOURCE=""
-INSTALL=""
-DEP_CONFIG=""
-
-VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all ci"
-
-MOGON1_DEPS=(
- "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
- "syscall_intercept" "date" "verbs"
-)
-
-MOGON2_DEPS=(
- "bzip2" "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
- "syscall_intercept" "date" "psm2"
-)
-
-NGIO_DEPS=(
- "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
- "syscall_intercept" "date" "agios" "psm2"
-)
-
-DIRECT_DEPS=(
- "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
-)
-
-ALL_DEPS=(
- "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb"
- "syscall_intercept" "date" "agios"
-)
-
-CI_DEPS=(
- "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios"
-)
+SOURCE_DIR=""
+INSTALL_DIR=""
+PERFORM_TEST=
+
+DEFAULT_PROFILE="default"
+DEFAULT_VERSION="latest"
+PROFILE_NAME=${DEFAULT_PROFILE}
+PROFILE_VERSION=${DEFAULT_VERSION}
+PROFILES_DIR="${PWD}/profiles"
+declare -a PROFILE_DEP_LIST
+declare -A PROFILE_DEP_NAMES
+declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES PROFILE_EXTRA_INSTALL_ARGS
+declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES
usage_short() {
echo "
-usage: compile_dep.sh [-h] [-l] [-n ] [-c ] [-d ] [-j ]
- source_path install_path
+usage: compile_dep.sh [-h]
+ [-l [PROFILE_NAME:[VERSION]]]
+ [-p PROFILE_NAME[:VERSION]]
+ [-d DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]
+ [-j COMPILE_CORES]
+ SOURCES_PATH INSTALL_PATH
"
}
@@ -80,87 +63,191 @@ help_msg() {
This script compiles all GekkoFS dependencies (excluding the fs itself)
positional arguments:
- source_path path to the cloned dependencies path from clone_dep.sh
- install_path path to the install path of the compiled dependencies
+ SOURCES_PATH path to the downloaded sources for the dependencies
+ INSTALL_PATH path to the installation directory for the built dependencies
optional arguments:
-h, --help shows this help message and exits
-l, --list-dependencies
list dependencies available for building and installation
- -n , --na
- network layer that is used for communication. Valid: {bmi,ofi,all}
- defaults to 'all'
- -c , --config
- allows additional configurations, e.g., for specific clusters
- supported values: {mogon1, mogon2, ngio, direct, all, ci}
- defaults to 'direct'
- -d , --dependency
- download a specific dependency and ignore --config setting. If unspecified
- all dependencies are built and installed based on set --config setting.
- Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\"
- -j , --compilecores
+ -p, --profile PROFILE_NAME[:VERSION]
+ allows installing a pre-defined set of dependencies as defined
+ in ${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to
+ deploy specific library versions and/or configurations,
+ using a recognizable name. Optionally, PROFILE_NAME may include
+ a specific version for the profile, e.g. 'mogon2:latest' or
+ 'ngio:0.8.0', which will download the dependencies defined for
+ that specific version. If unspecified, the 'default:latest' profile
+ will be used, which should include all the possible dependencies.
+ -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]
+ build and install a specific dependency, ignoring any --profile
+ option provided. If PROFILE_NAME is unspecified, the 'default'
+ profile will be used. Similarly, if VERSION is unspecified, the
+ 'latest' version of the specified profile will be used.
+ -j, --compilecores COMPILE_CORES
number of cores that are used to compile the dependencies
defaults to number of available cores
-t, --test Perform libraries tests.
"
}
-list_dependencies() {
+list_versions() {
- echo "Available dependencies: "
+ if [[ ! -d "${PROFILES_DIR}" ]]; then
+ echo "Directory '${PROFILES_DIR}' does not exist. No profiles available."
+ exit 1
+ fi
+
+ declare -A versions
+
+ while IFS= read -r -d '' filename; do
+ id="$(basename $(dirname ${filename}))"
+ profile="$(basename ${filename%%.specs})"
+
+ versions[$id]+="${profile} "
+ done < <(find -L "${PROFILES_DIR}" -type f -name "*.specs" -print0 | sort -z)
- echo -n " Mogon 1: "
- for d in "${MOGON1_DEPS[@]}"; do
- echo -n "$d "
+ echo -e "Available versions and configuration profiles:\n"
+
+ for id in "${!versions[@]}"; do
+ echo " ${id}:"
+ echo -e " ${versions[${id}]}\n"
done
- echo
- echo -n " Mogon 2: "
- for d in "${MOGON2_DEPS[@]}"; do
- echo -n "$d "
+
+ exit 0
+}
+
+list_profiles() {
+
+ local TAG=$1
+
+ if [[ "$TAG" =~ ^(.*):(.*)$ ]]; then
+ PROFILE="${BASH_REMATCH[1]}.specs"
+
+ if [[ -n ${BASH_REMATCH[2]} ]]; then
+ VERSION="${BASH_REMATCH[2]}"
+ else
+ VERSION="latest"
+ fi
+
+ else
+ VERSION="${TAG}"
+ fi
+
+ if [[ ! -d "${PROFILES_DIR}" ]]; then
+ echo "Directory '${PROFILES_DIR}' does not exist. No configuration profiles found."
+ exit 1
+ fi
+
+ if [[ ! -d "${PROFILES_DIR}/${VERSION}" ]]; then
+ echo "Version ${VERSION} does not exist. No configuration profiles found."
+ exit 1
+ fi
+
+ echo -e "Configuration profiles for '${VERSION}':\n"
+
+ find "${PROFILES_DIR}/${VERSION}/${PROFILE}" -type f -name "*.specs" -print0 | sort -z | while IFS= read -r -d '' filename; do
+ basename=$(basename "${filename}")
+ version=$(basename $(dirname "${filename}"))
+ profile="${basename%.*}"
+
+ echo "* ${profile}:${version} (${filename})"
+
+ source "${filename}"
+
+ if [[ -n "${comment}" ]]; then
+ echo -e "\n ${comment}\n"
+ fi
+
+ for d in "${order[@]}";
+ do
+ if [[ -n ${wgetdeps[${d}]} ]]; then
+ echo " ${d}: ${wgetdeps[${d}]}"
+ elif [[ -n ${clonedeps[${d}]} ]]; then
+ echo " ${d}: ${clonedeps[${d}]}"
+ else
+ echo " ${d}: ???"
+ fi
+ done
+
+ echo ""
+
+ unset wgetdeps
+ unset clonedeps
+ unset clonedeps_args
+ unset clonedeps_patches
+ unset comment
+ unset order
done
- echo
- echo -n " NGIO: "
- for d in "${NGIO_DEPS[@]}"; do
- echo -n "$d "
+ exit 0
+
+}
+
+load_profile() {
+
+ local profile=$1
+ local version=$2
+ shift
+
+ # make sure we are in a known state
+ PROFILE_DEP_NAMES=()
+ PROFILE_DEP_LIST=()
+ PROFILE_CLONEDEPS=()
+ PROFILE_CLONEDEPS_ARGS=()
+ PROFILE_CLONEDEPS_PATCHES=()
+ PROFILE_WGETDEPS=()
+
+ local filename="${PROFILES_DIR}/${version}/${profile}.specs"
+
+ if [[ ! -f "${filename}" ]]; then
+ echo "Profile '${profile}:${version}' does not exist."
+ exit 1
+ fi
+
+ source "${filename}"
+
+ # some checks
+ if [[ -z "${wgetdeps[*]}" && -z "${clonedeps[*]}" ]]; then
+ echo "Profile '${profile}' is invalid."
+ exit 1
+ fi
+
+ if [[ -z "${order[*]}" ]]; then
+ echo "Profile '${profile}' is invalid."
+ exit 1
+ fi
+
+ if [[ "$((${#wgetdeps[@]}+${#clonedeps[@]}))" -ne "${#order[@]}" ]]; then
+ echo "Profile '${profile}' is invalid."
+ exit 1
+ fi
+
+ # propagate results outside of function
+ for i in "${!order[@]}"; do
+ PROFILE_DEP_LIST[$i]="${order[${i}]}"
+ PROFILE_DEP_NAMES["${order[$i]}"]="$i"
done
- echo
- echo -n " Direct GekkoFS dependencies: "
- for d in "${DIRECT_DEPS[@]}"; do
- echo -n "$d "
+
+ for k in "${!clonedeps[@]}"; do
+ PROFILE_CLONEDEPS["${k}"]="${clonedeps[${k}]}"
done
- echo
- echo -n " All: "
- for d in "${ALL_DEPS[@]}"; do
- echo -n "$d "
+
+ for k in "${!clonedeps_args[@]}"; do
+ PROFILE_CLONEDEPS_ARGS["${k}"]="${clonedeps_args[${k}]}"
done
- echo
- echo -n " ci: "
- for d in "${CI_DEPS[@]}"; do
- echo -n "$d "
+
+ for k in "${!clonedeps_patches[@]}"; do
+ PROFILE_CLONEDEPS_PATCHES["${k}"]="${clonedeps_patches[${k}]}"
done
- echo
-}
-check_dependency() {
- local DEP=$1
- shift
- local DEP_CONFIG=("$@")
- # ignore template when specific dependency is set
- if [[ -n "${DEPENDENCY}" ]]; then
- # check if specific dependency was set and return from function
- if echo "${DEPENDENCY}" | grep -q "${DEP}"; then
- return
- fi
- else
- # if not check if dependency is part of dependency config
- for e in "${DEP_CONFIG[@]}"; do
- if [[ "${DEP}" == "${e}" ]]; then
- return
- fi
- done
- fi
- false
+ for k in "${!wgetdeps[@]}"; do
+ PROFILE_WGETDEPS["${k}"]="${wgetdeps[${k}]}"
+ done
+
+ for k in "${!extra_install_args[@]}"; do
+ PROFILE_EXTRA_INSTALL_ARGS["${k}"]="${extra_install_args["${k}"]}"
+ done
}
prepare_build_dir() {
@@ -180,35 +267,72 @@ find_cmake() {
echo "${CMAKE}"
}
+
POSITIONAL=()
while [[ $# -gt 0 ]]; do
key="$1"
case ${key} in
- -n | --na)
- NA_LAYER="$2"
- shift # past argument
- shift # past value
- ;;
- -c | --config)
+ -p | --profile)
if [[ -z "$2" ]]; then
- echo "ERROR: Missing argument for -c/--config option"
+ echo "ERROR: Missing argument for -p/--profile option"
exit 1
fi
- if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then
- echo "ERROR: Invalid argument for -c/--config option"
- exit 1
+
+ if [[ "$2" =~ ^(.*):(.*)$ ]]; then
+ PROFILE_NAME="${BASH_REMATCH[1]}"
+ PROFILE_VERSION="${BASH_REMATCH[2]}"
+ else
+ PROFILE_NAME="$2"
+ PROFILE_VERSION="${DEFAULT_VERSION}"
fi
- TMP_DEP_CONF="$2"
+
shift # past argument
shift # past value
;;
-d | --dependency)
+
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -d/--dependency option"
exit
fi
- DEPENDENCY="$2"
+
+ PROFILE_NAME=${DEFAULT_PROFILE}
+ PROFILE_VERSION=${DEFAULT_VERSION}
+
+ # e.g. mercury@mogon1:latest
+ if [[ "$2" =~ ^(.*)@(.*):(.*)$ ]]; then
+ if [[ -n "${BASH_REMATCH[1]}" ]]; then
+ DEPENDENCY="${BASH_REMATCH[1]}"
+ fi
+
+ if [[ -n "${BASH_REMATCH[2]}" ]]; then
+ PROFILE_NAME="${BASH_REMATCH[2]}"
+ fi
+
+ if [[ -n "${BASH_REMATCH[3]}" ]]; then
+ PROFILE_VERSION="${BASH_REMATCH[3]}"
+ fi
+
+ # e.g. mercury@mogon1
+ elif [[ "$2" =~ ^(.*)@(.*)$ ]]; then
+ if [[ -n "${BASH_REMATCH[1]}" ]]; then
+ DEPENDENCY="${BASH_REMATCH[1]}"
+ fi
+
+ if [[ -n "${BASH_REMATCH[2]}" ]]; then
+ PROFILE_NAME="${BASH_REMATCH[2]}"
+ fi
+ # e.g. mercury
+ else
+ DEPENDENCY="$2"
+ fi
+
+ if [[ ! -n "${DEPENDENCY}" ]]; then
+ echo "ERROR: Missing dependency name."
+ exit 1
+ fi
+
shift # past argument
shift # past value
;;
@@ -222,7 +346,11 @@ while [[ $# -gt 0 ]]; do
shift
;;
-l | --list-dependencies)
- list_dependencies
+ if [[ -z "$2" ]]; then
+ list_versions
+ else
+ list_profiles "$2"
+ fi
exit
;;
-h | --help)
@@ -244,14 +372,11 @@ if [[ (-z ${1+x}) || (-z ${2+x}) ]]; then
usage_short
exit 1
fi
-SOURCE="$(readlink -mn "${1}")"
-INSTALL="$(readlink -mn "${2}")"
+
+SOURCE_DIR="$(readlink -mn "${1}")"
+INSTALL_DIR="$(readlink -mn "${2}")"
# deal with optional arguments
-if [[ "${NA_LAYER}" == "" ]]; then
- echo "Defaulting NAPLUGIN to 'ofi'"
- NA_LAYER="ofi"
-fi
if [[ "${CORES}" == "" ]]; then
CORES=$(grep -c ^processor /proc/cpuinfo)
echo "CORES = ${CORES} (default)"
@@ -265,253 +390,53 @@ Input must be numeric and greater than 0."
echo CORES = "${CORES}"
fi
fi
-if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then
- echo NAPLUGIN = "${NA_LAYER}"
-else
- echo "ERROR: No valid NA plugin selected"
- usage_short
- exit
-fi
-# enable predefined dependency template
-case ${TMP_DEP_CONF} in
-mogon1)
- DEP_CONFIG=("${MOGON1_DEPS[@]}")
- echo "'Mogon1' dependencies are compiled"
- ;;
-mogon2)
- DEP_CONFIG=("${MOGON2_DEPS[@]}")
- echo "'Mogon2' dependencies are compiled"
- ;;
-ngio)
- DEP_CONFIG=("${NGIO_DEPS[@]}")
- echo "'NGIO' dependencies are compiled"
- ;;
-all)
- DEP_CONFIG=("${ALL_DEPS[@]}")
- echo "'All' dependencies are compiled"
- ;;
-ci)
- DEP_CONFIG=("${CI_DEPS[@]}")
- echo "'CI' dependencies are compiled"
- ;;
-direct | *)
- DEP_CONFIG=("${DIRECT_DEPS[@]}")
- echo "'Direct' GekkoFS dependencies are compiled (default)"
- ;;
-esac
-
-USE_BMI="-DNA_USE_BMI:BOOL=OFF"
-USE_OFI="-DNA_USE_OFI:BOOL=OFF"
+
+load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}"
CMAKE=$(find_cmake)
CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"
-echo "Source path = ${SOURCE}"
-echo "Install path = ${INSTALL}"
+echo "Sources download path = ${SOURCE_DIR}"
+echo "Installation path = ${INSTALL_DIR}"
echo "------------------------------------"
-mkdir -p "${SOURCE}"
+mkdir -p "${SOURCE_DIR}"
######### From now on exits on any error ########
set -e
-export CPATH="${CPATH}:${INSTALL}/include"
-export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64"
-export PKG_CONFIG_PATH="${INSTALL}/lib/pkgconfig:${PKG_CONFIG_PATH}"
-
-## Third party dependencies
+export CPATH="${CPATH}:${INSTALL_DIR}/include"
+export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL_DIR}/lib:${INSTALL_DIR}/lib64"
+export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}"
-# Set cluster dependencies first
-
-# build zstd for fast compression in rocksdb
-if check_dependency "zstd" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: zstd"
- CURR=${SOURCE}/zstd/build/cmake
- prepare_build_dir "${CURR}"
- cd "${CURR}"/build
- $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..
- make -j"${CORES}"
- make install
-fi
-
-# build zlib for rocksdb
-if check_dependency "lz4" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: lz4"
- CURR=${SOURCE}/lz4
- cd "${CURR}"
- make -j"${CORES}"
- make DESTDIR="${INSTALL}" PREFIX="" install
-fi
-
-# build snappy for rocksdb
-if check_dependency "snappy" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: snappy"
- CURR=${SOURCE}/snappy
- prepare_build_dir "${CURR}"
- cd "${CURR}"/build
- $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..
- make -j"${CORES}"
- make install
-fi
-
-# build bzip2 for rocksdb
-if check_dependency "bzip2" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: bzip2"
- CURR=${SOURCE}/bzip2
- cd "${CURR}"
- make install PREFIX="${INSTALL}"
-fi
-
-# build capstone for syscall-intercept
-if check_dependency "capstone" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: capstone"
- CURR=${SOURCE}/capstone
- prepare_build_dir "${CURR}"
- cd "${CURR}"/build
- $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..
- make -j"${CORES}" install
+if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then
+ echo "Dependency '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'"
+ exit
fi
-# build BMI
-if check_dependency "bmi" "${DEP_CONFIG[@]}"; then
- if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then
- USE_BMI="-DNA_USE_BMI:BOOL=ON"
- echo "############################################################ Installing: BMI"
- # BMI
- CURR=${SOURCE}/bmi
- prepare_build_dir "${CURR}"
- cd "${CURR}"
- ./prepare
- cd "${CURR}"/build
- CFLAGS="${CFLAGS} -w" ../configure --prefix="${INSTALL}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict
- make -j"${CORES}"
- make install
- fi
-fi
+for dep in "${PROFILE_DEP_LIST[@]}"; do
-# build ofi
-if check_dependency "ofi" "${DEP_CONFIG[@]}"; then
- if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then
- USE_OFI="-DNA_USE_OFI:BOOL=ON"
- echo "############################################################ Installing: LibFabric"
- #libfabric
- CURR=${SOURCE}/libfabric
- prepare_build_dir ${CURR}
- cd ${CURR}
- ./autogen.sh
- cd ${CURR}/build
- OFI_CONFIG="../configure --prefix=${INSTALL} --enable-tcp=yes"
- if check_dependency "verbs" "${DEP_CONFIG[@]}"; then
- OFI_CONFIG="${OFI_CONFIG} --enable-verbs=yes"
- elif check_dependency "psm2" "${DEP_CONFIG[@]}"; then
- OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes --with-psm2-src=${SOURCE}/psm2"
- elif check_dependency "psm2-system" "${DEP_CONFIG[@]}"; then
- OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes"
- fi
- ${OFI_CONFIG}
- make -j${CORES}
- make install
- [ "${PERFORM_TEST}" ] && make check
+ if [[ -n "${DEPENDENCY}" && "${dep}" != "${DEPENDENCY}" ]]; then
+ continue
fi
-fi
-# AGIOS
-if check_dependency "agios" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: AGIOS"
- CURR=${SOURCE}/agios
- prepare_build_dir "${CURR}"
- cd "${CURR}"/build
- $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" ..
- make install
-fi
+ install_script="${PROFILES_DIR}/${PROFILE_VERSION}/install/${dep}.install"
-# Mercury
-if check_dependency "mercury" "${DEP_CONFIG[@]}"; then
+ echo -e "\n\n######## Installing: ${dep} ###############################\n"
- if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then
- USE_BMI="-DNA_USE_BMI:BOOL=ON"
- fi
-
- if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then
- USE_OFI="-DNA_USE_OFI:BOOL=ON"
+ if [[ -f "${install_script}" ]]; then
+ source "${install_script}"
+ else
+ echo "WARNING: Install script for '${dep}' not found. Skipping."
+ continue
fi
- echo "############################################################ Installing: Mercury"
- CURR=${SOURCE}/mercury
- prepare_build_dir "${CURR}"
- cd "${CURR}"/build
- PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \
- -DCMAKE_BUILD_TYPE:STRING=Release \
- -DBUILD_TESTING:BOOL=ON \
- -DMERCURY_USE_SM_ROUTING:BOOL=ON \
- -DMERCURY_USE_SELF_FORWARD:BOOL=ON \
- -DMERCURY_USE_CHECKSUMS:BOOL=OFF \
- -DMERCURY_USE_BOOST_PP:BOOL=ON \
- -DMERCURY_USE_EAGER_BULK:BOOL=ON \
- -DBUILD_SHARED_LIBS:BOOL=ON \
- -DCMAKE_INSTALL_PREFIX=${INSTALL} \
- ${USE_BMI} ${USE_OFI} \
- ..
- make -j"${CORES}"
- make install
-fi
-
-# Argobots
-if check_dependency "argobots" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: Argobots"
- CURR=${SOURCE}/argobots
- prepare_build_dir "${CURR}"
- cd "${CURR}"
- ./autogen.sh
- cd "${CURR}"/build
- ../configure --prefix="${INSTALL}" --enable-perf-opt --disable-checks
- make -j"${CORES}"
- make install
- [ "${PERFORM_TEST}" ] && make check
-fi
-
-# Margo
-if check_dependency "margo" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: Margo"
- CURR=${SOURCE}/margo
- prepare_build_dir "${CURR}"
- cd "${CURR}"
- ./prepare.sh
- cd "${CURR}"/build
- ../configure --prefix="${INSTALL}" PKG_CONFIG_PATH="${INSTALL}"/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3"
- make -j"${CORES}"
- make install
- [ "${PERFORM_TEST}" ] && make check
-fi
-
-# Rocksdb
-if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: Rocksdb"
- CURR=${SOURCE}/rocksdb
- cd "${CURR}"
- make clean
- USE_RTTI=1 make -j"${CORES}" static_lib
- INSTALL_PATH="${INSTALL}" make install
-fi
+ pkg_install
-# syscall_intercept
-if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: Syscall_intercept"
- CURR=${SOURCE}/syscall_intercept
- prepare_build_dir "${CURR}"
- cd "${CURR}"/build
- $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL}" -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..
- make install
-fi
+ [ "${PERFORM_TEST}" ] && pkg_check
-# date
-if check_dependency "date" "${DEP_CONFIG[@]}"; then
- echo "############################################################ Installing: date"
- CURR=${SOURCE}/date
- prepare_build_dir "${CURR}"
- cd "${CURR}"/build
- $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..
- make install
-fi
+done
echo "Done"
+
+exit 0
diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh
index 9b4e90f34fd083481cd391b7664129893fe22866..14274ef8dfdea2277aad2d1d8bbe95703e9e0ad9 100755
--- a/scripts/dl_dep.sh
+++ b/scripts/dl_dep.sh
@@ -28,43 +28,19 @@
################################################################################
COMMON_CURL_FLAGS="--silent --fail --show-error --location -O"
-COMMON_GIT_FLAGS="--quiet --single-branch"
+COMMON_GIT_FLAGS="--quiet --single-branch -c advice.detachedHead=false"
PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PATCH_DIR="${PATCH_DIR}/patches"
DEPENDENCY=""
-NA_LAYER=""
-DEP_CONFIG=""
VERBOSE=false
-VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all ci"
-
-MOGON1_DEPS=(
- "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb"
- "syscall_intercept" "date"
-)
-
-MOGON2_DEPS=(
- "bzip2" "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb-experimental"
- "syscall_intercept" "date" "psm2"
-)
-
-NGIO_DEPS=(
- "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb"
- "syscall_intercept" "date" "psm2" "agios"
-
-)
-DIRECT_DEPS=(
- "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
-)
-
-ALL_DEPS=(
- "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb"
- "syscall_intercept" "date" "agios"
-)
-
-CI_DEPS=(
- "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios"
-)
+DEFAULT_PROFILE="default"
+DEFAULT_VERSION="latest"
+PROFILES_DIR="${PWD}/profiles"
+SOURCES_FILE="${PROFILES_DIR}/sources.list"
+declare -a PROFILE_DEP_NAMES
+declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES
+declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES
# Stop all backround jobs on interruption.
# "kill -- -$$" sends a SIGTERM to the whole process group,
@@ -85,61 +61,171 @@ error_exit() {
exit "${2:-1}" ## Return a code specified by $2 or 1 by default.
}
-list_dependencies() {
+list_versions() {
+
+ if [[ ! -d "${PROFILES_DIR}" ]]; then
+ echo "Directory '${PROFILES_DIR}' does not exist. No profiles available."
+ exit 1
+ fi
- echo "Available dependencies: "
+ declare -A versions
- echo -n " Mogon 1: "
- for d in "${MOGON1_DEPS[@]}"; do
- echo -n "$d "
+ while IFS= read -r -d '' filename; do
+ id="$(basename $(dirname ${filename}))"
+ profile="$(basename ${filename%%.specs})"
+
+ versions[$id]+="${profile} "
+ done < <(find -L "${PROFILES_DIR}" -type f -name "*.specs" -print0 | sort -z)
+
+ echo -e "Available versions and configuration profiles:\n"
+
+ for id in "${!versions[@]}"; do
+ echo " ${id}:"
+ echo -e " ${versions[${id}]}\n"
done
- echo
- echo -n " Mogon 2: "
- for d in "${MOGON2_DEPS[@]}"; do
- echo -n "$d "
+
+ exit 0
+}
+
+list_profiles() {
+
+ local TAG=$1
+
+ if [[ "$TAG" =~ ^(.*):(.*)$ ]]; then
+ PROFILE="${BASH_REMATCH[1]}.specs"
+
+ if [[ -n ${BASH_REMATCH[2]} ]]; then
+ VERSION="${BASH_REMATCH[2]}"
+ else
+ VERSION="latest"
+ fi
+
+ else
+ VERSION="${TAG}"
+ fi
+
+ if [[ ! -d "${PROFILES_DIR}" ]]; then
+ echo "Directory '${PROFILES_DIR}' does not exist. No configuration profiles found."
+ exit 1
+ fi
+
+ if [[ ! -d "${PROFILES_DIR}/${VERSION}" ]]; then
+ echo "Version ${VERSION} does not exist. No configuration profiles found."
+ exit 1
+ fi
+
+ echo -e "Configuration profiles for '${VERSION}':\n"
+
+ find "${PROFILES_DIR}/${VERSION}/${PROFILE}" -type f -name "*.specs" -print0 | sort -z | while IFS= read -r -d '' filename; do
+ basename=$(basename "${filename}")
+ version=$(basename $(dirname "${filename}"))
+ profile="${basename%.*}"
+
+ echo "* ${profile}:${version} (${filename})"
+
+ source "${filename}"
+
+ if [[ -n "${comment}" ]]; then
+ echo -e "\n ${comment}\n"
+ fi
+
+ for d in "${order[@]}";
+ do
+ if [[ -n ${wgetdeps[${d}]} ]]; then
+ echo " ${d}: ${wgetdeps[${d}]}"
+ elif [[ -n ${clonedeps[${d}]} ]]; then
+ echo " ${d}: ${clonedeps[${d}]}"
+ else
+ echo " ${d}: ???"
+ fi
+ done
+
+ echo ""
+
+ unset wgetdeps
+ unset clonedeps
+ unset clonedeps_args
+ unset clonedeps_patches
+ unset comment
+ unset order
+ done
+ exit 0
+
+}
+
+load_profile() {
+
+ local profile=$1
+ local version=$2
+ shift
+
+ # make sure we are in a known state
+ PROFILE_DEP_NAMES=()
+ PROFILE_CLONEDEPS=()
+ PROFILE_CLONEDEPS_ARGS=()
+ PROFILE_CLONEDEPS_PATCHES=()
+ PROFILE_WGETDEPS=()
+
+ local filename="${PROFILES_DIR}/${version}/${profile}.specs"
+
+ if [[ ! -f "${filename}" ]]; then
+ echo "Profile '${profile}:${version}' does not exist."
+ exit 1
+ fi
+
+ source "${filename}"
+
+ # some checks
+ if [[ -z "${wgetdeps[*]}" && -z "${clonedeps[*]}" ]]; then
+ echo "Profile '${profile}' is invalid."
+ exit 1
+ fi
+
+ if [[ -z "${order[*]}" ]]; then
+ echo "Profile '${profile}' is invalid."
+ exit 1
+ fi
+
+ if [[ "$((${#wgetdeps[@]}+${#clonedeps[@]}))" -ne "${#order[@]}" ]]; then
+ echo "Profile '${profile}' is invalid."
+ exit 1
+ fi
+
+ # propagate results outside of function
+ for i in "${!order[@]}"; do
+ PROFILE_DEP_NAMES[$i]="${order[${i}]}"
done
- echo
- echo -n " NGIO: "
- for d in "${NGIO_DEPS[@]}"; do
- echo -n "$d "
+
+ for k in "${!clonedeps[@]}"; do
+ PROFILE_CLONEDEPS["${k}"]="${clonedeps[${k}]}"
done
- echo
- echo -n " Direct GekkoFS dependencies: "
- for d in "${DIRECT_DEPS[@]}"; do
- echo -n "$d "
+
+ for k in "${!clonedeps_args[@]}"; do
+ PROFILE_CLONEDEPS_ARGS["${k}"]="${clonedeps_args[${k}]}"
done
- echo
- echo -n " All: "
- for d in "${ALL_DEPS[@]}"; do
- echo -n "$d "
+
+ for k in "${!clonedeps_patches[@]}"; do
+ PROFILE_CLONEDEPS_PATCHES["${k}"]="${clonedeps_patches[${k}]}"
done
- echo
- echo -n " ci: "
- for d in "${CI_DEPS[@]}"; do
- echo -n "$d "
+
+ for k in "${!wgetdeps[@]}"; do
+ PROFILE_WGETDEPS["${k}"]="${wgetdeps[${k}]}"
done
- echo
}
-check_dependency() {
- local DEP=$1
- shift
- local DEP_CONFIG=("$@")
- # ignore template when specific dependency is set
- if [[ -n "${DEPENDENCY}" ]]; then
- # check if specific dependency was set and return from function
- if echo "${DEPENDENCY}" | grep -q "${DEP}"; then
- return
- fi
- else
- # if not check if dependency is part of dependency config
- for e in "${DEP_CONFIG[@]}"; do
- if [[ "${DEP}" == "${e}" ]]; then
- return
- fi
- done
+load_sources() {
+
+ if [[ ! -f "${SOURCES_FILE}" ]]; then
+ echo "Missing dependency sources at '${SOURCES_FILE}'."
+ exit 1
fi
- false
+
+ source "${SOURCES_FILE}"
+
+ # propagate sources outside of function
+ for k in "${!sources[@]}"; do
+ PROFILE_SOURCES["${k}"]="${sources[${k}]}"
+ done
}
clonedeps() {
@@ -158,15 +244,15 @@ clonedeps() {
local ACTION
- if [[ -d "${SOURCE}/${FOLDER}/.git" ]]; then
- cd "${SOURCE}/${FOLDER}" && git fetch -q
+ if [[ -d "${SOURCE_DIR}/${FOLDER}/.git" ]]; then
+ cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q
ACTION="Pulled"
else
- git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE}/${FOLDER}"
+ git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}"
ACTION="Cloned"
fi
# fix the version
- cd "${SOURCE}/${FOLDER}" && git checkout -qf ${COMMIT}
+ cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}"
echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS}'"
# apply patch if provided
@@ -185,26 +271,29 @@ wgetdeps() {
FOLDER=$1
URL=$2
- if [[ -d "${SOURCE}/${FOLDER}" ]]; then
+ if [[ -d "${SOURCE_DIR}/${FOLDER}" ]]; then
# SC2115 Use "${var:?}" to ensure this never expands to /* .
- rm -rf "${SOURCE:?}/${FOLDER:?}"
+ rm -rf "${SOURCE_DIR:?}/${FOLDER:?}"
fi
- mkdir -p "${SOURCE}/${FOLDER}"
- cd "${SOURCE}"
- FILENAME=$(basename $URL)
- if [[ -f "${SOURCE}/$FILENAME" ]]; then
- rm -f "${SOURCE}/$FILENAME"
+ mkdir -p "${SOURCE_DIR}/${FOLDER}"
+ cd "${SOURCE_DIR}"
+ FILENAME="$(basename $URL)"
+ if [[ -f "${SOURCE_DIR}/$FILENAME" ]]; then
+ rm -f "${SOURCE_DIR}/$FILENAME"
fi
curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $?
- tar -xf "$FILENAME" --directory "${SOURCE}/${FOLDER}" --strip-components=1
+ tar -xf "$FILENAME" --directory "${SOURCE_DIR}/${FOLDER}" --strip-components=1
rm -f "$FILENAME"
echo "Downloaded '${URL}' to '${FOLDER}'"
}
usage_short() {
echo "
-usage: dl_dep.sh [-h] [-l] [-n ] [-c ] [-d ]
- source_path
+usage: dl_dep.sh [-h]
+ [-l [[PROFILE_NAME:]VERSION]]
+ [-p PROFILE_NAME[:VERSION]]
+ [-d DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]
+ DESTINATION_PATH
"
}
@@ -215,51 +304,61 @@ help_msg() {
This script gets all GekkoFS dependency sources (excluding the fs itself)
positional arguments:
- source_path path where the dependency downloads are put
+ DESTINATION_PATH path where dependencies should be downloaded
optional arguments:
-h, --help shows this help message and exits
- -l, --list-dependencies
- list dependencies available for download with descriptions
- -n , --na
- network layer that is used for communication. Valid: {bmi,ofi,all}
- defaults to 'ofi'
- -c , --config
- allows additional configurations, e.g., for specific clusters
- supported values: {mogon2, mogon1, ngio, direct, all, ci}
- defaults to 'direct'
- -d , --dependency
- download a specific dependency and ignore --config setting. If unspecified
- all dependencies are built and installed based on set --config setting.
- Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\"
+ -l, --list-dependencies [[PROFILE_NAME:]VERSION]
+ list dependency configuration profiles available for download
+ -p, --profile PROFILE_NAME[:VERSION]
+ allows downloading a pre-defined set of dependencies as defined
+ in ${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to
+ deploy specific library versions and/or configurations,
+ using a recognizable name. Optionally, PROFILE_NAME may include
+ a specific version for the profile, e.g. 'mogon2:latest' or
+ 'ngio:0.8.0', which will download the dependencies defined for
+ that specific version. If unspecified, the 'default:latest' profile
+ will be used, which should include all the possible dependencies.
+ -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]
+ build and install a specific dependency, ignoring any --profile
+ option provided. If PROFILE_NAME is unspecified, the 'default'
+ profile will be used. Similarly, if VERSION is unspecified, the
+ 'latest' version of the specified profile will be used.
-v, --verbose Increase download verbosity
"
}
+# load default profile for now, might be overridden later
+load_profile "${DEFAULT_PROFILE}" "${DEFAULT_VERSION}"
+
+# load source URLs for dependencies
+load_sources
+
POSITIONAL=()
while [[ $# -gt 0 ]]; do
key="$1"
case ${key} in
- -n | --na)
- NA_LAYER="$2"
- shift # past argument
- shift # past value
- ;;
- -c | --config)
+ -p | --profile)
if [[ -z "$2" ]]; then
- echo "ERROR: Missing argument for -c/--config option"
+ echo "ERROR: Missing argument for -p/--profile option"
exit 1
fi
- if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then
- echo "ERROR: Invalid argument for -c/--config option"
- exit 1
+
+ if [[ "$2" =~ ^(.*):(.*)$ ]]; then
+ PROFILE_NAME="${BASH_REMATCH[1]}"
+ PROFILE_VERSION="${BASH_REMATCH[2]}"
+ else
+ PROFILE_NAME="$2"
+ PROFILE_VERSION="${DEFAULT_VERSION}"
fi
- TMP_DEP_CONF="$2"
+
+ load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}"
shift # past argument
shift # past value
;;
+
-d | --dependency)
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -d/--dependency option"
@@ -270,7 +369,13 @@ while [[ $# -gt 0 ]]; do
shift # past value
;;
-l | --list-dependencies)
- list_dependencies
+
+ if [[ -z "$2" ]]; then
+ list_versions
+ else
+ list_profiles "$2"
+ fi
+
exit
;;
-h | --help)
@@ -295,145 +400,80 @@ if [[ -z ${1+x} ]]; then
usage_short
exit 1
fi
-SOURCE="$(readlink -mn "${1}")"
+SOURCE_DIR="$(readlink -mn "${1}")"
-# optional arguments
-if [[ "${NA_LAYER}" == "" ]]; then
- echo "Defaulting NAPLUGIN to 'ofi'"
- NA_LAYER="ofi"
-fi
+echo "Destination path is set to \"${SOURCE_DIR}\""
+echo "------------------------------------"
-# enable predefined dependency template
-case ${TMP_DEP_CONF} in
-mogon1)
- DEP_CONFIG=("${MOGON1_DEPS[@]}")
- [[ -z "${DEPENDENCY}" ]] && echo "'Mogon1' dependencies are downloaded"
- ;;
-mogon2)
- DEP_CONFIG=("${MOGON2_DEPS[@]}")
- [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded"
- ;;
-ngio)
- DEP_CONFIG=("${NGIO_DEPS[@]}")
- [[ -z "${DEPENDENCY}" ]] && echo "'NGIO' dependencies are downloaded"
- ;;
-all)
- DEP_CONFIG=("${ALL_DEPS[@]}")
- [[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded"
- ;;
-ci)
- DEP_CONFIG=("${CI_DEPS[@]}")
- [[ -z "${DEPENDENCY}" ]] && echo "'CI' dependencies are downloaded"
- ;;
-direct | *)
- DEP_CONFIG=("${DIRECT_DEPS[@]}")
- [[ -z "${DEPENDENCY}" ]] && echo "'Direct' GekkoFS dependencies are downloaded (default)"
- ;;
-esac
-
-# sanity checks
-if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then
- echo NAPLUGIN = "${NA_LAYER}"
-else
- echo "ERROR: No valid plugin selected"
- usage_short
- exit 1
-fi
+mkdir -p "${SOURCE_DIR}"
-echo "Source path is set to \"${SOURCE}\""
-echo "------------------------------------"
+## download dependencies
+for dep in "${PROFILE_DEP_NAMES[@]}"; do
-mkdir -p "${SOURCE}"
+ if [[ ! -z "${PROFILE_WGETDEPS[${dep}]:-}" ]]; then
-## Third party dependencies
+ # dependency names can include a TAG after a colon (e.g. ofi:verbs),
+ # remove it
+ dep_id=${dep%%:*}
-# get zstd for fast compression in rocksdb
-if check_dependency "zstd" "${DEP_CONFIG[@]}"; then
- wgetdeps "zstd" "https://github.com/facebook/zstd/archive/v1.3.2.tar.gz" &
-fi
+ # find required version for dependency
+ dep_version="${PROFILE_WGETDEPS[${dep}]}"
-# get zlib for rocksdb
-if check_dependency "lz4" "${DEP_CONFIG[@]}"; then
- wgetdeps "lz4" "https://github.com/lz4/lz4/archive/v1.8.0.tar.gz" &
-fi
+ # build URL for dependency
+ dep_url="${PROFILE_SOURCES[${dep_id}]}"
-# get snappy for rocksdb
-if check_dependency "snappy" "${DEP_CONFIG[@]}"; then
- wgetdeps "snappy" "https://github.com/google/snappy/archive/1.1.7.tar.gz" &
-fi
+ if [[ -z "${dep_url}" ]]; then
+ echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."
+ exit 1
+ fi
-# get bzip2 for rocksdb
-if check_dependency "bzip2" "${DEP_CONFIG[@]}"; then
- wgetdeps "bzip2" "https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz" &
-fi
+ dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"
-# get capstone for syscall-intercept
-if check_dependency "capstone" "${DEP_CONFIG[@]}"; then
- wgetdeps "capstone" "https://github.com/aquynh/capstone/archive/4.0.1.tar.gz" &
-fi
+ wgetdeps "${dep_id}" "${dep_url}" &
-## Direct GekkoFS dependencies
+ elif [[ ! -z "${PROFILE_CLONEDEPS[${dep}]:-}" ]]; then
-# get BMI
-if check_dependency "bmi" "${DEP_CONFIG[@]}"; then
- if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "all" ]; then
- clonedeps "bmi" "https://xgitlab.cels.anl.gov/sds/bmi.git" "81ad0575fc57a69269a16208417cbcbefa51f9ea" &
- fi
-fi
+ # dependency names can include a TAG after a colon (e.g. ofi:verbs),
+ # remove it
+ dep_id=${dep%%:*}
-# get libfabric
-if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then
- if check_dependency "ofi-experimental" "${DEP_CONFIG[@]}"; then
- clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.9.1" &
- elif check_dependency "ofi-verbs" "${DEP_CONFIG[@]}"; then
- # libibverbs 1.2.1-1 used on mogon 1i (installed on system) which is linked to libfabric
- # libfabric 1.8 random RPCs fail to be send. 1.9 RPC client cannot be started when in an MPI environment
- clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.7.2" &
- elif check_dependency "ofi" "${DEP_CONFIG[@]}"; then
- clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.8.1" &
- fi
-fi
+ dep_args=""
-if check_dependency "psm2" "${DEP_CONFIG[@]}"; then
- wgetdeps "psm2" "https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz" &
-fi
+ # find required version for dependency
+ dep_version="${PROFILE_CLONEDEPS[${dep}]}"
-# get Mercury
-if check_dependency "mercury" "${DEP_CONFIG[@]}"; then
- clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "41caa143a07ed179a3149cac4af0dc7aa3f946fd" "--recurse-submodules" &
-fi
+ # version may be a commit hash, a tag or something like HEAD@BRANCH_NAME
+ # if it's the latter, remove the @BRANCH_NAME
+ if [[ "${dep_version}" =~ ^(.*)@(.*)$ ]]; then
+ dep_args+="-b ${BASH_REMATCH[2]}"
+ dep_version=${BASH_REMATCH[1]}
+ fi
-# get Argobots
-if check_dependency "argobots" "${DEP_CONFIG[@]}"; then
- wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz" &
-fi
+ # build URL for dependency
+ dep_url="${PROFILE_SOURCES[${dep_id}]}"
-# get Margo
-if check_dependency "margo" "${DEP_CONFIG[@]}"; then
- clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "v0.6.3" &
-fi
+ if [[ -z "${dep_url}" ]]; then
+ echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."
+ exit 1
+ fi
-# get rocksdb
-if check_dependency "rocksdb-experimental" "${DEP_CONFIG[@]}"; then
- wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz" &
-elif check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then
- wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz" &
-fi
+ dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"
-# get syscall_intercept
-if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then
- clonedeps "syscall_intercept" "https://github.com/pmem/syscall_intercept.git" "f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" "" "syscall_intercept.patch" &
-fi
+ # check if extra args are required
+ dep_args+="${PROFILE_CLONEDEPS_ARGS[${dep}]}"
-# get AGIOS
-if check_dependency "agios" "${DEP_CONFIG[@]}"; then
- clonedeps "agios" "https://github.com/francielizanon/agios.git" "c26a6544200f823ebb8f890dd94e653d148bf226" "-b development" &
-fi
+ dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep}]}
+
+ clonedeps "${dep}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" &
+
+ else
+ echo "Unknown dependency '${dep}'."
+ exit 1
+ fi
+done
-# get date
-if check_dependency "date" "${DEP_CONFIG[@]}"; then
- clonedeps "date" "https://github.com/HowardHinnant/date.git" "e7e1482087f58913b80a20b04d5c58d9d6d90155" &
-fi
# Wait for all download to be completed
wait
echo "Done"
+
+exit 0
diff --git a/scripts/profiles/0.8.0/all.specs b/scripts/profiles/0.8.0/all.specs
new file mode 100644
index 0000000000000000000000000000000000000000..c6e1a181980856107319e972c4fa9c269a8140d2
--- /dev/null
+++ b/scripts/profiles/0.8.0/all.specs
@@ -0,0 +1,85 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="All dependencies (except transport-specific and experimental)"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["bzip2"]="1.0.6"
+ ["zstd"]="1.3.2"
+ ["lz4"]="1.8.0"
+ ["snappy"]="1.1.7"
+ ["capstone"]="4.0.1"
+ ["argobots"]="1.0rc1"
+ ["rocksdb"]="6.2.2"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8"
+ ["libfabric"]="HEAD@v1.8.1"
+ ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
+ ["margo"]="v0.6.3"
+ ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+ ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "libfabric" "mercury" "argobots"
+ "margo" "rocksdb" "syscall_intercept" "date" "agios"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+)
diff --git a/scripts/profiles/0.8.0/ci.specs b/scripts/profiles/0.8.0/ci.specs
new file mode 100644
index 0000000000000000000000000000000000000000..514bd5421821171a9ff08a1ec6bae03f42ab022c
--- /dev/null
+++ b/scripts/profiles/0.8.0/ci.specs
@@ -0,0 +1,79 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies required by the CI"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["argobots"]="1.0rc1"
+ ["rocksdb"]="6.2.2"
+ )
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.8.1"
+ ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
+ ["margo"]="v0.6.3"
+ ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+ ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
+ )
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept"
+ "date" "agios"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+)
diff --git a/scripts/profiles/0.8.0/default.specs b/scripts/profiles/0.8.0/default.specs
new file mode 100644
index 0000000000000000000000000000000000000000..4512c95b2a3fc3d080187022d8651ca9ffdf3de6
--- /dev/null
+++ b/scripts/profiles/0.8.0/default.specs
@@ -0,0 +1,96 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="All dependencies"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["bzip2"]="1.0.6"
+ ["zstd"]="1.3.2"
+ ["lz4"]="1.8.0"
+ ["snappy"]="1.1.7"
+ ["capstone"]="4.0.1"
+ ["argobots"]="1.0rc1"
+ ["rocksdb"]="6.2.2"
+ ["rocksdb:experimental"]="6.11.4"
+ ["psm2"]="11.2.86"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8"
+ ["libfabric"]="HEAD@v1.8.1"
+# ["libfabric:experimental"]="HEAD@v1.9.1"
+# ["libfabric:verbs"]="HEAD@v1.7.2"
+ ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
+ ["margo"]="v0.6.3"
+ ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+ ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi"
+ "libfabric"
+ #"libfabric:experimental"
+ #"libfabric:verbs"
+ "mercury" "argobots" "margo" "rocksdb" "rocksdb:experimental"
+ "syscall_intercept" "date" "psm2" "agios"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
+ #["libfabric:verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes"
+ #["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes"
+)
diff --git a/scripts/profiles/0.8.0/direct.specs b/scripts/profiles/0.8.0/direct.specs
new file mode 100644
index 0000000000000000000000000000000000000000..7c0b7318975d95282a6f64ce3c987debbed4be30
--- /dev/null
+++ b/scripts/profiles/0.8.0/direct.specs
@@ -0,0 +1,76 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Direct dependencies"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["argobots"]="1.0rc1"
+ ["rocksdb"]="6.2.2"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
+ ["margo"]="v0.6.3"
+ ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+)
diff --git a/scripts/profiles/0.8.0/install/agios.install b/scripts/profiles/0.8.0/install/agios.install
new file mode 100644
index 0000000000000000000000000000000000000000..4ceffc2c3c72aa9afc140148cdfe1b6167f63d6e
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/agios.install
@@ -0,0 +1,53 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/agios"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" ..
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/argobots.install b/scripts/profiles/0.8.0/install/argobots.install
new file mode 100644
index 0000000000000000000000000000000000000000..17c4d2be9f1ea36866307678816f6a6bfb8ca5a3
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/argobots.install
@@ -0,0 +1,56 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/argobots"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.8.0/install/bmi.install b/scripts/profiles/0.8.0/install/bmi.install
new file mode 100644
index 0000000000000000000000000000000000000000..411b4d880783b8559e43887be7a15daf6aea42da
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/bmi.install
@@ -0,0 +1,63 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/bmi"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./prepare
+ cd "${CURR}/build"
+
+ if [[ -n ${CFLAGS} ]]; then
+ CFLAGS="${CFLAGS} -w"
+ else
+ CFLAGS="-w"
+ fi
+
+ ../configure --prefix="${INSTALL_DIR}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/bzip2.install b/scripts/profiles/0.8.0/install/bzip2.install
new file mode 100644
index 0000000000000000000000000000000000000000..fa40bc2655c500b501ff7e12f55bfbc75518c752
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/bzip2.install
@@ -0,0 +1,51 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/bzip2"
+ cd "${CURR}"
+ make install PREFIX="${INSTALL_DIR}"
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/capstone.install b/scripts/profiles/0.8.0/install/capstone.install
new file mode 100644
index 0000000000000000000000000000000000000000..6a43867615efda4fe300de98c474689e9e67c6ab
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/capstone.install
@@ -0,0 +1,53 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/capstone"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release ..
+ make -j"${CORES}" install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/date.install b/scripts/profiles/0.8.0/install/date.install
new file mode 100644
index 0000000000000000000000000000000000000000..cab34851abf27c72a9009120f367764fddfc82ac
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/date.install
@@ -0,0 +1,53 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/date"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/libfabric.install b/scripts/profiles/0.8.0/install/libfabric.install
new file mode 100644
index 0000000000000000000000000000000000000000..09bf2a0890cf6f4ee560796bec5f4e30fe71e85e
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/libfabric.install
@@ -0,0 +1,64 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/libfabric"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes"
+
+ EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}"
+
+ if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then
+ OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}"
+ fi
+
+ ${OFI_CONFIG}
+ make -j${CORES}
+ make install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.8.0/install/libfabric:experimental.install b/scripts/profiles/0.8.0/install/libfabric:experimental.install
new file mode 100644
index 0000000000000000000000000000000000000000..09bf2a0890cf6f4ee560796bec5f4e30fe71e85e
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/libfabric:experimental.install
@@ -0,0 +1,64 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/libfabric"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes"
+
+ EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}"
+
+ if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then
+ OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}"
+ fi
+
+ ${OFI_CONFIG}
+ make -j${CORES}
+ make install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.8.0/install/libfabric:verbs.install b/scripts/profiles/0.8.0/install/libfabric:verbs.install
new file mode 100644
index 0000000000000000000000000000000000000000..09bf2a0890cf6f4ee560796bec5f4e30fe71e85e
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/libfabric:verbs.install
@@ -0,0 +1,64 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/libfabric"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes"
+
+ EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}"
+
+ if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then
+ OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}"
+ fi
+
+ ${OFI_CONFIG}
+ make -j${CORES}
+ make install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.8.0/install/lz4.install b/scripts/profiles/0.8.0/install/lz4.install
new file mode 100644
index 0000000000000000000000000000000000000000..26e549ba3f5a960b3eb9e394a847a3314ad033b5
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/lz4.install
@@ -0,0 +1,52 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/lz4"
+ cd "${CURR}"
+ make -j"${CORES}"
+ make DESTDIR="${INSTALL_DIR}" PREFIX="" install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/margo.install b/scripts/profiles/0.8.0/install/margo.install
new file mode 100644
index 0000000000000000000000000000000000000000..ea1a51d5b2fbaad28dfaba08d0fc83a783b65a25
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/margo.install
@@ -0,0 +1,56 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/margo"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./prepare.sh
+ cd "${CURR}/build"
+ ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3"
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.8.0/install/mercury.install b/scripts/profiles/0.8.0/install/mercury.install
new file mode 100644
index 0000000000000000000000000000000000000000..d2677fd63103b3f6b25018846365da1be1aa39c2
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/mercury.install
@@ -0,0 +1,82 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+
+ set -x
+
+ # if the profile compiles bmi, enable it
+ if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then
+ USE_BMI="-DNA_USE_BMI:BOOL=ON"
+ else
+ USE_BMI="-DNA_USE_BMI:BOOL=OFF"
+ fi
+
+ # if the profile compiles libfabric, enable it
+ if [[ -n "${PROFILE_DEP_NAMES['libfabric']}" ]]; then
+ USE_OFI="-DNA_USE_OFI:BOOL=ON"
+ else
+ USE_OFI="-DNA_USE_OFI:BOOL=OFF"
+ fi
+
+ CURR="${SOURCE_DIR}/mercury"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \
+ -DCMAKE_BUILD_TYPE:STRING=Release \
+ -DBUILD_TESTING:BOOL=ON \
+ -DMERCURY_USE_SM_ROUTING:BOOL=ON \
+ -DMERCURY_USE_SELF_FORWARD:BOOL=ON \
+ -DMERCURY_USE_CHECKSUMS:BOOL=OFF \
+ -DMERCURY_USE_BOOST_PP:BOOL=ON \
+ -DMERCURY_USE_EAGER_BULK:BOOL=ON \
+ -DBUILD_SHARED_LIBS:BOOL=ON \
+ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+ ${USE_BMI} ${USE_OFI} \
+ ..
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/rocksdb.install b/scripts/profiles/0.8.0/install/rocksdb.install
new file mode 100644
index 0000000000000000000000000000000000000000..1a413834210cf6eab951d97c1d7ee902ccb48e29
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/rocksdb.install
@@ -0,0 +1,53 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR=${SOURCE_DIR}/rocksdb
+ cd "${CURR}"
+ make clean
+ USE_RTTI=1 make -j"${CORES}" static_lib
+ INSTALL_PATH="${INSTALL_DIR}" make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/snappy.install b/scripts/profiles/0.8.0/install/snappy.install
new file mode 100644
index 0000000000000000000000000000000000000000..078c728516c818b9200064ee650341c1547a4067
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/snappy.install
@@ -0,0 +1,54 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR=${SOURCE_DIR}/snappy
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"/build
+ $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release ..
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/syscall_intercept.install b/scripts/profiles/0.8.0/install/syscall_intercept.install
new file mode 100644
index 0000000000000000000000000000000000000000..72d5551a640f8ab326f8a383d9042b9710315171
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/syscall_intercept.install
@@ -0,0 +1,53 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR=${SOURCE_DIR}/syscall_intercept
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"/build
+ $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/install/zstd.install b/scripts/profiles/0.8.0/install/zstd.install
new file mode 100644
index 0000000000000000000000000000000000000000..2ab403dbfbca27fb8b8b8f386dc2f9596c349fba
--- /dev/null
+++ b/scripts/profiles/0.8.0/install/zstd.install
@@ -0,0 +1,54 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/zstd/build/cmake"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"/build
+ $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release ..
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.8.0/mogon1.specs b/scripts/profiles/0.8.0/mogon1.specs
new file mode 100644
index 0000000000000000000000000000000000000000..65bee82fbcb83cd2aefa68d21917bc82da716ac0
--- /dev/null
+++ b/scripts/profiles/0.8.0/mogon1.specs
@@ -0,0 +1,83 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Mogon 1 supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["zstd"]="1.3.2"
+ ["lz4"]="1.8.0"
+ ["snappy"]="1.1.7"
+ ["capstone"]="4.0.1"
+ ["argobots"]="1.0rc1"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric:verbs"]="HEAD@v1.7.2"
+ ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
+ ["margo"]="v0.6.3"
+ ["rocksdb"]="6.2.2"
+ ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading/installing
+order=(
+ "zstd" "lz4" "snappy" "capstone" "libfabric:verbs" "mercury"
+ "argobots" "margo" "rocksdb" "syscall_intercept" "date"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric:verbs"]="--enable-verbs=yes"
+)
diff --git a/scripts/profiles/0.8.0/mogon2.specs b/scripts/profiles/0.8.0/mogon2.specs
new file mode 100644
index 0000000000000000000000000000000000000000..0e43065844e294cd324e0a3ba15068d83697e892
--- /dev/null
+++ b/scripts/profiles/0.8.0/mogon2.specs
@@ -0,0 +1,86 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Mogon 2 supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["bzip2"]="1.0.6"
+ ["zstd"]="1.3.2"
+ ["lz4"]="1.8.0"
+ ["snappy"]="1.1.7"
+ ["capstone"]="4.0.1"
+ ["argobots"]="1.0rc1"
+ ["rocksdb:experimental"]="6.11.4"
+ ["psm2"]="11.2.86"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric:experimental"]="HEAD@v1.9.1"
+ ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
+ ["margo"]="v0.6.3"
+ ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "bzip2" "zstd" "lz4" "snappy" "capstone" "libfabric:experimental"
+ "mercury" "argobots" "margo" "rocksdb:experimental" "syscall_intercept"
+ "date" "psm2"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
+)
diff --git a/scripts/profiles/0.8.0/ngio.specs b/scripts/profiles/0.8.0/ngio.specs
new file mode 100644
index 0000000000000000000000000000000000000000..715c08a3467ef7f09298ac788a38c4ae1e1102af
--- /dev/null
+++ b/scripts/profiles/0.8.0/ngio.specs
@@ -0,0 +1,85 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for NEXTGenIO prototype cluster"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["zstd"]="1.3.2"
+ ["lz4"]="1.8.0"
+ ["snappy"]="1.1.7"
+ ["capstone"]="4.0.1"
+ ["argobots"]="1.0rc1"
+ ["rocksdb"]="6.2.2"
+ ["psm2"]="11.2.86"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric:experimental"]="HEAD@v1.9.1"
+ ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
+ ["margo"]="v0.6.3"
+ ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+ ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "zstd" "lz4" "snappy" "capstone" "libfabric:experimental" "mercury"
+ "argobots" "margo" "rocksdb" "syscall_intercept" "date" "psm2" "agios"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
+)
diff --git a/scripts/profiles/latest b/scripts/profiles/latest
new file mode 120000
index 0000000000000000000000000000000000000000..8adc70fdd9d6a086cc636d251adf0c7a7bca8319
--- /dev/null
+++ b/scripts/profiles/latest
@@ -0,0 +1 @@
+0.8.0
\ No newline at end of file
diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list
new file mode 100644
index 0000000000000000000000000000000000000000..4485f3d7794f8d204e4767c19e044e424dc75d1a
--- /dev/null
+++ b/scripts/profiles/sources.list
@@ -0,0 +1,73 @@
+################################################################################
+# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+declare -A sources
+
+sources=(
+ ["bzip2"]="https://sourceforge.net/projects/bzip2/files/bzip2-{{VERSION}}.tar.gz"
+ ["zstd"]="https://github.com/facebook/zstd/archive/v{{VERSION}}.tar.gz"
+ ["lz4"]="https://github.com/lz4/lz4/archive/v{{VERSION}}.tar.gz"
+ ["snappy"]="https://github.com/google/snappy/archive/{{VERSION}}.tar.gz"
+ ["capstone"]="https://github.com/aquynh/capstone/archive/{{VERSION}}.tar.gz"
+ ["argobots"]="https://github.com/pmodels/argobots/archive/v{{VERSION}}.tar.gz"
+ ["rocksdb"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz"
+ ["psm2"]="https://github.com/intel/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz"
+ ["bmi"]="https://github.com/radix-io/bmi/"
+ ["libfabric"]="https://github.com/ofiwg/libfabric.git"
+ ["libfabric:experimental"]="https://github.com/ofiwg/libfabric.git"
+ ["libfabric:verbs"]="https://github.com/ofiwg/libfabric.git"
+ ["mercury"]="https://github.com/mercury-hpc/mercury"
+ ["margo"]="https://xgitlab.cels.anl.gov/sds/margo.git"
+ ["syscall_intercept"]="https://github.com/pmem/syscall_intercept.git"
+ ["date"]="https://github.com/HowardHinnant/date.git"
+ ["agios"]="https://github.com/francielizanon/agios.git"
+)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py
index 70096922aac21c0d410220932f9f3190ee94ab80..fd55f160f8db1a29c222defcb525f8c7ae1e7b6e 100644
--- a/tests/integration/harness/gkfs.py
+++ b/tests/integration/harness/gkfs.py
@@ -267,7 +267,7 @@ class Daemon:
logger.debug("waiting for daemon to be ready")
try:
- self.wait_until_active(self._proc.pid, 10.0)
+ self.wait_until_active(self._proc.pid, 60.0)
except Exception as ex:
logger.error(f"daemon initialization failed: {ex}")
diff --git a/tests/integration/pytest.ini.in b/tests/integration/pytest.ini.in
index d7901fa8f089a69b1604172972092fe3440165dd..e86bcb47697da580f6abc72a1c7f5fdb6634297d 100644
--- a/tests/integration/pytest.ini.in
+++ b/tests/integration/pytest.ini.in
@@ -15,3 +15,4 @@
testpaths = @CMAKE_CURRENT_SOURCE_DIR@
addopts = @PYTEST_BINDIR_ARGS@ @PYTEST_LIBDIR_ARGS@ --interface=@GKFS_TESTS_INTERFACE@
enable_assertion_pass_hook = true
+junit_family = xunit1
diff --git a/tests/integration/pytest.install.ini.in b/tests/integration/pytest.install.ini.in
index 23e87ec25a245c66b43c20037c70ee685e9b4214..7886d3371d5c9925f02bc18d120b8059386b140d 100644
--- a/tests/integration/pytest.install.ini.in
+++ b/tests/integration/pytest.install.ini.in
@@ -15,3 +15,4 @@
testpaths = @CMAKE_INSTALL_FULL_DATAROOTDIR@/gkfs/tests
addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_INTERFACE@
enable_assertion_pass_hook = true
+junit_family = xunit1