From e4eebd359cf8eb497d09e587366ac21aea9bd516 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 7 Oct 2021 12:18:05 +0200 Subject: [PATCH 01/17] Parallax backend added Add command line option for kreon Resolve CRTP specific function name Testing Infrastructure for different databases add path Reactivated destructors, kreondb is destroyed to reduce CI space Returning string_view in fs_data added dbbackend property Updated Readme and MetatadaFactory Adding extra options for kreondb, README+CHANGELOG updated Update .gitlab-ci.yml updating script test... Updated dl_dep tests Kreon test (for metadata) backend. Add command line option for kreon Resolve CRTP specific function name Testing Infrastructure for different databases add path Reactivated destructors, kreondb is destroyed to reduce CI space Returning string_view in fs_data added dbbackend property Adding extra options for kreondb, README+CHANGELOG updated updating script test... Updated dl_dep tests Remove Kreon - Add Parallax Add Parallax Add DockerFile adding parallaxdb option Docker update Missing Parallax hpp Removed Kreon from code, lib conflicts Updated Parallax master Updated DockerFile deps Reformat Changed Base Added ps Reduced Logging and added ps in docker Add Path increase testing timeout Create options.yml automatically Updated out tests --- .gitlab-ci.yml | 25 +- CHANGELOG.md | 2 + CMake/FindParallax.cmake | 51 ++ CMakeLists.txt | 5 +- README.md | 10 + docker/0.9.0-experimental/core/Dockerfile | 41 ++ docker/0.9.0-experimental/core/Makefile | 4 + docker/0.9.0-experimental/coverage/Dockerfile | 20 + docker/0.9.0-experimental/coverage/Makefile | 4 + docker/0.9.0-experimental/deps/Dockerfile | 36 ++ docker/0.9.0-experimental/deps/Makefile | 17 + docker/0.9.0-experimental/linter/Dockerfile | 19 + docker/0.9.0-experimental/linter/Makefile | 4 + docker/0.9.0-experimental/testing/Dockerfile | 14 + docker/0.9.0-experimental/testing/Makefile | 4 + include/client/preload_util.hpp | 2 +- include/daemon/backend/metadata/db.hpp | 42 +- .../backend/metadata/metadata_backend.hpp | 145 ++++++ .../backend/metadata/parallax_backend.hpp | 196 +++++++ .../backend/metadata/rocksdb_backend.hpp | 180 +++++++ include/daemon/classes/fs_data.hpp | 31 ++ scripts/profiles/0.9.0-experimental/all.specs | 86 ++++ scripts/profiles/0.9.0-experimental/ci.specs | 80 +++ .../profiles/0.9.0-experimental/default.specs | 97 ++++ .../profiles/0.9.0-experimental/direct.specs | 77 +++ .../0.9.0-experimental/install/agios.install | 53 ++ .../install/argobots.install | 56 ++ .../0.9.0-experimental/install/bmi.install | 63 +++ .../0.9.0-experimental/install/bzip2.install | 51 ++ .../install/capstone.install | 53 ++ .../0.9.0-experimental/install/date.install | 53 ++ .../install/libfabric.install | 64 +++ .../install/libfabric:experimental.install | 64 +++ .../install/libfabric:verbs.install | 64 +++ .../0.9.0-experimental/install/lz4.install | 52 ++ .../0.9.0-experimental/install/margo.install | 56 ++ .../install/mercury.install | 82 +++ .../install/parallax.install | 58 +++ .../install/rocksdb.install | 53 ++ .../0.9.0-experimental/install/snappy.install | 54 ++ .../install/syscall_intercept.install | 53 ++ .../0.9.0-experimental/install/zstd.install | 54 ++ .../profiles/0.9.0-experimental/mogon1.specs | 84 +++ .../profiles/0.9.0-experimental/mogon2.specs | 87 ++++ .../profiles/0.9.0-experimental/ngio.specs | 86 ++++ scripts/profiles/latest | 2 +- scripts/profiles/sources.list | 1 + src/daemon/backend/metadata/CMakeLists.txt | 11 +- src/daemon/backend/metadata/db.cpp | 201 ++------ .../backend/metadata/parallax_backend.cpp | 487 ++++++++++++++++++ .../backend/metadata/rocksdb_backend.cpp | 348 +++++++++++++ src/daemon/classes/fs_data.cpp | 41 ++ src/daemon/daemon.cpp | 35 +- tests/integration/conftest.py | 21 +- tests/integration/harness/gkfs.py | 17 +- .../compile_dep.sh/0.9.0-experimental/all.out | 14 + .../compile_dep.sh/0.9.0-experimental/ci.out | 8 + .../0.9.0-experimental/default.out | 17 + .../0.9.0-experimental/direct.out | 6 + .../0.9.0-experimental/mogon1.out | 11 + .../0.9.0-experimental/mogon2.out | 14 + .../0.9.0-experimental/ngio.out | 14 + .../dl_dep.sh/0.9.0-experimental/all.out | 15 + .../dl_dep.sh/0.9.0-experimental/ci.out | 9 + .../dl_dep.sh/0.9.0-experimental/default.out | 17 + .../dl_dep.sh/0.9.0-experimental/direct.out | 7 + .../dl_dep.sh/0.9.0-experimental/mogon1.out | 12 + .../dl_dep.sh/0.9.0-experimental/mogon2.out | 14 + .../dl_dep.sh/0.9.0-experimental/ngio.out | 14 + 69 files changed, 3555 insertions(+), 213 deletions(-) create mode 100644 CMake/FindParallax.cmake create mode 100644 docker/0.9.0-experimental/core/Dockerfile create mode 100644 docker/0.9.0-experimental/core/Makefile create mode 100644 docker/0.9.0-experimental/coverage/Dockerfile create mode 100644 docker/0.9.0-experimental/coverage/Makefile create mode 100644 docker/0.9.0-experimental/deps/Dockerfile create mode 100644 docker/0.9.0-experimental/deps/Makefile create mode 100644 docker/0.9.0-experimental/linter/Dockerfile create mode 100644 docker/0.9.0-experimental/linter/Makefile create mode 100644 docker/0.9.0-experimental/testing/Dockerfile create mode 100644 docker/0.9.0-experimental/testing/Makefile create mode 100644 include/daemon/backend/metadata/metadata_backend.hpp create mode 100644 include/daemon/backend/metadata/parallax_backend.hpp create mode 100644 include/daemon/backend/metadata/rocksdb_backend.hpp create mode 100644 scripts/profiles/0.9.0-experimental/all.specs create mode 100644 scripts/profiles/0.9.0-experimental/ci.specs create mode 100644 scripts/profiles/0.9.0-experimental/default.specs create mode 100644 scripts/profiles/0.9.0-experimental/direct.specs create mode 100644 scripts/profiles/0.9.0-experimental/install/agios.install create mode 100644 scripts/profiles/0.9.0-experimental/install/argobots.install create mode 100644 scripts/profiles/0.9.0-experimental/install/bmi.install create mode 100644 scripts/profiles/0.9.0-experimental/install/bzip2.install create mode 100644 scripts/profiles/0.9.0-experimental/install/capstone.install create mode 100644 scripts/profiles/0.9.0-experimental/install/date.install create mode 100644 scripts/profiles/0.9.0-experimental/install/libfabric.install create mode 100644 scripts/profiles/0.9.0-experimental/install/libfabric:experimental.install create mode 100644 scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install create mode 100644 scripts/profiles/0.9.0-experimental/install/lz4.install create mode 100644 scripts/profiles/0.9.0-experimental/install/margo.install create mode 100644 scripts/profiles/0.9.0-experimental/install/mercury.install create mode 100644 scripts/profiles/0.9.0-experimental/install/parallax.install create mode 100644 scripts/profiles/0.9.0-experimental/install/rocksdb.install create mode 100644 scripts/profiles/0.9.0-experimental/install/snappy.install create mode 100644 scripts/profiles/0.9.0-experimental/install/syscall_intercept.install create mode 100644 scripts/profiles/0.9.0-experimental/install/zstd.install create mode 100644 scripts/profiles/0.9.0-experimental/mogon1.specs create mode 100644 scripts/profiles/0.9.0-experimental/mogon2.specs create mode 100644 scripts/profiles/0.9.0-experimental/ngio.specs create mode 100644 src/daemon/backend/metadata/parallax_backend.cpp create mode 100644 src/daemon/backend/metadata/rocksdb_backend.cpp create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/all.out create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/default.out create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/mogon1.out create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/all.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/default.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00a5d74b4..c56ee4d74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.0 +image: gekkofs/core:0.9.0-experimental ################################################################################ ## Validating @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.0 + image: gekkofs/deps:0.9.0-experimental interruptible: true needs: [] script: @@ -71,7 +71,7 @@ gkfs: gkfwd: stage: build - image: gekkofs/deps:0.9.0 + image: gekkofs/deps:0.9.0-experimental interruptible: true needs: [] script: @@ -104,15 +104,13 @@ gkfwd: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.0-experimental needs: [] - parallel: - matrix: - - SUBTEST: [ dl_dep.sh, compile_dep.sh ] script: - mkdir -p ${BUILD_PATH}/tests/scripts - cd ${BUILD_PATH}/tests/scripts - - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml + - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit + - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit artifacts: expire_in: 1 week reports: @@ -122,7 +120,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.0-experimental interruptible: true needs: ['gkfs'] parallel: @@ -131,9 +129,10 @@ gkfs:integration: script: ## run tests + - export PATH=${PATH}:/usr/local/bin - mkdir -p ${BUILD_PATH}/tests/run - cd ${BUILD_PATH}/tests/integration - - ${PYTEST} -v -n $(nproc) + - ${PYTEST} -v -n 1 ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST} --basetemp=${BUILD_PATH}/tests/run/${SUBTEST} --junit-xml=report.xml @@ -167,7 +166,7 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.0-experimental interruptible: true needs: ['gkfwd'] parallel: @@ -212,9 +211,11 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.0-experimental needs: ['gkfs'] script: + ## Add path to mkfs.kreon + - export PATH=${PATH}:/usr/local/bin ## run actual tests - cd ${BUILD_PATH}/tests/unit - ctest -j $(nproc) -L unit::all --output-junit report.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index ead8b557e..aabccf79f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,8 @@ Note that tests still require `Boost_preprocessor`. ([!116](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/116)). - Fixed an issue where `LOG_OUTPUT_TRUNC` did not work as expected ([!118](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/118)). +- Added new metadata backend, kreon. + ## [0.8.0] - 2020-09-15 ### New - Both client library and daemon have been extended to support the ofi+verbs diff --git a/CMake/FindParallax.cmake b/CMake/FindParallax.cmake new file mode 100644 index 000000000..e6cce2d25 --- /dev/null +++ b/CMake/FindParallax.cmake @@ -0,0 +1,51 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +find_path(PARALLAX_INCLUDE_DIR + NAMES parallax.h + ) + +find_library(PARALLAX_LIBRARY + NAMES parallax + ) + +find_library(LOG_LIBRARY + NAMES log + ) + +set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR}) +set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY}) +set(PARALLAX_LIBRARIES ${LOG_LIBRARY}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY LOG_LIBRARY PARALLAX_INCLUDE_DIR) + +mark_as_advanced( + PARALLAX_LIBRARY + PARALLAX_INCLUDE_DIR + LOG_LIBRARY +) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f7b9ca6d..440397a40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,8 +128,10 @@ find_package(Mercury REQUIRED) find_package(Abt REQUIRED) find_package(Margo REQUIRED) find_package(Syscall_intercept REQUIRED) +find_package(Parallax REQUIRED) find_package(Threads REQUIRED) - +# parallax +find_package(yaml REQUIRED) # some compilers need extra flags for std::filesystem, such as -lstdc++fs, this # produces a std::filesystem imported target that takes care of all these # details transparently @@ -208,6 +210,7 @@ target_link_libraries(RocksDB ${LZ4_LIBRARIES} ) + if (${JeMalloc_FOUND}) target_link_libraries(RocksDB INTERFACE diff --git a/README.md b/README.md index f2c861414..e47526930 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,16 @@ Options: --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) -c,--clean-rootdir Cleans Rootdir >before< launching the deamon --version Print version and exit. + + --dbbackend 'rocksdb' (default) or 'kreondb' can be specified as + metadata backend. + --keepmd 'kreondb' specific, persist the metadata file + (default off, file is deleted at the end) + --reusemd 'kreondb' specific, do not recreate the + metadata file, (default re/create) + --kreonsize 'kreondb' specific, size of the metadata file in GB + (default 16, 16 GB) + --version Print version and exit. ``` It is possible to run multiple independent GekkoFS instances on the same node. Note, that when these GekkoFS instances diff --git a/docker/0.9.0-experimental/core/Dockerfile b/docker/0.9.0-experimental/core/Dockerfile new file mode 100644 index 000000000..1c42f9edd --- /dev/null +++ b/docker/0.9.0-experimental/core/Dockerfile @@ -0,0 +1,41 @@ +FROM debian:buster-slim + +LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ + # RocksDB dependencies + libsnappy-dev \ + liblz4-dev \ + libzstd-dev \ + libbz2-dev \ + zlib1g-dev \ + # syscall_intercept dependencies + libcapstone-dev \ + # GekkoFS dependencies + libboost-filesystem-dev \ + libboost-program-options-dev \ + uuid-dev && \ + # install cmake 3.14 since it's needed for some dependencies + curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \ + ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean && \ + rm ./cmake-3.14.5-Linux-x86_64.sh diff --git a/docker/0.9.0-experimental/core/Makefile b/docker/0.9.0-experimental/core/Makefile new file mode 100644 index 000000000..808a84458 --- /dev/null +++ b/docker/0.9.0-experimental/core/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + docker build -t gekkofs/core:0.9.0-experimental . diff --git a/docker/0.9.0-experimental/coverage/Dockerfile b/docker/0.9.0-experimental/coverage/Dockerfile new file mode 100644 index 000000000..fb935e0ce --- /dev/null +++ b/docker/0.9.0-experimental/coverage/Dockerfile @@ -0,0 +1,20 @@ +FROM debian:buster-slim + +LABEL Description="Environment to generate coverage reports in GekkoFS" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + git \ + cmake \ + gcc \ + g++ \ + lcov \ + python3 \ + python3-pip \ + python3-setuptools && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoclean && \ + python3 -m pip install --upgrade pip && \ + pip3 install gcovr diff --git a/docker/0.9.0-experimental/coverage/Makefile b/docker/0.9.0-experimental/coverage/Makefile new file mode 100644 index 000000000..cc8d4af22 --- /dev/null +++ b/docker/0.9.0-experimental/coverage/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + docker build -t gekkofs/coverage:0.8.0 . diff --git a/docker/0.9.0-experimental/deps/Dockerfile b/docker/0.9.0-experimental/deps/Dockerfile new file mode 100644 index 000000000..7cd2fbc82 --- /dev/null +++ b/docker/0.9.0-experimental/deps/Dockerfile @@ -0,0 +1,36 @@ +FROM gekkofs/core:0.9.0-experimental + +LABEL Description="Debian-based environment to build GekkoFS" + +ENV GKFS_PATH /opt/gkfs +ENV GKFS_VERSION 0.9.0-experimental + +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH /usr/local + +COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/patches ${SCRIPTS_PATH}/patches +COPY scripts/profiles ${SCRIPTS_PATH}/profiles + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python3-setuptools \ + libnuma-dev libyaml-dev procps && \ + python3 -m pip install --upgrade pip && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && apt-get autoclean + +# Download and build dependencies +RUN cd ${SCRIPTS_PATH} && \ + /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \ + /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \ + rm -rf ${DEPS_SRC_PATH} && \ + rm -rf ${SCRIPTS_PATH} && \ + rmdir ${GKFS_PATH} && \ + ldconfig diff --git a/docker/0.9.0-experimental/deps/Makefile b/docker/0.9.0-experimental/deps/Makefile new file mode 100644 index 000000000..8f869fa9d --- /dev/null +++ b/docker/0.9.0-experimental/deps/Makefile @@ -0,0 +1,17 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts umount-scripts + +mount-scripts: + mkdir -p $(CWD)/scripts + mount --bind $(GIT_ROOT)/scripts $(CWD)/scripts + +build: mount-scripts + docker build -t gekkofs/deps:0.9.0-experimental . + +umount-scripts: + - umount $(CWD)/scripts + - rmdir $(CWD)/scripts diff --git a/docker/0.9.0-experimental/linter/Dockerfile b/docker/0.9.0-experimental/linter/Dockerfile new file mode 100644 index 000000000..4f93de54d --- /dev/null +++ b/docker/0.9.0-experimental/linter/Dockerfile @@ -0,0 +1,19 @@ +FROM gekkofs/core:0.8.0 + +LABEL Description="Debian-based environment to check the formatting of GekkoFS code" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # clang 10 deps for clang-format + lsb-release \ + wget \ + software-properties-common \ + gnupg2 && \ + # add clang-10 repos + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 10 && \ + # install clang-format + apt-get update && apt-get install -y --no-install-recommends clang-format-10 && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.0-experimental/linter/Makefile b/docker/0.9.0-experimental/linter/Makefile new file mode 100644 index 000000000..f6752471b --- /dev/null +++ b/docker/0.9.0-experimental/linter/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + docker build -t gekkofs/linter:0.8.0 . diff --git a/docker/0.9.0-experimental/testing/Dockerfile b/docker/0.9.0-experimental/testing/Dockerfile new file mode 100644 index 000000000..775653b20 --- /dev/null +++ b/docker/0.9.0-experimental/testing/Dockerfile @@ -0,0 +1,14 @@ +FROM gekkofs/deps:0.9.0-experimental + +LABEL Description="Debian-based environment to test GekkoFS" + +RUN \ + # install cmake 3.21 since we need to produce JUnit XML files + curl -OL https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.sh && \ + chmod u+x ./cmake-3.21.1-linux-x86_64.sh && \ + ./cmake-3.21.1-linux-x86_64.sh --skip-license --prefix=/usr && \ + # install gcovr + # (required for partial coverage reports in parallel runs) + pip3 install gcovr && \ + # cleanup + rm ./cmake-3.21.1-linux-x86_64.sh diff --git a/docker/0.9.0-experimental/testing/Makefile b/docker/0.9.0-experimental/testing/Makefile new file mode 100644 index 000000000..416376397 --- /dev/null +++ b/docker/0.9.0-experimental/testing/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + docker build -t gekkofs/testing:0.9.0-experimental . diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index 9ccb8833b..2849c87e0 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -32,7 +32,7 @@ #include #include - +#include #include #include #include diff --git a/include/daemon/backend/metadata/db.hpp b/include/daemon/backend/metadata/db.hpp index 80e06f740..6b87af686 100644 --- a/include/daemon/backend/metadata/db.hpp +++ b/include/daemon/backend/metadata/db.hpp @@ -34,50 +34,28 @@ #include #include -#include #include #include +#include +#include +#include namespace rdb = rocksdb; namespace gkfs::metadata { -/** - * @brief MetadataDB class providing an abstraction layer to the KV store - * RocksDB. - */ + class MetadataDB { private: - std::unique_ptr db; ///< RocksDB instance - rdb::Options options; ///< RocksDB configuration - rdb::WriteOptions write_opts; ///< RocksDB write configuration - std::string path; ///< Path to where RocksDB persists its data - std::shared_ptr log_; ///< MetadataDB internal logger + std::string path_; + std::shared_ptr log_; + std::unique_ptr backend_; - /** - * @brief Sets up specific settings to optimize RocksDB instance to - * environment on launch. - * @param options RocksDB configurations - */ - static void - optimize_rocksdb_options(rdb::Options& options); public: - /** - * @brief Exception wrapper on Status object. - * @param s RocksDB status - * @throws Throws NotFoundException if s.IsNotFound(), general DBException - * otherwise - */ - static inline void - throw_rdb_status_excpt(const rdb::Status& s); + MetadataDB(const std::string& path, const std::string_view database); - /** - * @brief Constructor, called when daemon is started and connects to KV - * store. - * @param path Path to where RocksDB persists its data - */ - explicit MetadataDB(const std::string& path); + ~MetadataDB(); /** * @brief Gets the KV store value for a key. @@ -182,7 +160,7 @@ public: * is therefore unused. */ void - iterate_all(); + iterate_all() const; }; } // namespace gkfs::metadata diff --git a/include/daemon/backend/metadata/metadata_backend.hpp b/include/daemon/backend/metadata/metadata_backend.hpp new file mode 100644 index 000000000..a22392496 --- /dev/null +++ b/include/daemon/backend/metadata/metadata_backend.hpp @@ -0,0 +1,145 @@ +/* + Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_METADATA_BACKEND_HPP +#define GEKKOFS_METADATA_BACKEND_HPP + +#include +#include +#include +#include + +namespace gkfs::metadata { + + +class AbstractMetadataBackend { +public: + virtual ~AbstractMetadataBackend() = default; + + virtual std::string + get(const std::string& key) const = 0; + + virtual void + put(const std::string& key, const std::string& val) = 0; + + virtual void + put_no_exist(const std::string& key, const std::string& val) = 0; + + virtual void + remove(const std::string& key) = 0; + + virtual bool + exists(const std::string& key) = 0; + + virtual void + update(const std::string& old_key, const std::string& new_key, + const std::string& val) = 0; + + virtual void + increase_size(const std::string& key, size_t size, bool append) = 0; + + virtual void + decrease_size(const std::string& key, size_t size) = 0; + + virtual std::vector> + get_dirents(const std::string& dir) const = 0; + + virtual std::vector> + get_dirents_extended(const std::string& dir) const = 0; + + virtual void + iterate_all() const = 0; +}; + +template +class MetadataBackend : public AbstractMetadataBackend { +private: + std::string path; + std::shared_ptr log_; + +public: + std::string + get(const std::string& key) const { + return static_cast(*this).get_impl(key); + } + + void + put(const std::string& key, const std::string& val) { + static_cast(*this).put_impl(key, val); + } + + void + put_no_exist(const std::string& key, const std::string& val) { + static_cast(*this).put_no_exist_impl(key, val); + } + + void + remove(const std::string& key) { + static_cast(*this).remove_impl(key); + } + + bool + exists(const std::string& key) { + return static_cast(*this).exists_impl(key); + } + + void + update(const std::string& old_key, const std::string& new_key, + const std::string& val) { + static_cast(*this).update_impl(old_key, new_key, val); + } + + void + increase_size(const std::string& key, size_t size, bool append) { + static_cast(*this).increase_size_impl(key, size, append); + } + + void + decrease_size(const std::string& key, size_t size) { + static_cast(*this).decrease_size_impl(key, size); + } + + std::vector> + get_dirents(const std::string& dir) const { + return static_cast(*this).get_dirents_impl(dir); + } + + std::vector> + get_dirents_extended(const std::string& dir) const { + return static_cast(*this).get_dirents_extended_impl(dir); + } + + void + iterate_all() const { + static_cast(*this).iterate_all_impl(); + } +}; + +} // namespace gkfs::metadata + +#endif // GEKKOFS_METADATA_BACKEND_HPP diff --git a/include/daemon/backend/metadata/parallax_backend.hpp b/include/daemon/backend/metadata/parallax_backend.hpp new file mode 100644 index 000000000..deb55bfbd --- /dev/null +++ b/include/daemon/backend/metadata/parallax_backend.hpp @@ -0,0 +1,196 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_METADATA_PARALLAXBACKEND_HPP +#define GEKKOFS_METADATA_PARALLAXBACKEND_HPP + +#include +#include +#include +#include +extern "C" { +#include +} + +namespace rdb = rocksdb; +namespace gkfs::metadata { + +class ParallaxBackend : public MetadataBackend { +private: + par_handle par_db_; + par_db_options par_options_; + std::string par_path_; + + /** + * Convert a String to klc_key + * @param key + * @param klc_key struct + */ + inline void + str2par(const std::string& value, struct par_value& V) const; + + + /** + * Convert a String to klc_value + * @param value + * @param klc_value struct + */ + inline void + str2par(const std::string& key, struct par_key& K) const; + +public: + /** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ + explicit ParallaxBackend(const std::string& path); + + /** + * Exception wrapper on Status object. Throws NotFoundException if + * s == "Not Found", general DBException otherwise + * @param String with status + * @throws DBException + */ + static inline void + throw_status_excpt(const std::string& s); + + /** + * @brief Destroy the Kreon Backend:: Kreon Backend object + * We remove the file, too large for the CI. + * TODO: Insert option + */ + virtual ~ParallaxBackend(); + + /** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + std::string + get_impl(const std::string& key) const; + + /** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ + void + put_impl(const std::string& key, const std::string& val); + + /** + * Puts an entry into the KV store if it doesn't exist. This function does + * not use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ + void + put_no_exist_impl(const std::string& key, const std::string& val); + + /** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + remove_impl(const std::string& key); + + /** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ + bool + exists_impl(const std::string& key); + + /** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + update_impl(const std::string& old_key, const std::string& new_key, + const std::string& val); + + /** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ + void + increase_size_impl(const std::string& key, size_t size, bool append); + + /** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ + void + decrease_size_impl(const std::string& key, size_t size); + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_impl(const std::string& dir) const; + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_extended_impl(const std::string& dir) const; + + /** + * Code example for iterating all entries in KV store. This is for debug + * only as it is too expensive + */ + void + iterate_all_impl() const; +}; + +} // namespace gkfs::metadata + +#endif // GEKKOFS_METADATA_KREONBACKEND_HPP diff --git a/include/daemon/backend/metadata/rocksdb_backend.hpp b/include/daemon/backend/metadata/rocksdb_backend.hpp new file mode 100644 index 000000000..d889c2263 --- /dev/null +++ b/include/daemon/backend/metadata/rocksdb_backend.hpp @@ -0,0 +1,180 @@ +/* + Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_METADATA_ROCKSDBBACKEND_HPP +#define GEKKOFS_METADATA_ROCKSDBBACKEND_HPP + +#include +#include +#include +#include +#include + +namespace rdb = rocksdb; + +namespace gkfs::metadata { + +/** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ +class RocksDBBackend : public MetadataBackend { +private: + std::unique_ptr db_; + rdb::Options options_; + rdb::WriteOptions write_opts_; + +public: + explicit RocksDBBackend(const std::string& path); + + virtual ~RocksDBBackend(); + + /** + * Exception wrapper on Status object. Throws NotFoundException if + * s.IsNotFound(), general DBException otherwise + * @param RocksDB status + * @throws DBException + */ + static inline void + throw_status_excpt(const rdb::Status& s); + + /** + * Used for setting KV store settings + */ + void + optimize_database_impl(); + + /** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + std::string + get_impl(const std::string& key) const; + + /** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ + void + put_impl(const std::string& key, const std::string& val); + + /** + * Puts an entry into the KV store if it doesn't exist. This function does + * not use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ + void + put_no_exist_impl(const std::string& key, const std::string& val); + + /** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + remove_impl(const std::string& key); + + /** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ + bool + exists_impl(const std::string& key); + + + /** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + update_impl(const std::string& old_key, const std::string& new_key, + const std::string& val); + + /** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ + void + increase_size_impl(const std::string& key, size_t size, bool append); + + /** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ + void + decrease_size_impl(const std::string& key, size_t size); + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_impl(const std::string& dir) const; + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_extended_impl(const std::string& dir) const; + + /** + * Code example for iterating all entries in KV store. This is for debug + * only as it is too expensive + */ + void + iterate_all_impl() const; +}; + +} // namespace gkfs::metadata + +#endif // GEKKOFS_METADATA_ROCKSDBBACKEND_HPP diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index e38fea795..071ca8821 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -34,6 +34,7 @@ #include #include #include //std::hash +#include /* Forward declarations */ namespace gkfs { @@ -69,6 +70,12 @@ private: // Database std::shared_ptr mdb_; + std::string dbbackend_; + // Kreon + bool reuse_md_ = false; + bool keep_md_ = false; + unsigned long long size_md_ = 8589934592ull; + // Storage backend std::shared_ptr storage_; @@ -123,6 +130,12 @@ public: void metadir(const std::string& metadir_); + std::string_view + dbbackend() const; + + void + dbbackend(const std::string& dbbackend_); + const std::shared_ptr& mdb() const; @@ -191,6 +204,24 @@ public: void blocks_state(bool blocks_state); + + bool + kreon_keep_md() const; + + void + kreon_keep_md(bool keep_md); + + bool + kreon_reuse_md() const; + + void + kreon_reuse_md(bool reuse_md); + + unsigned long long + kreon_size_md() const; + + void + kreon_size_md(unsigned int size_md); }; } // namespace daemon diff --git a/scripts/profiles/0.9.0-experimental/all.specs b/scripts/profiles/0.9.0-experimental/all.specs new file mode 100644 index 000000000..6eafe8f3a --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/all.specs @@ -0,0 +1,86 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies (except transport-specific and experimental)" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["bzip2"]="1.0.6" + ["zstd"]="1.3.2" + ["lz4"]="1.8.0" + ["snappy"]="1.1.7" + ["capstone"]="4.0.1" + ["argobots"]="1.0rc1" + ["rocksdb"]="6.2.2" +) + +# Dependencies that must be cloned +clonedeps=( + ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8" + ["libfabric"]="HEAD@v1.8.1" + ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" + ["margo"]="v0.6.3" + ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "libfabric" "mercury" "argobots" + "margo" "rocksdb" "syscall_intercept" "date" "agios" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.0-experimental/ci.specs b/scripts/profiles/0.9.0-experimental/ci.specs new file mode 100644 index 000000000..625ae6234 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/ci.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies required by the CI" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.0rc1" + ["rocksdb"]="6.2.2" + ) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.8.1" + ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" + ["margo"]="v0.6.3" + ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "date" "agios" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.0-experimental/default.specs b/scripts/profiles/0.9.0-experimental/default.specs new file mode 100644 index 000000000..29ed724e4 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/default.specs @@ -0,0 +1,97 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["bzip2"]="1.0.6" + ["zstd"]="1.3.2" + ["lz4"]="1.8.0" + ["snappy"]="1.1.7" + ["capstone"]="4.0.1" + ["argobots"]="1.0rc1" + ["rocksdb"]="6.2.2" + ["rocksdb%experimental"]="6.11.4" + ["psm2"]="11.2.86" +) + +# Dependencies that must be cloned +clonedeps=( + ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8" + ["libfabric"]="HEAD@v1.8.1" + ["libfabric%experimental"]="HEAD@v1.9.1" + ["libfabric%verbs"]="HEAD@v1.7.2" + ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" + ["margo"]="v0.6.3" + ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" + "libfabric" + "libfabric%experimental" + "libfabric%verbs" + "mercury" "argobots" "margo" "rocksdb" "rocksdb%experimental" + "syscall_intercept" "date" "psm2" "agios" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" + ["libfabric%verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" + ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" +) diff --git a/scripts/profiles/0.9.0-experimental/direct.specs b/scripts/profiles/0.9.0-experimental/direct.specs new file mode 100644 index 000000000..aeed1bdf6 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/direct.specs @@ -0,0 +1,77 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Direct dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.0rc1" + ["rocksdb"]="6.2.2" +) + +# Dependencies that must be cloned +clonedeps=( + ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" + ["margo"]="v0.6.3" + ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.0-experimental/install/agios.install b/scripts/profiles/0.9.0-experimental/install/agios.install new file mode 100644 index 000000000..4ceffc2c3 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/agios.install @@ -0,0 +1,53 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/agios" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/argobots.install b/scripts/profiles/0.9.0-experimental/install/argobots.install new file mode 100644 index 000000000..17c4d2be9 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/argobots.install @@ -0,0 +1,56 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/argobots" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.0-experimental/install/bmi.install b/scripts/profiles/0.9.0-experimental/install/bmi.install new file mode 100644 index 000000000..411b4d880 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/bmi.install @@ -0,0 +1,63 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/bmi" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare + cd "${CURR}/build" + + if [[ -n ${CFLAGS} ]]; then + CFLAGS="${CFLAGS} -w" + else + CFLAGS="-w" + fi + + ../configure --prefix="${INSTALL_DIR}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/bzip2.install b/scripts/profiles/0.9.0-experimental/install/bzip2.install new file mode 100644 index 000000000..fa40bc265 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/bzip2.install @@ -0,0 +1,51 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/bzip2" + cd "${CURR}" + make install PREFIX="${INSTALL_DIR}" +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/capstone.install b/scripts/profiles/0.9.0-experimental/install/capstone.install new file mode 100644 index 000000000..6a4386761 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/capstone.install @@ -0,0 +1,53 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/capstone" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/date.install b/scripts/profiles/0.9.0-experimental/install/date.install new file mode 100644 index 000000000..cab34851a --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/date.install @@ -0,0 +1,53 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/date" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON .. + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/libfabric.install b/scripts/profiles/0.9.0-experimental/install/libfabric.install new file mode 100644 index 000000000..09bf2a089 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/libfabric.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/libfabric" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j${CORES} + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.0-experimental/install/libfabric:experimental.install b/scripts/profiles/0.9.0-experimental/install/libfabric:experimental.install new file mode 100644 index 000000000..09bf2a089 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/libfabric:experimental.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/libfabric" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j${CORES} + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install b/scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install new file mode 100644 index 000000000..09bf2a089 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/libfabric" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j${CORES} + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.0-experimental/install/lz4.install b/scripts/profiles/0.9.0-experimental/install/lz4.install new file mode 100644 index 000000000..26e549ba3 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/lz4.install @@ -0,0 +1,52 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/lz4" + cd "${CURR}" + make -j"${CORES}" + make DESTDIR="${INSTALL_DIR}" PREFIX="" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/margo.install b/scripts/profiles/0.9.0-experimental/install/margo.install new file mode 100644 index 000000000..ea1a51d5b --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/margo.install @@ -0,0 +1,56 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/margo" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.0-experimental/install/mercury.install b/scripts/profiles/0.9.0-experimental/install/mercury.install new file mode 100644 index 000000000..d2677fd63 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/mercury.install @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + set -x + + # if the profile compiles bmi, enable it + if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then + USE_BMI="-DNA_USE_BMI:BOOL=ON" + else + USE_BMI="-DNA_USE_BMI:BOOL=OFF" + fi + + # if the profile compiles libfabric, enable it + if [[ -n "${PROFILE_DEP_NAMES['libfabric']}" ]]; then + USE_OFI="-DNA_USE_OFI:BOOL=ON" + else + USE_OFI="-DNA_USE_OFI:BOOL=OFF" + fi + + CURR="${SOURCE_DIR}/mercury" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_SM_ROUTING:BOOL=ON \ + -DMERCURY_USE_SELF_FORWARD:BOOL=ON \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DMERCURY_USE_EAGER_BULK:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + ${USE_BMI} ${USE_OFI} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/parallax.install b/scripts/profiles/0.9.0-experimental/install/parallax.install new file mode 100644 index 000000000..d03d42ba1 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/parallax.install @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + CURR="${SOURCE_DIR}/parallax" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DBUILD_SHARED_LIBS:BOOL=OFF \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/rocksdb.install b/scripts/profiles/0.9.0-experimental/install/rocksdb.install new file mode 100644 index 000000000..7e14bbaab --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/rocksdb.install @@ -0,0 +1,53 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR=${SOURCE_DIR}/rocksdb + cd "${CURR}" + make clean + PORTABLE=1 USE_SSE=1 USE_RTTI=1 make -j"${CORES}" static_lib + INSTALL_PATH="${INSTALL_DIR}" make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/snappy.install b/scripts/profiles/0.9.0-experimental/install/snappy.install new file mode 100644 index 000000000..078c72851 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/snappy.install @@ -0,0 +1,54 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR=${SOURCE_DIR}/snappy + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/syscall_intercept.install b/scripts/profiles/0.9.0-experimental/install/syscall_intercept.install new file mode 100644 index 000000000..72d5551a6 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/syscall_intercept.install @@ -0,0 +1,53 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR=${SOURCE_DIR}/syscall_intercept + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/install/zstd.install b/scripts/profiles/0.9.0-experimental/install/zstd.install new file mode 100644 index 000000000..2ab403dbf --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/zstd.install @@ -0,0 +1,54 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/zstd/build/cmake" + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/mogon1.specs b/scripts/profiles/0.9.0-experimental/mogon1.specs new file mode 100644 index 000000000..722cbe3b6 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/mogon1.specs @@ -0,0 +1,84 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 1 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["zstd"]="1.3.2" + ["lz4"]="1.8.0" + ["snappy"]="1.1.7" + ["capstone"]="4.0.1" + ["argobots"]="1.0rc1" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%verbs"]="HEAD@v1.7.2" + ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" + ["margo"]="v0.6.3" + ["rocksdb"]="6.2.2" + ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading/installing +order=( + "zstd" "lz4" "snappy" "capstone" "libfabric%verbs" "mercury" + "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%verbs"]="--enable-verbs=yes" +) diff --git a/scripts/profiles/0.9.0-experimental/mogon2.specs b/scripts/profiles/0.9.0-experimental/mogon2.specs new file mode 100644 index 000000000..22b16a4a9 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/mogon2.specs @@ -0,0 +1,87 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["bzip2"]="1.0.6" + ["zstd"]="1.3.2" + ["lz4"]="1.8.0" + ["snappy"]="1.1.7" + ["capstone"]="4.0.1" + ["argobots"]="1.0rc1" + ["rocksdb%experimental"]="6.11.4" + ["psm2"]="11.2.86" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%experimental"]="HEAD@v1.9.1" + ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" + ["margo"]="v0.6.3" + ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "bzip2" "zstd" "lz4" "snappy" "capstone" "libfabric%experimental" + "mercury" "argobots" "margo" "rocksdb%experimental" "syscall_intercept" + "date" "psm2" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.0-experimental/ngio.specs b/scripts/profiles/0.9.0-experimental/ngio.specs new file mode 100644 index 000000000..114a27707 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/ngio.specs @@ -0,0 +1,86 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for NEXTGenIO prototype cluster" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["zstd"]="1.3.2" + ["lz4"]="1.8.0" + ["snappy"]="1.1.7" + ["capstone"]="4.0.1" + ["argobots"]="1.0rc1" + ["rocksdb"]="6.2.2" + ["psm2"]="11.2.86" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%experimental"]="HEAD@v1.9.1" + ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" + ["margo"]="v0.6.3" + ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "zstd" "lz4" "snappy" "capstone" "libfabric%experimental" "mercury" + "argobots" "margo" "rocksdb" "syscall_intercept" "date" "psm2" "agios" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/latest b/scripts/profiles/latest index 899f24fc7..9abbf6810 120000 --- a/scripts/profiles/latest +++ b/scripts/profiles/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.0-experimental \ No newline at end of file diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index 0c8788837..2226c2c30 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -50,6 +50,7 @@ sources=( ["date"]="https://github.com/HowardHinnant/date.git" ["agios"]="https://github.com/francielizanon/agios.git" ["json-c"]="https://github.com/json-c/json-c/archive/json-c-{{VERSION}}.tar.gz" + ["parallax"]="https://github.com/CARV-ICS-FORTH/parallax.git" ) diff --git a/src/daemon/backend/metadata/CMakeLists.txt b/src/daemon/backend/metadata/CMakeLists.txt index fc4961fc8..9219958d8 100644 --- a/src/daemon/backend/metadata/CMakeLists.txt +++ b/src/daemon/backend/metadata/CMakeLists.txt @@ -47,10 +47,15 @@ target_sources(metadata_db PUBLIC ${INCLUDE_DIR}/daemon/backend/metadata/db.hpp ${INCLUDE_DIR}/daemon/backend/exceptions.hpp + ${INCLUDE_DIR}/daemon/backend/metadata/metadata_backend.hpp + ${INCLUDE_DIR}/daemon/backend/metadata/rocksdb_backend.hpp + ${INCLUDE_DIR}/daemon/backend/metadata/parallax_backend.hpp PRIVATE ${INCLUDE_DIR}/daemon/backend/metadata/merge.hpp ${CMAKE_CURRENT_LIST_DIR}/merge.cpp ${CMAKE_CURRENT_LIST_DIR}/db.cpp + ${CMAKE_CURRENT_LIST_DIR}/rocksdb_backend.cpp + ${CMAKE_CURRENT_LIST_DIR}/parallax_backend.cpp ) add_library(metadata_module @@ -68,9 +73,13 @@ target_link_libraries(metadata_module PUBLIC spdlog ) +add_library(Parallax INTERFACE IMPORTED GLOBAL) target_link_libraries(metadata_db - # Required by RocksDB + ${PARALLAX_LIBRARY} + ${LOG_LIBRARY} + yaml + # Required by RocksDB metadata_module -ldl metadata diff --git a/src/daemon/backend/metadata/db.cpp b/src/daemon/backend/metadata/db.cpp index 440c97ad7..19ac84506 100644 --- a/src/daemon/backend/metadata/db.cpp +++ b/src/daemon/backend/metadata/db.cpp @@ -33,14 +33,31 @@ #include #include +#include extern "C" { #include } + namespace gkfs::metadata { -// private functions +/** + * Factory to create DB instances + * @param path where KV store data is stored + * @param id parallax or rocksdb (default) backend + */ +struct MetadataDBFactory { + static std::unique_ptr + create(const std::string& path, const std::string_view id) { + + if(id == "parallaxdb") { + return std::make_unique(path); + } + // Default rocksdb + return std::make_unique(path); + } +}; /** * @internal @@ -49,56 +66,31 @@ namespace gkfs::metadata { * see here: https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide * @endinternal */ -void -MetadataDB::optimize_rocksdb_options(rdb::Options& options) { - options.max_successive_merges = 128; -} - -// public functions - -void -MetadataDB::throw_rdb_status_excpt(const rdb::Status& s) { - assert(!s.ok()); - - if(s.IsNotFound()) { - throw NotFoundException(s.ToString()); - } else { - throw DBException(s.ToString()); - } -} +MetadataDB::MetadataDB(const std::string& path, const std::string_view database) + : path_(path) { -MetadataDB::MetadataDB(const std::string& path) : path(path) { + backend_ = MetadataDBFactory::create(path, database); /* Get logger instance and set it for data module and chunk storage */ GKFS_METADATA_MOD->log(spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME)); assert(GKFS_METADATA_MOD->log()); log_ = spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME); assert(log_); +} - // Optimize RocksDB. This is the easiest way to get RocksDB to perform well - options.IncreaseParallelism(); - options.OptimizeLevelStyleCompaction(); - // create the DB if it's not already present - options.create_if_missing = true; - options.merge_operator.reset(new MetadataMergeOperator); - MetadataDB::optimize_rocksdb_options(options); - write_opts.disableWAL = !(gkfs::config::rocksdb::use_write_ahead_log); - rdb::DB* rdb_ptr = nullptr; - auto s = rocksdb::DB::Open(options, path, &rdb_ptr); - if(!s.ok()) { - throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); - } - this->db.reset(rdb_ptr); +MetadataDB::~MetadataDB() { + backend_.reset(); } +/** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ std::string MetadataDB::get(const std::string& key) const { - std::string val; - auto s = db->Get(rdb::ReadOptions(), key, &val); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } - return val; + return backend_->get(key); } void @@ -106,11 +98,7 @@ MetadataDB::put(const std::string& key, const std::string& val) { assert(gkfs::path::is_absolute(key)); assert(key == "/" || !gkfs::path::has_trailing_slash(key)); - auto cop = CreateOperand(val); - auto s = db->Merge(write_opts, key, cop.serialize()); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + backend_->put(key, val); } /** @@ -120,44 +108,27 @@ MetadataDB::put(const std::string& key, const std::string& val) { */ void MetadataDB::put_no_exist(const std::string& key, const std::string& val) { - if(exists(key)) - throw ExistsException(key); - put(key, val); + + backend_->put_no_exist(key, val); } void MetadataDB::remove(const std::string& key) { - auto s = db->Delete(write_opts, key); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->remove(key); } bool MetadataDB::exists(const std::string& key) { - std::string val; - auto s = db->Get(rdb::ReadOptions(), key, &val); - if(!s.ok()) { - if(s.IsNotFound()) { - return false; - } else { - MetadataDB::throw_rdb_status_excpt(s); - } - } - return true; + + return backend_->exists(key); } void MetadataDB::update(const std::string& old_key, const std::string& new_key, const std::string& val) { - // TODO use rdb::Put() method - rdb::WriteBatch batch; - batch.Delete(old_key); - batch.Put(new_key, val); - auto s = db->Write(write_opts, &batch); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->update(old_key, new_key, val); } /** @@ -167,11 +138,8 @@ MetadataDB::update(const std::string& old_key, const std::string& new_key, */ void MetadataDB::increase_size(const std::string& key, size_t size, bool append) { - auto uop = IncreaseSizeOperand(size, append); - auto s = db->Merge(write_opts, key, uop.serialize()); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->increase_size(key, size, append); } /** @@ -181,11 +149,8 @@ MetadataDB::increase_size(const std::string& key, size_t size, bool append) { */ void MetadataDB::decrease_size(const std::string& key, size_t size) { - auto uop = DecreaseSizeOperand(size); - auto s = db->Merge(write_opts, key, uop.serialize()); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->decrease_size(key, size); } std::vector> @@ -198,38 +163,7 @@ MetadataDB::get_dirents(const std::string& dir) const { root_path.push_back('/'); } - rocksdb::ReadOptions ropts; - auto it = db->NewIterator(ropts); - - std::vector> entries; - - for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); - it->Next()) { - - if(it->key().size() == root_path.size()) { - // we skip this path cause it is exactly the root_path - continue; - } - - /***** Get File name *****/ - auto name = it->key().ToString(); - if(name.find_first_of('/', root_path.size()) != std::string::npos) { - // skip stuff deeper then one level depth - continue; - } - // remove prefix - name = name.substr(root_path.size()); - - // relative path of directory entries must not be empty - assert(!name.empty()); - - Metadata md(it->value().ToString()); - auto is_dir = S_ISDIR(md.mode()); - - entries.emplace_back(std::move(name), is_dir); - } - assert(it->status().ok()); - return entries; + return backend_->get_dirents(root_path); } std::vector> @@ -242,39 +176,7 @@ MetadataDB::get_dirents_extended(const std::string& dir) const { root_path.push_back('/'); } - rocksdb::ReadOptions ropts; - auto it = db->NewIterator(ropts); - - std::vector> entries; - - for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); - it->Next()) { - - if(it->key().size() == root_path.size()) { - // we skip this path cause it is exactly the root_path - continue; - } - - /***** Get File name *****/ - auto name = it->key().ToString(); - if(name.find_first_of('/', root_path.size()) != std::string::npos) { - // skip stuff deeper then one level depth - continue; - } - // remove prefix - name = name.substr(root_path.size()); - - // relative path of directory entries must not be empty - assert(!name.empty()); - - Metadata md(it->value().ToString()); - auto is_dir = S_ISDIR(md.mode()); - - entries.emplace_back(std::forward_as_tuple(std::move(name), is_dir, - md.size(), md.ctime())); - } - assert(it->status().ok()); - return entries; + return backend_->get_dirents_extended(root_path); } @@ -285,15 +187,8 @@ MetadataDB::get_dirents_extended(const std::string& dir) const { * @endinternal */ void -MetadataDB::iterate_all() { - std::string key; - std::string val; - // Do RangeScan on parent inode - auto iter = db->NewIterator(rdb::ReadOptions()); - for(iter->SeekToFirst(); iter->Valid(); iter->Next()) { - key = iter->key().ToString(); - val = iter->value().ToString(); - } +MetadataDB::iterate_all() const { + backend_->iterate_all(); } } // namespace gkfs::metadata diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp new file mode 100644 index 000000000..c27040e00 --- /dev/null +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -0,0 +1,487 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; +extern "C" { +#include +} + +std::recursive_mutex parallax_mutex_; + +namespace gkfs::metadata { +/** + * @brief Destroy the Kreon Backend:: Kreon Backend object + * We remove the file, too large for the CI. + * TODO: Insert option + */ +ParallaxBackend::~ParallaxBackend() { + par_close(par_db_); + if(GKFS_DATA->kreon_keep_md() == false) + unlink(par_path_.c_str()); +} + +/** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ +ParallaxBackend::ParallaxBackend(const std::string& path) { + + + // We try to open options.yml if it exists, if not we create it by default + int options = open("options.yml", O_RDWR | O_CREAT, 0644); + int64_t sizeOptions; + sizeOptions = lseek(options, 0, SEEK_END); + if(sizeOptions == 0) { + std::string optcontent = + "level0_size: 1\ngc_interval: 10\ngrowth_factor: 4\nmedium_log_LRU_cache_size: 400\nlevel_medium_inplace: 3\n"; + write(options, optcontent.c_str(), optcontent.length()); + } + + close(options); + + // Kreon + par_path_ = path + "x"; // file is rocksdb (add an x) + int64_t size; + + int fd = open(par_path_.c_str(), O_RDWR | O_CREAT, 0644); + if(fd == -1) { + perror("open"); + exit(EXIT_FAILURE); + } + + // Check size if we want to reuse it + if(GKFS_DATA->kreon_reuse_md()) { + + size = lseek(fd, 0, SEEK_END); + if(size == -1) { + printf("[%s:%s:%d] failed to determine volume size exiting...\n", + __FILE__, __func__, __LINE__); + perror("ioctl"); + exit(EXIT_FAILURE); + } + } + if(GKFS_DATA->kreon_reuse_md() && size == 0) { + GKFS_METADATA_MOD->log()->error("KREON database reused but size is 0"); + exit(EXIT_FAILURE); + } + + if(GKFS_DATA->kreon_reuse_md() == false) { + size = GKFS_DATA->kreon_size_md(); + + lseek(fd, size - 1, SEEK_SET); + std::string tmp = "x"; + write(fd, tmp.c_str(), 1); + close(fd); + + // We format the database + std::string cmd = "kv_format.parallax --device " + par_path_ + + " --max_regions_num 1 "; + system(cmd.c_str()); + + /* + fd = open(par_path_.c_str(), O_RDONLY); + size = lseek(fd, 0, SEEK_END); + close(fd); + */ + } + + par_options_.create_flag = PAR_CREATE_DB; + par_options_.db_name = "test"; + par_options_.volume_name = (char*) malloc(par_path_.size() + 1); + strcpy(par_options_.volume_name, par_path_.c_str()); + par_options_.volume_start = 0; + par_options_.volume_size = 0; + par_db_ = par_open(&par_options_); +} + + +/** + * Exception wrapper on Status object. Throws NotFoundException if + * s == "Not Found", general DBException otherwise + * @param String with status + * @throws DBException + */ +void +ParallaxBackend::throw_status_excpt(const std::string& s) { + if(s == "Not Found") { + throw NotFoundException(s); + } else { + throw DBException(s); + } +} + +/** + * Convert a String to klc_key + * @param key + * @param par_key struct + */ +inline void +ParallaxBackend::str2par(const std::string& key, struct par_key& K) const { + K.size = key.size(); + K.data = key.c_str(); +} + +/** + * Convert a String to klc_value + * @param value + * @param par_value struct + */ +inline void +ParallaxBackend::str2par(const std::string& value, struct par_value& V) const { + V.val_size = value.size() + 1; + V.val_buffer = (char*) value.c_str(); +} + +/** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +std::string +ParallaxBackend::get_impl(const std::string& key) const { + std::string val; + + struct par_key K; + struct par_value V; + V.val_buffer = NULL; + str2par(key, K); + + par_ret_code ret = par_get(par_db_, &K, &V); + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } else { + val = V.val_buffer; + free(V.val_buffer); + } + return val; +} + +/** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ +void +ParallaxBackend::put_impl(const std::string& key, const std::string& val) { + + struct par_key_value key_value; + + str2par(key, key_value.k); + str2par(val, key_value.v); + par_ret_code ret = par_put(par_db_, &key_value); + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } +} + +/** + * Puts an entry into the KV store if it doesn't exist. This function does not + * use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ +void +ParallaxBackend::put_no_exist_impl(const std::string& key, + const std::string& val) { + + struct par_key_value key_value; + str2par(key, key_value.k); + str2par(val, key_value.v); + + par_ret_code ret = par_exists(par_db_, &key_value.k); + if(ret == PAR_KEY_NOT_FOUND) { + par_put(par_db_, &key_value); + } else + throw ExistsException(key); +} + +/** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +ParallaxBackend::remove_impl(const std::string& key) { + + struct par_key k; + + str2par(key, k); + par_ret_code ret = par_delete(par_db_, &k); + + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } +} + +/** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ +bool +ParallaxBackend::exists_impl(const std::string& key) { + + struct par_key k; + + str2par(key, k); + + par_ret_code ret = par_exists(par_db_, &k); + if(ret == PAR_KEY_NOT_FOUND) { + return true; + } + + return false; // TODO it is not the only case, we can have errors +} + +/** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +ParallaxBackend::update_impl(const std::string& old_key, + const std::string& new_key, + const std::string& val) { + + // TODO: Check Parallax transactions/Batches + + struct par_key_value n_key_value; + struct par_key o_key; + + str2par(new_key, n_key_value.k); + str2par(val, n_key_value.v); + + str2par(old_key, o_key); + par_delete(par_db_, &o_key); + par_ret_code ret = par_put(par_db_, &n_key_value); + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } +} + +/** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ +void +ParallaxBackend::increase_size_impl(const std::string& key, size_t size, + bool append) { + lock_guard lock_guard(parallax_mutex_); + + auto value = get(key); + // Decompress string + Metadata md(value); + if(append) + size += md.size(); + md.size(size); + update(key, key, md.serialize()); +} + +/** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ +void +ParallaxBackend::decrease_size_impl(const std::string& key, size_t size) { + lock_guard lock_guard(parallax_mutex_); + + auto value = get(key); + // Decompress string + Metadata md(value); + md.size(size); + update(key, key, md.serialize()); +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +ParallaxBackend::get_dirents_impl(const std::string& dir) const { + auto root_path = dir; + // lock_guard lock_guard(kreon_mutex_); + + struct par_key K; + + str2par(root_path, K); + + par_scanner S = par_init_scanner(par_db_, &K, PAR_GREATER_OR_EQUAL); + + std::vector> entries; + + while(par_is_valid(S)) { + struct par_key K2 = par_get_key(S); + struct par_value value = par_get_value(S); + + std::string k(K2.data, K2.size); + std::string v(value.val_buffer, value.val_size); + if(k.size() < root_path.size() || + k.substr(0, root_path.size()) != root_path) { + break; + } + + if(k.size() == root_path.size()) { + par_get_next(S); + continue; + } + + /***** Get File name *****/ + auto name = k; + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + par_get_next(S); + continue; + } + + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(v); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::move(name), is_dir); + + par_get_next(S); + } + // If we don't close the scanner we cannot delete keys + par_close_scanner(S); + + return entries; +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +ParallaxBackend::get_dirents_extended_impl(const std::string& dir) const { + auto root_path = dir; + // assert(gkfs::path::is_absolute(root_path)); + // add trailing slash if missing + if(!gkfs::path::has_trailing_slash(root_path) && root_path.size() != 1) { + // add trailing slash only if missing and is not the root_folder "/" + root_path.push_back('/'); + } + + struct par_key K; + + str2par(root_path, K); + + par_scanner S = par_init_scanner(par_db_, &K, PAR_GREATER_OR_EQUAL); + + std::vector> entries; + + while(par_is_valid(S)) { + struct par_key K2 = par_get_key(S); + struct par_value value = par_get_value(S); + + std::string k(K2.data, K2.size); + std::string v(value.val_buffer, value.val_size); + + if(k.size() < root_path.size() || + k.substr(0, root_path.size()) != root_path) { + break; + } + + if(k.size() == root_path.size()) { + if(par_get_next(S) && !par_is_valid(S)) + break; + continue; + } + + /***** Get File name *****/ + auto name = k; + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + if(par_get_next(S) && !par_is_valid(S)) + break; + continue; + } + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(v); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::forward_as_tuple(std::move(name), is_dir, + md.size(), md.ctime())); + + if(par_get_next(S) && !par_is_valid(S)) + break; + } + // If we don't close the scanner we cannot delete keys + par_close_scanner(S); + + return entries; +} + + +/** + * Code example for iterating all entries in KV store. This is for debug only as + * it is too expensive + */ +void +ParallaxBackend::iterate_all_impl() const {} + + +} // namespace gkfs::metadata diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp new file mode 100644 index 000000000..0a4deca4d --- /dev/null +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -0,0 +1,348 @@ +/* + Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include +#include +#include +#include + +#include +#include +#include +#include +extern "C" { +#include +} + +namespace gkfs::metadata { + +/** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ +RocksDBBackend::RocksDBBackend(const std::string& path) { + + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + options_.IncreaseParallelism(); + options_.OptimizeLevelStyleCompaction(); + // create the DB if it's not already present + options_.create_if_missing = true; + options_.merge_operator.reset(new MetadataMergeOperator); + optimize_database_impl(); + write_opts_.disableWAL = !(gkfs::config::rocksdb::use_write_ahead_log); + rdb::DB* rdb_ptr = nullptr; + auto s = rocksdb::DB::Open(options_, path, &rdb_ptr); + if(!s.ok()) { + throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); + } + this->db_.reset(rdb_ptr); +} + + +RocksDBBackend::~RocksDBBackend() { + this->db_.reset(); +} + +/** + * Exception wrapper on Status object. Throws NotFoundException if + * s.IsNotFound(), general DBException otherwise + * @param RocksDB status + * @throws DBException + */ +void +RocksDBBackend::throw_status_excpt(const rdb::Status& s) { + assert(!s.ok()); + + if(s.IsNotFound()) { + throw NotFoundException(s.ToString()); + } else { + throw DBException(s.ToString()); + } +} + + +/** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +std::string +RocksDBBackend::get_impl(const std::string& key) const { + std::string val; + + auto s = db_->Get(rdb::ReadOptions(), key, &val); + if(!s.ok()) { + throw_status_excpt(s); + } + + return val; +} + +/** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ +void +RocksDBBackend::put_impl(const std::string& key, const std::string& val) { + + auto cop = CreateOperand(val); + auto s = db_->Merge(write_opts_, key, cop.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Puts an entry into the KV store if it doesn't exist. This function does not + * use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ +void +RocksDBBackend::put_no_exist_impl(const std::string& key, + const std::string& val) { + + if(exists(key)) + throw ExistsException(key); + put(key, val); +} + +/** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +RocksDBBackend::remove_impl(const std::string& key) { + + auto s = db_->Delete(write_opts_, key); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ +bool +RocksDBBackend::exists_impl(const std::string& key) { + + std::string val; + + auto s = db_->Get(rdb::ReadOptions(), key, &val); + if(!s.ok()) { + if(s.IsNotFound()) { + return false; + } else { + throw_status_excpt(s); + } + } + return true; +} + +/** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +RocksDBBackend::update_impl(const std::string& old_key, + const std::string& new_key, + const std::string& val) { + + // TODO use rdb::Put() method + rdb::WriteBatch batch; + batch.Delete(old_key); + batch.Put(new_key, val); + auto s = db_->Write(write_opts_, &batch); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ +void +RocksDBBackend::increase_size_impl(const std::string& key, size_t size, + bool append) { + + auto uop = IncreaseSizeOperand(size, append); + auto s = db_->Merge(write_opts_, key, uop.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ +void +RocksDBBackend::decrease_size_impl(const std::string& key, size_t size) { + + auto uop = DecreaseSizeOperand(size); + auto s = db_->Merge(write_opts_, key, uop.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +RocksDBBackend::get_dirents_impl(const std::string& dir) const { + auto root_path = dir; + rocksdb::ReadOptions ropts; + auto it = db_->NewIterator(ropts); + + std::vector> entries; + for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); + it->Next()) { + + if(it->key().size() == root_path.size()) { + // we skip this path cause it is exactly the root_path + continue; + } + + /***** Get File name *****/ + auto name = it->key().ToString(); + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + continue; + } + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(it->value().ToString()); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::move(name), is_dir); + } + assert(it->status().ok()); + return entries; +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +RocksDBBackend::get_dirents_extended_impl(const std::string& dir) const { + auto root_path = dir; + rocksdb::ReadOptions ropts; + auto it = db_->NewIterator(ropts); + + std::vector> entries; + + for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); + it->Next()) { + + if(it->key().size() == root_path.size()) { + // we skip this path cause it is exactly the root_path + continue; + } + + /***** Get File name *****/ + auto name = it->key().ToString(); + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + continue; + } + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(it->value().ToString()); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::forward_as_tuple(std::move(name), is_dir, + md.size(), md.ctime())); + } + assert(it->status().ok()); + return entries; +} + + +/** + * Code example for iterating all entries in KV store. This is for debug only as + * it is too expensive + */ +void +RocksDBBackend::iterate_all_impl() const { + std::string key; + std::string val; + // Do RangeScan on parent inode + auto iter = db_->NewIterator(rdb::ReadOptions()); + for(iter->SeekToFirst(); iter->Valid(); iter->Next()) { + key = iter->key().ToString(); + val = iter->value().ToString(); + std::cout << key << std::endl; + } +} + +/** + * Used for setting KV store settings + */ +void +RocksDBBackend::optimize_database_impl() { + options_.max_successive_merges = 128; +} + + +} // namespace gkfs::metadata diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 98ac82a53..09421f417 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -110,6 +110,16 @@ FsData::metadir(const std::string& metadir) { FsData::metadir_ = metadir; } +std::string_view +FsData::dbbackend() const { + return dbbackend_; +} + +void +FsData::dbbackend(const std::string& dbbackend) { + FsData::dbbackend_ = dbbackend; +} + const std::string& FsData::rpc_protocol() const { return rpc_protocol_; @@ -200,4 +210,35 @@ FsData::blocks_state(bool blocks_state) { FsData::blocks_state_ = blocks_state; } +bool +FsData::kreon_keep_md() const { + return keep_md_; +} + +void +FsData::kreon_keep_md(bool keep_md) { + FsData::keep_md_ = keep_md; +} + +bool +FsData::kreon_reuse_md() const { + return reuse_md_; +} + +void +FsData::kreon_reuse_md(bool reuse_md) { + FsData::reuse_md_ = reuse_md; +} + +unsigned long long +FsData::kreon_size_md() const { + return size_md_; +} + +void +FsData::kreon_size_md(unsigned int size_md) { + FsData::size_md_ = + (unsigned long long) size_md * 1024ull * 1024ull * 1024ull; +} + } // namespace gkfs::daemon diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5144f4bcc..5f314f77c 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -78,6 +78,8 @@ struct cli_options { string listen; string hosts_file; string rpc_protocol; + string dbbackend; + string kreonsize; }; /** @@ -245,8 +247,8 @@ init_environment() { GKFS_DATA->spdlogger()->debug("{}() Initializing metadata DB: '{}'", __func__, metadata_path); try { - GKFS_DATA->mdb( - std::make_shared(metadata_path)); + GKFS_DATA->mdb(std::make_shared( + metadata_path, GKFS_DATA->dbbackend())); } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( "{}() Failed to initialize metadata DB: {}", __func__, @@ -587,6 +589,22 @@ parse_input(const cli_options& opts, const CLI::App& desc) { GKFS_DATA->metadir(GKFS_DATA->rootdir()); #endif } + + if(desc.count("--dbbackend")) { + auto dbbackend = opts.dbbackend; + GKFS_DATA->dbbackend(dbbackend); + } else + GKFS_DATA->dbbackend("rocksdb"); + + if(desc.count("--keepmd")) { + GKFS_DATA->kreon_keep_md(true); + } + if(desc.count("--reusemd")) { + GKFS_DATA->kreon_reuse_md(true); + } + if(desc.count("--kreonsize")) { // Size in GB + GKFS_DATA->kreon_size_md(stoi(opts.kreonsize)); + } } /** @@ -605,7 +623,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { */ int main(int argc, const char* argv[]) { - // Define arg parsing CLI::App desc{"Allowed options"}; cli_options opts{}; // clang-format off @@ -644,6 +661,16 @@ main(int argc, const char* argv[]) { desc.add_flag( "--clean-rootdir,-c", "Cleans Rootdir >before< launching the deamon"); + desc.add_option( + "--dbbackend,-d", opts.dbbackend, + "Database Backend to use. If not set, rocksdb is used. For parallaxdb, a file called rocksdb with 8GB is needed in metadir"); + desc.add_flag("--keepmd", "Kreondb - Keeps metadir (default off)"); + desc.add_flag("--reusemd", + "Kreondb - Avoids initializing the metadata file, (default off)"); + desc.add_option("--kreonsize",opts.kreonsize, + "Kreondb - Metatada file size in GB (default 8), " + "used only with new files"); + desc.add_flag("--version", "Print version and exit."); // clang-format on try { @@ -651,6 +678,8 @@ main(int argc, const char* argv[]) { } catch(const CLI::ParseError& e) { return desc.exit(e); } + + if(desc.count("--version")) { cout << GKFS_VERSION_STRING << endl; #ifndef NDEBUG diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a3c4e5900..9236ccec6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -95,17 +95,34 @@ def test_workspace(tmp_path, request): request.config.getoption('--lib-dir')) @pytest.fixture -def gkfs_daemon(test_workspace, request): +def gkfs_daemon_rocksdb(test_workspace, request): """ Initializes a local gekkofs daemon """ interface = request.config.getoption('--interface') - daemon = Daemon(interface, test_workspace) + daemon = Daemon(interface, "rocksdb", test_workspace) yield daemon.run() daemon.shutdown() +@pytest.fixture +def gkfs_daemon_parallaxdb(test_workspace, request): + """ + Initializes a local gekkofs daemon + """ + + interface = request.config.getoption('--interface') + daemon = Daemon(interface, "parallaxdb", test_workspace) + + yield daemon.run() + daemon.shutdown() + +@pytest.fixture(params=['gkfs_daemon_rocksdb', 'gkfs_daemon_parallaxdb']) +def gkfs_daemon(request): + return request.getfixturevalue(request.param) + + @pytest.fixture def gkfs_client(test_workspace): """ diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index c9ce5c04c..eaf862fb8 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -225,14 +225,14 @@ class FwdClientCreator: class Daemon: - def __init__(self, interface, workspace): + def __init__(self, interface, database, workspace): self._address = get_ephemeral_address(interface) self._workspace = workspace - + self._database = database self._cmd = sh.Command(gkfs_daemon_cmd, self._workspace.bindirs) self._env = os.environ.copy() - + self._metadir = self.rootdir libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + [str(p) for p in self._workspace.libdirs])) @@ -249,7 +249,9 @@ class Daemon: args = [ '--mountdir', self.mountdir, '--rootdir', self.rootdir, - '-l', self._address ] + '-l', self._address, + '--metadir', self._metadir, + '--dbbackend', self._database ] logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -267,7 +269,7 @@ class Daemon: logger.debug("waiting for daemon to be ready") try: - self.wait_until_active(self._proc.pid, 60.0) + self.wait_until_active(self._proc.pid, 720.0) except Exception as ex: logger.error(f"daemon initialization failed: {ex}") @@ -307,7 +309,7 @@ class Daemon: while perf_counter() - init_time < timeout: try: - logger.debug(f"checking log file") + # logger.debug(f"checking log file") with open(self.logdir / gkfs_daemon_log_file) as log: for line in islice(log, max_lines): if re.search(gkfs_daemon_active_log_pattern, line) is not None: @@ -366,6 +368,7 @@ class _proxy_exec(): def __call__(self, *args, **kwargs): return self._client.run(self._name, *args, **kwargs) + class Client: """ A class to represent a GekkoFS client process with a patched LD_PRELOAD. @@ -776,7 +779,7 @@ class FwdDaemon: while perf_counter() - init_time < timeout: try: - logger.debug(f"checking log file") +# logger.debug(f"checking log file") with open(self.logdir / gkfwd_daemon_log_file) as log: for line in islice(log, max_lines): if re.search(gkfwd_daemon_active_log_pattern, line) is not None: diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/all.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/all.out new file mode 100644 index 000000000..847a6c563 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/all.out @@ -0,0 +1,14 @@ +######## Installing: bzip2 ############################### +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: bmi ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out new file mode 100644 index 000000000..8c7072d87 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out @@ -0,0 +1,8 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out new file mode 100644 index 000000000..a1781600a --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out @@ -0,0 +1,17 @@ +######## Installing: bzip2 ############################### +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: bmi ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: rocksdb%experimental ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out new file mode 100644 index 000000000..d830e84df --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out @@ -0,0 +1,6 @@ +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon1.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon1.out new file mode 100644 index 000000000..3afaabd70 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon1.out @@ -0,0 +1,11 @@ +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: libfabric%verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out new file mode 100644 index 000000000..925ef005d --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out @@ -0,0 +1,14 @@ +######## Installing: bzip2 ############################### +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: libfabric%experimental ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb%experimental ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out new file mode 100644 index 000000000..ec0cb2c4d --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out @@ -0,0 +1,14 @@ +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: libfabric%experimental ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: agios ############################### diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/all.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/all.out new file mode 100644 index 000000000..166a2b6a6 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/all.out @@ -0,0 +1,15 @@ +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/radix-io/bmi/' to 'bmi' with commit '[6ea0b78fce1b964e45102828cdd05df7040a94c8]' and flags '' +Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out new file mode 100644 index 000000000..3b639df25 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out @@ -0,0 +1,9 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out new file mode 100644 index 000000000..9fd9f0f26 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out @@ -0,0 +1,17 @@ +Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out new file mode 100644 index 000000000..cca6da718 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out @@ -0,0 +1,7 @@ +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out new file mode 100644 index 000000000..e2f1a07df --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Cloned 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' with commit '[6.2.2]' and flags '' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out new file mode 100644 index 000000000..14297f1c2 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out @@ -0,0 +1,14 @@ +Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out new file mode 100644 index 000000000..4d4931128 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out @@ -0,0 +1,14 @@ +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' -- GitLab From f3a9c72347fe89dee6915ea53b84cb3ee777b612 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 1 Feb 2022 11:46:01 +0100 Subject: [PATCH 02/17] Added includes for I/O ops --- src/daemon/backend/metadata/parallax_backend.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp index c27040e00..2e1aa4627 100644 --- a/src/daemon/backend/metadata/parallax_backend.cpp +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -35,7 +35,8 @@ #include #include #include - +#include +#include using namespace std; extern "C" { #include -- GitLab From 6fec7bf1182250fd3dabc81cab325df499c92a10 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 1 Feb 2022 12:06:07 +0100 Subject: [PATCH 03/17] Updated scripts for 0.9.0-experimental --- .../{all.specs => agios.specs} | 30 +++---- scripts/profiles/0.9.0-experimental/ci.specs | 18 ++-- .../profiles/0.9.0-experimental/default.specs | 42 +++------- .../profiles/0.9.0-experimental/direct.specs | 20 ++--- .../{mogon1.specs => infiniband_verbs.specs} | 31 ++++--- .../0.9.0-experimental/marenostrum4.specs | 83 +++++++++++++++++++ .../profiles/0.9.0-experimental/mogon2.specs | 34 ++++---- .../profiles/0.9.0-experimental/ngio.specs | 33 ++++---- .../0.9.0-experimental/omnipath_psm2.specs | 83 +++++++++++++++++++ .../dl_dep.sh/0.9.0-experimental/agios.out | 13 +++ .../dl_dep.sh/0.9.0-experimental/all.out | 15 ---- .../dl_dep.sh/0.9.0-experimental/ci.out | 15 ++-- .../dl_dep.sh/0.9.0-experimental/default.out | 27 +++--- .../dl_dep.sh/0.9.0-experimental/direct.out | 15 ++-- .../0.9.0-experimental/infiniband_verbs.out | 12 +++ .../0.9.0-experimental/marenostrum4.out | 13 +++ .../dl_dep.sh/0.9.0-experimental/mogon1.out | 12 --- .../dl_dep.sh/0.9.0-experimental/mogon2.out | 25 +++--- .../dl_dep.sh/0.9.0-experimental/ngio.out | 25 +++--- .../0.9.0-experimental/omnipath_psm2.out | 13 +++ 20 files changed, 358 insertions(+), 201 deletions(-) rename scripts/profiles/0.9.0-experimental/{all.specs => agios.specs} (81%) rename scripts/profiles/0.9.0-experimental/{mogon1.specs => infiniband_verbs.specs} (82%) create mode 100644 scripts/profiles/0.9.0-experimental/marenostrum4.specs create mode 100644 scripts/profiles/0.9.0-experimental/omnipath_psm2.specs create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out delete mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/all.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out delete mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out create mode 100644 tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out diff --git a/scripts/profiles/0.9.0-experimental/all.specs b/scripts/profiles/0.9.0-experimental/agios.specs similarity index 81% rename from scripts/profiles/0.9.0-experimental/all.specs rename to scripts/profiles/0.9.0-experimental/agios.specs index 6eafe8f3a..79e062517 100644 --- a/scripts/profiles/0.9.0-experimental/all.specs +++ b/scripts/profiles/0.9.0-experimental/agios.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,25 +37,22 @@ comment="All dependencies (except transport-specific and experimental)" # Dependencies that must be downloaded directly wgetdeps=( - ["bzip2"]="1.0.6" - ["zstd"]="1.3.2" - ["lz4"]="1.8.0" - ["snappy"]="1.1.7" - ["capstone"]="4.0.1" - ["argobots"]="1.0rc1" - ["rocksdb"]="6.2.2" + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8" - ["libfabric"]="HEAD@v1.8.1" - ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" - ["margo"]="v0.6.3" - ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" ) # Extra arguments for git clone @@ -70,8 +67,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "libfabric" "mercury" "argobots" - "margo" "rocksdb" "syscall_intercept" "date" "agios" "parallax" + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.0-experimental/ci.specs b/scripts/profiles/0.9.0-experimental/ci.specs index 625ae6234..3292e6bd0 100644 --- a/scripts/profiles/0.9.0-experimental/ci.specs +++ b/scripts/profiles/0.9.0-experimental/ci.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,19 +37,19 @@ comment="Dependencies required by the CI" # Dependencies that must be downloaded directly wgetdeps=( - ["argobots"]="1.0rc1" - ["rocksdb"]="6.2.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.8.1" - ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" - ["margo"]="v0.6.3" - ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/default.specs b/scripts/profiles/0.9.0-experimental/default.specs index 29ed724e4..ac4e40121 100644 --- a/scripts/profiles/0.9.0-experimental/default.specs +++ b/scripts/profiles/0.9.0-experimental/default.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,29 +37,21 @@ comment="All dependencies" # Dependencies that must be downloaded directly wgetdeps=( - ["bzip2"]="1.0.6" - ["zstd"]="1.3.2" - ["lz4"]="1.8.0" - ["snappy"]="1.1.7" - ["capstone"]="4.0.1" - ["argobots"]="1.0rc1" - ["rocksdb"]="6.2.2" - ["rocksdb%experimental"]="6.11.4" - ["psm2"]="11.2.86" + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8" - ["libfabric"]="HEAD@v1.8.1" - ["libfabric%experimental"]="HEAD@v1.9.1" - ["libfabric%verbs"]="HEAD@v1.7.2" - ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" - ["margo"]="v0.6.3" - ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" ) # Extra arguments for git clone @@ -74,12 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" - "libfabric" - "libfabric%experimental" - "libfabric%verbs" - "mercury" "argobots" "margo" "rocksdb" "rocksdb%experimental" - "syscall_intercept" "date" "psm2" "agios" "parallax" + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" ) # Extra arguments passed to the installation script. As such, they can @@ -91,7 +78,4 @@ order=( # - CORES: the number of cores to use when building # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( - ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" - ["libfabric%verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" - ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" ) diff --git a/scripts/profiles/0.9.0-experimental/direct.specs b/scripts/profiles/0.9.0-experimental/direct.specs index aeed1bdf6..19fb562cc 100644 --- a/scripts/profiles/0.9.0-experimental/direct.specs +++ b/scripts/profiles/0.9.0-experimental/direct.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,17 +37,17 @@ comment="Direct dependencies" # Dependencies that must be downloaded directly wgetdeps=( - ["argobots"]="1.0rc1" - ["rocksdb"]="6.2.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" ) # Dependencies that must be cloned clonedeps=( - ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" - ["margo"]="v0.6.3" - ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" - ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" ) # Extra arguments for git clone @@ -62,7 +62,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.0-experimental/mogon1.specs b/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs similarity index 82% rename from scripts/profiles/0.9.0-experimental/mogon1.specs rename to scripts/profiles/0.9.0-experimental/infiniband_verbs.specs index 722cbe3b6..4cb3146ab 100644 --- a/scripts/profiles/0.9.0-experimental/mogon1.specs +++ b/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -33,26 +33,25 @@ declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_arg declare -a order # Comment that should be displayed when printing the profile -comment="Dependencies for Mogon 1 supercomputer" +comment="Dependencies for Infiniband supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["zstd"]="1.3.2" - ["lz4"]="1.8.0" - ["snappy"]="1.1.7" - ["capstone"]="4.0.1" - ["argobots"]="1.0rc1" + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric%verbs"]="HEAD@v1.7.2" - ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" - ["margo"]="v0.6.3" - ["rocksdb"]="6.2.2" - ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["libfabric%verbs"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" ) # Extra arguments for git clone @@ -67,8 +66,8 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading/installing order=( - "zstd" "lz4" "snappy" "capstone" "libfabric%verbs" "mercury" - "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" + "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.0-experimental/marenostrum4.specs b/scripts/profiles/0.9.0-experimental/marenostrum4.specs new file mode 100644 index 000000000..524cf74d6 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/marenostrum4.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Marenostrum 4 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/0.9.0-experimental/mogon2.specs b/scripts/profiles/0.9.0-experimental/mogon2.specs index 22b16a4a9..486a37b03 100644 --- a/scripts/profiles/0.9.0-experimental/mogon2.specs +++ b/scripts/profiles/0.9.0-experimental/mogon2.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,24 +37,22 @@ comment="Dependencies for Mogon 2 supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["bzip2"]="1.0.6" - ["zstd"]="1.3.2" - ["lz4"]="1.8.0" - ["snappy"]="1.1.7" - ["capstone"]="4.0.1" - ["argobots"]="1.0rc1" - ["rocksdb%experimental"]="6.11.4" - ["psm2"]="11.2.86" + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric%experimental"]="HEAD@v1.9.1" - ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" - ["margo"]="v0.6.3" - ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" ) # Extra arguments for git clone @@ -69,9 +67,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "bzip2" "zstd" "lz4" "snappy" "capstone" "libfabric%experimental" - "mercury" "argobots" "margo" "rocksdb%experimental" "syscall_intercept" - "date" "psm2" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" ) # Extra arguments passed to the installation script. As such, they can @@ -83,5 +79,5 @@ order=( # - CORES: the number of cores to use when building # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( - ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" ) diff --git a/scripts/profiles/0.9.0-experimental/ngio.specs b/scripts/profiles/0.9.0-experimental/ngio.specs index 114a27707..4f40f811f 100644 --- a/scripts/profiles/0.9.0-experimental/ngio.specs +++ b/scripts/profiles/0.9.0-experimental/ngio.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,24 +37,22 @@ comment="Dependencies for NEXTGenIO prototype cluster" # Dependencies that must be downloaded directly wgetdeps=( - ["zstd"]="1.3.2" - ["lz4"]="1.8.0" - ["snappy"]="1.1.7" - ["capstone"]="4.0.1" - ["argobots"]="1.0rc1" - ["rocksdb"]="6.2.2" - ["psm2"]="11.2.86" + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric%experimental"]="HEAD@v1.9.1" - ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" - ["margo"]="v0.6.3" - ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="56e8b7b30908b1bc48f6c58ab03262498f44dad9" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" ) # Extra arguments for git clone @@ -69,8 +67,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "zstd" "lz4" "snappy" "capstone" "libfabric%experimental" "mercury" - "argobots" "margo" "rocksdb" "syscall_intercept" "date" "psm2" "agios" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" ) # Extra arguments passed to the installation script. As such, they can @@ -82,5 +79,5 @@ order=( # - CORES: the number of cores to use when building # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( - ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" ) diff --git a/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs b/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs new file mode 100644 index 000000000..cfe5e3980 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Omnipath supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out new file mode 100644 index 000000000..6e40c07a8 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/all.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/all.out deleted file mode 100644 index 166a2b6a6..000000000 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/all.out +++ /dev/null @@ -1,15 +0,0 @@ -Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' -Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' -Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/radix-io/bmi/' to 'bmi' with commit '[6ea0b78fce1b964e45102828cdd05df7040a94c8]' and flags '' -Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' -Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out index 3b639df25..c16a2b723 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out @@ -1,9 +1,10 @@ -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out index 9fd9f0f26..5060a0cf0 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out @@ -1,17 +1,12 @@ -Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' -Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' -Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' -Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' -Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' -Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out index cca6da718..c18e4d209 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out @@ -1,7 +1,8 @@ -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out new file mode 100644 index 000000000..0c18881e3 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out new file mode 100644 index 000000000..65689b961 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out deleted file mode 100644 index e2f1a07df..000000000 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon1.out +++ /dev/null @@ -1,12 +0,0 @@ -Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' -Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' -Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' -Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Cloned 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' with commit '[6.2.2]' and flags '' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out index 14297f1c2..65689b961 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out @@ -1,14 +1,13 @@ -Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' -Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' -Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' -Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' -Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out index 4d4931128..65689b961 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out @@ -1,14 +1,13 @@ -Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' -Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' -Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' -Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[56e8b7b30908b1bc48f6c58ab03262498f44dad9]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out new file mode 100644 index 000000000..65689b961 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Done \ No newline at end of file -- GitLab From d1848c916c4f22bf358d332135cc6c58669d84d0 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 1 Feb 2022 13:14:15 +0100 Subject: [PATCH 04/17] Updated base ci image from 0.9 --- docker/0.9.0-experimental/core/Dockerfile | 10 +++--- scripts/profiles/0.9.0-experimental/ci.specs | 2 +- .../install/rocksdb.install | 36 ++++++++++++++++--- .../0.9.0-experimental/{all.out => agios.out} | 7 ++-- .../compile_dep.sh/0.9.0-experimental/ci.out | 2 +- .../0.9.0-experimental/default.out | 11 ++---- .../0.9.0-experimental/direct.out | 4 +-- .../{mogon1.out => infiniband_verbs.out} | 5 ++- .../0.9.0-experimental/marenostrum4.out | 12 +++++++ .../0.9.0-experimental/mogon2.out | 14 ++++---- .../0.9.0-experimental/ngio.out | 12 +++---- .../0.9.0-experimental/omnipath_psm2.out | 12 +++++++ .../dl_dep.sh/0.9.0-experimental/ci.out | 2 +- 13 files changed, 81 insertions(+), 48 deletions(-) rename tests/scripts/compile_dep.sh/0.9.0-experimental/{all.out => agios.out} (65%) rename tests/scripts/compile_dep.sh/0.9.0-experimental/{mogon1.out => infiniband_verbs.out} (74%) create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/marenostrum4.out create mode 100644 tests/scripts/compile_dep.sh/0.9.0-experimental/omnipath_psm2.out diff --git a/docker/0.9.0-experimental/core/Dockerfile b/docker/0.9.0-experimental/core/Dockerfile index 1c42f9edd..bf1b2327e 100644 --- a/docker/0.9.0-experimental/core/Dockerfile +++ b/docker/0.9.0-experimental/core/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster-slim +FROM debian:bullseye-slim LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" @@ -13,21 +13,19 @@ RUN apt-get update && \ automake \ gcc \ g++ \ + procps \ # AGIOS dependencies libconfig-dev \ # Mercury dependencies libltdl-dev \ lbzip2 \ + # Margo dependencies \ + libjson-c-dev \ # RocksDB dependencies - libsnappy-dev \ liblz4-dev \ - libzstd-dev \ - libbz2-dev \ - zlib1g-dev \ # syscall_intercept dependencies libcapstone-dev \ # GekkoFS dependencies - libboost-filesystem-dev \ libboost-program-options-dev \ uuid-dev && \ # install cmake 3.14 since it's needed for some dependencies diff --git a/scripts/profiles/0.9.0-experimental/ci.specs b/scripts/profiles/0.9.0-experimental/ci.specs index 3292e6bd0..e281e0284 100644 --- a/scripts/profiles/0.9.0-experimental/ci.specs +++ b/scripts/profiles/0.9.0-experimental/ci.specs @@ -65,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" - "date" "agios" "parallax" + "date" "agios" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.0-experimental/install/rocksdb.install b/scripts/profiles/0.9.0-experimental/install/rocksdb.install index 7e14bbaab..ab87e7bb8 100644 --- a/scripts/profiles/0.9.0-experimental/install/rocksdb.install +++ b/scripts/profiles/0.9.0-experimental/install/rocksdb.install @@ -37,15 +37,41 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR=${SOURCE_DIR}/rocksdb - cd "${CURR}" - make clean - PORTABLE=1 USE_SSE=1 USE_RTTI=1 make -j"${CORES}" static_lib - INSTALL_PATH="${INSTALL_DIR}" make install + CXXFLAGS='' + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move' + fi + + # TODO use SSE? + CURR="${SOURCE_DIR}/rocksdb" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \ + -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \ + -DROCKSDB_BUILD_SHARED=OFF \ + -DWITH_LZ4=ON \ + -DWITH_GFLAGS=OFF \ + -DUSE_RTTI=1 \ + -DPORTABLE=1 \ + -DWITH_ALL_TESTS=OFF \ + -DWITH_BENCHMARK_TOOLS=OFF \ + -DWITH_TOOLS=OFF .. + make -j"${CORES}" install + + } pkg_check() { diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/all.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/agios.out similarity index 65% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/all.out rename to tests/scripts/compile_dep.sh/0.9.0-experimental/agios.out index 847a6c563..b3a81c6ab 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/all.out +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/agios.out @@ -1,9 +1,6 @@ -######## Installing: bzip2 ############################### -######## Installing: zstd ############################### ######## Installing: lz4 ############################### -######## Installing: snappy ############################### ######## Installing: capstone ############################### -######## Installing: bmi ############################### +######## Installing: json-c ############################### ######## Installing: libfabric ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### @@ -11,4 +8,4 @@ ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### -######## Installing: agios ############################### +######## Installing: agios ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out index 8c7072d87..c72b99ba7 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out @@ -5,4 +5,4 @@ ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### -######## Installing: agios ############################### +######## Installing: agios ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out index a1781600a..902132395 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out @@ -1,17 +1,10 @@ -######## Installing: bzip2 ############################### -######## Installing: zstd ############################### ######## Installing: lz4 ############################### -######## Installing: snappy ############################### ######## Installing: capstone ############################### -######## Installing: bmi ############################### +######## Installing: json-c ############################### ######## Installing: libfabric ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### ######## Installing: rocksdb ############################### -######## Installing: rocksdb%experimental ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### -######## Installing: psm2 ############################### -WARNING: Install script for 'psm2' not found. Skipping. -######## Installing: agios ############################### +######## Installing: date ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out index d830e84df..c972d1bce 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out @@ -1,6 +1,6 @@ +######## Installing: libfabric ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### ######## Installing: rocksdb ############################### -######## Installing: syscall_intercept ############################### -######## Installing: date ############################### +######## Installing: syscall_intercept ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon1.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/infiniband_verbs.out similarity index 74% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/mogon1.out rename to tests/scripts/compile_dep.sh/0.9.0-experimental/infiniband_verbs.out index 3afaabd70..22551fdf8 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon1.out +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/infiniband_verbs.out @@ -1,11 +1,10 @@ -######## Installing: zstd ############################### ######## Installing: lz4 ############################### -######## Installing: snappy ############################### ######## Installing: capstone ############################### +######## Installing: json-c ############################### ######## Installing: libfabric%verbs ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### +######## Installing: date ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/marenostrum4.out new file mode 100644 index 000000000..f0221c735 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/marenostrum4.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out index 925ef005d..f0221c735 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out @@ -1,14 +1,12 @@ -######## Installing: bzip2 ############################### -######## Installing: zstd ############################### ######## Installing: lz4 ############################### -######## Installing: snappy ############################### ######## Installing: capstone ############################### -######## Installing: libfabric%experimental ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### -######## Installing: rocksdb%experimental ############################### +######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### -######## Installing: psm2 ############################### -WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: date ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out index ec0cb2c4d..f0221c735 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out @@ -1,14 +1,12 @@ -######## Installing: zstd ############################### ######## Installing: lz4 ############################### -######## Installing: snappy ############################### ######## Installing: capstone ############################### -######## Installing: libfabric%experimental ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### -######## Installing: psm2 ############################### -WARNING: Install script for 'psm2' not found. Skipping. -######## Installing: agios ############################### +######## Installing: date ############################### \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.0-experimental/omnipath_psm2.out new file mode 100644 index 000000000..f0221c735 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.0-experimental/omnipath_psm2.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out index c16a2b723..3bcf87b49 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out @@ -7,4 +7,4 @@ Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' wi Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' -Done \ No newline at end of file +Done -- GitLab From e4bb613559d4f0080e49d05807b29774eb2bb7a1 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 14 Feb 2022 12:38:27 +0100 Subject: [PATCH 05/17] Update Parallax (solves delete-scan bug) --- .../profiles/0.9.0-experimental/agios.specs | 2 +- scripts/profiles/0.9.0-experimental/ci.specs | 2 +- .../profiles/0.9.0-experimental/default.specs | 2 +- .../profiles/0.9.0-experimental/direct.specs | 2 +- .../0.9.0-experimental/infiniband_verbs.specs | 2 +- .../0.9.0-experimental/install/json-c.install | 60 +++++++++++++++++++ .../0.9.0-experimental/marenostrum4.specs | 2 +- .../profiles/0.9.0-experimental/mogon2.specs | 2 +- .../profiles/0.9.0-experimental/ngio.specs | 2 +- .../0.9.0-experimental/omnipath_psm2.specs | 2 +- .../dl_dep.sh/0.9.0-experimental/agios.out | 2 +- .../dl_dep.sh/0.9.0-experimental/ci.out | 2 +- .../dl_dep.sh/0.9.0-experimental/default.out | 2 +- .../dl_dep.sh/0.9.0-experimental/direct.out | 2 +- .../0.9.0-experimental/infiniband_verbs.out | 2 +- .../0.9.0-experimental/marenostrum4.out | 2 +- .../dl_dep.sh/0.9.0-experimental/mogon2.out | 2 +- .../dl_dep.sh/0.9.0-experimental/ngio.out | 2 +- .../0.9.0-experimental/omnipath_psm2.out | 2 +- 19 files changed, 78 insertions(+), 18 deletions(-) create mode 100644 scripts/profiles/0.9.0-experimental/install/json-c.install diff --git a/scripts/profiles/0.9.0-experimental/agios.specs b/scripts/profiles/0.9.0-experimental/agios.specs index 79e062517..a5da166e5 100644 --- a/scripts/profiles/0.9.0-experimental/agios.specs +++ b/scripts/profiles/0.9.0-experimental/agios.specs @@ -52,7 +52,7 @@ clonedeps=( ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/ci.specs b/scripts/profiles/0.9.0-experimental/ci.specs index e281e0284..33cbe74fc 100644 --- a/scripts/profiles/0.9.0-experimental/ci.specs +++ b/scripts/profiles/0.9.0-experimental/ci.specs @@ -49,7 +49,7 @@ clonedeps=( ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/default.specs b/scripts/profiles/0.9.0-experimental/default.specs index ac4e40121..317f3c5b4 100644 --- a/scripts/profiles/0.9.0-experimental/default.specs +++ b/scripts/profiles/0.9.0-experimental/default.specs @@ -51,7 +51,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/direct.specs b/scripts/profiles/0.9.0-experimental/direct.specs index 19fb562cc..2016780da 100644 --- a/scripts/profiles/0.9.0-experimental/direct.specs +++ b/scripts/profiles/0.9.0-experimental/direct.specs @@ -47,7 +47,7 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs b/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs index 4cb3146ab..af847d771 100644 --- a/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs +++ b/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs @@ -51,7 +51,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/install/json-c.install b/scripts/profiles/0.9.0-experimental/install/json-c.install new file mode 100644 index 000000000..ccb5aad49 --- /dev/null +++ b/scripts/profiles/0.9.0-experimental/install/json-c.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/json-c" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install + # Margo doesn't search in both directories, so we make it available in both lib and lib64 + if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then + cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/ + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.0-experimental/marenostrum4.specs b/scripts/profiles/0.9.0-experimental/marenostrum4.specs index 524cf74d6..7408f83f5 100644 --- a/scripts/profiles/0.9.0-experimental/marenostrum4.specs +++ b/scripts/profiles/0.9.0-experimental/marenostrum4.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/mogon2.specs b/scripts/profiles/0.9.0-experimental/mogon2.specs index 486a37b03..291ff460f 100644 --- a/scripts/profiles/0.9.0-experimental/mogon2.specs +++ b/scripts/profiles/0.9.0-experimental/mogon2.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/ngio.specs b/scripts/profiles/0.9.0-experimental/ngio.specs index 4f40f811f..62d4ba126 100644 --- a/scripts/profiles/0.9.0-experimental/ngio.specs +++ b/scripts/profiles/0.9.0-experimental/ngio.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs b/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs index cfe5e3980..1e2fb04cf 100644 --- a/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs +++ b/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="0ab2f77aa866dfa697c130dc2f319b530b3007e5" + ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" ) # Extra arguments for git clone diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out index 6e40c07a8..f88c1e711 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out @@ -9,5 +9,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out index 3bcf87b49..64d9ab9eb 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out @@ -6,5 +6,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out index 5060a0cf0..43d83881e 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out @@ -8,5 +8,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out index c18e4d209..23c01815e 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out @@ -4,5 +4,5 @@ Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobot Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out index 0c18881e3..b62be6b8b 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out @@ -8,5 +8,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out index 65689b961..55015f542 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out index 65689b961..55015f542 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out index 65689b961..55015f542 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out index 65689b961..55015f542 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[0ab2f77aa866dfa697c130dc2f319b530b3007e5]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' Done \ No newline at end of file -- GitLab From 135ce55ef9ad0765abf0f261819efb3615988400 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 15 Feb 2022 08:59:33 +0100 Subject: [PATCH 06/17] Increased parallax key size to POSIX MAX_LENGTH --- scripts/profiles/0.9.0-experimental/install/parallax.install | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/profiles/0.9.0-experimental/install/parallax.install b/scripts/profiles/0.9.0-experimental/install/parallax.install index d03d42ba1..4b7ec619a 100644 --- a/scripts/profiles/0.9.0-experimental/install/parallax.install +++ b/scripts/profiles/0.9.0-experimental/install/parallax.install @@ -43,11 +43,13 @@ pkg_install() { CURR="${SOURCE_DIR}/parallax" + sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h prepare_build_dir "${CURR}" cd "${CURR}/build" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ -DBUILD_SHARED_LIBS:BOOL=OFF \ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ .. make -j"${CORES}" make install -- GitLab From 68b85cabee748ef1c838c86aeb2d1d875cc524e5 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 24 Feb 2022 10:27:13 +0100 Subject: [PATCH 07/17] Prepare Parallax Support to 0.9.1 --- .gitlab-ci.yml | 20 +++--- CHANGELOG.md | 5 +- README.md | 13 ++-- docker/0.9.0-experimental/core/Makefile | 4 -- docker/0.9.0-experimental/coverage/Makefile | 4 -- docker/0.9.0-experimental/deps/Makefile | 17 ----- docker/0.9.0-experimental/linter/Makefile | 4 -- docker/0.9.0-experimental/testing/Makefile | 4 -- .../core/Dockerfile | 0 docker/0.9.1/core/Makefile | 10 +++ .../coverage/Dockerfile | 2 +- docker/0.9.1/coverage/Makefile | 10 +++ .../deps/Dockerfile | 11 ++-- docker/0.9.1/deps/Makefile | 23 +++++++ docker/0.9.1/docs/Dockerfile | 33 ++++++++++ docker/0.9.1/docs/Makefile | 10 +++ .../linter/Dockerfile | 6 +- docker/0.9.1/linter/Makefile | 10 +++ .../testing/Dockerfile | 2 +- docker/0.9.1/testing/Makefile | 10 +++ docker/latest | 2 +- scripts/out | 12 ++++ .../0.9.0-experimental/install/bmi.install | 63 ------------------ .../0.9.0-experimental/install/bzip2.install | 51 --------------- .../install/libfabric:verbs.install | 64 ------------------- .../0.9.0-experimental/install/snappy.install | 54 ---------------- .../0.9.0-experimental/install/zstd.install | 54 ---------------- .../{0.9.0-experimental => 0.9.1}/agios.specs | 2 +- .../{0.9.0-experimental => 0.9.1}/ci.specs | 2 +- .../default.specs | 2 +- .../direct.specs | 2 +- .../infiniband_verbs.specs | 2 +- .../install/agios.install | 10 ++- .../install/argobots.install | 10 ++- .../install/capstone.install | 10 ++- .../install/date.install | 18 ++++-- .../install/json-c.install | 4 +- .../install/libfabric%verbs.install} | 15 +++-- .../install/libfabric.install | 15 +++-- .../install/lz4.install | 15 ++++- .../install/margo.install | 13 ++-- .../install/mercury.install | 20 +++--- .../install/parallax.install | 2 +- .../install/rocksdb.install | 4 +- .../install/syscall_intercept.install | 18 ++++-- .../marenostrum4.specs | 2 +- .../mogon2.specs | 2 +- .../{0.9.0-experimental => 0.9.1}/ngio.specs | 2 +- .../omnipath_psm2.specs | 2 +- scripts/profiles/latest | 2 +- .../backend/metadata/parallax_backend.cpp | 3 +- .../default.out => 0.9.1/agios.out} | 5 +- .../{0.9.0-experimental => 0.9.1}/ci.out | 3 +- .../agios.out => 0.9.1/default.out} | 2 +- .../{0.9.0-experimental => 0.9.1}/direct.out | 3 +- .../infiniband_verbs.out | 3 +- .../marenostrum4.out | 3 +- .../{0.9.0-experimental => 0.9.1}/mogon2.out | 3 +- .../{0.9.0-experimental => 0.9.1}/ngio.out | 3 +- .../omnipath_psm2.out | 3 +- tests/scripts/compile_dep.sh/latest | 2 +- .../{0.9.0-experimental => 0.9.1}/agios.out | 2 +- .../{0.9.0-experimental => 0.9.1}/ci.out | 4 +- .../{0.9.0-experimental => 0.9.1}/default.out | 2 +- .../{0.9.0-experimental => 0.9.1}/direct.out | 2 +- .../infiniband_verbs.out | 2 +- .../marenostrum4.out | 2 +- .../{0.9.0-experimental => 0.9.1}/mogon2.out | 2 +- .../{0.9.0-experimental => 0.9.1}/ngio.out | 2 +- .../omnipath_psm2.out | 2 +- tests/scripts/dl_dep.sh/latest | 2 +- 71 files changed, 293 insertions(+), 434 deletions(-) delete mode 100644 docker/0.9.0-experimental/core/Makefile delete mode 100644 docker/0.9.0-experimental/coverage/Makefile delete mode 100644 docker/0.9.0-experimental/deps/Makefile delete mode 100644 docker/0.9.0-experimental/linter/Makefile delete mode 100644 docker/0.9.0-experimental/testing/Makefile rename docker/{0.9.0-experimental => 0.9.1}/core/Dockerfile (100%) create mode 100644 docker/0.9.1/core/Makefile rename docker/{0.9.0-experimental => 0.9.1}/coverage/Dockerfile (94%) create mode 100644 docker/0.9.1/coverage/Makefile rename docker/{0.9.0-experimental => 0.9.1}/deps/Dockerfile (84%) create mode 100644 docker/0.9.1/deps/Makefile create mode 100644 docker/0.9.1/docs/Dockerfile create mode 100644 docker/0.9.1/docs/Makefile rename docker/{0.9.0-experimental => 0.9.1}/linter/Dockerfile (89%) create mode 100644 docker/0.9.1/linter/Makefile rename docker/{0.9.0-experimental => 0.9.1}/testing/Dockerfile (93%) create mode 100644 docker/0.9.1/testing/Makefile create mode 100644 scripts/out delete mode 100644 scripts/profiles/0.9.0-experimental/install/bmi.install delete mode 100644 scripts/profiles/0.9.0-experimental/install/bzip2.install delete mode 100644 scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install delete mode 100644 scripts/profiles/0.9.0-experimental/install/snappy.install delete mode 100644 scripts/profiles/0.9.0-experimental/install/zstd.install rename scripts/profiles/{0.9.0-experimental => 0.9.1}/agios.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/ci.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/default.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/direct.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/infiniband_verbs.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/agios.install (88%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/argobots.install (88%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/capstone.install (88%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/date.install (82%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/json-c.install (96%) rename scripts/profiles/{0.9.0-experimental/install/libfabric:experimental.install => 0.9.1/install/libfabric%verbs.install} (86%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/libfabric.install (86%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/lz4.install (81%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/margo.install (88%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/mercury.install (86%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/parallax.install (97%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/rocksdb.install (96%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/install/syscall_intercept.install (82%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/marenostrum4.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/mogon2.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/ngio.specs (98%) rename scripts/profiles/{0.9.0-experimental => 0.9.1}/omnipath_psm2.specs (98%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental/default.out => 0.9.1/agios.out} (75%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental => 0.9.1}/ci.out (78%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental/agios.out => 0.9.1/default.out} (90%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental => 0.9.1}/direct.out (84%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental => 0.9.1}/infiniband_verbs.out (82%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental => 0.9.1}/marenostrum4.out (84%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental => 0.9.1}/mogon2.out (84%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental => 0.9.1}/ngio.out (84%) rename tests/scripts/compile_dep.sh/{0.9.0-experimental => 0.9.1}/omnipath_psm2.out (84%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/agios.out (94%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/ci.out (93%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/default.out (94%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/direct.out (91%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/infiniband_verbs.out (94%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/marenostrum4.out (94%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/mogon2.out (94%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/ngio.out (94%) rename tests/scripts/dl_dep.sh/{0.9.0-experimental => 0.9.1}/omnipath_psm2.out (94%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c56ee4d74..297088798 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.0-experimental +image: gekkofs/core:0.9.1 ################################################################################ ## Validating ################################################################################ check format: stage: lint - image: gekkofs/linter:0.9.0 + image: gekkofs/linter:0.9.1 needs: [] script: - ${SCRIPTS_DIR}/check_format.sh @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.0-experimental + image: gekkofs/deps:0.9.1 interruptible: true needs: [] script: @@ -71,7 +71,7 @@ gkfs: gkfwd: stage: build - image: gekkofs/deps:0.9.0-experimental + image: gekkofs/deps:0.9.1 interruptible: true needs: [] script: @@ -104,7 +104,7 @@ gkfwd: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.0-experimental + image: gekkofs/testing:0.9.1 needs: [] script: - mkdir -p ${BUILD_PATH}/tests/scripts @@ -120,7 +120,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.0-experimental + image: gekkofs/testing:0.9.1 interruptible: true needs: ['gkfs'] parallel: @@ -166,7 +166,7 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.0-experimental + image: gekkofs/testing:0.9.1 interruptible: true needs: ['gkfwd'] parallel: @@ -211,7 +211,7 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.0-experimental + image: gekkofs/testing:0.9.1 needs: ['gkfs'] script: ## Add path to mkfs.kreon @@ -241,7 +241,7 @@ gkfs:unit: ################################################################################ documentation: stage: docs - image: gekkofs/docs:0.9.0 + image: gekkofs/docs:0.9.1 needs: [] rules: # we only build the documentation automatically if we are on the @@ -277,7 +277,7 @@ documentation: ################################################################################ coverage: stage: report - image: gekkofs/coverage:0.9.0 + image: gekkofs/coverage:0.9.1 needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ] script: - cd ${BUILD_PATH} diff --git a/CHANGELOG.md b/CHANGELOG.md index aabccf79f..a384b8d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] - +- Parallax experimental integration +Support for different databases backend +New Docker Images and scripts (0.9.1) +([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)). ## [0.9.0] - 2022-02-22 ### New diff --git a/README.md b/README.md index e47526930..7ba8c48f7 100644 --- a/README.md +++ b/README.md @@ -106,14 +106,15 @@ Options: -c,--clean-rootdir Cleans Rootdir >before< launching the deamon --version Print version and exit. - --dbbackend 'rocksdb' (default) or 'kreondb' can be specified as - metadata backend. - --keepmd 'kreondb' specific, persist the metadata file + --dbbackend 'rocksdb' (default) or 'parallaxdb' can be specified as + metadata backend, in that case a file in 'metadir' named + rocksdbx is created. Parallaxdb support is experimental. + --keepmd 'parallaxdb' specific, persist the metadata file (default off, file is deleted at the end) - --reusemd 'kreondb' specific, do not recreate the + --reusemd 'parallaxdb' specific, do not recreate the metadata file, (default re/create) - --kreonsize 'kreondb' specific, size of the metadata file in GB - (default 16, 16 GB) + --kreonsize 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB + (default 8, 8 GB) --version Print version and exit. ``` diff --git a/docker/0.9.0-experimental/core/Makefile b/docker/0.9.0-experimental/core/Makefile deleted file mode 100644 index 808a84458..000000000 --- a/docker/0.9.0-experimental/core/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -.PHONY: all - -all: - docker build -t gekkofs/core:0.9.0-experimental . diff --git a/docker/0.9.0-experimental/coverage/Makefile b/docker/0.9.0-experimental/coverage/Makefile deleted file mode 100644 index cc8d4af22..000000000 --- a/docker/0.9.0-experimental/coverage/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -.PHONY: all - -all: - docker build -t gekkofs/coverage:0.8.0 . diff --git a/docker/0.9.0-experimental/deps/Makefile b/docker/0.9.0-experimental/deps/Makefile deleted file mode 100644 index 8f869fa9d..000000000 --- a/docker/0.9.0-experimental/deps/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -CWD:=$(shell pwd) -GIT_ROOT:=$(shell git rev-parse --show-toplevel) - -.PHONY: all build mount-scripts umount-scripts - -all: build mount-scripts umount-scripts - -mount-scripts: - mkdir -p $(CWD)/scripts - mount --bind $(GIT_ROOT)/scripts $(CWD)/scripts - -build: mount-scripts - docker build -t gekkofs/deps:0.9.0-experimental . - -umount-scripts: - - umount $(CWD)/scripts - - rmdir $(CWD)/scripts diff --git a/docker/0.9.0-experimental/linter/Makefile b/docker/0.9.0-experimental/linter/Makefile deleted file mode 100644 index f6752471b..000000000 --- a/docker/0.9.0-experimental/linter/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -.PHONY: all - -all: - docker build -t gekkofs/linter:0.8.0 . diff --git a/docker/0.9.0-experimental/testing/Makefile b/docker/0.9.0-experimental/testing/Makefile deleted file mode 100644 index 416376397..000000000 --- a/docker/0.9.0-experimental/testing/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -.PHONY: all - -all: - docker build -t gekkofs/testing:0.9.0-experimental . diff --git a/docker/0.9.0-experimental/core/Dockerfile b/docker/0.9.1/core/Dockerfile similarity index 100% rename from docker/0.9.0-experimental/core/Dockerfile rename to docker/0.9.1/core/Dockerfile diff --git a/docker/0.9.1/core/Makefile b/docker/0.9.1/core/Makefile new file mode 100644 index 000000000..0f32dcbeb --- /dev/null +++ b/docker/0.9.1/core/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/core:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/core:0.9.1 . + +all: + docker build -t gekkofs/core:0.9.1 . diff --git a/docker/0.9.0-experimental/coverage/Dockerfile b/docker/0.9.1/coverage/Dockerfile similarity index 94% rename from docker/0.9.0-experimental/coverage/Dockerfile rename to docker/0.9.1/coverage/Dockerfile index fb935e0ce..651480713 100644 --- a/docker/0.9.0-experimental/coverage/Dockerfile +++ b/docker/0.9.1/coverage/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster-slim +FROM debian:bullseye-slim LABEL Description="Environment to generate coverage reports in GekkoFS" diff --git a/docker/0.9.1/coverage/Makefile b/docker/0.9.1/coverage/Makefile new file mode 100644 index 000000000..67ba36bba --- /dev/null +++ b/docker/0.9.1/coverage/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/coverage:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/coverage:0.9.1 . + +all: + docker build -t gekkofs/coverage:0.9.1 . \ No newline at end of file diff --git a/docker/0.9.0-experimental/deps/Dockerfile b/docker/0.9.1/deps/Dockerfile similarity index 84% rename from docker/0.9.0-experimental/deps/Dockerfile rename to docker/0.9.1/deps/Dockerfile index 7cd2fbc82..fe81b563b 100644 --- a/docker/0.9.0-experimental/deps/Dockerfile +++ b/docker/0.9.1/deps/Dockerfile @@ -1,9 +1,9 @@ -FROM gekkofs/core:0.9.0-experimental +FROM gekkofs/core:0.9.1 LABEL Description="Debian-based environment to build GekkoFS" ENV GKFS_PATH /opt/gkfs -ENV GKFS_VERSION 0.9.0-experimental +ENV GKFS_VERSION 0.9.1 ENV SCRIPTS_PATH ${GKFS_PATH}/scripts ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src @@ -20,11 +20,12 @@ RUN apt-get update && \ python3-pip \ python3-dev \ python3-venv \ - python3-setuptools \ - libnuma-dev libyaml-dev procps && \ + python3-setuptools \ + libnuma-dev libyaml-dev \ + procps && \ python3 -m pip install --upgrade pip && \ rm -rf /var/lib/apt/lists/* && \ - apt-get clean && apt-get autoclean + apt-get clean && apt-get autoclean # Download and build dependencies RUN cd ${SCRIPTS_PATH} && \ diff --git a/docker/0.9.1/deps/Makefile b/docker/0.9.1/deps/Makefile new file mode 100644 index 000000000..6cd0dc7db --- /dev/null +++ b/docker/0.9.1/deps/Makefile @@ -0,0 +1,23 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts remove-scripts +amd64: build-amd64 mount-scripts remove-scripts +aarch64: build-aarch64 mount-scripts remove-scripts + +copy-scripts: + cp -R $(GIT_ROOT)/scripts $(CWD)/scripts + +build: copy-scripts + docker build -t gekkofs/deps:0.9.1 . + +build-amd64: copy-scripts + docker build --platform amd64 -t gekkofs/deps:0.9.1 . + +build-aarch64: copy-scripts + docker build --platform aarch64 -t gekkofs/deps:0.9.1 . + +remove-scripts: + - rm -rf $(CWD)/scripts diff --git a/docker/0.9.1/docs/Dockerfile b/docker/0.9.1/docs/Dockerfile new file mode 100644 index 000000000..21389dbaf --- /dev/null +++ b/docker/0.9.1/docs/Dockerfile @@ -0,0 +1,33 @@ +FROM gekkofs/deps:0.9.1 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -OL https://www.doxygen.nl/files/doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.33.1' \ + 'exhale==0.3.1' \ + 'sphinx-copybutton==0.5.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.17.0' && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.1/docs/Makefile b/docker/0.9.1/docs/Makefile new file mode 100644 index 000000000..9770b2a5b --- /dev/null +++ b/docker/0.9.1/docs/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/docs:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/docs:0.9.1 . + +all: + docker build -t gekkofs/docs:0.9.1 . \ No newline at end of file diff --git a/docker/0.9.0-experimental/linter/Dockerfile b/docker/0.9.1/linter/Dockerfile similarity index 89% rename from docker/0.9.0-experimental/linter/Dockerfile rename to docker/0.9.1/linter/Dockerfile index 4f93de54d..fa39907d3 100644 --- a/docker/0.9.0-experimental/linter/Dockerfile +++ b/docker/0.9.1/linter/Dockerfile @@ -1,4 +1,4 @@ -FROM gekkofs/core:0.8.0 +FROM gekkofs/core:0.9.1 LABEL Description="Debian-based environment to check the formatting of GekkoFS code" @@ -10,9 +10,9 @@ RUN apt-get update && \ software-properties-common \ gnupg2 && \ # add clang-10 repos - wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 10 && \ + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 11 && \ # install clang-format - apt-get update && apt-get install -y --no-install-recommends clang-format-10 && \ + apt-get update && apt-get install -y --no-install-recommends clang-format-11 && \ # Clean apt cache to reduce image layer size rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ # Clean apt caches of packages diff --git a/docker/0.9.1/linter/Makefile b/docker/0.9.1/linter/Makefile new file mode 100644 index 000000000..449c1cadc --- /dev/null +++ b/docker/0.9.1/linter/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/linter:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/linter:0.9.1 . + +all: + docker build -t gekkofs/linter:0.9.1 . diff --git a/docker/0.9.0-experimental/testing/Dockerfile b/docker/0.9.1/testing/Dockerfile similarity index 93% rename from docker/0.9.0-experimental/testing/Dockerfile rename to docker/0.9.1/testing/Dockerfile index 775653b20..4c28d99ef 100644 --- a/docker/0.9.0-experimental/testing/Dockerfile +++ b/docker/0.9.1/testing/Dockerfile @@ -1,4 +1,4 @@ -FROM gekkofs/deps:0.9.0-experimental +FROM gekkofs/deps:0.9.1 LABEL Description="Debian-based environment to test GekkoFS" diff --git a/docker/0.9.1/testing/Makefile b/docker/0.9.1/testing/Makefile new file mode 100644 index 000000000..7f507aa4b --- /dev/null +++ b/docker/0.9.1/testing/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/testing:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/testing:0.9.1 . + +all: + docker build -t gekkofs/testing:0.9.1 . \ No newline at end of file diff --git a/docker/latest b/docker/latest index 899f24fc7..f514a2f0b 120000 --- a/docker/latest +++ b/docker/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.1 \ No newline at end of file diff --git a/scripts/out b/scripts/out new file mode 100644 index 000000000..bd0b741eb --- /dev/null +++ b/scripts/out @@ -0,0 +1,12 @@ +Destination path is set to "/home/rnou/depsx" +Profile name: direct +Profile version: 0.9.0-experimental +------------------------------------ +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/kreon.git' to 'kreon' with commit '[889353d39052f1c4b3bb9f95821e935dc5a5565e]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Done diff --git a/scripts/profiles/0.9.0-experimental/install/bmi.install b/scripts/profiles/0.9.0-experimental/install/bmi.install deleted file mode 100644 index 411b4d880..000000000 --- a/scripts/profiles/0.9.0-experimental/install/bmi.install +++ /dev/null @@ -1,63 +0,0 @@ -################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ -# vi: ft=bash - -################################################################################ -## The installation script must define both a pkg_install function and -## pkg_check function that, as their name implies, must specify how -## a dependency package should be installed and tested. ## ## The following -## variables can be used in the installation script: -## - CMAKE: a variable that expands to the cmake binary -## - SOURCE_DIR: the directory where the sources for the package were -## downloaded -## - INSTALL_DIR: the directory where the package should be installed -## - CORES: the number of cores to use when building -## - PERFORM_TEST: whether tests for the package should be executed -################################################################################ - -pkg_install() { - CURR="${SOURCE_DIR}/bmi" - prepare_build_dir "${CURR}" - cd "${CURR}" - ./prepare - cd "${CURR}/build" - - if [[ -n ${CFLAGS} ]]; then - CFLAGS="${CFLAGS} -w" - else - CFLAGS="-w" - fi - - ../configure --prefix="${INSTALL_DIR}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict - make -j"${CORES}" - make install -} - -pkg_check() { - : -} diff --git a/scripts/profiles/0.9.0-experimental/install/bzip2.install b/scripts/profiles/0.9.0-experimental/install/bzip2.install deleted file mode 100644 index fa40bc265..000000000 --- a/scripts/profiles/0.9.0-experimental/install/bzip2.install +++ /dev/null @@ -1,51 +0,0 @@ -################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ -# vi: ft=bash - -################################################################################ -## The installation script must define both a pkg_install function and -## pkg_check function that, as their name implies, must specify how -## a dependency package should be installed and tested. ## ## The following -## variables can be used in the installation script: -## - CMAKE: a variable that expands to the cmake binary -## - SOURCE_DIR: the directory where the sources for the package were -## downloaded -## - INSTALL_DIR: the directory where the package should be installed -## - CORES: the number of cores to use when building -## - PERFORM_TEST: whether tests for the package should be executed -################################################################################ - -pkg_install() { - CURR="${SOURCE_DIR}/bzip2" - cd "${CURR}" - make install PREFIX="${INSTALL_DIR}" -} - -pkg_check() { - : -} diff --git a/scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install b/scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install deleted file mode 100644 index 09bf2a089..000000000 --- a/scripts/profiles/0.9.0-experimental/install/libfabric:verbs.install +++ /dev/null @@ -1,64 +0,0 @@ -################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ -# vi: ft=bash - -################################################################################ -## The installation script must define both a pkg_install function and -## pkg_check function that, as their name implies, must specify how -## a dependency package should be installed and tested. ## ## The following -## variables can be used in the installation script: -## - CMAKE: a variable that expands to the cmake binary -## - SOURCE_DIR: the directory where the sources for the package were -## downloaded -## - INSTALL_DIR: the directory where the package should be installed -## - CORES: the number of cores to use when building -## - PERFORM_TEST: whether tests for the package should be executed -################################################################################ - -pkg_install() { - CURR="${SOURCE_DIR}/libfabric" - prepare_build_dir "${CURR}" - cd "${CURR}" - ./autogen.sh - cd "${CURR}/build" - OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" - - EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" - - if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then - OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" - fi - - ${OFI_CONFIG} - make -j${CORES} - make install -} - -pkg_check() { - make check -} diff --git a/scripts/profiles/0.9.0-experimental/install/snappy.install b/scripts/profiles/0.9.0-experimental/install/snappy.install deleted file mode 100644 index 078c72851..000000000 --- a/scripts/profiles/0.9.0-experimental/install/snappy.install +++ /dev/null @@ -1,54 +0,0 @@ -################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ -# vi: ft=bash - -################################################################################ -## The installation script must define both a pkg_install function and -## pkg_check function that, as their name implies, must specify how -## a dependency package should be installed and tested. ## ## The following -## variables can be used in the installation script: -## - CMAKE: a variable that expands to the cmake binary -## - SOURCE_DIR: the directory where the sources for the package were -## downloaded -## - INSTALL_DIR: the directory where the package should be installed -## - CORES: the number of cores to use when building -## - PERFORM_TEST: whether tests for the package should be executed -################################################################################ - -pkg_install() { - CURR=${SOURCE_DIR}/snappy - prepare_build_dir "${CURR}" - cd "${CURR}"/build - $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. - make -j"${CORES}" - make install -} - -pkg_check() { - : -} diff --git a/scripts/profiles/0.9.0-experimental/install/zstd.install b/scripts/profiles/0.9.0-experimental/install/zstd.install deleted file mode 100644 index 2ab403dbf..000000000 --- a/scripts/profiles/0.9.0-experimental/install/zstd.install +++ /dev/null @@ -1,54 +0,0 @@ -################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ -# vi: ft=bash - -################################################################################ -## The installation script must define both a pkg_install function and -## pkg_check function that, as their name implies, must specify how -## a dependency package should be installed and tested. ## ## The following -## variables can be used in the installation script: -## - CMAKE: a variable that expands to the cmake binary -## - SOURCE_DIR: the directory where the sources for the package were -## downloaded -## - INSTALL_DIR: the directory where the package should be installed -## - CORES: the number of cores to use when building -## - PERFORM_TEST: whether tests for the package should be executed -################################################################################ - -pkg_install() { - CURR="${SOURCE_DIR}/zstd/build/cmake" - prepare_build_dir "${CURR}" - cd "${CURR}"/build - $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. - make -j"${CORES}" - make install -} - -pkg_check() { - : -} diff --git a/scripts/profiles/0.9.0-experimental/agios.specs b/scripts/profiles/0.9.1/agios.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/agios.specs rename to scripts/profiles/0.9.1/agios.specs index a5da166e5..2721d3880 100644 --- a/scripts/profiles/0.9.0-experimental/agios.specs +++ b/scripts/profiles/0.9.1/agios.specs @@ -52,7 +52,7 @@ clonedeps=( ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/ci.specs b/scripts/profiles/0.9.1/ci.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/ci.specs rename to scripts/profiles/0.9.1/ci.specs index 33cbe74fc..5f320f817 100644 --- a/scripts/profiles/0.9.0-experimental/ci.specs +++ b/scripts/profiles/0.9.1/ci.specs @@ -49,7 +49,7 @@ clonedeps=( ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/default.specs b/scripts/profiles/0.9.1/default.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/default.specs rename to scripts/profiles/0.9.1/default.specs index 317f3c5b4..aee56bff4 100644 --- a/scripts/profiles/0.9.0-experimental/default.specs +++ b/scripts/profiles/0.9.1/default.specs @@ -51,7 +51,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/direct.specs b/scripts/profiles/0.9.1/direct.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/direct.specs rename to scripts/profiles/0.9.1/direct.specs index 2016780da..1698e21f7 100644 --- a/scripts/profiles/0.9.0-experimental/direct.specs +++ b/scripts/profiles/0.9.1/direct.specs @@ -47,7 +47,7 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs b/scripts/profiles/0.9.1/infiniband_verbs.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/infiniband_verbs.specs rename to scripts/profiles/0.9.1/infiniband_verbs.specs index af847d771..0bcca15a3 100644 --- a/scripts/profiles/0.9.0-experimental/infiniband_verbs.specs +++ b/scripts/profiles/0.9.1/infiniband_verbs.specs @@ -51,7 +51,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/install/agios.install b/scripts/profiles/0.9.1/install/agios.install similarity index 88% rename from scripts/profiles/0.9.0-experimental/install/agios.install rename to scripts/profiles/0.9.1/install/agios.install index 4ceffc2c3..3d0392ff5 100644 --- a/scripts/profiles/0.9.0-experimental/install/agios.install +++ b/scripts/profiles/0.9.1/install/agios.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,11 +37,15 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/agios" + ID="agios" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. diff --git a/scripts/profiles/0.9.0-experimental/install/argobots.install b/scripts/profiles/0.9.1/install/argobots.install similarity index 88% rename from scripts/profiles/0.9.0-experimental/install/argobots.install rename to scripts/profiles/0.9.1/install/argobots.install index 17c4d2be9..96ccea920 100644 --- a/scripts/profiles/0.9.0-experimental/install/argobots.install +++ b/scripts/profiles/0.9.1/install/argobots.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,11 +37,15 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/argobots" + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh diff --git a/scripts/profiles/0.9.0-experimental/install/capstone.install b/scripts/profiles/0.9.1/install/capstone.install similarity index 88% rename from scripts/profiles/0.9.0-experimental/install/capstone.install rename to scripts/profiles/0.9.1/install/capstone.install index 6a4386761..ff299d82b 100644 --- a/scripts/profiles/0.9.0-experimental/install/capstone.install +++ b/scripts/profiles/0.9.1/install/capstone.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,11 +37,15 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/capstone" + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. diff --git a/scripts/profiles/0.9.0-experimental/install/date.install b/scripts/profiles/0.9.1/install/date.install similarity index 82% rename from scripts/profiles/0.9.0-experimental/install/date.install rename to scripts/profiles/0.9.1/install/date.install index cab34851a..c4f12524c 100644 --- a/scripts/profiles/0.9.0-experimental/install/date.install +++ b/scripts/profiles/0.9.1/install/date.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,15 +37,23 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/date" + ID="date" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" - ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON .. - make install + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_CXX_STANDARD:STRING=14 \ + -DUSE_SYSTEM_TZ_DB:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON .. + make -j"${CORES}" install } pkg_check() { diff --git a/scripts/profiles/0.9.0-experimental/install/json-c.install b/scripts/profiles/0.9.1/install/json-c.install similarity index 96% rename from scripts/profiles/0.9.0-experimental/install/json-c.install rename to scripts/profiles/0.9.1/install/json-c.install index ccb5aad49..7280cda81 100644 --- a/scripts/profiles/0.9.0-experimental/install/json-c.install +++ b/scripts/profiles/0.9.1/install/json-c.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.0-experimental/install/libfabric:experimental.install b/scripts/profiles/0.9.1/install/libfabric%verbs.install similarity index 86% rename from scripts/profiles/0.9.0-experimental/install/libfabric:experimental.install rename to scripts/profiles/0.9.1/install/libfabric%verbs.install index 09bf2a089..15257eb81 100644 --- a/scripts/profiles/0.9.0-experimental/install/libfabric:experimental.install +++ b/scripts/profiles/0.9.1/install/libfabric%verbs.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,26 +37,29 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/libfabric" + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh cd "${CURR}/build" OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" - EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" fi ${OFI_CONFIG} - make -j${CORES} - make install + make -j"${CORES}" install } pkg_check() { diff --git a/scripts/profiles/0.9.0-experimental/install/libfabric.install b/scripts/profiles/0.9.1/install/libfabric.install similarity index 86% rename from scripts/profiles/0.9.0-experimental/install/libfabric.install rename to scripts/profiles/0.9.1/install/libfabric.install index 09bf2a089..7db89438e 100644 --- a/scripts/profiles/0.9.0-experimental/install/libfabric.install +++ b/scripts/profiles/0.9.1/install/libfabric.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,26 +37,29 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/libfabric" + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh cd "${CURR}/build" OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" - EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" fi ${OFI_CONFIG} - make -j${CORES} - make install + make -j"${CORES}" install } pkg_check() { diff --git a/scripts/profiles/0.9.0-experimental/install/lz4.install b/scripts/profiles/0.9.1/install/lz4.install similarity index 81% rename from scripts/profiles/0.9.0-experimental/install/lz4.install rename to scripts/profiles/0.9.1/install/lz4.install index 26e549ba3..13e4e53b3 100644 --- a/scripts/profiles/0.9.0-experimental/install/lz4.install +++ b/scripts/profiles/0.9.1/install/lz4.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,12 +37,21 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/lz4" + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" cd "${CURR}" + # try to remove binaries first in case they already exist. Otherwise install fails. + LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4) + for LZ4_BIN in "${LZ4_BINS[@]}"; do + [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN" + done make -j"${CORES}" make DESTDIR="${INSTALL_DIR}" PREFIX="" install } diff --git a/scripts/profiles/0.9.0-experimental/install/margo.install b/scripts/profiles/0.9.1/install/margo.install similarity index 88% rename from scripts/profiles/0.9.0-experimental/install/margo.install rename to scripts/profiles/0.9.1/install/margo.install index ea1a51d5b..d2c6e4b5f 100644 --- a/scripts/profiles/0.9.0-experimental/install/margo.install +++ b/scripts/profiles/0.9.1/install/margo.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,18 +37,21 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/margo" + ID="margo" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./prepare.sh cd "${CURR}/build" ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" - make -j"${CORES}" - make install + make -j"${CORES}" install } pkg_check() { diff --git a/scripts/profiles/0.9.0-experimental/install/mercury.install b/scripts/profiles/0.9.1/install/mercury.install similarity index 86% rename from scripts/profiles/0.9.0-experimental/install/mercury.install rename to scripts/profiles/0.9.1/install/mercury.install index d2677fd63..601305b83 100644 --- a/scripts/profiles/0.9.0-experimental/install/mercury.install +++ b/scripts/profiles/0.9.1/install/mercury.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,13 +37,14 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - set -x - # if the profile compiles bmi, enable it if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then USE_BMI="-DNA_USE_BMI:BOOL=ON" @@ -51,24 +52,23 @@ pkg_install() { USE_BMI="-DNA_USE_BMI:BOOL=OFF" fi - # if the profile compiles libfabric, enable it - if [[ -n "${PROFILE_DEP_NAMES['libfabric']}" ]]; then + # if the profile provides any flavour of libfabric, enable it + if profile_has_dependency "^libfabric.*$"; then USE_OFI="-DNA_USE_OFI:BOOL=ON" else USE_OFI="-DNA_USE_OFI:BOOL=OFF" fi - CURR="${SOURCE_DIR}/mercury" + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DBUILD_TESTING:BOOL=ON \ - -DMERCURY_USE_SM_ROUTING:BOOL=ON \ - -DMERCURY_USE_SELF_FORWARD:BOOL=ON \ -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ -DMERCURY_USE_BOOST_PP:BOOL=ON \ - -DMERCURY_USE_EAGER_BULK:BOOL=ON \ -DBUILD_SHARED_LIBS:BOOL=ON \ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ ${USE_BMI} ${USE_OFI} \ diff --git a/scripts/profiles/0.9.0-experimental/install/parallax.install b/scripts/profiles/0.9.1/install/parallax.install similarity index 97% rename from scripts/profiles/0.9.0-experimental/install/parallax.install rename to scripts/profiles/0.9.1/install/parallax.install index 4b7ec619a..1726549f4 100644 --- a/scripts/profiles/0.9.0-experimental/install/parallax.install +++ b/scripts/profiles/0.9.1/install/parallax.install @@ -43,7 +43,7 @@ pkg_install() { CURR="${SOURCE_DIR}/parallax" - sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h + # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h prepare_build_dir "${CURR}" cd "${CURR}/build" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ diff --git a/scripts/profiles/0.9.0-experimental/install/rocksdb.install b/scripts/profiles/0.9.1/install/rocksdb.install similarity index 96% rename from scripts/profiles/0.9.0-experimental/install/rocksdb.install rename to scripts/profiles/0.9.1/install/rocksdb.install index ab87e7bb8..9350e9feb 100644 --- a/scripts/profiles/0.9.0-experimental/install/rocksdb.install +++ b/scripts/profiles/0.9.1/install/rocksdb.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.0-experimental/install/syscall_intercept.install b/scripts/profiles/0.9.1/install/syscall_intercept.install similarity index 82% rename from scripts/profiles/0.9.0-experimental/install/syscall_intercept.install rename to scripts/profiles/0.9.1/install/syscall_intercept.install index 72d5551a6..9dd287271 100644 --- a/scripts/profiles/0.9.0-experimental/install/syscall_intercept.install +++ b/scripts/profiles/0.9.1/install/syscall_intercept.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,15 +37,23 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR=${SOURCE_DIR}/syscall_intercept + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}"/build - $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. - make install + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DBUILD_TESTS:BOOK=OFF .. + make -j"${CORES}" install } pkg_check() { diff --git a/scripts/profiles/0.9.0-experimental/marenostrum4.specs b/scripts/profiles/0.9.1/marenostrum4.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/marenostrum4.specs rename to scripts/profiles/0.9.1/marenostrum4.specs index 7408f83f5..9a1208601 100644 --- a/scripts/profiles/0.9.0-experimental/marenostrum4.specs +++ b/scripts/profiles/0.9.1/marenostrum4.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/mogon2.specs b/scripts/profiles/0.9.1/mogon2.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/mogon2.specs rename to scripts/profiles/0.9.1/mogon2.specs index 291ff460f..c0257d813 100644 --- a/scripts/profiles/0.9.0-experimental/mogon2.specs +++ b/scripts/profiles/0.9.1/mogon2.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/ngio.specs b/scripts/profiles/0.9.1/ngio.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/ngio.specs rename to scripts/profiles/0.9.1/ngio.specs index 62d4ba126..946d9b6ba 100644 --- a/scripts/profiles/0.9.0-experimental/ngio.specs +++ b/scripts/profiles/0.9.1/ngio.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs b/scripts/profiles/0.9.1/omnipath_psm2.specs similarity index 98% rename from scripts/profiles/0.9.0-experimental/omnipath_psm2.specs rename to scripts/profiles/0.9.1/omnipath_psm2.specs index 1e2fb04cf..1b1ba467e 100644 --- a/scripts/profiles/0.9.0-experimental/omnipath_psm2.specs +++ b/scripts/profiles/0.9.1/omnipath_psm2.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="86a219fcddcd850895195821d393b8a061dc00ba" + ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" ) # Extra arguments for git clone diff --git a/scripts/profiles/latest b/scripts/profiles/latest index 9abbf6810..f514a2f0b 120000 --- a/scripts/profiles/latest +++ b/scripts/profiles/latest @@ -1 +1 @@ -0.9.0-experimental \ No newline at end of file +0.9.1 \ No newline at end of file diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp index 2e1aa4627..17780d49e 100644 --- a/src/daemon/backend/metadata/parallax_backend.cpp +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -62,14 +62,13 @@ ParallaxBackend::~ParallaxBackend() { */ ParallaxBackend::ParallaxBackend(const std::string& path) { - // We try to open options.yml if it exists, if not we create it by default int options = open("options.yml", O_RDWR | O_CREAT, 0644); int64_t sizeOptions; sizeOptions = lseek(options, 0, SEEK_END); if(sizeOptions == 0) { std::string optcontent = - "level0_size: 1\ngc_interval: 10\ngrowth_factor: 4\nmedium_log_LRU_cache_size: 400\nlevel_medium_inplace: 3\n"; + "level0_size: 64\ngc_interval: 10\ngrowth_factor: 4\nmedium_log_LRU_cache_size: 400\nlevel_medium_inplace: 3\n"; write(options, optcontent.c_str(), optcontent.length()); } diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out b/tests/scripts/compile_dep.sh/0.9.1/agios.out similarity index 75% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/default.out rename to tests/scripts/compile_dep.sh/0.9.1/agios.out index 902132395..179154be4 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/default.out +++ b/tests/scripts/compile_dep.sh/0.9.1/agios.out @@ -7,4 +7,7 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### \ No newline at end of file +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/compile_dep.sh/0.9.1/ci.out similarity index 78% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out rename to tests/scripts/compile_dep.sh/0.9.1/ci.out index c72b99ba7..d3c276800 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/ci.out +++ b/tests/scripts/compile_dep.sh/0.9.1/ci.out @@ -5,4 +5,5 @@ ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### -######## Installing: agios ############################### \ No newline at end of file +######## Installing: agios ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/agios.out b/tests/scripts/compile_dep.sh/0.9.1/default.out similarity index 90% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/agios.out rename to tests/scripts/compile_dep.sh/0.9.1/default.out index b3a81c6ab..351b957bd 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/agios.out +++ b/tests/scripts/compile_dep.sh/0.9.1/default.out @@ -8,4 +8,4 @@ ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### -######## Installing: agios ############################### \ No newline at end of file +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out b/tests/scripts/compile_dep.sh/0.9.1/direct.out similarity index 84% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out rename to tests/scripts/compile_dep.sh/0.9.1/direct.out index c972d1bce..868efc2ac 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/direct.out +++ b/tests/scripts/compile_dep.sh/0.9.1/direct.out @@ -3,4 +3,5 @@ ######## Installing: argobots ############################### ######## Installing: margo ############################### ######## Installing: rocksdb ############################### -######## Installing: syscall_intercept ############################### \ No newline at end of file +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out similarity index 82% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/infiniband_verbs.out rename to tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out index 22551fdf8..3543af15c 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/infiniband_verbs.out +++ b/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out @@ -7,4 +7,5 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### \ No newline at end of file +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.1/marenostrum4.out similarity index 84% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/marenostrum4.out rename to tests/scripts/compile_dep.sh/0.9.1/marenostrum4.out index f0221c735..38f67ce3d 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/marenostrum4.out +++ b/tests/scripts/compile_dep.sh/0.9.1/marenostrum4.out @@ -9,4 +9,5 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### \ No newline at end of file +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out b/tests/scripts/compile_dep.sh/0.9.1/mogon2.out similarity index 84% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out rename to tests/scripts/compile_dep.sh/0.9.1/mogon2.out index f0221c735..38f67ce3d 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/mogon2.out +++ b/tests/scripts/compile_dep.sh/0.9.1/mogon2.out @@ -9,4 +9,5 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### \ No newline at end of file +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out b/tests/scripts/compile_dep.sh/0.9.1/ngio.out similarity index 84% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out rename to tests/scripts/compile_dep.sh/0.9.1/ngio.out index f0221c735..38f67ce3d 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/ngio.out +++ b/tests/scripts/compile_dep.sh/0.9.1/ngio.out @@ -9,4 +9,5 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### \ No newline at end of file +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.0-experimental/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out similarity index 84% rename from tests/scripts/compile_dep.sh/0.9.0-experimental/omnipath_psm2.out rename to tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out index f0221c735..38f67ce3d 100644 --- a/tests/scripts/compile_dep.sh/0.9.0-experimental/omnipath_psm2.out +++ b/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out @@ -9,4 +9,5 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### \ No newline at end of file +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/latest b/tests/scripts/compile_dep.sh/latest index 899f24fc7..f514a2f0b 120000 --- a/tests/scripts/compile_dep.sh/latest +++ b/tests/scripts/compile_dep.sh/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.1 \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out b/tests/scripts/dl_dep.sh/0.9.1/agios.out similarity index 94% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out rename to tests/scripts/dl_dep.sh/0.9.1/agios.out index f88c1e711..fba9ecce5 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/agios.out +++ b/tests/scripts/dl_dep.sh/0.9.1/agios.out @@ -9,5 +9,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out b/tests/scripts/dl_dep.sh/0.9.1/ci.out similarity index 93% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out rename to tests/scripts/dl_dep.sh/0.9.1/ci.out index 64d9ab9eb..b10c43848 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/ci.out +++ b/tests/scripts/dl_dep.sh/0.9.1/ci.out @@ -6,5 +6,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' -Done +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out b/tests/scripts/dl_dep.sh/0.9.1/default.out similarity index 94% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/default.out rename to tests/scripts/dl_dep.sh/0.9.1/default.out index 43d83881e..388c7e778 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/default.out +++ b/tests/scripts/dl_dep.sh/0.9.1/default.out @@ -8,5 +8,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out b/tests/scripts/dl_dep.sh/0.9.1/direct.out similarity index 91% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out rename to tests/scripts/dl_dep.sh/0.9.1/direct.out index 23c01815e..4af1d0d5f 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/direct.out +++ b/tests/scripts/dl_dep.sh/0.9.1/direct.out @@ -4,5 +4,5 @@ Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobot Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out similarity index 94% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out rename to tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out index b62be6b8b..d31c171a4 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/infiniband_verbs.out +++ b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out @@ -8,5 +8,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out similarity index 94% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out rename to tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out index 55015f542..a569ac92b 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out similarity index 94% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out rename to tests/scripts/dl_dep.sh/0.9.1/mogon2.out index 55015f542..a569ac92b 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out b/tests/scripts/dl_dep.sh/0.9.1/ngio.out similarity index 94% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out rename to tests/scripts/dl_dep.sh/0.9.1/ngio.out index 55015f542..a569ac92b 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.1/ngio.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out similarity index 94% rename from tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out rename to tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out index 55015f542..a569ac92b 100644 --- a/tests/scripts/dl_dep.sh/0.9.0-experimental/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[86a219fcddcd850895195821d393b8a061dc00ba]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/latest b/tests/scripts/dl_dep.sh/latest index 899f24fc7..f514a2f0b 120000 --- a/tests/scripts/dl_dep.sh/latest +++ b/tests/scripts/dl_dep.sh/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.1 \ No newline at end of file -- GitLab From f61afc37f437dcd22472308f94eeb1073025be60 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 24 Feb 2022 18:26:34 +0100 Subject: [PATCH 08/17] Added CMAKE options to enable only one metadata backend --- .gitlab-ci.yml | 2 + CMakeLists.txt | 28 +-- README.md | 8 + include/daemon/backend/metadata/db.hpp | 3 +- src/daemon/backend/metadata/CMakeLists.txt | 70 ++++++- src/daemon/backend/metadata/db.cpp | 8 + .../backend/metadata/parallax_backend.cpp | 3 +- tests/integration/CMakeLists.txt | 32 ++++ tests/integration/conftest.template | 173 ++++++++++++++++++ 9 files changed, 293 insertions(+), 34 deletions(-) create mode 100644 tests/integration/conftest.template diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 297088798..b44b0bae1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,6 +59,8 @@ gkfs: -DGKFS_INSTALL_TESTS:BOOL=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} -DGKFS_USE_GUIDED_DISTRIBUTION:BOOL=ON + -DGKFS_ENABLE_PARALLAX:BOOL=ON + -DGKFS_ENABLE_ROCKSDB:BOOL=ON ${CI_PROJECT_DIR} - make -j$(nproc) install # reduce artifacts size diff --git a/CMakeLists.txt b/CMakeLists.txt index 440397a40..6fd797641 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,16 +119,12 @@ include(gkfs-code-coverage) set(CMAKE_EXPORT_COMPILE_COMMANDS 0) -# Rocksdb dependencies -find_package(LZ4 REQUIRED) -find_package(JeMalloc) # required if rocksdb has been build with jemalloc -find_package(RocksDB REQUIRED) + # margo dependencies find_package(Mercury REQUIRED) find_package(Abt REQUIRED) find_package(Margo REQUIRED) find_package(Syscall_intercept REQUIRED) -find_package(Parallax REQUIRED) find_package(Threads REQUIRED) # parallax find_package(yaml REQUIRED) @@ -184,6 +180,8 @@ if (GKFS_ENABLE_AGIOS) find_package(AGIOS REQUIRED) endif () +option(GKFS_ENABLE_PARALLAX "Enable Parallax db backend" OFF) +option(GKFS_ENABLE_ROCKSDB "Enable ROCKSDB backend" ON) set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library") add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE}) @@ -201,27 +199,7 @@ endif() configure_file(include/common/cmake_configure.hpp.in include/common/cmake_configure.hpp) -# Imported target -add_library(RocksDB INTERFACE IMPORTED GLOBAL) -target_link_libraries(RocksDB - INTERFACE - ${ROCKSDB_LIBRARIES} - # rocksdb libs - ${LZ4_LIBRARIES} - ) - -if (${JeMalloc_FOUND}) - target_link_libraries(RocksDB - INTERFACE - ${JEMALLOC_LIBRARIES} - ) -endif () -# we cannot use target_include_directories with CMake < 3.11 -set_target_properties(RocksDB - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} - ) add_library(CLI11 INTERFACE) # we cannot use target_include_directories with CMake < 3.11 diff --git a/README.md b/README.md index 7ba8c48f7..8aebec804 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,14 @@ Then, the `examples/distributors/guided/generate.py` scrpt is used to create the Finally, modify `guided_config.txt` to your distribution requirements. +### Metadata Backends +There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH` +is available. +To enable it use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=ON`. + +Once it is enabled, `--dbbackend` option will be functional. + + ### Acknowledgment This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu). diff --git a/include/daemon/backend/metadata/db.hpp b/include/daemon/backend/metadata/db.hpp index 6b87af686..1373832c5 100644 --- a/include/daemon/backend/metadata/db.hpp +++ b/include/daemon/backend/metadata/db.hpp @@ -38,8 +38,9 @@ #include #include #include +#ifdef GKFS_ENABLE_PARALLAX #include - +#endif namespace rdb = rocksdb; namespace gkfs::metadata { diff --git a/src/daemon/backend/metadata/CMakeLists.txt b/src/daemon/backend/metadata/CMakeLists.txt index 9219958d8..107849522 100644 --- a/src/daemon/backend/metadata/CMakeLists.txt +++ b/src/daemon/backend/metadata/CMakeLists.txt @@ -43,20 +43,75 @@ if (HAS_LIBURING) set(LIBURING "-luring") endif() -target_sources(metadata_db +set(DB_DEFAULT_SOURCES PUBLIC ${INCLUDE_DIR}/daemon/backend/metadata/db.hpp ${INCLUDE_DIR}/daemon/backend/exceptions.hpp ${INCLUDE_DIR}/daemon/backend/metadata/metadata_backend.hpp - ${INCLUDE_DIR}/daemon/backend/metadata/rocksdb_backend.hpp - ${INCLUDE_DIR}/daemon/backend/metadata/parallax_backend.hpp PRIVATE ${INCLUDE_DIR}/daemon/backend/metadata/merge.hpp ${CMAKE_CURRENT_LIST_DIR}/merge.cpp ${CMAKE_CURRENT_LIST_DIR}/db.cpp - ${CMAKE_CURRENT_LIST_DIR}/rocksdb_backend.cpp - ${CMAKE_CURRENT_LIST_DIR}/parallax_backend.cpp - ) +) + + +if (GKFS_ENABLE_ROCKSDB) + # Rocksdb dependencies + find_package(LZ4 REQUIRED) + find_package(JeMalloc) # required if rocksdb has been build with jemalloc + find_package(RocksDB REQUIRED) + # Imported target + add_library(RocksDB INTERFACE IMPORTED GLOBAL) + target_link_libraries(RocksDB + INTERFACE + ${ROCKSDB_LIBRARIES} + # rocksdb libs + ${LZ4_LIBRARIES} + ) + + + if (${JeMalloc_FOUND}) + target_link_libraries(RocksDB + INTERFACE + ${JEMALLOC_LIBRARIES} + ) + endif () + # we cannot use target_include_directories with CMake < 3.11 + set_target_properties(RocksDB + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} + ) + + set(DB_ROCKSDB_SOURCES + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/rocksdb_backend.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/rocksdb_backend.cpp + ) + target_compile_definitions(metadata_db + PUBLIC + GKFS_ENABLE_ROCKSDB + ) + +endif() + +if (GKFS_ENABLE_PARALLAX) + find_package(Parallax REQUIRED) + set(DB_PARALLAX_SOURCES + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/parallax_backend.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/parallax_backend.cpp + ) + target_compile_definitions(metadata_db + PUBLIC + GKFS_ENABLE_PARALLAX + ) + add_library(Parallax INTERFACE IMPORTED GLOBAL) +endif() + +target_sources(metadata_db +${DB_DEFAULT_SOURCES} ${DB_PARALLAX_SOURCES} ${DB_ROCKSDB_SOURCES}) add_library(metadata_module STATIC @@ -73,7 +128,8 @@ target_link_libraries(metadata_module PUBLIC spdlog ) -add_library(Parallax INTERFACE IMPORTED GLOBAL) + + target_link_libraries(metadata_db ${PARALLAX_LIBRARY} diff --git a/src/daemon/backend/metadata/db.cpp b/src/daemon/backend/metadata/db.cpp index 19ac84506..c77ac31b0 100644 --- a/src/daemon/backend/metadata/db.cpp +++ b/src/daemon/backend/metadata/db.cpp @@ -51,9 +51,17 @@ struct MetadataDBFactory { static std::unique_ptr create(const std::string& path, const std::string_view id) { + if(id == "parallaxdb") { +#ifdef GKFS_ENABLE_PARALLAX return std::make_unique(path); +#else + GKFS_METADATA_MOD->log()->error("PARALLAX not compiled"); + exit(EXIT_FAILURE); +#endif } + + // Default rocksdb return std::make_unique(path); } diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp index 17780d49e..56986a54a 100644 --- a/src/daemon/backend/metadata/parallax_backend.cpp +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -96,7 +96,8 @@ ParallaxBackend::ParallaxBackend(const std::string& path) { } } if(GKFS_DATA->kreon_reuse_md() && size == 0) { - GKFS_METADATA_MOD->log()->error("KREON database reused but size is 0"); + GKFS_METADATA_MOD->log()->error( + "PARALLAX database reused but size is 0"); exit(EXIT_FAILURE); } diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 37033091a..497a8e558 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -29,6 +29,33 @@ # ensure helper programs in the testing harness get built add_subdirectory(harness) +# Create daemon factories from template, depending of the databases +unset(DBS) +if (GKFS_ENABLE_ROCKSDB) + set (DBS "'gkfs_daemon_rocksdb'") +endif() +if (GKFS_ENABLE_PARALLAX) + if (DBS) + set (DBS "${DBS},'gkfs_daemon_parallaxdb'") + else() + set (DBS "'gkfs_daemon_parallaxdb'") + endif() +endif() + +if (GKFS_ENABLE_PARALLAX AND GKFS_ENABLE_ROCKSDB) + set (DBS "'gkfs_daemon_rocksdb','gkfs_daemon_parallaxdb'") +endif() + +# We enable rocksdb by default +if (NOT DBS) + set (DBS "'gkfs_daemon_rocksdb'") +endif() + +FIND_PATH(BUILD_PATH CMakeLists.txt . ) +FILE(READ ${BUILD_PATH}/conftest.template CONF_TEST_FILE) +STRING(REGEX REPLACE "'gkfs_daemon_rocksdb'" "${DBS}" MOD_CONF_TEST_FILE "${CONF_TEST_FILE}" ) +FILE(WRITE ${BUILD_PATH}/conftest.py "${MOD_CONF_TEST_FILE}") + gkfs_enable_python_testing( BINARY_DIRECTORIES ${CMAKE_BINARY_DIR}/src/daemon/ ${CMAKE_BINARY_DIR}/src/client/ @@ -171,4 +198,9 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) endif () + + endif() + + + diff --git a/tests/integration/conftest.template b/tests/integration/conftest.template new file mode 100644 index 000000000..b012a3783 --- /dev/null +++ b/tests/integration/conftest.template @@ -0,0 +1,173 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +import pytest +import logging +from collections import namedtuple +from _pytest.logging import caplog as _caplog +from pathlib import Path +from harness.logger import logger, initialize_logging, finalize_logging +from harness.cli import add_cli_options, set_default_log_formatter +from harness.workspace import Workspace, FileCreator +from harness.gkfs import Daemon, Client, ShellClient, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator +from harness.reporter import report_test_status, report_test_headline, report_assertion_pass + +def pytest_configure(config): + """ + Some configurations for our particular usage of pytest + """ + set_default_log_formatter(config, "%(message)s") + +def pytest_assertion_pass(item, lineno, orig, expl): + + location = namedtuple( + 'Location', ['path', 'module', 'function', 'lineno'])( + str(item.parent.fspath), item.parent.name, item.name, lineno) + + report_assertion_pass(logger, location, orig, expl) + +def pytest_addoption(parser): + """ + Adds extra options from the GKFS harness to the py.test CLI. + """ + add_cli_options(parser) + +@pytest.fixture(autouse=True) +def caplog(test_workspace, request, _caplog): + + # we don't need to see the logs from sh + _caplog.set_level(logging.CRITICAL, 'sh.command') + _caplog.set_level(logging.CRITICAL, 'sh.command.process') + _caplog.set_level(logging.CRITICAL, 'sh.command.process.streamreader') + _caplog.set_level(logging.CRITICAL, 'sh.stream_bufferer') + _caplog.set_level(logging.CRITICAL, 'sh.streamreader') + + test_log_path = test_workspace.logdir / (request.node.name + ".log") + + h = initialize_logging(logger, test_log_path) + report_test_headline(logger, request.node.nodeid, request.config, test_workspace) + + yield _caplog + + finalize_logging(logger, h) + +def pytest_runtest_logreport(report): + """ + Pytest hook called after a test phase (setup, call, teardownd) + has completed. + """ + + report_test_status(logger, report) + +@pytest.fixture +def test_workspace(tmp_path, request): + """ + Initializes a test workspace by creating a temporary directory for it. + """ + + yield Workspace(tmp_path, + request.config.getoption('--bin-dir'), + request.config.getoption('--lib-dir')) + +@pytest.fixture +def gkfs_daemon_rocksdb(test_workspace, request): + """ + Initializes a local gekkofs daemon + """ + + interface = request.config.getoption('--interface') + daemon = Daemon(interface, "rocksdb", test_workspace) + + yield daemon.run() + daemon.shutdown() + +@pytest.fixture +def gkfs_daemon_parallaxdb(test_workspace, request): + """ + Initializes a local gekkofs daemon + """ + + interface = request.config.getoption('--interface') + daemon = Daemon(interface, "parallaxdb", test_workspace) + + yield daemon.run() + daemon.shutdown() + +@pytest.fixture(params=['gkfs_daemon_rocksdb']) +def gkfs_daemon(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def gkfs_client(test_workspace): + """ + Sets up a gekkofs client environment so that + operations (system calls, library calls, ...) can + be requested from a co-running daemon. + """ + + return Client(test_workspace) + +@pytest.fixture +def gkfs_shell(test_workspace): + """ + Sets up a gekkofs environment so that shell commands + (stat, ls, mkdir, etc.) can be issued to a co-running daemon. + """ + + return ShellClient(test_workspace) + +@pytest.fixture +def file_factory(test_workspace): + """ + Returns a factory that can create custom input files + in the test workspace. + """ + + return FileCreator(test_workspace) + +@pytest.fixture +def gkfwd_daemon_factory(test_workspace, request): + """ + Returns a factory that can create forwarding daemons + in the test workspace. + """ + + interface = request.config.getoption('--interface') + + return FwdDaemonCreator(interface, test_workspace) + +@pytest.fixture +def gkfwd_client_factory(test_workspace): + """ + Sets up a gekkofs client environment so that + operations (system calls, library calls, ...) can + be requested from a co-running daemon. + """ + + return FwdClientCreator(test_workspace) -- GitLab From d0c0e9ffbc9a7135e53f70baf3e9ddf4d14aef4e Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 28 Feb 2022 13:09:26 +0100 Subject: [PATCH 09/17] New parameter to remove rootdir at the end --- .gitlab-ci.yml | 2 + CHANGELOG.md | 1 + README.md | 7 +--- docs/sphinx/users/building.rst | 5 +++ docs/sphinx/users/running.md | 6 +++ include/client/preload_util.hpp | 1 - include/daemon/backend/metadata/db.hpp | 4 +- include/daemon/classes/fs_data.hpp | 14 ++----- scripts/out | 12 ------ src/daemon/backend/metadata/db.cpp | 12 ++++-- .../backend/metadata/parallax_backend.cpp | 28 +++----------- src/daemon/classes/fs_data.cpp | 18 ++------- src/daemon/daemon.cpp | 37 +++++++++++++------ tests/integration/harness/gkfs.py | 4 +- 14 files changed, 70 insertions(+), 81 deletions(-) delete mode 100644 scripts/out diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b44b0bae1..b5a077a34 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -87,6 +87,8 @@ gkfwd: -DGKFS_INSTALL_TESTS:BOOL=ON -DGKFS_ENABLE_FORWARDING:BOOL=ON -DGKFS_ENABLE_AGIOS:BOOL=ON + -DGKFS_ENABLE_PARALLAX:BOOL=OFF + -DGKFS_ENABLE_ROCKSDB:BOOL=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} ${CI_PROJECT_DIR} - make -j$(nproc) install diff --git a/CHANGELOG.md b/CHANGELOG.md index a384b8d23..0596326cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Support for different databases backend New Docker Images and scripts (0.9.1) ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)). +- Added `--clean-rootdir-finish` argument to remove rootdir/metadir at the end. ## [0.9.0] - 2022-02-22 ### New diff --git a/README.md b/README.md index 8aebec804..b349e1be7 100644 --- a/README.md +++ b/README.md @@ -104,15 +104,12 @@ Options: Libfabric must have enabled support verbs or psm2. --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) -c,--clean-rootdir Cleans Rootdir >before< launching the deamon + -f, --clean-rootdir-finish Cleans Rootdir >After< the deamon finishes --version Print version and exit. --dbbackend 'rocksdb' (default) or 'parallaxdb' can be specified as metadata backend, in that case a file in 'metadir' named rocksdbx is created. Parallaxdb support is experimental. - --keepmd 'parallaxdb' specific, persist the metadata file - (default off, file is deleted at the end) - --reusemd 'parallaxdb' specific, do not recreate the - metadata file, (default re/create) --kreonsize 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB (default 8, 8 GB) --version Print version and exit. @@ -239,7 +236,7 @@ Finally, modify `guided_config.txt` to your distribution requirements. ### Metadata Backends There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH` is available. -To enable it use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=ON`. +To enable it use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=OFF`. Once it is enabled, `--dbbackend` option will be functional. diff --git a/docs/sphinx/users/building.rst b/docs/sphinx/users/building.rst index bcd13438b..259e84d1a 100644 --- a/docs/sphinx/users/building.rst +++ b/docs/sphinx/users/building.rst @@ -108,6 +108,11 @@ dependencies: - `AGIOS `_ (commit c26a654 or newer) to enable the :code:`GekkoFWD` I/O forwarding mode. +- `PARALLAX` There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH` +is available. To enable it, use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=OFF`. + Once it is enabled, `--dbbackend` option will be functional. + + .. _step_by_step_installation: Step-by-step installation diff --git a/docs/sphinx/users/running.md b/docs/sphinx/users/running.md index 286353a98..f023c5ba4 100644 --- a/docs/sphinx/users/running.md +++ b/docs/sphinx/users/running.md @@ -74,6 +74,12 @@ Further options are available Libfabric must have enabled support verbs or psm2. --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) -c,--clean-rootdir Cleans Rootdir >before< launching the deamon + -f,--clean-rootdir-finish Cleans Rootdir >after< finishing the deamon + --dbbackend 'rocksdb' (default) or 'parallaxdb' can be specified as + metadata backend, in that case a file in 'metadir' named + rocksdbx is created. Parallaxdb support is experimental. + --kreonsize 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB + (default 8, 8 GB) --version Print version and exit. ```` diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index 2849c87e0..d4bb154a7 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -32,7 +32,6 @@ #include #include -#include #include #include #include diff --git a/include/daemon/backend/metadata/db.hpp b/include/daemon/backend/metadata/db.hpp index 1373832c5..e226ba6b0 100644 --- a/include/daemon/backend/metadata/db.hpp +++ b/include/daemon/backend/metadata/db.hpp @@ -37,11 +37,13 @@ #include #include #include +#ifdef GKFS_ENABLE_ROCKSDB #include +#endif #ifdef GKFS_ENABLE_PARALLAX #include #endif -namespace rdb = rocksdb; + namespace gkfs::metadata { diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 071ca8821..879d155cd 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -71,9 +71,9 @@ private: // Database std::shared_ptr mdb_; std::string dbbackend_; + bool keep_rootdir_ = true; // Kreon - bool reuse_md_ = false; - bool keep_md_ = false; + unsigned long long size_md_ = 8589934592ull; // Storage backend @@ -206,16 +206,10 @@ public: blocks_state(bool blocks_state); bool - kreon_keep_md() const; - - void - kreon_keep_md(bool keep_md); - - bool - kreon_reuse_md() const; + keep_rootdir() const; void - kreon_reuse_md(bool reuse_md); + keep_rootdir(bool keep_rootdir_); unsigned long long kreon_size_md() const; diff --git a/scripts/out b/scripts/out deleted file mode 100644 index bd0b741eb..000000000 --- a/scripts/out +++ /dev/null @@ -1,12 +0,0 @@ -Destination path is set to "/home/rnou/depsx" -Profile name: direct -Profile version: 0.9.0-experimental ------------------------------------- -Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' -Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/kreon.git' to 'kreon' with commit '[889353d39052f1c4b3bb9f95821e935dc5a5565e]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' -Done diff --git a/src/daemon/backend/metadata/db.cpp b/src/daemon/backend/metadata/db.cpp index c77ac31b0..6a440eb30 100644 --- a/src/daemon/backend/metadata/db.cpp +++ b/src/daemon/backend/metadata/db.cpp @@ -58,12 +58,18 @@ struct MetadataDBFactory { #else GKFS_METADATA_MOD->log()->error("PARALLAX not compiled"); exit(EXIT_FAILURE); +#endif + } else if(id == "rocksdb") { +#ifdef GKFS_ENABLE_ROCKSDB + return std::make_unique(path); +#else + GKFS_METADATA_MOD->log()->error("ROCKSDB not compiled"); + exit(EXIT_FAILURE); #endif } - - // Default rocksdb - return std::make_unique(path); + GKFS_METADATA_MOD->log()->error("No valid metadata backend selected"); + exit(EXIT_FAILURE); } }; diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp index 56986a54a..31ded67c5 100644 --- a/src/daemon/backend/metadata/parallax_backend.cpp +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -52,8 +52,6 @@ namespace gkfs::metadata { */ ParallaxBackend::~ParallaxBackend() { par_close(par_db_); - if(GKFS_DATA->kreon_keep_md() == false) - unlink(par_path_.c_str()); } /** @@ -85,23 +83,15 @@ ParallaxBackend::ParallaxBackend(const std::string& path) { } // Check size if we want to reuse it - if(GKFS_DATA->kreon_reuse_md()) { - - size = lseek(fd, 0, SEEK_END); - if(size == -1) { - printf("[%s:%s:%d] failed to determine volume size exiting...\n", - __FILE__, __func__, __LINE__); - perror("ioctl"); - exit(EXIT_FAILURE); - } - } - if(GKFS_DATA->kreon_reuse_md() && size == 0) { - GKFS_METADATA_MOD->log()->error( - "PARALLAX database reused but size is 0"); + size = lseek(fd, 0, SEEK_END); + if(size == -1) { + printf("[%s:%s:%d] failed to determine volume size exiting...\n", + __FILE__, __func__, __LINE__); + perror("ioctl"); exit(EXIT_FAILURE); } - if(GKFS_DATA->kreon_reuse_md() == false) { + if(size == 0) { size = GKFS_DATA->kreon_size_md(); lseek(fd, size - 1, SEEK_SET); @@ -113,12 +103,6 @@ ParallaxBackend::ParallaxBackend(const std::string& path) { std::string cmd = "kv_format.parallax --device " + par_path_ + " --max_regions_num 1 "; system(cmd.c_str()); - - /* - fd = open(par_path_.c_str(), O_RDONLY); - size = lseek(fd, 0, SEEK_END); - close(fd); - */ } par_options_.create_flag = PAR_CREATE_DB; diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 09421f417..fd14a2c35 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -211,23 +211,13 @@ FsData::blocks_state(bool blocks_state) { } bool -FsData::kreon_keep_md() const { - return keep_md_; +FsData::keep_rootdir() const { + return keep_rootdir_; } void -FsData::kreon_keep_md(bool keep_md) { - FsData::keep_md_ = keep_md; -} - -bool -FsData::kreon_reuse_md() const { - return reuse_md_; -} - -void -FsData::kreon_reuse_md(bool reuse_md) { - FsData::reuse_md_ = reuse_md; +FsData::keep_rootdir(bool keep_rootdir) { + FsData::keep_rootdir_ = keep_rootdir; } unsigned long long diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5f314f77c..da5fc7478 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -409,6 +409,19 @@ destroy_enviroment() { GKFS_DATA->spdlogger()->info("{}() Closing metadata DB", __func__); GKFS_DATA->close_mdb(); + + + // Delete rootdir/metadir if requested + if(!GKFS_DATA->keep_rootdir()) { + GKFS_DATA->spdlogger()->info("{}() Removing RootDir/MetaDir/MetaFile", + __func__); + fs::remove_all(GKFS_DATA->metadir(), ecode); +#ifdef GKFS_ENABLE_PARALLAX + // some metadata backends uses a file instead of a dir. + fs::remove_all(GKFS_DATA->metadir() + "x", ecode); +#endif + fs::remove_all(GKFS_DATA->rootdir(), ecode); + } } /** @@ -558,6 +571,11 @@ parse_input(const cli_options& opts, const CLI::App& desc) { fs::remove_all(rootdir_path.native()); GKFS_DATA->spdlogger()->info("{}() Rootdir cleaned.", __func__); } + + if(desc.count("--clean-rootdir-finish")) { + GKFS_DATA->keep_rootdir(false); + } + GKFS_DATA->spdlogger()->debug("{}() Root directory: '{}'", __func__, rootdir_path.native()); fs::create_directories(rootdir_path); @@ -596,12 +614,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { } else GKFS_DATA->dbbackend("rocksdb"); - if(desc.count("--keepmd")) { - GKFS_DATA->kreon_keep_md(true); - } - if(desc.count("--reusemd")) { - GKFS_DATA->kreon_reuse_md(true); - } if(desc.count("--kreonsize")) { // Size in GB GKFS_DATA->kreon_size_md(stoi(opts.kreonsize)); } @@ -661,16 +673,17 @@ main(int argc, const char* argv[]) { desc.add_flag( "--clean-rootdir,-c", "Cleans Rootdir >before< launching the deamon"); + desc.add_flag( + "--clean-rootdir-finish,-f", + "Cleans Rootdir >after< the deamon finishes"); desc.add_option( "--dbbackend,-d", opts.dbbackend, - "Database Backend to use. If not set, rocksdb is used. For parallaxdb, a file called rocksdb with 8GB is needed in metadir"); - desc.add_flag("--keepmd", "Kreondb - Keeps metadir (default off)"); - desc.add_flag("--reusemd", - "Kreondb - Avoids initializing the metadata file, (default off)"); + "Database Backend to use. If not set, rocksdb is used. For parallaxdb, a file called rocksdbx with 8GB will be created in metadir"); + + desc.add_option("--kreonsize",opts.kreonsize, - "Kreondb - Metatada file size in GB (default 8), " + "parallaxdb - Metatada file size in GB (default 8), " "used only with new files"); - desc.add_flag("--version", "Print version and exit."); // clang-format on try { diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index eaf862fb8..6f5624dae 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -251,7 +251,9 @@ class Daemon: '--rootdir', self.rootdir, '-l', self._address, '--metadir', self._metadir, - '--dbbackend', self._database ] + '--dbbackend', self._database] + if self._database == "parallaxdb" : + args.append('--clean-rootdir-finish') logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) -- GitLab From 14a087137d7be134e9d4213b01629eb09a4185cd Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 15 Mar 2022 08:14:14 +0100 Subject: [PATCH 10/17] Updated PARALLAX --- scripts/profiles/0.9.1/agios.specs | 2 +- scripts/profiles/0.9.1/ci.specs | 2 +- scripts/profiles/0.9.1/default.specs | 2 +- scripts/profiles/0.9.1/direct.specs | 2 +- scripts/profiles/0.9.1/infiniband_verbs.specs | 2 +- scripts/profiles/0.9.1/marenostrum4.specs | 2 +- scripts/profiles/0.9.1/mogon2.specs | 2 +- scripts/profiles/0.9.1/ngio.specs | 2 +- scripts/profiles/0.9.1/omnipath_psm2.specs | 2 +- tests/scripts/dl_dep.sh/0.9.1/agios.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/ci.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/default.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/direct.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/mogon2.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/ngio.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/profiles/0.9.1/agios.specs b/scripts/profiles/0.9.1/agios.specs index 2721d3880..b4aded64f 100644 --- a/scripts/profiles/0.9.1/agios.specs +++ b/scripts/profiles/0.9.1/agios.specs @@ -52,7 +52,7 @@ clonedeps=( ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/ci.specs b/scripts/profiles/0.9.1/ci.specs index 5f320f817..99c91f716 100644 --- a/scripts/profiles/0.9.1/ci.specs +++ b/scripts/profiles/0.9.1/ci.specs @@ -49,7 +49,7 @@ clonedeps=( ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/default.specs b/scripts/profiles/0.9.1/default.specs index aee56bff4..cad5a23da 100644 --- a/scripts/profiles/0.9.1/default.specs +++ b/scripts/profiles/0.9.1/default.specs @@ -51,7 +51,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/direct.specs b/scripts/profiles/0.9.1/direct.specs index 1698e21f7..18c0a5393 100644 --- a/scripts/profiles/0.9.1/direct.specs +++ b/scripts/profiles/0.9.1/direct.specs @@ -47,7 +47,7 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/infiniband_verbs.specs b/scripts/profiles/0.9.1/infiniband_verbs.specs index 0bcca15a3..9beeb344d 100644 --- a/scripts/profiles/0.9.1/infiniband_verbs.specs +++ b/scripts/profiles/0.9.1/infiniband_verbs.specs @@ -51,7 +51,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/marenostrum4.specs b/scripts/profiles/0.9.1/marenostrum4.specs index 9a1208601..2fd272eff 100644 --- a/scripts/profiles/0.9.1/marenostrum4.specs +++ b/scripts/profiles/0.9.1/marenostrum4.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/mogon2.specs b/scripts/profiles/0.9.1/mogon2.specs index c0257d813..71ebf7a65 100644 --- a/scripts/profiles/0.9.1/mogon2.specs +++ b/scripts/profiles/0.9.1/mogon2.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/ngio.specs b/scripts/profiles/0.9.1/ngio.specs index 946d9b6ba..72c99cebd 100644 --- a/scripts/profiles/0.9.1/ngio.specs +++ b/scripts/profiles/0.9.1/ngio.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/scripts/profiles/0.9.1/omnipath_psm2.specs b/scripts/profiles/0.9.1/omnipath_psm2.specs index 1b1ba467e..50dbef5cc 100644 --- a/scripts/profiles/0.9.1/omnipath_psm2.specs +++ b/scripts/profiles/0.9.1/omnipath_psm2.specs @@ -52,7 +52,7 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="24750a64efee4352fcc7ac5533acb887ff6d4647" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone diff --git a/tests/scripts/dl_dep.sh/0.9.1/agios.out b/tests/scripts/dl_dep.sh/0.9.1/agios.out index fba9ecce5..3ed872e6b 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/agios.out +++ b/tests/scripts/dl_dep.sh/0.9.1/agios.out @@ -9,5 +9,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/ci.out b/tests/scripts/dl_dep.sh/0.9.1/ci.out index b10c43848..aa2acc0cd 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/ci.out +++ b/tests/scripts/dl_dep.sh/0.9.1/ci.out @@ -6,5 +6,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/default.out b/tests/scripts/dl_dep.sh/0.9.1/default.out index 388c7e778..4f301b9a1 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/default.out +++ b/tests/scripts/dl_dep.sh/0.9.1/default.out @@ -8,5 +8,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/direct.out b/tests/scripts/dl_dep.sh/0.9.1/direct.out index 4af1d0d5f..e98962560 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/direct.out +++ b/tests/scripts/dl_dep.sh/0.9.1/direct.out @@ -4,5 +4,5 @@ Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobot Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out index d31c171a4..e812c1f9e 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out +++ b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out @@ -8,5 +8,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out index a569ac92b..1d4ad26f5 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out index a569ac92b..1d4ad26f5 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/ngio.out b/tests/scripts/dl_dep.sh/0.9.1/ngio.out index a569ac92b..1d4ad26f5 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.1/ngio.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out index a569ac92b..1d4ad26f5 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out @@ -9,5 +9,5 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[24750a64efee4352fcc7ac5533acb887ff6d4647]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file -- GitLab From be7a9eab72728e5e0af181ab24befd2e27df2367 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Wed, 16 Mar 2022 20:42:05 +0100 Subject: [PATCH 11/17] Refactoring CMake files for metadata db --- CMake/FindParallax.cmake | 10 +- CMakeLists.txt | 2 - .../profiles/0.9.1/install/parallax.install | 5 +- src/daemon/backend/metadata/CMakeLists.txt | 217 +++++++++--------- 4 files changed, 122 insertions(+), 112 deletions(-) diff --git a/CMake/FindParallax.cmake b/CMake/FindParallax.cmake index e6cce2d25..562ad6b9c 100644 --- a/CMake/FindParallax.cmake +++ b/CMake/FindParallax.cmake @@ -33,19 +33,19 @@ find_library(PARALLAX_LIBRARY NAMES parallax ) -find_library(LOG_LIBRARY +find_library(PARALLAX_LOG_LIBRARY NAMES log ) set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR}) set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY}) -set(PARALLAX_LIBRARIES ${LOG_LIBRARY}) +set(PARALLAX_LIBRARIES ${PARALLAX_LOG_LIBRARY}) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY LOG_LIBRARY PARALLAX_INCLUDE_DIR) +find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY PARALLAX_LOG_LIBRARY PARALLAX_INCLUDE_DIR) mark_as_advanced( PARALLAX_LIBRARY - PARALLAX_INCLUDE_DIR - LOG_LIBRARY + PARALLAX_INCLUDE_DIR + PARALLAX_LOG_LIBRARY ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fd797641..7589df519 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,8 +126,6 @@ find_package(Abt REQUIRED) find_package(Margo REQUIRED) find_package(Syscall_intercept REQUIRED) find_package(Threads REQUIRED) -# parallax -find_package(yaml REQUIRED) # some compilers need extra flags for std::filesystem, such as -lstdc++fs, this # produces a std::filesystem imported target that takes care of all these # details transparently diff --git a/scripts/profiles/0.9.1/install/parallax.install b/scripts/profiles/0.9.1/install/parallax.install index 1726549f4..0c552cc78 100644 --- a/scripts/profiles/0.9.1/install/parallax.install +++ b/scripts/profiles/0.9.1/install/parallax.install @@ -49,12 +49,13 @@ pkg_install() { PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ -DBUILD_SHARED_LIBS:BOOL=OFF \ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \ .. make -j"${CORES}" make install } pkg_check() { - : + : } diff --git a/src/daemon/backend/metadata/CMakeLists.txt b/src/daemon/backend/metadata/CMakeLists.txt index 107849522..5d3c68767 100644 --- a/src/daemon/backend/metadata/CMakeLists.txt +++ b/src/daemon/backend/metadata/CMakeLists.txt @@ -26,24 +26,23 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -add_library(metadata_db STATIC) -# If it exists, RocksDB has been build with liburing support. -# If check succeeds, we need to link with liburing with the daemon -set(CMAKE_REQUIRED_FLAGS "-luring") -check_cxx_source_compiles(" - #include - int main() { - struct io_uring ring; - io_uring_queue_init(1, &ring, 0); - return 0; - } - " HAS_LIBURING) -set(liburing "") -if (HAS_LIBURING) - set(LIBURING "-luring") -endif() -set(DB_DEFAULT_SOURCES +# Define metadata module library first +add_library(metadata_module STATIC) +target_sources(metadata_module + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/metadata_module.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/metadata_module.cpp + ) +target_link_libraries(metadata_module + PUBLIC + spdlog + ) + +# Define metadata_db and its common dependencies and sources +add_library(metadata_db STATIC) +set(DB_COMMON_SOURCES PUBLIC ${INCLUDE_DIR}/daemon/backend/metadata/db.hpp ${INCLUDE_DIR}/daemon/backend/exceptions.hpp @@ -52,95 +51,107 @@ set(DB_DEFAULT_SOURCES ${INCLUDE_DIR}/daemon/backend/metadata/merge.hpp ${CMAKE_CURRENT_LIST_DIR}/merge.cpp ${CMAKE_CURRENT_LIST_DIR}/db.cpp -) - - -if (GKFS_ENABLE_ROCKSDB) - # Rocksdb dependencies - find_package(LZ4 REQUIRED) - find_package(JeMalloc) # required if rocksdb has been build with jemalloc - find_package(RocksDB REQUIRED) - # Imported target - add_library(RocksDB INTERFACE IMPORTED GLOBAL) - target_link_libraries(RocksDB - INTERFACE - ${ROCKSDB_LIBRARIES} - # rocksdb libs - ${LZ4_LIBRARIES} - ) - - - if (${JeMalloc_FOUND}) - target_link_libraries(RocksDB - INTERFACE - ${JEMALLOC_LIBRARIES} - ) - endif () - # we cannot use target_include_directories with CMake < 3.11 - set_target_properties(RocksDB - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} - ) - - set(DB_ROCKSDB_SOURCES - PUBLIC - ${INCLUDE_DIR}/daemon/backend/metadata/rocksdb_backend.hpp - PRIVATE - ${CMAKE_CURRENT_LIST_DIR}/rocksdb_backend.cpp - ) - target_compile_definitions(metadata_db - PUBLIC - GKFS_ENABLE_ROCKSDB - ) - -endif() - -if (GKFS_ENABLE_PARALLAX) - find_package(Parallax REQUIRED) - set(DB_PARALLAX_SOURCES - PUBLIC - ${INCLUDE_DIR}/daemon/backend/metadata/parallax_backend.hpp - PRIVATE - ${CMAKE_CURRENT_LIST_DIR}/parallax_backend.cpp - ) - target_compile_definitions(metadata_db - PUBLIC - GKFS_ENABLE_PARALLAX - ) - add_library(Parallax INTERFACE IMPORTED GLOBAL) -endif() - -target_sources(metadata_db -${DB_DEFAULT_SOURCES} ${DB_PARALLAX_SOURCES} ${DB_ROCKSDB_SOURCES}) - -add_library(metadata_module - STATIC -) - -target_sources(metadata_module - PUBLIC - ${INCLUDE_DIR}/daemon/backend/metadata/metadata_module.hpp - PRIVATE - ${CMAKE_CURRENT_LIST_DIR}/metadata_module.cpp -) - -target_link_libraries(metadata_module - PUBLIC - spdlog -) - - - -target_link_libraries(metadata_db - ${PARALLAX_LIBRARY} - ${LOG_LIBRARY} - yaml - # Required by RocksDB + ) +# lib links for all DBs. Extended below for specific DB +set(DB_LINK_LIBRARIES + PRIVATE + # Required by RocksDB metadata_module -ldl metadata - RocksDB spdlog path_util - ${LIBURING} ) + +# When RocksDB is used +if (GKFS_ENABLE_ROCKSDB) + # If it exists, RocksDB has been build with liburing support. + # If check succeeds, we need to link with liburing with the daemon + set(CMAKE_REQUIRED_FLAGS "-luring") + check_cxx_source_compiles(" + #include + int main() { + struct io_uring ring; + io_uring_queue_init(1, &ring, 0); + return 0; + } + " HAS_LIBURING) + if (HAS_LIBURING) + set(DB_LINK_LIBRARIES + ${DB_LINK_LIBRARIES} + -luring + ) + endif () + # Rocksdb dependencies + find_package(LZ4 REQUIRED) + find_package(JeMalloc) # required if rocksdb has been build with jemalloc + find_package(RocksDB REQUIRED) + # Imported target + add_library(RocksDB INTERFACE IMPORTED GLOBAL) + target_link_libraries(RocksDB + INTERFACE + ${ROCKSDB_LIBRARIES} + # rocksdb libs + ${LZ4_LIBRARIES} + ) + if (${JeMalloc_FOUND}) + target_link_libraries(RocksDB + INTERFACE + ${JEMALLOC_LIBRARIES} + ) + endif () + # we cannot use target_include_directories with CMake < 3.11 + set_target_properties(RocksDB + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} + ) + set(DB_ROCKSDB_SOURCES + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/rocksdb_backend.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/rocksdb_backend.cpp + ) + target_compile_definitions(metadata_db + PUBLIC + GKFS_ENABLE_ROCKSDB + ) + set(DB_LINK_LIBRARIES + ${DB_LINK_LIBRARIES} + RocksDB + ) +endif () + +# When Parallax is used +if (GKFS_ENABLE_PARALLAX) + find_package(yaml REQUIRED) + find_package(Parallax REQUIRED) + set(DB_PARALLAX_SOURCES + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/parallax_backend.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/parallax_backend.cpp + ) + target_compile_definitions(metadata_db + PUBLIC + GKFS_ENABLE_PARALLAX + ) + add_library(Parallax INTERFACE IMPORTED GLOBAL) + target_link_libraries(Parallax + INTERFACE + ${PARALLAX_LIBRARY} + ${PARALLAX_LOG_LIBRARY} + yaml + ) + set_target_properties(Parallax + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${PARALLAX_INCLUDE_DIRS} + ) + set(DB_LINK_LIBRARIES + ${DB_LINK_LIBRARIES} + Parallax + ) +endif () + +target_sources(metadata_db + ${DB_COMMON_SOURCES} ${DB_PARALLAX_SOURCES} ${DB_ROCKSDB_SOURCES}) +target_link_libraries(metadata_db ${DB_LINK_LIBRARIES}) \ No newline at end of file -- GitLab From f5a24bddb3fcac116823d46e58b54624b5d9dabb Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Wed, 16 Mar 2022 22:01:42 +0100 Subject: [PATCH 12/17] Sanitizing --dbbackend input, refactor metadata db path usage --- include/config.hpp | 5 +- include/daemon/backend/metadata/db.hpp | 4 +- src/daemon/backend/metadata/db.cpp | 32 +++++++------ .../backend/metadata/parallax_backend.cpp | 9 ++-- src/daemon/daemon.cpp | 48 +++++++++++++------ 5 files changed, 61 insertions(+), 37 deletions(-) diff --git a/include/config.hpp b/include/config.hpp index 10d672a25..e8fcd01fb 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -57,6 +57,9 @@ constexpr auto daemon_log_level = 4; // info } // namespace log namespace metadata { +// directory name where the metadata db instance is placed +constexpr auto dir = "metadata"; + // which metadata should be considered apart from size and mode constexpr auto use_atime = false; constexpr auto use_ctime = false; @@ -98,8 +101,6 @@ constexpr auto daemon_handler_xstreams = 4; namespace rocksdb { // Write-ahead logging of rocksdb constexpr auto use_write_ahead_log = false; -// directory name where the rocksdb instance is placed -constexpr auto data_dir = "rocksdb"; } // namespace rocksdb } // namespace gkfs::config diff --git a/include/daemon/backend/metadata/db.hpp b/include/daemon/backend/metadata/db.hpp index e226ba6b0..1114a9e31 100644 --- a/include/daemon/backend/metadata/db.hpp +++ b/include/daemon/backend/metadata/db.hpp @@ -47,6 +47,9 @@ namespace gkfs::metadata { +constexpr auto rocksdb_backend = "rocksdb"; +constexpr auto parallax_backend = "parallaxdb"; + class MetadataDB { private: @@ -54,7 +57,6 @@ private: std::shared_ptr log_; std::unique_ptr backend_; - public: MetadataDB(const std::string& path, const std::string_view database); diff --git a/src/daemon/backend/metadata/db.cpp b/src/daemon/backend/metadata/db.cpp index 6a440eb30..fcf629fa3 100644 --- a/src/daemon/backend/metadata/db.cpp +++ b/src/daemon/backend/metadata/db.cpp @@ -34,11 +34,12 @@ #include #include #include +#include extern "C" { #include } - +namespace fs = std::filesystem; namespace gkfs::metadata { @@ -51,23 +52,24 @@ struct MetadataDBFactory { static std::unique_ptr create(const std::string& path, const std::string_view id) { - - if(id == "parallaxdb") { + if(id == gkfs::metadata::parallax_backend) { #ifdef GKFS_ENABLE_PARALLAX - return std::make_unique(path); -#else - GKFS_METADATA_MOD->log()->error("PARALLAX not compiled"); - exit(EXIT_FAILURE); + auto metadata_path = fmt::format("{}/{}", path, + gkfs::metadata::parallax_backend); + GKFS_METADATA_MOD->log()->trace("Using Parallax file '{}'", + metadata_path); + return std::make_unique(metadata_path); #endif - } else if(id == "rocksdb") { + } else if(id == gkfs::metadata::rocksdb_backend) { #ifdef GKFS_ENABLE_ROCKSDB - return std::make_unique(path); -#else - GKFS_METADATA_MOD->log()->error("ROCKSDB not compiled"); - exit(EXIT_FAILURE); + auto metadata_path = + fmt::format("{}/{}", path, gkfs::metadata::rocksdb_backend); + fs::create_directories(metadata_path); + GKFS_METADATA_MOD->log()->trace("Using RocksDB directory '{}'", + metadata_path); + return std::make_unique(metadata_path); #endif } - GKFS_METADATA_MOD->log()->error("No valid metadata backend selected"); exit(EXIT_FAILURE); } @@ -83,13 +85,13 @@ struct MetadataDBFactory { MetadataDB::MetadataDB(const std::string& path, const std::string_view database) : path_(path) { - backend_ = MetadataDBFactory::create(path, database); - /* Get logger instance and set it for data module and chunk storage */ GKFS_METADATA_MOD->log(spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME)); assert(GKFS_METADATA_MOD->log()); log_ = spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME); assert(log_); + + backend_ = MetadataDBFactory::create(path, database); } MetadataDB::~MetadataDB() { diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp index 31ded67c5..0ba26673a 100644 --- a/src/daemon/backend/metadata/parallax_backend.cpp +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -58,7 +58,8 @@ ParallaxBackend::~ParallaxBackend() { * Called when the daemon is started: Connects to the KV store * @param path where KV store data is stored */ -ParallaxBackend::ParallaxBackend(const std::string& path) { +ParallaxBackend::ParallaxBackend(const std::string& path) + : par_path_(std::move(path)) { // We try to open options.yml if it exists, if not we create it by default int options = open("options.yml", O_RDWR | O_CREAT, 0644); @@ -71,9 +72,6 @@ ParallaxBackend::ParallaxBackend(const std::string& path) { } close(options); - - // Kreon - par_path_ = path + "x"; // file is rocksdb (add an x) int64_t size; int fd = open(par_path_.c_str(), O_RDWR | O_CREAT, 0644); @@ -99,7 +97,8 @@ ParallaxBackend::ParallaxBackend(const std::string& path) { write(fd, tmp.c_str(), 1); close(fd); - // We format the database + // We format the database TODO this doesn't work kv_format.parallax is + // not in path std::string cmd = "kv_format.parallax --device " + par_path_ + " --max_regions_num 1 "; system(cmd.c_str()); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index da5fc7478..adfe92473 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -243,7 +243,8 @@ void init_environment() { // Initialize metadata db auto metadata_path = fmt::format("{}/{}", GKFS_DATA->metadir(), - gkfs::config::rocksdb::data_dir); + gkfs::config::metadata::dir); + fs::create_directories(metadata_path); GKFS_DATA->spdlogger()->debug("{}() Initializing metadata DB: '{}'", __func__, metadata_path); try { @@ -413,13 +414,9 @@ destroy_enviroment() { // Delete rootdir/metadir if requested if(!GKFS_DATA->keep_rootdir()) { - GKFS_DATA->spdlogger()->info("{}() Removing RootDir/MetaDir/MetaFile", + GKFS_DATA->spdlogger()->info("{}() Removing rootdir and metadir ...", __func__); fs::remove_all(GKFS_DATA->metadir(), ecode); -#ifdef GKFS_ENABLE_PARALLAX - // some metadata backends uses a file instead of a dir. - fs::remove_all(GKFS_DATA->metadir() + "x", ecode); -#endif fs::remove_all(GKFS_DATA->rootdir(), ecode); } } @@ -551,7 +548,7 @@ parse_input(const cli_options& opts, const CLI::App& desc) { auto rootdir_path = fs::path(rootdir); if(desc.count("--rootdir-suffix")) { if(opts.rootdir_suffix == gkfs::config::data::chunk_dir || - opts.rootdir_suffix == gkfs::config::rocksdb::data_dir) + opts.rootdir_suffix == gkfs::config::metadata::dir) throw runtime_error(fmt::format( "rootdir_suffix '{}' is reserved and not allowed.", opts.rootdir_suffix)); @@ -609,10 +606,33 @@ parse_input(const cli_options& opts, const CLI::App& desc) { } if(desc.count("--dbbackend")) { - auto dbbackend = opts.dbbackend; - GKFS_DATA->dbbackend(dbbackend); + if(opts.dbbackend == gkfs::metadata::rocksdb_backend || + opts.dbbackend == gkfs::metadata::parallax_backend) { +#ifndef GKFS_ENABLE_PARALLAX + if(opts.dbbackend == gkfs::metadata::parallax_backend) { + throw runtime_error(fmt::format( + "dbbackend '{}' was not compiled and is disabled. " + "Pass -DGKFS_ENABLE_PARALLAX:BOOL=ON to CMake to enable.", + opts.dbbackend)); + } +#endif +#ifndef GKFS_ENABLE_ROCKSDB + if(opts.dbbackend == gkfs::metadata::rocksdb_backend) { + throw runtime_error(fmt::format( + "dbbackend '{}' was not compiled and is disabled. " + "Pass -DGKFS_ENABLE_ROCKSDB:BOOL=ON to CMake to enable.", + opts.dbbackend)); + } +#endif + GKFS_DATA->dbbackend(opts.dbbackend); + } else { + throw runtime_error( + fmt::format("dbbackend '{}' is not valid. Consult `--help`", + opts.dbbackend)); + } + } else - GKFS_DATA->dbbackend("rocksdb"); + GKFS_DATA->dbbackend(gkfs::metadata::rocksdb_backend); if(desc.count("--kreonsize")) { // Size in GB GKFS_DATA->kreon_size_md(stoi(opts.kreonsize)); @@ -678,11 +698,11 @@ main(int argc, const char* argv[]) { "Cleans Rootdir >after< the deamon finishes"); desc.add_option( "--dbbackend,-d", opts.dbbackend, - "Database Backend to use. If not set, rocksdb is used. For parallaxdb, a file called rocksdbx with 8GB will be created in metadir"); - - + "Metadata database backend to use. Available: {rocksdb, parallaxdb}'\n" + "RocksDB is default if not set. Parallax support is experimental.\n" + "Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir."); desc.add_option("--kreonsize",opts.kreonsize, - "parallaxdb - Metatada file size in GB (default 8), " + "parallaxdb - metadata file size in GB (default 8GB), " "used only with new files"); desc.add_flag("--version", "Print version and exit."); // clang-format on -- GitLab From 40d499ee41ab9ae2eafed8e46331465bcbf82e8a Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Wed, 16 Mar 2022 22:16:20 +0100 Subject: [PATCH 13/17] Making keep_rootdir non-global, clean also metadir if given, -c now cleans rootdir after finish --- include/daemon/classes/fs_data.hpp | 6 ------ src/daemon/classes/fs_data.cpp | 10 ---------- src/daemon/daemon.cpp | 19 +++++++++++++------ 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 879d155cd..9324db108 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -205,12 +205,6 @@ public: void blocks_state(bool blocks_state); - bool - keep_rootdir() const; - - void - keep_rootdir(bool keep_rootdir_); - unsigned long long kreon_size_md() const; diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index fd14a2c35..179e31a75 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -210,16 +210,6 @@ FsData::blocks_state(bool blocks_state) { FsData::blocks_state_ = blocks_state; } -bool -FsData::keep_rootdir() const { - return keep_rootdir_; -} - -void -FsData::keep_rootdir(bool keep_rootdir) { - FsData::keep_rootdir_ = keep_rootdir; -} - unsigned long long FsData::kreon_size_md() const { return size_md_; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index adfe92473..159af4cd8 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -69,6 +69,7 @@ namespace fs = std::filesystem; static condition_variable shutdown_please; // handler for shutdown signaling static mutex mtx; // mutex to wait on shutdown conditional variable +static bool keep_rootdir = true; struct cli_options { string mountdir; @@ -413,7 +414,7 @@ destroy_enviroment() { // Delete rootdir/metadir if requested - if(!GKFS_DATA->keep_rootdir()) { + if(!keep_rootdir) { GKFS_DATA->spdlogger()->info("{}() Removing rootdir and metadir ...", __func__); fs::remove_all(GKFS_DATA->metadir(), ecode); @@ -566,11 +567,11 @@ parse_input(const cli_options& opts, const CLI::App& desc) { GKFS_DATA->spdlogger()->debug("{}() Cleaning rootdir '{}' ...", __func__, rootdir_path.native()); fs::remove_all(rootdir_path.native()); - GKFS_DATA->spdlogger()->info("{}() Rootdir cleaned.", __func__); + GKFS_DATA->spdlogger()->info("{}() rootdir cleaned.", __func__); } if(desc.count("--clean-rootdir-finish")) { - GKFS_DATA->keep_rootdir(false); + keep_rootdir = false; } GKFS_DATA->spdlogger()->debug("{}() Root directory: '{}'", __func__, @@ -586,7 +587,13 @@ parse_input(const cli_options& opts, const CLI::App& desc) { #else auto metadir_path = fs::path(metadir); #endif - + if(desc.count("--clean-rootdir")) { + // may throw exception (caught in main) + GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...", + __func__, metadir_path.native()); + fs::remove_all(metadir_path.native()); + GKFS_DATA->spdlogger()->info("{}() metadir cleaned.", __func__); + } fs::create_directories(metadir_path); GKFS_DATA->metadir(fs::canonical(metadir_path).native()); @@ -691,10 +698,10 @@ main(int argc, const char* argv[]) { "Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, " "instead of using the RPC protocol. (Default off)"); desc.add_flag( - "--clean-rootdir,-c", + "--clean-rootdir", "Cleans Rootdir >before< launching the deamon"); desc.add_flag( - "--clean-rootdir-finish,-f", + "--clean-rootdir-finish,-c", "Cleans Rootdir >after< the deamon finishes"); desc.add_option( "--dbbackend,-d", opts.dbbackend, -- GitLab From 2a28c4447d7a153947d7d80992817da3dd8a71fa Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 17 Mar 2022 10:24:45 +0100 Subject: [PATCH 14/17] Changelog formatting, changed kreon appearances to parallax --- CHANGELOG.md | 42 ++++++++++++------- README.md | 2 +- docs/sphinx/users/running.md | 2 +- include/daemon/classes/fs_data.hpp | 9 ++-- .../backend/metadata/parallax_backend.cpp | 4 +- src/daemon/classes/fs_data.cpp | 10 ++--- src/daemon/daemon.cpp | 8 ++-- 7 files changed, 43 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0596326cf..05f997a69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,25 +1,39 @@ # Changelog + All notable changes to GekkoFS project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres +to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -- Parallax experimental integration -Support for different databases backend -New Docker Images and scripts (0.9.1) -([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)). -- Added `--clean-rootdir-finish` argument to remove rootdir/metadir at the end. + +### New + +- Added new experimental metadata backend: + Parallax ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)). + - Added support to use multiple metadata backends. + - Added `--clean-rootdir-finish` argument to remove rootdir/metadir at the end when the daemon finishes. + +### Changed + +- `-c` argument has been moved to `--clean-rootdir-finish` and is now used to clean rootdir/metadir on daemon + shutdown ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)). + +### Removed + +### Fixed ## [0.9.0] - 2022-02-22 + ### New + - GekkoFS now uses C++17 (!74). -- Added a new `dirents_extended` function which can improve `find` operations. -A corresponding example how to use this function can be found at -`examples/gfind/gfind.cpp` with a non-mpi version at `examples/gfind/sfind.cpp` -([!73](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/73)). -- Code coverage reports for the source code are now generated and tracked -([!77](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/77)). +- Added a new `dirents_extended` function which can improve `find` operations. A corresponding example how to use this + function can be found at + `examples/gfind/gfind.cpp` with a non-mpi version at `examples/gfind/sfind.cpp` + ([!73](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/73)). +- Code coverage reports for the source code are now generated and tracked + ([!77](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/77)). - Considerable overhaul and new features of the GekkoFS testing facilities ([!120](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/120), [!121](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/121)). @@ -82,8 +96,6 @@ Note that tests still require `Boost_preprocessor`. ([!116](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/116)). - Fixed an issue where `LOG_OUTPUT_TRUNC` did not work as expected ([!118](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/118)). -- Added new metadata backend, kreon. - ## [0.8.0] - 2020-09-15 ### New - Both client library and daemon have been extended to support the ofi+verbs diff --git a/README.md b/README.md index b349e1be7..cda198f38 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Options: --dbbackend 'rocksdb' (default) or 'parallaxdb' can be specified as metadata backend, in that case a file in 'metadir' named rocksdbx is created. Parallaxdb support is experimental. - --kreonsize 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB + --parallax_size 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB (default 8, 8 GB) --version Print version and exit. ``` diff --git a/docs/sphinx/users/running.md b/docs/sphinx/users/running.md index f023c5ba4..9ffb0caee 100644 --- a/docs/sphinx/users/running.md +++ b/docs/sphinx/users/running.md @@ -78,7 +78,7 @@ Further options are available --dbbackend 'rocksdb' (default) or 'parallaxdb' can be specified as metadata backend, in that case a file in 'metadir' named rocksdbx is created. Parallaxdb support is experimental. - --kreonsize 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB + --parallax_size 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB (default 8, 8 GB) --version Print version and exit. ```` diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 9324db108..06e71468c 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -71,10 +71,9 @@ private: // Database std::shared_ptr mdb_; std::string dbbackend_; - bool keep_rootdir_ = true; - // Kreon - unsigned long long size_md_ = 8589934592ull; + // Parallax + unsigned long long parallax_size_md_ = 8589934592ull; // Storage backend std::shared_ptr storage_; @@ -206,10 +205,10 @@ public: blocks_state(bool blocks_state); unsigned long long - kreon_size_md() const; + parallax_size_md() const; void - kreon_size_md(unsigned int size_md); + parallax_size_md(unsigned int size_md); }; } // namespace daemon diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp index 0ba26673a..ed3f27c10 100644 --- a/src/daemon/backend/metadata/parallax_backend.cpp +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -90,7 +90,7 @@ ParallaxBackend::ParallaxBackend(const std::string& path) } if(size == 0) { - size = GKFS_DATA->kreon_size_md(); + size = GKFS_DATA->parallax_size_md(); lseek(fd, size - 1, SEEK_SET); std::string tmp = "x"; @@ -334,8 +334,6 @@ ParallaxBackend::decrease_size_impl(const std::string& key, size_t size) { std::vector> ParallaxBackend::get_dirents_impl(const std::string& dir) const { auto root_path = dir; - // lock_guard lock_guard(kreon_mutex_); - struct par_key K; str2par(root_path, K); diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 179e31a75..1611e9e8a 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -211,14 +211,14 @@ FsData::blocks_state(bool blocks_state) { } unsigned long long -FsData::kreon_size_md() const { - return size_md_; +FsData::parallax_size_md() const { + return parallax_size_md_; } void -FsData::kreon_size_md(unsigned int size_md) { - FsData::size_md_ = - (unsigned long long) size_md * 1024ull * 1024ull * 1024ull; +FsData::parallax_size_md(unsigned int size_md) { + FsData::parallax_size_md_ = static_cast( + size_md * 1024ull * 1024ull * 1024ull); } } // namespace gkfs::daemon diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 159af4cd8..5931ba879 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -80,7 +80,7 @@ struct cli_options { string hosts_file; string rpc_protocol; string dbbackend; - string kreonsize; + string parallax_size; }; /** @@ -641,8 +641,8 @@ parse_input(const cli_options& opts, const CLI::App& desc) { } else GKFS_DATA->dbbackend(gkfs::metadata::rocksdb_backend); - if(desc.count("--kreonsize")) { // Size in GB - GKFS_DATA->kreon_size_md(stoi(opts.kreonsize)); + if(desc.count("--parallaxsize")) { // Size in GB + GKFS_DATA->parallax_size_md(stoi(opts.parallax_size)); } } @@ -708,7 +708,7 @@ main(int argc, const char* argv[]) { "Metadata database backend to use. Available: {rocksdb, parallaxdb}'\n" "RocksDB is default if not set. Parallax support is experimental.\n" "Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir."); - desc.add_option("--kreonsize",opts.kreonsize, + desc.add_option("--parallaxsize", opts.parallax_size, "parallaxdb - metadata file size in GB (default 8GB), " "used only with new files"); desc.add_flag("--version", "Print version and exit."); -- GitLab From 724f75404c429765586d0f1dabfc559132bd5595 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 17 Mar 2022 10:50:32 +0100 Subject: [PATCH 15/17] Updating daemon help and documentation --- README.md | 15 +++++------- docs/sphinx/users/running.md | 47 ++++++++++++++++++------------------ src/daemon/daemon.cpp | 4 +-- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index cda198f38..24857eba0 100644 --- a/README.md +++ b/README.md @@ -103,16 +103,13 @@ Options: Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets) Libfabric must have enabled support verbs or psm2. --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) - -c,--clean-rootdir Cleans Rootdir >before< launching the deamon - -f, --clean-rootdir-finish Cleans Rootdir >After< the deamon finishes + --clean-rootdir Cleans Rootdir >before< launching the deamon + -c,--clean-rootdir-finish Cleans Rootdir >after< the deamon finishes + -d,--dbbackend TEXT Metadata database backend to use. Available: {rocksdb, parallaxdb} + RocksDB is default if not set. Parallax support is experimental. + Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir. + --parallaxsize TEXT parallaxdb - metadata file size in GB (default 8GB), used only with new files --version Print version and exit. - - --dbbackend 'rocksdb' (default) or 'parallaxdb' can be specified as - metadata backend, in that case a file in 'metadir' named - rocksdbx is created. Parallaxdb support is experimental. - --parallax_size 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB - (default 8, 8 GB) - --version Print version and exit. ``` It is possible to run multiple independent GekkoFS instances on the same node. Note, that when these GekkoFS instances diff --git a/docs/sphinx/users/running.md b/docs/sphinx/users/running.md index 9ffb0caee..4750c151c 100644 --- a/docs/sphinx/users/running.md +++ b/docs/sphinx/users/running.md @@ -57,30 +57,29 @@ metadata locally, and at which path clients can access the file system (mount po Further options are available ```bash - Allowed options - Usage: src/daemon/gkfs_daemon [OPTIONS] - - Options: - -h,--help Print this help message and exit - -m,--mountdir TEXT REQUIRED Virtual mounting directory where GekkoFS is available. - -r,--rootdir TEXT REQUIRED Local data directory where GekkoFS data for this daemon is stored. - -s,--rootdir-suffix TEXT Creates an additional directory within the rootdir, allowing multiple daemons on one node. - -i,--metadir TEXT Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used. - -l,--listen TEXT Address or interface to bind the daemon to. Default: local hostname. - When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly which associates the verbs device with the network interface. In case FI_VERBS_IFACE is already defined, the argument is ignored. Default 'ib'. - -H,--hosts-file TEXT Shared file used by deamons to register their endpoints. (default './gkfs_hosts.txt') - -P,--rpc-protocol TEXT Used RPC protocol for inter-node communication. - Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets) - Libfabric must have enabled support verbs or psm2. - --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) - -c,--clean-rootdir Cleans Rootdir >before< launching the deamon - -f,--clean-rootdir-finish Cleans Rootdir >after< finishing the deamon - --dbbackend 'rocksdb' (default) or 'parallaxdb' can be specified as - metadata backend, in that case a file in 'metadir' named - rocksdbx is created. Parallaxdb support is experimental. - --parallax_size 'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB - (default 8, 8 GB) - --version Print version and exit. +Allowed options +Usage: src/daemon/gkfs_daemon [OPTIONS] + +Options: + -h,--help Print this help message and exit + -m,--mountdir TEXT REQUIRED Virtual mounting directory where GekkoFS is available. + -r,--rootdir TEXT REQUIRED Local data directory where GekkoFS data for this daemon is stored. + -s,--rootdir-suffix TEXT Creates an additional directory within the rootdir, allowing multiple daemons on one node. + -i,--metadir TEXT Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used. + -l,--listen TEXT Address or interface to bind the daemon to. Default: local hostname. + When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly which associates the verbs device with the network interface. In case FI_VERBS_IFACE is already defined, the argument is ignored. Default 'ib'. + -H,--hosts-file TEXT Shared file used by deamons to register their endpoints. (default './gkfs_hosts.txt') + -P,--rpc-protocol TEXT Used RPC protocol for inter-node communication. + Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets) + Libfabric must have enabled support verbs or psm2. + --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) + --clean-rootdir Cleans Rootdir >before< launching the deamon + -c,--clean-rootdir-finish Cleans Rootdir >after< the deamon finishes + -d,--dbbackend TEXT Metadata database backend to use. Available: {rocksdb, parallaxdb} + RocksDB is default if not set. Parallax support is experimental. + Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir. + --parallaxsize TEXT parallaxdb - metadata file size in GB (default 8GB), used only with new files + --version Print version and exit. ```` Shut it down by gracefully killing the process (SIGTERM). diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5931ba879..1ed24e9a2 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -677,7 +677,7 @@ main(int argc, const char* argv[]) { "Creates an additional directory within the rootdir, allowing multiple daemons on one node."); desc.add_option( "--metadir,-i", opts.metadir, - "Metadata directory where GekkoFS' RocksDB data directory is located. If not set, rootdir is used."); + "Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used."); desc.add_option( "--listen,-l", opts.listen, "Address or interface to bind the daemon to. Default: local hostname.\n" @@ -705,7 +705,7 @@ main(int argc, const char* argv[]) { "Cleans Rootdir >after< the deamon finishes"); desc.add_option( "--dbbackend,-d", opts.dbbackend, - "Metadata database backend to use. Available: {rocksdb, parallaxdb}'\n" + "Metadata database backend to use. Available: {rocksdb, parallaxdb}\n" "RocksDB is default if not set. Parallax support is experimental.\n" "Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir."); desc.add_option("--parallaxsize", opts.parallax_size, -- GitLab From 363a45f0bf569531d8f0d811cf9299155f58abc4 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 17 Mar 2022 11:03:43 +0100 Subject: [PATCH 16/17] Refactoring 0.9.1 profiles, adding new `all` profile --- scripts/profiles/0.9.1/agios.specs | 3 +- scripts/profiles/0.9.1/all.specs | 85 +++++++++++++++++++ scripts/profiles/0.9.1/default.specs | 3 +- scripts/profiles/0.9.1/direct.specs | 3 +- scripts/profiles/0.9.1/infiniband_verbs.specs | 3 +- scripts/profiles/0.9.1/omnipath_psm2.specs | 3 +- tests/scripts/compile_dep.sh/0.9.1/agios.out | 1 - tests/scripts/compile_dep.sh/0.9.1/all.out | 15 ++++ .../scripts/compile_dep.sh/0.9.1/default.out | 1 - tests/scripts/compile_dep.sh/0.9.1/direct.out | 1 - .../compile_dep.sh/0.9.1/infiniband_verbs.out | 1 - .../compile_dep.sh/0.9.1/omnipath_psm2.out | 1 - tests/scripts/dl_dep.sh/0.9.1/agios.out | 1 - tests/scripts/dl_dep.sh/0.9.1/all.out | 14 +++ tests/scripts/dl_dep.sh/0.9.1/default.out | 1 - tests/scripts/dl_dep.sh/0.9.1/direct.out | 1 - .../dl_dep.sh/0.9.1/infiniband_verbs.out | 1 - .../scripts/dl_dep.sh/0.9.1/omnipath_psm2.out | 1 - 18 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 scripts/profiles/0.9.1/all.specs create mode 100644 tests/scripts/compile_dep.sh/0.9.1/all.out create mode 100644 tests/scripts/dl_dep.sh/0.9.1/all.out diff --git a/scripts/profiles/0.9.1/agios.specs b/scripts/profiles/0.9.1/agios.specs index b4aded64f..e967029cd 100644 --- a/scripts/profiles/0.9.1/agios.specs +++ b/scripts/profiles/0.9.1/agios.specs @@ -52,7 +52,6 @@ clonedeps=( ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" - ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone @@ -67,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" "parallax" + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.1/all.specs b/scripts/profiles/0.9.1/all.specs new file mode 100644 index 000000000..1beb78451 --- /dev/null +++ b/scripts/profiles/0.9.1/all.specs @@ -0,0 +1,85 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "agios" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.1/default.specs b/scripts/profiles/0.9.1/default.specs index cad5a23da..c65bf556e 100644 --- a/scripts/profiles/0.9.1/default.specs +++ b/scripts/profiles/0.9.1/default.specs @@ -51,7 +51,6 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone @@ -66,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.1/direct.specs b/scripts/profiles/0.9.1/direct.specs index 18c0a5393..6f2a48123 100644 --- a/scripts/profiles/0.9.1/direct.specs +++ b/scripts/profiles/0.9.1/direct.specs @@ -47,7 +47,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone @@ -62,7 +61,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.1/infiniband_verbs.specs b/scripts/profiles/0.9.1/infiniband_verbs.specs index 9beeb344d..c34c4ff6d 100644 --- a/scripts/profiles/0.9.1/infiniband_verbs.specs +++ b/scripts/profiles/0.9.1/infiniband_verbs.specs @@ -51,7 +51,6 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone @@ -67,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading/installing order=( "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "parallax" + "syscall_intercept" "date" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.1/omnipath_psm2.specs b/scripts/profiles/0.9.1/omnipath_psm2.specs index 50dbef5cc..10163c302 100644 --- a/scripts/profiles/0.9.1/omnipath_psm2.specs +++ b/scripts/profiles/0.9.1/omnipath_psm2.specs @@ -52,7 +52,6 @@ clonedeps=( ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" - ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone @@ -67,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) # Extra arguments passed to the installation script. As such, they can diff --git a/tests/scripts/compile_dep.sh/0.9.1/agios.out b/tests/scripts/compile_dep.sh/0.9.1/agios.out index 179154be4..b098aecfa 100644 --- a/tests/scripts/compile_dep.sh/0.9.1/agios.out +++ b/tests/scripts/compile_dep.sh/0.9.1/agios.out @@ -9,5 +9,4 @@ ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### ######## Installing: agios ############################### -######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/all.out b/tests/scripts/compile_dep.sh/0.9.1/all.out new file mode 100644 index 000000000..5cdf4c5ff --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/all.out @@ -0,0 +1,15 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### +Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.1/default.out b/tests/scripts/compile_dep.sh/0.9.1/default.out index 351b957bd..787e4a08f 100644 --- a/tests/scripts/compile_dep.sh/0.9.1/default.out +++ b/tests/scripts/compile_dep.sh/0.9.1/default.out @@ -8,4 +8,3 @@ ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### -######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/direct.out b/tests/scripts/compile_dep.sh/0.9.1/direct.out index 868efc2ac..816266773 100644 --- a/tests/scripts/compile_dep.sh/0.9.1/direct.out +++ b/tests/scripts/compile_dep.sh/0.9.1/direct.out @@ -4,4 +4,3 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out index 3543af15c..e82dd020f 100644 --- a/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out +++ b/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out @@ -8,4 +8,3 @@ ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### -######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out index 38f67ce3d..7138da9e2 100644 --- a/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out +++ b/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out @@ -10,4 +10,3 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### -######## Installing: parallax ############################### diff --git a/tests/scripts/dl_dep.sh/0.9.1/agios.out b/tests/scripts/dl_dep.sh/0.9.1/agios.out index 3ed872e6b..07bf82bac 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/agios.out +++ b/tests/scripts/dl_dep.sh/0.9.1/agios.out @@ -9,5 +9,4 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/all.out b/tests/scripts/dl_dep.sh/0.9.1/all.out new file mode 100644 index 000000000..f6ddad018 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/all.out @@ -0,0 +1,14 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/default.out b/tests/scripts/dl_dep.sh/0.9.1/default.out index 4f301b9a1..7c047de89 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/default.out +++ b/tests/scripts/dl_dep.sh/0.9.1/default.out @@ -8,5 +8,4 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/direct.out b/tests/scripts/dl_dep.sh/0.9.1/direct.out index e98962560..42b333576 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/direct.out +++ b/tests/scripts/dl_dep.sh/0.9.1/direct.out @@ -4,5 +4,4 @@ Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobot Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out index e812c1f9e..da3c3e52c 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out +++ b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out @@ -8,5 +8,4 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out index 1d4ad26f5..b4f4e9c0e 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out @@ -9,5 +9,4 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file -- GitLab From bb5912fe8691f7b1b93bd7abb5ca45a3c76e65b9 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 17 Mar 2022 11:51:33 +0100 Subject: [PATCH 17/17] Fix Parallax code warnings --- .../backend/metadata/parallax_backend.cpp | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp index ed3f27c10..7f2226460 100644 --- a/src/daemon/backend/metadata/parallax_backend.cpp +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -68,25 +68,31 @@ ParallaxBackend::ParallaxBackend(const std::string& path) if(sizeOptions == 0) { std::string optcontent = "level0_size: 64\ngc_interval: 10\ngrowth_factor: 4\nmedium_log_LRU_cache_size: 400\nlevel_medium_inplace: 3\n"; - write(options, optcontent.c_str(), optcontent.length()); + auto write_size = + write(options, optcontent.c_str(), optcontent.length()); + if(write_size < 0 || + static_cast(write_size) < optcontent.length()) { + throw std::runtime_error(fmt::format( + "Failed to write Parallax options.yml: err '{}'", + write_size)); + } } close(options); int64_t size; int fd = open(par_path_.c_str(), O_RDWR | O_CREAT, 0644); - if(fd == -1) { - perror("open"); - exit(EXIT_FAILURE); + if(fd < 0) { + throw std::runtime_error( + fmt::format("Failed to open Parallax DB file. fd '{}'", fd)); } // Check size if we want to reuse it size = lseek(fd, 0, SEEK_END); if(size == -1) { - printf("[%s:%s:%d] failed to determine volume size exiting...\n", - __FILE__, __func__, __LINE__); - perror("ioctl"); - exit(EXIT_FAILURE); + throw std::runtime_error(fmt::format( + "[{}:{}:{}] failed to determine volume size exiting...", + __FILE__, __func__, __LINE__)); } if(size == 0) { @@ -94,14 +100,23 @@ ParallaxBackend::ParallaxBackend(const std::string& path) lseek(fd, size - 1, SEEK_SET); std::string tmp = "x"; - write(fd, tmp.c_str(), 1); + auto write_size = write(fd, tmp.c_str(), 1); + if(write_size < 1) { + throw std::runtime_error( + fmt::format("Failed to write to Parallax db file: err '{}'", + write_size)); + } close(fd); // We format the database TODO this doesn't work kv_format.parallax is // not in path std::string cmd = "kv_format.parallax --device " + par_path_ + " --max_regions_num 1 "; - system(cmd.c_str()); + auto err = system(cmd.c_str()); + if(err < 0) { + throw std::runtime_error(fmt::format( + "Failed to format parallax device: err '{}'", err)); + } } par_options_.create_flag = PAR_CREATE_DB; -- GitLab