diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00a5d74b471ab41f67c7c1baf7d558b969b24ab6..b5a077a342dac3a7572f798af64e19dbc1d7b116 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.0 +image: gekkofs/core:0.9.1 ################################################################################ ## Validating ################################################################################ check format: stage: lint - image: gekkofs/linter:0.9.0 + image: gekkofs/linter:0.9.1 needs: [] script: - ${SCRIPTS_DIR}/check_format.sh @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.0 + image: gekkofs/deps:0.9.1 interruptible: true needs: [] script: @@ -59,6 +59,8 @@ gkfs: -DGKFS_INSTALL_TESTS:BOOL=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} -DGKFS_USE_GUIDED_DISTRIBUTION:BOOL=ON + -DGKFS_ENABLE_PARALLAX:BOOL=ON + -DGKFS_ENABLE_ROCKSDB:BOOL=ON ${CI_PROJECT_DIR} - make -j$(nproc) install # reduce artifacts size @@ -71,7 +73,7 @@ gkfs: gkfwd: stage: build - image: gekkofs/deps:0.9.0 + image: gekkofs/deps:0.9.1 interruptible: true needs: [] script: @@ -85,6 +87,8 @@ gkfwd: -DGKFS_INSTALL_TESTS:BOOL=ON -DGKFS_ENABLE_FORWARDING:BOOL=ON -DGKFS_ENABLE_AGIOS:BOOL=ON + -DGKFS_ENABLE_PARALLAX:BOOL=OFF + -DGKFS_ENABLE_ROCKSDB:BOOL=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} ${CI_PROJECT_DIR} - make -j$(nproc) install @@ -104,15 +108,13 @@ gkfwd: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.1 needs: [] - parallel: - matrix: - - SUBTEST: [ dl_dep.sh, compile_dep.sh ] script: - mkdir -p ${BUILD_PATH}/tests/scripts - cd ${BUILD_PATH}/tests/scripts - - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml + - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit + - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit artifacts: expire_in: 1 week reports: @@ -122,7 +124,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.1 interruptible: true needs: ['gkfs'] parallel: @@ -131,9 +133,10 @@ gkfs:integration: script: ## run tests + - export PATH=${PATH}:/usr/local/bin - mkdir -p ${BUILD_PATH}/tests/run - cd ${BUILD_PATH}/tests/integration - - ${PYTEST} -v -n $(nproc) + - ${PYTEST} -v -n 1 ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST} --basetemp=${BUILD_PATH}/tests/run/${SUBTEST} --junit-xml=report.xml @@ -167,7 +170,7 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.1 interruptible: true needs: ['gkfwd'] parallel: @@ -212,9 +215,11 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.0 + image: gekkofs/testing:0.9.1 needs: ['gkfs'] script: + ## Add path to mkfs.kreon + - export PATH=${PATH}:/usr/local/bin ## run actual tests - cd ${BUILD_PATH}/tests/unit - ctest -j $(nproc) -L unit::all --output-junit report.xml @@ -240,7 +245,7 @@ gkfs:unit: ################################################################################ documentation: stage: docs - image: gekkofs/docs:0.9.0 + image: gekkofs/docs:0.9.1 needs: [] rules: # we only build the documentation automatically if we are on the @@ -276,7 +281,7 @@ documentation: ################################################################################ coverage: stage: report - image: gekkofs/coverage:0.9.0 + image: gekkofs/coverage:0.9.1 needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ] script: - cd ${BUILD_PATH} diff --git a/CHANGELOG.md b/CHANGELOG.md index ead8b557ee8af8969af4b3fc14388125c920be79..05f997a69f40ace3a5131f9fb19824c51cd2e63a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,39 @@ # Changelog + All notable changes to GekkoFS project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres +to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New + +- Added new experimental metadata backend: + Parallax ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)). + - Added support to use multiple metadata backends. + - Added `--clean-rootdir-finish` argument to remove rootdir/metadir at the end when the daemon finishes. + +### Changed + +- `-c` argument has been moved to `--clean-rootdir-finish` and is now used to clean rootdir/metadir on daemon + shutdown ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)). + +### Removed + +### Fixed ## [0.9.0] - 2022-02-22 + ### New + - GekkoFS now uses C++17 (!74). -- Added a new `dirents_extended` function which can improve `find` operations. -A corresponding example how to use this function can be found at -`examples/gfind/gfind.cpp` with a non-mpi version at `examples/gfind/sfind.cpp` -([!73](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/73)). -- Code coverage reports for the source code are now generated and tracked -([!77](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/77)). +- Added a new `dirents_extended` function which can improve `find` operations. A corresponding example how to use this + function can be found at + `examples/gfind/gfind.cpp` with a non-mpi version at `examples/gfind/sfind.cpp` + ([!73](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/73)). +- Code coverage reports for the source code are now generated and tracked + ([!77](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/77)). - Considerable overhaul and new features of the GekkoFS testing facilities ([!120](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/120), [!121](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/121)). diff --git a/CMake/FindParallax.cmake b/CMake/FindParallax.cmake new file mode 100644 index 0000000000000000000000000000000000000000..562ad6b9ce2f9c8e5f9a8469094cb32dc312c7a3 --- /dev/null +++ b/CMake/FindParallax.cmake @@ -0,0 +1,51 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +find_path(PARALLAX_INCLUDE_DIR + NAMES parallax.h + ) + +find_library(PARALLAX_LIBRARY + NAMES parallax + ) + +find_library(PARALLAX_LOG_LIBRARY + NAMES log + ) + +set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR}) +set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY}) +set(PARALLAX_LIBRARIES ${PARALLAX_LOG_LIBRARY}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY PARALLAX_LOG_LIBRARY PARALLAX_INCLUDE_DIR) + +mark_as_advanced( + PARALLAX_LIBRARY + PARALLAX_INCLUDE_DIR + PARALLAX_LOG_LIBRARY +) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f7b9ca6df7c3a7db783fd1ce183b0e1e0fb7840..7589df5191bf2878d9c6db29e98e3f3e5b85cdae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,17 +119,13 @@ include(gkfs-code-coverage) set(CMAKE_EXPORT_COMPILE_COMMANDS 0) -# Rocksdb dependencies -find_package(LZ4 REQUIRED) -find_package(JeMalloc) # required if rocksdb has been build with jemalloc -find_package(RocksDB REQUIRED) + # margo dependencies find_package(Mercury REQUIRED) find_package(Abt REQUIRED) find_package(Margo REQUIRED) find_package(Syscall_intercept REQUIRED) find_package(Threads REQUIRED) - # some compilers need extra flags for std::filesystem, such as -lstdc++fs, this # produces a std::filesystem imported target that takes care of all these # details transparently @@ -182,6 +178,8 @@ if (GKFS_ENABLE_AGIOS) find_package(AGIOS REQUIRED) endif () +option(GKFS_ENABLE_PARALLAX "Enable Parallax db backend" OFF) +option(GKFS_ENABLE_ROCKSDB "Enable ROCKSDB backend" ON) set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library") add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE}) @@ -199,26 +197,7 @@ endif() configure_file(include/common/cmake_configure.hpp.in include/common/cmake_configure.hpp) -# Imported target -add_library(RocksDB INTERFACE IMPORTED GLOBAL) -target_link_libraries(RocksDB - INTERFACE - ${ROCKSDB_LIBRARIES} - # rocksdb libs - ${LZ4_LIBRARIES} - ) -if (${JeMalloc_FOUND}) - target_link_libraries(RocksDB - INTERFACE - ${JEMALLOC_LIBRARIES} - ) -endif () -# we cannot use target_include_directories with CMake < 3.11 -set_target_properties(RocksDB - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} - ) add_library(CLI11 INTERFACE) # we cannot use target_include_directories with CMake < 3.11 diff --git a/README.md b/README.md index f2c861414fb06c7c361ed88dba2da108c1a93452..24857eba041ce9a1ac8c2a19f756208f2b5f1c10 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,12 @@ Options: Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets) Libfabric must have enabled support verbs or psm2. --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) - -c,--clean-rootdir Cleans Rootdir >before< launching the deamon + --clean-rootdir Cleans Rootdir >before< launching the deamon + -c,--clean-rootdir-finish Cleans Rootdir >after< the deamon finishes + -d,--dbbackend TEXT Metadata database backend to use. Available: {rocksdb, parallaxdb} + RocksDB is default if not set. Parallax support is experimental. + Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir. + --parallaxsize TEXT parallaxdb - metadata file size in GB (default 8GB), used only with new files --version Print version and exit. ``` @@ -225,6 +230,14 @@ Then, the `examples/distributors/guided/generate.py` scrpt is used to create the Finally, modify `guided_config.txt` to your distribution requirements. +### Metadata Backends +There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH` +is available. +To enable it use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=OFF`. + +Once it is enabled, `--dbbackend` option will be functional. + + ### Acknowledgment This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu). diff --git a/docker/0.9.1/core/Dockerfile b/docker/0.9.1/core/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bf1b2327ef9f7a79973049405c3f8696a0178fa8 --- /dev/null +++ b/docker/0.9.1/core/Dockerfile @@ -0,0 +1,39 @@ +FROM debian:bullseye-slim + +LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + procps \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ + # Margo dependencies \ + libjson-c-dev \ + # RocksDB dependencies + liblz4-dev \ + # syscall_intercept dependencies + libcapstone-dev \ + # GekkoFS dependencies + libboost-program-options-dev \ + uuid-dev && \ + # install cmake 3.14 since it's needed for some dependencies + curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \ + ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean && \ + rm ./cmake-3.14.5-Linux-x86_64.sh diff --git a/docker/0.9.1/core/Makefile b/docker/0.9.1/core/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0f32dcbebdc083ba0a053952a3a9f985aeaa67d4 --- /dev/null +++ b/docker/0.9.1/core/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/core:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/core:0.9.1 . + +all: + docker build -t gekkofs/core:0.9.1 . diff --git a/docker/0.9.1/coverage/Dockerfile b/docker/0.9.1/coverage/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6514807130f821e3dd79dc259a3b5366bb22a4b3 --- /dev/null +++ b/docker/0.9.1/coverage/Dockerfile @@ -0,0 +1,20 @@ +FROM debian:bullseye-slim + +LABEL Description="Environment to generate coverage reports in GekkoFS" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + git \ + cmake \ + gcc \ + g++ \ + lcov \ + python3 \ + python3-pip \ + python3-setuptools && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoclean && \ + python3 -m pip install --upgrade pip && \ + pip3 install gcovr diff --git a/docker/0.9.1/coverage/Makefile b/docker/0.9.1/coverage/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..67ba36bba92843dd970b95343920b3b1ccbe3933 --- /dev/null +++ b/docker/0.9.1/coverage/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/coverage:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/coverage:0.9.1 . + +all: + docker build -t gekkofs/coverage:0.9.1 . \ No newline at end of file diff --git a/docker/0.9.1/deps/Dockerfile b/docker/0.9.1/deps/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..fe81b563b4d2beca8272b4ea4803666617ea6bff --- /dev/null +++ b/docker/0.9.1/deps/Dockerfile @@ -0,0 +1,37 @@ +FROM gekkofs/core:0.9.1 + +LABEL Description="Debian-based environment to build GekkoFS" + +ENV GKFS_PATH /opt/gkfs +ENV GKFS_VERSION 0.9.1 + +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH /usr/local + +COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/patches ${SCRIPTS_PATH}/patches +COPY scripts/profiles ${SCRIPTS_PATH}/profiles + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python3-setuptools \ + libnuma-dev libyaml-dev \ + procps && \ + python3 -m pip install --upgrade pip && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && apt-get autoclean + +# Download and build dependencies +RUN cd ${SCRIPTS_PATH} && \ + /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \ + /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \ + rm -rf ${DEPS_SRC_PATH} && \ + rm -rf ${SCRIPTS_PATH} && \ + rmdir ${GKFS_PATH} && \ + ldconfig diff --git a/docker/0.9.1/deps/Makefile b/docker/0.9.1/deps/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..6cd0dc7dbdeb9d9afb59be3d2326459ab14028d1 --- /dev/null +++ b/docker/0.9.1/deps/Makefile @@ -0,0 +1,23 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts remove-scripts +amd64: build-amd64 mount-scripts remove-scripts +aarch64: build-aarch64 mount-scripts remove-scripts + +copy-scripts: + cp -R $(GIT_ROOT)/scripts $(CWD)/scripts + +build: copy-scripts + docker build -t gekkofs/deps:0.9.1 . + +build-amd64: copy-scripts + docker build --platform amd64 -t gekkofs/deps:0.9.1 . + +build-aarch64: copy-scripts + docker build --platform aarch64 -t gekkofs/deps:0.9.1 . + +remove-scripts: + - rm -rf $(CWD)/scripts diff --git a/docker/0.9.1/docs/Dockerfile b/docker/0.9.1/docs/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..21389dbafdb46a601c761fc3f2829023ba6dfd4d --- /dev/null +++ b/docker/0.9.1/docs/Dockerfile @@ -0,0 +1,33 @@ +FROM gekkofs/deps:0.9.1 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -OL https://www.doxygen.nl/files/doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.33.1' \ + 'exhale==0.3.1' \ + 'sphinx-copybutton==0.5.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.17.0' && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.1/docs/Makefile b/docker/0.9.1/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9770b2a5bc2cfdad5f8e15d0f6edc2cab67efa24 --- /dev/null +++ b/docker/0.9.1/docs/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/docs:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/docs:0.9.1 . + +all: + docker build -t gekkofs/docs:0.9.1 . \ No newline at end of file diff --git a/docker/0.9.1/linter/Dockerfile b/docker/0.9.1/linter/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..fa39907d335239922d58292fa701a6fc117ea54f --- /dev/null +++ b/docker/0.9.1/linter/Dockerfile @@ -0,0 +1,19 @@ +FROM gekkofs/core:0.9.1 + +LABEL Description="Debian-based environment to check the formatting of GekkoFS code" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # clang 10 deps for clang-format + lsb-release \ + wget \ + software-properties-common \ + gnupg2 && \ + # add clang-10 repos + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 11 && \ + # install clang-format + apt-get update && apt-get install -y --no-install-recommends clang-format-11 && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.1/linter/Makefile b/docker/0.9.1/linter/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..449c1cadcb13f7bef53a533dd1bd7e5b01b39baf --- /dev/null +++ b/docker/0.9.1/linter/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/linter:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/linter:0.9.1 . + +all: + docker build -t gekkofs/linter:0.9.1 . diff --git a/docker/0.9.1/testing/Dockerfile b/docker/0.9.1/testing/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..4c28d99ef8f0e1db68d6b724e330b44b08501352 --- /dev/null +++ b/docker/0.9.1/testing/Dockerfile @@ -0,0 +1,14 @@ +FROM gekkofs/deps:0.9.1 + +LABEL Description="Debian-based environment to test GekkoFS" + +RUN \ + # install cmake 3.21 since we need to produce JUnit XML files + curl -OL https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.sh && \ + chmod u+x ./cmake-3.21.1-linux-x86_64.sh && \ + ./cmake-3.21.1-linux-x86_64.sh --skip-license --prefix=/usr && \ + # install gcovr + # (required for partial coverage reports in parallel runs) + pip3 install gcovr && \ + # cleanup + rm ./cmake-3.21.1-linux-x86_64.sh diff --git a/docker/0.9.1/testing/Makefile b/docker/0.9.1/testing/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7f507aa4b8f43bc9989aedfc86e64bc60a8558e0 --- /dev/null +++ b/docker/0.9.1/testing/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/testing:0.9.1 . + +aarch64: + docker build --platform aarch64 -t gekkofs/testing:0.9.1 . + +all: + docker build -t gekkofs/testing:0.9.1 . \ No newline at end of file diff --git a/docker/latest b/docker/latest index 899f24fc754a14b89617093824d258aaa3aa2943..f514a2f0bd053cdaebb7ab7bcffca86fed32620a 120000 --- a/docker/latest +++ b/docker/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.1 \ No newline at end of file diff --git a/docs/sphinx/users/building.rst b/docs/sphinx/users/building.rst index bcd13438b87112fc300d8ef6bdfdc339fcf8af76..259e84d1a14845bb226484af8f27d87e6fd6f9c3 100644 --- a/docs/sphinx/users/building.rst +++ b/docs/sphinx/users/building.rst @@ -108,6 +108,11 @@ dependencies: - `AGIOS `_ (commit c26a654 or newer) to enable the :code:`GekkoFWD` I/O forwarding mode. +- `PARALLAX` There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH` +is available. To enable it, use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=OFF`. + Once it is enabled, `--dbbackend` option will be functional. + + .. _step_by_step_installation: Step-by-step installation diff --git a/docs/sphinx/users/running.md b/docs/sphinx/users/running.md index 286353a9835bf9926d11b235fb27272c1cb31806..4750c151cd839d473e6b08cdfee4d7348cfcd4dc 100644 --- a/docs/sphinx/users/running.md +++ b/docs/sphinx/users/running.md @@ -57,24 +57,29 @@ metadata locally, and at which path clients can access the file system (mount po Further options are available ```bash - Allowed options - Usage: src/daemon/gkfs_daemon [OPTIONS] - - Options: - -h,--help Print this help message and exit - -m,--mountdir TEXT REQUIRED Virtual mounting directory where GekkoFS is available. - -r,--rootdir TEXT REQUIRED Local data directory where GekkoFS data for this daemon is stored. - -s,--rootdir-suffix TEXT Creates an additional directory within the rootdir, allowing multiple daemons on one node. - -i,--metadir TEXT Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used. - -l,--listen TEXT Address or interface to bind the daemon to. Default: local hostname. - When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly which associates the verbs device with the network interface. In case FI_VERBS_IFACE is already defined, the argument is ignored. Default 'ib'. - -H,--hosts-file TEXT Shared file used by deamons to register their endpoints. (default './gkfs_hosts.txt') - -P,--rpc-protocol TEXT Used RPC protocol for inter-node communication. - Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets) - Libfabric must have enabled support verbs or psm2. - --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) - -c,--clean-rootdir Cleans Rootdir >before< launching the deamon - --version Print version and exit. +Allowed options +Usage: src/daemon/gkfs_daemon [OPTIONS] + +Options: + -h,--help Print this help message and exit + -m,--mountdir TEXT REQUIRED Virtual mounting directory where GekkoFS is available. + -r,--rootdir TEXT REQUIRED Local data directory where GekkoFS data for this daemon is stored. + -s,--rootdir-suffix TEXT Creates an additional directory within the rootdir, allowing multiple daemons on one node. + -i,--metadir TEXT Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used. + -l,--listen TEXT Address or interface to bind the daemon to. Default: local hostname. + When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly which associates the verbs device with the network interface. In case FI_VERBS_IFACE is already defined, the argument is ignored. Default 'ib'. + -H,--hosts-file TEXT Shared file used by deamons to register their endpoints. (default './gkfs_hosts.txt') + -P,--rpc-protocol TEXT Used RPC protocol for inter-node communication. + Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets) + Libfabric must have enabled support verbs or psm2. + --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off) + --clean-rootdir Cleans Rootdir >before< launching the deamon + -c,--clean-rootdir-finish Cleans Rootdir >after< the deamon finishes + -d,--dbbackend TEXT Metadata database backend to use. Available: {rocksdb, parallaxdb} + RocksDB is default if not set. Parallax support is experimental. + Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir. + --parallaxsize TEXT parallaxdb - metadata file size in GB (default 8GB), used only with new files + --version Print version and exit. ```` Shut it down by gracefully killing the process (SIGTERM). diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index 9ccb8833b2a52b77f4c6b190c0fa1dc728d3db53..d4bb154a7aa2e921b832eceeaa8934e04fc751d1 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -32,7 +32,6 @@ #include #include - #include #include #include diff --git a/include/config.hpp b/include/config.hpp index 10d672a25b179f615ac0e165b0a65e22d4222c47..e8fcd01fbcfa22b0263c8d51ce03b642215800ad 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -57,6 +57,9 @@ constexpr auto daemon_log_level = 4; // info } // namespace log namespace metadata { +// directory name where the metadata db instance is placed +constexpr auto dir = "metadata"; + // which metadata should be considered apart from size and mode constexpr auto use_atime = false; constexpr auto use_ctime = false; @@ -98,8 +101,6 @@ constexpr auto daemon_handler_xstreams = 4; namespace rocksdb { // Write-ahead logging of rocksdb constexpr auto use_write_ahead_log = false; -// directory name where the rocksdb instance is placed -constexpr auto data_dir = "rocksdb"; } // namespace rocksdb } // namespace gkfs::config diff --git a/include/daemon/backend/metadata/db.hpp b/include/daemon/backend/metadata/db.hpp index 80e06f7403830de1a8bb6df47ed640eeb672cdff..1114a9e3189b10bfba61087c22b997178a67928f 100644 --- a/include/daemon/backend/metadata/db.hpp +++ b/include/daemon/backend/metadata/db.hpp @@ -34,50 +34,33 @@ #include #include -#include #include #include +#include +#ifdef GKFS_ENABLE_ROCKSDB +#include +#endif +#ifdef GKFS_ENABLE_PARALLAX +#include +#endif -namespace rdb = rocksdb; namespace gkfs::metadata { -/** - * @brief MetadataDB class providing an abstraction layer to the KV store - * RocksDB. - */ +constexpr auto rocksdb_backend = "rocksdb"; +constexpr auto parallax_backend = "parallaxdb"; + + class MetadataDB { private: - std::unique_ptr db; ///< RocksDB instance - rdb::Options options; ///< RocksDB configuration - rdb::WriteOptions write_opts; ///< RocksDB write configuration - std::string path; ///< Path to where RocksDB persists its data - std::shared_ptr log_; ///< MetadataDB internal logger - - /** - * @brief Sets up specific settings to optimize RocksDB instance to - * environment on launch. - * @param options RocksDB configurations - */ - static void - optimize_rocksdb_options(rdb::Options& options); + std::string path_; + std::shared_ptr log_; + std::unique_ptr backend_; public: - /** - * @brief Exception wrapper on Status object. - * @param s RocksDB status - * @throws Throws NotFoundException if s.IsNotFound(), general DBException - * otherwise - */ - static inline void - throw_rdb_status_excpt(const rdb::Status& s); + MetadataDB(const std::string& path, const std::string_view database); - /** - * @brief Constructor, called when daemon is started and connects to KV - * store. - * @param path Path to where RocksDB persists its data - */ - explicit MetadataDB(const std::string& path); + ~MetadataDB(); /** * @brief Gets the KV store value for a key. @@ -182,7 +165,7 @@ public: * is therefore unused. */ void - iterate_all(); + iterate_all() const; }; } // namespace gkfs::metadata diff --git a/include/daemon/backend/metadata/metadata_backend.hpp b/include/daemon/backend/metadata/metadata_backend.hpp new file mode 100644 index 0000000000000000000000000000000000000000..a22392496c9a094b0edd474ffb6c5c833f96d10a --- /dev/null +++ b/include/daemon/backend/metadata/metadata_backend.hpp @@ -0,0 +1,145 @@ +/* + Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_METADATA_BACKEND_HPP +#define GEKKOFS_METADATA_BACKEND_HPP + +#include +#include +#include +#include + +namespace gkfs::metadata { + + +class AbstractMetadataBackend { +public: + virtual ~AbstractMetadataBackend() = default; + + virtual std::string + get(const std::string& key) const = 0; + + virtual void + put(const std::string& key, const std::string& val) = 0; + + virtual void + put_no_exist(const std::string& key, const std::string& val) = 0; + + virtual void + remove(const std::string& key) = 0; + + virtual bool + exists(const std::string& key) = 0; + + virtual void + update(const std::string& old_key, const std::string& new_key, + const std::string& val) = 0; + + virtual void + increase_size(const std::string& key, size_t size, bool append) = 0; + + virtual void + decrease_size(const std::string& key, size_t size) = 0; + + virtual std::vector> + get_dirents(const std::string& dir) const = 0; + + virtual std::vector> + get_dirents_extended(const std::string& dir) const = 0; + + virtual void + iterate_all() const = 0; +}; + +template +class MetadataBackend : public AbstractMetadataBackend { +private: + std::string path; + std::shared_ptr log_; + +public: + std::string + get(const std::string& key) const { + return static_cast(*this).get_impl(key); + } + + void + put(const std::string& key, const std::string& val) { + static_cast(*this).put_impl(key, val); + } + + void + put_no_exist(const std::string& key, const std::string& val) { + static_cast(*this).put_no_exist_impl(key, val); + } + + void + remove(const std::string& key) { + static_cast(*this).remove_impl(key); + } + + bool + exists(const std::string& key) { + return static_cast(*this).exists_impl(key); + } + + void + update(const std::string& old_key, const std::string& new_key, + const std::string& val) { + static_cast(*this).update_impl(old_key, new_key, val); + } + + void + increase_size(const std::string& key, size_t size, bool append) { + static_cast(*this).increase_size_impl(key, size, append); + } + + void + decrease_size(const std::string& key, size_t size) { + static_cast(*this).decrease_size_impl(key, size); + } + + std::vector> + get_dirents(const std::string& dir) const { + return static_cast(*this).get_dirents_impl(dir); + } + + std::vector> + get_dirents_extended(const std::string& dir) const { + return static_cast(*this).get_dirents_extended_impl(dir); + } + + void + iterate_all() const { + static_cast(*this).iterate_all_impl(); + } +}; + +} // namespace gkfs::metadata + +#endif // GEKKOFS_METADATA_BACKEND_HPP diff --git a/include/daemon/backend/metadata/parallax_backend.hpp b/include/daemon/backend/metadata/parallax_backend.hpp new file mode 100644 index 0000000000000000000000000000000000000000..deb55bfbd7b2df64ba820f7d0e06bc0bec33e9ae --- /dev/null +++ b/include/daemon/backend/metadata/parallax_backend.hpp @@ -0,0 +1,196 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_METADATA_PARALLAXBACKEND_HPP +#define GEKKOFS_METADATA_PARALLAXBACKEND_HPP + +#include +#include +#include +#include +extern "C" { +#include +} + +namespace rdb = rocksdb; +namespace gkfs::metadata { + +class ParallaxBackend : public MetadataBackend { +private: + par_handle par_db_; + par_db_options par_options_; + std::string par_path_; + + /** + * Convert a String to klc_key + * @param key + * @param klc_key struct + */ + inline void + str2par(const std::string& value, struct par_value& V) const; + + + /** + * Convert a String to klc_value + * @param value + * @param klc_value struct + */ + inline void + str2par(const std::string& key, struct par_key& K) const; + +public: + /** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ + explicit ParallaxBackend(const std::string& path); + + /** + * Exception wrapper on Status object. Throws NotFoundException if + * s == "Not Found", general DBException otherwise + * @param String with status + * @throws DBException + */ + static inline void + throw_status_excpt(const std::string& s); + + /** + * @brief Destroy the Kreon Backend:: Kreon Backend object + * We remove the file, too large for the CI. + * TODO: Insert option + */ + virtual ~ParallaxBackend(); + + /** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + std::string + get_impl(const std::string& key) const; + + /** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ + void + put_impl(const std::string& key, const std::string& val); + + /** + * Puts an entry into the KV store if it doesn't exist. This function does + * not use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ + void + put_no_exist_impl(const std::string& key, const std::string& val); + + /** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + remove_impl(const std::string& key); + + /** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ + bool + exists_impl(const std::string& key); + + /** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + update_impl(const std::string& old_key, const std::string& new_key, + const std::string& val); + + /** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ + void + increase_size_impl(const std::string& key, size_t size, bool append); + + /** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ + void + decrease_size_impl(const std::string& key, size_t size); + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_impl(const std::string& dir) const; + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_extended_impl(const std::string& dir) const; + + /** + * Code example for iterating all entries in KV store. This is for debug + * only as it is too expensive + */ + void + iterate_all_impl() const; +}; + +} // namespace gkfs::metadata + +#endif // GEKKOFS_METADATA_KREONBACKEND_HPP diff --git a/include/daemon/backend/metadata/rocksdb_backend.hpp b/include/daemon/backend/metadata/rocksdb_backend.hpp new file mode 100644 index 0000000000000000000000000000000000000000..d889c2263d862ab1022d749628fef26c7baa5ee0 --- /dev/null +++ b/include/daemon/backend/metadata/rocksdb_backend.hpp @@ -0,0 +1,180 @@ +/* + Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_METADATA_ROCKSDBBACKEND_HPP +#define GEKKOFS_METADATA_ROCKSDBBACKEND_HPP + +#include +#include +#include +#include +#include + +namespace rdb = rocksdb; + +namespace gkfs::metadata { + +/** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ +class RocksDBBackend : public MetadataBackend { +private: + std::unique_ptr db_; + rdb::Options options_; + rdb::WriteOptions write_opts_; + +public: + explicit RocksDBBackend(const std::string& path); + + virtual ~RocksDBBackend(); + + /** + * Exception wrapper on Status object. Throws NotFoundException if + * s.IsNotFound(), general DBException otherwise + * @param RocksDB status + * @throws DBException + */ + static inline void + throw_status_excpt(const rdb::Status& s); + + /** + * Used for setting KV store settings + */ + void + optimize_database_impl(); + + /** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + std::string + get_impl(const std::string& key) const; + + /** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ + void + put_impl(const std::string& key, const std::string& val); + + /** + * Puts an entry into the KV store if it doesn't exist. This function does + * not use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ + void + put_no_exist_impl(const std::string& key, const std::string& val); + + /** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + remove_impl(const std::string& key); + + /** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ + bool + exists_impl(const std::string& key); + + + /** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ + void + update_impl(const std::string& old_key, const std::string& new_key, + const std::string& val); + + /** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ + void + increase_size_impl(const std::string& key, size_t size, bool append); + + /** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ + void + decrease_size_impl(const std::string& key, size_t size); + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_impl(const std::string& dir) const; + + /** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ + std::vector> + get_dirents_extended_impl(const std::string& dir) const; + + /** + * Code example for iterating all entries in KV store. This is for debug + * only as it is too expensive + */ + void + iterate_all_impl() const; +}; + +} // namespace gkfs::metadata + +#endif // GEKKOFS_METADATA_ROCKSDBBACKEND_HPP diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index e38fea7951c5596b62627fa9c213557a4810ebfd..06e71468c693058661b79b65184d1eaaf2de969f 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -34,6 +34,7 @@ #include #include #include //std::hash +#include /* Forward declarations */ namespace gkfs { @@ -69,6 +70,11 @@ private: // Database std::shared_ptr mdb_; + std::string dbbackend_; + + // Parallax + unsigned long long parallax_size_md_ = 8589934592ull; + // Storage backend std::shared_ptr storage_; @@ -123,6 +129,12 @@ public: void metadir(const std::string& metadir_); + std::string_view + dbbackend() const; + + void + dbbackend(const std::string& dbbackend_); + const std::shared_ptr& mdb() const; @@ -191,6 +203,12 @@ public: void blocks_state(bool blocks_state); + + unsigned long long + parallax_size_md() const; + + void + parallax_size_md(unsigned int size_md); }; } // namespace daemon diff --git a/scripts/profiles/0.9.1/agios.specs b/scripts/profiles/0.9.1/agios.specs new file mode 100644 index 0000000000000000000000000000000000000000..e967029cd6b56035f8a9fbfeb4f19b401c29ab37 --- /dev/null +++ b/scripts/profiles/0.9.1/agios.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies (except transport-specific and experimental)" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.1/all.specs b/scripts/profiles/0.9.1/all.specs new file mode 100644 index 0000000000000000000000000000000000000000..1beb784514031897f5157b9d17766fdc366d05b9 --- /dev/null +++ b/scripts/profiles/0.9.1/all.specs @@ -0,0 +1,85 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "agios" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.1/ci.specs b/scripts/profiles/0.9.1/ci.specs new file mode 100644 index 0000000000000000000000000000000000000000..99c91f716ce5f77a26862f4bbdeac47350a61ac6 --- /dev/null +++ b/scripts/profiles/0.9.1/ci.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies required by the CI" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" + ) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "date" "agios" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.1/default.specs b/scripts/profiles/0.9.1/default.specs new file mode 100644 index 0000000000000000000000000000000000000000..c65bf556ee555e7c4980317908da48cd7527524b --- /dev/null +++ b/scripts/profiles/0.9.1/default.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.1/direct.specs b/scripts/profiles/0.9.1/direct.specs new file mode 100644 index 0000000000000000000000000000000000000000..6f2a48123e3ae1f3ec0d9e55ee9478c16dc5daf2 --- /dev/null +++ b/scripts/profiles/0.9.1/direct.specs @@ -0,0 +1,76 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Direct dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.1/infiniband_verbs.specs b/scripts/profiles/0.9.1/infiniband_verbs.specs new file mode 100644 index 0000000000000000000000000000000000000000..c34c4ff6d8c76566843fcb0bb4471dc0d0e3cb1a --- /dev/null +++ b/scripts/profiles/0.9.1/infiniband_verbs.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Infiniband supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%verbs"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading/installing +order=( + "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%verbs"]="--enable-verbs=yes" +) diff --git a/scripts/profiles/0.9.1/install/agios.install b/scripts/profiles/0.9.1/install/agios.install new file mode 100644 index 0000000000000000000000000000000000000000..3d0392ff5305710794c19dc1e8b96985722fa5b7 --- /dev/null +++ b/scripts/profiles/0.9.1/install/agios.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="agios" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/argobots.install b/scripts/profiles/0.9.1/install/argobots.install new file mode 100644 index 0000000000000000000000000000000000000000..96ccea920eeb4493c7e6287737cf091945e6b06a --- /dev/null +++ b/scripts/profiles/0.9.1/install/argobots.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.1/install/capstone.install b/scripts/profiles/0.9.1/install/capstone.install new file mode 100644 index 0000000000000000000000000000000000000000..ff299d82b47f76389b3650553014438c65e92ae7 --- /dev/null +++ b/scripts/profiles/0.9.1/install/capstone.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/date.install b/scripts/profiles/0.9.1/install/date.install new file mode 100644 index 0000000000000000000000000000000000000000..c4f12524cf738888185be97b2c2a7c3aed4f6a04 --- /dev/null +++ b/scripts/profiles/0.9.1/install/date.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="date" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_CXX_STANDARD:STRING=14 \ + -DUSE_SYSTEM_TZ_DB:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/json-c.install b/scripts/profiles/0.9.1/install/json-c.install new file mode 100644 index 0000000000000000000000000000000000000000..7280cda81e54d91500ef5137a45ad56cad298bc1 --- /dev/null +++ b/scripts/profiles/0.9.1/install/json-c.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/json-c" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install + # Margo doesn't search in both directories, so we make it available in both lib and lib64 + if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then + cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/ + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/libfabric%verbs.install b/scripts/profiles/0.9.1/install/libfabric%verbs.install new file mode 100644 index 0000000000000000000000000000000000000000..15257eb817549eefcab8264671c5df97307e49db --- /dev/null +++ b/scripts/profiles/0.9.1/install/libfabric%verbs.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.1/install/libfabric.install b/scripts/profiles/0.9.1/install/libfabric.install new file mode 100644 index 0000000000000000000000000000000000000000..7db89438e194931bb6311ed581e8d0d81576fa85 --- /dev/null +++ b/scripts/profiles/0.9.1/install/libfabric.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.1/install/lz4.install b/scripts/profiles/0.9.1/install/lz4.install new file mode 100644 index 0000000000000000000000000000000000000000..13e4e53b381f7df45aac175f3a9af84498b8abd4 --- /dev/null +++ b/scripts/profiles/0.9.1/install/lz4.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + # try to remove binaries first in case they already exist. Otherwise install fails. + LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4) + for LZ4_BIN in "${LZ4_BINS[@]}"; do + [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN" + done + make -j"${CORES}" + make DESTDIR="${INSTALL_DIR}" PREFIX="" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/margo.install b/scripts/profiles/0.9.1/install/margo.install new file mode 100644 index 0000000000000000000000000000000000000000..d2c6e4b5f2d21ff0e0f02865c53fe374d161b976 --- /dev/null +++ b/scripts/profiles/0.9.1/install/margo.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="margo" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.1/install/mercury.install b/scripts/profiles/0.9.1/install/mercury.install new file mode 100644 index 0000000000000000000000000000000000000000..601305b830c8cf1e320ca81cbdb45368b335cc3c --- /dev/null +++ b/scripts/profiles/0.9.1/install/mercury.install @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + # if the profile compiles bmi, enable it + if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then + USE_BMI="-DNA_USE_BMI:BOOL=ON" + else + USE_BMI="-DNA_USE_BMI:BOOL=OFF" + fi + + # if the profile provides any flavour of libfabric, enable it + if profile_has_dependency "^libfabric.*$"; then + USE_OFI="-DNA_USE_OFI:BOOL=ON" + else + USE_OFI="-DNA_USE_OFI:BOOL=OFF" + fi + + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + ${USE_BMI} ${USE_OFI} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/parallax.install b/scripts/profiles/0.9.1/install/parallax.install new file mode 100644 index 0000000000000000000000000000000000000000..0c552cc78118c8f153a02f2bd72f96452ca4b399 --- /dev/null +++ b/scripts/profiles/0.9.1/install/parallax.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + CURR="${SOURCE_DIR}/parallax" + # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DBUILD_SHARED_LIBS:BOOL=OFF \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/rocksdb.install b/scripts/profiles/0.9.1/install/rocksdb.install new file mode 100644 index 0000000000000000000000000000000000000000..9350e9feb8a34491d61f447be06efd33de315bfb --- /dev/null +++ b/scripts/profiles/0.9.1/install/rocksdb.install @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CXXFLAGS='' + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move' + fi + + # TODO use SSE? + CURR="${SOURCE_DIR}/rocksdb" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \ + -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \ + -DROCKSDB_BUILD_SHARED=OFF \ + -DWITH_LZ4=ON \ + -DWITH_GFLAGS=OFF \ + -DUSE_RTTI=1 \ + -DPORTABLE=1 \ + -DWITH_ALL_TESTS=OFF \ + -DWITH_BENCHMARK_TOOLS=OFF \ + -DWITH_TOOLS=OFF .. + make -j"${CORES}" install + + +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/install/syscall_intercept.install b/scripts/profiles/0.9.1/install/syscall_intercept.install new file mode 100644 index 0000000000000000000000000000000000000000..9dd287271c61838a29cedd5e3bf3146ffb598f08 --- /dev/null +++ b/scripts/profiles/0.9.1/install/syscall_intercept.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DBUILD_TESTS:BOOK=OFF .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.1/marenostrum4.specs b/scripts/profiles/0.9.1/marenostrum4.specs new file mode 100644 index 0000000000000000000000000000000000000000..2fd272eff2539038687724bb33bde698e9241277 --- /dev/null +++ b/scripts/profiles/0.9.1/marenostrum4.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Marenostrum 4 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/0.9.1/mogon2.specs b/scripts/profiles/0.9.1/mogon2.specs new file mode 100644 index 0000000000000000000000000000000000000000..71ebf7a65e326a93b03fce9f283d4e80d7fc0d36 --- /dev/null +++ b/scripts/profiles/0.9.1/mogon2.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.1/ngio.specs b/scripts/profiles/0.9.1/ngio.specs new file mode 100644 index 0000000000000000000000000000000000000000..72c99cebdb4f79b33b86c81f618f869a5ff674d2 --- /dev/null +++ b/scripts/profiles/0.9.1/ngio.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for NEXTGenIO prototype cluster" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.1/omnipath_psm2.specs b/scripts/profiles/0.9.1/omnipath_psm2.specs new file mode 100644 index 0000000000000000000000000000000000000000..10163c302156f7e50faac52a8337068c615461e8 --- /dev/null +++ b/scripts/profiles/0.9.1/omnipath_psm2.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Omnipath supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/latest b/scripts/profiles/latest index 899f24fc754a14b89617093824d258aaa3aa2943..f514a2f0bd053cdaebb7ab7bcffca86fed32620a 120000 --- a/scripts/profiles/latest +++ b/scripts/profiles/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.1 \ No newline at end of file diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index 0c87888374e83d48a0724862a01817030e72a8d1..2226c2c30a1bbb448538557cf35ba7d4429e5d31 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -50,6 +50,7 @@ sources=( ["date"]="https://github.com/HowardHinnant/date.git" ["agios"]="https://github.com/francielizanon/agios.git" ["json-c"]="https://github.com/json-c/json-c/archive/json-c-{{VERSION}}.tar.gz" + ["parallax"]="https://github.com/CARV-ICS-FORTH/parallax.git" ) diff --git a/src/daemon/backend/metadata/CMakeLists.txt b/src/daemon/backend/metadata/CMakeLists.txt index fc4961fc89ec20fb2f86984f2173688fffab4650..5d3c6876737b96b5dfcac3b7735b238628e20982 100644 --- a/src/daemon/backend/metadata/CMakeLists.txt +++ b/src/daemon/backend/metadata/CMakeLists.txt @@ -26,56 +26,132 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -add_library(metadata_db STATIC) -# If it exists, RocksDB has been build with liburing support. -# If check succeeds, we need to link with liburing with the daemon -set(CMAKE_REQUIRED_FLAGS "-luring") -check_cxx_source_compiles(" - #include - int main() { - struct io_uring ring; - io_uring_queue_init(1, &ring, 0); - return 0; - } - " HAS_LIBURING) -set(liburing "") -if (HAS_LIBURING) - set(LIBURING "-luring") -endif() -target_sources(metadata_db +# Define metadata module library first +add_library(metadata_module STATIC) +target_sources(metadata_module + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/metadata_module.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/metadata_module.cpp + ) +target_link_libraries(metadata_module + PUBLIC + spdlog + ) + +# Define metadata_db and its common dependencies and sources +add_library(metadata_db STATIC) +set(DB_COMMON_SOURCES PUBLIC ${INCLUDE_DIR}/daemon/backend/metadata/db.hpp ${INCLUDE_DIR}/daemon/backend/exceptions.hpp + ${INCLUDE_DIR}/daemon/backend/metadata/metadata_backend.hpp PRIVATE ${INCLUDE_DIR}/daemon/backend/metadata/merge.hpp ${CMAKE_CURRENT_LIST_DIR}/merge.cpp ${CMAKE_CURRENT_LIST_DIR}/db.cpp ) - -add_library(metadata_module - STATIC -) - -target_sources(metadata_module - PUBLIC - ${INCLUDE_DIR}/daemon/backend/metadata/metadata_module.hpp - PRIVATE - ${CMAKE_CURRENT_LIST_DIR}/metadata_module.cpp -) - -target_link_libraries(metadata_module - PUBLIC - spdlog -) - -target_link_libraries(metadata_db +# lib links for all DBs. Extended below for specific DB +set(DB_LINK_LIBRARIES + PRIVATE # Required by RocksDB metadata_module -ldl metadata - RocksDB spdlog path_util - ${LIBURING} ) + +# When RocksDB is used +if (GKFS_ENABLE_ROCKSDB) + # If it exists, RocksDB has been build with liburing support. + # If check succeeds, we need to link with liburing with the daemon + set(CMAKE_REQUIRED_FLAGS "-luring") + check_cxx_source_compiles(" + #include + int main() { + struct io_uring ring; + io_uring_queue_init(1, &ring, 0); + return 0; + } + " HAS_LIBURING) + if (HAS_LIBURING) + set(DB_LINK_LIBRARIES + ${DB_LINK_LIBRARIES} + -luring + ) + endif () + # Rocksdb dependencies + find_package(LZ4 REQUIRED) + find_package(JeMalloc) # required if rocksdb has been build with jemalloc + find_package(RocksDB REQUIRED) + # Imported target + add_library(RocksDB INTERFACE IMPORTED GLOBAL) + target_link_libraries(RocksDB + INTERFACE + ${ROCKSDB_LIBRARIES} + # rocksdb libs + ${LZ4_LIBRARIES} + ) + if (${JeMalloc_FOUND}) + target_link_libraries(RocksDB + INTERFACE + ${JEMALLOC_LIBRARIES} + ) + endif () + # we cannot use target_include_directories with CMake < 3.11 + set_target_properties(RocksDB + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} + ) + set(DB_ROCKSDB_SOURCES + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/rocksdb_backend.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/rocksdb_backend.cpp + ) + target_compile_definitions(metadata_db + PUBLIC + GKFS_ENABLE_ROCKSDB + ) + set(DB_LINK_LIBRARIES + ${DB_LINK_LIBRARIES} + RocksDB + ) +endif () + +# When Parallax is used +if (GKFS_ENABLE_PARALLAX) + find_package(yaml REQUIRED) + find_package(Parallax REQUIRED) + set(DB_PARALLAX_SOURCES + PUBLIC + ${INCLUDE_DIR}/daemon/backend/metadata/parallax_backend.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/parallax_backend.cpp + ) + target_compile_definitions(metadata_db + PUBLIC + GKFS_ENABLE_PARALLAX + ) + add_library(Parallax INTERFACE IMPORTED GLOBAL) + target_link_libraries(Parallax + INTERFACE + ${PARALLAX_LIBRARY} + ${PARALLAX_LOG_LIBRARY} + yaml + ) + set_target_properties(Parallax + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${PARALLAX_INCLUDE_DIRS} + ) + set(DB_LINK_LIBRARIES + ${DB_LINK_LIBRARIES} + Parallax + ) +endif () + +target_sources(metadata_db + ${DB_COMMON_SOURCES} ${DB_PARALLAX_SOURCES} ${DB_ROCKSDB_SOURCES}) +target_link_libraries(metadata_db ${DB_LINK_LIBRARIES}) \ No newline at end of file diff --git a/src/daemon/backend/metadata/db.cpp b/src/daemon/backend/metadata/db.cpp index 440c97ad78269646dc00475bf30faf8b7ff8deb1..fcf629fa37bf80f732b0c87b98dbc006edd5737f 100644 --- a/src/daemon/backend/metadata/db.cpp +++ b/src/daemon/backend/metadata/db.cpp @@ -33,14 +33,47 @@ #include #include +#include +#include extern "C" { #include } +namespace fs = std::filesystem; namespace gkfs::metadata { -// private functions +/** + * Factory to create DB instances + * @param path where KV store data is stored + * @param id parallax or rocksdb (default) backend + */ +struct MetadataDBFactory { + static std::unique_ptr + create(const std::string& path, const std::string_view id) { + + if(id == gkfs::metadata::parallax_backend) { +#ifdef GKFS_ENABLE_PARALLAX + auto metadata_path = fmt::format("{}/{}", path, + gkfs::metadata::parallax_backend); + GKFS_METADATA_MOD->log()->trace("Using Parallax file '{}'", + metadata_path); + return std::make_unique(metadata_path); +#endif + } else if(id == gkfs::metadata::rocksdb_backend) { +#ifdef GKFS_ENABLE_ROCKSDB + auto metadata_path = + fmt::format("{}/{}", path, gkfs::metadata::rocksdb_backend); + fs::create_directories(metadata_path); + GKFS_METADATA_MOD->log()->trace("Using RocksDB directory '{}'", + metadata_path); + return std::make_unique(metadata_path); +#endif + } + GKFS_METADATA_MOD->log()->error("No valid metadata backend selected"); + exit(EXIT_FAILURE); + } +}; /** * @internal @@ -49,25 +82,8 @@ namespace gkfs::metadata { * see here: https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide * @endinternal */ -void -MetadataDB::optimize_rocksdb_options(rdb::Options& options) { - options.max_successive_merges = 128; -} - -// public functions - -void -MetadataDB::throw_rdb_status_excpt(const rdb::Status& s) { - assert(!s.ok()); - - if(s.IsNotFound()) { - throw NotFoundException(s.ToString()); - } else { - throw DBException(s.ToString()); - } -} - -MetadataDB::MetadataDB(const std::string& path) : path(path) { +MetadataDB::MetadataDB(const std::string& path, const std::string_view database) + : path_(path) { /* Get logger instance and set it for data module and chunk storage */ GKFS_METADATA_MOD->log(spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME)); @@ -75,30 +91,22 @@ MetadataDB::MetadataDB(const std::string& path) : path(path) { log_ = spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME); assert(log_); - // Optimize RocksDB. This is the easiest way to get RocksDB to perform well - options.IncreaseParallelism(); - options.OptimizeLevelStyleCompaction(); - // create the DB if it's not already present - options.create_if_missing = true; - options.merge_operator.reset(new MetadataMergeOperator); - MetadataDB::optimize_rocksdb_options(options); - write_opts.disableWAL = !(gkfs::config::rocksdb::use_write_ahead_log); - rdb::DB* rdb_ptr = nullptr; - auto s = rocksdb::DB::Open(options, path, &rdb_ptr); - if(!s.ok()) { - throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); - } - this->db.reset(rdb_ptr); + backend_ = MetadataDBFactory::create(path, database); } +MetadataDB::~MetadataDB() { + backend_.reset(); +} + +/** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ std::string MetadataDB::get(const std::string& key) const { - std::string val; - auto s = db->Get(rdb::ReadOptions(), key, &val); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } - return val; + return backend_->get(key); } void @@ -106,11 +114,7 @@ MetadataDB::put(const std::string& key, const std::string& val) { assert(gkfs::path::is_absolute(key)); assert(key == "/" || !gkfs::path::has_trailing_slash(key)); - auto cop = CreateOperand(val); - auto s = db->Merge(write_opts, key, cop.serialize()); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + backend_->put(key, val); } /** @@ -120,44 +124,27 @@ MetadataDB::put(const std::string& key, const std::string& val) { */ void MetadataDB::put_no_exist(const std::string& key, const std::string& val) { - if(exists(key)) - throw ExistsException(key); - put(key, val); + + backend_->put_no_exist(key, val); } void MetadataDB::remove(const std::string& key) { - auto s = db->Delete(write_opts, key); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->remove(key); } bool MetadataDB::exists(const std::string& key) { - std::string val; - auto s = db->Get(rdb::ReadOptions(), key, &val); - if(!s.ok()) { - if(s.IsNotFound()) { - return false; - } else { - MetadataDB::throw_rdb_status_excpt(s); - } - } - return true; + + return backend_->exists(key); } void MetadataDB::update(const std::string& old_key, const std::string& new_key, const std::string& val) { - // TODO use rdb::Put() method - rdb::WriteBatch batch; - batch.Delete(old_key); - batch.Put(new_key, val); - auto s = db->Write(write_opts, &batch); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->update(old_key, new_key, val); } /** @@ -167,11 +154,8 @@ MetadataDB::update(const std::string& old_key, const std::string& new_key, */ void MetadataDB::increase_size(const std::string& key, size_t size, bool append) { - auto uop = IncreaseSizeOperand(size, append); - auto s = db->Merge(write_opts, key, uop.serialize()); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->increase_size(key, size, append); } /** @@ -181,11 +165,8 @@ MetadataDB::increase_size(const std::string& key, size_t size, bool append) { */ void MetadataDB::decrease_size(const std::string& key, size_t size) { - auto uop = DecreaseSizeOperand(size); - auto s = db->Merge(write_opts, key, uop.serialize()); - if(!s.ok()) { - MetadataDB::throw_rdb_status_excpt(s); - } + + backend_->decrease_size(key, size); } std::vector> @@ -198,38 +179,7 @@ MetadataDB::get_dirents(const std::string& dir) const { root_path.push_back('/'); } - rocksdb::ReadOptions ropts; - auto it = db->NewIterator(ropts); - - std::vector> entries; - - for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); - it->Next()) { - - if(it->key().size() == root_path.size()) { - // we skip this path cause it is exactly the root_path - continue; - } - - /***** Get File name *****/ - auto name = it->key().ToString(); - if(name.find_first_of('/', root_path.size()) != std::string::npos) { - // skip stuff deeper then one level depth - continue; - } - // remove prefix - name = name.substr(root_path.size()); - - // relative path of directory entries must not be empty - assert(!name.empty()); - - Metadata md(it->value().ToString()); - auto is_dir = S_ISDIR(md.mode()); - - entries.emplace_back(std::move(name), is_dir); - } - assert(it->status().ok()); - return entries; + return backend_->get_dirents(root_path); } std::vector> @@ -242,39 +192,7 @@ MetadataDB::get_dirents_extended(const std::string& dir) const { root_path.push_back('/'); } - rocksdb::ReadOptions ropts; - auto it = db->NewIterator(ropts); - - std::vector> entries; - - for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); - it->Next()) { - - if(it->key().size() == root_path.size()) { - // we skip this path cause it is exactly the root_path - continue; - } - - /***** Get File name *****/ - auto name = it->key().ToString(); - if(name.find_first_of('/', root_path.size()) != std::string::npos) { - // skip stuff deeper then one level depth - continue; - } - // remove prefix - name = name.substr(root_path.size()); - - // relative path of directory entries must not be empty - assert(!name.empty()); - - Metadata md(it->value().ToString()); - auto is_dir = S_ISDIR(md.mode()); - - entries.emplace_back(std::forward_as_tuple(std::move(name), is_dir, - md.size(), md.ctime())); - } - assert(it->status().ok()); - return entries; + return backend_->get_dirents_extended(root_path); } @@ -285,15 +203,8 @@ MetadataDB::get_dirents_extended(const std::string& dir) const { * @endinternal */ void -MetadataDB::iterate_all() { - std::string key; - std::string val; - // Do RangeScan on parent inode - auto iter = db->NewIterator(rdb::ReadOptions()); - for(iter->SeekToFirst(); iter->Valid(); iter->Next()) { - key = iter->key().ToString(); - val = iter->value().ToString(); - } +MetadataDB::iterate_all() const { + backend_->iterate_all(); } } // namespace gkfs::metadata diff --git a/src/daemon/backend/metadata/parallax_backend.cpp b/src/daemon/backend/metadata/parallax_backend.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7f2226460de2e84ea58ee3f6886b0d2141c3fe8c --- /dev/null +++ b/src/daemon/backend/metadata/parallax_backend.cpp @@ -0,0 +1,484 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +using namespace std; +extern "C" { +#include +} + +std::recursive_mutex parallax_mutex_; + +namespace gkfs::metadata { +/** + * @brief Destroy the Kreon Backend:: Kreon Backend object + * We remove the file, too large for the CI. + * TODO: Insert option + */ +ParallaxBackend::~ParallaxBackend() { + par_close(par_db_); +} + +/** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ +ParallaxBackend::ParallaxBackend(const std::string& path) + : par_path_(std::move(path)) { + + // We try to open options.yml if it exists, if not we create it by default + int options = open("options.yml", O_RDWR | O_CREAT, 0644); + int64_t sizeOptions; + sizeOptions = lseek(options, 0, SEEK_END); + if(sizeOptions == 0) { + std::string optcontent = + "level0_size: 64\ngc_interval: 10\ngrowth_factor: 4\nmedium_log_LRU_cache_size: 400\nlevel_medium_inplace: 3\n"; + auto write_size = + write(options, optcontent.c_str(), optcontent.length()); + if(write_size < 0 || + static_cast(write_size) < optcontent.length()) { + throw std::runtime_error(fmt::format( + "Failed to write Parallax options.yml: err '{}'", + write_size)); + } + } + + close(options); + int64_t size; + + int fd = open(par_path_.c_str(), O_RDWR | O_CREAT, 0644); + if(fd < 0) { + throw std::runtime_error( + fmt::format("Failed to open Parallax DB file. fd '{}'", fd)); + } + + // Check size if we want to reuse it + size = lseek(fd, 0, SEEK_END); + if(size == -1) { + throw std::runtime_error(fmt::format( + "[{}:{}:{}] failed to determine volume size exiting...", + __FILE__, __func__, __LINE__)); + } + + if(size == 0) { + size = GKFS_DATA->parallax_size_md(); + + lseek(fd, size - 1, SEEK_SET); + std::string tmp = "x"; + auto write_size = write(fd, tmp.c_str(), 1); + if(write_size < 1) { + throw std::runtime_error( + fmt::format("Failed to write to Parallax db file: err '{}'", + write_size)); + } + close(fd); + + // We format the database TODO this doesn't work kv_format.parallax is + // not in path + std::string cmd = "kv_format.parallax --device " + par_path_ + + " --max_regions_num 1 "; + auto err = system(cmd.c_str()); + if(err < 0) { + throw std::runtime_error(fmt::format( + "Failed to format parallax device: err '{}'", err)); + } + } + + par_options_.create_flag = PAR_CREATE_DB; + par_options_.db_name = "test"; + par_options_.volume_name = (char*) malloc(par_path_.size() + 1); + strcpy(par_options_.volume_name, par_path_.c_str()); + par_options_.volume_start = 0; + par_options_.volume_size = 0; + par_db_ = par_open(&par_options_); +} + + +/** + * Exception wrapper on Status object. Throws NotFoundException if + * s == "Not Found", general DBException otherwise + * @param String with status + * @throws DBException + */ +void +ParallaxBackend::throw_status_excpt(const std::string& s) { + if(s == "Not Found") { + throw NotFoundException(s); + } else { + throw DBException(s); + } +} + +/** + * Convert a String to klc_key + * @param key + * @param par_key struct + */ +inline void +ParallaxBackend::str2par(const std::string& key, struct par_key& K) const { + K.size = key.size(); + K.data = key.c_str(); +} + +/** + * Convert a String to klc_value + * @param value + * @param par_value struct + */ +inline void +ParallaxBackend::str2par(const std::string& value, struct par_value& V) const { + V.val_size = value.size() + 1; + V.val_buffer = (char*) value.c_str(); +} + +/** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +std::string +ParallaxBackend::get_impl(const std::string& key) const { + std::string val; + + struct par_key K; + struct par_value V; + V.val_buffer = NULL; + str2par(key, K); + + par_ret_code ret = par_get(par_db_, &K, &V); + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } else { + val = V.val_buffer; + free(V.val_buffer); + } + return val; +} + +/** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ +void +ParallaxBackend::put_impl(const std::string& key, const std::string& val) { + + struct par_key_value key_value; + + str2par(key, key_value.k); + str2par(val, key_value.v); + par_ret_code ret = par_put(par_db_, &key_value); + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } +} + +/** + * Puts an entry into the KV store if it doesn't exist. This function does not + * use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ +void +ParallaxBackend::put_no_exist_impl(const std::string& key, + const std::string& val) { + + struct par_key_value key_value; + str2par(key, key_value.k); + str2par(val, key_value.v); + + par_ret_code ret = par_exists(par_db_, &key_value.k); + if(ret == PAR_KEY_NOT_FOUND) { + par_put(par_db_, &key_value); + } else + throw ExistsException(key); +} + +/** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +ParallaxBackend::remove_impl(const std::string& key) { + + struct par_key k; + + str2par(key, k); + par_ret_code ret = par_delete(par_db_, &k); + + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } +} + +/** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ +bool +ParallaxBackend::exists_impl(const std::string& key) { + + struct par_key k; + + str2par(key, k); + + par_ret_code ret = par_exists(par_db_, &k); + if(ret == PAR_KEY_NOT_FOUND) { + return true; + } + + return false; // TODO it is not the only case, we can have errors +} + +/** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +ParallaxBackend::update_impl(const std::string& old_key, + const std::string& new_key, + const std::string& val) { + + // TODO: Check Parallax transactions/Batches + + struct par_key_value n_key_value; + struct par_key o_key; + + str2par(new_key, n_key_value.k); + str2par(val, n_key_value.v); + + str2par(old_key, o_key); + par_delete(par_db_, &o_key); + par_ret_code ret = par_put(par_db_, &n_key_value); + if(ret != PAR_SUCCESS) { + throw_status_excpt("Not Found"); + } +} + +/** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ +void +ParallaxBackend::increase_size_impl(const std::string& key, size_t size, + bool append) { + lock_guard lock_guard(parallax_mutex_); + + auto value = get(key); + // Decompress string + Metadata md(value); + if(append) + size += md.size(); + md.size(size); + update(key, key, md.serialize()); +} + +/** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ +void +ParallaxBackend::decrease_size_impl(const std::string& key, size_t size) { + lock_guard lock_guard(parallax_mutex_); + + auto value = get(key); + // Decompress string + Metadata md(value); + md.size(size); + update(key, key, md.serialize()); +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +ParallaxBackend::get_dirents_impl(const std::string& dir) const { + auto root_path = dir; + struct par_key K; + + str2par(root_path, K); + + par_scanner S = par_init_scanner(par_db_, &K, PAR_GREATER_OR_EQUAL); + + std::vector> entries; + + while(par_is_valid(S)) { + struct par_key K2 = par_get_key(S); + struct par_value value = par_get_value(S); + + std::string k(K2.data, K2.size); + std::string v(value.val_buffer, value.val_size); + if(k.size() < root_path.size() || + k.substr(0, root_path.size()) != root_path) { + break; + } + + if(k.size() == root_path.size()) { + par_get_next(S); + continue; + } + + /***** Get File name *****/ + auto name = k; + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + par_get_next(S); + continue; + } + + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(v); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::move(name), is_dir); + + par_get_next(S); + } + // If we don't close the scanner we cannot delete keys + par_close_scanner(S); + + return entries; +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +ParallaxBackend::get_dirents_extended_impl(const std::string& dir) const { + auto root_path = dir; + // assert(gkfs::path::is_absolute(root_path)); + // add trailing slash if missing + if(!gkfs::path::has_trailing_slash(root_path) && root_path.size() != 1) { + // add trailing slash only if missing and is not the root_folder "/" + root_path.push_back('/'); + } + + struct par_key K; + + str2par(root_path, K); + + par_scanner S = par_init_scanner(par_db_, &K, PAR_GREATER_OR_EQUAL); + + std::vector> entries; + + while(par_is_valid(S)) { + struct par_key K2 = par_get_key(S); + struct par_value value = par_get_value(S); + + std::string k(K2.data, K2.size); + std::string v(value.val_buffer, value.val_size); + + if(k.size() < root_path.size() || + k.substr(0, root_path.size()) != root_path) { + break; + } + + if(k.size() == root_path.size()) { + if(par_get_next(S) && !par_is_valid(S)) + break; + continue; + } + + /***** Get File name *****/ + auto name = k; + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + if(par_get_next(S) && !par_is_valid(S)) + break; + continue; + } + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(v); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::forward_as_tuple(std::move(name), is_dir, + md.size(), md.ctime())); + + if(par_get_next(S) && !par_is_valid(S)) + break; + } + // If we don't close the scanner we cannot delete keys + par_close_scanner(S); + + return entries; +} + + +/** + * Code example for iterating all entries in KV store. This is for debug only as + * it is too expensive + */ +void +ParallaxBackend::iterate_all_impl() const {} + + +} // namespace gkfs::metadata diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0a4deca4d926ad03169843cdaeb2bbca8158fe93 --- /dev/null +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -0,0 +1,348 @@ +/* + Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include +#include +#include +#include + +#include +#include +#include +#include +extern "C" { +#include +} + +namespace gkfs::metadata { + +/** + * Called when the daemon is started: Connects to the KV store + * @param path where KV store data is stored + */ +RocksDBBackend::RocksDBBackend(const std::string& path) { + + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + options_.IncreaseParallelism(); + options_.OptimizeLevelStyleCompaction(); + // create the DB if it's not already present + options_.create_if_missing = true; + options_.merge_operator.reset(new MetadataMergeOperator); + optimize_database_impl(); + write_opts_.disableWAL = !(gkfs::config::rocksdb::use_write_ahead_log); + rdb::DB* rdb_ptr = nullptr; + auto s = rocksdb::DB::Open(options_, path, &rdb_ptr); + if(!s.ok()) { + throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); + } + this->db_.reset(rdb_ptr); +} + + +RocksDBBackend::~RocksDBBackend() { + this->db_.reset(); +} + +/** + * Exception wrapper on Status object. Throws NotFoundException if + * s.IsNotFound(), general DBException otherwise + * @param RocksDB status + * @throws DBException + */ +void +RocksDBBackend::throw_status_excpt(const rdb::Status& s) { + assert(!s.ok()); + + if(s.IsNotFound()) { + throw NotFoundException(s.ToString()); + } else { + throw DBException(s.ToString()); + } +} + + +/** + * Gets a KV store value for a key + * @param key + * @return value + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +std::string +RocksDBBackend::get_impl(const std::string& key) const { + std::string val; + + auto s = db_->Get(rdb::ReadOptions(), key, &val); + if(!s.ok()) { + throw_status_excpt(s); + } + + return val; +} + +/** + * Puts an entry into the KV store + * @param key + * @param val + * @throws DBException on failure + */ +void +RocksDBBackend::put_impl(const std::string& key, const std::string& val) { + + auto cop = CreateOperand(val); + auto s = db_->Merge(write_opts_, key, cop.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Puts an entry into the KV store if it doesn't exist. This function does not + * use a mutex. + * @param key + * @param val + * @throws DBException on failure, ExistException if entry already exists + */ +void +RocksDBBackend::put_no_exist_impl(const std::string& key, + const std::string& val) { + + if(exists(key)) + throw ExistsException(key); + put(key, val); +} + +/** + * Removes an entry from the KV store + * @param key + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +RocksDBBackend::remove_impl(const std::string& key) { + + auto s = db_->Delete(write_opts_, key); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * checks for existence of an entry + * @param key + * @return true if exists + * @throws DBException on failure + */ +bool +RocksDBBackend::exists_impl(const std::string& key) { + + std::string val; + + auto s = db_->Get(rdb::ReadOptions(), key, &val); + if(!s.ok()) { + if(s.IsNotFound()) { + return false; + } else { + throw_status_excpt(s); + } + } + return true; +} + +/** + * Updates a metadentry atomically and also allows to change keys + * @param old_key + * @param new_key + * @param val + * @throws DBException on failure, NotFoundException if entry doesn't exist + */ +void +RocksDBBackend::update_impl(const std::string& old_key, + const std::string& new_key, + const std::string& val) { + + // TODO use rdb::Put() method + rdb::WriteBatch batch; + batch.Delete(old_key); + batch.Put(new_key, val); + auto s = db_->Write(write_opts_, &batch); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Updates the size on the metadata + * Operation. E.g., called before a write() call + * @param key + * @param size + * @param append + * @throws DBException on failure + */ +void +RocksDBBackend::increase_size_impl(const std::string& key, size_t size, + bool append) { + + auto uop = IncreaseSizeOperand(size, append); + auto s = db_->Merge(write_opts_, key, uop.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Decreases the size on the metadata + * Operation E.g., called before a truncate() call + * @param key + * @param size + * @throws DBException on failure + */ +void +RocksDBBackend::decrease_size_impl(const std::string& key, size_t size) { + + auto uop = DecreaseSizeOperand(size); + auto s = db_->Merge(write_opts_, key, uop.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +RocksDBBackend::get_dirents_impl(const std::string& dir) const { + auto root_path = dir; + rocksdb::ReadOptions ropts; + auto it = db_->NewIterator(ropts); + + std::vector> entries; + for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); + it->Next()) { + + if(it->key().size() == root_path.size()) { + // we skip this path cause it is exactly the root_path + continue; + } + + /***** Get File name *****/ + auto name = it->key().ToString(); + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + continue; + } + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(it->value().ToString()); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::move(name), is_dir); + } + assert(it->status().ok()); + return entries; +} + +/** + * Return all the first-level entries of the directory @dir + * + * @return vector of pair , + * where name is the name of the entries and is_dir + * is true in the case the entry is a directory. + */ +std::vector> +RocksDBBackend::get_dirents_extended_impl(const std::string& dir) const { + auto root_path = dir; + rocksdb::ReadOptions ropts; + auto it = db_->NewIterator(ropts); + + std::vector> entries; + + for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); + it->Next()) { + + if(it->key().size() == root_path.size()) { + // we skip this path cause it is exactly the root_path + continue; + } + + /***** Get File name *****/ + auto name = it->key().ToString(); + if(name.find_first_of('/', root_path.size()) != std::string::npos) { + // skip stuff deeper then one level depth + continue; + } + // remove prefix + name = name.substr(root_path.size()); + + // relative path of directory entries must not be empty + assert(!name.empty()); + + Metadata md(it->value().ToString()); + auto is_dir = S_ISDIR(md.mode()); + + entries.emplace_back(std::forward_as_tuple(std::move(name), is_dir, + md.size(), md.ctime())); + } + assert(it->status().ok()); + return entries; +} + + +/** + * Code example for iterating all entries in KV store. This is for debug only as + * it is too expensive + */ +void +RocksDBBackend::iterate_all_impl() const { + std::string key; + std::string val; + // Do RangeScan on parent inode + auto iter = db_->NewIterator(rdb::ReadOptions()); + for(iter->SeekToFirst(); iter->Valid(); iter->Next()) { + key = iter->key().ToString(); + val = iter->value().ToString(); + std::cout << key << std::endl; + } +} + +/** + * Used for setting KV store settings + */ +void +RocksDBBackend::optimize_database_impl() { + options_.max_successive_merges = 128; +} + + +} // namespace gkfs::metadata diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 98ac82a53f8ee6bed2f16dc239ff8bd9a85dff62..1611e9e8a3bf94fabda6dc0386b31bf8c4a5aece 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -110,6 +110,16 @@ FsData::metadir(const std::string& metadir) { FsData::metadir_ = metadir; } +std::string_view +FsData::dbbackend() const { + return dbbackend_; +} + +void +FsData::dbbackend(const std::string& dbbackend) { + FsData::dbbackend_ = dbbackend; +} + const std::string& FsData::rpc_protocol() const { return rpc_protocol_; @@ -200,4 +210,15 @@ FsData::blocks_state(bool blocks_state) { FsData::blocks_state_ = blocks_state; } +unsigned long long +FsData::parallax_size_md() const { + return parallax_size_md_; +} + +void +FsData::parallax_size_md(unsigned int size_md) { + FsData::parallax_size_md_ = static_cast( + size_md * 1024ull * 1024ull * 1024ull); +} + } // namespace gkfs::daemon diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5144f4bcc6dce45516cbf1c932013d701d8f0c50..1ed24e9a22af443cc1a1ce16630f592c5fd56d6a 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -69,6 +69,7 @@ namespace fs = std::filesystem; static condition_variable shutdown_please; // handler for shutdown signaling static mutex mtx; // mutex to wait on shutdown conditional variable +static bool keep_rootdir = true; struct cli_options { string mountdir; @@ -78,6 +79,8 @@ struct cli_options { string listen; string hosts_file; string rpc_protocol; + string dbbackend; + string parallax_size; }; /** @@ -241,12 +244,13 @@ void init_environment() { // Initialize metadata db auto metadata_path = fmt::format("{}/{}", GKFS_DATA->metadir(), - gkfs::config::rocksdb::data_dir); + gkfs::config::metadata::dir); + fs::create_directories(metadata_path); GKFS_DATA->spdlogger()->debug("{}() Initializing metadata DB: '{}'", __func__, metadata_path); try { - GKFS_DATA->mdb( - std::make_shared(metadata_path)); + GKFS_DATA->mdb(std::make_shared( + metadata_path, GKFS_DATA->dbbackend())); } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( "{}() Failed to initialize metadata DB: {}", __func__, @@ -407,6 +411,15 @@ destroy_enviroment() { GKFS_DATA->spdlogger()->info("{}() Closing metadata DB", __func__); GKFS_DATA->close_mdb(); + + + // Delete rootdir/metadir if requested + if(!keep_rootdir) { + GKFS_DATA->spdlogger()->info("{}() Removing rootdir and metadir ...", + __func__); + fs::remove_all(GKFS_DATA->metadir(), ecode); + fs::remove_all(GKFS_DATA->rootdir(), ecode); + } } /** @@ -536,7 +549,7 @@ parse_input(const cli_options& opts, const CLI::App& desc) { auto rootdir_path = fs::path(rootdir); if(desc.count("--rootdir-suffix")) { if(opts.rootdir_suffix == gkfs::config::data::chunk_dir || - opts.rootdir_suffix == gkfs::config::rocksdb::data_dir) + opts.rootdir_suffix == gkfs::config::metadata::dir) throw runtime_error(fmt::format( "rootdir_suffix '{}' is reserved and not allowed.", opts.rootdir_suffix)); @@ -554,8 +567,13 @@ parse_input(const cli_options& opts, const CLI::App& desc) { GKFS_DATA->spdlogger()->debug("{}() Cleaning rootdir '{}' ...", __func__, rootdir_path.native()); fs::remove_all(rootdir_path.native()); - GKFS_DATA->spdlogger()->info("{}() Rootdir cleaned.", __func__); + GKFS_DATA->spdlogger()->info("{}() rootdir cleaned.", __func__); + } + + if(desc.count("--clean-rootdir-finish")) { + keep_rootdir = false; } + GKFS_DATA->spdlogger()->debug("{}() Root directory: '{}'", __func__, rootdir_path.native()); fs::create_directories(rootdir_path); @@ -569,7 +587,13 @@ parse_input(const cli_options& opts, const CLI::App& desc) { #else auto metadir_path = fs::path(metadir); #endif - + if(desc.count("--clean-rootdir")) { + // may throw exception (caught in main) + GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...", + __func__, metadir_path.native()); + fs::remove_all(metadir_path.native()); + GKFS_DATA->spdlogger()->info("{}() metadir cleaned.", __func__); + } fs::create_directories(metadir_path); GKFS_DATA->metadir(fs::canonical(metadir_path).native()); @@ -587,6 +611,39 @@ parse_input(const cli_options& opts, const CLI::App& desc) { GKFS_DATA->metadir(GKFS_DATA->rootdir()); #endif } + + if(desc.count("--dbbackend")) { + if(opts.dbbackend == gkfs::metadata::rocksdb_backend || + opts.dbbackend == gkfs::metadata::parallax_backend) { +#ifndef GKFS_ENABLE_PARALLAX + if(opts.dbbackend == gkfs::metadata::parallax_backend) { + throw runtime_error(fmt::format( + "dbbackend '{}' was not compiled and is disabled. " + "Pass -DGKFS_ENABLE_PARALLAX:BOOL=ON to CMake to enable.", + opts.dbbackend)); + } +#endif +#ifndef GKFS_ENABLE_ROCKSDB + if(opts.dbbackend == gkfs::metadata::rocksdb_backend) { + throw runtime_error(fmt::format( + "dbbackend '{}' was not compiled and is disabled. " + "Pass -DGKFS_ENABLE_ROCKSDB:BOOL=ON to CMake to enable.", + opts.dbbackend)); + } +#endif + GKFS_DATA->dbbackend(opts.dbbackend); + } else { + throw runtime_error( + fmt::format("dbbackend '{}' is not valid. Consult `--help`", + opts.dbbackend)); + } + + } else + GKFS_DATA->dbbackend(gkfs::metadata::rocksdb_backend); + + if(desc.count("--parallaxsize")) { // Size in GB + GKFS_DATA->parallax_size_md(stoi(opts.parallax_size)); + } } /** @@ -605,7 +662,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { */ int main(int argc, const char* argv[]) { - // Define arg parsing CLI::App desc{"Allowed options"}; cli_options opts{}; // clang-format off @@ -621,7 +677,7 @@ main(int argc, const char* argv[]) { "Creates an additional directory within the rootdir, allowing multiple daemons on one node."); desc.add_option( "--metadir,-i", opts.metadir, - "Metadata directory where GekkoFS' RocksDB data directory is located. If not set, rootdir is used."); + "Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used."); desc.add_option( "--listen,-l", opts.listen, "Address or interface to bind the daemon to. Default: local hostname.\n" @@ -642,8 +698,19 @@ main(int argc, const char* argv[]) { "Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, " "instead of using the RPC protocol. (Default off)"); desc.add_flag( - "--clean-rootdir,-c", + "--clean-rootdir", "Cleans Rootdir >before< launching the deamon"); + desc.add_flag( + "--clean-rootdir-finish,-c", + "Cleans Rootdir >after< the deamon finishes"); + desc.add_option( + "--dbbackend,-d", opts.dbbackend, + "Metadata database backend to use. Available: {rocksdb, parallaxdb}\n" + "RocksDB is default if not set. Parallax support is experimental.\n" + "Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir."); + desc.add_option("--parallaxsize", opts.parallax_size, + "parallaxdb - metadata file size in GB (default 8GB), " + "used only with new files"); desc.add_flag("--version", "Print version and exit."); // clang-format on try { @@ -651,6 +718,8 @@ main(int argc, const char* argv[]) { } catch(const CLI::ParseError& e) { return desc.exit(e); } + + if(desc.count("--version")) { cout << GKFS_VERSION_STRING << endl; #ifndef NDEBUG diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 37033091a43ffe869d356619f01bce7f9bc9682a..497a8e558cfd5110e22166b3d87892898bd2d29a 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -29,6 +29,33 @@ # ensure helper programs in the testing harness get built add_subdirectory(harness) +# Create daemon factories from template, depending of the databases +unset(DBS) +if (GKFS_ENABLE_ROCKSDB) + set (DBS "'gkfs_daemon_rocksdb'") +endif() +if (GKFS_ENABLE_PARALLAX) + if (DBS) + set (DBS "${DBS},'gkfs_daemon_parallaxdb'") + else() + set (DBS "'gkfs_daemon_parallaxdb'") + endif() +endif() + +if (GKFS_ENABLE_PARALLAX AND GKFS_ENABLE_ROCKSDB) + set (DBS "'gkfs_daemon_rocksdb','gkfs_daemon_parallaxdb'") +endif() + +# We enable rocksdb by default +if (NOT DBS) + set (DBS "'gkfs_daemon_rocksdb'") +endif() + +FIND_PATH(BUILD_PATH CMakeLists.txt . ) +FILE(READ ${BUILD_PATH}/conftest.template CONF_TEST_FILE) +STRING(REGEX REPLACE "'gkfs_daemon_rocksdb'" "${DBS}" MOD_CONF_TEST_FILE "${CONF_TEST_FILE}" ) +FILE(WRITE ${BUILD_PATH}/conftest.py "${MOD_CONF_TEST_FILE}") + gkfs_enable_python_testing( BINARY_DIRECTORIES ${CMAKE_BINARY_DIR}/src/daemon/ ${CMAKE_BINARY_DIR}/src/client/ @@ -171,4 +198,9 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) endif () + + endif() + + + diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a3c4e590074ecc15d5724d88bd56ab3d27eec262..9236ccec6b2fe4b9b4932fcc88f108a745e5201c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -95,17 +95,34 @@ def test_workspace(tmp_path, request): request.config.getoption('--lib-dir')) @pytest.fixture -def gkfs_daemon(test_workspace, request): +def gkfs_daemon_rocksdb(test_workspace, request): """ Initializes a local gekkofs daemon """ interface = request.config.getoption('--interface') - daemon = Daemon(interface, test_workspace) + daemon = Daemon(interface, "rocksdb", test_workspace) yield daemon.run() daemon.shutdown() +@pytest.fixture +def gkfs_daemon_parallaxdb(test_workspace, request): + """ + Initializes a local gekkofs daemon + """ + + interface = request.config.getoption('--interface') + daemon = Daemon(interface, "parallaxdb", test_workspace) + + yield daemon.run() + daemon.shutdown() + +@pytest.fixture(params=['gkfs_daemon_rocksdb', 'gkfs_daemon_parallaxdb']) +def gkfs_daemon(request): + return request.getfixturevalue(request.param) + + @pytest.fixture def gkfs_client(test_workspace): """ diff --git a/tests/integration/conftest.template b/tests/integration/conftest.template new file mode 100644 index 0000000000000000000000000000000000000000..b012a3783c3c497ea149ccc871d1bac111878445 --- /dev/null +++ b/tests/integration/conftest.template @@ -0,0 +1,173 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +import pytest +import logging +from collections import namedtuple +from _pytest.logging import caplog as _caplog +from pathlib import Path +from harness.logger import logger, initialize_logging, finalize_logging +from harness.cli import add_cli_options, set_default_log_formatter +from harness.workspace import Workspace, FileCreator +from harness.gkfs import Daemon, Client, ShellClient, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator +from harness.reporter import report_test_status, report_test_headline, report_assertion_pass + +def pytest_configure(config): + """ + Some configurations for our particular usage of pytest + """ + set_default_log_formatter(config, "%(message)s") + +def pytest_assertion_pass(item, lineno, orig, expl): + + location = namedtuple( + 'Location', ['path', 'module', 'function', 'lineno'])( + str(item.parent.fspath), item.parent.name, item.name, lineno) + + report_assertion_pass(logger, location, orig, expl) + +def pytest_addoption(parser): + """ + Adds extra options from the GKFS harness to the py.test CLI. + """ + add_cli_options(parser) + +@pytest.fixture(autouse=True) +def caplog(test_workspace, request, _caplog): + + # we don't need to see the logs from sh + _caplog.set_level(logging.CRITICAL, 'sh.command') + _caplog.set_level(logging.CRITICAL, 'sh.command.process') + _caplog.set_level(logging.CRITICAL, 'sh.command.process.streamreader') + _caplog.set_level(logging.CRITICAL, 'sh.stream_bufferer') + _caplog.set_level(logging.CRITICAL, 'sh.streamreader') + + test_log_path = test_workspace.logdir / (request.node.name + ".log") + + h = initialize_logging(logger, test_log_path) + report_test_headline(logger, request.node.nodeid, request.config, test_workspace) + + yield _caplog + + finalize_logging(logger, h) + +def pytest_runtest_logreport(report): + """ + Pytest hook called after a test phase (setup, call, teardownd) + has completed. + """ + + report_test_status(logger, report) + +@pytest.fixture +def test_workspace(tmp_path, request): + """ + Initializes a test workspace by creating a temporary directory for it. + """ + + yield Workspace(tmp_path, + request.config.getoption('--bin-dir'), + request.config.getoption('--lib-dir')) + +@pytest.fixture +def gkfs_daemon_rocksdb(test_workspace, request): + """ + Initializes a local gekkofs daemon + """ + + interface = request.config.getoption('--interface') + daemon = Daemon(interface, "rocksdb", test_workspace) + + yield daemon.run() + daemon.shutdown() + +@pytest.fixture +def gkfs_daemon_parallaxdb(test_workspace, request): + """ + Initializes a local gekkofs daemon + """ + + interface = request.config.getoption('--interface') + daemon = Daemon(interface, "parallaxdb", test_workspace) + + yield daemon.run() + daemon.shutdown() + +@pytest.fixture(params=['gkfs_daemon_rocksdb']) +def gkfs_daemon(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def gkfs_client(test_workspace): + """ + Sets up a gekkofs client environment so that + operations (system calls, library calls, ...) can + be requested from a co-running daemon. + """ + + return Client(test_workspace) + +@pytest.fixture +def gkfs_shell(test_workspace): + """ + Sets up a gekkofs environment so that shell commands + (stat, ls, mkdir, etc.) can be issued to a co-running daemon. + """ + + return ShellClient(test_workspace) + +@pytest.fixture +def file_factory(test_workspace): + """ + Returns a factory that can create custom input files + in the test workspace. + """ + + return FileCreator(test_workspace) + +@pytest.fixture +def gkfwd_daemon_factory(test_workspace, request): + """ + Returns a factory that can create forwarding daemons + in the test workspace. + """ + + interface = request.config.getoption('--interface') + + return FwdDaemonCreator(interface, test_workspace) + +@pytest.fixture +def gkfwd_client_factory(test_workspace): + """ + Sets up a gekkofs client environment so that + operations (system calls, library calls, ...) can + be requested from a co-running daemon. + """ + + return FwdClientCreator(test_workspace) diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index c9ce5c04c83e633e11fe5dfa866a54692a36a098..6f5624daec3a17f0ca0d24ef9367f7ca63f4ce7d 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -225,14 +225,14 @@ class FwdClientCreator: class Daemon: - def __init__(self, interface, workspace): + def __init__(self, interface, database, workspace): self._address = get_ephemeral_address(interface) self._workspace = workspace - + self._database = database self._cmd = sh.Command(gkfs_daemon_cmd, self._workspace.bindirs) self._env = os.environ.copy() - + self._metadir = self.rootdir libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + [str(p) for p in self._workspace.libdirs])) @@ -249,7 +249,11 @@ class Daemon: args = [ '--mountdir', self.mountdir, '--rootdir', self.rootdir, - '-l', self._address ] + '-l', self._address, + '--metadir', self._metadir, + '--dbbackend', self._database] + if self._database == "parallaxdb" : + args.append('--clean-rootdir-finish') logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -267,7 +271,7 @@ class Daemon: logger.debug("waiting for daemon to be ready") try: - self.wait_until_active(self._proc.pid, 60.0) + self.wait_until_active(self._proc.pid, 720.0) except Exception as ex: logger.error(f"daemon initialization failed: {ex}") @@ -307,7 +311,7 @@ class Daemon: while perf_counter() - init_time < timeout: try: - logger.debug(f"checking log file") + # logger.debug(f"checking log file") with open(self.logdir / gkfs_daemon_log_file) as log: for line in islice(log, max_lines): if re.search(gkfs_daemon_active_log_pattern, line) is not None: @@ -366,6 +370,7 @@ class _proxy_exec(): def __call__(self, *args, **kwargs): return self._client.run(self._name, *args, **kwargs) + class Client: """ A class to represent a GekkoFS client process with a patched LD_PRELOAD. @@ -776,7 +781,7 @@ class FwdDaemon: while perf_counter() - init_time < timeout: try: - logger.debug(f"checking log file") +# logger.debug(f"checking log file") with open(self.logdir / gkfwd_daemon_log_file) as log: for line in islice(log, max_lines): if re.search(gkfwd_daemon_active_log_pattern, line) is not None: diff --git a/tests/scripts/compile_dep.sh/0.9.1/agios.out b/tests/scripts/compile_dep.sh/0.9.1/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..b098aecfac7bbfa0a0ce7623d4bd3895af481285 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/agios.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.1/all.out b/tests/scripts/compile_dep.sh/0.9.1/all.out new file mode 100644 index 0000000000000000000000000000000000000000..5cdf4c5ffcdb5d1a480196ae8d3f861bb089f347 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/all.out @@ -0,0 +1,15 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### +Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.1/ci.out b/tests/scripts/compile_dep.sh/0.9.1/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..d3c276800bec7125a9c3247dc2b2218fe77fb5f2 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/ci.out @@ -0,0 +1,9 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/default.out b/tests/scripts/compile_dep.sh/0.9.1/default.out new file mode 100644 index 0000000000000000000000000000000000000000..787e4a08f0946c4b6155af6181afabab9842ea28 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/default.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/direct.out b/tests/scripts/compile_dep.sh/0.9.1/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..816266773b569d3d098245c7b868f7983753ff12 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/direct.out @@ -0,0 +1,6 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..e82dd020fa7dabd6cbd95dae0a07f38e2fabb825 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/infiniband_verbs.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric%verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.1/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..38f67ce3df9c4cbeee8d31ce3e1da6725d8f5819 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/marenostrum4.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/mogon2.out b/tests/scripts/compile_dep.sh/0.9.1/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..38f67ce3df9c4cbeee8d31ce3e1da6725d8f5819 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/mogon2.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/ngio.out b/tests/scripts/compile_dep.sh/0.9.1/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..38f67ce3df9c4cbeee8d31ce3e1da6725d8f5819 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/ngio.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..7138da9e21f4b03645a9cfe0ff07bb66276ca00a --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.1/omnipath_psm2.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/latest b/tests/scripts/compile_dep.sh/latest index 899f24fc754a14b89617093824d258aaa3aa2943..f514a2f0bd053cdaebb7ab7bcffca86fed32620a 120000 --- a/tests/scripts/compile_dep.sh/latest +++ b/tests/scripts/compile_dep.sh/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.1 \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/agios.out b/tests/scripts/dl_dep.sh/0.9.1/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..07bf82bac2c7eb153bfaef8bc0496d280a83b025 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/agios.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/all.out b/tests/scripts/dl_dep.sh/0.9.1/all.out new file mode 100644 index 0000000000000000000000000000000000000000..f6ddad01822139e754348a3fbc0ca6734c8d9d8d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/all.out @@ -0,0 +1,14 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/ci.out b/tests/scripts/dl_dep.sh/0.9.1/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..aa2acc0cd8b9bd43f2d05948a348fde1f68e119b --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/ci.out @@ -0,0 +1,10 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/default.out b/tests/scripts/dl_dep.sh/0.9.1/default.out new file mode 100644 index 0000000000000000000000000000000000000000..7c047de89939362fcceb01308271a229d798389c --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/default.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/direct.out b/tests/scripts/dl_dep.sh/0.9.1/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..42b3335768596af57f653b2409cd51492c3fb0b5 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/direct.out @@ -0,0 +1,7 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..da3c3e52c97232c28d407379a31c34bd217e167e --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/infiniband_verbs.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..1d4ad26f5aab87331a48265a2b749c5f64f5c2e8 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..1d4ad26f5aab87331a48265a2b749c5f64f5c2e8 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/ngio.out b/tests/scripts/dl_dep.sh/0.9.1/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..1d4ad26f5aab87331a48265a2b749c5f64f5c2e8 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/ngio.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..b4f4e9c0ea4b6bbb21148e1a1d4cec9c771c856c --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/latest b/tests/scripts/dl_dep.sh/latest index 899f24fc754a14b89617093824d258aaa3aa2943..f514a2f0bd053cdaebb7ab7bcffca86fed32620a 120000 --- a/tests/scripts/dl_dep.sh/latest +++ b/tests/scripts/dl_dep.sh/latest @@ -1 +1 @@ -0.9.0 \ No newline at end of file +0.9.1 \ No newline at end of file