Parallax backend added

Add command line option for kreon

Resolve CRTP specific function name

Testing Infrastructure for different databases

add path

Reactivated destructors, kreondb is destroyed to reduce CI space

Returning string_view in fs_data added dbbackend property

Updated Readme and MetatadaFactory

Adding extra options for kreondb, README+CHANGELOG updated

Update .gitlab-ci.yml

updating script test...

Updated dl_dep tests

Kreon test (for metadata) backend.

Add command line option for kreon

Resolve CRTP specific function name

Testing Infrastructure for different databases

add path

Reactivated destructors, kreondb is destroyed to reduce CI space

Returning string_view in fs_data added dbbackend property

Adding extra options for kreondb, README+CHANGELOG updated

updating script test...

Updated dl_dep tests

Remove Kreon - Add Parallax

Add Parallax

Add DockerFile

adding parallaxdb option

Docker update

Missing Parallax hpp

Removed Kreon from code, lib conflicts

Updated Parallax master

Updated DockerFile deps

Reformat

Changed Base

Added ps

Reduced Logging and added ps in docker

Add Path

increase testing timeout

Create options.yml automatically

Updated out tests

.gitlab-ci.yml

@@ -24,7 +24,7 @@ variables:
   GIT_SUBMODULE_STRATEGY:       recursive
 
 # base image
-image: gekkofs/core:0.9.0
+image: gekkofs/core:0.9.0-experimental
 
 ################################################################################
 ## Validating
@@ -45,7 +45,7 @@ check format:
 ################################################################################
 gkfs:
   stage: build
-  image: gekkofs/deps:0.9.0
+  image: gekkofs/deps:0.9.0-experimental
   interruptible: true
   needs: []
   script:
@@ -71,7 +71,7 @@ gkfs:
 
 gkfwd:
   stage: build
-  image: gekkofs/deps:0.9.0
+  image: gekkofs/deps:0.9.0-experimental
   interruptible: true
   needs: []
   script:
@@ -104,15 +104,13 @@ gkfwd:
 ## == tests for scripts ====================
 scripts:
   stage: test
-  image: gekkofs/testing:0.9.0
+  image: gekkofs/testing:0.9.0-experimental
   needs: []
-  parallel:
-    matrix:
-      - SUBTEST: [ dl_dep.sh, compile_dep.sh ]
   script:
     - mkdir -p ${BUILD_PATH}/tests/scripts
     - cd ${BUILD_PATH}/tests/scripts
-    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml
+    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit
+    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit
   artifacts:
     expire_in: 1 week
     reports:
@@ -122,7 +120,7 @@ scripts:
 ## == integration tests for gkfs ===========
 gkfs:integration:
   stage: test
-  image: gekkofs/testing:0.9.0
+  image: gekkofs/testing:0.9.0-experimental
   interruptible: true
   needs: ['gkfs']
   parallel:
@@ -131,9 +129,10 @@ gkfs:integration:
 
   script:
     ## run tests
+    - export PATH=${PATH}:/usr/local/bin
     - mkdir -p ${BUILD_PATH}/tests/run
     - cd ${BUILD_PATH}/tests/integration
-    - ${PYTEST} -v -n $(nproc)
+    - ${PYTEST} -v -n 1
           ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}
           --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}
           --junit-xml=report.xml
@@ -167,7 +166,7 @@ gkfs:integration:
 ## == integration tests for gkfwd ==========
 gkfwd:integration:
   stage: test
-  image: gekkofs/testing:0.9.0
+  image: gekkofs/testing:0.9.0-experimental
   interruptible: true
   needs: ['gkfwd']
   parallel:
@@ -212,9 +211,11 @@ gkfwd:integration:
 ## == unit tests for gkfs ==================
 gkfs:unit:
   stage: test
-  image: gekkofs/testing:0.9.0
+  image: gekkofs/testing:0.9.0-experimental
   needs: ['gkfs']
   script:
+    ## Add path to mkfs.kreon
+    - export PATH=${PATH}:/usr/local/bin
     ## run actual tests
     - cd ${BUILD_PATH}/tests/unit
     - ctest -j $(nproc) -L unit::all --output-junit report.xml

CHANGELOG.md

@@ -78,6 +78,8 @@ Note that tests still require `Boost_preprocessor`.
 ([!116](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/116)).
 - Fixed an issue where `LOG_OUTPUT_TRUNC` did not work as expected ([!118](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/118)).
 
+- Added new metadata backend, kreon.
+
 ## [0.8.0] - 2020-09-15
 ### New
 - Both client library and daemon have been extended to support the ofi+verbs

CMake/FindParallax.cmake

+################################################################################
+# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain            #
+# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany          #
+#                                                                              #
+# This software was partially supported by the                                 #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #
+#                                                                              #
+# This software was partially supported by the                                 #
+# ADA-FS project under the SPPEXA project funded by the DFG.                   #
+#                                                                              #
+# This file is part of GekkoFS.                                                #
+#                                                                              #
+# GekkoFS is free software: you can redistribute it and/or modify              #
+# it under the terms of the GNU General Public License as published by         #
+# the Free Software Foundation, either version 3 of the License, or            #
+# (at your option) any later version.                                          #
+#                                                                              #
+# GekkoFS is distributed in the hope that it will be useful,                   #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #
+#                                                                              #
+# SPDX-License-Identifier: GPL-3.0-or-later                                    #
+################################################################################
+find_path(PARALLAX_INCLUDE_DIR
+        NAMES parallax.h
+        )
+
+find_library(PARALLAX_LIBRARY
+        NAMES parallax
+        )
+
+find_library(LOG_LIBRARY
+	NAMES log
+	)
+
+set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR})
+set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY})
+set(PARALLAX_LIBRARIES ${LOG_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY LOG_LIBRARY PARALLAX_INCLUDE_DIR)
+
+mark_as_advanced(
+        PARALLAX_LIBRARY
+        PARALLAX_INCLUDE_DIR
+	LOG_LIBRARY
+)

CMakeLists.txt

@@ -128,8 +128,10 @@ find_package(Mercury REQUIRED)
 find_package(Abt REQUIRED)
 find_package(Margo REQUIRED)
 find_package(Syscall_intercept REQUIRED)
+find_package(Parallax REQUIRED)
 find_package(Threads REQUIRED)
-
+# parallax
+find_package(yaml REQUIRED)
 # some compilers need extra flags for std::filesystem, such as -lstdc++fs, this
 # produces a std::filesystem imported target that takes care of all these
 # details transparently
@@ -208,6 +210,7 @@ target_link_libraries(RocksDB
     ${LZ4_LIBRARIES}
     )
 
+
 if (${JeMalloc_FOUND})
     target_link_libraries(RocksDB
         INTERFACE

README.md

@@ -105,6 +105,16 @@ Options:
   --auto-sm                   Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off)
   -c,--clean-rootdir          Cleans Rootdir >before< launching the deamon
   --version                   Print version and exit.
+
+  --dbbackend               'rocksdb' (default) or 'kreondb' can be specified as
+                            metadata backend.
+  --keepmd                  'kreondb' specific, persist the metadata file
+                            (default off, file is deleted at the end)
+  --reusemd                 'kreondb' specific, do not recreate the 
+                            metadata file, (default re/create)
+  --kreonsize               'kreondb' specific, size of the metadata file in GB
+                            (default 16, 16 GB)
+  --version                 Print version and exit.
 ```
 
 It is possible to run multiple independent GekkoFS instances on the same node. Note, that when these GekkoFS instances

docker/0.9.0-experimental/core/Dockerfile

+FROM debian:buster-slim
+
+LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		git \
+		curl \
+		ca-certificates \
+		libtool \
+		pkg-config \
+		make \
+		automake \
+		gcc \
+		g++ \
+		# AGIOS dependencies
+		libconfig-dev \
+		# Mercury dependencies
+		libltdl-dev \
+		lbzip2 \
+		# RocksDB dependencies
+		libsnappy-dev \
+		liblz4-dev \
+		libzstd-dev \
+		libbz2-dev \
+		zlib1g-dev \
+		# syscall_intercept dependencies
+		libcapstone-dev \
+		# GekkoFS dependencies
+		libboost-filesystem-dev \
+		libboost-program-options-dev \
+		uuid-dev && \
+    # install cmake 3.14 since it's needed for some dependencies
+    curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \
+    chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \
+    ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean && \
+    rm ./cmake-3.14.5-Linux-x86_64.sh

docker/0.9.0-experimental/core/Makefile

+.PHONY: all
+
+all:
+	docker build -t gekkofs/core:0.9.0-experimental .

docker/0.9.0-experimental/coverage/Dockerfile

+FROM debian:buster-slim
+
+LABEL Description="Environment to generate coverage reports in GekkoFS"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        wget \
+        git \
+        cmake \
+        gcc \
+        g++ \
+        lcov \
+        python3 \
+        python3-pip \
+        python3-setuptools && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean && \
+    apt-get autoclean && \
+    python3 -m pip install --upgrade pip && \
+    pip3 install gcovr

docker/0.9.0-experimental/coverage/Makefile

+.PHONY: all
+
+all:
+	docker build -t gekkofs/coverage:0.8.0 .

docker/0.9.0-experimental/deps/Dockerfile

+FROM gekkofs/core:0.9.0-experimental
+
+LABEL Description="Debian-based environment to build GekkoFS"
+
+ENV GKFS_PATH	/opt/gkfs
+ENV GKFS_VERSION 0.9.0-experimental
+
+ENV SCRIPTS_PATH	${GKFS_PATH}/scripts
+ENV DEPS_SRC_PATH	${GKFS_PATH}/deps_src
+ENV INSTALL_PATH	/usr/local
+
+COPY scripts/dl_dep.sh		${SCRIPTS_PATH}/
+COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/patches        ${SCRIPTS_PATH}/patches
+COPY scripts/profiles       ${SCRIPTS_PATH}/profiles
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		python3 \
+		python3-pip \
+		python3-dev \
+		python3-venv \
+		python3-setuptools \ 
+		libnuma-dev libyaml-dev procps && \
+    python3 -m pip install --upgrade pip && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean && apt-get autoclean 
+ 
+# Download and build dependencies
+RUN cd ${SCRIPTS_PATH} && \
+    /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \
+    /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \
+    rm -rf ${DEPS_SRC_PATH} && \
+    rm -rf ${SCRIPTS_PATH} && \
+    rmdir ${GKFS_PATH} && \
+    ldconfig

docker/0.9.0-experimental/deps/Makefile

+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts umount-scripts
+
+mount-scripts:
+	mkdir -p $(CWD)/scripts
+	mount --bind $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: mount-scripts
+	docker build -t gekkofs/deps:0.9.0-experimental .
+
+umount-scripts:
+	- umount $(CWD)/scripts
+	- rmdir $(CWD)/scripts

docker/0.9.0-experimental/linter/Dockerfile

+FROM gekkofs/core:0.8.0
+
+LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		# clang 10 deps for clang-format
+		lsb-release \
+		wget \
+		software-properties-common \
+		gnupg2 && \
+    # add clang-10 repos
+    wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 10 && \
+    # install clang-format
+    apt-get update && apt-get install -y --no-install-recommends clang-format-10 && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean

docker/0.9.0-experimental/linter/Makefile

+.PHONY: all
+
+all:
+	docker build -t gekkofs/linter:0.8.0 .

docker/0.9.0-experimental/testing/Dockerfile

+FROM gekkofs/deps:0.9.0-experimental
+
+LABEL Description="Debian-based environment to test GekkoFS"
+
+RUN \
+    # install cmake 3.21 since we need to produce JUnit XML files
+    curl -OL https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.sh && \
+    chmod u+x ./cmake-3.21.1-linux-x86_64.sh && \
+    ./cmake-3.21.1-linux-x86_64.sh --skip-license --prefix=/usr && \
+    # install gcovr
+    # (required for partial coverage reports in parallel runs)
+    pip3 install gcovr && \
+    # cleanup
+    rm ./cmake-3.21.1-linux-x86_64.sh

docker/0.9.0-experimental/testing/Makefile

+.PHONY: all
+
+all:
+	docker build -t gekkofs/testing:0.9.0-experimental .

include/client/preload_util.hpp

@@ -32,7 +32,7 @@
 
 #include <client/preload.hpp>
 #include <common/metadata.hpp>
-
+#include <optional>
 #include <string>
 #include <iostream>
 #include <map>

include/daemon/backend/metadata/db.hpp

@@ -34,50 +34,28 @@
 
 #include <memory>
 #include <spdlog/spdlog.h>
-#include <rocksdb/db.h>
 #include <daemon/backend/exceptions.hpp>
 #include <tuple>
+#include <daemon/backend/metadata/metadata_backend.hpp>
+#include <daemon/backend/metadata/rocksdb_backend.hpp>
+#include <daemon/backend/metadata/parallax_backend.hpp>
 
 namespace rdb = rocksdb;
 
 namespace gkfs::metadata {
 
-/**
- * @brief MetadataDB class providing an abstraction layer to the KV store
- * RocksDB.
- */
+
 class MetadataDB {
 private:
-    std::unique_ptr<rdb::DB> db;  ///< RocksDB instance
-    rdb::Options options;         ///< RocksDB configuration
-    rdb::WriteOptions write_opts; ///< RocksDB write configuration
-    std::string path;             ///< Path to where RocksDB persists its data
-    std::shared_ptr<spdlog::logger> log_; ///< MetadataDB internal logger
+    std::string path_;
+    std::shared_ptr<spdlog::logger> log_;
+    std::unique_ptr<AbstractMetadataBackend> backend_;
 
-    /**
-     * @brief Sets up specific settings to optimize RocksDB instance to
-     * environment on launch.
-     * @param options RocksDB configurations
-     */
-    static void
-    optimize_rocksdb_options(rdb::Options& options);
 
 public:
-    /**
-     * @brief Exception wrapper on Status object.
-     * @param s RocksDB status
-     * @throws Throws NotFoundException if s.IsNotFound(), general DBException
-     * otherwise
-     */
-    static inline void
-    throw_rdb_status_excpt(const rdb::Status& s);
+    MetadataDB(const std::string& path, const std::string_view database);
 
-    /**
-     * @brief Constructor, called when daemon is started and connects to KV
-     * store.
-     * @param path Path to where RocksDB persists its data
-     */
-    explicit MetadataDB(const std::string& path);
+    ~MetadataDB();
 
     /**
      * @brief Gets the KV store value for a key.
@@ -182,7 +160,7 @@ public:
      * is therefore unused.
      */
     void
-    iterate_all();
+    iterate_all() const;
 };
 
 } // namespace gkfs::metadata

include/daemon/backend/metadata/metadata_backend.hpp

+/*
+  Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  This file is part of GekkoFS.
+
+  GekkoFS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  GekkoFS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.
+
+  SPDX-License-Identifier: GPL-3.0-or-later
+*/
+
+#ifndef GEKKOFS_METADATA_BACKEND_HPP
+#define GEKKOFS_METADATA_BACKEND_HPP
+
+#include <memory>
+#include <spdlog/spdlog.h>
+#include <daemon/backend/exceptions.hpp>
+#include <tuple>
+
+namespace gkfs::metadata {
+
+
+class AbstractMetadataBackend {
+public:
+    virtual ~AbstractMetadataBackend() = default;
+
+    virtual std::string
+    get(const std::string& key) const = 0;
+
+    virtual void
+    put(const std::string& key, const std::string& val) = 0;
+
+    virtual void
+    put_no_exist(const std::string& key, const std::string& val) = 0;
+
+    virtual void
+    remove(const std::string& key) = 0;
+
+    virtual bool
+    exists(const std::string& key) = 0;
+
+    virtual void
+    update(const std::string& old_key, const std::string& new_key,
+           const std::string& val) = 0;
+
+    virtual void
+    increase_size(const std::string& key, size_t size, bool append) = 0;
+
+    virtual void
+    decrease_size(const std::string& key, size_t size) = 0;
+
+    virtual std::vector<std::pair<std::string, bool>>
+    get_dirents(const std::string& dir) const = 0;
+
+    virtual std::vector<std::tuple<std::string, bool, size_t, time_t>>
+    get_dirents_extended(const std::string& dir) const = 0;
+
+    virtual void
+    iterate_all() const = 0;
+};
+
+template <typename T>
+class MetadataBackend : public AbstractMetadataBackend {
+private:
+    std::string path;
+    std::shared_ptr<spdlog::logger> log_;
+
+public:
+    std::string
+    get(const std::string& key) const {
+        return static_cast<T const&>(*this).get_impl(key);
+    }
+
+    void
+    put(const std::string& key, const std::string& val) {
+        static_cast<T&>(*this).put_impl(key, val);
+    }
+
+    void
+    put_no_exist(const std::string& key, const std::string& val) {
+        static_cast<T&>(*this).put_no_exist_impl(key, val);
+    }
+
+    void
+    remove(const std::string& key) {
+        static_cast<T&>(*this).remove_impl(key);
+    }
+
+    bool
+    exists(const std::string& key) {
+        return static_cast<T&>(*this).exists_impl(key);
+    }
+
+    void
+    update(const std::string& old_key, const std::string& new_key,
+           const std::string& val) {
+        static_cast<T&>(*this).update_impl(old_key, new_key, val);
+    }
+
+    void
+    increase_size(const std::string& key, size_t size, bool append) {
+        static_cast<T&>(*this).increase_size_impl(key, size, append);
+    }
+
+    void
+    decrease_size(const std::string& key, size_t size) {
+        static_cast<T&>(*this).decrease_size_impl(key, size);
+    }
+
+    std::vector<std::pair<std::string, bool>>
+    get_dirents(const std::string& dir) const {
+        return static_cast<T const&>(*this).get_dirents_impl(dir);
+    }
+
+    std::vector<std::tuple<std::string, bool, size_t, time_t>>
+    get_dirents_extended(const std::string& dir) const {
+        return static_cast<T const&>(*this).get_dirents_extended_impl(dir);
+    }
+
+    void
+    iterate_all() const {
+        static_cast<T const&>(*this).iterate_all_impl();
+    }
+};
+
+} // namespace gkfs::metadata
+
+#endif // GEKKOFS_METADATA_BACKEND_HPP

include/daemon/backend/metadata/parallax_backend.hpp

+/*
+  Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  This file is part of GekkoFS.
+
+  GekkoFS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  GekkoFS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License