diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 00a5d74b471ab41f67c7c1baf7d558b969b24ab6..b5a077a342dac3a7572f798af64e19dbc1d7b116 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,14 +24,14 @@ variables:
GIT_SUBMODULE_STRATEGY: recursive
# base image
-image: gekkofs/core:0.9.0
+image: gekkofs/core:0.9.1
################################################################################
## Validating
################################################################################
check format:
stage: lint
- image: gekkofs/linter:0.9.0
+ image: gekkofs/linter:0.9.1
needs: []
script:
- ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format:
################################################################################
gkfs:
stage: build
- image: gekkofs/deps:0.9.0
+ image: gekkofs/deps:0.9.1
interruptible: true
needs: []
script:
@@ -59,6 +59,8 @@ gkfs:
-DGKFS_INSTALL_TESTS:BOOL=ON
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
-DGKFS_USE_GUIDED_DISTRIBUTION:BOOL=ON
+ -DGKFS_ENABLE_PARALLAX:BOOL=ON
+ -DGKFS_ENABLE_ROCKSDB:BOOL=ON
${CI_PROJECT_DIR}
- make -j$(nproc) install
# reduce artifacts size
@@ -71,7 +73,7 @@ gkfs:
gkfwd:
stage: build
- image: gekkofs/deps:0.9.0
+ image: gekkofs/deps:0.9.1
interruptible: true
needs: []
script:
@@ -85,6 +87,8 @@ gkfwd:
-DGKFS_INSTALL_TESTS:BOOL=ON
-DGKFS_ENABLE_FORWARDING:BOOL=ON
-DGKFS_ENABLE_AGIOS:BOOL=ON
+ -DGKFS_ENABLE_PARALLAX:BOOL=OFF
+ -DGKFS_ENABLE_ROCKSDB:BOOL=ON
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
${CI_PROJECT_DIR}
- make -j$(nproc) install
@@ -104,15 +108,13 @@ gkfwd:
## == tests for scripts ====================
scripts:
stage: test
- image: gekkofs/testing:0.9.0
+ image: gekkofs/testing:0.9.1
needs: []
- parallel:
- matrix:
- - SUBTEST: [ dl_dep.sh, compile_dep.sh ]
script:
- mkdir -p ${BUILD_PATH}/tests/scripts
- cd ${BUILD_PATH}/tests/scripts
- - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml
+ - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit
+ - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit
artifacts:
expire_in: 1 week
reports:
@@ -122,7 +124,7 @@ scripts:
## == integration tests for gkfs ===========
gkfs:integration:
stage: test
- image: gekkofs/testing:0.9.0
+ image: gekkofs/testing:0.9.1
interruptible: true
needs: ['gkfs']
parallel:
@@ -131,9 +133,10 @@ gkfs:integration:
script:
## run tests
+ - export PATH=${PATH}:/usr/local/bin
- mkdir -p ${BUILD_PATH}/tests/run
- cd ${BUILD_PATH}/tests/integration
- - ${PYTEST} -v -n $(nproc)
+ - ${PYTEST} -v -n 1
${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}
--basetemp=${BUILD_PATH}/tests/run/${SUBTEST}
--junit-xml=report.xml
@@ -167,7 +170,7 @@ gkfs:integration:
## == integration tests for gkfwd ==========
gkfwd:integration:
stage: test
- image: gekkofs/testing:0.9.0
+ image: gekkofs/testing:0.9.1
interruptible: true
needs: ['gkfwd']
parallel:
@@ -212,9 +215,11 @@ gkfwd:integration:
## == unit tests for gkfs ==================
gkfs:unit:
stage: test
- image: gekkofs/testing:0.9.0
+ image: gekkofs/testing:0.9.1
needs: ['gkfs']
script:
+ ## Add path to mkfs.kreon
+ - export PATH=${PATH}:/usr/local/bin
## run actual tests
- cd ${BUILD_PATH}/tests/unit
- ctest -j $(nproc) -L unit::all --output-junit report.xml
@@ -240,7 +245,7 @@ gkfs:unit:
################################################################################
documentation:
stage: docs
- image: gekkofs/docs:0.9.0
+ image: gekkofs/docs:0.9.1
needs: []
rules:
# we only build the documentation automatically if we are on the
@@ -276,7 +281,7 @@ documentation:
################################################################################
coverage:
stage: report
- image: gekkofs/coverage:0.9.0
+ image: gekkofs/coverage:0.9.1
needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]
script:
- cd ${BUILD_PATH}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ead8b557ee8af8969af4b3fc14388125c920be79..05f997a69f40ace3a5131f9fb19824c51cd2e63a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,21 +1,39 @@
# Changelog
+
All notable changes to GekkoFS project will be documented in this file.
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres
+to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
+### New
+
+- Added new experimental metadata backend:
+ Parallax ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)).
+ - Added support to use multiple metadata backends.
+ - Added `--clean-rootdir-finish` argument to remove rootdir/metadir at the end when the daemon finishes.
+
+### Changed
+
+- `-c` argument has been moved to `--clean-rootdir-finish` and is now used to clean rootdir/metadir on daemon
+ shutdown ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)).
+
+### Removed
+
+### Fixed
## [0.9.0] - 2022-02-22
+
### New
+
- GekkoFS now uses C++17 (!74).
-- Added a new `dirents_extended` function which can improve `find` operations.
-A corresponding example how to use this function can be found at
-`examples/gfind/gfind.cpp` with a non-mpi version at `examples/gfind/sfind.cpp`
-([!73](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/73)).
-- Code coverage reports for the source code are now generated and tracked
-([!77](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/77)).
+- Added a new `dirents_extended` function which can improve `find` operations. A corresponding example how to use this
+ function can be found at
+ `examples/gfind/gfind.cpp` with a non-mpi version at `examples/gfind/sfind.cpp`
+ ([!73](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/73)).
+- Code coverage reports for the source code are now generated and tracked
+ ([!77](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/77)).
- Considerable overhaul and new features of the GekkoFS testing facilities
([!120](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/120),
[!121](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/121)).
diff --git a/CMake/FindParallax.cmake b/CMake/FindParallax.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..562ad6b9ce2f9c8e5f9a8469094cb32dc312c7a3
--- /dev/null
+++ b/CMake/FindParallax.cmake
@@ -0,0 +1,51 @@
+################################################################################
+# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+find_path(PARALLAX_INCLUDE_DIR
+ NAMES parallax.h
+ )
+
+find_library(PARALLAX_LIBRARY
+ NAMES parallax
+ )
+
+find_library(PARALLAX_LOG_LIBRARY
+ NAMES log
+ )
+
+set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR})
+set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY})
+set(PARALLAX_LIBRARIES ${PARALLAX_LOG_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY PARALLAX_LOG_LIBRARY PARALLAX_INCLUDE_DIR)
+
+mark_as_advanced(
+ PARALLAX_LIBRARY
+ PARALLAX_INCLUDE_DIR
+ PARALLAX_LOG_LIBRARY
+)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2f7b9ca6df7c3a7db783fd1ce183b0e1e0fb7840..7589df5191bf2878d9c6db29e98e3f3e5b85cdae 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -119,17 +119,13 @@ include(gkfs-code-coverage)
set(CMAKE_EXPORT_COMPILE_COMMANDS 0)
-# Rocksdb dependencies
-find_package(LZ4 REQUIRED)
-find_package(JeMalloc) # required if rocksdb has been build with jemalloc
-find_package(RocksDB REQUIRED)
+
# margo dependencies
find_package(Mercury REQUIRED)
find_package(Abt REQUIRED)
find_package(Margo REQUIRED)
find_package(Syscall_intercept REQUIRED)
find_package(Threads REQUIRED)
-
# some compilers need extra flags for std::filesystem, such as -lstdc++fs, this
# produces a std::filesystem imported target that takes care of all these
# details transparently
@@ -182,6 +178,8 @@ if (GKFS_ENABLE_AGIOS)
find_package(AGIOS REQUIRED)
endif ()
+option(GKFS_ENABLE_PARALLAX "Enable Parallax db backend" OFF)
+option(GKFS_ENABLE_ROCKSDB "Enable ROCKSDB backend" ON)
set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library")
add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE})
@@ -199,26 +197,7 @@ endif()
configure_file(include/common/cmake_configure.hpp.in include/common/cmake_configure.hpp)
-# Imported target
-add_library(RocksDB INTERFACE IMPORTED GLOBAL)
-target_link_libraries(RocksDB
- INTERFACE
- ${ROCKSDB_LIBRARIES}
- # rocksdb libs
- ${LZ4_LIBRARIES}
- )
-if (${JeMalloc_FOUND})
- target_link_libraries(RocksDB
- INTERFACE
- ${JEMALLOC_LIBRARIES}
- )
-endif ()
-# we cannot use target_include_directories with CMake < 3.11
-set_target_properties(RocksDB
- PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS}
- )
add_library(CLI11 INTERFACE)
# we cannot use target_include_directories with CMake < 3.11
diff --git a/README.md b/README.md
index f2c861414fb06c7c361ed88dba2da108c1a93452..24857eba041ce9a1ac8c2a19f756208f2b5f1c10 100644
--- a/README.md
+++ b/README.md
@@ -103,7 +103,12 @@ Options:
Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets)
Libfabric must have enabled support verbs or psm2.
--auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off)
- -c,--clean-rootdir Cleans Rootdir >before< launching the deamon
+ --clean-rootdir Cleans Rootdir >before< launching the deamon
+ -c,--clean-rootdir-finish Cleans Rootdir >after< the deamon finishes
+ -d,--dbbackend TEXT Metadata database backend to use. Available: {rocksdb, parallaxdb}
+ RocksDB is default if not set. Parallax support is experimental.
+ Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir.
+ --parallaxsize TEXT parallaxdb - metadata file size in GB (default 8GB), used only with new files
--version Print version and exit.
```
@@ -225,6 +230,14 @@ Then, the `examples/distributors/guided/generate.py` scrpt is used to create the
Finally, modify `guided_config.txt` to your distribution requirements.
+### Metadata Backends
+There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH`
+is available.
+To enable it use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=OFF`.
+
+Once it is enabled, `--dbbackend` option will be functional.
+
+
### Acknowledgment
This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu).
diff --git a/docker/0.9.1/core/Dockerfile b/docker/0.9.1/core/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..bf1b2327ef9f7a79973049405c3f8696a0178fa8
--- /dev/null
+++ b/docker/0.9.1/core/Dockerfile
@@ -0,0 +1,39 @@
+FROM debian:bullseye-slim
+
+LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ git \
+ curl \
+ ca-certificates \
+ libtool \
+ pkg-config \
+ make \
+ automake \
+ gcc \
+ g++ \
+ procps \
+ # AGIOS dependencies
+ libconfig-dev \
+ # Mercury dependencies
+ libltdl-dev \
+ lbzip2 \
+ # Margo dependencies \
+ libjson-c-dev \
+ # RocksDB dependencies
+ liblz4-dev \
+ # syscall_intercept dependencies
+ libcapstone-dev \
+ # GekkoFS dependencies
+ libboost-program-options-dev \
+ uuid-dev && \
+ # install cmake 3.14 since it's needed for some dependencies
+ curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \
+ chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \
+ ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean && \
+ rm ./cmake-3.14.5-Linux-x86_64.sh
diff --git a/docker/0.9.1/core/Makefile b/docker/0.9.1/core/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..0f32dcbebdc083ba0a053952a3a9f985aeaa67d4
--- /dev/null
+++ b/docker/0.9.1/core/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/core:0.9.1 .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/core:0.9.1 .
+
+all:
+ docker build -t gekkofs/core:0.9.1 .
diff --git a/docker/0.9.1/coverage/Dockerfile b/docker/0.9.1/coverage/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..6514807130f821e3dd79dc259a3b5366bb22a4b3
--- /dev/null
+++ b/docker/0.9.1/coverage/Dockerfile
@@ -0,0 +1,20 @@
+FROM debian:bullseye-slim
+
+LABEL Description="Environment to generate coverage reports in GekkoFS"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ wget \
+ git \
+ cmake \
+ gcc \
+ g++ \
+ lcov \
+ python3 \
+ python3-pip \
+ python3-setuptools && \
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && \
+ apt-get autoclean && \
+ python3 -m pip install --upgrade pip && \
+ pip3 install gcovr
diff --git a/docker/0.9.1/coverage/Makefile b/docker/0.9.1/coverage/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..67ba36bba92843dd970b95343920b3b1ccbe3933
--- /dev/null
+++ b/docker/0.9.1/coverage/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/coverage:0.9.1 .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/coverage:0.9.1 .
+
+all:
+ docker build -t gekkofs/coverage:0.9.1 .
\ No newline at end of file
diff --git a/docker/0.9.1/deps/Dockerfile b/docker/0.9.1/deps/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..fe81b563b4d2beca8272b4ea4803666617ea6bff
--- /dev/null
+++ b/docker/0.9.1/deps/Dockerfile
@@ -0,0 +1,37 @@
+FROM gekkofs/core:0.9.1
+
+LABEL Description="Debian-based environment to build GekkoFS"
+
+ENV GKFS_PATH /opt/gkfs
+ENV GKFS_VERSION 0.9.1
+
+ENV SCRIPTS_PATH ${GKFS_PATH}/scripts
+ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src
+ENV INSTALL_PATH /usr/local
+
+COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/patches ${SCRIPTS_PATH}/patches
+COPY scripts/profiles ${SCRIPTS_PATH}/profiles
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ python3 \
+ python3-pip \
+ python3-dev \
+ python3-venv \
+ python3-setuptools \
+ libnuma-dev libyaml-dev \
+ procps && \
+ python3 -m pip install --upgrade pip && \
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && apt-get autoclean
+
+# Download and build dependencies
+RUN cd ${SCRIPTS_PATH} && \
+ /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \
+ /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \
+ rm -rf ${DEPS_SRC_PATH} && \
+ rm -rf ${SCRIPTS_PATH} && \
+ rmdir ${GKFS_PATH} && \
+ ldconfig
diff --git a/docker/0.9.1/deps/Makefile b/docker/0.9.1/deps/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..6cd0dc7dbdeb9d9afb59be3d2326459ab14028d1
--- /dev/null
+++ b/docker/0.9.1/deps/Makefile
@@ -0,0 +1,23 @@
+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts remove-scripts
+amd64: build-amd64 mount-scripts remove-scripts
+aarch64: build-aarch64 mount-scripts remove-scripts
+
+copy-scripts:
+ cp -R $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: copy-scripts
+ docker build -t gekkofs/deps:0.9.1 .
+
+build-amd64: copy-scripts
+ docker build --platform amd64 -t gekkofs/deps:0.9.1 .
+
+build-aarch64: copy-scripts
+ docker build --platform aarch64 -t gekkofs/deps:0.9.1 .
+
+remove-scripts:
+ - rm -rf $(CWD)/scripts
diff --git a/docker/0.9.1/docs/Dockerfile b/docker/0.9.1/docs/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..21389dbafdb46a601c761fc3f2829023ba6dfd4d
--- /dev/null
+++ b/docker/0.9.1/docs/Dockerfile
@@ -0,0 +1,33 @@
+FROM gekkofs/deps:0.9.1
+
+LABEL Description="Debian-based environment suitable to build GekkoFS' documentation"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # install dependencies for Doxygen
+ python \
+ flex \
+ bison \
+ graphviz && \
+ # install doxygen (repo version is kind of old)
+ cd /tmp && curl -OL https://www.doxygen.nl/files/doxygen-1.9.2.src.tar.gz && \
+ tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \
+ mkdir -p /tmp/doxygen-1.9.2/build && \
+ cd /tmp/doxygen-1.9.2/build && \
+ cmake -G "Unix Makefiles" .. && \
+ make -j8 install && \
+ # install sphinx, breathe and exhale
+ pip3 install \
+ 'sphinx==4.4.0' \
+ sphinx_rtd_theme \
+ 'breathe==4.33.1' \
+ 'exhale==0.3.1' \
+ 'sphinx-copybutton==0.5.0' \
+ 'sphinx-multiversion==0.2.4' \
+ 'myst_parser==0.17.0' && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && \
+ rm -rf /tmp/doxygen-1.9.2 && \
+ rm /tmp/doxygen-1.9.2.src.tar.gz && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.9.1/docs/Makefile b/docker/0.9.1/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..9770b2a5bc2cfdad5f8e15d0f6edc2cab67efa24
--- /dev/null
+++ b/docker/0.9.1/docs/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/docs:0.9.1 .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/docs:0.9.1 .
+
+all:
+ docker build -t gekkofs/docs:0.9.1 .
\ No newline at end of file
diff --git a/docker/0.9.1/linter/Dockerfile b/docker/0.9.1/linter/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..fa39907d335239922d58292fa701a6fc117ea54f
--- /dev/null
+++ b/docker/0.9.1/linter/Dockerfile
@@ -0,0 +1,19 @@
+FROM gekkofs/core:0.9.1
+
+LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # clang 10 deps for clang-format
+ lsb-release \
+ wget \
+ software-properties-common \
+ gnupg2 && \
+ # add clang-10 repos
+ wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 11 && \
+ # install clang-format
+ apt-get update && apt-get install -y --no-install-recommends clang-format-11 && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.9.1/linter/Makefile b/docker/0.9.1/linter/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..449c1cadcb13f7bef53a533dd1bd7e5b01b39baf
--- /dev/null
+++ b/docker/0.9.1/linter/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/linter:0.9.1 .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/linter:0.9.1 .
+
+all:
+ docker build -t gekkofs/linter:0.9.1 .
diff --git a/docker/0.9.1/testing/Dockerfile b/docker/0.9.1/testing/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..4c28d99ef8f0e1db68d6b724e330b44b08501352
--- /dev/null
+++ b/docker/0.9.1/testing/Dockerfile
@@ -0,0 +1,14 @@
+FROM gekkofs/deps:0.9.1
+
+LABEL Description="Debian-based environment to test GekkoFS"
+
+RUN \
+ # install cmake 3.21 since we need to produce JUnit XML files
+ curl -OL https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.sh && \
+ chmod u+x ./cmake-3.21.1-linux-x86_64.sh && \
+ ./cmake-3.21.1-linux-x86_64.sh --skip-license --prefix=/usr && \
+ # install gcovr
+ # (required for partial coverage reports in parallel runs)
+ pip3 install gcovr && \
+ # cleanup
+ rm ./cmake-3.21.1-linux-x86_64.sh
diff --git a/docker/0.9.1/testing/Makefile b/docker/0.9.1/testing/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..7f507aa4b8f43bc9989aedfc86e64bc60a8558e0
--- /dev/null
+++ b/docker/0.9.1/testing/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/testing:0.9.1 .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/testing:0.9.1 .
+
+all:
+ docker build -t gekkofs/testing:0.9.1 .
\ No newline at end of file
diff --git a/docker/latest b/docker/latest
index 899f24fc754a14b89617093824d258aaa3aa2943..f514a2f0bd053cdaebb7ab7bcffca86fed32620a 120000
--- a/docker/latest
+++ b/docker/latest
@@ -1 +1 @@
-0.9.0
\ No newline at end of file
+0.9.1
\ No newline at end of file
diff --git a/docs/sphinx/users/building.rst b/docs/sphinx/users/building.rst
index bcd13438b87112fc300d8ef6bdfdc339fcf8af76..259e84d1a14845bb226484af8f27d87e6fd6f9c3 100644
--- a/docs/sphinx/users/building.rst
+++ b/docs/sphinx/users/building.rst
@@ -108,6 +108,11 @@ dependencies:
- `AGIOS `_ (commit c26a654 or
newer) to enable the :code:`GekkoFWD` I/O forwarding mode.
+- `PARALLAX` There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH`
+is available. To enable it, use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=OFF`.
+ Once it is enabled, `--dbbackend` option will be functional.
+
+
.. _step_by_step_installation:
Step-by-step installation
diff --git a/docs/sphinx/users/running.md b/docs/sphinx/users/running.md
index 286353a9835bf9926d11b235fb27272c1cb31806..4750c151cd839d473e6b08cdfee4d7348cfcd4dc 100644
--- a/docs/sphinx/users/running.md
+++ b/docs/sphinx/users/running.md
@@ -57,24 +57,29 @@ metadata locally, and at which path clients can access the file system (mount po
Further options are available
```bash
- Allowed options
- Usage: src/daemon/gkfs_daemon [OPTIONS]
-
- Options:
- -h,--help Print this help message and exit
- -m,--mountdir TEXT REQUIRED Virtual mounting directory where GekkoFS is available.
- -r,--rootdir TEXT REQUIRED Local data directory where GekkoFS data for this daemon is stored.
- -s,--rootdir-suffix TEXT Creates an additional directory within the rootdir, allowing multiple daemons on one node.
- -i,--metadir TEXT Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used.
- -l,--listen TEXT Address or interface to bind the daemon to. Default: local hostname.
- When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly which associates the verbs device with the network interface. In case FI_VERBS_IFACE is already defined, the argument is ignored. Default 'ib'.
- -H,--hosts-file TEXT Shared file used by deamons to register their endpoints. (default './gkfs_hosts.txt')
- -P,--rpc-protocol TEXT Used RPC protocol for inter-node communication.
- Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets)
- Libfabric must have enabled support verbs or psm2.
- --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off)
- -c,--clean-rootdir Cleans Rootdir >before< launching the deamon
- --version Print version and exit.
+Allowed options
+Usage: src/daemon/gkfs_daemon [OPTIONS]
+
+Options:
+ -h,--help Print this help message and exit
+ -m,--mountdir TEXT REQUIRED Virtual mounting directory where GekkoFS is available.
+ -r,--rootdir TEXT REQUIRED Local data directory where GekkoFS data for this daemon is stored.
+ -s,--rootdir-suffix TEXT Creates an additional directory within the rootdir, allowing multiple daemons on one node.
+ -i,--metadir TEXT Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used.
+ -l,--listen TEXT Address or interface to bind the daemon to. Default: local hostname.
+ When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly which associates the verbs device with the network interface. In case FI_VERBS_IFACE is already defined, the argument is ignored. Default 'ib'.
+ -H,--hosts-file TEXT Shared file used by deamons to register their endpoints. (default './gkfs_hosts.txt')
+ -P,--rpc-protocol TEXT Used RPC protocol for inter-node communication.
+ Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets)
+ Libfabric must have enabled support verbs or psm2.
+ --auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off)
+ --clean-rootdir Cleans Rootdir >before< launching the deamon
+ -c,--clean-rootdir-finish Cleans Rootdir >after< the deamon finishes
+ -d,--dbbackend TEXT Metadata database backend to use. Available: {rocksdb, parallaxdb}
+ RocksDB is default if not set. Parallax support is experimental.
+ Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir.
+ --parallaxsize TEXT parallaxdb - metadata file size in GB (default 8GB), used only with new files
+ --version Print version and exit.
````
Shut it down by gracefully killing the process (SIGTERM).
diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp
index 9ccb8833b2a52b77f4c6b190c0fa1dc728d3db53..d4bb154a7aa2e921b832eceeaa8934e04fc751d1 100644
--- a/include/client/preload_util.hpp
+++ b/include/client/preload_util.hpp
@@ -32,7 +32,6 @@
#include
#include
-
#include
#include
#include