Commit 4d760bc6 authored by Ramon Nou's avatar Ramon Nou
Browse files

Merge branch 'rnou/173-support-other-kv-databases' into 'master' 

Resolve "Support other KV Databases"

Closes #173

This `MR` adds support for different metadata backends. Actually, `PARALLAX` from `FORTH` is supported as experimental.
`PARALLAX` needs a 8GB file (minimum size) to store metadata. It is created in the `metadir` directory as `rocksdbx` name.
Inside the install script there is a commented line, to modify an internal parameter of `PARALLAX` increasing the key size drom 255 bytes to 4096 (Max PATH name in POSIX). However, `PARALLAX` is not (fully) tested with such values.

This backend should work in all the scenarios, but there are some issues that appear under high load mdtest.

This `MR` includes the optional compilation/dependencies of each of the backends. Testing is generated dynamically from conftest.template depending on the options selected.

This `MR` includes a new argument for the daemon to remove the rootdir/metadir files directories when the daemon is finished. This reduces the data from artifacts when running databases that uses a fixed size file. (i.e., Parallax)

See merge request !110
parents 19a732c5 bb5912fe

.gitlab-ci.yml

+20 −15
Original line number Diff line number Diff line
@@ -24,14 +24,14 @@ variables: 
  GIT_SUBMODULE_STRATEGY:       recursive



# base image

image: gekkofs/core:0.9.0

image: gekkofs/core:0.9.1



################################################################################

## Validating

################################################################################

check format:

  stage: lint

  image: gekkofs/linter:0.9.0

  image: gekkofs/linter:0.9.1

  needs: []

  script:

    - ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format: 
################################################################################

gkfs:

  stage: build

  image: gekkofs/deps:0.9.0

  image: gekkofs/deps:0.9.1

  interruptible: true

  needs: []

  script:
@@ -59,6 +59,8 @@ gkfs: 
      -DGKFS_INSTALL_TESTS:BOOL=ON

      -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}

      -DGKFS_USE_GUIDED_DISTRIBUTION:BOOL=ON

      -DGKFS_ENABLE_PARALLAX:BOOL=ON

      -DGKFS_ENABLE_ROCKSDB:BOOL=ON

      ${CI_PROJECT_DIR}

    - make -j$(nproc) install

    # reduce artifacts size
@@ -71,7 +73,7 @@ gkfs: 


gkfwd:

  stage: build

  image: gekkofs/deps:0.9.0

  image: gekkofs/deps:0.9.1

  interruptible: true

  needs: []

  script:
@@ -85,6 +87,8 @@ gkfwd: 
      -DGKFS_INSTALL_TESTS:BOOL=ON

      -DGKFS_ENABLE_FORWARDING:BOOL=ON

      -DGKFS_ENABLE_AGIOS:BOOL=ON

      -DGKFS_ENABLE_PARALLAX:BOOL=OFF

      -DGKFS_ENABLE_ROCKSDB:BOOL=ON

      -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}

      ${CI_PROJECT_DIR}

    - make -j$(nproc) install
@@ -104,15 +108,13 @@ gkfwd: 
## == tests for scripts ====================

scripts:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.1

  needs: []

  parallel:

    matrix:

      - SUBTEST: [ dl_dep.sh, compile_dep.sh ]

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts

    - cd ${BUILD_PATH}/tests/scripts

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit

  artifacts:

    expire_in: 1 week

    reports:
@@ -122,7 +124,7 @@ scripts: 
## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.1

  interruptible: true

  needs: ['gkfs']

  parallel:
@@ -131,9 +133,10 @@ gkfs:integration: 


  script:

    ## run tests

    - export PATH=${PATH}:/usr/local/bin

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

    - ${PYTEST} -v -n 1

          ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}

          --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}

          --junit-xml=report.xml
@@ -167,7 +170,7 @@ gkfs:integration: 
## == integration tests for gkfwd ==========

gkfwd:integration:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.1

  interruptible: true

  needs: ['gkfwd']

  parallel:
@@ -212,9 +215,11 @@ gkfwd:integration: 
## == unit tests for gkfs ==================

gkfs:unit:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.1

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon

    - export PATH=${PATH}:/usr/local/bin

    ## run actual tests

    - cd ${BUILD_PATH}/tests/unit

    - ctest -j $(nproc) -L unit::all --output-junit report.xml
@@ -240,7 +245,7 @@ gkfs:unit: 
################################################################################

documentation:

  stage: docs

  image: gekkofs/docs:0.9.0

  image: gekkofs/docs:0.9.1

  needs: []

  rules:

    # we only build the documentation automatically if we are on the
@@ -276,7 +281,7 @@ documentation: 
################################################################################

coverage:

  stage: report

  image: gekkofs/coverage:0.9.0

  image: gekkofs/coverage:0.9.1

  needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]

  script:

    - cd ${BUILD_PATH}

CHANGELOG.md

+26 −8
Original line number Diff line number Diff line 
# Changelog



All notable changes to GekkoFS project will be documented in this file.



The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres

to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).



## [Unreleased]



### New



- Added new experimental metadata backend:

  Parallax ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)).

    - Added support to use multiple metadata backends.

    - Added `--clean-rootdir-finish` argument to remove rootdir/metadir at the end when the daemon finishes.



### Changed



- `-c` argument has been moved to `--clean-rootdir-finish` and is now used to clean rootdir/metadir on daemon

  shutdown ([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)).



### Removed



### Fixed



## [0.9.0] - 2022-02-22



### New



- GekkoFS now uses C++17 (!74).

- Added a new `dirents_extended` function which can improve `find` operations.

A corresponding example how to use this function can be found at 

- Added a new `dirents_extended` function which can improve `find` operations. A corresponding example how to use this

  function can be found at

  `examples/gfind/gfind.cpp` with a non-mpi version at `examples/gfind/sfind.cpp`

  ([!73](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/73)).

- Code coverage reports for the source code are now generated and tracked

CMake/FindParallax.cmake

0 → 100644
+51 −0
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################

find_path(PARALLAX_INCLUDE_DIR

        NAMES parallax.h

        )



find_library(PARALLAX_LIBRARY

        NAMES parallax

        )



find_library(PARALLAX_LOG_LIBRARY

	NAMES log

	)



set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR})

set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY})

set(PARALLAX_LIBRARIES ${PARALLAX_LOG_LIBRARY})



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY PARALLAX_LOG_LIBRARY PARALLAX_INCLUDE_DIR)



mark_as_advanced(

        PARALLAX_LIBRARY

	PARALLAX_INCLUDE_DIR

	PARALLAX_LOG_LIBRARY

)

CMakeLists.txt

+3 −24
Original line number Diff line number Diff line
@@ -119,17 +119,13 @@ include(gkfs-code-coverage) 


set(CMAKE_EXPORT_COMPILE_COMMANDS 0)



# Rocksdb dependencies

find_package(LZ4 REQUIRED)

find_package(JeMalloc) # required if rocksdb has been build with jemalloc

find_package(RocksDB REQUIRED)



# margo dependencies

find_package(Mercury REQUIRED)

find_package(Abt REQUIRED)

find_package(Margo REQUIRED)

find_package(Syscall_intercept REQUIRED)

find_package(Threads REQUIRED)



# some compilers need extra flags for std::filesystem, such as -lstdc++fs, this

# produces a std::filesystem imported target that takes care of all these

# details transparently
@@ -182,6 +178,8 @@ if (GKFS_ENABLE_AGIOS) 
    find_package(AGIOS REQUIRED)

endif ()



option(GKFS_ENABLE_PARALLAX "Enable Parallax db backend" OFF)

option(GKFS_ENABLE_ROCKSDB "Enable ROCKSDB backend" ON)



set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library")

add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE})
@@ -199,26 +197,7 @@ endif() 


configure_file(include/common/cmake_configure.hpp.in include/common/cmake_configure.hpp)



# Imported target

add_library(RocksDB INTERFACE IMPORTED GLOBAL)

target_link_libraries(RocksDB

    INTERFACE

    ${ROCKSDB_LIBRARIES}

    # rocksdb libs

    ${LZ4_LIBRARIES}

    )



if (${JeMalloc_FOUND})

    target_link_libraries(RocksDB

        INTERFACE

        ${JEMALLOC_LIBRARIES}

        )

endif ()

# we cannot use target_include_directories with CMake < 3.11

set_target_properties(RocksDB

    PROPERTIES

    INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS}

    )



add_library(CLI11 INTERFACE)

# we cannot use target_include_directories with CMake < 3.11

README.md

+14 −1
Original line number Diff line number Diff line
@@ -103,7 +103,12 @@ Options: 
                              Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets)

                              Libfabric must have enabled support verbs or psm2.

  --auto-sm                   Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off)

  -c,--clean-rootdir          Cleans Rootdir >before< launching the deamon

  --clean-rootdir             Cleans Rootdir >before< launching the deamon

  -c,--clean-rootdir-finish   Cleans Rootdir >after< the deamon finishes

  -d,--dbbackend TEXT         Metadata database backend to use. Available: {rocksdb, parallaxdb}

                              RocksDB is default if not set. Parallax support is experimental.

                              Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir.

  --parallaxsize TEXT         parallaxdb - metadata file size in GB (default 8GB), used only with new files

  --version                   Print version and exit.

```
@@ -225,6 +230,14 @@ Then, the `examples/distributors/guided/generate.py` scrpt is used to create the 


Finally, modify `guided_config.txt` to your distribution requirements.



### Metadata Backends

There are two different metadata backends in GekkoFS. The default one uses `rocksdb`, however an alternative based on `PARALLAX` from `FORTH` 

is available.

To enable it use the `-DGKFS_ENABLE_PARALLAX:BOOL=ON` option, you can also disable `rocksdb` with `-DGKFS_ENABLE_ROCKSDB:BOOL=OFF`.



Once it is enabled, `--dbbackend` option will be functional.





### Acknowledgment



This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu).