Commit e4eebd35 authored by Ramon Nou's avatar Ramon Nou
Browse files

Parallax backend added 

Add command line option for kreon

Resolve CRTP specific function name

Testing Infrastructure for different databases

add path

Reactivated destructors, kreondb is destroyed to reduce CI space

Returning string_view in fs_data added dbbackend property

Updated Readme and MetatadaFactory

Adding extra options for kreondb, README+CHANGELOG updated

Update .gitlab-ci.yml

updating script test...

Updated dl_dep tests

Kreon test (for metadata) backend.

Add command line option for kreon

Resolve CRTP specific function name

Testing Infrastructure for different databases

add path

Reactivated destructors, kreondb is destroyed to reduce CI space

Returning string_view in fs_data added dbbackend property

Adding extra options for kreondb, README+CHANGELOG updated

updating script test...

Updated dl_dep tests

Remove Kreon - Add Parallax

Add Parallax

Add DockerFile

adding parallaxdb option

Docker update

Missing Parallax hpp

Removed Kreon from code, lib conflicts

Updated Parallax master

Updated DockerFile deps

Reformat

Changed Base

Added ps

Reduced Logging and added ps in docker

Add Path

increase testing timeout

Create options.yml automatically

Updated out tests
parent 19a732c5

.gitlab-ci.yml

+13 −12
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ variables: 
  GIT_SUBMODULE_STRATEGY:       recursive



# base image

image: gekkofs/core:0.9.0

image: gekkofs/core:0.9.0-experimental



################################################################################

## Validating
@@ -45,7 +45,7 @@ check format: 
################################################################################

gkfs:

  stage: build

  image: gekkofs/deps:0.9.0

  image: gekkofs/deps:0.9.0-experimental

  interruptible: true

  needs: []

  script:
@@ -71,7 +71,7 @@ gkfs: 


gkfwd:

  stage: build

  image: gekkofs/deps:0.9.0

  image: gekkofs/deps:0.9.0-experimental

  interruptible: true

  needs: []

  script:
@@ -104,15 +104,13 @@ gkfwd: 
## == tests for scripts ====================

scripts:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.0-experimental

  needs: []

  parallel:

    matrix:

      - SUBTEST: [ dl_dep.sh, compile_dep.sh ]

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts

    - cd ${BUILD_PATH}/tests/scripts

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit

  artifacts:

    expire_in: 1 week

    reports:
@@ -122,7 +120,7 @@ scripts: 
## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.0-experimental

  interruptible: true

  needs: ['gkfs']

  parallel:
@@ -131,9 +129,10 @@ gkfs:integration: 


  script:

    ## run tests

    - export PATH=${PATH}:/usr/local/bin

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

    - ${PYTEST} -v -n 1

          ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}

          --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}

          --junit-xml=report.xml
@@ -167,7 +166,7 @@ gkfs:integration: 
## == integration tests for gkfwd ==========

gkfwd:integration:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.0-experimental

  interruptible: true

  needs: ['gkfwd']

  parallel:
@@ -212,9 +211,11 @@ gkfwd:integration: 
## == unit tests for gkfs ==================

gkfs:unit:

  stage: test

  image: gekkofs/testing:0.9.0

  image: gekkofs/testing:0.9.0-experimental

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon

    - export PATH=${PATH}:/usr/local/bin

    ## run actual tests

    - cd ${BUILD_PATH}/tests/unit

    - ctest -j $(nproc) -L unit::all --output-junit report.xml

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -78,6 +78,8 @@ Note that tests still require `Boost_preprocessor`. 
([!116](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/116)).

- Fixed an issue where `LOG_OUTPUT_TRUNC` did not work as expected ([!118](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/118)).



- Added new metadata backend, kreon.



## [0.8.0] - 2020-09-15

### New

- Both client library and daemon have been extended to support the ofi+verbs

CMake/FindParallax.cmake

0 → 100644
+51 −0
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################

find_path(PARALLAX_INCLUDE_DIR

        NAMES parallax.h

        )



find_library(PARALLAX_LIBRARY

        NAMES parallax

        )



find_library(LOG_LIBRARY

	NAMES log

	)



set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR})

set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY})

set(PARALLAX_LIBRARIES ${LOG_LIBRARY})



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY LOG_LIBRARY PARALLAX_INCLUDE_DIR)



mark_as_advanced(

        PARALLAX_LIBRARY

        PARALLAX_INCLUDE_DIR

	LOG_LIBRARY

)

CMakeLists.txt

+4 −1
Original line number Diff line number Diff line
@@ -128,8 +128,10 @@ find_package(Mercury REQUIRED) 
find_package(Abt REQUIRED)

find_package(Margo REQUIRED)

find_package(Syscall_intercept REQUIRED)

find_package(Parallax REQUIRED)

find_package(Threads REQUIRED)



# parallax

find_package(yaml REQUIRED)

# some compilers need extra flags for std::filesystem, such as -lstdc++fs, this

# produces a std::filesystem imported target that takes care of all these

# details transparently
@@ -208,6 +210,7 @@ target_link_libraries(RocksDB 
    ${LZ4_LIBRARIES}

    )





if (${JeMalloc_FOUND})

    target_link_libraries(RocksDB

        INTERFACE

README.md

+10 −0
Original line number Diff line number Diff line
@@ -105,6 +105,16 @@ Options: 
  --auto-sm                   Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off)

  -c,--clean-rootdir          Cleans Rootdir >before< launching the deamon

  --version                   Print version and exit.



  --dbbackend               'rocksdb' (default) or 'kreondb' can be specified as

                            metadata backend.

  --keepmd                  'kreondb' specific, persist the metadata file

                            (default off, file is deleted at the end)

  --reusemd                 'kreondb' specific, do not recreate the 

                            metadata file, (default re/create)

  --kreonsize               'kreondb' specific, size of the metadata file in GB

                            (default 16, 16 GB)

  --version                 Print version and exit.

```



It is possible to run multiple independent GekkoFS instances on the same node. Note, that when these GekkoFS instances