Resolve "Support other KV Databases"

Ramon Nourequested to merge
rnou/173-support-other-kv-databases into master

Closes #173 (closed)

This MR adds support for different metadata backends. Actually, PARALLAX from FORTH is supported as experimental. PARALLAX needs a 8GB file (minimum size) to store metadata. It is created in the metadir directory as rocksdbx name. Inside the install script there is a commented line, to modify an internal parameter of PARALLAX increasing the key size drom 255 bytes to 4096 (Max PATH name in POSIX). However, PARALLAX is not (fully) tested with such values.

This backend should work in all the scenarios, but there are some issues that appear under high load mdtest.

This MR includes the optional compilation/dependencies of each of the backends. Testing is generated dynamically from conftest.template depending on the options selected.

This MR includes a new argument for the daemon to remove the rootdir/metadir files directories when the daemon is finished. This reduces the data from artifacts when running databases that uses a fixed size file. (i.e., Parallax)

Edited by Ramon Nou

Merge request reports

CMake/FindParallax.cmake

0 → 100644
+51 −0
Changes for CMake/FindParallax.cmake: 51 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################

find_path(PARALLAX_INCLUDE_DIR

        NAMES parallax.h

        )



find_library(PARALLAX_LIBRARY

        NAMES parallax

        )



find_library(PARALLAX_LOG_LIBRARY

	NAMES log

	)



set(PARALLAX_INCLUDE_DIRS ${PARALLAX_INCLUDE_DIR})

set(PARALLAX_LIBRARIES ${PARALLAX_LIBRARY})

set(PARALLAX_LIBRARIES ${PARALLAX_LOG_LIBRARY})



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(Parallax DEFAULT_MSG PARALLAX_LIBRARY PARALLAX_LOG_LIBRARY PARALLAX_INCLUDE_DIR)



mark_as_advanced(

        PARALLAX_LIBRARY

	PARALLAX_INCLUDE_DIR

	PARALLAX_LOG_LIBRARY

)

docker/0.9.1/core/Dockerfile

0 → 100644
+39 −0
Changes for docker/0.9.1/core/Dockerfile: 39 added lines, 0 removed lines.
Original line number Diff line number Diff line 
FROM debian:bullseye-slim



LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"



RUN apt-get update && \

    apt-get install -y --no-install-recommends \

		git \

		curl \

		ca-certificates \

		libtool \

		pkg-config \

		make \

		automake \

		gcc \

		g++ \

		procps \

		# AGIOS dependencies

		libconfig-dev \

		# Mercury dependencies

		libltdl-dev \

		lbzip2 \

        # Margo dependencies \

        libjson-c-dev \

		# RocksDB dependencies

		liblz4-dev \

		# syscall_intercept dependencies

		libcapstone-dev \

		# GekkoFS dependencies

		libboost-program-options-dev \

		uuid-dev && \

    # install cmake 3.14 since it's needed for some dependencies

    curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \

    chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \

    ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \

    # Clean apt cache to reduce image layer size

    rm -rf /var/lib/apt/lists/* && \

    # Clean apt caches of packages

    apt-get clean && apt-get autoclean && \

    rm ./cmake-3.14.5-Linux-x86_64.sh

docker/0.9.1/core/Makefile

0 → 100644
+10 −0
Changes for docker/0.9.1/core/Makefile: 10 added lines, 0 removed lines.
Original line number Diff line number Diff line 
.PHONY: all



amd64:

	docker build --platform amd64 -t gekkofs/core:0.9.1 .



aarch64:

	docker build --platform aarch64 -t gekkofs/core:0.9.1 .



all:

	docker build -t gekkofs/core:0.9.1 .

docker/0.9.1/coverage/Dockerfile

0 → 100644
+20 −0
Changes for docker/0.9.1/coverage/Dockerfile: 20 added lines, 0 removed lines.
Original line number Diff line number Diff line 
FROM debian:bullseye-slim



LABEL Description="Environment to generate coverage reports in GekkoFS"



RUN apt-get update && \

    apt-get install -y --no-install-recommends \

        wget \

        git \

        cmake \

        gcc \

        g++ \

        lcov \

        python3 \

        python3-pip \

        python3-setuptools && \

    rm -rf /var/lib/apt/lists/* && \

    apt-get clean && \

    apt-get autoclean && \

    python3 -m pip install --upgrade pip && \

    pip3 install gcovr

docker/0.9.1/coverage/Makefile

0 → 100644
+10 −0
Changes for docker/0.9.1/coverage/Makefile: 10 added lines, 0 removed lines.
Original line number Diff line number Diff line 
.PHONY: all



amd64:

	docker build --platform amd64 -t gekkofs/coverage:0.9.1 .



aarch64:

	docker build --platform aarch64 -t gekkofs/coverage:0.9.1 .



all:

	docker build -t gekkofs/coverage:0.9.1 .
 
 No newline at end of file