Prepare Parallax Support to 0.9.1

.gitlab-ci.yml

@@ -24,14 +24,14 @@ variables:
   GIT_SUBMODULE_STRATEGY:       recursive
 
 # base image
-image: gekkofs/core:0.9.0-experimental
+image: gekkofs/core:0.9.1
 
 ################################################################################
 ## Validating
 ################################################################################
 check format:
   stage: lint
-  image: gekkofs/linter:0.9.0
+  image: gekkofs/linter:0.9.1
   needs: []
   script:
     - ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format:
 ################################################################################
 gkfs:
   stage: build
-  image: gekkofs/deps:0.9.0-experimental
+  image: gekkofs/deps:0.9.1
   interruptible: true
   needs: []
   script:
@@ -71,7 +71,7 @@ gkfs:
 
 gkfwd:
   stage: build
-  image: gekkofs/deps:0.9.0-experimental
+  image: gekkofs/deps:0.9.1
   interruptible: true
   needs: []
   script:
@@ -104,7 +104,7 @@ gkfwd:
 ## == tests for scripts ====================
 scripts:
   stage: test
-  image: gekkofs/testing:0.9.0-experimental
+  image: gekkofs/testing:0.9.1
   needs: []
   script:
     - mkdir -p ${BUILD_PATH}/tests/scripts
@@ -120,7 +120,7 @@ scripts:
 ## == integration tests for gkfs ===========
 gkfs:integration:
   stage: test
-  image: gekkofs/testing:0.9.0-experimental
+  image: gekkofs/testing:0.9.1
   interruptible: true
   needs: ['gkfs']
   parallel:
@@ -166,7 +166,7 @@ gkfs:integration:
 ## == integration tests for gkfwd ==========
 gkfwd:integration:
   stage: test
-  image: gekkofs/testing:0.9.0-experimental
+  image: gekkofs/testing:0.9.1
   interruptible: true
   needs: ['gkfwd']
   parallel:
@@ -211,7 +211,7 @@ gkfwd:integration:
 ## == unit tests for gkfs ==================
 gkfs:unit:
   stage: test
-  image: gekkofs/testing:0.9.0-experimental
+  image: gekkofs/testing:0.9.1
   needs: ['gkfs']
   script:
     ## Add path to mkfs.kreon
@@ -241,7 +241,7 @@ gkfs:unit:
 ################################################################################
 documentation:
   stage: docs
-  image: gekkofs/docs:0.9.0
+  image: gekkofs/docs:0.9.1
   needs: []
   rules:
     # we only build the documentation automatically if we are on the
@@ -277,7 +277,7 @@ documentation:
 ################################################################################
 coverage:
   stage: report
-  image: gekkofs/coverage:0.9.0
+  image: gekkofs/coverage:0.9.1
   needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]
   script:
     - cd ${BUILD_PATH}

CHANGELOG.md

@@ -5,7 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
-
+- Parallax experimental integration 
+Support for different databases backend
+New Docker Images and scripts (0.9.1)
+([!110](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/110)).
 
 ## [0.9.0] - 2022-02-22
 ### New

README.md

@@ -106,14 +106,15 @@ Options:
   -c,--clean-rootdir          Cleans Rootdir >before< launching the deamon
   --version                   Print version and exit.
 
-  --dbbackend               'rocksdb' (default) or 'kreondb' can be specified as
-                            metadata backend.
-  --keepmd                  'kreondb' specific, persist the metadata file
+  --dbbackend               'rocksdb' (default) or 'parallaxdb' can be specified as
+                            metadata backend, in that case a file in 'metadir' named
+                            rocksdbx is created. Parallaxdb support is experimental.
+  --keepmd                  'parallaxdb' specific, persist the metadata file
                             (default off, file is deleted at the end)
-  --reusemd                 'kreondb' specific, do not recreate the 
+  --reusemd                 'parallaxdb' specific, do not recreate the 
                             metadata file, (default re/create)
-  --kreonsize               'kreondb' specific, size of the metadata file in GB
-                            (default 16, 16 GB)
+  --kreonsize               'parallaxdb' specific, size of the metadata file in GB. Minimal is 8 GB
+                            (default 8, 8 GB)
   --version                 Print version and exit.
 ```
 

docker/0.9.0-experimental/core/Makefile

-.PHONY: all
-
-all:
-	docker build -t gekkofs/core:0.9.0-experimental .

docker/0.9.0-experimental/coverage/Makefile

-.PHONY: all
-
-all:
-	docker build -t gekkofs/coverage:0.8.0 .

docker/0.9.0-experimental/deps/Makefile

-CWD:=$(shell pwd)
-GIT_ROOT:=$(shell git rev-parse --show-toplevel)
-
-.PHONY: all build mount-scripts umount-scripts
-
-all: build mount-scripts umount-scripts
-
-mount-scripts:
-	mkdir -p $(CWD)/scripts
-	mount --bind $(GIT_ROOT)/scripts $(CWD)/scripts
-
-build: mount-scripts
-	docker build -t gekkofs/deps:0.9.0-experimental .
-
-umount-scripts:
-	- umount $(CWD)/scripts
-	- rmdir $(CWD)/scripts

docker/0.9.0-experimental/linter/Makefile

-.PHONY: all
-
-all:
-	docker build -t gekkofs/linter:0.8.0 .

docker/0.9.0-experimental/testing/Makefile

-.PHONY: all
-
-all:
-	docker build -t gekkofs/testing:0.9.0-experimental .

docker/0.9.1/core/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/core:0.9.1 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/core:0.9.1 .
+
+all:
+	docker build -t gekkofs/core:0.9.1 .

docker/0.9.0-experimental/coverage/Dockerfile → docker/0.9.1/coverage/Dockerfile

-FROM debian:buster-slim
+FROM debian:bullseye-slim
 
 LABEL Description="Environment to generate coverage reports in GekkoFS"
 

docker/0.9.1/coverage/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/coverage:0.9.1 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/coverage:0.9.1 .
+
+all:
+	docker build -t gekkofs/coverage:0.9.1 .
\ No newline at end of file

docker/0.9.0-experimental/deps/Dockerfile → docker/0.9.1/deps/Dockerfile

-FROM gekkofs/core:0.9.0-experimental
+FROM gekkofs/core:0.9.1
 
 LABEL Description="Debian-based environment to build GekkoFS"
 
 ENV GKFS_PATH	/opt/gkfs
-ENV GKFS_VERSION 0.9.0-experimental
+ENV GKFS_VERSION 0.9.1
 
 ENV SCRIPTS_PATH	${GKFS_PATH}/scripts
 ENV DEPS_SRC_PATH	${GKFS_PATH}/deps_src
@@ -20,11 +20,12 @@ RUN apt-get update && \
 		python3-pip \
 		python3-dev \
 		python3-venv \
-		python3-setuptools \ 
-		libnuma-dev libyaml-dev procps && \
+		python3-setuptools \
+    libnuma-dev libyaml-dev \
+    procps && \
     python3 -m pip install --upgrade pip && \
     rm -rf /var/lib/apt/lists/* && \
-    apt-get clean && apt-get autoclean 
+    apt-get clean && apt-get autoclean
  
 # Download and build dependencies
 RUN cd ${SCRIPTS_PATH} && \

docker/0.9.1/deps/Makefile

+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts remove-scripts
+amd64: build-amd64 mount-scripts remove-scripts
+aarch64: build-aarch64 mount-scripts remove-scripts
+
+copy-scripts:
+	cp -R $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: copy-scripts
+	docker build -t gekkofs/deps:0.9.1 .
+
+build-amd64: copy-scripts
+	docker build --platform amd64 -t gekkofs/deps:0.9.1 .
+
+build-aarch64: copy-scripts
+	docker build --platform aarch64 -t gekkofs/deps:0.9.1 .
+
+remove-scripts:
+	- rm -rf $(CWD)/scripts

docker/0.9.1/docs/Dockerfile

+FROM gekkofs/deps:0.9.1
+
+LABEL Description="Debian-based environment suitable to build GekkoFS' documentation"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        # install dependencies for Doxygen
+        python \
+        flex \
+        bison \
+		graphviz && \
+	# install doxygen (repo version is kind of old)
+	cd /tmp && curl -OL https://www.doxygen.nl/files/doxygen-1.9.2.src.tar.gz && \
+	tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \
+	mkdir -p /tmp/doxygen-1.9.2/build && \
+	cd /tmp/doxygen-1.9.2/build && \
+	cmake -G "Unix Makefiles" .. && \
+	make -j8 install && \
+    # install sphinx, breathe and exhale
+    pip3 install \
+        'sphinx==4.4.0' \
+        sphinx_rtd_theme \
+        'breathe==4.33.1' \
+        'exhale==0.3.1' \
+        'sphinx-copybutton==0.5.0' \
+        'sphinx-multiversion==0.2.4' \
+        'myst_parser==0.17.0' && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/doxygen-1.9.2 && \
+    rm /tmp/doxygen-1.9.2.src.tar.gz && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean

docker/0.9.1/docs/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/docs:0.9.1 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/docs:0.9.1 .
+
+all:
+	docker build -t gekkofs/docs:0.9.1 .
\ No newline at end of file

docker/0.9.0-experimental/linter/Dockerfile → docker/0.9.1/linter/Dockerfile

-FROM gekkofs/core:0.8.0
+FROM gekkofs/core:0.9.1
 
 LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
 
@@ -10,9 +10,9 @@ RUN apt-get update && \
 		software-properties-common \
 		gnupg2 && \
     # add clang-10 repos
-    wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 10 && \
+    wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 11 && \
     # install clang-format
-    apt-get update && apt-get install -y --no-install-recommends clang-format-10 && \
+    apt-get update && apt-get install -y --no-install-recommends clang-format-11 && \
     # Clean apt cache to reduce image layer size
     rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
     # Clean apt caches of packages

docker/0.9.1/linter/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/linter:0.9.1 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/linter:0.9.1 .
+
+all:
+	docker build -t gekkofs/linter:0.9.1 .

docker/0.9.0-experimental/testing/Dockerfile → docker/0.9.1/testing/Dockerfile

-FROM gekkofs/deps:0.9.0-experimental
+FROM gekkofs/deps:0.9.1
 
 LABEL Description="Debian-based environment to test GekkoFS"
 

docker/0.9.1/testing/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/testing:0.9.1 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/testing:0.9.1 .
+
+all:
+	docker build -t gekkofs/testing:0.9.1 .
\ No newline at end of file