CI docker tests tmp

.gitlab-ci.yml

@@ -22,14 +22,14 @@ variables:
   GIT_SUBMODULE_STRATEGY:       recursive
 
 # base image
-image: gekkofs/core:0.8.0
+image: gekkofs/core:0.9.0
 
 ################################################################################
 ## Validating
 ################################################################################
 check format:
   stage: lint
-  image: gekkofs/linter:0.8.0
+  image: gekkofs/linter:0.9.0
   needs: []
   script:
     - ${SCRIPTS_DIR}/check_format.sh
@@ -43,7 +43,7 @@ check format:
 ################################################################################
 gkfs:
   stage: build
-  image: gekkofs/deps:0.8.0
+  image: gekkofs/deps:0.9.0
   interruptible: true
   needs: []
   script:
@@ -68,7 +68,7 @@ gkfs:
 
 gkfwd:
   stage: build
-  image: gekkofs/deps:0.8.0
+  image: gekkofs/deps:0.9.0
   interruptible: true
   needs: []
   script:
@@ -101,7 +101,7 @@ gkfwd:
 ## == tests for scripts ====================
 scripts:
   stage: test
-  image: gekkofs/testing:0.8.0
+  image: gekkofs/testing:0.9.0
   needs: []
   parallel:
     matrix:
@@ -119,7 +119,7 @@ scripts:
 ## == integration tests for gkfs ===========
 gkfs:integration:
   stage: test
-  image: gekkofs/testing:0.8.0
+  image: gekkofs/testing:0.9.0
   interruptible: true
   needs: ['gkfs']
   parallel:
@@ -164,7 +164,7 @@ gkfs:integration:
 ## == integration tests for gkfwd ==========
 gkfwd:integration:
   stage: test
-  image: gekkofs/testing:0.8.0
+  image: gekkofs/testing:0.9.0
   interruptible: true
   needs: ['gkfwd']
   parallel:
@@ -209,7 +209,7 @@ gkfwd:integration:
 ## == unit tests for gkfs ==================
 gkfs:unit:
   stage: test
-  image: gekkofs/testing:0.8.0
+  image: gekkofs/testing:0.9.0
   needs: ['gkfs']
   script:
     ## run actual tests
@@ -237,7 +237,7 @@ gkfs:unit:
 ################################################################################
 coverage:
   stage: report
-  image: gekkofs/coverage:0.8.0
+  image: gekkofs/coverage:0.9.0
   needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]
   script:
     - cd ${BUILD_PATH}

docker/0.9.0/core/Dockerfile

+FROM debian:bullseye-slim
+
+LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		git \
+		curl \
+		ca-certificates \
+		libtool \
+		pkg-config \
+		make \
+		automake \
+		gcc \
+		g++ \
+		# AGIOS dependencies
+		libconfig-dev \
+		# Mercury dependencies
+		libltdl-dev \
+		lbzip2 \
+        # Margo dependencies \
+        libjson-c-dev \
+		# RocksDB dependencies
+		liblz4-dev \
+		# syscall_intercept dependencies
+		libcapstone-dev \
+		# GekkoFS dependencies
+		libboost-program-options-dev \
+		uuid-dev && \
+    # install cmake 3.14 since it's needed for some dependencies
+    curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \
+    chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \
+    ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean && \
+    rm ./cmake-3.14.5-Linux-x86_64.sh

docker/0.9.0/core/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/core:0.9.0 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/core:0.9.0 .
+
+all:
+	docker build -t gekkofs/core:0.9.0 .

docker/0.9.0/coverage/Dockerfile

+FROM debian:bullseye-slim
+
+LABEL Description="Environment to generate coverage reports in GekkoFS"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        wget \
+        git \
+        cmake \
+        gcc \
+        g++ \
+        lcov \
+        python3 \
+        python3-pip \
+        python3-setuptools && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean && \
+    apt-get autoclean && \
+    python3 -m pip install --upgrade pip && \
+    pip3 install gcovr

docker/0.9.0/coverage/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/coverage:0.9.0 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/coverage:0.9.0 .
+
+all:
+	docker build -t gekkofs/coverage:0.9.0 .
\ No newline at end of file

docker/0.9.0/deps/Dockerfile

+FROM gekkofs/core:0.9.0
+
+LABEL Description="Debian-based environment to build GekkoFS"
+
+ENV GKFS_PATH	/opt/gkfs
+ENV GKFS_VERSION 0.9.0
+
+ENV SCRIPTS_PATH	${GKFS_PATH}/scripts
+ENV DEPS_SRC_PATH	${GKFS_PATH}/deps_src
+ENV INSTALL_PATH	/usr/local
+
+COPY scripts/dl_dep.sh		${SCRIPTS_PATH}/
+COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/patches        ${SCRIPTS_PATH}/patches
+COPY scripts/profiles       ${SCRIPTS_PATH}/profiles
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		python3 \
+		python3-pip \
+		python3-dev \
+		python3-venv \
+		python3-setuptools && \
+    python3 -m pip install --upgrade pip && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean && apt-get autoclean
+ 
+# Download and build dependencies
+RUN cd ${SCRIPTS_PATH} && \
+    /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \
+    /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \
+    rm -rf ${DEPS_SRC_PATH} && \
+    rm -rf ${SCRIPTS_PATH} && \
+    rmdir ${GKFS_PATH} && \
+    ldconfig

docker/0.9.0/deps/Makefile

+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts remove-scripts
+amd64: build-amd64 mount-scripts remove-scripts
+aarch64: build-aarch64 mount-scripts remove-scripts
+
+copy-scripts:
+	cp -R $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: copy-scripts
+	docker build -t gekkofs/deps:0.9.0 .
+
+build-amd64: copy-scripts
+	docker build --platform amd64 -t gekkofs/deps:0.9.0 .
+
+build-aarch64: copy-scripts
+	docker build --platform aarch64 -t gekkofs/deps:0.9.0 .
+
+remove-scripts:
+	- rm -rf $(CWD)/scripts

docker/0.9.0/linter/Dockerfile

+FROM gekkofs/core:0.9.0
+
+LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		# clang 10 deps for clang-format
+		lsb-release \
+		wget \
+		software-properties-common \
+		gnupg2 && \
+    # add clang-10 repos
+    wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 11 && \
+    # install clang-format
+    apt-get update && apt-get install -y --no-install-recommends clang-format-11 && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean

docker/0.9.0/linter/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/linter:0.9.0 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/linter:0.9.0 .
+
+all:
+	docker build -t gekkofs/linter:0.9.0 .

docker/0.9.0/testing/Dockerfile

+FROM gekkofs/deps:0.9.0
+
+LABEL Description="Debian-based environment to test GekkoFS"
+
+RUN \
+    # install cmake 3.21 since we need to produce JUnit XML files
+    curl -OL https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.sh && \
+    chmod u+x ./cmake-3.21.1-linux-x86_64.sh && \
+    ./cmake-3.21.1-linux-x86_64.sh --skip-license --prefix=/usr && \
+    # install gcovr
+    # (required for partial coverage reports in parallel runs)
+    pip3 install gcovr && \
+    # cleanup
+    rm ./cmake-3.21.1-linux-x86_64.sh

docker/0.9.0/testing/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/testing:0.9.0 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/testing:0.9.0 .
+
+all:
+	docker build -t gekkofs/testing:0.9.0 .
\ No newline at end of file

scripts/check_format.sh

@@ -110,7 +110,7 @@ set -- "${POSITIONAL[@]}" # restore positional parameters
 if [[ -z $CLANG_FORMAT_BIN ]]; then
     CLANG_FORMAT_BIN=$(command -v clang-format)
     if [[ -z $CLANG_FORMAT_BIN ]]; then
-        CLANG_FORMAT_BIN=$(command -v clang-format-10)
+        CLANG_FORMAT_BIN=$(command -v clang-format-11)
         # if it still doesn't exist exit
         if [[ -z $CLANG_FORMAT_BIN ]]; then
             echo "*** ERR: clang-format not found! Exiting ..."

scripts/profiles/0.9.0/ci.specs

@@ -37,16 +37,16 @@ comment="Dependencies required by the CI"
 
 # Dependencies that must be downloaded directly
 wgetdeps=(
-    ["argobots"]="1.0rc1"
-    ["rocksdb"]="6.2.2"
+    ["argobots"]="1.1"
+    ["rocksdb"]="6.26.1"
     )
 
 # Dependencies that must be cloned
 clonedeps=(
-    ["libfabric"]="HEAD@v1.8.1"
-    ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
-    ["margo"]="v0.6.3"
-    ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
+    ["libfabric"]="HEAD@v1.13.2"
+    ["mercury"]="v2.1.0rc4"
+    ["margo"]="v0.9.6"
+    ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
     ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
     ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
     )