Resolve "Bump all dependencies"

Comparison of dependency versions, used for performance comparison. We aim to bump them to newest image

Compatibilities of dependencies with various GCC versions. GekkoFS tests succeeded in all cases: image

Changelog:

  • ofi+tcp support
  • liburing support for RocksDB if available on the system
  • Adding 0.9.0 profiles for dependency upgrades
  • Fix Regex when locating the C++ compiler in compile_dep.sh script
  • Introducing new experimental gkfs_dep.sh script which combines dl_dep.sh and compile_dep.sh
  • Fixing various GCC warnings
  • General CMake improvements and streamlining
  • Re-adding Boost dependency, but only if the Guided Distributor is used
  • Using new margo_init_ext since margo_init_opt is deprecated
  • GekkoFS tests:
    • Support for Python 3.10 and corresponding Pytest fixes
    • Updating bats tests for 0.9.0
  • Hermes submodule update
  • Spdlog update from v1.31 to v1.92
  • Optimization: Reducing RPC handler xstreams to 4
  • Update Readme

Tested:

  • Ubuntu 20.04 from scratch
  • CentOS 8 from scratch

New:

  • ofi+tcp support

Closes #179 (closed) #145 (closed) #46 (closed)

Edited by Marc Vef

Merge request reports

Loading

CMake/gkfs-utils.cmake

0 → 100644
+148 −0
Changes for CMake/gkfs-utils.cmake: 148 added lines, 0 removed lines.
Original line number Diff line number Diff line
################################################################################
# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain            #
# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany          #
#                                                                              #
# This software was partially supported by the                                 #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #
#                                                                              #
# This software was partially supported by the                                 #
# ADA-FS project under the SPPEXA project funded by the DFG.                   #
#                                                                              #
# This file is part of GekkoFS.                                                #
#                                                                              #
# GekkoFS is free software: you can redistribute it and/or modify              #
# it under the terms of the GNU General Public License as published by         #
# the Free Software Foundation, either version 3 of the License, or            #
# (at your option) any later version.                                          #
#                                                                              #
# GekkoFS is distributed in the hope that it will be useful,                   #
# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
# GNU General Public License for more details.                                 #
#                                                                              #
# You should have received a copy of the GNU General Public License            #
# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #
#                                                                              #
# SPDX-License-Identifier: GPL-3.0-or-later                                    #
################################################################################

include(CMakeParseArguments)

#[=======================================================================[.rst:

  get_cmake_variables(OUTPUT_VARIABLE
                     [ REGEX <regular_expression> [EXCLUDE] ])

Initialize ``OUTPUT_VARIABLE`` to a list of all currently defined CMake
variables. The function accepts a ``<regular_expression>`` to allow filtering
the results. Furthermore, if the ``EXCLUDE`` flag is used, the function will
return all variables NOT MATCHING the provided ``<regular_expression>``.
#]=======================================================================]
function(get_cmake_variables OUTPUT_VARIABLE)

  set(OPTIONS EXCLUDE)
  set(SINGLE_VALUE REGEX)
  set(MULTI_VALUE) # no multiple value args for now

  cmake_parse_arguments(
    ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}
  )

  if(ARGS_UNPARSED_ARGUMENTS)
    message(WARNING "Unparsed arguments in get_cmake_variables(): "
                    "this often indicates typos!\n"
                    "Unparsed arguments: ${ARGS_UNPARSED_ARGUMENTS}"
    )
  endif()

  get_cmake_property(_var_names VARIABLES)

  if(NOT ARGS_REGEX)
    set(${OUTPUT_VARIABLE}
        ${_var_names}
        PARENT_SCOPE
    )
    return()
  endif()

  if(ARGS_EXCLUDE)
    set(_mode EXCLUDE)
  else()
    set(_mode INCLUDE)
  endif()

  list(FILTER _var_names ${_mode} REGEX ${ARGS_REGEX})
  set(${OUTPUT_VARIABLE}
      ${_var_names}
      PARENT_SCOPE
  )
endfunction()

#[=======================================================================[.rst:

  dump_cmake_variables([ REGEX <regular_expression> [EXCLUDE] ])

Print all currently defined CMake variables. The function accepts a
``<regular_expression>`` to allow filtering the results. Furthermore, if the
``EXCLUDE`` flag is used, the function will print all variables NOT MATCHING
the provided ``<regular_expression>``.
#]=======================================================================]
function(dump_cmake_variables)

  set(OPTIONS EXCLUDE)
  set(SINGLE_VALUE REGEX)
  set(MULTI_VALUE) # no multiple value args for now

  cmake_parse_arguments(
    ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}
  )

  if(ARGS_UNPARSED_ARGUMENTS)
    message(WARNING "Unparsed arguments in dump_cmake_variables(): "
                    "this often indicates typos!"
                    "Unparsed arguments: ${ARGS_UNPARSED_ARGUMENTS}"
    )
  endif()

  if(ARGS_EXCLUDE AND NOT ARGS_REGEX)
    message(ERROR "EXCLUDE option doesn't make sense without REGEX.")
  endif()

  get_cmake_variables(_var_names REGEX ${ARGS_REGEX} ${ARGS_EXCLUDE})

  foreach(_var ${_var_names})
    message(STATUS "${_var}=${${_var}}")
  endforeach()
endfunction()

#[=======================================================================[.rst:

  mark_variables_as_advanced(REGEX <regular_expression>)

Mark all CMake variables matching ``regular_expression`` as advanced.
#]=======================================================================]
function(mark_variables_as_advanced)

  set(OPTIONS) # no options for now
  set(SINGLE_VALUE REGEX)
  set(MULTI_VALUE) # no multiple value args for now

  cmake_parse_arguments(
    ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}
  )

  if(ARGS_UNPARSED_ARGUMENTS)
    message(WARNING "Unparsed arguments in mark_variables_as_advanced(): "
                    "this often indicates typos!\n"
                    "Unparsed arguments: ${ARGS_UNPARSED_ARGUMENTS}"
    )
  endif()

  get_cmake_property(_var_names VARIABLES)

  list(FILTER _var_names INCLUDE REGEX ${ARGS_REGEX})

  foreach(_var ${_var_names})
    mark_as_advanced(${_var})
  endforeach()
endfunction()
+39 −0
Changes for docker/0.9.0/core/Dockerfile: 39 added lines, 0 removed lines.
Original line number Diff line number Diff line
FROM debian:bullseye-slim

LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"

RUN apt-get update && \
    apt-get install -y --no-install-recommends \
		git \
		curl \
		ca-certificates \
		libtool \
		pkg-config \
		make \
		automake \
		gcc \
		g++ \
		procps \
		# AGIOS dependencies
		libconfig-dev \
		# Mercury dependencies
		libltdl-dev \
		lbzip2 \
        # Margo dependencies \
        libjson-c-dev \
		# RocksDB dependencies
		liblz4-dev \
		# syscall_intercept dependencies
		libcapstone-dev \
		# GekkoFS dependencies
		libboost-program-options-dev \
		uuid-dev && \
    # install cmake 3.14 since it's needed for some dependencies
    curl -OL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh && \
    chmod u+x ./cmake-3.14.5-Linux-x86_64.sh && \
    ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix=/usr && \
    # Clean apt cache to reduce image layer size
    rm -rf /var/lib/apt/lists/* && \
    # Clean apt caches of packages
    apt-get clean && apt-get autoclean && \
    rm ./cmake-3.14.5-Linux-x86_64.sh
+10 −0
Changes for docker/0.9.0/core/Makefile: 10 added lines, 0 removed lines.
Original line number Diff line number Diff line
.PHONY: all

amd64:
	docker build --platform amd64 -t gekkofs/core:0.9.0 .

aarch64:
	docker build --platform aarch64 -t gekkofs/core:0.9.0 .

all:
	docker build -t gekkofs/core:0.9.0 .
+20 −0
Changes for docker/0.9.0/coverage/Dockerfile: 20 added lines, 0 removed lines.
Original line number Diff line number Diff line
FROM debian:bullseye-slim

LABEL Description="Environment to generate coverage reports in GekkoFS"

RUN apt-get update && \
    apt-get install -y --no-install-recommends \
        wget \
        git \
        cmake \
        gcc \
        g++ \
        lcov \
        python3 \
        python3-pip \
        python3-setuptools && \
    rm -rf /var/lib/apt/lists/* && \
    apt-get clean && \
    apt-get autoclean && \
    python3 -m pip install --upgrade pip && \
    pip3 install gcovr
+10 −0
Changes for docker/0.9.0/coverage/Makefile: 10 added lines, 0 removed lines.
Original line number Diff line number Diff line
.PHONY: all

amd64:
	docker build --platform amd64 -t gekkofs/coverage:0.9.0 .

aarch64:
	docker build --platform aarch64 -t gekkofs/coverage:0.9.0 .

all:
	docker build -t gekkofs/coverage:0.9.0 .
 No newline at end of file
Loading
Loading