diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 68eaca4ed3a2a944e32510b744c9294bf0e4d417..19c076f05914df9170c43104dda58ca13eae5890 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,14 +24,14 @@ variables:
GIT_SUBMODULE_STRATEGY: recursive
# base image
-image: gekkofs/core:0.9.2
+image: gekkofs/core:0.9.3
################################################################################
## Validating
################################################################################
check format:
stage: lint
- image: gekkofs/linter:0.9.2
+ image: gekkofs/linter:0.9.3
needs: []
script:
- ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format:
################################################################################
gkfs:
stage: build
- image: gekkofs/deps:0.9.2
+ image: gekkofs/deps:0.9.3
interruptible: true
needs: []
script:
@@ -65,23 +65,6 @@ gkfs:
- ${INSTALL_PATH}
expire_in: 1 week
-gkfwd:
- stage: build
- image: gekkofs/deps:0.9.2
- interruptible: true
- needs: []
- script:
- - cmake --preset ci-forwarding-coverage
- - cmake --build ${BUILD_PATH} -j $(nproc) --target install
- # reduce artifacts size
- - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}
- artifacts:
- paths:
- - ${BUILD_PATH}
- - ${INSTALL_PATH}
- expire_in: 1 week
-
-
################################################################################
## Testing
################################################################################
@@ -89,7 +72,7 @@ gkfwd:
## == tests for scripts ====================
scripts:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3
needs: []
script:
- mkdir -p ${BUILD_PATH}/tests/scripts
@@ -105,7 +88,7 @@ scripts:
## == integration tests for gkfs ===========
gkfs:integration:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3
interruptible: true
needs: ['gkfs']
parallel:
@@ -154,14 +137,14 @@ gkfs:integration:
## == integration tests for gkfwd ==========
gkfwd:integration:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3
interruptible: true
- needs: ['gkfwd']
+ needs: ['gkfs']
parallel:
matrix:
- SUBTEST: [ forwarding ]
- rules:
- - when: never
+ # rules:
+ # - when: never
script:
## run tests
@@ -204,7 +187,7 @@ gkfwd:integration:
## == unit tests for gkfs ==================
gkfs:unit:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3
needs: ['gkfs']
script:
## Add path to mkfs.kreon
@@ -242,7 +225,7 @@ gkfs:unit:
################################################################################
documentation:
stage: docs
- image: gekkofs/docs:0.9.2
+ image: gekkofs/docs:0.9.3
needs: []
rules:
# we only build the documentation automatically if we are on the
@@ -272,9 +255,9 @@ documentation:
## == coverage baseline ====================
coverage:baseline:
stage: report
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3
interruptible: true
- needs: ['gkfs', 'gkfwd']
+ needs: ['gkfs']
script:
## capture initial coverage information to establish a baseline
@@ -298,10 +281,10 @@ coverage:baseline:
coverage:
stage: report
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3
# needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration',
# 'gkfs:unit' ]
- needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ]
+ needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']
script:
- cd ${CI_PROJECT_DIR}
- cmake
diff --git a/.gitmodules b/.gitmodules
index 18f937ae7b811d4f78fc6fce7f89600199f28abd..2763952410498bb4175ac57565a5f3bfb7d2ea1f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,6 @@
[submodule "external/hermes"]
path = external/hermes
- url = https://github.com/bsc-ssrg/hermes.git
+ url = https://github.com/gekkofs/hermes.git
[submodule "tests/scripts/bats"]
path = tests/scripts/bats
url = https://github.com/bats-core/bats-core.git
@@ -22,3 +22,6 @@
[submodule "external/CLI11"]
path = external/CLI11
url = https://github.com/CLIUtils/CLI11.git
+[submodule "external/MessagePack"]
+ path = external/MessagePack
+ url = https://github.com/GekkoFS/MessagePackCPP.git
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 98ae0762a2456b0efd8172251cd3a3d5d079d7a8..93e0446d56f634b8b9652c0ce5839d69c6593ffc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).
- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server.
- Added reattemp support in get_fs_config to other servers, when the initial server fails.
+- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable
+`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)).
+- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)).
### New
### Changed
diff --git a/CMake/FindMercury.cmake b/CMake/FindMercury.cmake
index 7eb96cde545a7ea7bb863df725f9285368c47a59..451ed6d403ab2a3d4f4f2fb6cabb351179593add 100644
--- a/CMake/FindMercury.cmake
+++ b/CMake/FindMercury.cmake
@@ -177,7 +177,7 @@ endmacro()
include(SelectLibraryConfigurations)
-set(_mercury_components na mercury_util mercury_hl)
+set(_mercury_components na mercury_util)
# prevent repeating work if the main CMakeLists.txt already called
# find_package(PkgConfig)
diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake
index 30da5afc387621004040ca75bfc3f43630722189..398b60ac039573f55c1b7ab3cf072f648e6d1d61 100644
--- a/CMake/gkfs-options.cmake
+++ b/CMake/gkfs-options.cmake
@@ -227,6 +227,13 @@ gkfs_define_option(
DEFAULT_VALUE OFF
)
+# build tools
+gkfs_define_option(
+ GKFS_BUILD_TOOLS
+ HELP_TEXT "Enable ${PROJECT_NAME} tools compilation"
+ DEFAULT_VALUE OFF
+)
+
cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF)
@@ -348,14 +355,6 @@ gkfs_define_option(
# I/O forwarding
################################################################################
-## Forwarding support
-gkfs_define_option(
- GKFS_ENABLE_FORWARDING
- HELP_TEXT "Enable I/O forwarding mode"
- DEFAULT_VALUE OFF
- DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer"
-)
-
## Scheduling in I/O forwarding mode
gkfs_define_option(
GKFS_ENABLE_AGIOS
@@ -364,4 +363,13 @@ gkfs_define_option(
DESCRIPTION "If GKFS_ENABLE_FORWARDING is ON, use AGIOS for scheduling I/Os"
)
+################################################################################
+# MSGPack client metrics
+################################################################################
+gkfs_define_option(
+ GKFS_ENABLE_CLIENT_METRICS
+ HELP_TEXT "Enable client metrics via MSGPack"
+ DEFAULT_VALUE ON # TODO disable by default
+ DESCRIPTION "If GKFS_ENABLE_CLIENT_METRICS is ON, use MSGPack to dump client read/write metrics"
+)
\ No newline at end of file
diff --git a/CMake/load_nlohmann_json.cmake b/CMake/load_nlohmann_json.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..951900998bad144acff05e0845504dd87d3f2a7d
--- /dev/null
+++ b/CMake/load_nlohmann_json.cmake
@@ -0,0 +1,52 @@
+################################################################################
+# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+include(FetchContent)
+
+set(FETCHCONTENT_QUIET OFF)
+
+FetchContent_Declare(nlohmann_json
+ DOWNLOAD_EXTRACT_TIMESTAMP ON
+ URL https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz)
+
+FetchContent_GetProperties(nlohmann_json)
+
+if (NOT nlohmann_json_POPULATED)
+ FetchContent_Populate(nlohmann_json)
+ message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}")
+ message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}")
+
+ # we don't really care so much about a third party library's tests to be
+ # run from our own project's code
+ set(JSON_BuildTests OFF CACHE INTERNAL "")
+
+ # we also don't need to install it when our main project gets installed
+ set(JSON_Install OFF CACHE INTERNAL "")
+
+ add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR})
+endif ()
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index afdda9c02ef8e0137d8bca31f8b2bab90bb8cce2..c0ca49b4dfc330c257f3291da3e5e4e77094817d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,16 +32,17 @@ cmake_minimum_required(VERSION 3.13)
project(
GekkoFS
VERSION 0.9.2
+ LANGUAGES ASM CXX C
)
enable_testing()
-if (NOT CMAKE_COMPILER_IS_GNUCC)
- message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported")
-endif ()
-if (NOT CMAKE_COMPILER_IS_GNUCXX)
- message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported")
-endif ()
+#if (NOT CMAKE_COMPILER_IS_GNUCC)
+# message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported")
+#endif ()
+#if (NOT CMAKE_COMPILER_IS_GNUCXX)
+# message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported")
+#endif ()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -160,16 +161,18 @@ find_package(Margo 0.9.6 REQUIRED)
message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept")
find_package(Syscall_intercept REQUIRED)
-### Date (required by the libgkfs_intercept.so)
-message(STATUS "[${PROJECT_NAME}] Checking for Date")
-find_package(Date REQUIRED)
-
### AGIOS: required for scheduling I/O requests
if (GKFS_ENABLE_AGIOS)
message(STATUS "[${PROJECT_NAME}] Checking for Agios")
find_package(AGIOS REQUIRED)
endif()
+if (GKFS_ENABLE_CLIENT_METRICS)
+ ### zeromq: required for sending client metrics to FTIO
+ message(STATUS "[${PROJECT_NAME}] Checking for cppzmq")
+ find_package(cppzmq)
+endif ()
+
### Metadata backends
if(GKFS_ENABLE_ROCKSDB)
message(STATUS "[${PROJECT_NAME}] Checking for RocksDB")
@@ -225,7 +228,7 @@ include_from_source(fmt
MESSAGE "[${PROJECT_NAME}] Searching for {fmt}"
SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt
GIT_REPOSITORY https://github.com/fmtlib/fmt
- GIT_TAG f94b7364b9409f05207c3af3fa4666730e11a854 # v6.1.2
+ GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0
)
# ensure that fmt is linked as PIC
@@ -247,6 +250,15 @@ include_from_source(cli11
GIT_TAG v2.2.0
)
+if (GKFS_ENABLE_CLIENT_METRICS)
+ ### MessagePack: used for monitoring information on the client
+ include_from_source(MessagePack
+ MESSAGE "[${PROJECT_NAME}] Searching for MessagePackCPP"
+ SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/MessagePack
+ GIT_REPOSITORY https://github.com/GekkoFS/MessagePackCPP
+ GIT_TAG 51655ec8f2b8fed84f685e3bcf8f226b139a263b # latest HEAD
+ )
+endif ()
################################################################################
## Check configured variables/options and act accordingly
@@ -305,10 +317,14 @@ set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include")
include_directories(
${INCLUDE_DIR}
${CMAKE_BINARY_DIR}/include
+ /usr/include
)
add_subdirectory(src)
add_subdirectory(include)
+if (GKFS_BUILD_TOOLS)
+ add_subdirectory(tools)
+endif ()
### Mark any CMake variables imported from {fmt} and spdlog as advanced, so
### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT
@@ -338,11 +354,7 @@ if (GKFS_BUILD_TESTS)
message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")
message(STATUS "[gekkofs] Check for forwarding tests...")
- if (GKFS_ENABLE_FORWARDING)
- set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
- else ()
- set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
- endif ()
+ set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}")
message(STATUS "[gekkofs] Check for guided distributor tests...")
if (GKFS_USE_GUIDED_DISTRIBUTION)
diff --git a/CMakePresets.json b/CMakePresets.json
index 9f12e9c734ceabc71595221f384db23c67ff7bf9..da7bf79b3ef83ca3a0f51bcc014d255dccd8ece6 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -27,14 +27,6 @@
"deprecated": true
}
},
- {
- "name": "forwarding",
- "inherits": "default",
- "hidden": true,
- "cacheVariables": {
- "GKFS_ENABLE_FORWARDING": true
- }
- },
{
"name": "release",
"hidden": true,
@@ -113,30 +105,6 @@
"release"
]
},
- {
- "name": "forwarding-debug",
- "displayName": "Forwarding gekkofs (debug)",
- "inherits": [
- "forwarding",
- "debug"
- ]
- },
- {
- "name": "forwarding-coverage",
- "displayName": "Forwarding gekkofs (coverage)",
- "inherits": [
- "forwarding",
- "coverage"
- ]
- },
- {
- "name": "forwarding-release",
- "displayName": "Forwarding gekkofs (release)",
- "inherits": [
- "forwarding",
- "release"
- ]
- },
{
"name": "ci-debug",
"displayName": "Default gekkofs (debug, CI flags)",
@@ -171,33 +139,6 @@
"default",
"release"
]
- },
- {
- "name": "ci-forwarding-debug",
- "displayName": "Forwarding gekkofs (debug, CI flags)",
- "inherits": [
- "ci",
- "forwarding",
- "debug"
- ]
- },
- {
- "name": "ci-forwarding-coverage",
- "displayName": "Forwarding gekkofs (coverage, CI flags)",
- "inherits": [
- "ci",
- "forwarding",
- "coverage"
- ]
- },
- {
- "name": "ci-forwarding-release",
- "displayName": "Forwarding gekkofs (release, CI flags)",
- "inherits": [
- "ci",
- "forwarding",
- "release"
- ]
}
]
}
diff --git a/config/GekkoFWD/agios/agios.conf b/config/GekkoFWD/agios/agios.conf
index e0fc38d0a854d12d19a73597e2eab05ac3dc9df5..41c747da4e899d62715cefb7dcb19d8cbbfe9780 100644
--- a/config/GekkoFWD/agios/agios.conf
+++ b/config/GekkoFWD/agios/agios.conf
@@ -15,7 +15,7 @@ library_options:
#should the prediction module try to predict aggregations? (requires predict_read_traces = true)
predict_request_aggregation = false ;
- #should the prediction module create simplified traces with information (the metrics) it obtained from the real traces?
+ #should the prediction module create simplified traces with information (the client_metrics) it obtained from the real traces?
predict_write_simplified_traces = false;
#the tolerance for arrival times difference when checking if two predicted requests are the same (in %)
diff --git a/docker/0.9.3/core/Dockerfile b/docker/0.9.3/core/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..a55434924b2cfe720195c371de6736ac049826f7
--- /dev/null
+++ b/docker/0.9.3/core/Dockerfile
@@ -0,0 +1,40 @@
+FROM debian:bullseye-slim
+
+LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ git \
+ curl \
+ ca-certificates \
+ libtool \
+ pkg-config \
+ make \
+ automake \
+ gcc \
+ g++ \
+ ninja-build \
+ procps \
+ # AGIOS dependencies
+ libconfig-dev \
+ # Mercury dependencies
+ libltdl-dev \
+ lbzip2 \
+ # Margo dependencies \
+ libjson-c-dev \
+ # RocksDB dependencies
+ liblz4-dev \
+ # syscall_intercept dependencies
+ libcapstone-dev \
+ # GekkoFS dependencies
+ libboost-program-options-dev \
+ uuid-dev && \
+ # install cmake 3.14+ since it's needed for some dependencies
+ curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \
+ chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \
+ ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean && \
+ rm ./cmake-3.25.2-Linux-x86_64.sh
diff --git a/docker/0.9.3/core/Makefile b/docker/0.9.3/core/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..9c27a583a8ae1069865ad13cc0196214d40b9362
--- /dev/null
+++ b/docker/0.9.3/core/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker buildx build --platform amd64 -t gekkofs/core:0.9.3 .
+
+aarch64:
+ docker buildx build --platform aarch64 -t gekkofs/core:0.9.3 .
+
+all:
+ docker buildx build -t gekkofs/core:0.9.3 .
diff --git a/docker/0.9.3/deps/Dockerfile b/docker/0.9.3/deps/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..ca4ecc8a646db53669fa642eaefdd2bfcc7e8804
--- /dev/null
+++ b/docker/0.9.3/deps/Dockerfile
@@ -0,0 +1,38 @@
+FROM gekkofs/core:0.9.3
+
+LABEL Description="Debian-based environment to build GekkoFS"
+
+ENV GKFS_PATH /opt/gkfs
+ENV GKFS_VERSION 0.9.3
+
+ENV SCRIPTS_PATH ${GKFS_PATH}/scripts
+ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src
+ENV INSTALL_PATH /usr/local
+
+COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/patches ${SCRIPTS_PATH}/patches
+COPY scripts/profiles ${SCRIPTS_PATH}/profiles
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ python3 \
+ python3-pip \
+ python3-dev \
+ python3-venv \
+ python3-setuptools \
+ libnuma-dev libyaml-dev libcurl4-openssl-dev \
+ procps && \
+ python3 -m pip install --upgrade pip && \
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && apt-get autoclean
+
+# Download and build dependencies
+RUN cd ${SCRIPTS_PATH} && \
+ /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \
+ /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \
+ cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\
+ rm -rf ${DEPS_SRC_PATH} && \
+ rm -rf ${SCRIPTS_PATH} && \
+ rmdir ${GKFS_PATH} && \
+ ldconfig
diff --git a/docker/0.9.3/deps/Makefile b/docker/0.9.3/deps/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..a97d3195f66a9cf7c5b44ed7b32d25abb49e6ab0
--- /dev/null
+++ b/docker/0.9.3/deps/Makefile
@@ -0,0 +1,23 @@
+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts remove-scripts
+amd64: build-amd64 mount-scripts remove-scripts
+aarch64: build-aarch64 mount-scripts remove-scripts
+
+copy-scripts:
+ cp -R $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: copy-scripts
+ docker build -t gekkofs/deps:0.9.3 .
+
+build-amd64: copy-scripts
+ docker build --platform amd64 -t gekkofs/deps:0.9.3 .
+
+build-aarch64: copy-scripts
+ docker build --platform aarch64 -t gekkofs/deps:0.9.3 .
+
+remove-scripts:
+ - rm -rf $(CWD)/scripts
diff --git a/docker/0.9.3/docs/Dockerfile b/docker/0.9.3/docs/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..f770f3a23de51eb7ee8f13d0f3b8ebf8623d2649
--- /dev/null
+++ b/docker/0.9.3/docs/Dockerfile
@@ -0,0 +1,34 @@
+FROM gekkofs/deps:0.9.3
+
+LABEL Description="Debian-based environment suitable to build GekkoFS' documentation"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # install dependencies for Doxygen
+ python \
+ flex \
+ bison \
+ graphviz && \
+ # install doxygen (repo version is kind of old)
+ cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \
+ tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \
+ mkdir -p /tmp/doxygen-1.9.2/build && \
+ cd /tmp/doxygen-1.9.2/build && \
+ cmake -G "Unix Makefiles" .. && \
+ make -j8 install && \
+ # install sphinx, breathe and exhale
+ pip3 install \
+ 'sphinx==4.4.0' \
+ sphinx_rtd_theme \
+ 'breathe==4.33.1' \
+ 'exhale==0.3.1' \
+ 'sphinx-copybutton==0.5.0' \
+ 'sphinx-multiversion==0.2.4' \
+ 'myst_parser==0.17.0' \
+ attrs && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && \
+ rm -rf /tmp/doxygen-1.9.2 && \
+ rm /tmp/doxygen-1.9.2.src.tar.gz && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.9.3/docs/Makefile b/docker/0.9.3/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..c6c89926af84c32ee55331a647a3d041ea6e0555
--- /dev/null
+++ b/docker/0.9.3/docs/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/docs:0.9.3 .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/docs:0.9.3 .
+
+all:
+ docker build -t gekkofs/docs:0.9.3 .
\ No newline at end of file
diff --git a/docker/0.9.3/linter/Dockerfile b/docker/0.9.3/linter/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..f5d57118e518d36fb28f7a0f2bc95716dc99f2d0
--- /dev/null
+++ b/docker/0.9.3/linter/Dockerfile
@@ -0,0 +1,19 @@
+FROM gekkofs/core:0.9.3
+
+LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # clang 15 deps for clang-format
+ lsb-release \
+ wget \
+ software-properties-common \
+ gnupg2 && \
+ # add clang-15 repos
+ wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \
+ # install clang-format
+ apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.9.3/linter/Makefile b/docker/0.9.3/linter/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..f427eb387a7f8aebffa6a4522c428467835903e4
--- /dev/null
+++ b/docker/0.9.3/linter/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker buildx build --platform amd64 -t gekkofs/linter:0.9.3 .
+
+aarch64:
+ docker buildx build --platform aarch64 -t gekkofs/linter:0.9.3 .
+
+all:
+ docker buildx build -t gekkofs/linter:0.9.3 .
diff --git a/docker/0.9.3/testing/Dockerfile b/docker/0.9.3/testing/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..971944a86b0c5102eba84a912b33a53187078701
--- /dev/null
+++ b/docker/0.9.3/testing/Dockerfile
@@ -0,0 +1,32 @@
+FROM gekkofs/deps:0.9.3
+
+LABEL Description="Debian-based environment to test GekkoFS"
+
+RUN \
+ apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # required by lcov's genhtml
+ libgd-perl && \
+ # install cmake 3.21+ since we need to produce JUnit XML files
+ curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \
+ chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \
+ ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \
+ # install loguru
+ # (required by several of our scripts)
+ pip3 install loguru && \
+ # install lcov_cobertura
+ # (required to produce Cobertura XML reports)
+ pip3 install lcov_cobertura && \
+ # install lcov
+ # (required to produce partial coverage reports in parallel runs)
+ curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \
+ tar xfz lcov-1.16.tar.gz && \
+ cd lcov-1.16 && \
+ make install && \
+ cd .. && \
+ # cleanup
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && \
+ apt-get autoclean && \
+ rm ./cmake-3.25.2-Linux-x86_64.sh && \
+ rm -rf ./lcov-1.16.*
diff --git a/docker/0.9.3/testing/Makefile b/docker/0.9.3/testing/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..b5908b5399d36a19a96cc61a8f929fd9a6c4a581
--- /dev/null
+++ b/docker/0.9.3/testing/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/testing:0.9.3 .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/testing:0.9.3 .
+
+all:
+ docker build -t gekkofs/testing:0.9.3 .
diff --git a/docs/sphinx/users/dependencies.rst b/docs/sphinx/users/dependencies.rst
index 0d3732410f1b133cc755047a1df1ad2601fa2f0b..3490ef5d6300b3153f35f37f5ff252854252e306 100644
--- a/docs/sphinx/users/dependencies.rst
+++ b/docs/sphinx/users/dependencies.rst
@@ -43,4 +43,4 @@ Required
Optional
--------
-- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode.
+- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling.
diff --git a/docs/sphinx/users/forwarding.rst b/docs/sphinx/users/forwarding.rst
index db0afc7b22e587f629bcacadfde6de82c62f7e80..e850b4d04d7b4649fd7b600f3dd9cd6c7dbad8f5 100644
--- a/docs/sphinx/users/forwarding.rst
+++ b/docs/sphinx/users/forwarding.rst
@@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes.
Enabling GekkoFWD
------------------
-To enable the I/O forwarding mode of GekkoFS, the
-:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when
-configuring ` the build:
-
-.. code-block:: console
-
- $ cmake -DENABLE_FORWARDING:BOOL=ON
+To enable the I/O forwarding mode of GekkoFS, an environment
+variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided
+to allow users to identify the map file on each client.
+This environment variable will enable GekkoFS forwarding, then we
+only need to use :code:`--enable-forwarding`` option in the servers.
I/O Scheduling
--------------
diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt
index d9e63e317b8045762757231c1196b0a973faca7a..d4788ddd03ea5a92df9b7b407420a64547519df2 100644
--- a/examples/gfind/CMakeLists.txt
+++ b/examples/gfind/CMakeLists.txt
@@ -26,11 +26,19 @@
# SPDX-License-Identifier: GPL-3.0-or-later #
################################################################################
-set (CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
add_executable(sfind sfind.cpp)
+add_executable(gkfs_lib_example gkfs_lib_example.cpp)
+
+target_link_libraries(gkfs_lib_example
+ PRIVATE gkfs_user_lib
+)
if(GKFS_INSTALL_TESTS)
install(TARGETS sfind
DESTINATION ${CMAKE_INSTALL_BINDIR}
)
+ install(TARGETS gkfs_lib_example
+ DESTINATION ${CMAKE_INSTALL_BINDIR}
+ )
endif()
diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f064e3e3347dfdb82a065de50ba2cfecd47db545
--- /dev/null
+++ b/examples/gfind/gkfs_lib_example.cpp
@@ -0,0 +1,115 @@
+/*
+ Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
+ Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
+
+ This software was partially supported by the
+ EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+ This software was partially supported by the
+ ADA-FS project under the SPPEXA project funded by the DFG.
+
+ This file is part of GekkoFS.
+
+ GekkoFS is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ GekkoFS is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GekkoFS. If not, see .
+
+ SPDX-License-Identifier: GPL-3.0-or-later
+*/
+
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+using namespace std;
+
+void
+write_file(std::string filename) {
+ // Open File
+ int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT);
+
+ cout << "FD open " << fd << endl;
+
+ char* bufwrite = (char*) malloc(10);
+ strncpy(bufwrite, "testing\0", 8);
+
+
+ int size = gkfs::syscall::gkfs_write(fd, bufwrite, 7);
+
+ cout << "Writting size " << size << endl;
+
+ free(bufwrite);
+ gkfs::syscall::gkfs_close(fd);
+}
+
+
+void
+read_file(std::string filename) {
+ int fdread = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDONLY);
+ if(fdread == -1)
+ return;
+ char* bufread = (char*) malloc(10);
+ int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7);
+
+ cout << "Reading Size: " << sizeread << " Content: " << bufread << endl;
+
+ free(bufread);
+ gkfs::syscall::gkfs_close(fdread);
+}
+
+int
+main(int argc, char** argv) {
+ cout << "GekkoFS Client library test" << endl;
+
+ auto res = gkfs_init();
+
+ cout << "Init result " << res << endl;
+
+ write_file("/test.tmp");
+
+ read_file("/test.tmp");
+
+
+ write_file("/secondfile.tmp");
+
+ auto f_list = gkfs::syscall::gkfs_get_file_list("/");
+
+ for(auto f : f_list) {
+ cout << "File: " << f << endl;
+ struct stat buf;
+ memset(&buf, 0, sizeof(struct stat));
+
+ gkfs::syscall::gkfs_stat("/" + f, &buf, true);
+
+ cout << "Size: " << buf.st_size << " Mode: " << buf.st_mode << endl;
+ cout << "Atime: " << buf.st_atime << " Mtime: " << buf.st_mtime
+ << " Ctime: " << buf.st_ctime << endl
+ << " ****** " << endl;
+ }
+
+ res = gkfs_end();
+
+ cout << "End result " << res << endl;
+}
diff --git a/external/MessagePack b/external/MessagePack
new file mode 160000
index 0000000000000000000000000000000000000000..26527c71eb74fdc625ea994fa5caf45db318f7f1
--- /dev/null
+++ b/external/MessagePack
@@ -0,0 +1 @@
+Subproject commit 26527c71eb74fdc625ea994fa5caf45db318f7f1
diff --git a/external/hermes b/external/hermes
index d8755039c82501323cdc9a4f5a2b922b9b3bcac7..a133c9dd0cefc7e8a3554badfb8995c9f2c27be6 160000
--- a/external/hermes
+++ b/external/hermes
@@ -1 +1 @@
-Subproject commit d8755039c82501323cdc9a4f5a2b922b9b3bcac7
+Subproject commit a133c9dd0cefc7e8a3554badfb8995c9f2c27be6
diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt
index 4cc56ac387e724c02d549d4e5d4774a68c62be61..1ae9e3b40e76552f804dc6c3cd4c7dd053c291a3 100644
--- a/include/CMakeLists.txt
+++ b/include/CMakeLists.txt
@@ -34,7 +34,3 @@ add_subdirectory(daemon)
add_subdirectory(client)
target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in)
-
-if(GKFS_ENABLE_FORWARDING)
- target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in)
-endif()
diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt
index f5df6ee5c796e2dd72c909ac4e8b1b15bb3912c2..6815fc95321a27ecce55b407d10075cb53cad2ca 100644
--- a/include/client/CMakeLists.txt
+++ b/include/client/CMakeLists.txt
@@ -54,30 +54,30 @@ target_sources(
syscalls/detail/syscall_info.h
)
-if(GKFS_ENABLE_FORWARDING)
- target_sources(
- gkfwd_intercept
- PUBLIC gkfs_functions.hpp
- env.hpp
- hooks.hpp
- intercept.hpp
- logging.hpp
- make_array.hpp
- open_file_map.hpp
- open_dir.hpp
- path.hpp
- preload.hpp
- preload_context.hpp
- preload_util.hpp
- rpc/rpc_types.hpp
- rpc/forward_management.hpp
- rpc/forward_metadata.hpp
- rpc/forward_data.hpp
- syscalls/args.hpp
- syscalls/decoder.hpp
- syscalls/errno.hpp
- syscalls/rets.hpp
- syscalls/syscall.hpp
- syscalls/detail/syscall_info.h
- )
-endif()
+target_sources(
+ gkfs_user_lib
+ PUBLIC gkfs_functions.hpp
+ env.hpp
+ hooks.hpp
+ intercept.hpp
+ logging.hpp
+ make_array.hpp
+ open_file_map.hpp
+ open_dir.hpp
+ path.hpp
+ preload.hpp
+ preload_context.hpp
+ preload_util.hpp
+ rpc/rpc_types.hpp
+ rpc/forward_management.hpp
+ rpc/forward_metadata.hpp
+ rpc/forward_data.hpp
+ syscalls/args.hpp
+ syscalls/decoder.hpp
+ syscalls/errno.hpp
+ syscalls/rets.hpp
+ syscalls/syscall.hpp
+ syscalls/detail/syscall_info.h
+ void_syscall_intercept.hpp
+ user_functions.hpp
+)
diff --git a/include/client/env.hpp b/include/client/env.hpp
index 34a986facb0855c117370a2871fc36527c77d5df..4f24d9ab13b0a3d910d37006bbcac39a92d672dc 100644
--- a/include/client/env.hpp
+++ b/include/client/env.hpp
@@ -49,9 +49,15 @@ static constexpr auto LOG_PER_PROCESS = ADD_PREFIX("LOG_PER_PROCESS");
static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC");
static constexpr auto CWD = ADD_PREFIX("CWD");
static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE");
-#ifdef GKFS_ENABLE_FORWARDING
static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+static constexpr auto ENABLE_METRICS = ADD_PREFIX("ENABLE_METRICS");
+static constexpr auto METRICS_FLUSH_INTERVAL =
+ ADD_PREFIX("METRICS_FLUSH_INTERVAL");
+static constexpr auto METRICS_PATH = ADD_PREFIX("METRICS_PATH");
+static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT");
#endif
+
static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");
} // namespace gkfs::env
diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp
index 63b9b655f55d5851d780f0c5379b4b35d7225533..c616e0c0a0de6e43bfae9e1f5cd0ba8c9ad64c79 100644
--- a/include/client/gkfs_functions.hpp
+++ b/include/client/gkfs_functions.hpp
@@ -105,11 +105,13 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize);
#endif
ssize_t
-gkfs_pwrite(std::shared_ptr file, const char* buf,
- size_t count, off64_t offset, bool update_pos = false);
+gkfs_do_write(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos);
ssize_t
-gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset);
+gkfs_write_ws(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos = false);
+
+ssize_t
+gkfs_pwrite(int fd, const void* buf, size_t count, off64_t offset);
ssize_t
gkfs_write(int fd, const void* buf, size_t count);
@@ -121,11 +123,13 @@ ssize_t
gkfs_writev(int fd, const struct iovec* iov, int iovcnt);
ssize_t
-gkfs_pread(std::shared_ptr file, char* buf,
- size_t count, off64_t offset);
+gkfs_do_read(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset);
ssize_t
-gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset);
+gkfs_read_ws(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset);
+
+ssize_t
+gkfs_pread(int fd, void* buf, size_t count, off64_t offset);
ssize_t
gkfs_read(int fd, void* buf, size_t count);
@@ -149,14 +153,22 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp,
int
gkfs_rmdir(const std::string& path);
+int
+gkfs_close(unsigned int fd);
+
+std::vector
+gkfs_get_file_list(const std::string& path);
+
#ifdef HAS_RENAME
int
gkfs_rename(const std::string& old_path, const std::string& new_path);
#endif // HAS_RENAME
+
} // namespace gkfs::syscall
// gkfs_getsingleserverdir is using extern "C" to demangle it for C usage
extern "C" int
gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp,
unsigned int count, int server);
+
#endif // GEKKOFS_GKFS_FUNCTIONS_HPP
diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp
index e71a6a08d68b2b0875b213b7f19e260a13c73b2b..fa9ec424bc606a941650edb253a20781af54cf8d 100644
--- a/include/client/hooks.hpp
+++ b/include/client/hooks.hpp
@@ -36,7 +36,13 @@ extern "C" {
#include
#include
}
+
+#ifndef BYPASS_SYSCALL
#include
+#else
+#include
+#endif
+
/*
* For PowerPC, syscall_no_intercept_wrapper() is defined in the
diff --git a/include/client/logging.hpp b/include/client/logging.hpp
index c99e2e4a034cf5567008ba1fa16ea6f7219105f9..9d1f464e52649e33268769b9a2cadea711b31fee 100644
--- a/include/client/logging.hpp
+++ b/include/client/logging.hpp
@@ -30,7 +30,11 @@
#ifndef LIBGKFS_LOGGING_HPP
#define LIBGKFS_LOGGING_HPP
+#ifndef BYPASS_SYSCALL
#include
+#else
+#include
+#endif
#include
#include
@@ -38,7 +42,6 @@
#include
#include
#include
-#include
#include
#ifdef GKFS_DEBUG_BUILD
@@ -175,6 +178,138 @@ log_buffer(int fd, const void* buffer, std::size_t length) {
::syscall_no_intercept(SYS_write, fd, buffer, length);
}
+/**
+ * @brief convert a time_t to a tm
+ * It is not POSIX compliant, but it dows not uses any syscall or timezone
+ * Converts a Unix timestamp (number of seconds since the beginning of 1970
+ * CE) to a Gregorian civil date-time tuple in GMT (UTC) time zone.
+ *
+ * This conforms to C89 (and C99...) and POSIX.
+ *
+ * This implementation works, and doesn't overflow for any sizeof(time_t).
+ * It doesn't check for overflow/underflow in tm->tm_year output. Other than
+ * that, it never overflows or underflows. It assumes that that time_t is
+ * signed.
+ *
+ * This implements the inverse of the POSIX formula
+ * (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_15)
+ * for all time_t values, no matter the size, as long as tm->tm_year doesn't
+ * overflow or underflow. The formula is: tm_sec + tm_min*60 + tm_hour*3600
+ * + tm_yday*86400 + (tm_year-70)*31536000 + ((tm_year-69)/4)*86400 -
+ * ((tm_year-1)/100)*86400 + ((tm_year+299)/400)*86400.
+ *
+ * License : GNU General Public License v2.0 from
+ * https://github.com/pts/minilibc686/
+ * @param time_t
+ * @return tm
+ */
+
+static inline struct tm*
+mini_gmtime_r(const time_t* timep, struct tm* tm) {
+ const time_t ts = *timep;
+ time_t t = ts / 86400;
+ unsigned hms =
+ ts %
+ 86400; /* -86399 <= hms <= 86399. This needs sizeof(int) >= 4. */
+ time_t c, f;
+ unsigned yday; /* 0 <= yday <= 426. Also fits to an `unsigned short', but
+ `int' is faster. */
+ unsigned a; /* 0 <= a <= 2133. Also fits to an `unsigned short', but `int'
+ is faster. */
+ if((int) hms < 0) {
+ --t;
+ hms += 86400;
+ } /* Fix quotient and negative remainder if ts was negative (i.e. before
+ year 1970 CE). */
+ /* Now: -24856 <= t <= 24855. */
+ tm->tm_sec = hms % 60;
+ hms /= 60;
+ tm->tm_min = hms % 60;
+ tm->tm_hour = hms / 60;
+ if(sizeof(time_t) >
+ 4) { /* Optimization. For int32_t, this would keep t intact, so we won't
+ have to do it. This produces unreachable code. */
+ f = (t + 4) % 7;
+ if(f < 0)
+ f += 7; /* Fix negative remainder if (t + 4) was negative. */
+ /* Now 0 <= f <= 6. */
+ tm->tm_wday = f;
+ c = (t << 2) + 102032;
+ f = c / 146097;
+ if(c % 146097 < 0)
+ --f; /* Fix negative remainder if c was negative. */
+ --f;
+ t += f;
+ f >>= 2;
+ t -= f;
+ f = (t << 2) + 102035;
+ c = f / 1461;
+ if(f % 1461 < 0)
+ --c; /* Fix negative remainder if f was negative. */
+ } else {
+ tm->tm_wday = (t + 24861) % 7; /* t + 24861 >= 0. */
+ /* Now: -24856 <= t <= 24855. */
+ c = ((t << 2) + 102035) / 1461;
+ }
+ yday = t - 365 * c - (c >> 2) + 25568;
+ /* Now: 0 <= yday <= 425. */
+ a = yday * 5 + 8;
+ /* Now: 8 <= a <= 2133. */
+ tm->tm_mon = a / 153;
+ a %= 153; /* No need to fix if a < 0, because a cannot be negative here. */
+ /* Now: 2 <= tm->tm_mon <= 13. */
+ /* Now: 0 <= a <= 152. */
+ tm->tm_mday = 1 + a / 5; /* No need to fix if a < 0, because a cannot be
+ negative here. */
+ /* Now: 1 <= tm->tm_mday <= 31. */
+ if(tm->tm_mon >= 12) {
+ tm->tm_mon -= 12;
+ /* Now: 0 <= tm->tm_mon <= 1. */
+ ++c;
+ yday -= 366;
+ } else { /* Check for leap year (in c). */
+ /* Now: 2 <= tm->tm_mon <= 11. */
+ /* 1903: not leap; 1904: leap, 1900: not leap; 2000: leap */
+ /* With sizeof(time_t) == 4, we have 1901 <= year <= 2038; of these
+ * years only 2000 is divisble by 100, and that's a leap year, no we
+ * optimize the check to `(c & 3) == 0' only.
+ */
+ if(!((c & 3) == 0 &&
+ (sizeof(time_t) <= 4 || c % 100 != 0 || (c + 300) % 400 == 0)))
+ --yday; /* These `== 0' comparisons work even if c < 0. */
+ }
+ tm->tm_year =
+ c; /* This assignment may overflow or underflow, we don't check it.
+ Example: time_t is a huge int64_t, tm->tm_year is int32_t. */
+ /* Now: 0 <= tm->tm_mon <= 11. */
+ /* Now: 0 <= yday <= 365. */
+ tm->tm_yday = yday;
+ tm->tm_isdst = 0;
+ return tm;
+}
+
+static inline struct tm*
+mini_gmtime(const time_t* timep) {
+ static struct tm tm;
+ return mini_gmtime_r(timep, &tm);
+}
+
+static inline ssize_t
+format_timeval(struct timeval* tv, char* buf, size_t sz) {
+ ssize_t written = -1;
+ struct tm* gm = mini_gmtime(&tv->tv_sec);
+
+
+ written = (ssize_t) strftime(buf, sz, "%Y-%m-%d %H:%M:%S", gm);
+ if((written > 0) && ((size_t) written < sz)) {
+ int w = snprintf(buf + written, sz - (size_t) written, ".%06ld",
+ tv->tv_usec);
+ written = (w > 0) ? written + w : -1;
+ }
+
+ return written;
+}
+
/**
* format_timestamp_to - safely format a timestamp for logging messages
*
@@ -192,13 +327,10 @@ log_buffer(int fd, const void* buffer, std::size_t length) {
* one thread exactly, and we pass it as an argument whenever we need to
* format a timestamp. If no timezone is provided, we just format the epoch.
*
- * NOTE: we use the date C++ library to query the timezone database and
- * to format the timestamps.
*/
template
static inline void
-format_timestamp_to(Buffer&& buffer,
- const date::time_zone* const timezone = nullptr) {
+format_timestamp_to(Buffer&& buffer) {
struct ::timeval tv;
@@ -208,17 +340,11 @@ format_timestamp_to(Buffer&& buffer,
return;
}
- date::sys_time now{
- std::chrono::seconds{tv.tv_sec} +
- std::chrono::microseconds{tv.tv_usec}};
+ char buf[28];
- if(!timezone) {
- fmt::format_to(buffer, "[{}] ", now.time_since_epoch().count());
- return;
+ if(format_timeval(&tv, buf, sizeof(buf)) > 0) {
+ fmt::format_to(std::back_inserter(buffer), "[{}] ", buf);
}
-
- fmt::format_to(buffer, "[{}] ",
- date::zoned_time{timezone, now});
}
template
@@ -226,7 +352,7 @@ static inline void
format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) {
const auto ttid = syscall_no_intercept(SYS_gettid);
- fmt::format_to(buffer, "[{}] [syscall] ", ttid);
+ fmt::format_to(std::back_inserter(buffer), "[{}] [syscall] ", ttid);
char o;
char t;
@@ -256,20 +382,15 @@ format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) {
}
const std::array tmp = {'[', o, t, ']', ' '};
- fmt::format_to(buffer, fmt::string_view(tmp.data(), tmp.size()));
+ fmt::format_to(std::back_inserter(buffer),
+ fmt::string_view(tmp.data(), tmp.size()));
}
} // namespace detail
enum { max_buffer_size = LIBGKFS_LOG_MESSAGE_SIZE };
-struct static_buffer : public fmt::basic_memory_buffer {
-
-protected:
- void
- grow(std::size_t size) override final;
-};
-
+using static_buffer = fmt::basic_memory_buffer;
struct logger {
@@ -293,16 +414,17 @@ struct logger {
}
static_buffer buffer;
- detail::format_timestamp_to(buffer, timezone_);
- fmt::format_to(buffer, "[{}] [{}] ", log_process_id_,
- lookup_level_name(level));
+ detail::format_timestamp_to(buffer);
+ fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ",
+ log_process_id_, lookup_level_name(level));
if(!!(level & log::debug)) {
- fmt::format_to(buffer, "<{}():{}> ", func, lineno);
+ fmt::format_to(std::back_inserter(buffer), "<{}():{}> ", func,
+ lineno);
}
- fmt::format_to(buffer, std::forward(args)...);
- fmt::format_to(buffer, "\n");
+ fmt::format_to(std::back_inserter(buffer), std::forward(args)...);
+ fmt::format_to(std::back_inserter(buffer), "\n");
detail::log_buffer(log_fd_, buffer);
}
@@ -337,8 +459,8 @@ struct logger {
static_buffer prefix;
detail::format_timestamp_to(prefix);
- fmt::format_to(prefix, "[{}] [{}] ", log_process_id_,
- lookup_level_name(level));
+ fmt::format_to(std::back_inserter(prefix), "[{}] [{}] ",
+ log_process_id_, lookup_level_name(level));
char buffer[max_buffer_size];
const int n = vsnprintf(buffer, sizeof(buffer), fmt, ap);
@@ -387,8 +509,8 @@ struct logger {
}
static_buffer buffer;
- fmt::format_to(buffer, std::forward(args)...);
- fmt::format_to(buffer, "\n");
+ fmt::format_to(std::back_inserter(buffer), std::forward(args)...);
+ fmt::format_to(std::back_inserter(buffer), "\n");
detail::log_buffer(fd, buffer);
}
@@ -410,8 +532,6 @@ struct logger {
std::bitset<512> filtered_syscalls_;
int debug_verbosity_;
#endif
-
- const date::time_zone* timezone_;
};
// the following static functions can be used to interact
@@ -440,23 +560,6 @@ destroy_global_logger() {
logger::global_logger().reset();
}
-inline void
-static_buffer::grow(std::size_t size) {
-
- const auto logger = get_global_logger();
-
- if(logger) {
- logger->log_mask_ &= ~(syscall | syscall_at_entry);
- }
-
- std::fprintf(
- stderr,
- "FATAL: message too long for gkfs::log::static_buffer, increase the size of\n"
- "LIBGKFS_LOG_MESSAGE_SIZE in CMake or reduce the length of the offending "
- "message.\n");
- abort();
-}
-
} // namespace gkfs::log
#define LOG(XXX, ...) LOG_##XXX(__VA_ARGS__)
diff --git a/include/client/preload.hpp b/include/client/preload.hpp
index 175cd8f6edd89e441993f2055bb9eb325e611e24..afad97c684bbb5736bceef4079e67adc7b120b3c 100644
--- a/include/client/preload.hpp
+++ b/include/client/preload.hpp
@@ -31,7 +31,8 @@
#define IOINTERCEPT_PRELOAD_HPP
#include
-
+#include
+#include
#define EUNKNOWN (-1)
#define CTX gkfs::preload::PreloadContext::getInstance()
@@ -40,11 +41,12 @@ void
init_ld_env_if_needed();
} // namespace gkfs::preload
+#ifndef BYPASS_SYSCALL
void
init_preload() __attribute__((constructor));
void
destroy_preload() __attribute__((destructor));
-
+#endif
#endif // IOINTERCEPT_PRELOAD_HPP
diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp
index 1fa0b1925e4c3e18fb464bfd4fef939ea5d4c973..f73288032085456ca2a64506fa582c044276dd25 100644
--- a/include/client/preload_context.hpp
+++ b/include/client/preload_context.hpp
@@ -51,6 +51,9 @@ class Distributor;
namespace log {
struct logger;
}
+namespace messagepack {
+class ClientMetrics;
+}
namespace preload {
/*
@@ -107,6 +110,10 @@ private:
std::string hostname;
int replicas_;
+ std::shared_ptr write_metrics_;
+ std::shared_ptr read_metrics_;
+
+
public:
static PreloadContext*
getInstance() {
@@ -119,9 +126,14 @@ public:
void
operator=(PreloadContext const&) = delete;
+ ~PreloadContext();
+
void
init_logging();
+ bool
+ init_metrics();
+
void
mountdir(const std::string& path);
@@ -223,6 +235,12 @@ public:
int
get_replicas();
+
+ const std::shared_ptr
+ write_metrics();
+
+ const std::shared_ptr
+ read_metrics();
};
} // namespace preload
diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp
index a3fa9b64a1104d892696af9bf5daf3c4eb674a64..4c756dec22d8fefbe3aeac29c2ed6c6bf6c6e6dd 100644
--- a/include/client/rpc/rpc_types.hpp
+++ b/include/client/rpc/rpc_types.hpp
@@ -85,10 +85,10 @@ struct fs_config {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 3033006080;
+ constexpr static const uint64_t public_id = 1;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::fs_config;
@@ -259,10 +259,10 @@ struct create {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 796590080;
+ constexpr static const uint64_t public_id = 2;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::create;
@@ -375,10 +375,10 @@ struct stat {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 1396244480;
+ constexpr static const uint64_t public_id = 3;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::stat;
@@ -494,10 +494,10 @@ struct remove_metadata {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 2087845888;
+ constexpr static const uint64_t public_id = 4;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::remove_metadata;
@@ -618,10 +618,10 @@ struct decr_size {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 1291649024;
+ constexpr static const uint64_t public_id = 5;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::decr_size;
@@ -734,10 +734,10 @@ struct update_metadentry {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 99483648;
+ constexpr static const uint64_t public_id = 6;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::update_metadentry;
@@ -960,10 +960,10 @@ struct get_metadentry_size {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 3426484224;
+ constexpr static const uint64_t public_id = 7;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::get_metadentry_size;
@@ -1076,10 +1076,10 @@ struct update_metadentry_size {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 2760900608;
+ constexpr static const uint64_t public_id = 8;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::update_metadentry_size;
@@ -1216,10 +1216,10 @@ struct mk_symlink {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 3207004160;
+ constexpr static const uint64_t public_id = 9;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::mk_symlink;
@@ -1334,10 +1334,10 @@ struct remove_data {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 2649292800;
+ constexpr static const uint64_t public_id = 10;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::remove_data;
@@ -1442,10 +1442,10 @@ struct write_data {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 3716481024;
+ constexpr static const uint64_t public_id = 11;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::write;
@@ -1627,10 +1627,10 @@ struct read_data {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 892207104;
+ constexpr static const uint64_t public_id = 12;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::read;
@@ -1812,10 +1812,10 @@ struct trunc_data {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 1850933248;
+ constexpr static const uint64_t public_id = 13;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::truncate;
@@ -1931,10 +1931,10 @@ struct get_dirents {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 4121034752;
+ constexpr static const uint64_t public_id = 14;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::get_dirents;
@@ -2055,10 +2055,10 @@ struct get_dirents_extended {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 1463943168;
+ constexpr static const uint64_t public_id = 15;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::get_dirents_extended;
@@ -2181,10 +2181,10 @@ struct chunk_stat {
// RPC public identifier
// (N.B: we reuse the same IDs assigned by Margo so that the daemon
// understands Hermes RPCs)
- constexpr static const uint64_t public_id = 532742144;
+ constexpr static const uint64_t public_id = 16;
// RPC internal Mercury identifier
- constexpr static const hg_id_t mercury_id = public_id;
+ constexpr static const hg_id_t mercury_id = 0;
// RPC name
constexpr static const auto name = gkfs::rpc::tag::get_chunk_stat;
diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp
index 05fa8296337d7ab2cfe689f394dc1d8365bdb61b..53ed726c3b1484bc7ec64b92508e26bf908ed73a 100644
--- a/include/client/syscalls/args.hpp
+++ b/include/client/syscalls/args.hpp
@@ -228,7 +228,7 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) {
// we assume that if a flag value is zero, its printable
// name will always be at position 0 in the array
if(flag == 0 && desc[0].flag_ == 0) {
- fmt::format_to(buffer, "{}", desc[0].name_);
+ fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_);
return;
}
@@ -239,12 +239,12 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) {
}
if((flag == desc[i].flag_)) {
- fmt::format_to(buffer, "{}", desc[i].name_);
+ fmt::format_to(std::back_inserter(buffer), "{}", desc[i].name_);
return;
}
}
- fmt::format_to(buffer, "{:#x}", flag);
+ fmt::format_to(std::back_inserter(buffer), "{:#x}", flag);
}
template
@@ -254,7 +254,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) {
// we assume that if a flag value is zero, its printable
// name will always be at position 0 in the array
if(flags == 0 && desc[0].flag_ == 0) {
- fmt::format_to(buffer, "{}", desc[0].name_);
+ fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_);
return;
}
@@ -269,7 +269,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) {
}
if((flags & desc[i].flag_) != 0) {
- fmt::format_to(buffer, "{}{}",
+ fmt::format_to(std::back_inserter(buffer), "{}{}",
buffer.size() != buffer_start ? "|" : "",
desc[i].name_);
flags &= ~desc[i].flag_;
@@ -280,15 +280,15 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) {
if(flags != 0) {
if(buffer.size() != buffer_start) {
- fmt::format_to(buffer, "|");
+ fmt::format_to(std::back_inserter(buffer), "|");
}
- fmt::format_to(buffer, "{:#x}", flags);
+ fmt::format_to(std::back_inserter(buffer), "{:#x}", flags);
return;
}
if(buffer_start == buffer.size()) {
- fmt::format_to(buffer, "0x0");
+ fmt::format_to(std::back_inserter(buffer), "0x0");
}
}
@@ -311,7 +311,7 @@ format_whence_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
FLAG_ENTRY(SEEK_END)
);
- fmt::format_to(buffer, "{}=", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=", parg.name);
format_flag_set(buffer, parg.value, flag_names);
}
@@ -334,7 +334,7 @@ format_mmap_prot_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
FLAG_ENTRY(PROT_WRITE),
FLAG_ENTRY(PROT_EXEC));
- fmt::format_to(buffer, "{}=", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=", parg.name);
format_flag_set(buffer, parg.value, flag_names);
return;
@@ -376,7 +376,7 @@ format_mmap_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
#endif
);
- fmt::format_to(buffer, "{}=", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=", parg.name);
format_flag_set(buffer, parg.value, flag_names);
return;
}
@@ -420,14 +420,14 @@ format_clone_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
FLAG_ENTRY(CLONE_NEWNET),
FLAG_ENTRY(CLONE_IO));
- fmt::format_to(buffer, "{}=", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=", parg.name);
// the low byte in clone flags contains the number of the termination
// signal sent to the parent when the child dies
format_flag_set(buffer, parg.value & ~0x11l, flag_names);
if((parg.value & 0x11l) != 0) {
- fmt::format_to(buffer, "|", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "|", parg.name);
format_signum_arg_to(buffer, {"", parg.value & 0x11l});
}
return;
@@ -479,7 +479,7 @@ format_signum_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
FLAG_ENTRY(SIGSYS));
if(std::strcmp(parg.name, "")) {
- fmt::format_to(buffer, "{}=", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=", parg.name);
}
format_flag(buffer, parg.value, flag_names);
@@ -504,7 +504,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
FLAG_ENTRY(SIG_UNBLOCK),
FLAG_ENTRY(SIG_SETMASK));
- fmt::format_to(buffer, "{}=", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=", parg.name);
format_flag(buffer, parg.value, flag_names);
return;
}
@@ -518,7 +518,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
template
inline void
format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
- fmt::format_to(buffer, "void");
+ fmt::format_to(std::back_inserter(buffer), "void");
}
@@ -531,7 +531,7 @@ format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
template
inline void
format_fd_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
- fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value));
+ fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value));
}
@@ -546,11 +546,11 @@ inline void
format_atfd_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
if(static_cast(parg.value) == AT_FDCWD) {
- fmt::format_to(buffer, "{}=AT_FDCWD", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=AT_FDCWD", parg.name);
return;
}
- fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value));
+ fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value));
}
@@ -565,12 +565,12 @@ inline void
format_cstr_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
if(LIKELY(reinterpret_cast(parg.value) != nullptr)) {
- fmt::format_to(buffer, "{}=\"{}\"", parg.name,
+ fmt::format_to(std::back_inserter(buffer), "{}=\"{}\"", parg.name,
reinterpret_cast(parg.value));
return;
}
- fmt::format_to(buffer, "{}=NULL", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name);
}
/**
@@ -618,7 +618,7 @@ format_open_flags_to(FmtBuffer& buffer,
long flags = parg.value;
- fmt::format_to(buffer, "{}=", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=", parg.name);
format_flag(buffer, flags & O_ACCMODE, flag_names);
flags &= ~O_ACCMODE;
@@ -636,7 +636,7 @@ format_open_flags_to(FmtBuffer& buffer,
#endif // !O_TMPFILE
if(flags != 0) {
- fmt::format_to(buffer, "|", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "|", parg.name);
format_flag_set(buffer, flags, extra_flag_names);
}
}
@@ -650,7 +650,7 @@ format_open_flags_to(FmtBuffer& buffer,
template
inline void
format_octal_mode_to(FmtBuffer& buffer, const printable_arg& parg) {
- fmt::format_to(buffer, "{}={:#04o}", parg.name, parg.value);
+ fmt::format_to(std::back_inserter(buffer), "{}={:#04o}", parg.name, parg.value);
}
/**
@@ -664,12 +664,12 @@ inline void
format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
if(LIKELY(reinterpret_cast(parg.value) != nullptr)) {
- fmt::format_to(buffer, "{}={}", parg.name,
+ fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name,
reinterpret_cast(parg.value));
return;
}
- fmt::format_to(buffer, "{}=NULL", parg.name);
+ fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name);
}
@@ -682,7 +682,7 @@ format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
template
inline void
format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
- fmt::format_to(buffer, "{}={}", parg.name, parg.value);
+ fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, parg.value);
}
@@ -695,7 +695,7 @@ format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
template
inline void
format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
- fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value));
+ fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value));
}
@@ -708,7 +708,7 @@ format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
template
inline void
format_arg_to(FmtBuffer& buffer, const printable_arg& parg) {
- fmt::format_to(buffer, "{}={:#x}", parg.name, parg.value);
+ fmt::format_to(std::back_inserter(buffer), "{}={:#x}", parg.name, parg.value);
}
#undef FLAG_ENTRY
diff --git a/include/client/syscalls/decoder.hpp b/include/client/syscalls/decoder.hpp
index eebdd6fdbc9551514b9c8d89cdc29d4a1fea6381..3172cc2df291c118dcf3de9c13cfa19f7177b10c 100644
--- a/include/client/syscalls/decoder.hpp
+++ b/include/client/syscalls/decoder.hpp
@@ -60,19 +60,19 @@ decode(FmtBuffer& buffer, const long syscall_number,
const auto sc = lookup_by_number(syscall_number, argv);
- fmt::format_to(buffer, "{}(", sc.name());
+ fmt::format_to(std::back_inserter(buffer), "{}(", sc.name());
for(int i = 0; i < sc.num_args(); ++i) {
- const auto& arg = sc.args().at(i);
+ const auto arg = sc.args().at(i);
arg.formatter()(buffer, {arg.name(), argv[i]});
if(i < sc.num_args() - 1) {
- fmt::format_to(buffer, ", ");
+ fmt::format_to(std::back_inserter(buffer), ", ");
}
}
- fmt::format_to(buffer, ") = ?");
+ fmt::format_to(std::back_inserter(buffer), ") = ?");
}
template
@@ -84,30 +84,31 @@ decode(FmtBuffer& buffer, const long syscall_number, const long argv[MAX_ARGS],
const auto sc = lookup_by_number(syscall_number, argv);
- fmt::format_to(buffer, "{}(", sc.name());
+ fmt::format_to(std::back_inserter(buffer), "{}(", sc.name());
for(int i = 0; i < sc.num_args(); ++i) {
- const auto& arg = sc.args().at(i);
+ const auto arg = sc.args().at(i);
arg.formatter()(buffer, {arg.name(), argv[i]});
if(i < sc.num_args() - 1) {
- fmt::format_to(buffer, ", ");
+ fmt::format_to(std::back_inserter(buffer), ", ");
}
}
if(never_returns(syscall_number)) {
- fmt::format_to(buffer, ") = ?");
+ fmt::format_to(std::back_inserter(buffer), ") = ?");
return;
}
if(error_code(result) != 0) {
- fmt::format_to(buffer, ") = {} {} ({})", static_cast(-1),
- errno_name(-result), errno_message(-result));
+ fmt::format_to(std::back_inserter(buffer), ") = {} {} ({})",
+ static_cast(-1), errno_name(-result),
+ errno_message(-result));
return;
}
- fmt::format_to(buffer, ") = ");
+ fmt::format_to(std::back_inserter(buffer), ") = ");
const auto& ret = sc.return_type();
ret.formatter()(buffer, result);
}
diff --git a/include/client/syscalls/rets.hpp b/include/client/syscalls/rets.hpp
index adf77647c8928748d671effedaa73e45db1b26e8..1e40f77b02f6af0d457fd75b124f3ae14aaf85ef 100644
--- a/include/client/syscalls/rets.hpp
+++ b/include/client/syscalls/rets.hpp
@@ -121,7 +121,7 @@ template
inline void
format_none_ret_to(FmtBuffer& buffer,
long val) {
- fmt::format_to(buffer, "void");
+ fmt::format_to(std::back_inserter(buffer), "void");
}
template
@@ -129,18 +129,18 @@ inline void
format_ptr_ret_to(FmtBuffer& buffer,
long val) {
if(LIKELY(reinterpret_cast(val) != nullptr)) {
- fmt::format_to(buffer, "{}", reinterpret_cast(val));
+ fmt::format_to(std::back_inserter(buffer), "{}", reinterpret_cast(val));
return;
}
- fmt::format_to(buffer, "NULL");
+ fmt::format_to(std::back_inserter(buffer), "NULL");
}
template
inline void
format_dec_ret_to(FmtBuffer& buffer,
long val) {
- fmt::format_to(buffer, "{}", val);
+ fmt::format_to(std::back_inserter(buffer), "{}", val);
}
#undef LIKELY
diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..de5f717840afcb2a1386a164292bdd37e2a9ccab
--- /dev/null
+++ b/include/client/user_functions.hpp
@@ -0,0 +1,85 @@
+/*
+ Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
+ Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
+
+ This software was partially supported by the
+ EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+ This software was partially supported by the
+ ADA-FS project under the SPPEXA project funded by the DFG.
+
+ This file is part of GekkoFS' POSIX interface.
+
+ GekkoFS' POSIX interface is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of the License,
+ or (at your option) any later version.
+
+ GekkoFS' POSIX interface is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with GekkoFS' POSIX interface. If not, see
+ .
+
+ SPDX-License-Identifier: LGPL-3.0-or-later
+*/
+
+#ifndef GEKKOFS_USER_FUNCTIONS_HPP
+#define GEKKOFS_USER_FUNCTIONS_HPP
+#include
+#include
+#include
+#include
+
+struct linux_dirent64;
+
+namespace gkfs::syscall {
+
+int
+gkfs_open(const std::string& path, mode_t mode, int flags);
+
+int
+gkfs_create(const std::string& path, mode_t mode);
+
+int
+gkfs_remove(const std::string& path);
+
+ssize_t
+gkfs_write(int fd, const void* buf, size_t count);
+
+ssize_t
+gkfs_read(int fd, void* buf, size_t count);
+
+int
+gkfs_close(unsigned int fd);
+
+off64_t
+gkfs_lseek(unsigned int fd, off64_t offset, unsigned int whence);
+
+ssize_t
+gkfs_pwrite(int fd, const void* buf, size_t count, off64_t offset);
+
+ssize_t
+gkfs_pread(int fd, void* buf, size_t count, off64_t offset);
+
+int
+gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true);
+
+int
+gkfs_remove(const std::string& path);
+
+std::vector
+gkfs_get_file_list(const std::string& path);
+} // namespace gkfs::syscall
+
+
+extern "C" int
+gkfs_init();
+
+extern "C" int
+gkfs_end();
+
+#endif // GEKKOFS_USER_FUNCTIONS_HPP
diff --git a/include/client/void_syscall_intercept.hpp b/include/client/void_syscall_intercept.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..16cd01b5bd59da0a1e31ac831c66ed707462e488
--- /dev/null
+++ b/include/client/void_syscall_intercept.hpp
@@ -0,0 +1,45 @@
+/*
+ Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
+ Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
+
+ This software was partially supported by the
+ EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+ This software was partially supported by the
+ ADA-FS project under the SPPEXA project funded by the DFG.
+
+ This file is part of GekkoFS' POSIX interface.
+
+ GekkoFS' POSIX interface is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of the License,
+ or (at your option) any later version.
+
+ GekkoFS' POSIX interface is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with GekkoFS' POSIX interface. If not, see
+ .
+
+ SPDX-License-Identifier: LGPL-3.0-or-later
+*/
+#ifndef SYSCALL_BYPASS_HPP
+#define SYSCALL_BYPASS_HPP
+
+static inline int
+syscall_error_code(long result) {
+ if(result < 0 && result >= -0x1000)
+ return (int) -result;
+
+ return 0;
+}
+
+extern "C" {
+long
+syscall_no_intercept(long syscall_number, ...);
+}
+
+#endif
\ No newline at end of file
diff --git a/include/common/CMakeLists.txt b/include/common/CMakeLists.txt
index 9561361970c92e4ba89f02c1434508c2aaa1eb0e..38a23bc300c90a2a9f193bb440044a4890b07fa4 100644
--- a/include/common/CMakeLists.txt
+++ b/include/common/CMakeLists.txt
@@ -30,10 +30,3 @@ target_sources(
gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp
rpc/rpc_util.hpp
)
-
-if(GKFS_ENABLE_FORWARDING)
- target_sources(
- gkfwd_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp
- rpc/rpc_types.hpp rpc/rpc_util.hpp
- )
-endif()
diff --git a/include/common/common_defs.hpp b/include/common/common_defs.hpp
index 649f49f75fc49e58b500a70392eb75328ceb0ea2..21c09a7438fcca08d98712c9a3b264edab231899 100644
--- a/include/common/common_defs.hpp
+++ b/include/common/common_defs.hpp
@@ -58,11 +58,18 @@ constexpr auto get_chunk_stat = "rpc_srv_chunk_stat";
} // namespace tag
namespace protocol {
-constexpr auto ofi_psm2 = "ofi+psm2";
+constexpr auto na_sm = "na+sm";
constexpr auto ofi_sockets = "ofi+sockets";
constexpr auto ofi_tcp = "ofi+tcp";
constexpr auto ofi_verbs = "ofi+verbs";
-constexpr auto na_sm = "na+sm";
+constexpr auto ofi_psm2 = "ofi+psm2";
+constexpr auto ucx_all = "ucx+all";
+constexpr auto ucx_tcp = "ucx+tcp";
+constexpr auto ucx_rc = "ucx+rc";
+constexpr auto ucx_ud = "ucx+ud";
+constexpr auto all_remote_protocols = {ofi_sockets, ofi_tcp, ofi_verbs,
+ ofi_psm2, ucx_all, ucx_tcp,
+ ucx_rc, ucx_ud};
} // namespace protocol
} // namespace gkfs::rpc
diff --git a/include/common/env_util.hpp b/include/common/env_util.hpp
index e60fac7dc5fd81681181143b2e7c15d6b1f7e6ea..e9f667fb8500cd143f659052114079728a185e13 100644
--- a/include/common/env_util.hpp
+++ b/include/common/env_util.hpp
@@ -36,6 +36,9 @@ namespace gkfs::env {
std::string
get_var(const std::string& name, const std::string& default_value = "");
+bool
+var_is_set(const std::string& name);
+
} // namespace gkfs::env
#endif // GKFS_COMMON_ENV_UTIL_HPP
diff --git a/include/common/msgpack_util.hpp b/include/common/msgpack_util.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..841369b4efc439289a7be375637bf009af0b9ac6
--- /dev/null
+++ b/include/common/msgpack_util.hpp
@@ -0,0 +1,150 @@
+/*
+ Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain
+ Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany
+
+ This software was partially supported by the
+ EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+ This software was partially supported by the
+ ADA-FS project under the SPPEXA project funded by the DFG.
+
+ This file is part of GekkoFS.
+
+ GekkoFS is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ GekkoFS is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GekkoFS. If not, see .
+
+ SPDX-License-Identifier: GPL-3.0-or-later
+*/
+
+#ifndef GKFS_COMMON_MSGPACK_HPP
+#define GKFS_COMMON_MSGPACK_HPP
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+
+namespace gkfs::messagepack {
+
+enum class client_metric_io_type { write, read };
+enum class client_metric_flush_type { file, socket };
+
+class ClientMetrics {
+
+public:
+ /*
+ * MessagePack data structure for client metrics. Includes only what is
+ * actually sent
+ */
+ struct msgpack_data {
+ uint32_t flush_t_;
+ std::string hostname_;
+ int pid_;
+ std::string io_type_;
+ std::vector start_t_{};
+ std::vector end_t_{};
+ std::vector req_size_{};
+ uint32_t total_bytes_{};
+ int total_iops_{0};
+
+ template
+ void
+ pack(T& pack) {
+ pack(flush_t_, hostname_, pid_, io_type_, start_t_, end_t_,
+ req_size_, total_iops_, total_bytes_);
+ }
+
+ std::vector
+ pack_msgpack() {
+ return msgpack::pack(*this);
+ }
+ };
+
+private:
+ bool metrics_enabled_{false};
+
+ msgpack_data msgpack_data_{};
+
+ // Initialization time used to compute relative timestamps
+ std::chrono::time_point init_t_;
+
+ std::mutex data_mtx_{};
+ std::thread flush_thread_{};
+ std::condition_variable flush_thread_cv_{};
+ std::mutex flush_thread_cv_mutex_{};
+ std::atomic flush_thread_running_{false};
+
+ client_metric_flush_type flush_type_{client_metric_flush_type::file};
+ int flush_interval_{};
+ std::unique_ptr zmq_flush_context_ = nullptr;
+ std::unique_ptr zmq_flush_socket_ = nullptr;
+ std::string flush_path_{};
+ int flush_count_{0};
+
+
+public:
+ ClientMetrics() = default;
+
+ explicit ClientMetrics(client_metric_io_type io_type,
+ client_metric_flush_type flush_type =
+ client_metric_flush_type::file,
+ int flush_interval = 5);
+
+ ~ClientMetrics();
+
+ void
+ add_event(size_t size,
+ std::chrono::time_point start);
+
+ void
+ reset_metrics();
+
+ void
+ flush_msgpack();
+
+ void
+ flush_loop();
+
+ void
+ enable();
+
+ void
+ disable();
+
+ void
+ zmq_connect(const std::string& ip_port);
+
+ bool
+ zmq_is_connected();
+
+ [[nodiscard]] const std::string&
+ path() const;
+
+ void
+ path(const std::string& path, const std::string prefix = "");
+
+ int
+ flush_count() const;
+};
+
+} // namespace gkfs::messagepack
+
+#endif // GKFS_COMMON_MSGPACK_HPP
diff --git a/include/config.hpp b/include/config.hpp
index 7ec761f64e035c533adcebfa3a9da7b9c267816e..edf7b2c69a3a9174d7a79cce65cb3491fae12148 100644
--- a/include/config.hpp
+++ b/include/config.hpp
@@ -39,7 +39,16 @@
namespace gkfs::config {
constexpr auto hostfile_path = "./gkfs_hosts.txt";
-constexpr auto forwarding_file_path = "./gkfs_forwarding.map";
+// We do not default this, ENV variable always required.
+constexpr auto forwarding_file_path = "";
+
+namespace client_metrics {
+// Default directory where client metrics are stored. Can be set via
+// LIBGKFS_METRICS_PATH. Filename consists of starting time, pid, and hostname
+// Note: when LIBGKFS_METRICS_IP is given, ZeroMQ is used instead
+constexpr auto flush_path = "/tmp/gkfs_client_metrics";
+constexpr auto flush_interval = 5; // in seconds
+} // namespace client_metrics
namespace io {
/*
diff --git a/include/daemon/CMakeLists.txt b/include/daemon/CMakeLists.txt
index 9cdfe06a94e699e3c0280ac3bdbe3daf512b00ee..26466ba16138af272d680b78b9f081a4f516c454 100644
--- a/include/daemon/CMakeLists.txt
+++ b/include/daemon/CMakeLists.txt
@@ -38,21 +38,6 @@ target_sources(
handler/rpc_util.hpp
)
-if(GKFS_ENABLE_FORWARDING)
- target_sources(
- gkfwd_daemon
- PUBLIC daemon.hpp
- util.hpp
- ops/data.hpp
- ops/metadentry.hpp
- classes/fs_data.hpp
- classes/rpc_data.hpp
- handler/rpc_defs.hpp
- handler/rpc_util.hpp
- )
-
- if(GKFS_ENABLE_AGIOS)
- target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp)
- endif()
-
+if(GKFS_ENABLE_AGIOS)
+ target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp)
endif()
diff --git a/include/daemon/backend/metadata/merge.hpp b/include/daemon/backend/metadata/merge.hpp
index f655f7d4070593273fd28ad8431a5dd2c4f1f2f4..5d30aaef1ca19acc1edab3d5a2c959044eeb8ac8 100644
--- a/include/daemon/backend/metadata/merge.hpp
+++ b/include/daemon/backend/metadata/merge.hpp
@@ -33,6 +33,8 @@
#include
#include
+#include
+
namespace rdb = rocksdb;
namespace gkfs::metadata {
@@ -43,7 +45,9 @@ namespace gkfs::metadata {
enum class OperandID : char {
increase_size = 'i',
decrease_size = 'd',
- create = 'c'
+ create = 'c',
+ update_time = 't'
+
};
/**
@@ -156,6 +160,33 @@ public:
std::string
serialize_params() const override;
};
+
+/**
+ * @brief Update time operand
+ */
+class UpdateTimeOperand : public MergeOperand {
+private:
+ time_t mtime_{};
+
+public:
+ UpdateTimeOperand() = default;
+
+ explicit UpdateTimeOperand(time_t mtime);
+
+ explicit UpdateTimeOperand(const rdb::Slice& serialized_op);
+
+ OperandID
+ id() const override;
+
+ std::string
+ serialize_params() const override;
+
+ time_t
+ mtime() const {
+ return mtime_;
+ }
+};
+
/**
* @brief Merge operator class passed to RocksDB, used during merge operations
*/
diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp
index 13f8023d1aa2e4dd9c52cbb15d6c570eb37327a0..ef89b5e282e0b1f9cc7dd82f6b6c66c4287a9553 100644
--- a/include/daemon/classes/fs_data.hpp
+++ b/include/daemon/classes/fs_data.hpp
@@ -95,6 +95,7 @@ private:
bool enable_stats_ = false;
bool enable_chunkstats_ = false;
bool enable_prometheus_ = false;
+ bool enable_forwarding_ = false;
std::string stats_file_;
// Prometheus
@@ -246,6 +247,12 @@ public:
void
enable_chunkstats(bool enable_chunkstats);
+ bool
+ enable_forwarding() const;
+
+ void
+ enable_forwarding(bool enable_forwarding);
+
bool
enable_prometheus() const;
diff --git a/scripts/profiles/0.9.3/agios.specs b/scripts/profiles/0.9.3/agios.specs
new file mode 100644
index 0000000000000000000000000000000000000000..68921e298efc685cec9c99ac4f82213070a5442d
--- /dev/null
+++ b/scripts/profiles/0.9.3/agios.specs
@@ -0,0 +1,80 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="All dependencies (except transport-specific and experimental)"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+ ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "agios"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+)
diff --git a/scripts/profiles/0.9.3/all.specs b/scripts/profiles/0.9.3/all.specs
new file mode 100644
index 0000000000000000000000000000000000000000..67396b8dc377d0a5d95010d37a280a486cac4c8f
--- /dev/null
+++ b/scripts/profiles/0.9.3/all.specs
@@ -0,0 +1,86 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Mogon 2 supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["psm2"]="11.2.185"
+ ["json-c"]="0.17-20230812"
+ ["curl"]="7.88.1"
+ ["prometheus-cpp"]="v1.0.0"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+ ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
+ ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept"
+ "agios" "curl" "prometheus-cpp" "parallax"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
+)
diff --git a/scripts/profiles/0.9.3/arm.specs b/scripts/profiles/0.9.3/arm.specs
new file mode 100644
index 0000000000000000000000000000000000000000..1d5432de89d1e4cbf40c9f2137bef4d5418e082d
--- /dev/null
+++ b/scripts/profiles/0.9.3/arm.specs
@@ -0,0 +1,82 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for PowerPC supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+ ["psm2"]="11.2.185"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="fb73c3924b502e2840a5dc8a18746e395b06a8a6"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch.arm64"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=no --enable-sockets=yes"
+ ["syscall_intercept"]="ARM64"
+)
diff --git a/scripts/profiles/0.9.3/ci.specs b/scripts/profiles/0.9.3/ci.specs
new file mode 100644
index 0000000000000000000000000000000000000000..dc9fcedffa4a46fda15bbdbefc973fe98617e61e
--- /dev/null
+++ b/scripts/profiles/0.9.3/ci.specs
@@ -0,0 +1,80 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies required by the CI"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.5.4"
+ ["prometheus-cpp"]="v1.0.0"
+ )
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.17.0"
+ ["mercury"]="v2.3.0"
+ ["margo"]="v0.14.1"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+ ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
+ ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd"
+ )
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept"
+ "agios" "parallax" "prometheus-cpp"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+)
diff --git a/scripts/profiles/0.9.3/default.specs b/scripts/profiles/0.9.3/default.specs
new file mode 100644
index 0000000000000000000000000000000000000000..59d8175fe8bee6a4dedc6b530323e507a583b850
--- /dev/null
+++ b/scripts/profiles/0.9.3/default.specs
@@ -0,0 +1,79 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="All dependencies"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+)
+
+# Dependencies that must be cloned.
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+)
diff --git a/scripts/profiles/0.9.3/default_zmq.specs b/scripts/profiles/0.9.3/default_zmq.specs
new file mode 100644
index 0000000000000000000000000000000000000000..7185ddb19b9355a5b5e33ec331a7acc330fc51aa
--- /dev/null
+++ b/scripts/profiles/0.9.3/default_zmq.specs
@@ -0,0 +1,81 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="All dependencies"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+ ["libzmq"]="4.3.5"
+ ["cppzmq"]="4.10.0"
+)
+
+# Dependencies that must be cloned.
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "libzmq" "cppzmq"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+)
diff --git a/scripts/profiles/0.9.3/infiniband_verbs.specs b/scripts/profiles/0.9.3/infiniband_verbs.specs
new file mode 100644
index 0000000000000000000000000000000000000000..abb374413860c22a20fec3f36eae13e2343489a1
--- /dev/null
+++ b/scripts/profiles/0.9.3/infiniband_verbs.specs
@@ -0,0 +1,81 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Infiniband supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.5.4"
+ ["json-c"]="0.17-20230812"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric%verbs"]="HEAD@v1.17.0"
+ ["mercury"]="v2.3.0"
+ ["margo"]="v0.14.1"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading/installing
+order=(
+ "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb"
+ "syscall_intercept"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric%verbs"]="--enable-verbs=yes"
+)
diff --git a/scripts/profiles/0.9.3/install/agios.install b/scripts/profiles/0.9.3/install/agios.install
new file mode 100644
index 0000000000000000000000000000000000000000..b5ed7d7ee930fd2eb79a4113a3aaccd533f6fbe3
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/agios.install
@@ -0,0 +1,57 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="agios"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" ..
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/argobots.install b/scripts/profiles/0.9.3/install/argobots.install
new file mode 100644
index 0000000000000000000000000000000000000000..9ff03b5a09576c4e07b3fceaea6c2922ca2ee753
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/argobots.install
@@ -0,0 +1,60 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="argobots"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.9.3/install/capstone.install b/scripts/profiles/0.9.3/install/capstone.install
new file mode 100644
index 0000000000000000000000000000000000000000..afd7794aff4c51cfc24b9351fd875a4a8724f1dd
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/capstone.install
@@ -0,0 +1,57 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="capstone"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release ..
+ make -j"${CORES}" install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/cppzmq.install b/scripts/profiles/0.9.3/install/cppzmq.install
new file mode 100644
index 0000000000000000000000000000000000000000..bc236806db28e51fb46d6bcb247631034af6ce6f
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/cppzmq.install
@@ -0,0 +1,62 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="cppzmq"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ $CMAKE \
+ -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \
+ -DCMAKE_BUILD_TYPE:STRING=Release \
+ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+ ..
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/curl.install b/scripts/profiles/0.9.3/install/curl.install
new file mode 100644
index 0000000000000000000000000000000000000000..44fab7ce95e891e8372cbffc6ac3bd60825ab0e6
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/curl.install
@@ -0,0 +1,58 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="curl"
+ CURR="${SOURCE_DIR}/${ID}"
+ cd "${CURR}"
+ autoreconf -fi
+ ./configure --prefix="${INSTALL_DIR}" --without-ssl
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/json-c.install b/scripts/profiles/0.9.3/install/json-c.install
new file mode 100644
index 0000000000000000000000000000000000000000..9ba99fed64b2d4c573c4e7a247be1b6d0f949a94
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/json-c.install
@@ -0,0 +1,60 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CURR="${SOURCE_DIR}/json-c"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release ..
+ make -j"${CORES}" install
+ # Margo doesn't search in both directories, so we make it available in both lib and lib64
+ if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then
+ cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/
+ fi
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/libfabric%verbs.install b/scripts/profiles/0.9.3/install/libfabric%verbs.install
new file mode 100644
index 0000000000000000000000000000000000000000..74a1921023aa08f5098474d1041577a2a1986dea
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/libfabric%verbs.install
@@ -0,0 +1,67 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="libfabric%verbs"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes"
+
+ EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}"
+
+ if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then
+ OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}"
+ fi
+
+ ${OFI_CONFIG}
+ make -j"${CORES}" install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.9.3/install/libfabric.install b/scripts/profiles/0.9.3/install/libfabric.install
new file mode 100644
index 0000000000000000000000000000000000000000..6fee0bf9e77375a776e91db574615aaf53988025
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/libfabric.install
@@ -0,0 +1,67 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID=libfabric
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes"
+
+ EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}"
+
+ if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then
+ OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}"
+ fi
+
+ ${OFI_CONFIG}
+ make -j"${CORES}" install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.9.3/install/libzmq.install b/scripts/profiles/0.9.3/install/libzmq.install
new file mode 100644
index 0000000000000000000000000000000000000000..9f98a196d259e01d2190977879e0edb3b2e95a50
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/libzmq.install
@@ -0,0 +1,59 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="libzmq"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ ../configure --prefix="${INSTALL_DIR}" CFLAGS="${CFLAGS} -Wall -O3"
+ make -j"${CORES}" install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/lz4.install b/scripts/profiles/0.9.3/install/lz4.install
new file mode 100644
index 0000000000000000000000000000000000000000..893a1b13bc4fa570657908c6a6ce8986d4bd1b57
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/lz4.install
@@ -0,0 +1,61 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="lz4"
+ CURR="${SOURCE_DIR}/${ID}"
+ cd "${CURR}"
+ # try to remove binaries first in case they already exist. Otherwise install fails.
+ LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4)
+ for LZ4_BIN in "${LZ4_BINS[@]}"; do
+ [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN"
+ done
+ make -j"${CORES}"
+ make DESTDIR="${INSTALL_DIR}" PREFIX="" install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/margo.install b/scripts/profiles/0.9.3/install/margo.install
new file mode 100644
index 0000000000000000000000000000000000000000..014621c68c61b93b0a322e64b60d2f8a332dd212
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/margo.install
@@ -0,0 +1,59 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="margo"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./prepare.sh
+ cd "${CURR}/build"
+ ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3"
+ make -j"${CORES}" install
+}
+
+pkg_check() {
+ make check
+}
diff --git a/scripts/profiles/0.9.3/install/mercury.install b/scripts/profiles/0.9.3/install/mercury.install
new file mode 100644
index 0000000000000000000000000000000000000000..9fc5588c9b2a8ffdbc1ce36697cf7725a0e83853
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/mercury.install
@@ -0,0 +1,67 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="mercury"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \
+ -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \
+ -DCMAKE_BUILD_TYPE:STRING=Release \
+ -DBUILD_TESTING:BOOL=ON \
+ -DMERCURY_USE_CHECKSUMS:BOOL=OFF \
+ -DMERCURY_USE_BOOST_PP:BOOL=ON \
+ -DBUILD_SHARED_LIBS:BOOL=ON \
+ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+ -DNA_USE_OFI:BOOL=ON -DNA_USE_UCX:BOOL=ON \
+ ..
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/parallax.install b/scripts/profiles/0.9.3/install/parallax.install
new file mode 100644
index 0000000000000000000000000000000000000000..8c0cc9e95296b2553c651f4d8e1858fee8334035
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/parallax.install
@@ -0,0 +1,64 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+
+ CURR="${SOURCE_DIR}/parallax"
+ # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \
+ -DBUILD_SHARED_LIBS:BOOL=ON \
+ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+ -DCMAKE_BUILD_TYPE="Release" \
+ -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \
+ -DDISABLE_LOGGING:BOOL=ON \
+ ..
+ make -j"${CORES}"
+ make install
+ # We need to copy this file as it is not installed
+ cp ${CURR}/lib/include/parallax/structures.h ${INSTALL_DIR}/include/
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/prometheus-cpp.install b/scripts/profiles/0.9.3/install/prometheus-cpp.install
new file mode 100644
index 0000000000000000000000000000000000000000..154db65dfeadf13a1e4515e33203251d3ef8fead
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/prometheus-cpp.install
@@ -0,0 +1,61 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="prometheus-cpp"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ ${CMAKE} \
+ -DCMAKE_BUILD_TYPE:STRING=Release \
+ -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+ -DBUILD_SHARED_LIBS:BOOL=ON \
+ ..
+ make -j"${CORES}" install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/rocksdb.install b/scripts/profiles/0.9.3/install/rocksdb.install
new file mode 100644
index 0000000000000000000000000000000000000000..7c3004fee34a937dd813fc560714ce9aac8e3009
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/rocksdb.install
@@ -0,0 +1,79 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ CXXFLAGS=''
+ # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move
+ if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) ||
+ ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then
+ CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move -Wno-error=maybe-uninitialized'
+ fi
+
+ # TODO use SSE?
+ CURR="${SOURCE_DIR}/rocksdb"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}/build"
+ PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \
+ -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \
+ -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \
+ -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \
+ -DROCKSDB_BUILD_SHARED=OFF \
+ -DWITH_LZ4=ON \
+ -DWITH_GFLAGS=OFF \
+ -DUSE_RTTI=1 \
+ -DPORTABLE=1 \
+ -DWITH_ALL_TESTS=OFF \
+ -DWITH_BENCHMARK_TOOLS=OFF \
+ -DWITH_TOOLS=OFF ..
+ make -j"${CORES}" install
+
+
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/syscall_intercept.install b/scripts/profiles/0.9.3/install/syscall_intercept.install
new file mode 100644
index 0000000000000000000000000000000000000000..f69393d3a9ddb2c68f397973b26108035ea4ce4e
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/syscall_intercept.install
@@ -0,0 +1,71 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+ ID="syscall_intercept"
+ CURR="${SOURCE_DIR}/${ID}"
+ EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}"
+ prepare_build_dir "${CURR}"
+ if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then
+ cd "${CURR}"/arch/aarch64/
+ mkdir -p build
+ cd build
+ else
+ cd "${CURR}"/build
+ fi
+ $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \
+ -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+ -DCMAKE_BUILD_TYPE:STRING=Debug \
+ -DBUILD_EXAMPLES:BOOL=OFF \
+ -DBUILD_TESTS:BOOK=OFF ..
+ make -j"${CORES}" install
+ if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then
+ cp "${CURR}"/arch/aarch64/include/libsyscall_intercept_hook_point.h ${INSTALL_DIR}/include
+ fi
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/install/ucx.install b/scripts/profiles/0.9.3/install/ucx.install
new file mode 100644
index 0000000000000000000000000000000000000000..fd90d6d3f62dd7552becf46bb02ee8c9fdc7a82c
--- /dev/null
+++ b/scripts/profiles/0.9.3/install/ucx.install
@@ -0,0 +1,61 @@
+################################################################################
+# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+# vi: ft=bash
+
+################################################################################
+## The installation script must define both a pkg_install function and
+## pkg_check function that, as their name implies, must specify how
+## a dependency package should be installed and tested. ## ## The following
+## variables can be used in the installation script:
+## - CMAKE: a variable that expands to the cmake binary
+## - SOURCE_DIR: the directory where the sources for the package were
+## downloaded
+## - INSTALL_DIR: the directory where the package should be installed
+## - CORES: the number of cores to use when building
+## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.)
+## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0)
+## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9)
+## - PERFORM_TEST: whether tests for the package should be executed
+################################################################################
+
+pkg_install() {
+
+ ID="ucx"
+ CURR="${SOURCE_DIR}/${ID}"
+ prepare_build_dir "${CURR}"
+ cd "${CURR}"
+ ./autogen.sh
+ cd "${CURR}/build"
+ ../contrib/configure-release --prefix=${INSTALL_DIR}
+ make -j"${CORES}"
+ make install
+}
+
+pkg_check() {
+ :
+}
diff --git a/scripts/profiles/0.9.3/marenostrum4.specs b/scripts/profiles/0.9.3/marenostrum4.specs
new file mode 100644
index 0000000000000000000000000000000000000000..15814ec63e526f254d74abccee0711e156599fa3
--- /dev/null
+++ b/scripts/profiles/0.9.3/marenostrum4.specs
@@ -0,0 +1,83 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Marenostrum 4 supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.5.4"
+ ["json-c"]="0.17-20230812"
+ ["psm2"]="11.2.185"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.17.0"
+ ["mercury"]="v2.3.0"
+ ["margo"]="v0.14.1"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+ ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=no --enable-sockets=yes"
+)
diff --git a/scripts/profiles/0.9.3/mogon2.specs b/scripts/profiles/0.9.3/mogon2.specs
new file mode 100644
index 0000000000000000000000000000000000000000..f7d9b2cadd509c0c52486d8e712902244f42d45b
--- /dev/null
+++ b/scripts/profiles/0.9.3/mogon2.specs
@@ -0,0 +1,85 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Mogon 2 supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+# ["psm2"]="11.2.230"
+# ["psm2"]="11.2.185"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+# ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=yes --enable-opx=yes"
+# ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
+)
diff --git a/scripts/profiles/0.9.3/mogon3.specs b/scripts/profiles/0.9.3/mogon3.specs
new file mode 100644
index 0000000000000000000000000000000000000000..8d1ca5497a719f3cc4e0b285fa4d5ac382c1869f
--- /dev/null
+++ b/scripts/profiles/0.9.3/mogon3.specs
@@ -0,0 +1,82 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Mogon 2 supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+ ["libzmq"]="4.3.5"
+ ["cppzmq"]="4.10.0"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+# ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "libzmq" "cppzmq"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+# ["libfabric"]=""
+)
diff --git a/scripts/profiles/0.9.3/ngio.specs b/scripts/profiles/0.9.3/ngio.specs
new file mode 100644
index 0000000000000000000000000000000000000000..fd1f3a3353a462a82a6080f8193f908936f74e0c
--- /dev/null
+++ b/scripts/profiles/0.9.3/ngio.specs
@@ -0,0 +1,83 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for NEXTGenIO prototype cluster"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+ ["psm2"]="11.2.185"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+ ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
+)
diff --git a/scripts/profiles/0.9.3/omnipath_psm2.specs b/scripts/profiles/0.9.3/omnipath_psm2.specs
new file mode 100644
index 0000000000000000000000000000000000000000..8cdac16d86074198212610762b7c38a98adb9bda
--- /dev/null
+++ b/scripts/profiles/0.9.3/omnipath_psm2.specs
@@ -0,0 +1,82 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for Omnipath supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+ ["psm2"]="11.2.185"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
+)
diff --git a/scripts/profiles/0.9.3/p9.specs b/scripts/profiles/0.9.3/p9.specs
new file mode 100644
index 0000000000000000000000000000000000000000..cf80be5a9c2b08a3ea03eb739177aa7f59e902d2
--- /dev/null
+++ b/scripts/profiles/0.9.3/p9.specs
@@ -0,0 +1,82 @@
+################################################################################
+# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+# vi: ft=bash
+
+# Variables to be imported into the scripts
+declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args
+declare -a order
+
+# Comment that should be displayed when printing the profile
+comment="Dependencies for PowerPC supercomputer"
+
+# Dependencies that must be downloaded directly
+wgetdeps=(
+ ["lz4"]="1.9.4"
+ ["capstone"]="4.0.2"
+ ["argobots"]="1.1"
+ ["rocksdb"]="8.10.0"
+ ["json-c"]="0.17-20230812"
+ ["psm2"]="11.2.185"
+)
+
+# Dependencies that must be cloned
+clonedeps=(
+ ["libfabric"]="HEAD@v1.20.1"
+ ["mercury"]="v2.3.1"
+ ["margo"]="v0.15.0"
+ ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f"
+ ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
+)
+
+# Extra arguments for git clone
+clonedeps_args=(
+ ["mercury"]="--recurse-submodules"
+)
+
+# Patches that should be applied post-clone
+clonedeps_patches=(
+ ["syscall_intercept"]="syscall_intercept.patch"
+)
+
+# Ordering that MUST be followed when downloading
+order=(
+ "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
+)
+
+# Extra arguments passed to the installation script. As such, they can
+# reference the following variables:
+# - CMAKE: a variable that expands to the cmake binary
+# - SOURCE_DIR: the directory where the sources for the package were
+# downloaded
+# - INSTALL_DIR: the directory where the package should be installed
+# - CORES: the number of cores to use when building
+# - PERFORM_TEST: whether tests for the package should be executed
+extra_install_args=(
+ ["libfabric"]="--enable-psm2=no --enable-sockets=yes"
+)
diff --git a/scripts/profiles/latest b/scripts/profiles/latest
index f76f9131742ee389211cdafc141553a15268098f..b3ec1638fda741e3680a70960377c66e3b41c4c7 120000
--- a/scripts/profiles/latest
+++ b/scripts/profiles/latest
@@ -1 +1 @@
-0.9.2
\ No newline at end of file
+0.9.3
\ No newline at end of file
diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list
index b659f38fbc0f4f6a0cd194047dbf381940285f77..385ac8afb1ccd5d07349c82f03e601835227879e 100644
--- a/scripts/profiles/sources.list
+++ b/scripts/profiles/sources.list
@@ -39,7 +39,7 @@ sources=(
["argobots"]="https://github.com/pmodels/argobots/archive/v{{VERSION}}.tar.gz"
["rocksdb"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz"
["rocksdb%experimental"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz"
- ["psm2"]="https://github.com/intel/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz"
+ ["psm2"]="https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz"
["bmi"]="https://github.com/radix-io/bmi/"
["libfabric"]="https://github.com/ofiwg/libfabric.git"
["libfabric%experimental"]="https://github.com/ofiwg/libfabric.git"
@@ -53,6 +53,9 @@ sources=(
["parallax"]="https://github.com/CARV-ICS-FORTH/parallax.git"
["prometheus-cpp"]="https://github.com/jupp0r/prometheus-cpp/releases/download/{{VERSION}}/prometheus-cpp-with-submodules.tar.gz"
["curl"]="https://curl.se/download/curl-{{VERSION}}.tar.gz"
+ ["ucx"]="https://github.com/openucx/ucx/archive/{{VERSION}}.tar.gz"
+ ["libzmq"]="https://github.com/zeromq/libzmq/archive/v{{VERSION}}.tar.gz"
+ ["cppzmq"]="https://github.com/zeromq/cppzmq/archive/v{{VERSION}}.tar.gz"
)
diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt
index 0108fb9c9fa5b8a543fcf0170dca8422c9941554..2a518c1ab7e7016ba30c86a70463b791e2b4e588 100644
--- a/src/client/CMakeLists.txt
+++ b/src/client/CMakeLists.txt
@@ -31,13 +31,33 @@
# This builds the `libgkfs_intercept.so` library: the primary GekkoFS client
# based on syscall interception.
# ##############################################################################
-add_library(gkfs_intercept SHARED)
-target_sources(
- gkfs_intercept
- PRIVATE gkfs_functions.cpp
+add_library (gkfs_intercept SHARED)
+add_library (gkfs_user_lib SHARED)
+
+
+target_sources(gkfs_intercept
+ PRIVATE gkfs_functions.cpp
+ intercept.cpp
hooks.cpp
+ logging.cpp
+ open_file_map.cpp
+ open_dir.cpp
+ path.cpp
+ preload.cpp
+ preload_context.cpp
+ preload_util.cpp
+ rpc/rpc_types.cpp
+ rpc/forward_data.cpp
+ rpc/forward_management.cpp
+ rpc/forward_metadata.cpp
+ syscalls/detail/syscall_info.c)
+
+target_sources(
+ gkfs_user_lib
+ PRIVATE gkfs_functions.cpp
intercept.cpp
+ hooks.cpp
logging.cpp
open_file_map.cpp
open_dir.cpp
@@ -49,19 +69,44 @@ target_sources(
rpc/forward_data.cpp
rpc/forward_management.cpp
rpc/forward_metadata.cpp
- syscalls/detail/syscall_info.c
+ syscalls/detail/syscall_info.c syscalls/util.S
)
+target_compile_definitions(gkfs_user_lib PUBLIC BYPASS_SYSCALL)
+
+if(GKFS_ENABLE_AGIOS)
+ target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS)
+endif()
+# Enable MSGPack metrics for intercept only
+
target_link_libraries(
gkfs_intercept
PRIVATE metadata distributor env_util arithmetic path_util rpc_utils
- PUBLIC Syscall_intercept::Syscall_intercept
- dl
+ PUBLIC dl
+ Mercury::Mercury
+ hermes
+ fmt::fmt
+ Threads::Threads
+ Syscall_intercept::Syscall_intercept
+)
+# Enable MSGPack metrics for intercept only
+if (GKFS_ENABLE_CLIENT_METRICS)
+ target_link_libraries(
+ gkfs_intercept
+ PUBLIC
+ msgpack_util
+ )
+ target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_CLIENT_METRICS)
+endif ()
+
+target_link_libraries(
+ gkfs_user_lib
+ PRIVATE metadata distributor env_util arithmetic path_util rpc_utils
+ PUBLIC dl
Mercury::Mercury
hermes
fmt::fmt
Threads::Threads
- Date::TZ
)
install(
@@ -71,54 +116,17 @@ install(
PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs
)
-# ##############################################################################
-# This builds the `libgkfwd_intercept.so` library: the client for GekkoFS'
-# forwarding mode based on syscall interception.
-# ##############################################################################
-
-if(GKFS_ENABLE_FORWARDING)
- add_library(gkfwd_intercept SHARED)
-
- target_sources(
- gkfwd_intercept
- PRIVATE gkfs_functions.cpp
- hooks.cpp
- intercept.cpp
- logging.cpp
- open_file_map.cpp
- open_dir.cpp
- path.cpp
- preload.cpp
- preload_context.cpp
- preload_util.cpp
- rpc/rpc_types.cpp
- rpc/forward_data.cpp
- rpc/forward_management.cpp
- rpc/forward_metadata.cpp
- syscalls/detail/syscall_info.c
- )
- target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_FORWARDING)
- if(GKFS_ENABLE_AGIOS)
- target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_AGIOS)
- endif()
+set_target_properties(gkfs_user_lib
+ PROPERTIES
+ PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp"
+ PUBLIC_HEADER "../../include/client/user_functions.hpp"
+)
- target_link_libraries(
- gkfwd_intercept
- PRIVATE metadata distributor env_util arithmetic path_util rpc_utils
- PUBLIC Syscall_intercept::Syscall_intercept
- dl
- Mercury::Mercury
- hermes
- fmt::fmt
- Threads::Threads
- Date::TZ
- )
- install(
- TARGETS gkfwd_intercept
- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
- PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs
- )
-endif()
+install(
+ TARGETS gkfs_user_lib
+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs
+)
diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp
index b03e8eaebe906bf24ffe9fdaadec01799541a4da..ca8b5855e4d83e502fcfc3648d05fcc912426113 100644
--- a/src/client/gkfs_functions.cpp
+++ b/src/client/gkfs_functions.cpp
@@ -37,6 +37,10 @@
#include
#include
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+#include
+#endif
+#include
extern "C" {
#include // used for file types in the getdents{,64}() functions
@@ -859,7 +863,7 @@ gkfs_dup2(const int oldfd, const int newfd) {
}
/**
- * Wrapper function for all gkfs write operations
+ * Actual write function for all gkfs write operations
* errno may be set
* @param file
* @param buf
@@ -870,16 +874,15 @@ gkfs_dup2(const int oldfd, const int newfd) {
* @return written size or -1 on error
*/
ssize_t
-gkfs_pwrite(std::shared_ptr file, const char* buf,
- size_t count, off64_t offset, bool update_pos) {
- if(file->type() != gkfs::filemap::FileType::regular) {
- assert(file->type() == gkfs::filemap::FileType::directory);
+gkfs_do_write(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos) {
+ if(file.type() != gkfs::filemap::FileType::regular) {
+ assert(file.type() == gkfs::filemap::FileType::directory);
LOG(WARNING, "Cannot write to directory");
errno = EISDIR;
return -1;
}
- auto path = make_unique(file->path());
- auto is_append = file->get_flag(gkfs::filemap::OpenFile_flags::append);
+ auto path = make_unique(file.path());
+ auto is_append = file.get_flag(gkfs::filemap::OpenFile_flags::append);
auto write_size = 0;
auto num_replicas = CTX->get_replicas();
@@ -929,7 +932,7 @@ gkfs_pwrite(std::shared_ptr file, const char* buf,
}
if(update_pos) {
// Update offset in file descriptor in the file map
- file->pos(offset + write_size);
+ file.pos(offset + write_size);
}
if(static_cast(write_size) != count) {
LOG(WARNING,
@@ -939,6 +942,29 @@ gkfs_pwrite(std::shared_ptr file, const char* buf,
return write_size; // return written size
}
+/**
+ * Wrapper function for all gkfs write operations
+ * errno may be set
+ * @param file
+ * @param buf
+ * @param count
+ * @param offset
+ * @param update_pos pos should only be updated for some write operations (see
+ * man 2 pwrite)
+ * @return written size or -1 on error
+ */
+ssize_t
+gkfs_write_ws(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos) {
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+ auto start_t = std::chrono::high_resolution_clock::now();
+ auto written = gkfs_do_write(file, buf, count, offset, update_pos);
+ CTX->write_metrics()->add_event(written, start_t);
+ return written;
+#else
+ return gkfs_do_write(file, buf, count, offset, update_pos);
+#endif
+}
+
/**
* gkfs wrapper for pwrite() system calls
* errno may be set
@@ -949,9 +975,10 @@ gkfs_pwrite(std::shared_ptr file, const char* buf,
* @return written size or -1 on error
*/
ssize_t
-gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset) {
+gkfs_pwrite(int fd, const void* buf, size_t count, off64_t offset) {
auto file = CTX->file_map()->get(fd);
- return gkfs_pwrite(file, reinterpret_cast(buf), count, offset);
+ return gkfs_write_ws(*file, reinterpret_cast(buf), count,
+ offset);
}
/**
@@ -964,10 +991,10 @@ gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset) {
*/
ssize_t
gkfs_write(int fd, const void* buf, size_t count) {
- auto gkfs_fd = CTX->file_map()->get(fd);
+ auto gkfs_file = CTX->file_map()->get(fd);
// call pwrite and update pos
- auto ret = gkfs_pwrite(gkfs_fd, reinterpret_cast(buf), count,
- gkfs_fd->pos(), true);
+ auto ret = gkfs_write_ws(*gkfs_file, reinterpret_cast(buf),
+ count, gkfs_file->pos(), true);
return ret;
}
@@ -993,7 +1020,7 @@ gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset) {
continue;
}
auto buf = (iov + i)->iov_base;
- ret = gkfs_pwrite(file, reinterpret_cast(buf), count, pos);
+ ret = gkfs_write_ws(*file, reinterpret_cast(buf), count, pos);
if(ret == -1) {
break;
}
@@ -1034,7 +1061,7 @@ gkfs_writev(int fd, const struct iovec* iov, int iovcnt) {
}
/**
- * Wrapper function for all gkfs read operations
+ * Actual read function for all gkfs read operations
* @param file
* @param buf
* @param count
@@ -1042,10 +1069,9 @@ gkfs_writev(int fd, const struct iovec* iov, int iovcnt) {
* @return read size or -1 on error
*/
ssize_t
-gkfs_pread(std::shared_ptr file, char* buf,
- size_t count, off64_t offset) {
- if(file->type() != gkfs::filemap::FileType::regular) {
- assert(file->type() == gkfs::filemap::FileType::directory);
+gkfs_do_read(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset) {
+ if(file.type() != gkfs::filemap::FileType::regular) {
+ assert(file.type() == gkfs::filemap::FileType::directory);
LOG(WARNING, "Cannot read from directory");
errno = EISDIR;
return -1;
@@ -1060,17 +1086,17 @@ gkfs_pread(std::shared_ptr file, char* buf,
std::set failed; // set with failed targets.
if(CTX->get_replicas() != 0) {
- ret = gkfs::rpc::forward_read(file->path(), buf, offset, count,
+ ret = gkfs::rpc::forward_read(file.path(), buf, offset, count,
CTX->get_replicas(), failed);
while(ret.first == EIO) {
- ret = gkfs::rpc::forward_read(file->path(), buf, offset, count,
+ ret = gkfs::rpc::forward_read(file.path(), buf, offset, count,
CTX->get_replicas(), failed);
LOG(WARNING, "gkfs::rpc::forward_read() failed with ret '{}'",
ret.first);
}
} else {
- ret = gkfs::rpc::forward_read(file->path(), buf, offset, count, 0,
+ ret = gkfs::rpc::forward_read(file.path(), buf, offset, count, 0,
failed);
}
@@ -1084,6 +1110,41 @@ gkfs_pread(std::shared_ptr file, char* buf,
return ret.second; // return read size
}
+/**
+ * Wrapper function for all gkfs read operations
+ * @param file
+ * @param buf
+ * @param count
+ * @param offset
+ * @return read size or -1 on error
+ */
+ssize_t
+gkfs_read_ws(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset) {
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+ auto start_t = std::chrono::high_resolution_clock::now();
+ auto read = gkfs_do_read(file, buf, count, offset);
+ CTX->read_metrics()->add_event(read, start_t);
+ return read;
+#else
+ return gkfs_do_read(file, buf, count, offset);
+#endif
+}
+
+/**
+ * gkfs wrapper for pread() system calls
+ * errno may be set
+ * @param fd
+ * @param buf
+ * @param count
+ * @param offset
+ * @return read size or -1 on error
+ */
+ssize_t
+gkfs_pread(int fd, void* buf, size_t count, off64_t offset) {
+ auto gkfs_fd = CTX->file_map()->get(fd);
+ return gkfs_read_ws(*gkfs_fd, reinterpret_cast(buf), count, offset);
+}
+
/**
* gkfs wrapper for read() system calls
* errno may be set
@@ -1096,7 +1157,7 @@ ssize_t
gkfs_read(int fd, void* buf, size_t count) {
auto gkfs_fd = CTX->file_map()->get(fd);
auto pos = gkfs_fd->pos(); // retrieve the current offset
- auto ret = gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, pos);
+ auto ret = gkfs_read_ws(*gkfs_fd, reinterpret_cast(buf), count, pos);
// Update offset in file descriptor in the file map
if(ret > 0) {
gkfs_fd->pos(pos + ret);
@@ -1126,7 +1187,7 @@ gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) {
continue;
}
auto buf = (iov + i)->iov_base;
- ret = gkfs_pread(file, reinterpret_cast(buf), count, pos);
+ ret = gkfs_read_ws(*file, reinterpret_cast(buf), count, pos);
if(ret == -1) {
break;
}
@@ -1166,21 +1227,6 @@ gkfs_readv(int fd, const struct iovec* iov, int iovcnt) {
return ret;
}
-/**
- * gkfs wrapper for pread() system calls
- * errno may be set
- * @param fd
- * @param buf
- * @param count
- * @param offset
- * @return read size or -1 on error
- */
-ssize_t
-gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) {
- auto gkfs_fd = CTX->file_map()->get(fd);
- return gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, offset);
-}
-
/**
* wrapper function for opening directories
* errno may be set
@@ -1394,6 +1440,28 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp,
return written;
}
+/**
+ * @brief Closes an fd. To be used externally
+ *
+ * @param fd
+ * @return int
+ */
+int
+gkfs_close(unsigned int fd) {
+ if(CTX->file_map()->exist(fd)) {
+ // No call to the daemon is required
+ CTX->file_map()->remove(fd);
+ return 0;
+ }
+
+ if(CTX->is_internal_fd(fd)) {
+ // the client application (for some reason) is trying to close an
+ // internal fd: ignore it
+ return 0;
+ }
+
+ return -1;
+}
#ifdef HAS_SYMLINKS
#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS
@@ -1482,6 +1550,28 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize) {
#endif
#endif
+
+std::vector
+gkfs_get_file_list(const std::string& path) {
+ auto ret = gkfs::rpc::forward_get_dirents(path);
+ auto err = ret.first;
+ if(err) {
+ errno = err;
+ return {};
+ }
+
+ auto open_dir = ret.second;
+
+ std::vector file_list;
+ unsigned int pos = 0;
+
+ while(pos < open_dir->size()) {
+ auto de = open_dir->getdent(pos++);
+ file_list.push_back(de.name());
+ }
+ return file_list;
+}
+
} // namespace gkfs::syscall
diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp
index 56e1f829b3319d2a91d21504a756639a927b1a35..a20503a5dbd3133c72496db769d7da1b6e0d0893 100644
--- a/src/client/hooks.cpp
+++ b/src/client/hooks.cpp
@@ -91,17 +91,10 @@ hook_close(int fd) {
LOG(DEBUG, "{}() called with fd: {}", __func__, fd);
- if(CTX->file_map()->exist(fd)) {
- // No call to the daemon is required
- CTX->file_map()->remove(fd);
- return 0;
- }
+ auto ret = gkfs::syscall::gkfs_close(fd);
- if(CTX->is_internal_fd(fd)) {
- // the client application (for some reason) is trying to close an
- // internal fd: ignore it
+ if(ret == 0)
return 0;
- }
return syscall_no_intercept_wrapper(SYS_close, fd);
}
@@ -242,7 +235,7 @@ hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos) {
fd, fmt::ptr(buf), count, pos);
if(CTX->file_map()->exist(fd)) {
- return with_errno(gkfs::syscall::gkfs_pread_ws(fd, buf, count, pos));
+ return with_errno(gkfs::syscall::gkfs_pread(fd, buf, count, pos));
}
/* Since kernel 2.6: pread() became pread64(), and pwrite() became
* pwrite64(). */
@@ -296,7 +289,7 @@ hook_pwrite(unsigned int fd, const char* buf, size_t count, loff_t pos) {
fd, fmt::ptr(buf), count, pos);
if(CTX->file_map()->exist(fd)) {
- return with_errno(gkfs::syscall::gkfs_pwrite_ws(fd, buf, count, pos));
+ return with_errno(gkfs::syscall::gkfs_pwrite(fd, buf, count, pos));
}
/* Since kernel 2.6: pread() became pread64(), and pwrite() became
* pwrite64(). */
@@ -878,7 +871,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname,
"newname \"{}\", flags {}",
__func__, olddfd, oldname, newdfd, newname, flags);
- const char* oldpath_pass;
+ const char* oldpath_pass = oldname;
std::string oldpath_resolved;
auto oldpath_status =
CTX->relativize_fd_path(olddfd, oldname, oldpath_resolved);
@@ -902,7 +895,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname,
return -EINVAL;
}
- const char* newpath_pass;
+ const char* newpath_pass = newname;
std::string newpath_resolved;
auto newpath_status =
CTX->relativize_fd_path(newdfd, newname, newpath_resolved);
diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp
index 316c5e398403761b00d5f32917f153cca9c22c7b..7a8a455fe4aa63f6f3eab2bec381023dfbb888f0 100644
--- a/src/client/intercept.cpp
+++ b/src/client/intercept.cpp
@@ -44,6 +44,24 @@ extern "C" {
#include
}
+
+#ifdef BYPASS_SYSCALL
+int (*intercept_hook_point)(long syscall_number, long arg0, long arg1,
+ long arg2, long arg3, long arg4, long arg5,
+ long* result){};
+
+void (*intercept_hook_point_clone_child)(unsigned long flags, void* child_stack,
+ int* ptid, int* ctid, long newtls){};
+
+void (*intercept_hook_point_clone_parent)(unsigned long flags,
+ void* child_stack, int* ptid,
+ int* ctid, long newtls,
+ long returned_pid){};
+
+void (*intercept_hook_point_post_kernel)(long syscall_number, long arg0,
+ long arg1, long arg2, long arg3,
+ long arg4, long arg5, long result){};
+#endif
namespace {
thread_local bool reentrance_guard_flag;
diff --git a/src/client/logging.cpp b/src/client/logging.cpp
index 88f9947140e070bbca5abc8f6410e7a0fd942588..830d5691ecd281c9974bc98a5ca299935fdc5b01 100644
--- a/src/client/logging.cpp
+++ b/src/client/logging.cpp
@@ -32,11 +32,7 @@
#include
#include
#include
-
-extern "C" {
-#include
-#include
-}
+#include
#ifdef GKFS_ENABLE_LOGGING
@@ -289,8 +285,7 @@ logger::logger(const std::string& opts, const std::string& path,
,
const std::string& filter, int verbosity
#endif
- )
- : timezone_(nullptr) {
+) {
/* use stderr by default */
log_fd_ = 2;
@@ -335,34 +330,6 @@ logger::logger(const std::string& opts, const std::string& path,
log_fd_ = fd;
}
- // Finding the current timezone implies accessing OS files (i.e. syscalls),
- // but current_zone() doesn't actually retrieve the time zone but rather
- // provides a descriptor to it that is **atomically initialized** upon its
- // first use. Thus, if we don't force the initialization here, logging the
- // first intercepted syscall will produce a call to date::time_zone::init()
- // (under std::call_once) which internally ends up calling fopen(). Since
- // fopen() ends up calling sys_open(), we will need to generate another
- // timestamp for a system call log entry, which will attempt to call
- // date::time_zone::init() since the prior initialization (under the same
- // std::call_once) has not yet completed.
- //
- // Unfortunately, date::time_zone doesn't provide a function to prevent
- // this lazy initialization, therefore we force it by requesting
- // information from an arbitrary timepoint (January 1st 1970) which forces
- // the initialization. This doesn't do any actual work and could safely be
- // removed if the date API ends up providing this functionality.
- try {
- timezone_ = date::current_zone();
-#ifdef GKFS_DEBUG_BUILD
- using namespace date;
- timezone_->get_info(date::sys_days{January / 1 / 1970});
-#endif // GKFS_DEBUG_BUILD
- } catch(const std::exception& ex) {
- // if timezone initialization fails, setting timezone_ to nullptr
- // makes format_timestamp_to() default to producing epoch timestamps
- timezone_ = nullptr;
- }
-
#ifdef GKFS_ENABLE_LOGGING
const auto log_hermes_message =
[](const std::string& msg, hermes::log::level l, int severity,
@@ -490,9 +457,9 @@ logger::log_syscall(syscall::info info, const long syscall_number,
print_syscall:
- static_buffer buffer;
+ fmt::basic_memory_buffer buffer;
- detail::format_timestamp_to(buffer, timezone_);
+ detail::format_timestamp_to(buffer);
detail::format_syscall_info_to(buffer, info);
if(result) {
@@ -501,7 +468,7 @@ print_syscall:
syscall::decode(buffer, syscall_number, args);
}
- fmt::format_to(buffer, "\n");
+ fmt::format_to(std::back_inserter(buffer), "\n");
::syscall_no_intercept(SYS_write, log_fd_, buffer.data(), buffer.size());
}
diff --git a/src/client/path.cpp b/src/client/path.cpp
index 4f72b4807c3cc09da756ab769aebe89cd9871e05..85357209a24ae47a87ed8358cd8d3cad255f232a 100644
--- a/src/client/path.cpp
+++ b/src/client/path.cpp
@@ -39,9 +39,14 @@
#include
#include
+#ifndef BYPASS_SYSCALL
+#include
+#else
+#include
+#endif
+
extern "C" {
#include
-#include
}
using namespace std;
diff --git a/src/client/preload.cpp b/src/client/preload.cpp
index 4de0df2c31168fe6f07a434a59e0eeb2331be27c..8f3e1e9881baa652104134771b384b4fbe6ad536 100644
--- a/src/client/preload.cpp
+++ b/src/client/preload.cpp
@@ -36,6 +36,9 @@
#include
#include
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+#include
+#endif
#include
#include
@@ -50,13 +53,13 @@ std::unique_ptr ld_network_service; // extern variable
namespace {
-#ifdef GKFS_ENABLE_FORWARDING
+// FORWARDING
pthread_t mapper;
bool forwarding_running;
pthread_mutex_t remap_mutex;
pthread_cond_t remap_signal;
-#endif
+// END FORWARDING
inline void
exit_error_msg(int errcode, const string& msg) {
@@ -67,8 +70,10 @@ exit_error_msg(int errcode, const string& msg) {
// if we don't disable interception before calling ::exit()
// syscall hooks may find an inconsistent in shared state
// (e.g. the logger) and thus, crash
- gkfs::preload::stop_interception();
- CTX->disable_interception();
+ if(CTX->interception_enabled()) {
+ gkfs::preload::stop_interception();
+ CTX->disable_interception();
+ }
::exit(errcode);
}
@@ -102,7 +107,6 @@ init_hermes_client() {
return true;
}
-#ifdef GKFS_ENABLE_FORWARDING
void*
forwarding_mapper(void* p) {
struct timespec timeout;
@@ -115,7 +119,7 @@ forwarding_mapper(void* p) {
try {
gkfs::utils::load_forwarding_map();
- if(previous != CTX->fwd_host_id()) {
+ if(previous != (int64_t) CTX->fwd_host_id()) {
LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id());
previous = CTX->fwd_host_id();
@@ -133,18 +137,14 @@ forwarding_mapper(void* p) {
return nullptr;
}
-#endif
-#ifdef GKFS_ENABLE_FORWARDING
void
init_forwarding_mapper() {
forwarding_running = true;
pthread_create(&mapper, NULL, forwarding_mapper, NULL);
}
-#endif
-#ifdef GKFS_ENABLE_FORWARDING
void
destroy_forwarding_mapper() {
forwarding_running = false;
@@ -153,7 +153,6 @@ destroy_forwarding_mapper() {
pthread_join(mapper, NULL);
}
-#endif
void
log_prog_name() {
@@ -207,31 +206,34 @@ init_environment() {
}
/* Setup distributor */
-#ifdef GKFS_ENABLE_FORWARDING
- try {
- gkfs::utils::load_forwarding_map();
+ auto forwarding_map_file = gkfs::env::get_var(
+ gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path);
- LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id());
- } catch(std::exception& e) {
- exit_error_msg(
- EXIT_FAILURE,
- fmt::format("Unable set the forwarding host '{}'", e.what()));
- }
+ if(!forwarding_map_file.empty()) {
+ try {
+ gkfs::utils::load_forwarding_map();
+
+ LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id());
+ } catch(std::exception& e) {
+ exit_error_msg(EXIT_FAILURE,
+ fmt::format("Unable set the forwarding host '{}'",
+ e.what()));
+ }
+
+ auto forwarder_dist = std::make_shared(
+ CTX->fwd_host_id(), CTX->hosts().size());
+ CTX->distributor(forwarder_dist);
+ } else {
- auto forwarder_dist = std::make_shared(
- CTX->fwd_host_id(), CTX->hosts().size());
- CTX->distributor(forwarder_dist);
-#else
#ifdef GKFS_USE_GUIDED_DISTRIBUTION
- auto distributor = std::make_shared(
- CTX->local_host_id(), CTX->hosts().size());
+ auto distributor = std::make_shared(
+ CTX->local_host_id(), CTX->hosts().size());
#else
- auto distributor = std::make_shared(
- CTX->local_host_id(), CTX->hosts().size());
-#endif
- CTX->distributor(distributor);
+ auto distributor = std::make_shared(
+ CTX->local_host_id(), CTX->hosts().size());
#endif
-
+ CTX->distributor(distributor);
+ }
LOG(INFO, "Retrieving file system configuration...");
@@ -290,12 +292,20 @@ init_preload() {
CTX->unprotect_user_fds();
-#ifdef GKFS_ENABLE_FORWARDING
- init_forwarding_mapper();
-#endif
+ auto forwarding_map_file = gkfs::env::get_var(
+ gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path);
+ if(!forwarding_map_file.empty()) {
+ init_forwarding_mapper();
+ }
gkfs::preload::start_interception();
errno = oerrno;
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+ if(!CTX->init_metrics()) {
+ exit_error_msg(EXIT_FAILURE,
+ "Unable to initialize client metrics. Exiting...");
+ }
+#endif
}
/**
@@ -304,19 +314,60 @@ init_preload() {
*/
void
destroy_preload() {
-#ifdef GKFS_ENABLE_FORWARDING
- destroy_forwarding_mapper();
+ auto forwarding_map_file = gkfs::env::get_var(
+ gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path);
+ if(!forwarding_map_file.empty()) {
+ destroy_forwarding_mapper();
+ }
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+ LOG(INFO, "Flushing final metrics...");
+ CTX->write_metrics()->flush_msgpack();
+ CTX->read_metrics()->flush_msgpack();
+ LOG(INFO, "Metrics flushed. Total flush operations: {}",
+ CTX->write_metrics()->flush_count());
#endif
-
CTX->clear_hosts();
LOG(DEBUG, "Peer information deleted");
ld_network_service.reset();
LOG(DEBUG, "RPC subsystem shut down");
- gkfs::preload::stop_interception();
- CTX->disable_interception();
- LOG(DEBUG, "Syscall interception stopped");
+ if(CTX->interception_enabled()) {
+ gkfs::preload::stop_interception();
+ CTX->disable_interception();
+ LOG(DEBUG, "Syscall interception stopped");
+ }
LOG(INFO, "All subsystems shut down. Client shutdown complete.");
}
+
+
+/**
+ * @brief External functions to call linking the library
+ *
+ */
+extern "C" int
+gkfs_init() {
+ CTX->init_logging();
+
+ // from here ownwards it is safe to print messages
+ LOG(DEBUG, "Logging subsystem initialized");
+
+ gkfs::preload::init_environment();
+
+ return 0;
+}
+
+
+extern "C" int
+gkfs_end() {
+ CTX->clear_hosts();
+ LOG(DEBUG, "Peer information deleted");
+
+ ld_network_service.reset();
+ LOG(DEBUG, "RPC subsystem shut down");
+
+ LOG(INFO, "All subsystems shut down. Client shutdown complete.");
+
+ return 0;
+}
diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp
index 7101041dcdb77864a55ed671bdd014a250db5623..bbf9fab0b0224cbc5d7f927027940fc72ab0adf0 100644
--- a/src/client/preload_context.cpp
+++ b/src/client/preload_context.cpp
@@ -36,14 +36,23 @@
#include
#include
-#include
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+#include
+#endif
+#include
#include
#include
+#include
-extern "C" {
+#ifndef BYPASS_SYSCALL
#include
+#else
+#include
+#endif
+
+extern "C" {
#include
}
@@ -69,6 +78,10 @@ PreloadContext::PreloadContext()
std::stoi(gkfs::env::get_var(gkfs::env::NUM_REPL, "0")));
}
+// Destructor set here to allow unique_ptr of forward declared classes in the
+// header. T must be complete at the point of deletion.
+PreloadContext::~PreloadContext() = default;
+
void
PreloadContext::init_logging() {
@@ -107,6 +120,74 @@ PreloadContext::init_logging() {
);
}
+bool
+PreloadContext::init_metrics() {
+#ifdef GKFS_ENABLE_CLIENT_METRICS
+ auto flush_interval = std::stoi(gkfs::env::get_var(
+ gkfs::env::METRICS_FLUSH_INTERVAL,
+ std::to_string(gkfs::config::client_metrics::flush_interval)));
+ if(gkfs::env::var_is_set(gkfs::env::METRICS_IP_PORT)) {
+ write_metrics_ = std::make_unique(
+ gkfs::messagepack::client_metric_io_type::write,
+ gkfs::messagepack::client_metric_flush_type::socket,
+ flush_interval);
+ read_metrics_ = std::make_unique(
+ gkfs::messagepack::client_metric_io_type::read,
+ gkfs::messagepack::client_metric_flush_type::socket,
+ flush_interval);
+ if(gkfs::env::var_is_set(gkfs::env::ENABLE_METRICS)) {
+ LOG(INFO,
+ "Client metrics enabled with ZeroMQ flushing. Initializing...");
+ write_metrics_->enable();
+ read_metrics_->enable();
+ auto metrics_ip = gkfs::env::get_var(gkfs::env::METRICS_IP_PORT);
+ write_metrics_->zmq_connect(metrics_ip);
+ if(!write_metrics_->zmq_is_connected()) {
+ LOG(ERROR, "Client write metrics failed to connect to : {}",
+ metrics_ip);
+ return false;
+ }
+ LOG(INFO, "Client write metrics connected to : {}", metrics_ip);
+ read_metrics_->zmq_connect(metrics_ip);
+ if(!read_metrics_->zmq_is_connected()) {
+ LOG(ERROR, "Client read metrics failed to connect to : {}",
+ metrics_ip);
+ return false;
+ }
+ LOG(INFO, "Client read metrics connected to : {}", metrics_ip);
+ }
+ } else {
+ write_metrics_ = std::make_unique(
+ gkfs::messagepack::client_metric_io_type::write,
+ gkfs::messagepack::client_metric_flush_type::file,
+ flush_interval);
+ read_metrics_ = std::make_unique(
+ gkfs::messagepack::client_metric_io_type::read,
+ gkfs::messagepack::client_metric_flush_type::file,
+ flush_interval);
+ if(gkfs::env::var_is_set(gkfs::env::ENABLE_METRICS)) {
+ LOG(INFO,
+ "Client metrics enabled with file flushing. Initializing...");
+ write_metrics_->enable();
+ read_metrics_->enable();
+ if(!gkfs::env::var_is_set(gkfs::env::METRICS_PATH)) {
+ LOG(WARNING, "No metrics path set. Using default path at {}",
+ gkfs::config::client_metrics::flush_path);
+ }
+ auto metrics_path = gkfs::env::get_var(
+ gkfs::env::METRICS_PATH,
+ gkfs::config::client_metrics::flush_path);
+ std::filesystem::create_directories(metrics_path);
+ write_metrics_->path(metrics_path, "write");
+ LOG(INFO, "Client write metrics path: {}", write_metrics_->path());
+ read_metrics_->path(metrics_path, "read");
+ LOG(INFO, "Client read metrics path: {}", read_metrics_->path());
+ }
+ }
+#endif
+ return true;
+}
+
void
PreloadContext::mountdir(const std::string& path) {
assert(gkfs::path::is_absolute(path));
@@ -448,7 +529,6 @@ PreloadContext::unprotect_user_fds() {
internal_fds_must_relocate_ = true;
}
-
std::string
PreloadContext::get_hostname() {
return hostname;
@@ -464,5 +544,15 @@ PreloadContext::get_replicas() {
return replicas_;
}
+const std::shared_ptr
+PreloadContext::write_metrics() {
+ return write_metrics_;
+}
+
+const std::shared_ptr
+PreloadContext::read_metrics() {
+ return read_metrics_;
+}
+
} // namespace preload
} // namespace gkfs
diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp
index aa09d0e0727d69e509aea64a1133063bc8db3e71..b70ddf2d9acaba9081176c30caea1ea75f0f3422 100644
--- a/src/client/preload_util.cpp
+++ b/src/client/preload_util.cpp
@@ -104,16 +104,15 @@ extract_protocol(const string& uri) {
throw runtime_error(fmt::format("Invalid format for URI: '{}'", uri));
}
string protocol{};
-
- if(uri.find(gkfs::rpc::protocol::ofi_sockets) != string::npos) {
- protocol = gkfs::rpc::protocol::ofi_sockets;
- } else if(uri.find(gkfs::rpc::protocol::ofi_psm2) != string::npos) {
- protocol = gkfs::rpc::protocol::ofi_psm2;
- } else if(uri.find(gkfs::rpc::protocol::ofi_verbs) != string::npos) {
- protocol = gkfs::rpc::protocol::ofi_verbs;
+ for(const auto& valid_protocol :
+ gkfs::rpc::protocol::all_remote_protocols) {
+ if(uri.find(valid_protocol) != string::npos) {
+ protocol = valid_protocol;
+ break;
+ }
}
- // check for shared memory protocol. Can be plain shared memory or real ofi
- // protocol + auto_sm
+ // check for shared memory protocol. Can be plain shared memory or real
+ // ofi protocol + auto_sm
if(uri.find(gkfs::rpc::protocol::na_sm) != string::npos) {
if(protocol.empty())
protocol = gkfs::rpc::protocol::na_sm;
@@ -286,7 +285,6 @@ metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md,
return 0;
}
-#ifdef GKFS_ENABLE_FORWARDING
map
load_forwarding_map_file(const std::string& lfpath) {
@@ -320,9 +318,7 @@ load_forwarding_map_file(const std::string& lfpath) {
}
return forwarding_map;
}
-#endif
-#ifdef GKFS_ENABLE_FORWARDING
void
load_forwarding_map() {
string forwarding_map_file;
@@ -359,7 +355,6 @@ load_forwarding_map() {
CTX->fwd_host_id(forwarding_map[local_hostname]);
}
-#endif
vector>
read_hosts_file() {
diff --git a/src/client/rpc/forward_management.cpp b/src/client/rpc/forward_management.cpp
index c975842e560d15a9f107b3138ba1016b853071ed..7504b1f6b6bb869d99faab62c946a620ecace192 100644
--- a/src/client/rpc/forward_management.cpp
+++ b/src/client/rpc/forward_management.cpp
@@ -47,7 +47,7 @@ forward_get_fs_config() {
bool found = false;
size_t idx = 0;
- while(!found && idx <= CTX->hosts().size()) {
+ while(!found && idx < CTX->hosts().size()) {
try {
LOG(DEBUG, "Retrieving file system configurations from daemon");
// TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that
diff --git a/src/client/syscalls/util.S b/src/client/syscalls/util.S
new file mode 100644
index 0000000000000000000000000000000000000000..67c9b279afdcad35a0409dd800c392f079029956
--- /dev/null
+++ b/src/client/syscalls/util.S
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2016-2017, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+.global syscall_no_intercept;
+.type syscall_no_intercept, @function
+
+.text
+
+syscall_no_intercept:
+ movq %rdi, %rax /* convert from linux ABI calling */
+ movq %rsi, %rdi /* convention to syscall calling convention */
+ movq %rdx, %rsi
+ movq %rcx, %rdx
+ movq %r8, %r10
+ movq %r9, %r8
+ movq 8(%rsp), %r9
+ syscall
+ ret
+
+.size syscall_no_intercept, .-syscall_no_intercept
diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 73b7348dea21e749b40a02e431bb85b1bb0bd297..5254f93f532e19c09b48782e5c8a0b3182fa5cd0 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -104,14 +104,29 @@ target_link_libraries(metadata
fmt::fmt
)
-add_library(path_util
- STATIC
- )
+add_library(path_util STATIC)
set_property(TARGET path_util PROPERTY POSITION_INDEPENDENT_CODE ON)
-
target_sources(path_util
PRIVATE
path_util.cpp
PUBLIC
${INCLUDE_DIR}/common/path_util.hpp
)
+
+if (GKFS_ENABLE_CLIENT_METRICS)
+ add_library(msgpack_util STATIC)
+ set_property(TARGET msgpack_util PROPERTY POSITION_INDEPENDENT_CODE ON)
+ set_target_properties(msgpack_util PROPERTIES LINKER_LANGUAGE CXX)
+ target_sources(msgpack_util
+ PUBLIC
+ ${INCLUDE_DIR}/common/msgpack_util.hpp
+ PRIVATE
+ ${CMAKE_CURRENT_LIST_DIR}/msgpack_util.cpp
+ )
+ target_link_libraries(msgpack_util
+ PUBLIC
+ Msgpack::Msgpack
+ cppzmq
+ rpc_utils
+ )
+endif ()
diff --git a/src/common/env_util.cpp b/src/common/env_util.cpp
index 352217f292e0846caf81902ad7606566ca7671bd..63f5c6cda7a65007359b9896e2617a30e6f73874 100644
--- a/src/common/env_util.cpp
+++ b/src/common/env_util.cpp
@@ -32,8 +32,18 @@
using namespace std;
+namespace gkfs::env {
+
string
-gkfs::env::get_var(const string& name, const string& default_value) {
+get_var(const string& name, const string& default_value) {
const char* const val = ::secure_getenv(name.c_str());
return val != nullptr ? string(val) : default_value;
}
+
+bool
+var_is_set(const string& name) {
+ const char* const val = ::secure_getenv(name.c_str());
+ return val != nullptr;
+}
+
+} // namespace gkfs::env
diff --git a/src/common/msgpack_util.cpp b/src/common/msgpack_util.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..18bd3fa0d6a66944eed7158d14a6595715b9b149
--- /dev/null
+++ b/src/common/msgpack_util.cpp
@@ -0,0 +1,208 @@
+/*
+ Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain
+ Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany
+
+ This software was partially supported by the
+ EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+ This software was partially supported by the
+ ADA-FS project under the SPPEXA project funded by the DFG.
+
+ This file is part of GekkoFS' POSIX interface.
+
+ GekkoFS' POSIX interface is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of the License,
+ or (at your option) any later version.
+
+ GekkoFS' POSIX interface is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with GekkoFS' POSIX interface. If not, see
+ .
+
+ SPDX-License-Identifier: LGPL-3.0-or-later
+*/
+
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+
+extern "C" {
+#include
+#include
+}
+
+using namespace std;
+
+namespace gkfs::messagepack {
+
+ClientMetrics::ClientMetrics(client_metric_io_type io_type,
+ client_metric_flush_type ftype, int flush_interval)
+ : flush_interval_(flush_interval) {
+ init_t_ = std::chrono::system_clock::now();
+ msgpack_data_.hostname_ = gkfs::rpc::get_my_hostname(true);
+ msgpack_data_.pid_ = getpid();
+ if(io_type == client_metric_io_type::write) {
+ msgpack_data_.io_type_ = "w";
+ } else {
+ msgpack_data_.io_type_ = "r";
+ }
+ if(ftype == client_metric_flush_type::file) {
+ // set default path
+ flush_type_ = client_metric_flush_type::file;
+ flush_path_ = gkfs::config::client_metrics::flush_path;
+ } else {
+ flush_type_ = client_metric_flush_type::socket;
+ zmq_flush_context_ = std::make_unique(1);
+ zmq_flush_socket_ =
+ std::make_unique(*zmq_flush_context_, ZMQ_PUSH);
+ }
+ flush_thread_running_ = true;
+ flush_thread_ = std::thread(&ClientMetrics::flush_loop, this);
+}
+
+ClientMetrics::~ClientMetrics() {
+ flush_thread_running_ = false;
+ flush_thread_cv_.notify_one();
+ if(flush_thread_.joinable())
+ flush_thread_.join();
+ if(flush_type_ == client_metric_flush_type::socket) {
+ zmq_flush_socket_->close();
+ zmq_flush_context_->close();
+ }
+}
+
+void
+ClientMetrics::add_event(
+ size_t size, std::chrono::time_point start) {
+ if(!metrics_enabled_)
+ return;
+ auto end = std::chrono::system_clock::now();
+ std::lock_guard const data_lock(data_mtx_);
+ auto start_offset =
+ std::chrono::duration(start - init_t_);
+ auto end_offset = std::chrono::duration(end - init_t_);
+ auto duration = std::chrono::duration(end_offset -
+ start_offset);
+ msgpack_data_.total_bytes_ += size;
+ // auto size_mib = size / (1024 * 1024); // in MiB
+ // auto duration_s = duration.count() / 1000; // in seconds
+ // throw away decimals
+ msgpack_data_.start_t_.emplace_back(
+ static_cast(start_offset.count()));
+ msgpack_data_.end_t_.emplace_back(static_cast(end_offset.count()));
+ msgpack_data_.req_size_.emplace_back(size);
+ msgpack_data_.total_iops_ += 1;
+}
+
+void
+ClientMetrics::reset_metrics() {
+ msgpack_data_.start_t_.clear();
+ msgpack_data_.end_t_.clear();
+ msgpack_data_.flush_t_ = 0;
+ msgpack_data_.req_size_.clear();
+ msgpack_data_.total_bytes_ = 0;
+ msgpack_data_.total_iops_ = 0;
+}
+
+void
+ClientMetrics::flush_msgpack() {
+ if(!metrics_enabled_)
+ return;
+ std::lock_guard const data_lock(data_mtx_);
+ if(msgpack_data_.total_iops_ == 0)
+ return;
+ auto flush_t_now = std::chrono::system_clock::now();
+ msgpack_data_.flush_t_ = static_cast(
+ std::chrono::duration(flush_t_now - init_t_)
+ .count());
+ auto data = msgpack_data_.pack_msgpack();
+ if(flush_type_ == client_metric_flush_type::file) {
+ auto fd =
+ open(flush_path_.c_str(), O_CREAT | O_WRONLY | O_APPEND, 0666);
+ if(fd < 0) {
+ // cout << "error open" << endl;
+ exit(1);
+ }
+ write(fd, data.data(), data.size());
+ // auto written = write(fd, data.data(), data.size());
+ // cout << "written: " << written << endl;
+ close(fd);
+ } else {
+ zmq::message_t message(data.size());
+ // copy data from serialized msgpack to zmq message
+ memcpy(message.data(), data.data(), data.size());
+ // non-blocking zmq send
+ if(zmq_flush_socket_->send(message, zmq::send_flags::none) == -1) {
+ std::cerr << "Failed to send zmq message" << std::endl;
+ }
+ }
+ reset_metrics();
+ flush_count_++;
+}
+
+void
+ClientMetrics::flush_loop() {
+ while(flush_thread_running_) {
+ std::unique_lock wait_lock(
+ flush_thread_cv_mutex_); // Lock only for wait_for
+ flush_thread_cv_.wait_for(wait_lock,
+ std::chrono::seconds(flush_interval_),
+ [&] { return !flush_thread_running_; });
+ flush_msgpack();
+ }
+}
+
+void
+ClientMetrics::enable() {
+ metrics_enabled_ = true;
+}
+
+void
+ClientMetrics::disable() {
+ metrics_enabled_ = false;
+}
+
+void
+ClientMetrics::zmq_connect(const string& ip_port) {
+ auto address = "tcp://" + ip_port;
+ zmq_flush_socket_->connect(address);
+}
+
+bool
+ClientMetrics::zmq_is_connected() {
+ return zmq_flush_socket_->handle() != nullptr;
+}
+
+const string&
+ClientMetrics::path() const {
+ return flush_path_;
+}
+void
+ClientMetrics::path(const string& path, const string prefix) {
+ const std::time_t t = std::chrono::system_clock::to_time_t(init_t_);
+ std::stringstream init_t_stream;
+ init_t_stream << std::put_time(std::localtime(&t), "%F_%T");
+ flush_path_ = path + "/" + prefix + "_" + init_t_stream.str() + "_" +
+ msgpack_data_.hostname_ + "_" +
+ to_string(msgpack_data_.pid_) + ".msgpack";
+}
+int
+ClientMetrics::flush_count() const {
+ return flush_count_;
+}
+
+
+} // namespace gkfs::messagepack
diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt
index 95c4ade7542ead463ea3565842481a82738fa102..89af71c7624109df1d90b7c4bae1b29fd030b193 100644
--- a/src/daemon/CMakeLists.txt
+++ b/src/daemon/CMakeLists.txt
@@ -72,63 +72,15 @@ target_link_libraries(
Threads::Threads
)
+if(GKFS_ENABLE_AGIOS)
+ target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp)
+ target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS)
+ target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS)
+endif()
+
if(GKFS_ENABLE_CODE_COVERAGE)
target_code_coverage(gkfs_daemon AUTO)
endif()
install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
-# ##############################################################################
-# This builds the `gkfwd_daemon` executable: the daemon for GekkoFS data
-# forwarding mode.
-# ##############################################################################
-
-if(GKFS_ENABLE_FORWARDING)
-
- add_executable(gkfwd_daemon)
-
- # We need to add here any files that may have different compile definitions
- target_sources(
- gkfwd_daemon
- PRIVATE ../common/rpc/rpc_util.cpp
- daemon.cpp
- util.cpp
- ops/metadentry.cpp
- ops/data.cpp
- classes/fs_data.cpp
- classes/rpc_data.cpp
- handler/srv_metadata.cpp
- handler/srv_management.cpp
- handler/srv_data.cpp
- )
-
- target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_FORWARDING)
- target_link_libraries(
- gkfwd_daemon
- PUBLIC # internal libs
- metadata
- metadata_backend
- storage
- distributor
- statistics
- log_util
- env_util
- path_util
- # external libs
- CLI11::CLI11
- fmt::fmt
- Mercury::Mercury
- Argobots::Argobots
- Margo::Margo
- # others
- Threads::Threads
- )
-
- if(GKFS_ENABLE_AGIOS)
- target_sources(gkfwd_daemon PRIVATE scheduler/agios.cpp)
- target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_AGIOS)
- target_link_libraries(gkfwd_daemon PRIVATE AGIOS::AGIOS)
- endif()
-
- install(TARGETS gkfwd_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
-endif ()
diff --git a/src/daemon/backend/metadata/merge.cpp b/src/daemon/backend/metadata/merge.cpp
index 1c1a2a6fc03e283112de8df70e23c99c62beca9a..84fd5c66f79a5c6d8e5681d5c10645989367a07b 100644
--- a/src/daemon/backend/metadata/merge.cpp
+++ b/src/daemon/backend/metadata/merge.cpp
@@ -137,6 +137,26 @@ CreateOperand::serialize_params() const {
return metadata;
}
+UpdateTimeOperand::UpdateTimeOperand(time_t mtime) : mtime_(mtime) {}
+
+UpdateTimeOperand::UpdateTimeOperand(const rdb::Slice& serialized_op) {
+ // Parse size
+ size_t read = 0;
+ mtime_ = ::stoul(serialized_op.ToString(), &read);
+ // check that we consumed all the input string
+ assert(read == serialized_op.size());
+}
+
+OperandID
+UpdateTimeOperand::id() const {
+ return OperandID::update_time;
+}
+
+string
+UpdateTimeOperand::serialize_params() const {
+ return ::to_string(mtime_);
+}
+
/**
* @internal
* Merges all operands in chronological order for the same key.
@@ -181,10 +201,6 @@ MetadataMergeOperator::FullMergeV2(const MergeOperationInput& merge_in,
auto operand_id = MergeOperand::get_id(serialized_op);
auto parameters = MergeOperand::get_params(serialized_op);
- if constexpr(gkfs::config::metadata::use_mtime) {
- md.update_mtime_now();
- }
-
if(operand_id == OperandID::increase_size) {
auto op = IncreaseSizeOperand(parameters);
if(op.append()) {
@@ -204,6 +220,9 @@ MetadataMergeOperator::FullMergeV2(const MergeOperationInput& merge_in,
fsize = op.size();
} else if(operand_id == OperandID::create) {
continue;
+ } else if(operand_id == OperandID::update_time) {
+ auto op = UpdateTimeOperand(parameters);
+ md.mtime(op.mtime());
} else {
throw ::runtime_error("Unrecognized merge operand ID: " +
(char) operand_id);
diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp
index 3e8e0d687d7c7a6f610904526cfec9624f978b6e..508e7bd82df15a91806a44f68a7fb1e63a63b778 100644
--- a/src/daemon/backend/metadata/rocksdb_backend.cpp
+++ b/src/daemon/backend/metadata/rocksdb_backend.cpp
@@ -34,7 +34,7 @@
#include
#include
#include
-#include
+#include
extern "C" {
#include
}
@@ -244,6 +244,15 @@ RocksDBBackend::increase_size_impl(const std::string& key, size_t io_size,
if(!s.ok()) {
throw_status_excpt(s);
}
+ if constexpr(gkfs::config::metadata::use_mtime) {
+ // get current time and update mtime for this file
+ time_t now = time(nullptr);
+ auto t_op = UpdateTimeOperand(now);
+ s = db_->Merge(write_opts_, key, t_op.serialize());
+ if(!s.ok()) {
+ throw_status_excpt(s);
+ }
+ }
}
return out_offset;
}
@@ -263,6 +272,15 @@ RocksDBBackend::decrease_size_impl(const std::string& key, size_t size) {
if(!s.ok()) {
throw_status_excpt(s);
}
+ if constexpr(gkfs::config::metadata::use_mtime) {
+ // get current time and update mtime for this file
+ time_t now = time(nullptr);
+ auto t_op = UpdateTimeOperand(now);
+ s = db_->Merge(write_opts_, key, t_op.serialize());
+ if(!s.ok()) {
+ throw_status_excpt(s);
+ }
+ }
}
/**
diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp
index 0d38c718c5af2b76b9c8be8414088a4239ee1c52..33b43bf5942789507cc951f737ac1da2c7188a9e 100644
--- a/src/daemon/classes/fs_data.cpp
+++ b/src/daemon/classes/fs_data.cpp
@@ -256,6 +256,15 @@ FsData::enable_chunkstats(bool enable_chunkstats) {
FsData::enable_chunkstats_ = enable_chunkstats;
}
+bool
+FsData::enable_forwarding() const {
+ return enable_forwarding_;
+}
+
+void
+FsData::enable_forwarding(bool enable_forwarding) {
+ FsData::enable_forwarding_ = enable_forwarding;
+}
bool
FsData::enable_prometheus() const {
return enable_prometheus_;
diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp
index 49a87fa604e8b6d747c6d3e9dd19b66f9ee27211..6bc229cd9b237fabe147a9221cd3bbe96c3c841f 100644
--- a/src/daemon/daemon.cpp
+++ b/src/daemon/daemon.cpp
@@ -261,25 +261,6 @@ init_environment() {
throw;
}
- GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ", __func__);
- try {
-#ifdef GKFS_USE_GUIDED_DISTRIBUTION
- auto distributor = std::make_shared();
-#else
- auto distributor = std::make_shared();
-#endif
- RPC_DATA->distributor(distributor);
- } catch(const std::exception& e) {
- GKFS_DATA->spdlogger()->error(
- "{}() Failed to initialize Distributor: {}", __func__,
- e.what());
- throw;
- }
-
-#ifdef GKFS_ENABLE_FORWARDING
- GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__);
-#endif
-
#ifdef GKFS_ENABLE_AGIOS
// Initialize AGIOS scheduler
GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'",
@@ -392,9 +373,10 @@ agios_initialize() {
*/
void
destroy_enviroment() {
- GKFS_DATA->spdlogger()->debug("{}() Removing mount directory", __func__);
+ // GKFS_DATA->spdlogger()->debug("{}() Removing mount directory",
+ // __func__);
std::error_code ecode;
- fs::remove_all(GKFS_DATA->mountdir(), ecode);
+ // fs::remove_all(GKFS_DATA->mountdir(), ecode);
GKFS_DATA->spdlogger()->debug("{}() Freeing I/O executions streams",
__func__);
for(unsigned int i = 0; i < RPC_DATA->io_streams().size(); i++) {
@@ -496,13 +478,18 @@ parse_input(const cli_options& opts, const CLI::App& desc) {
auto rpc_protocol = string(gkfs::rpc::protocol::ofi_sockets);
if(desc.count("--rpc-protocol")) {
rpc_protocol = opts.rpc_protocol;
- if(rpc_protocol != gkfs::rpc::protocol::ofi_verbs &&
- rpc_protocol != gkfs::rpc::protocol::ofi_sockets &&
- rpc_protocol != gkfs::rpc::protocol::ofi_psm2) {
+ auto protocol_found = false;
+ for(const auto& valid_protocol :
+ gkfs::rpc::protocol::all_remote_protocols) {
+ if(rpc_protocol == valid_protocol) {
+ protocol_found = true;
+ break;
+ }
+ }
+ if(!protocol_found)
throw runtime_error(fmt::format(
"Given RPC protocol '{}' not supported. Check --help for supported protocols.",
rpc_protocol));
- }
}
auto use_auto_sm = desc.count("--auto-sm") != 0;
@@ -552,10 +539,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) {
assert(desc.count("--rootdir"));
auto rootdir = opts.rootdir;
-#ifdef GKFS_ENABLE_FORWARDING
- // In forwarding mode, the backend is shared
- auto rootdir_path = fs::path(rootdir);
-#else
auto rootdir_path = fs::path(rootdir);
if(desc.count("--rootdir-suffix")) {
if(opts.rootdir_suffix == gkfs::config::data::chunk_dir ||
@@ -570,7 +553,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) {
rootdir_path /= opts.rootdir_suffix;
GKFS_DATA->rootdir_suffix(opts.rootdir_suffix);
}
-#endif
if(desc.count("--clean-rootdir")) {
// may throw exception (caught in main)
@@ -589,14 +571,23 @@ parse_input(const cli_options& opts, const CLI::App& desc) {
fs::create_directories(rootdir_path);
GKFS_DATA->rootdir(rootdir_path.native());
+ if(desc.count("--enable-forwarding")) {
+ GKFS_DATA->enable_forwarding(true);
+ GKFS_DATA->spdlogger()->info("{}() Forwarding mode enabled", __func__);
+ }
+
if(desc.count("--metadir")) {
auto metadir = opts.metadir;
-#ifdef GKFS_ENABLE_FORWARDING
- auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str();
-#else
+
auto metadir_path = fs::path(metadir);
-#endif
+ if(GKFS_DATA->enable_forwarding()) {
+ // As we store normally he metadata to the pfs, we need to put each
+ // daemon in a separate directory.
+ metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str();
+ }
+
+
if(desc.count("--clean-rootdir")) {
// may throw exception (caught in main)
GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...",
@@ -613,13 +604,16 @@ parse_input(const cli_options& opts, const CLI::App& desc) {
// use rootdir as metadata dir
auto metadir = opts.rootdir;
-#ifdef GKFS_ENABLE_FORWARDING
- auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str();
- fs::create_directories(metadir_path);
- GKFS_DATA->metadir(fs::canonical(metadir_path).native());
-#else
- GKFS_DATA->metadir(GKFS_DATA->rootdir());
-#endif
+
+ if(GKFS_DATA->enable_forwarding()) {
+ // As we store normally he metadata to the pfs, we need to put each
+ // daemon in a separate directory.
+ auto metadir_path =
+ fs::path(metadir) / fmt::format_int(getpid()).str();
+ fs::create_directories(metadir_path);
+ GKFS_DATA->metadir(fs::canonical(metadir_path).native());
+ } else
+ GKFS_DATA->metadir(GKFS_DATA->rootdir());
}
if(desc.count("--dbbackend")) {
@@ -787,6 +781,9 @@ main(int argc, const char* argv[]) {
desc.add_option(
"--output-stats", opts.stats_file,
"Creates a thread that outputs the server stats each 10s to the specified file.");
+ desc.add_flag(
+ "--enable-forwarding",
+ "Enables forwarding mode, so the metadata is stored in a separate directory (pid).");
#ifdef GKFS_ENABLE_PROMETHEUS
desc.add_flag(
"--enable-prometheus",
diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp
index 9a4e1697355d95c535e74ad44a73ad924de39c9d..b59f5c551071222630fe11c439769d2a0a910d02 100644
--- a/src/daemon/handler/srv_data.cpp
+++ b/src/daemon/handler/srv_data.cpp
@@ -109,7 +109,7 @@ rpc_srv_write(hg_handle_t handle) {
return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle);
}
auto hgi = margo_get_info(handle);
- auto mid = margo_hg_info_get_instance(hgi);
+ auto mid = margo_hg_handle_get_instance(handle);
auto bulk_size = margo_bulk_get_size(in.bulk_handle);
GKFS_DATA->spdlogger()->debug(
"{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'",
@@ -244,8 +244,6 @@ rpc_srv_write(hg_handle_t handle) {
GKFS_DATA->stats()->add_write(in.path, chnk_id_file);
}
- GKFS_DATA->spdlogger()->error("{}() Processing at host {} -> {}",
- __func__, host_id, chnk_id_file);
chnk_ids_host[chnk_id_curr] =
chnk_id_file; // save this id to host chunk list
// offset case. Only relevant in the first iteration of the loop and if
@@ -414,7 +412,7 @@ rpc_srv_read(hg_handle_t handle) {
return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle);
}
auto hgi = margo_get_info(handle);
- auto mid = margo_hg_info_get_instance(hgi);
+ auto mid = margo_hg_handle_get_instance(handle);
auto bulk_size = margo_bulk_get_size(in.bulk_handle);
GKFS_DATA->spdlogger()->debug(
diff --git a/src/daemon/handler/srv_management.cpp b/src/daemon/handler/srv_management.cpp
index 585646aeddee5c3094a5573b3fd3cd51877c1af7..ef5f8735d0fdcccb4eebda2690ce6756eca6b8d6 100644
--- a/src/daemon/handler/srv_management.cpp
+++ b/src/daemon/handler/srv_management.cpp
@@ -72,8 +72,9 @@ rpc_srv_get_fs_config(hg_handle_t handle) {
out.blocks_state = static_cast(GKFS_DATA->blocks_state());
out.uid = getuid();
out.gid = getgid();
- GKFS_DATA->spdlogger()->debug("{}() Sending output configs back to library",
- __func__);
+ GKFS_DATA->spdlogger()->debug(
+ "{}() Sending output configs back to library. mountdir '{}' rootdir '{}'",
+ __func__, out.mountdir, out.rootdir);
auto hret = margo_respond(handle, &out);
if(hret != HG_SUCCESS) {
GKFS_DATA->spdlogger()->error(
diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp
index e68069797fc8104894399f10a38ee25d323234b8..1339170b94aa61072451c759f1f1b8b1ba849afd 100644
--- a/src/daemon/handler/srv_metadata.cpp
+++ b/src/daemon/handler/srv_metadata.cpp
@@ -537,7 +537,7 @@ rpc_srv_get_dirents(hg_handle_t handle) {
// Retrieve size of source buffer
auto hgi = margo_get_info(handle);
- auto mid = margo_hg_info_get_instance(hgi);
+ auto mid = margo_hg_handle_get_instance(handle);
auto bulk_size = margo_bulk_get_size(in.bulk_handle);
GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ",
__func__, in.path, bulk_size);
@@ -696,7 +696,7 @@ rpc_srv_get_dirents_extended(hg_handle_t handle) {
// Retrieve size of source buffer
auto hgi = margo_get_info(handle);
- auto mid = margo_hg_info_get_instance(hgi);
+ auto mid = margo_hg_handle_get_instance(handle);
auto bulk_size = margo_bulk_get_size(in.bulk_handle);
GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ",
__func__, in.path, bulk_size);
diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py
index bc8e0c86427bf14538ce3388b95755f78cabceac..4bc63e41d3590bae7eb7c88070bf9c7362c8b81f 100644
--- a/tests/integration/forwarding/test_map.py
+++ b/tests/integration/forwarding/test_map.py
@@ -165,7 +165,7 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory):
lines = f.readlines()
for line in lines:
- if 'Forward to' in line:
+ if 'forwarding_mapper() Forward to' in line:
ion = line.split()[-1]
assert ion == '0'
@@ -190,15 +190,16 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory):
ret = c00.write(file, buf, len(buf))
assert ret.retval == len(buf) # Return the number of read bytes
-
+
+ # log line should go to the end of the file
with open(c00.log) as f:
lines = f.readlines()
-
+ ion = 0
for line in lines:
- if 'Forward to' in line:
+ if 'forwarding_mapper() Forward to' in line:
ion = line.split()[-1]
- assert ion == '1'
+ assert ion == '1'
def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory):
"""Write files from one client and read in the other using two daemons"""
diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt
index 9c3e97aad140f5914f37ffcd90ce6b06024f0139..03d73b6bf96bed9c2439fecae4568350eec7b912 100644
--- a/tests/integration/harness/CMakeLists.txt
+++ b/tests/integration/harness/CMakeLists.txt
@@ -70,30 +70,7 @@ add_executable(gkfs.io
gkfs.io/rename.cpp
)
-include(FetchContent)
-
-set(FETCHCONTENT_QUIET OFF)
-
-FetchContent_Declare(nlohmann_json
- DOWNLOAD_EXTRACT_TIMESTAMP ON
- URL https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz)
-
-FetchContent_GetProperties(nlohmann_json)
-
-if(NOT nlohmann_json_POPULATED)
- FetchContent_Populate(nlohmann_json)
- message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}")
- message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}")
-
- # we don't really care so much about a third party library's tests to be
- # run from our own project's code
- set(JSON_BuildTests OFF CACHE INTERNAL "")
-
- # we also don't need to install it when our main project gets installed
- set(JSON_Install OFF CACHE INTERNAL "")
-
- add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR})
-endif()
+include(load_nlohmann_json)
target_include_directories(gkfs.io PRIVATE
${BOOST_PREPROCESSOR_INCLUDE_DIRS}
diff --git a/tests/integration/harness/gkfs.io/serialize.hpp b/tests/integration/harness/gkfs.io/serialize.hpp
index 3873b2ec85d484a52f748820f150738ea9a938ec..92328fcf86dc53b3d71d3acfa63a4ec5de4ca72a 100644
--- a/tests/integration/harness/gkfs.io/serialize.hpp
+++ b/tests/integration/harness/gkfs.io/serialize.hpp
@@ -178,13 +178,10 @@ template <>
struct adl_serializer {
static void
to_json(json& j, const struct ::statfs opt) {
- j = json{{"f_type", opt.f_type},
- {"f_bsize", opt.f_bsize},
- {"f_blocks", opt.f_blocks},
- {"f_bfree", opt.f_bfree},
- {"f_bavail", opt.f_bavail},
- {"f_files", opt.f_files},
- {"f_ffree", opt.f_ffree}};
+ j = json{{"f_type", opt.f_type}, {"f_bsize", opt.f_bsize},
+ {"f_blocks", opt.f_blocks}, {"f_bfree", opt.f_bfree},
+ {"f_bavail", opt.f_bavail}, {"f_files", opt.f_files},
+ {"f_ffree", opt.f_ffree}};
}
};
@@ -253,7 +250,8 @@ struct formatter {
template
auto
- format(const struct ::dirent& dirent, FormatContext& ctx) {
+ format(const struct ::dirent& dirent, FormatContext& ctx) const
+ -> format_context::iterator {
return format_to(ctx.out(),
"struct dirent {{\n"
" d_ino = {};\n"
diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py
index ef48c404255306f5301ea029c4c189feb7e10c20..a65a171f7b7ad50f5129f2d27ebe196e7d729088 100644
--- a/tests/integration/harness/gkfs.py
+++ b/tests/integration/harness/gkfs.py
@@ -48,15 +48,16 @@ gkfs_client_log_level = 'all'
gkfs_client_log_syscall_filter = 'epoll_wait,epoll_create'
gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.'
-gkfwd_daemon_cmd = 'gkfwd_daemon'
+gkfwd_daemon_cmd = 'gkfs_daemon'
gkfwd_client_cmd = 'gkfs.io'
-gkfwd_client_lib_file = 'libgkfwd_intercept.so'
+gkfwd_client_lib_file = 'libgkfs_intercept.so'
gkfwd_hosts_file = 'gkfs_hosts.txt'
gkfwd_forwarding_map_file = 'gkfs_forwarding.map'
-gkfwd_daemon_log_file = 'gkfwd_daemon.log'
+gkfwd_daemon_log_file = 'gkfs_daemon.log'
gkfwd_daemon_log_level = '100'
-gkfwd_client_log_file = 'gkfwd_client.log'
+gkfwd_client_log_file = 'gkfs_client.log'
gkfwd_client_log_level = 'all'
+gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create'
gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.'
@@ -520,7 +521,8 @@ class ShellClient:
'LD_PRELOAD' : str(self._preload_library),
'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file),
'LIBGKFS_LOG' : gkfs_client_log_level,
- 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file)
+ 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file),
+ 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter
}
self._env.update(self._patched_env)
@@ -732,7 +734,8 @@ class FwdDaemon:
args = [ '--mountdir', self.mountdir,
'--metadir', self.metadir,
'--rootdir', self.rootdir,
- '-l', self._address ]
+ '-l', self._address,
+ '--enable-forwarding']
logger.debug(f"spawning daemon")
logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args)))
@@ -902,11 +905,12 @@ class FwdClient:
self._patched_env = {
'LD_LIBRARY_PATH' : libdirs,
- 'LD_PRELOAD' : self._preload_library,
+ 'LD_PRELOAD' : str(self._preload_library),
'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file),
'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local),
'LIBGKFS_LOG' : gkfs_client_log_level,
- 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local)
+ 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local),
+ 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter
}
self._env.update(self._patched_env)
@@ -996,7 +1000,8 @@ class ShellFwdClient:
'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file),
'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file),
'LIBGKFS_LOG' : gkfwd_client_log_level,
- 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file)
+ 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file),
+ 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter
}
self._env.update(self._patched_env)
diff --git a/tests/scripts/compile_dep.sh/0.9.3/agios.out b/tests/scripts/compile_dep.sh/0.9.3/agios.out
new file mode 100644
index 0000000000000000000000000000000000000000..25a1cad3a58a9c7c07c75ba87bfa2c5a6a9643ca
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/agios.out
@@ -0,0 +1,11 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
+######## Installing: agios ###############################
+
diff --git a/tests/scripts/compile_dep.sh/0.9.3/all.out b/tests/scripts/compile_dep.sh/0.9.3/all.out
new file mode 100644
index 0000000000000000000000000000000000000000..2fcd55b6f7dc5e574eeea0233a628e75ea105f4e
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/all.out
@@ -0,0 +1,14 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: psm2 ###############################
+WARNING: Install script for 'psm2' not found. Skipping.
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
+######## Installing: agios ###############################
+######## Installing: parallax ###############################
+Done
\ No newline at end of file
diff --git a/tests/scripts/compile_dep.sh/0.9.3/arm.out b/tests/scripts/compile_dep.sh/0.9.3/arm.out
new file mode 100644
index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/arm.out
@@ -0,0 +1,11 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: psm2 ###############################
+WARNING: Install script for 'psm2' not found. Skipping.
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/ci.out b/tests/scripts/compile_dep.sh/0.9.3/ci.out
new file mode 100644
index 0000000000000000000000000000000000000000..9ec3c24bb12c16078d2b3869831331de15f5e305
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/ci.out
@@ -0,0 +1,8 @@
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
+######## Installing: agios ###############################
+######## Installing: parallax ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/default.out b/tests/scripts/compile_dep.sh/0.9.3/default.out
new file mode 100644
index 0000000000000000000000000000000000000000..ca393e65d3bc03aa72fe6a628efab5dad85174b9
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/default.out
@@ -0,0 +1,10 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
+
diff --git a/tests/scripts/compile_dep.sh/0.9.3/direct.out b/tests/scripts/compile_dep.sh/0.9.3/direct.out
new file mode 100644
index 0000000000000000000000000000000000000000..816266773b569d3d098245c7b868f7983753ff12
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/direct.out
@@ -0,0 +1,6 @@
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out
new file mode 100644
index 0000000000000000000000000000000000000000..ff149d28f1874e6fe4f94484c4f0b4f7abf49416
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out
@@ -0,0 +1,9 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: libfabric%verbs ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out
new file mode 100644
index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out
@@ -0,0 +1,12 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: psm2 ###############################
+WARNING: Install script for 'psm2' not found. Skipping.
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
+######## Installing: parallax ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/mogon2.out b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out
new file mode 100644
index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out
@@ -0,0 +1,12 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: psm2 ###############################
+WARNING: Install script for 'psm2' not found. Skipping.
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
+######## Installing: parallax ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/ngio.out b/tests/scripts/compile_dep.sh/0.9.3/ngio.out
new file mode 100644
index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/ngio.out
@@ -0,0 +1,12 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: psm2 ###############################
+WARNING: Install script for 'psm2' not found. Skipping.
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
+######## Installing: parallax ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out
new file mode 100644
index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out
@@ -0,0 +1,11 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: psm2 ###############################
+WARNING: Install script for 'psm2' not found. Skipping.
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
diff --git a/tests/scripts/compile_dep.sh/0.9.3/p9.out b/tests/scripts/compile_dep.sh/0.9.3/p9.out
new file mode 100644
index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538
--- /dev/null
+++ b/tests/scripts/compile_dep.sh/0.9.3/p9.out
@@ -0,0 +1,11 @@
+######## Installing: lz4 ###############################
+######## Installing: capstone ###############################
+######## Installing: json-c ###############################
+######## Installing: psm2 ###############################
+WARNING: Install script for 'psm2' not found. Skipping.
+######## Installing: libfabric ###############################
+######## Installing: mercury ###############################
+######## Installing: argobots ###############################
+######## Installing: margo ###############################
+######## Installing: rocksdb ###############################
+######## Installing: syscall_intercept ###############################
diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out
index 7ba8e6778a49f2f7ebaf516c6143f2dd095adef8..8369fe83959bf552ffa03f984ee038b6c55baaed 100644
--- a/tests/scripts/dl_dep.sh/0.8.0/default.out
+++ b/tests/scripts/dl_dep.sh/0.8.0/default.out
@@ -12,5 +12,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocks
Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental'
Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags ''
Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags ''
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2'
Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development'
diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out
index d8af0a79b2bed070d64eb40b2751a06fd5040207..d5844433bc81445c690efc0e30054a331000b4ce 100644
--- a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out
+++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out
@@ -10,4 +10,4 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6.
Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental'
Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags ''
Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags ''
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2'
diff --git a/tests/scripts/dl_dep.sh/0.8.0/ngio.out b/tests/scripts/dl_dep.sh/0.8.0/ngio.out
index f0c9f79592b10c1307d92dbafc560a5914c5974d..e6cd7bdcc2d1c625d552b6c85b256500df133683 100644
--- a/tests/scripts/dl_dep.sh/0.8.0/ngio.out
+++ b/tests/scripts/dl_dep.sh/0.8.0/ngio.out
@@ -8,6 +8,6 @@ Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa
Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6.3]' and flags ''
Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb'
Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags ''
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2'
Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags ''
Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development'
diff --git a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out
index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644
--- a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out
+++ b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out
index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644
--- a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out
+++ b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.0/ngio.out b/tests/scripts/dl_dep.sh/0.9.0/ngio.out
index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644
--- a/tests/scripts/dl_dep.sh/0.9.0/ngio.out
+++ b/tests/scripts/dl_dep.sh/0.9.0/ngio.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out
index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644
--- a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out
+++ b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.1/all.out b/tests/scripts/dl_dep.sh/0.9.1/all.out
index 35b6b521748a2bb789559c4df780c438d345b7b8..9cbff6342eb7dba75c4f9e617fddacfd446b13ff 100644
--- a/tests/scripts/dl_dep.sh/0.9.1/all.out
+++ b/tests/scripts/dl_dep.sh/0.9.1/all.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out
index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644
--- a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out
+++ b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out
index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644
--- a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out
+++ b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.1/ngio.out b/tests/scripts/dl_dep.sh/0.9.1/ngio.out
index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644
--- a/tests/scripts/dl_dep.sh/0.9.1/ngio.out
+++ b/tests/scripts/dl_dep.sh/0.9.1/ngio.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out
index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644
--- a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out
+++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.1/p9.out b/tests/scripts/dl_dep.sh/0.9.1/p9.out
index 3a99bb523089fa9d7aaa1a36a4a62fa9d6295732..b1eee01ba618b62af568de183d0d094947586123 100644
--- a/tests/scripts/dl_dep.sh/0.9.1/p9.out
+++ b/tests/scripts/dl_dep.sh/0.9.1/p9.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.2/all.out b/tests/scripts/dl_dep.sh/0.9.2/all.out
index 4a1ab9e6f6de6a14f9e7da7c7792bd01e6171c48..4b5a73fda0b5c2962d25cc0b7d5c786cdf5adb5b 100644
--- a/tests/scripts/dl_dep.sh/0.9.2/all.out
+++ b/tests/scripts/dl_dep.sh/0.9.2/all.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.2/arm.out b/tests/scripts/dl_dep.sh/0.9.2/arm.out
index 2c61edb9c437445a2347de92fdd11a66ac304576..8725e5892b82e32c3e2e8b15d7f805f72c09cc88 100644
--- a/tests/scripts/dl_dep.sh/0.9.2/arm.out
+++ b/tests/scripts/dl_dep.sh/0.9.2/arm.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out
index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644
--- a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out
+++ b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out
index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644
--- a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out
+++ b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.2/ngio.out b/tests/scripts/dl_dep.sh/0.9.2/ngio.out
index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644
--- a/tests/scripts/dl_dep.sh/0.9.2/ngio.out
+++ b/tests/scripts/dl_dep.sh/0.9.2/ngio.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out
index d26b498ca41977f2e48d86f1f0437c1bc0b7a563..96e22237b1da0070350c22e2809d62a69dcd24d3 100644
--- a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out
+++ b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.2/p9.out b/tests/scripts/dl_dep.sh/0.9.2/p9.out
index 3a99bb523089fa9d7aaa1a36a4a62fa9d6295732..b1eee01ba618b62af568de183d0d094947586123 100644
--- a/tests/scripts/dl_dep.sh/0.9.2/p9.out
+++ b/tests/scripts/dl_dep.sh/0.9.2/p9.out
@@ -1,7 +1,7 @@
Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
-Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
diff --git a/tests/scripts/dl_dep.sh/0.9.3/agios.out b/tests/scripts/dl_dep.sh/0.9.3/agios.out
new file mode 100644
index 0000000000000000000000000000000000000000..8d8e2486b502394977608f89057d0479487e736d
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/agios.out
@@ -0,0 +1,11 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development'
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/all.out b/tests/scripts/dl_dep.sh/0.9.3/all.out
new file mode 100644
index 0000000000000000000000000000000000000000..dbd2183188053b13c59cfb0dbb93e38d96f482d6
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/all.out
@@ -0,0 +1,13 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development'
+Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/arm.out b/tests/scripts/dl_dep.sh/0.9.3/arm.out
new file mode 100644
index 0000000000000000000000000000000000000000..7b1d7c5fcb554b36e7e8163f4f9c751c28628534
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/arm.out
@@ -0,0 +1,11 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[fb73c3924b502e2840a5dc8a18746e395b06a8a6]' and flags ''
+Done
diff --git a/tests/scripts/dl_dep.sh/0.9.3/ci.out b/tests/scripts/dl_dep.sh/0.9.3/ci.out
new file mode 100644
index 0000000000000000000000000000000000000000..0754a2860f268f916db2cc7ab121e1737b6b5705
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/ci.out
@@ -0,0 +1,9 @@
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development'
+Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/default.out b/tests/scripts/dl_dep.sh/0.9.3/default.out
new file mode 100644
index 0000000000000000000000000000000000000000..7b7330872c4578763b697f7394fb9d49f872f11d
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/default.out
@@ -0,0 +1,10 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/direct.out b/tests/scripts/dl_dep.sh/0.9.3/direct.out
new file mode 100644
index 0000000000000000000000000000000000000000..42a6d15450c591a23273ec6201faf713235fab58
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/direct.out
@@ -0,0 +1,7 @@
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out
new file mode 100644
index 0000000000000000000000000000000000000000..f2c3f6d3b2c69b9e934055bec1ef1cbfc2a20f63
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out
@@ -0,0 +1,10 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out
new file mode 100644
index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out
@@ -0,0 +1,12 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/mogon2.out b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out
new file mode 100644
index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out
@@ -0,0 +1,12 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/ngio.out b/tests/scripts/dl_dep.sh/0.9.3/ngio.out
new file mode 100644
index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/ngio.out
@@ -0,0 +1,12 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out
new file mode 100644
index 0000000000000000000000000000000000000000..8ef7035e06cb383d1d826f0354d222e9bdcb48b8
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out
@@ -0,0 +1,11 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags ''
+Done
\ No newline at end of file
diff --git a/tests/scripts/dl_dep.sh/0.9.3/p9.out b/tests/scripts/dl_dep.sh/0.9.3/p9.out
new file mode 100644
index 0000000000000000000000000000000000000000..1c939fc2332d564471238d6941f319b748348f7a
--- /dev/null
+++ b/tests/scripts/dl_dep.sh/0.9.3/p9.out
@@ -0,0 +1,11 @@
+Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4'
+Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone'
+Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c'
+Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2'
+Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2'
+Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules'
+Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots'
+Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags ''
+Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb'
+Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f]' and flags ''
+Done
diff --git a/CMake/FindDate.cmake b/tools/CMakeLists.txt
similarity index 68%
rename from CMake/FindDate.cmake
rename to tools/CMakeLists.txt
index 78b6a79d8c5cce0d27a491ba0b724881701e5f14..9c3e16519551b976c85b23601083fc1f8ee35bbe 100644
--- a/CMake/FindDate.cmake
+++ b/tools/CMakeLists.txt
@@ -1,6 +1,6 @@
################################################################################
-# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain #
-# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany #
+# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
@@ -26,43 +26,13 @@
# SPDX-License-Identifier: GPL-3.0-or-later #
################################################################################
-find_path(DATE_INCLUDE_DIR
- NAMES date/date.h
-)
-
-find_path(TZ_INCLUDE_DIR
- NAMES date/tz.h
-)
-
-find_library(TZ_LIBRARY
- NAMES tz
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args( Date
- DEFAULT_MSG
- DATE_INCLUDE_DIR
- TZ_INCLUDE_DIR
- TZ_LIBRARY
-)
-
-if(Date_FOUND)
- set(DATE_INCLUDE_DIRS ${DATE_INCLUDE_DIR})
- set(TZ_INCLUDE_DIRS ${TZ_INCLUDE_DIR})
- set(TZ_LIBRARIES ${TZ_LIBRARY})
-
- if(NOT TARGET Date::TZ)
- add_library(Date::TZ UNKNOWN IMPORTED)
- set_target_properties(Date::TZ PROPERTIES
- IMPORTED_LOCATION "${TZ_LIBRARY}"
- INTERFACE_COMPILE_DEFINITIONS "USE_OS_TZDB=1"
- INTERFACE_INCLUDE_DIRECTORIES "${TZ_INCLUDE_DIR}"
- )
- endif()
-endif()
-
-mark_as_advanced(
- DATE_INCLUDE_DIR
- TZ_INCLUDE_DIR
- TZ_LIBRARY
-)
+if (GKFS_ENABLE_CLIENT_METRICS)
+ include(load_nlohmann_json)
+ add_executable(gkfs_clientmetrics2json)
+ target_sources(gkfs_clientmetrics2json
+ PRIVATE
+ ${CMAKE_CURRENT_LIST_DIR}/clientmetrics2json.cpp
+ )
+ target_link_libraries(gkfs_clientmetrics2json PUBLIC msgpack_util nlohmann_json::nlohmann_json)
+ install(TARGETS gkfs_clientmetrics2json RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+endif ()
\ No newline at end of file
diff --git a/tools/clientmetrics2json.cpp b/tools/clientmetrics2json.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f74667b5fa94d0100888766371da4bd1089796e5
--- /dev/null
+++ b/tools/clientmetrics2json.cpp
@@ -0,0 +1,139 @@
+/*
+ Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain
+ Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany
+
+ This software was partially supported by the
+ EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+ This software was partially supported by the
+ ADA-FS project under the SPPEXA project funded by the DFG.
+
+ This file is part of GekkoFS' POSIX interface.
+
+ GekkoFS' POSIX interface is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of the License,
+ or (at your option) any later version.
+
+ GekkoFS' POSIX interface is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with GekkoFS' POSIX interface. If not, see
+ .
+
+ SPDX-License-Identifier: LGPL-3.0-or-later
+*/
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+extern "C" {
+#include
+#include
+}
+
+using json = nlohmann::json;
+
+void
+report_msgpack(std::vector& buffer) {
+ std::error_code ec{};
+ auto undata =
+ msgpack::unpack(
+ buffer, ec);
+ std::vector avg_thruput(undata.req_size_.size());
+ for(size_t i = 0; i < avg_thruput.size(); ++i) {
+ auto size_mib = undata.req_size_[i] / (1024.0 * 1024.0); // in MiB
+ auto duration_s = (undata.end_t_[i] - undata.start_t_[i]) / 1000000.0;
+ avg_thruput[i] = std::round((size_mib / duration_s) * 100.0) / 100.0;
+ }
+ json json_obj;
+ json_obj["hostname"] = undata.hostname_;
+ json_obj["pid"] = undata.pid_;
+ json_obj["io_type"] = undata.io_type_;
+ json_obj["total_bytes"] = undata.total_bytes_;
+ json_obj["total_iops"] = undata.total_iops_;
+ json_obj["start_t_micro"] = undata.start_t_;
+ json_obj["end_t_micro"] = undata.end_t_;
+ json_obj["flush_t"] = undata.flush_t_;
+ json_obj["req_size"] = undata.req_size_;
+ json_obj["[extra]avg_thruput_mib"] = avg_thruput;
+
+ std::cout << "Generated JSON:" << '\n';
+ for(const auto& item : json_obj.items()) {
+ std::cout << item.key() << ": " << item.value().dump() << '\n';
+ }
+}
+
+void
+read_file(const std::string& path) {
+ if(!std::filesystem::exists(path)) {
+ std::cerr << "Input file " << path << " does not exist" << '\n';
+ return;
+ }
+ std::ifstream file(path, std::ios::binary);
+ if(!file.is_open()) {
+ std::cout << "failed to open " << '\n';
+ return;
+ }
+ std::vector buffer((std::istreambuf_iterator(file)),
+ std::istreambuf_iterator());
+
+ file.close();
+ report_msgpack(buffer);
+}
+
+
+int
+main(int argc, char** argv) {
+ std::string mode{};
+ if(argc != 2) {
+ std::cerr << "Usage: " << argv[0]
+ << " \n"
+ << " ZMQ listing must start with tcp://\n";
+ return -1;
+ }
+ auto path_ip = std::string(argv[1]);
+ if(path_ip.find("tcp://") == 0) {
+ // Create a ZeroMQ context
+ zmq::context_t context(1);
+ // Create a socket to for the server to bind to
+ zmq::socket_t socket(context, ZMQ_PULL);
+ std::cout << "Binding to: " << path_ip << std::endl;
+ socket.bind(path_ip);
+
+ while(true) {
+ // Wait for a request message
+ zmq::message_t request;
+ std::cout << "Waiting for message... " << std::endl;
+ if(!socket.recv(request, zmq::recv_flags::none)) {
+ std::cerr << "Failed to receive message" << std::endl;
+ continue;
+ }
+ std::cout << "Received message with size " << request.size()
+ << '\n';
+
+ std::vector requestData(
+ static_cast(request.data()),
+ static_cast(request.data()) +
+ request.size());
+
+ report_msgpack(requestData);
+ std::cout
+ << "-------------------------------------------------------------------------------------\n";
+ }
+ } else {
+ read_file(path_ip);
+ }
+ // std::cout << json_obj.dump(4) << std::endl; // Use dump for pretty
+ // printing
+
+ return 0;
+}
\ No newline at end of file