diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68eaca4ed3a2a944e32510b744c9294bf0e4d417..19c076f05914df9170c43104dda58ca13eae5890 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.2 +image: gekkofs/core:0.9.3 ################################################################################ ## Validating ################################################################################ check format: stage: lint - image: gekkofs/linter:0.9.2 + image: gekkofs/linter:0.9.3 needs: [] script: - ${SCRIPTS_DIR}/check_format.sh @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.2 + image: gekkofs/deps:0.9.3 interruptible: true needs: [] script: @@ -65,23 +65,6 @@ gkfs: - ${INSTALL_PATH} expire_in: 1 week -gkfwd: - stage: build - image: gekkofs/deps:0.9.2 - interruptible: true - needs: [] - script: - - cmake --preset ci-forwarding-coverage - - cmake --build ${BUILD_PATH} -j $(nproc) --target install - # reduce artifacts size - - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH} - artifacts: - paths: - - ${BUILD_PATH} - - ${INSTALL_PATH} - expire_in: 1 week - - ################################################################################ ## Testing ################################################################################ @@ -89,7 +72,7 @@ gkfwd: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: [] script: - mkdir -p ${BUILD_PATH}/tests/scripts @@ -105,7 +88,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true needs: ['gkfs'] parallel: @@ -154,14 +137,14 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true - needs: ['gkfwd'] + needs: ['gkfs'] parallel: matrix: - SUBTEST: [ forwarding ] - rules: - - when: never + # rules: + # - when: never script: ## run tests @@ -204,7 +187,7 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: ['gkfs'] script: ## Add path to mkfs.kreon @@ -242,7 +225,7 @@ gkfs:unit: ################################################################################ documentation: stage: docs - image: gekkofs/docs:0.9.2 + image: gekkofs/docs:0.9.3 needs: [] rules: # we only build the documentation automatically if we are on the @@ -272,9 +255,9 @@ documentation: ## == coverage baseline ==================== coverage:baseline: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true - needs: ['gkfs', 'gkfwd'] + needs: ['gkfs'] script: ## capture initial coverage information to establish a baseline @@ -298,10 +281,10 @@ coverage:baseline: coverage: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 # needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', # 'gkfs:unit' ] - needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ] + needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] script: - cd ${CI_PROJECT_DIR} - cmake diff --git a/.gitmodules b/.gitmodules index 18f937ae7b811d4f78fc6fce7f89600199f28abd..2763952410498bb4175ac57565a5f3bfb7d2ea1f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "external/hermes"] path = external/hermes - url = https://github.com/bsc-ssrg/hermes.git + url = https://github.com/gekkofs/hermes.git [submodule "tests/scripts/bats"] path = tests/scripts/bats url = https://github.com/bats-core/bats-core.git @@ -22,3 +22,6 @@ [submodule "external/CLI11"] path = external/CLI11 url = https://github.com/CLIUtils/CLI11.git +[submodule "external/MessagePack"] + path = external/MessagePack + url = https://github.com/GekkoFS/MessagePackCPP.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ae0762a2456b0efd8172251cd3a3d5d079d7a8..93e0446d56f634b8b9652c0ce5839d69c6593ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). - Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. - Added reattemp support in get_fs_config to other servers, when the initial server fails. +- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable +`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). +- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)). ### New ### Changed diff --git a/CMake/FindMercury.cmake b/CMake/FindMercury.cmake index 7eb96cde545a7ea7bb863df725f9285368c47a59..451ed6d403ab2a3d4f4f2fb6cabb351179593add 100644 --- a/CMake/FindMercury.cmake +++ b/CMake/FindMercury.cmake @@ -177,7 +177,7 @@ endmacro() include(SelectLibraryConfigurations) -set(_mercury_components na mercury_util mercury_hl) +set(_mercury_components na mercury_util) # prevent repeating work if the main CMakeLists.txt already called # find_package(PkgConfig) diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake index 30da5afc387621004040ca75bfc3f43630722189..398b60ac039573f55c1b7ab3cf072f648e6d1d61 100644 --- a/CMake/gkfs-options.cmake +++ b/CMake/gkfs-options.cmake @@ -227,6 +227,13 @@ gkfs_define_option( DEFAULT_VALUE OFF ) +# build tools +gkfs_define_option( + GKFS_BUILD_TOOLS + HELP_TEXT "Enable ${PROJECT_NAME} tools compilation" + DEFAULT_VALUE OFF +) + cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF) @@ -348,14 +355,6 @@ gkfs_define_option( # I/O forwarding ################################################################################ -## Forwarding support -gkfs_define_option( - GKFS_ENABLE_FORWARDING - HELP_TEXT "Enable I/O forwarding mode" - DEFAULT_VALUE OFF - DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer" -) - ## Scheduling in I/O forwarding mode gkfs_define_option( GKFS_ENABLE_AGIOS @@ -364,4 +363,13 @@ gkfs_define_option( DESCRIPTION "If GKFS_ENABLE_FORWARDING is ON, use AGIOS for scheduling I/Os" ) +################################################################################ +# MSGPack client metrics +################################################################################ +gkfs_define_option( + GKFS_ENABLE_CLIENT_METRICS + HELP_TEXT "Enable client metrics via MSGPack" + DEFAULT_VALUE ON # TODO disable by default + DESCRIPTION "If GKFS_ENABLE_CLIENT_METRICS is ON, use MSGPack to dump client read/write metrics" +) \ No newline at end of file diff --git a/CMake/load_nlohmann_json.cmake b/CMake/load_nlohmann_json.cmake new file mode 100644 index 0000000000000000000000000000000000000000..951900998bad144acff05e0845504dd87d3f2a7d --- /dev/null +++ b/CMake/load_nlohmann_json.cmake @@ -0,0 +1,52 @@ +################################################################################ +# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +include(FetchContent) + +set(FETCHCONTENT_QUIET OFF) + +FetchContent_Declare(nlohmann_json + DOWNLOAD_EXTRACT_TIMESTAMP ON + URL https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz) + +FetchContent_GetProperties(nlohmann_json) + +if (NOT nlohmann_json_POPULATED) + FetchContent_Populate(nlohmann_json) + message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}") + message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}") + + # we don't really care so much about a third party library's tests to be + # run from our own project's code + set(JSON_BuildTests OFF CACHE INTERNAL "") + + # we also don't need to install it when our main project gets installed + set(JSON_Install OFF CACHE INTERNAL "") + + add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR}) +endif () \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index afdda9c02ef8e0137d8bca31f8b2bab90bb8cce2..c0ca49b4dfc330c257f3291da3e5e4e77094817d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,16 +32,17 @@ cmake_minimum_required(VERSION 3.13) project( GekkoFS VERSION 0.9.2 + LANGUAGES ASM CXX C ) enable_testing() -if (NOT CMAKE_COMPILER_IS_GNUCC) - message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") -endif () -if (NOT CMAKE_COMPILER_IS_GNUCXX) - message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") -endif () +#if (NOT CMAKE_COMPILER_IS_GNUCC) +# message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") +#endif () +#if (NOT CMAKE_COMPILER_IS_GNUCXX) +# message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") +#endif () set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -160,16 +161,18 @@ find_package(Margo 0.9.6 REQUIRED) message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept") find_package(Syscall_intercept REQUIRED) -### Date (required by the libgkfs_intercept.so) -message(STATUS "[${PROJECT_NAME}] Checking for Date") -find_package(Date REQUIRED) - ### AGIOS: required for scheduling I/O requests if (GKFS_ENABLE_AGIOS) message(STATUS "[${PROJECT_NAME}] Checking for Agios") find_package(AGIOS REQUIRED) endif() +if (GKFS_ENABLE_CLIENT_METRICS) + ### zeromq: required for sending client metrics to FTIO + message(STATUS "[${PROJECT_NAME}] Checking for cppzmq") + find_package(cppzmq) +endif () + ### Metadata backends if(GKFS_ENABLE_ROCKSDB) message(STATUS "[${PROJECT_NAME}] Checking for RocksDB") @@ -225,7 +228,7 @@ include_from_source(fmt MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG f94b7364b9409f05207c3af3fa4666730e11a854 # v6.1.2 + GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0 ) # ensure that fmt is linked as PIC @@ -247,6 +250,15 @@ include_from_source(cli11 GIT_TAG v2.2.0 ) +if (GKFS_ENABLE_CLIENT_METRICS) + ### MessagePack: used for monitoring information on the client + include_from_source(MessagePack + MESSAGE "[${PROJECT_NAME}] Searching for MessagePackCPP" + SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/MessagePack + GIT_REPOSITORY https://github.com/GekkoFS/MessagePackCPP + GIT_TAG 51655ec8f2b8fed84f685e3bcf8f226b139a263b # latest HEAD + ) +endif () ################################################################################ ## Check configured variables/options and act accordingly @@ -305,10 +317,14 @@ set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") include_directories( ${INCLUDE_DIR} ${CMAKE_BINARY_DIR}/include + /usr/include ) add_subdirectory(src) add_subdirectory(include) +if (GKFS_BUILD_TOOLS) + add_subdirectory(tools) +endif () ### Mark any CMake variables imported from {fmt} and spdlog as advanced, so ### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT @@ -338,11 +354,7 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}") message(STATUS "[gekkofs] Check for forwarding tests...") - if (GKFS_ENABLE_FORWARDING) - set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - else () - set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - endif () + set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}") message(STATUS "[gekkofs] Check for guided distributor tests...") if (GKFS_USE_GUIDED_DISTRIBUTION) diff --git a/CMakePresets.json b/CMakePresets.json index 9f12e9c734ceabc71595221f384db23c67ff7bf9..da7bf79b3ef83ca3a0f51bcc014d255dccd8ece6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -27,14 +27,6 @@ "deprecated": true } }, - { - "name": "forwarding", - "inherits": "default", - "hidden": true, - "cacheVariables": { - "GKFS_ENABLE_FORWARDING": true - } - }, { "name": "release", "hidden": true, @@ -113,30 +105,6 @@ "release" ] }, - { - "name": "forwarding-debug", - "displayName": "Forwarding gekkofs (debug)", - "inherits": [ - "forwarding", - "debug" - ] - }, - { - "name": "forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage)", - "inherits": [ - "forwarding", - "coverage" - ] - }, - { - "name": "forwarding-release", - "displayName": "Forwarding gekkofs (release)", - "inherits": [ - "forwarding", - "release" - ] - }, { "name": "ci-debug", "displayName": "Default gekkofs (debug, CI flags)", @@ -171,33 +139,6 @@ "default", "release" ] - }, - { - "name": "ci-forwarding-debug", - "displayName": "Forwarding gekkofs (debug, CI flags)", - "inherits": [ - "ci", - "forwarding", - "debug" - ] - }, - { - "name": "ci-forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage, CI flags)", - "inherits": [ - "ci", - "forwarding", - "coverage" - ] - }, - { - "name": "ci-forwarding-release", - "displayName": "Forwarding gekkofs (release, CI flags)", - "inherits": [ - "ci", - "forwarding", - "release" - ] } ] } diff --git a/config/GekkoFWD/agios/agios.conf b/config/GekkoFWD/agios/agios.conf index e0fc38d0a854d12d19a73597e2eab05ac3dc9df5..41c747da4e899d62715cefb7dcb19d8cbbfe9780 100644 --- a/config/GekkoFWD/agios/agios.conf +++ b/config/GekkoFWD/agios/agios.conf @@ -15,7 +15,7 @@ library_options: #should the prediction module try to predict aggregations? (requires predict_read_traces = true) predict_request_aggregation = false ; - #should the prediction module create simplified traces with information (the metrics) it obtained from the real traces? + #should the prediction module create simplified traces with information (the client_metrics) it obtained from the real traces? predict_write_simplified_traces = false; #the tolerance for arrival times difference when checking if two predicted requests are the same (in %) diff --git a/docker/0.9.3/core/Dockerfile b/docker/0.9.3/core/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a55434924b2cfe720195c371de6736ac049826f7 --- /dev/null +++ b/docker/0.9.3/core/Dockerfile @@ -0,0 +1,40 @@ +FROM debian:bullseye-slim + +LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + ninja-build \ + procps \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ + # Margo dependencies \ + libjson-c-dev \ + # RocksDB dependencies + liblz4-dev \ + # syscall_intercept dependencies + libcapstone-dev \ + # GekkoFS dependencies + libboost-program-options-dev \ + uuid-dev && \ + # install cmake 3.14+ since it's needed for some dependencies + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh diff --git a/docker/0.9.3/core/Makefile b/docker/0.9.3/core/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9c27a583a8ae1069865ad13cc0196214d40b9362 --- /dev/null +++ b/docker/0.9.3/core/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/core:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/core:0.9.3 . + +all: + docker buildx build -t gekkofs/core:0.9.3 . diff --git a/docker/0.9.3/deps/Dockerfile b/docker/0.9.3/deps/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ca4ecc8a646db53669fa642eaefdd2bfcc7e8804 --- /dev/null +++ b/docker/0.9.3/deps/Dockerfile @@ -0,0 +1,38 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to build GekkoFS" + +ENV GKFS_PATH /opt/gkfs +ENV GKFS_VERSION 0.9.3 + +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH /usr/local + +COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/patches ${SCRIPTS_PATH}/patches +COPY scripts/profiles ${SCRIPTS_PATH}/profiles + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python3-setuptools \ + libnuma-dev libyaml-dev libcurl4-openssl-dev \ + procps && \ + python3 -m pip install --upgrade pip && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && apt-get autoclean + +# Download and build dependencies +RUN cd ${SCRIPTS_PATH} && \ + /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \ + /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \ + cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\ + rm -rf ${DEPS_SRC_PATH} && \ + rm -rf ${SCRIPTS_PATH} && \ + rmdir ${GKFS_PATH} && \ + ldconfig diff --git a/docker/0.9.3/deps/Makefile b/docker/0.9.3/deps/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a97d3195f66a9cf7c5b44ed7b32d25abb49e6ab0 --- /dev/null +++ b/docker/0.9.3/deps/Makefile @@ -0,0 +1,23 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts remove-scripts +amd64: build-amd64 mount-scripts remove-scripts +aarch64: build-aarch64 mount-scripts remove-scripts + +copy-scripts: + cp -R $(GIT_ROOT)/scripts $(CWD)/scripts + +build: copy-scripts + docker build -t gekkofs/deps:0.9.3 . + +build-amd64: copy-scripts + docker build --platform amd64 -t gekkofs/deps:0.9.3 . + +build-aarch64: copy-scripts + docker build --platform aarch64 -t gekkofs/deps:0.9.3 . + +remove-scripts: + - rm -rf $(CWD)/scripts diff --git a/docker/0.9.3/docs/Dockerfile b/docker/0.9.3/docs/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f770f3a23de51eb7ee8f13d0f3b8ebf8623d2649 --- /dev/null +++ b/docker/0.9.3/docs/Dockerfile @@ -0,0 +1,34 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.33.1' \ + 'exhale==0.3.1' \ + 'sphinx-copybutton==0.5.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.17.0' \ + attrs && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/docs/Makefile b/docker/0.9.3/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..c6c89926af84c32ee55331a647a3d041ea6e0555 --- /dev/null +++ b/docker/0.9.3/docs/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/docs:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/docs:0.9.3 . + +all: + docker build -t gekkofs/docs:0.9.3 . \ No newline at end of file diff --git a/docker/0.9.3/linter/Dockerfile b/docker/0.9.3/linter/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f5d57118e518d36fb28f7a0f2bc95716dc99f2d0 --- /dev/null +++ b/docker/0.9.3/linter/Dockerfile @@ -0,0 +1,19 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to check the formatting of GekkoFS code" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # clang 15 deps for clang-format + lsb-release \ + wget \ + software-properties-common \ + gnupg2 && \ + # add clang-15 repos + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \ + # install clang-format + apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/linter/Makefile b/docker/0.9.3/linter/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f427eb387a7f8aebffa6a4522c428467835903e4 --- /dev/null +++ b/docker/0.9.3/linter/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/linter:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/linter:0.9.3 . + +all: + docker buildx build -t gekkofs/linter:0.9.3 . diff --git a/docker/0.9.3/testing/Dockerfile b/docker/0.9.3/testing/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..971944a86b0c5102eba84a912b33a53187078701 --- /dev/null +++ b/docker/0.9.3/testing/Dockerfile @@ -0,0 +1,32 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment to test GekkoFS" + +RUN \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + # required by lcov's genhtml + libgd-perl && \ + # install cmake 3.21+ since we need to produce JUnit XML files + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # install loguru + # (required by several of our scripts) + pip3 install loguru && \ + # install lcov_cobertura + # (required to produce Cobertura XML reports) + pip3 install lcov_cobertura && \ + # install lcov + # (required to produce partial coverage reports in parallel runs) + curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \ + tar xfz lcov-1.16.tar.gz && \ + cd lcov-1.16 && \ + make install && \ + cd .. && \ + # cleanup + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh && \ + rm -rf ./lcov-1.16.* diff --git a/docker/0.9.3/testing/Makefile b/docker/0.9.3/testing/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b5908b5399d36a19a96cc61a8f929fd9a6c4a581 --- /dev/null +++ b/docker/0.9.3/testing/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/testing:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/testing:0.9.3 . + +all: + docker build -t gekkofs/testing:0.9.3 . diff --git a/docs/sphinx/users/dependencies.rst b/docs/sphinx/users/dependencies.rst index 0d3732410f1b133cc755047a1df1ad2601fa2f0b..3490ef5d6300b3153f35f37f5ff252854252e306 100644 --- a/docs/sphinx/users/dependencies.rst +++ b/docs/sphinx/users/dependencies.rst @@ -43,4 +43,4 @@ Required Optional -------- -- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode. +- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling. diff --git a/docs/sphinx/users/forwarding.rst b/docs/sphinx/users/forwarding.rst index db0afc7b22e587f629bcacadfde6de82c62f7e80..e850b4d04d7b4649fd7b600f3dd9cd6c7dbad8f5 100644 --- a/docs/sphinx/users/forwarding.rst +++ b/docs/sphinx/users/forwarding.rst @@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes. Enabling GekkoFWD ------------------ -To enable the I/O forwarding mode of GekkoFS, the -:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when -configuring ` the build: - -.. code-block:: console - - $ cmake -DENABLE_FORWARDING:BOOL=ON +To enable the I/O forwarding mode of GekkoFS, an environment +variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided +to allow users to identify the map file on each client. +This environment variable will enable GekkoFS forwarding, then we +only need to use :code:`--enable-forwarding`` option in the servers. I/O Scheduling -------------- diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index d9e63e317b8045762757231c1196b0a973faca7a..d4788ddd03ea5a92df9b7b407420a64547519df2 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -26,11 +26,19 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -set (CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) add_executable(sfind sfind.cpp) +add_executable(gkfs_lib_example gkfs_lib_example.cpp) + +target_link_libraries(gkfs_lib_example + PRIVATE gkfs_user_lib +) if(GKFS_INSTALL_TESTS) install(TARGETS sfind DESTINATION ${CMAKE_INSTALL_BINDIR} ) + install(TARGETS gkfs_lib_example + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) endif() diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f064e3e3347dfdb82a065de50ba2cfecd47db545 --- /dev/null +++ b/examples/gfind/gkfs_lib_example.cpp @@ -0,0 +1,115 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +void +write_file(std::string filename) { + // Open File + int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT); + + cout << "FD open " << fd << endl; + + char* bufwrite = (char*) malloc(10); + strncpy(bufwrite, "testing\0", 8); + + + int size = gkfs::syscall::gkfs_write(fd, bufwrite, 7); + + cout << "Writting size " << size << endl; + + free(bufwrite); + gkfs::syscall::gkfs_close(fd); +} + + +void +read_file(std::string filename) { + int fdread = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDONLY); + if(fdread == -1) + return; + char* bufread = (char*) malloc(10); + int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); + + cout << "Reading Size: " << sizeread << " Content: " << bufread << endl; + + free(bufread); + gkfs::syscall::gkfs_close(fdread); +} + +int +main(int argc, char** argv) { + cout << "GekkoFS Client library test" << endl; + + auto res = gkfs_init(); + + cout << "Init result " << res << endl; + + write_file("/test.tmp"); + + read_file("/test.tmp"); + + + write_file("/secondfile.tmp"); + + auto f_list = gkfs::syscall::gkfs_get_file_list("/"); + + for(auto f : f_list) { + cout << "File: " << f << endl; + struct stat buf; + memset(&buf, 0, sizeof(struct stat)); + + gkfs::syscall::gkfs_stat("/" + f, &buf, true); + + cout << "Size: " << buf.st_size << " Mode: " << buf.st_mode << endl; + cout << "Atime: " << buf.st_atime << " Mtime: " << buf.st_mtime + << " Ctime: " << buf.st_ctime << endl + << " ****** " << endl; + } + + res = gkfs_end(); + + cout << "End result " << res << endl; +} diff --git a/external/MessagePack b/external/MessagePack new file mode 160000 index 0000000000000000000000000000000000000000..26527c71eb74fdc625ea994fa5caf45db318f7f1 --- /dev/null +++ b/external/MessagePack @@ -0,0 +1 @@ +Subproject commit 26527c71eb74fdc625ea994fa5caf45db318f7f1 diff --git a/external/hermes b/external/hermes index d8755039c82501323cdc9a4f5a2b922b9b3bcac7..a133c9dd0cefc7e8a3554badfb8995c9f2c27be6 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit d8755039c82501323cdc9a4f5a2b922b9b3bcac7 +Subproject commit a133c9dd0cefc7e8a3554badfb8995c9f2c27be6 diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 4cc56ac387e724c02d549d4e5d4774a68c62be61..1ae9e3b40e76552f804dc6c3cd4c7dd053c291a3 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -34,7 +34,3 @@ add_subdirectory(daemon) add_subdirectory(client) target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in) - -if(GKFS_ENABLE_FORWARDING) - target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in) -endif() diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index f5df6ee5c796e2dd72c909ac4e8b1b15bb3912c2..6815fc95321a27ecce55b407d10075cb53cad2ca 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -54,30 +54,30 @@ target_sources( syscalls/detail/syscall_info.h ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_intercept - PUBLIC gkfs_functions.hpp - env.hpp - hooks.hpp - intercept.hpp - logging.hpp - make_array.hpp - open_file_map.hpp - open_dir.hpp - path.hpp - preload.hpp - preload_context.hpp - preload_util.hpp - rpc/rpc_types.hpp - rpc/forward_management.hpp - rpc/forward_metadata.hpp - rpc/forward_data.hpp - syscalls/args.hpp - syscalls/decoder.hpp - syscalls/errno.hpp - syscalls/rets.hpp - syscalls/syscall.hpp - syscalls/detail/syscall_info.h - ) -endif() +target_sources( + gkfs_user_lib + PUBLIC gkfs_functions.hpp + env.hpp + hooks.hpp + intercept.hpp + logging.hpp + make_array.hpp + open_file_map.hpp + open_dir.hpp + path.hpp + preload.hpp + preload_context.hpp + preload_util.hpp + rpc/rpc_types.hpp + rpc/forward_management.hpp + rpc/forward_metadata.hpp + rpc/forward_data.hpp + syscalls/args.hpp + syscalls/decoder.hpp + syscalls/errno.hpp + syscalls/rets.hpp + syscalls/syscall.hpp + syscalls/detail/syscall_info.h + void_syscall_intercept.hpp + user_functions.hpp +) diff --git a/include/client/env.hpp b/include/client/env.hpp index 34a986facb0855c117370a2871fc36527c77d5df..4f24d9ab13b0a3d910d37006bbcac39a92d672dc 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -49,9 +49,15 @@ static constexpr auto LOG_PER_PROCESS = ADD_PREFIX("LOG_PER_PROCESS"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); -#ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); +#ifdef GKFS_ENABLE_CLIENT_METRICS +static constexpr auto ENABLE_METRICS = ADD_PREFIX("ENABLE_METRICS"); +static constexpr auto METRICS_FLUSH_INTERVAL = + ADD_PREFIX("METRICS_FLUSH_INTERVAL"); +static constexpr auto METRICS_PATH = ADD_PREFIX("METRICS_PATH"); +static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); #endif + static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); } // namespace gkfs::env diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 63b9b655f55d5851d780f0c5379b4b35d7225533..c616e0c0a0de6e43bfae9e1f5cd0ba8c9ad64c79 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -105,11 +105,13 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize); #endif ssize_t -gkfs_pwrite(std::shared_ptr file, const char* buf, - size_t count, off64_t offset, bool update_pos = false); +gkfs_do_write(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos); ssize_t -gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset); +gkfs_write_ws(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos = false); + +ssize_t +gkfs_pwrite(int fd, const void* buf, size_t count, off64_t offset); ssize_t gkfs_write(int fd, const void* buf, size_t count); @@ -121,11 +123,13 @@ ssize_t gkfs_writev(int fd, const struct iovec* iov, int iovcnt); ssize_t -gkfs_pread(std::shared_ptr file, char* buf, - size_t count, off64_t offset); +gkfs_do_read(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset); ssize_t -gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); +gkfs_read_ws(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset); + +ssize_t +gkfs_pread(int fd, void* buf, size_t count, off64_t offset); ssize_t gkfs_read(int fd, void* buf, size_t count); @@ -149,14 +153,22 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, int gkfs_rmdir(const std::string& path); +int +gkfs_close(unsigned int fd); + +std::vector +gkfs_get_file_list(const std::string& path); + #ifdef HAS_RENAME int gkfs_rename(const std::string& old_path, const std::string& new_path); #endif // HAS_RENAME + } // namespace gkfs::syscall // gkfs_getsingleserverdir is using extern "C" to demangle it for C usage extern "C" int gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp, unsigned int count, int server); + #endif // GEKKOFS_GKFS_FUNCTIONS_HPP diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index e71a6a08d68b2b0875b213b7f19e260a13c73b2b..fa9ec424bc606a941650edb253a20781af54cf8d 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -36,7 +36,13 @@ extern "C" { #include #include } + +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + /* * For PowerPC, syscall_no_intercept_wrapper() is defined in the diff --git a/include/client/logging.hpp b/include/client/logging.hpp index c99e2e4a034cf5567008ba1fa16ea6f7219105f9..9d1f464e52649e33268769b9a2cadea711b31fee 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -30,7 +30,11 @@ #ifndef LIBGKFS_LOGGING_HPP #define LIBGKFS_LOGGING_HPP +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif #include #include @@ -38,7 +42,6 @@ #include #include #include -#include #include #ifdef GKFS_DEBUG_BUILD @@ -175,6 +178,138 @@ log_buffer(int fd, const void* buffer, std::size_t length) { ::syscall_no_intercept(SYS_write, fd, buffer, length); } +/** + * @brief convert a time_t to a tm + * It is not POSIX compliant, but it dows not uses any syscall or timezone + * Converts a Unix timestamp (number of seconds since the beginning of 1970 + * CE) to a Gregorian civil date-time tuple in GMT (UTC) time zone. + * + * This conforms to C89 (and C99...) and POSIX. + * + * This implementation works, and doesn't overflow for any sizeof(time_t). + * It doesn't check for overflow/underflow in tm->tm_year output. Other than + * that, it never overflows or underflows. It assumes that that time_t is + * signed. + * + * This implements the inverse of the POSIX formula + * (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_15) + * for all time_t values, no matter the size, as long as tm->tm_year doesn't + * overflow or underflow. The formula is: tm_sec + tm_min*60 + tm_hour*3600 + * + tm_yday*86400 + (tm_year-70)*31536000 + ((tm_year-69)/4)*86400 - + * ((tm_year-1)/100)*86400 + ((tm_year+299)/400)*86400. + * + * License : GNU General Public License v2.0 from + * https://github.com/pts/minilibc686/ + * @param time_t + * @return tm + */ + +static inline struct tm* +mini_gmtime_r(const time_t* timep, struct tm* tm) { + const time_t ts = *timep; + time_t t = ts / 86400; + unsigned hms = + ts % + 86400; /* -86399 <= hms <= 86399. This needs sizeof(int) >= 4. */ + time_t c, f; + unsigned yday; /* 0 <= yday <= 426. Also fits to an `unsigned short', but + `int' is faster. */ + unsigned a; /* 0 <= a <= 2133. Also fits to an `unsigned short', but `int' + is faster. */ + if((int) hms < 0) { + --t; + hms += 86400; + } /* Fix quotient and negative remainder if ts was negative (i.e. before + year 1970 CE). */ + /* Now: -24856 <= t <= 24855. */ + tm->tm_sec = hms % 60; + hms /= 60; + tm->tm_min = hms % 60; + tm->tm_hour = hms / 60; + if(sizeof(time_t) > + 4) { /* Optimization. For int32_t, this would keep t intact, so we won't + have to do it. This produces unreachable code. */ + f = (t + 4) % 7; + if(f < 0) + f += 7; /* Fix negative remainder if (t + 4) was negative. */ + /* Now 0 <= f <= 6. */ + tm->tm_wday = f; + c = (t << 2) + 102032; + f = c / 146097; + if(c % 146097 < 0) + --f; /* Fix negative remainder if c was negative. */ + --f; + t += f; + f >>= 2; + t -= f; + f = (t << 2) + 102035; + c = f / 1461; + if(f % 1461 < 0) + --c; /* Fix negative remainder if f was negative. */ + } else { + tm->tm_wday = (t + 24861) % 7; /* t + 24861 >= 0. */ + /* Now: -24856 <= t <= 24855. */ + c = ((t << 2) + 102035) / 1461; + } + yday = t - 365 * c - (c >> 2) + 25568; + /* Now: 0 <= yday <= 425. */ + a = yday * 5 + 8; + /* Now: 8 <= a <= 2133. */ + tm->tm_mon = a / 153; + a %= 153; /* No need to fix if a < 0, because a cannot be negative here. */ + /* Now: 2 <= tm->tm_mon <= 13. */ + /* Now: 0 <= a <= 152. */ + tm->tm_mday = 1 + a / 5; /* No need to fix if a < 0, because a cannot be + negative here. */ + /* Now: 1 <= tm->tm_mday <= 31. */ + if(tm->tm_mon >= 12) { + tm->tm_mon -= 12; + /* Now: 0 <= tm->tm_mon <= 1. */ + ++c; + yday -= 366; + } else { /* Check for leap year (in c). */ + /* Now: 2 <= tm->tm_mon <= 11. */ + /* 1903: not leap; 1904: leap, 1900: not leap; 2000: leap */ + /* With sizeof(time_t) == 4, we have 1901 <= year <= 2038; of these + * years only 2000 is divisble by 100, and that's a leap year, no we + * optimize the check to `(c & 3) == 0' only. + */ + if(!((c & 3) == 0 && + (sizeof(time_t) <= 4 || c % 100 != 0 || (c + 300) % 400 == 0))) + --yday; /* These `== 0' comparisons work even if c < 0. */ + } + tm->tm_year = + c; /* This assignment may overflow or underflow, we don't check it. + Example: time_t is a huge int64_t, tm->tm_year is int32_t. */ + /* Now: 0 <= tm->tm_mon <= 11. */ + /* Now: 0 <= yday <= 365. */ + tm->tm_yday = yday; + tm->tm_isdst = 0; + return tm; +} + +static inline struct tm* +mini_gmtime(const time_t* timep) { + static struct tm tm; + return mini_gmtime_r(timep, &tm); +} + +static inline ssize_t +format_timeval(struct timeval* tv, char* buf, size_t sz) { + ssize_t written = -1; + struct tm* gm = mini_gmtime(&tv->tv_sec); + + + written = (ssize_t) strftime(buf, sz, "%Y-%m-%d %H:%M:%S", gm); + if((written > 0) && ((size_t) written < sz)) { + int w = snprintf(buf + written, sz - (size_t) written, ".%06ld", + tv->tv_usec); + written = (w > 0) ? written + w : -1; + } + + return written; +} + /** * format_timestamp_to - safely format a timestamp for logging messages * @@ -192,13 +327,10 @@ log_buffer(int fd, const void* buffer, std::size_t length) { * one thread exactly, and we pass it as an argument whenever we need to * format a timestamp. If no timezone is provided, we just format the epoch. * - * NOTE: we use the date C++ library to query the timezone database and - * to format the timestamps. */ template static inline void -format_timestamp_to(Buffer&& buffer, - const date::time_zone* const timezone = nullptr) { +format_timestamp_to(Buffer&& buffer) { struct ::timeval tv; @@ -208,17 +340,11 @@ format_timestamp_to(Buffer&& buffer, return; } - date::sys_time now{ - std::chrono::seconds{tv.tv_sec} + - std::chrono::microseconds{tv.tv_usec}}; + char buf[28]; - if(!timezone) { - fmt::format_to(buffer, "[{}] ", now.time_since_epoch().count()); - return; + if(format_timeval(&tv, buf, sizeof(buf)) > 0) { + fmt::format_to(std::back_inserter(buffer), "[{}] ", buf); } - - fmt::format_to(buffer, "[{}] ", - date::zoned_time{timezone, now}); } template @@ -226,7 +352,7 @@ static inline void format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { const auto ttid = syscall_no_intercept(SYS_gettid); - fmt::format_to(buffer, "[{}] [syscall] ", ttid); + fmt::format_to(std::back_inserter(buffer), "[{}] [syscall] ", ttid); char o; char t; @@ -256,20 +382,15 @@ format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { } const std::array tmp = {'[', o, t, ']', ' '}; - fmt::format_to(buffer, fmt::string_view(tmp.data(), tmp.size())); + fmt::format_to(std::back_inserter(buffer), + fmt::string_view(tmp.data(), tmp.size())); } } // namespace detail enum { max_buffer_size = LIBGKFS_LOG_MESSAGE_SIZE }; -struct static_buffer : public fmt::basic_memory_buffer { - -protected: - void - grow(std::size_t size) override final; -}; - +using static_buffer = fmt::basic_memory_buffer; struct logger { @@ -293,16 +414,17 @@ struct logger { } static_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); - fmt::format_to(buffer, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + detail::format_timestamp_to(buffer); + fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ", + log_process_id_, lookup_level_name(level)); if(!!(level & log::debug)) { - fmt::format_to(buffer, "<{}():{}> ", func, lineno); + fmt::format_to(std::back_inserter(buffer), "<{}():{}> ", func, + lineno); } - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(log_fd_, buffer); } @@ -337,8 +459,8 @@ struct logger { static_buffer prefix; detail::format_timestamp_to(prefix); - fmt::format_to(prefix, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + fmt::format_to(std::back_inserter(prefix), "[{}] [{}] ", + log_process_id_, lookup_level_name(level)); char buffer[max_buffer_size]; const int n = vsnprintf(buffer, sizeof(buffer), fmt, ap); @@ -387,8 +509,8 @@ struct logger { } static_buffer buffer; - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(fd, buffer); } @@ -410,8 +532,6 @@ struct logger { std::bitset<512> filtered_syscalls_; int debug_verbosity_; #endif - - const date::time_zone* timezone_; }; // the following static functions can be used to interact @@ -440,23 +560,6 @@ destroy_global_logger() { logger::global_logger().reset(); } -inline void -static_buffer::grow(std::size_t size) { - - const auto logger = get_global_logger(); - - if(logger) { - logger->log_mask_ &= ~(syscall | syscall_at_entry); - } - - std::fprintf( - stderr, - "FATAL: message too long for gkfs::log::static_buffer, increase the size of\n" - "LIBGKFS_LOG_MESSAGE_SIZE in CMake or reduce the length of the offending " - "message.\n"); - abort(); -} - } // namespace gkfs::log #define LOG(XXX, ...) LOG_##XXX(__VA_ARGS__) diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 175cd8f6edd89e441993f2055bb9eb325e611e24..afad97c684bbb5736bceef4079e67adc7b120b3c 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -31,7 +31,8 @@ #define IOINTERCEPT_PRELOAD_HPP #include - +#include +#include #define EUNKNOWN (-1) #define CTX gkfs::preload::PreloadContext::getInstance() @@ -40,11 +41,12 @@ void init_ld_env_if_needed(); } // namespace gkfs::preload +#ifndef BYPASS_SYSCALL void init_preload() __attribute__((constructor)); void destroy_preload() __attribute__((destructor)); - +#endif #endif // IOINTERCEPT_PRELOAD_HPP diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 1fa0b1925e4c3e18fb464bfd4fef939ea5d4c973..f73288032085456ca2a64506fa582c044276dd25 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -51,6 +51,9 @@ class Distributor; namespace log { struct logger; } +namespace messagepack { +class ClientMetrics; +} namespace preload { /* @@ -107,6 +110,10 @@ private: std::string hostname; int replicas_; + std::shared_ptr write_metrics_; + std::shared_ptr read_metrics_; + + public: static PreloadContext* getInstance() { @@ -119,9 +126,14 @@ public: void operator=(PreloadContext const&) = delete; + ~PreloadContext(); + void init_logging(); + bool + init_metrics(); + void mountdir(const std::string& path); @@ -223,6 +235,12 @@ public: int get_replicas(); + + const std::shared_ptr + write_metrics(); + + const std::shared_ptr + read_metrics(); }; } // namespace preload diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp index a3fa9b64a1104d892696af9bf5daf3c4eb674a64..4c756dec22d8fefbe3aeac29c2ed6c6bf6c6e6dd 100644 --- a/include/client/rpc/rpc_types.hpp +++ b/include/client/rpc/rpc_types.hpp @@ -85,10 +85,10 @@ struct fs_config { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3033006080; + constexpr static const uint64_t public_id = 1; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::fs_config; @@ -259,10 +259,10 @@ struct create { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 796590080; + constexpr static const uint64_t public_id = 2; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::create; @@ -375,10 +375,10 @@ struct stat { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1396244480; + constexpr static const uint64_t public_id = 3; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::stat; @@ -494,10 +494,10 @@ struct remove_metadata { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2087845888; + constexpr static const uint64_t public_id = 4; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::remove_metadata; @@ -618,10 +618,10 @@ struct decr_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1291649024; + constexpr static const uint64_t public_id = 5; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::decr_size; @@ -734,10 +734,10 @@ struct update_metadentry { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 99483648; + constexpr static const uint64_t public_id = 6; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::update_metadentry; @@ -960,10 +960,10 @@ struct get_metadentry_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3426484224; + constexpr static const uint64_t public_id = 7; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_metadentry_size; @@ -1076,10 +1076,10 @@ struct update_metadentry_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2760900608; + constexpr static const uint64_t public_id = 8; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::update_metadentry_size; @@ -1216,10 +1216,10 @@ struct mk_symlink { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3207004160; + constexpr static const uint64_t public_id = 9; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::mk_symlink; @@ -1334,10 +1334,10 @@ struct remove_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2649292800; + constexpr static const uint64_t public_id = 10; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::remove_data; @@ -1442,10 +1442,10 @@ struct write_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3716481024; + constexpr static const uint64_t public_id = 11; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::write; @@ -1627,10 +1627,10 @@ struct read_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 892207104; + constexpr static const uint64_t public_id = 12; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::read; @@ -1812,10 +1812,10 @@ struct trunc_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1850933248; + constexpr static const uint64_t public_id = 13; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::truncate; @@ -1931,10 +1931,10 @@ struct get_dirents { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 4121034752; + constexpr static const uint64_t public_id = 14; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_dirents; @@ -2055,10 +2055,10 @@ struct get_dirents_extended { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1463943168; + constexpr static const uint64_t public_id = 15; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_dirents_extended; @@ -2181,10 +2181,10 @@ struct chunk_stat { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 532742144; + constexpr static const uint64_t public_id = 16; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_chunk_stat; diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp index 05fa8296337d7ab2cfe689f394dc1d8365bdb61b..53ed726c3b1484bc7ec64b92508e26bf908ed73a 100644 --- a/include/client/syscalls/args.hpp +++ b/include/client/syscalls/args.hpp @@ -228,7 +228,7 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flag == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -239,12 +239,12 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { } if((flag == desc[i].flag_)) { - fmt::format_to(buffer, "{}", desc[i].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[i].name_); return; } } - fmt::format_to(buffer, "{:#x}", flag); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flag); } template @@ -254,7 +254,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flags == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -269,7 +269,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { } if((flags & desc[i].flag_) != 0) { - fmt::format_to(buffer, "{}{}", + fmt::format_to(std::back_inserter(buffer), "{}{}", buffer.size() != buffer_start ? "|" : "", desc[i].name_); flags &= ~desc[i].flag_; @@ -280,15 +280,15 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { if(flags != 0) { if(buffer.size() != buffer_start) { - fmt::format_to(buffer, "|"); + fmt::format_to(std::back_inserter(buffer), "|"); } - fmt::format_to(buffer, "{:#x}", flags); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flags); return; } if(buffer_start == buffer.size()) { - fmt::format_to(buffer, "0x0"); + fmt::format_to(std::back_inserter(buffer), "0x0"); } } @@ -311,7 +311,7 @@ format_whence_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SEEK_END) ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); } @@ -334,7 +334,7 @@ format_mmap_prot_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(PROT_WRITE), FLAG_ENTRY(PROT_EXEC)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; @@ -376,7 +376,7 @@ format_mmap_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { #endif ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; } @@ -420,14 +420,14 @@ format_clone_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(CLONE_NEWNET), FLAG_ENTRY(CLONE_IO)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); // the low byte in clone flags contains the number of the termination // signal sent to the parent when the child dies format_flag_set(buffer, parg.value & ~0x11l, flag_names); if((parg.value & 0x11l) != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_signum_arg_to(buffer, {"", parg.value & 0x11l}); } return; @@ -479,7 +479,7 @@ format_signum_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIGSYS)); if(std::strcmp(parg.name, "")) { - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); } format_flag(buffer, parg.value, flag_names); @@ -504,7 +504,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIG_UNBLOCK), FLAG_ENTRY(SIG_SETMASK)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, parg.value, flag_names); return; } @@ -518,7 +518,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } @@ -531,7 +531,7 @@ format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_fd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -546,11 +546,11 @@ inline void format_atfd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(static_cast(parg.value) == AT_FDCWD) { - fmt::format_to(buffer, "{}=AT_FDCWD", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=AT_FDCWD", parg.name); return; } - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -565,12 +565,12 @@ inline void format_cstr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}=\"{}\"", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}=\"{}\"", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } /** @@ -618,7 +618,7 @@ format_open_flags_to(FmtBuffer& buffer, long flags = parg.value; - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, flags & O_ACCMODE, flag_names); flags &= ~O_ACCMODE; @@ -636,7 +636,7 @@ format_open_flags_to(FmtBuffer& buffer, #endif // !O_TMPFILE if(flags != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_flag_set(buffer, flags, extra_flag_names); } } @@ -650,7 +650,7 @@ format_open_flags_to(FmtBuffer& buffer, template inline void format_octal_mode_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#04o}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#04o}", parg.name, parg.value); } /** @@ -664,12 +664,12 @@ inline void format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}={}", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } @@ -682,7 +682,7 @@ format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, parg.value); } @@ -695,7 +695,7 @@ format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -708,7 +708,7 @@ format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#x}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#x}", parg.name, parg.value); } #undef FLAG_ENTRY diff --git a/include/client/syscalls/decoder.hpp b/include/client/syscalls/decoder.hpp index eebdd6fdbc9551514b9c8d89cdc29d4a1fea6381..3172cc2df291c118dcf3de9c13cfa19f7177b10c 100644 --- a/include/client/syscalls/decoder.hpp +++ b/include/client/syscalls/decoder.hpp @@ -60,19 +60,19 @@ decode(FmtBuffer& buffer, const long syscall_number, const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); } template @@ -84,30 +84,31 @@ decode(FmtBuffer& buffer, const long syscall_number, const long argv[MAX_ARGS], const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } if(never_returns(syscall_number)) { - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); return; } if(error_code(result) != 0) { - fmt::format_to(buffer, ") = {} {} ({})", static_cast(-1), - errno_name(-result), errno_message(-result)); + fmt::format_to(std::back_inserter(buffer), ") = {} {} ({})", + static_cast(-1), errno_name(-result), + errno_message(-result)); return; } - fmt::format_to(buffer, ") = "); + fmt::format_to(std::back_inserter(buffer), ") = "); const auto& ret = sc.return_type(); ret.formatter()(buffer, result); } diff --git a/include/client/syscalls/rets.hpp b/include/client/syscalls/rets.hpp index adf77647c8928748d671effedaa73e45db1b26e8..1e40f77b02f6af0d457fd75b124f3ae14aaf85ef 100644 --- a/include/client/syscalls/rets.hpp +++ b/include/client/syscalls/rets.hpp @@ -121,7 +121,7 @@ template inline void format_none_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } template @@ -129,18 +129,18 @@ inline void format_ptr_ret_to(FmtBuffer& buffer, long val) { if(LIKELY(reinterpret_cast(val) != nullptr)) { - fmt::format_to(buffer, "{}", reinterpret_cast(val)); + fmt::format_to(std::back_inserter(buffer), "{}", reinterpret_cast(val)); return; } - fmt::format_to(buffer, "NULL"); + fmt::format_to(std::back_inserter(buffer), "NULL"); } template inline void format_dec_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "{}", val); + fmt::format_to(std::back_inserter(buffer), "{}", val); } #undef LIKELY diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp new file mode 100644 index 0000000000000000000000000000000000000000..de5f717840afcb2a1386a164292bdd37e2a9ccab --- /dev/null +++ b/include/client/user_functions.hpp @@ -0,0 +1,85 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + +#ifndef GEKKOFS_USER_FUNCTIONS_HPP +#define GEKKOFS_USER_FUNCTIONS_HPP +#include +#include +#include +#include + +struct linux_dirent64; + +namespace gkfs::syscall { + +int +gkfs_open(const std::string& path, mode_t mode, int flags); + +int +gkfs_create(const std::string& path, mode_t mode); + +int +gkfs_remove(const std::string& path); + +ssize_t +gkfs_write(int fd, const void* buf, size_t count); + +ssize_t +gkfs_read(int fd, void* buf, size_t count); + +int +gkfs_close(unsigned int fd); + +off64_t +gkfs_lseek(unsigned int fd, off64_t offset, unsigned int whence); + +ssize_t +gkfs_pwrite(int fd, const void* buf, size_t count, off64_t offset); + +ssize_t +gkfs_pread(int fd, void* buf, size_t count, off64_t offset); + +int +gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true); + +int +gkfs_remove(const std::string& path); + +std::vector +gkfs_get_file_list(const std::string& path); +} // namespace gkfs::syscall + + +extern "C" int +gkfs_init(); + +extern "C" int +gkfs_end(); + +#endif // GEKKOFS_USER_FUNCTIONS_HPP diff --git a/include/client/void_syscall_intercept.hpp b/include/client/void_syscall_intercept.hpp new file mode 100644 index 0000000000000000000000000000000000000000..16cd01b5bd59da0a1e31ac831c66ed707462e488 --- /dev/null +++ b/include/client/void_syscall_intercept.hpp @@ -0,0 +1,45 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ +#ifndef SYSCALL_BYPASS_HPP +#define SYSCALL_BYPASS_HPP + +static inline int +syscall_error_code(long result) { + if(result < 0 && result >= -0x1000) + return (int) -result; + + return 0; +} + +extern "C" { +long +syscall_no_intercept(long syscall_number, ...); +} + +#endif \ No newline at end of file diff --git a/include/common/CMakeLists.txt b/include/common/CMakeLists.txt index 9561361970c92e4ba89f02c1434508c2aaa1eb0e..38a23bc300c90a2a9f193bb440044a4890b07fa4 100644 --- a/include/common/CMakeLists.txt +++ b/include/common/CMakeLists.txt @@ -30,10 +30,3 @@ target_sources( gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp rpc/rpc_util.hpp ) - -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp - rpc/rpc_types.hpp rpc/rpc_util.hpp - ) -endif() diff --git a/include/common/common_defs.hpp b/include/common/common_defs.hpp index 649f49f75fc49e58b500a70392eb75328ceb0ea2..21c09a7438fcca08d98712c9a3b264edab231899 100644 --- a/include/common/common_defs.hpp +++ b/include/common/common_defs.hpp @@ -58,11 +58,18 @@ constexpr auto get_chunk_stat = "rpc_srv_chunk_stat"; } // namespace tag namespace protocol { -constexpr auto ofi_psm2 = "ofi+psm2"; +constexpr auto na_sm = "na+sm"; constexpr auto ofi_sockets = "ofi+sockets"; constexpr auto ofi_tcp = "ofi+tcp"; constexpr auto ofi_verbs = "ofi+verbs"; -constexpr auto na_sm = "na+sm"; +constexpr auto ofi_psm2 = "ofi+psm2"; +constexpr auto ucx_all = "ucx+all"; +constexpr auto ucx_tcp = "ucx+tcp"; +constexpr auto ucx_rc = "ucx+rc"; +constexpr auto ucx_ud = "ucx+ud"; +constexpr auto all_remote_protocols = {ofi_sockets, ofi_tcp, ofi_verbs, + ofi_psm2, ucx_all, ucx_tcp, + ucx_rc, ucx_ud}; } // namespace protocol } // namespace gkfs::rpc diff --git a/include/common/env_util.hpp b/include/common/env_util.hpp index e60fac7dc5fd81681181143b2e7c15d6b1f7e6ea..e9f667fb8500cd143f659052114079728a185e13 100644 --- a/include/common/env_util.hpp +++ b/include/common/env_util.hpp @@ -36,6 +36,9 @@ namespace gkfs::env { std::string get_var(const std::string& name, const std::string& default_value = ""); +bool +var_is_set(const std::string& name); + } // namespace gkfs::env #endif // GKFS_COMMON_ENV_UTIL_HPP diff --git a/include/common/msgpack_util.hpp b/include/common/msgpack_util.hpp new file mode 100644 index 0000000000000000000000000000000000000000..841369b4efc439289a7be375637bf009af0b9ac6 --- /dev/null +++ b/include/common/msgpack_util.hpp @@ -0,0 +1,150 @@ +/* + Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GKFS_COMMON_MSGPACK_HPP +#define GKFS_COMMON_MSGPACK_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace gkfs::messagepack { + +enum class client_metric_io_type { write, read }; +enum class client_metric_flush_type { file, socket }; + +class ClientMetrics { + +public: + /* + * MessagePack data structure for client metrics. Includes only what is + * actually sent + */ + struct msgpack_data { + uint32_t flush_t_; + std::string hostname_; + int pid_; + std::string io_type_; + std::vector start_t_{}; + std::vector end_t_{}; + std::vector req_size_{}; + uint32_t total_bytes_{}; + int total_iops_{0}; + + template + void + pack(T& pack) { + pack(flush_t_, hostname_, pid_, io_type_, start_t_, end_t_, + req_size_, total_iops_, total_bytes_); + } + + std::vector + pack_msgpack() { + return msgpack::pack(*this); + } + }; + +private: + bool metrics_enabled_{false}; + + msgpack_data msgpack_data_{}; + + // Initialization time used to compute relative timestamps + std::chrono::time_point init_t_; + + std::mutex data_mtx_{}; + std::thread flush_thread_{}; + std::condition_variable flush_thread_cv_{}; + std::mutex flush_thread_cv_mutex_{}; + std::atomic flush_thread_running_{false}; + + client_metric_flush_type flush_type_{client_metric_flush_type::file}; + int flush_interval_{}; + std::unique_ptr zmq_flush_context_ = nullptr; + std::unique_ptr zmq_flush_socket_ = nullptr; + std::string flush_path_{}; + int flush_count_{0}; + + +public: + ClientMetrics() = default; + + explicit ClientMetrics(client_metric_io_type io_type, + client_metric_flush_type flush_type = + client_metric_flush_type::file, + int flush_interval = 5); + + ~ClientMetrics(); + + void + add_event(size_t size, + std::chrono::time_point start); + + void + reset_metrics(); + + void + flush_msgpack(); + + void + flush_loop(); + + void + enable(); + + void + disable(); + + void + zmq_connect(const std::string& ip_port); + + bool + zmq_is_connected(); + + [[nodiscard]] const std::string& + path() const; + + void + path(const std::string& path, const std::string prefix = ""); + + int + flush_count() const; +}; + +} // namespace gkfs::messagepack + +#endif // GKFS_COMMON_MSGPACK_HPP diff --git a/include/config.hpp b/include/config.hpp index 7ec761f64e035c533adcebfa3a9da7b9c267816e..edf7b2c69a3a9174d7a79cce65cb3491fae12148 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -39,7 +39,16 @@ namespace gkfs::config { constexpr auto hostfile_path = "./gkfs_hosts.txt"; -constexpr auto forwarding_file_path = "./gkfs_forwarding.map"; +// We do not default this, ENV variable always required. +constexpr auto forwarding_file_path = ""; + +namespace client_metrics { +// Default directory where client metrics are stored. Can be set via +// LIBGKFS_METRICS_PATH. Filename consists of starting time, pid, and hostname +// Note: when LIBGKFS_METRICS_IP is given, ZeroMQ is used instead +constexpr auto flush_path = "/tmp/gkfs_client_metrics"; +constexpr auto flush_interval = 5; // in seconds +} // namespace client_metrics namespace io { /* diff --git a/include/daemon/CMakeLists.txt b/include/daemon/CMakeLists.txt index 9cdfe06a94e699e3c0280ac3bdbe3daf512b00ee..26466ba16138af272d680b78b9f081a4f516c454 100644 --- a/include/daemon/CMakeLists.txt +++ b/include/daemon/CMakeLists.txt @@ -38,21 +38,6 @@ target_sources( handler/rpc_util.hpp ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon - PUBLIC daemon.hpp - util.hpp - ops/data.hpp - ops/metadentry.hpp - classes/fs_data.hpp - classes/rpc_data.hpp - handler/rpc_defs.hpp - handler/rpc_util.hpp - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) - endif() - +if(GKFS_ENABLE_AGIOS) + target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) endif() diff --git a/include/daemon/backend/metadata/merge.hpp b/include/daemon/backend/metadata/merge.hpp index f655f7d4070593273fd28ad8431a5dd2c4f1f2f4..5d30aaef1ca19acc1edab3d5a2c959044eeb8ac8 100644 --- a/include/daemon/backend/metadata/merge.hpp +++ b/include/daemon/backend/metadata/merge.hpp @@ -33,6 +33,8 @@ #include #include +#include + namespace rdb = rocksdb; namespace gkfs::metadata { @@ -43,7 +45,9 @@ namespace gkfs::metadata { enum class OperandID : char { increase_size = 'i', decrease_size = 'd', - create = 'c' + create = 'c', + update_time = 't' + }; /** @@ -156,6 +160,33 @@ public: std::string serialize_params() const override; }; + +/** + * @brief Update time operand + */ +class UpdateTimeOperand : public MergeOperand { +private: + time_t mtime_{}; + +public: + UpdateTimeOperand() = default; + + explicit UpdateTimeOperand(time_t mtime); + + explicit UpdateTimeOperand(const rdb::Slice& serialized_op); + + OperandID + id() const override; + + std::string + serialize_params() const override; + + time_t + mtime() const { + return mtime_; + } +}; + /** * @brief Merge operator class passed to RocksDB, used during merge operations */ diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 13f8023d1aa2e4dd9c52cbb15d6c570eb37327a0..ef89b5e282e0b1f9cc7dd82f6b6c66c4287a9553 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -95,6 +95,7 @@ private: bool enable_stats_ = false; bool enable_chunkstats_ = false; bool enable_prometheus_ = false; + bool enable_forwarding_ = false; std::string stats_file_; // Prometheus @@ -246,6 +247,12 @@ public: void enable_chunkstats(bool enable_chunkstats); + bool + enable_forwarding() const; + + void + enable_forwarding(bool enable_forwarding); + bool enable_prometheus() const; diff --git a/scripts/profiles/0.9.3/agios.specs b/scripts/profiles/0.9.3/agios.specs new file mode 100644 index 0000000000000000000000000000000000000000..68921e298efc685cec9c99ac4f82213070a5442d --- /dev/null +++ b/scripts/profiles/0.9.3/agios.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies (except transport-specific and experimental)" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "agios" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/all.specs b/scripts/profiles/0.9.3/all.specs new file mode 100644 index 0000000000000000000000000000000000000000..67396b8dc377d0a5d95010d37a280a486cac4c8f --- /dev/null +++ b/scripts/profiles/0.9.3/all.specs @@ -0,0 +1,86 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.17-20230812" + ["curl"]="7.88.1" + ["prometheus-cpp"]="v1.0.0" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "agios" "curl" "prometheus-cpp" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/arm.specs b/scripts/profiles/0.9.3/arm.specs new file mode 100644 index 0000000000000000000000000000000000000000..1d5432de89d1e4cbf40c9f2137bef4d5418e082d --- /dev/null +++ b/scripts/profiles/0.9.3/arm.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="fb73c3924b502e2840a5dc8a18746e395b06a8a6" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch.arm64" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" + ["syscall_intercept"]="ARM64" +) diff --git a/scripts/profiles/0.9.3/ci.specs b/scripts/profiles/0.9.3/ci.specs new file mode 100644 index 0000000000000000000000000000000000000000..dc9fcedffa4a46fda15bbdbefc973fe98617e61e --- /dev/null +++ b/scripts/profiles/0.9.3/ci.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies required by the CI" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="8.5.4" + ["prometheus-cpp"]="v1.0.0" + ) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" + ) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "agios" "parallax" "prometheus-cpp" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/default.specs b/scripts/profiles/0.9.3/default.specs new file mode 100644 index 0000000000000000000000000000000000000000..59d8175fe8bee6a4dedc6b530323e507a583b850 --- /dev/null +++ b/scripts/profiles/0.9.3/default.specs @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" +) + +# Dependencies that must be cloned. +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/default_zmq.specs b/scripts/profiles/0.9.3/default_zmq.specs new file mode 100644 index 0000000000000000000000000000000000000000..7185ddb19b9355a5b5e33ec331a7acc330fc51aa --- /dev/null +++ b/scripts/profiles/0.9.3/default_zmq.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["libzmq"]="4.3.5" + ["cppzmq"]="4.10.0" +) + +# Dependencies that must be cloned. +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "libzmq" "cppzmq" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/infiniband_verbs.specs b/scripts/profiles/0.9.3/infiniband_verbs.specs new file mode 100644 index 0000000000000000000000000000000000000000..abb374413860c22a20fec3f36eae13e2343489a1 --- /dev/null +++ b/scripts/profiles/0.9.3/infiniband_verbs.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Infiniband supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.5.4" + ["json-c"]="0.17-20230812" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%verbs"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading/installing +order=( + "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%verbs"]="--enable-verbs=yes" +) diff --git a/scripts/profiles/0.9.3/install/agios.install b/scripts/profiles/0.9.3/install/agios.install new file mode 100644 index 0000000000000000000000000000000000000000..b5ed7d7ee930fd2eb79a4113a3aaccd533f6fbe3 --- /dev/null +++ b/scripts/profiles/0.9.3/install/agios.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="agios" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/argobots.install b/scripts/profiles/0.9.3/install/argobots.install new file mode 100644 index 0000000000000000000000000000000000000000..9ff03b5a09576c4e07b3fceaea6c2922ca2ee753 --- /dev/null +++ b/scripts/profiles/0.9.3/install/argobots.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/capstone.install b/scripts/profiles/0.9.3/install/capstone.install new file mode 100644 index 0000000000000000000000000000000000000000..afd7794aff4c51cfc24b9351fd875a4a8724f1dd --- /dev/null +++ b/scripts/profiles/0.9.3/install/capstone.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/cppzmq.install b/scripts/profiles/0.9.3/install/cppzmq.install new file mode 100644 index 0000000000000000000000000000000000000000..bc236806db28e51fb46d6bcb247631034af6ce6f --- /dev/null +++ b/scripts/profiles/0.9.3/install/cppzmq.install @@ -0,0 +1,62 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="cppzmq" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/curl.install b/scripts/profiles/0.9.3/install/curl.install new file mode 100644 index 0000000000000000000000000000000000000000..44fab7ce95e891e8372cbffc6ac3bd60825ab0e6 --- /dev/null +++ b/scripts/profiles/0.9.3/install/curl.install @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="curl" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + autoreconf -fi + ./configure --prefix="${INSTALL_DIR}" --without-ssl + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/json-c.install b/scripts/profiles/0.9.3/install/json-c.install new file mode 100644 index 0000000000000000000000000000000000000000..9ba99fed64b2d4c573c4e7a247be1b6d0f949a94 --- /dev/null +++ b/scripts/profiles/0.9.3/install/json-c.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/json-c" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install + # Margo doesn't search in both directories, so we make it available in both lib and lib64 + if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then + cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/ + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/libfabric%verbs.install b/scripts/profiles/0.9.3/install/libfabric%verbs.install new file mode 100644 index 0000000000000000000000000000000000000000..74a1921023aa08f5098474d1041577a2a1986dea --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric%verbs.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/libfabric.install b/scripts/profiles/0.9.3/install/libfabric.install new file mode 100644 index 0000000000000000000000000000000000000000..6fee0bf9e77375a776e91db574615aaf53988025 --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/libzmq.install b/scripts/profiles/0.9.3/install/libzmq.install new file mode 100644 index 0000000000000000000000000000000000000000..9f98a196d259e01d2190977879e0edb3b2e95a50 --- /dev/null +++ b/scripts/profiles/0.9.3/install/libzmq.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="libzmq" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/lz4.install b/scripts/profiles/0.9.3/install/lz4.install new file mode 100644 index 0000000000000000000000000000000000000000..893a1b13bc4fa570657908c6a6ce8986d4bd1b57 --- /dev/null +++ b/scripts/profiles/0.9.3/install/lz4.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + # try to remove binaries first in case they already exist. Otherwise install fails. + LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4) + for LZ4_BIN in "${LZ4_BINS[@]}"; do + [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN" + done + make -j"${CORES}" + make DESTDIR="${INSTALL_DIR}" PREFIX="" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/margo.install b/scripts/profiles/0.9.3/install/margo.install new file mode 100644 index 0000000000000000000000000000000000000000..014621c68c61b93b0a322e64b60d2f8a332dd212 --- /dev/null +++ b/scripts/profiles/0.9.3/install/margo.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="margo" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/mercury.install b/scripts/profiles/0.9.3/install/mercury.install new file mode 100644 index 0000000000000000000000000000000000000000..9fc5588c9b2a8ffdbc1ce36697cf7725a0e83853 --- /dev/null +++ b/scripts/profiles/0.9.3/install/mercury.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DNA_USE_OFI:BOOL=ON -DNA_USE_UCX:BOOL=ON \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/parallax.install b/scripts/profiles/0.9.3/install/parallax.install new file mode 100644 index 0000000000000000000000000000000000000000..8c0cc9e95296b2553c651f4d8e1858fee8334035 --- /dev/null +++ b/scripts/profiles/0.9.3/install/parallax.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + CURR="${SOURCE_DIR}/parallax" + # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \ + -DDISABLE_LOGGING:BOOL=ON \ + .. + make -j"${CORES}" + make install + # We need to copy this file as it is not installed + cp ${CURR}/lib/include/parallax/structures.h ${INSTALL_DIR}/include/ +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/prometheus-cpp.install b/scripts/profiles/0.9.3/install/prometheus-cpp.install new file mode 100644 index 0000000000000000000000000000000000000000..154db65dfeadf13a1e4515e33203251d3ef8fead --- /dev/null +++ b/scripts/profiles/0.9.3/install/prometheus-cpp.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="prometheus-cpp" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/rocksdb.install b/scripts/profiles/0.9.3/install/rocksdb.install new file mode 100644 index 0000000000000000000000000000000000000000..7c3004fee34a937dd813fc560714ce9aac8e3009 --- /dev/null +++ b/scripts/profiles/0.9.3/install/rocksdb.install @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CXXFLAGS='' + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move -Wno-error=maybe-uninitialized' + fi + + # TODO use SSE? + CURR="${SOURCE_DIR}/rocksdb" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \ + -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \ + -DROCKSDB_BUILD_SHARED=OFF \ + -DWITH_LZ4=ON \ + -DWITH_GFLAGS=OFF \ + -DUSE_RTTI=1 \ + -DPORTABLE=1 \ + -DWITH_ALL_TESTS=OFF \ + -DWITH_BENCHMARK_TOOLS=OFF \ + -DWITH_TOOLS=OFF .. + make -j"${CORES}" install + + +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/syscall_intercept.install b/scripts/profiles/0.9.3/install/syscall_intercept.install new file mode 100644 index 0000000000000000000000000000000000000000..f69393d3a9ddb2c68f397973b26108035ea4ce4e --- /dev/null +++ b/scripts/profiles/0.9.3/install/syscall_intercept.install @@ -0,0 +1,71 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + prepare_build_dir "${CURR}" + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cd "${CURR}"/arch/aarch64/ + mkdir -p build + cd build + else + cd "${CURR}"/build + fi + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DBUILD_TESTS:BOOK=OFF .. + make -j"${CORES}" install + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cp "${CURR}"/arch/aarch64/include/libsyscall_intercept_hook_point.h ${INSTALL_DIR}/include + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/ucx.install b/scripts/profiles/0.9.3/install/ucx.install new file mode 100644 index 0000000000000000000000000000000000000000..fd90d6d3f62dd7552becf46bb02ee8c9fdc7a82c --- /dev/null +++ b/scripts/profiles/0.9.3/install/ucx.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + ID="ucx" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../contrib/configure-release --prefix=${INSTALL_DIR} + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/marenostrum4.specs b/scripts/profiles/0.9.3/marenostrum4.specs new file mode 100644 index 0000000000000000000000000000000000000000..15814ec63e526f254d74abccee0711e156599fa3 --- /dev/null +++ b/scripts/profiles/0.9.3/marenostrum4.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Marenostrum 4 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.5.4" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/0.9.3/mogon2.specs b/scripts/profiles/0.9.3/mogon2.specs new file mode 100644 index 0000000000000000000000000000000000000000..f7d9b2cadd509c0c52486d8e712902244f42d45b --- /dev/null +++ b/scripts/profiles/0.9.3/mogon2.specs @@ -0,0 +1,85 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" +# ["psm2"]="11.2.230" +# ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +# ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --enable-opx=yes" +# ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/mogon3.specs b/scripts/profiles/0.9.3/mogon3.specs new file mode 100644 index 0000000000000000000000000000000000000000..8d1ca5497a719f3cc4e0b285fa4d5ac382c1869f --- /dev/null +++ b/scripts/profiles/0.9.3/mogon3.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["libzmq"]="4.3.5" + ["cppzmq"]="4.10.0" +) + +# Dependencies that must be cloned +clonedeps=( +# ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "libzmq" "cppzmq" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +# ["libfabric"]="" +) diff --git a/scripts/profiles/0.9.3/ngio.specs b/scripts/profiles/0.9.3/ngio.specs new file mode 100644 index 0000000000000000000000000000000000000000..fd1f3a3353a462a82a6080f8193f908936f74e0c --- /dev/null +++ b/scripts/profiles/0.9.3/ngio.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for NEXTGenIO prototype cluster" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/omnipath_psm2.specs b/scripts/profiles/0.9.3/omnipath_psm2.specs new file mode 100644 index 0000000000000000000000000000000000000000..8cdac16d86074198212610762b7c38a98adb9bda --- /dev/null +++ b/scripts/profiles/0.9.3/omnipath_psm2.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Omnipath supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/p9.specs b/scripts/profiles/0.9.3/p9.specs new file mode 100644 index 0000000000000000000000000000000000000000..cf80be5a9c2b08a3ea03eb739177aa7f59e902d2 --- /dev/null +++ b/scripts/profiles/0.9.3/p9.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/latest b/scripts/profiles/latest index f76f9131742ee389211cdafc141553a15268098f..b3ec1638fda741e3680a70960377c66e3b41c4c7 120000 --- a/scripts/profiles/latest +++ b/scripts/profiles/latest @@ -1 +1 @@ -0.9.2 \ No newline at end of file +0.9.3 \ No newline at end of file diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index b659f38fbc0f4f6a0cd194047dbf381940285f77..385ac8afb1ccd5d07349c82f03e601835227879e 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -39,7 +39,7 @@ sources=( ["argobots"]="https://github.com/pmodels/argobots/archive/v{{VERSION}}.tar.gz" ["rocksdb"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" ["rocksdb%experimental"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" - ["psm2"]="https://github.com/intel/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" + ["psm2"]="https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" ["bmi"]="https://github.com/radix-io/bmi/" ["libfabric"]="https://github.com/ofiwg/libfabric.git" ["libfabric%experimental"]="https://github.com/ofiwg/libfabric.git" @@ -53,6 +53,9 @@ sources=( ["parallax"]="https://github.com/CARV-ICS-FORTH/parallax.git" ["prometheus-cpp"]="https://github.com/jupp0r/prometheus-cpp/releases/download/{{VERSION}}/prometheus-cpp-with-submodules.tar.gz" ["curl"]="https://curl.se/download/curl-{{VERSION}}.tar.gz" + ["ucx"]="https://github.com/openucx/ucx/archive/{{VERSION}}.tar.gz" + ["libzmq"]="https://github.com/zeromq/libzmq/archive/v{{VERSION}}.tar.gz" + ["cppzmq"]="https://github.com/zeromq/cppzmq/archive/v{{VERSION}}.tar.gz" ) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0108fb9c9fa5b8a543fcf0170dca8422c9941554..2a518c1ab7e7016ba30c86a70463b791e2b4e588 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -31,13 +31,33 @@ # This builds the `libgkfs_intercept.so` library: the primary GekkoFS client # based on syscall interception. # ############################################################################## -add_library(gkfs_intercept SHARED) -target_sources( - gkfs_intercept - PRIVATE gkfs_functions.cpp +add_library (gkfs_intercept SHARED) +add_library (gkfs_user_lib SHARED) + + +target_sources(gkfs_intercept + PRIVATE gkfs_functions.cpp + intercept.cpp hooks.cpp + logging.cpp + open_file_map.cpp + open_dir.cpp + path.cpp + preload.cpp + preload_context.cpp + preload_util.cpp + rpc/rpc_types.cpp + rpc/forward_data.cpp + rpc/forward_management.cpp + rpc/forward_metadata.cpp + syscalls/detail/syscall_info.c) + +target_sources( + gkfs_user_lib + PRIVATE gkfs_functions.cpp intercept.cpp + hooks.cpp logging.cpp open_file_map.cpp open_dir.cpp @@ -49,19 +69,44 @@ target_sources( rpc/forward_data.cpp rpc/forward_management.cpp rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c + syscalls/detail/syscall_info.c syscalls/util.S ) +target_compile_definitions(gkfs_user_lib PUBLIC BYPASS_SYSCALL) + +if(GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) +endif() +# Enable MSGPack metrics for intercept only + target_link_libraries( gkfs_intercept PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl + PUBLIC dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads + Syscall_intercept::Syscall_intercept +) +# Enable MSGPack metrics for intercept only +if (GKFS_ENABLE_CLIENT_METRICS) + target_link_libraries( + gkfs_intercept + PUBLIC + msgpack_util + ) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_CLIENT_METRICS) +endif () + +target_link_libraries( + gkfs_user_lib + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC dl Mercury::Mercury hermes fmt::fmt Threads::Threads - Date::TZ ) install( @@ -71,54 +116,17 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -# ############################################################################## -# This builds the `libgkfwd_intercept.so` library: the client for GekkoFS' -# forwarding mode based on syscall interception. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - add_library(gkfwd_intercept SHARED) - - target_sources( - gkfwd_intercept - PRIVATE gkfs_functions.cpp - hooks.cpp - intercept.cpp - logging.cpp - open_file_map.cpp - open_dir.cpp - path.cpp - preload.cpp - preload_context.cpp - preload_util.cpp - rpc/rpc_types.cpp - rpc/forward_data.cpp - rpc/forward_management.cpp - rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c - ) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_FORWARDING) - if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_AGIOS) - endif() +set_target_properties(gkfs_user_lib + PROPERTIES + PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" + PUBLIC_HEADER "../../include/client/user_functions.hpp" +) - target_link_libraries( - gkfwd_intercept - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads - Date::TZ - ) - install( - TARGETS gkfwd_intercept - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs - ) -endif() +install( + TARGETS gkfs_user_lib + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs +) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index b03e8eaebe906bf24ffe9fdaadec01799541a4da..ca8b5855e4d83e502fcfc3648d05fcc912426113 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -37,6 +37,10 @@ #include #include +#ifdef GKFS_ENABLE_CLIENT_METRICS +#include +#endif +#include extern "C" { #include // used for file types in the getdents{,64}() functions @@ -859,7 +863,7 @@ gkfs_dup2(const int oldfd, const int newfd) { } /** - * Wrapper function for all gkfs write operations + * Actual write function for all gkfs write operations * errno may be set * @param file * @param buf @@ -870,16 +874,15 @@ gkfs_dup2(const int oldfd, const int newfd) { * @return written size or -1 on error */ ssize_t -gkfs_pwrite(std::shared_ptr file, const char* buf, - size_t count, off64_t offset, bool update_pos) { - if(file->type() != gkfs::filemap::FileType::regular) { - assert(file->type() == gkfs::filemap::FileType::directory); +gkfs_do_write(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos) { + if(file.type() != gkfs::filemap::FileType::regular) { + assert(file.type() == gkfs::filemap::FileType::directory); LOG(WARNING, "Cannot write to directory"); errno = EISDIR; return -1; } - auto path = make_unique(file->path()); - auto is_append = file->get_flag(gkfs::filemap::OpenFile_flags::append); + auto path = make_unique(file.path()); + auto is_append = file.get_flag(gkfs::filemap::OpenFile_flags::append); auto write_size = 0; auto num_replicas = CTX->get_replicas(); @@ -929,7 +932,7 @@ gkfs_pwrite(std::shared_ptr file, const char* buf, } if(update_pos) { // Update offset in file descriptor in the file map - file->pos(offset + write_size); + file.pos(offset + write_size); } if(static_cast(write_size) != count) { LOG(WARNING, @@ -939,6 +942,29 @@ gkfs_pwrite(std::shared_ptr file, const char* buf, return write_size; // return written size } +/** + * Wrapper function for all gkfs write operations + * errno may be set + * @param file + * @param buf + * @param count + * @param offset + * @param update_pos pos should only be updated for some write operations (see + * man 2 pwrite) + * @return written size or -1 on error + */ +ssize_t +gkfs_write_ws(gkfs::filemap::OpenFile& file, const char* buf, size_t count, off64_t offset, bool update_pos) { +#ifdef GKFS_ENABLE_CLIENT_METRICS + auto start_t = std::chrono::high_resolution_clock::now(); + auto written = gkfs_do_write(file, buf, count, offset, update_pos); + CTX->write_metrics()->add_event(written, start_t); + return written; +#else + return gkfs_do_write(file, buf, count, offset, update_pos); +#endif +} + /** * gkfs wrapper for pwrite() system calls * errno may be set @@ -949,9 +975,10 @@ gkfs_pwrite(std::shared_ptr file, const char* buf, * @return written size or -1 on error */ ssize_t -gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset) { +gkfs_pwrite(int fd, const void* buf, size_t count, off64_t offset) { auto file = CTX->file_map()->get(fd); - return gkfs_pwrite(file, reinterpret_cast(buf), count, offset); + return gkfs_write_ws(*file, reinterpret_cast(buf), count, + offset); } /** @@ -964,10 +991,10 @@ gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset) { */ ssize_t gkfs_write(int fd, const void* buf, size_t count) { - auto gkfs_fd = CTX->file_map()->get(fd); + auto gkfs_file = CTX->file_map()->get(fd); // call pwrite and update pos - auto ret = gkfs_pwrite(gkfs_fd, reinterpret_cast(buf), count, - gkfs_fd->pos(), true); + auto ret = gkfs_write_ws(*gkfs_file, reinterpret_cast(buf), + count, gkfs_file->pos(), true); return ret; } @@ -993,7 +1020,7 @@ gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset) { continue; } auto buf = (iov + i)->iov_base; - ret = gkfs_pwrite(file, reinterpret_cast(buf), count, pos); + ret = gkfs_write_ws(*file, reinterpret_cast(buf), count, pos); if(ret == -1) { break; } @@ -1034,7 +1061,7 @@ gkfs_writev(int fd, const struct iovec* iov, int iovcnt) { } /** - * Wrapper function for all gkfs read operations + * Actual read function for all gkfs read operations * @param file * @param buf * @param count @@ -1042,10 +1069,9 @@ gkfs_writev(int fd, const struct iovec* iov, int iovcnt) { * @return read size or -1 on error */ ssize_t -gkfs_pread(std::shared_ptr file, char* buf, - size_t count, off64_t offset) { - if(file->type() != gkfs::filemap::FileType::regular) { - assert(file->type() == gkfs::filemap::FileType::directory); +gkfs_do_read(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset) { + if(file.type() != gkfs::filemap::FileType::regular) { + assert(file.type() == gkfs::filemap::FileType::directory); LOG(WARNING, "Cannot read from directory"); errno = EISDIR; return -1; @@ -1060,17 +1086,17 @@ gkfs_pread(std::shared_ptr file, char* buf, std::set failed; // set with failed targets. if(CTX->get_replicas() != 0) { - ret = gkfs::rpc::forward_read(file->path(), buf, offset, count, + ret = gkfs::rpc::forward_read(file.path(), buf, offset, count, CTX->get_replicas(), failed); while(ret.first == EIO) { - ret = gkfs::rpc::forward_read(file->path(), buf, offset, count, + ret = gkfs::rpc::forward_read(file.path(), buf, offset, count, CTX->get_replicas(), failed); LOG(WARNING, "gkfs::rpc::forward_read() failed with ret '{}'", ret.first); } } else { - ret = gkfs::rpc::forward_read(file->path(), buf, offset, count, 0, + ret = gkfs::rpc::forward_read(file.path(), buf, offset, count, 0, failed); } @@ -1084,6 +1110,41 @@ gkfs_pread(std::shared_ptr file, char* buf, return ret.second; // return read size } +/** + * Wrapper function for all gkfs read operations + * @param file + * @param buf + * @param count + * @param offset + * @return read size or -1 on error + */ +ssize_t +gkfs_read_ws(const gkfs::filemap::OpenFile& file, char* buf, size_t count, off64_t offset) { +#ifdef GKFS_ENABLE_CLIENT_METRICS + auto start_t = std::chrono::high_resolution_clock::now(); + auto read = gkfs_do_read(file, buf, count, offset); + CTX->read_metrics()->add_event(read, start_t); + return read; +#else + return gkfs_do_read(file, buf, count, offset); +#endif +} + +/** + * gkfs wrapper for pread() system calls + * errno may be set + * @param fd + * @param buf + * @param count + * @param offset + * @return read size or -1 on error + */ +ssize_t +gkfs_pread(int fd, void* buf, size_t count, off64_t offset) { + auto gkfs_fd = CTX->file_map()->get(fd); + return gkfs_read_ws(*gkfs_fd, reinterpret_cast(buf), count, offset); +} + /** * gkfs wrapper for read() system calls * errno may be set @@ -1096,7 +1157,7 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { auto gkfs_fd = CTX->file_map()->get(fd); auto pos = gkfs_fd->pos(); // retrieve the current offset - auto ret = gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, pos); + auto ret = gkfs_read_ws(*gkfs_fd, reinterpret_cast(buf), count, pos); // Update offset in file descriptor in the file map if(ret > 0) { gkfs_fd->pos(pos + ret); @@ -1126,7 +1187,7 @@ gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { continue; } auto buf = (iov + i)->iov_base; - ret = gkfs_pread(file, reinterpret_cast(buf), count, pos); + ret = gkfs_read_ws(*file, reinterpret_cast(buf), count, pos); if(ret == -1) { break; } @@ -1166,21 +1227,6 @@ gkfs_readv(int fd, const struct iovec* iov, int iovcnt) { return ret; } -/** - * gkfs wrapper for pread() system calls - * errno may be set - * @param fd - * @param buf - * @param count - * @param offset - * @return read size or -1 on error - */ -ssize_t -gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) { - auto gkfs_fd = CTX->file_map()->get(fd); - return gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, offset); -} - /** * wrapper function for opening directories * errno may be set @@ -1394,6 +1440,28 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, return written; } +/** + * @brief Closes an fd. To be used externally + * + * @param fd + * @return int + */ +int +gkfs_close(unsigned int fd) { + if(CTX->file_map()->exist(fd)) { + // No call to the daemon is required + CTX->file_map()->remove(fd); + return 0; + } + + if(CTX->is_internal_fd(fd)) { + // the client application (for some reason) is trying to close an + // internal fd: ignore it + return 0; + } + + return -1; +} #ifdef HAS_SYMLINKS #ifdef GKFS_ENABLE_UNUSED_FUNCTIONS @@ -1482,6 +1550,28 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize) { #endif #endif + +std::vector +gkfs_get_file_list(const std::string& path) { + auto ret = gkfs::rpc::forward_get_dirents(path); + auto err = ret.first; + if(err) { + errno = err; + return {}; + } + + auto open_dir = ret.second; + + std::vector file_list; + unsigned int pos = 0; + + while(pos < open_dir->size()) { + auto de = open_dir->getdent(pos++); + file_list.push_back(de.name()); + } + return file_list; +} + } // namespace gkfs::syscall diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 56e1f829b3319d2a91d21504a756639a927b1a35..a20503a5dbd3133c72496db769d7da1b6e0d0893 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -91,17 +91,10 @@ hook_close(int fd) { LOG(DEBUG, "{}() called with fd: {}", __func__, fd); - if(CTX->file_map()->exist(fd)) { - // No call to the daemon is required - CTX->file_map()->remove(fd); - return 0; - } + auto ret = gkfs::syscall::gkfs_close(fd); - if(CTX->is_internal_fd(fd)) { - // the client application (for some reason) is trying to close an - // internal fd: ignore it + if(ret == 0) return 0; - } return syscall_no_intercept_wrapper(SYS_close, fd); } @@ -242,7 +235,7 @@ hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos) { fd, fmt::ptr(buf), count, pos); if(CTX->file_map()->exist(fd)) { - return with_errno(gkfs::syscall::gkfs_pread_ws(fd, buf, count, pos)); + return with_errno(gkfs::syscall::gkfs_pread(fd, buf, count, pos)); } /* Since kernel 2.6: pread() became pread64(), and pwrite() became * pwrite64(). */ @@ -296,7 +289,7 @@ hook_pwrite(unsigned int fd, const char* buf, size_t count, loff_t pos) { fd, fmt::ptr(buf), count, pos); if(CTX->file_map()->exist(fd)) { - return with_errno(gkfs::syscall::gkfs_pwrite_ws(fd, buf, count, pos)); + return with_errno(gkfs::syscall::gkfs_pwrite(fd, buf, count, pos)); } /* Since kernel 2.6: pread() became pread64(), and pwrite() became * pwrite64(). */ @@ -878,7 +871,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, "newname \"{}\", flags {}", __func__, olddfd, oldname, newdfd, newname, flags); - const char* oldpath_pass; + const char* oldpath_pass = oldname; std::string oldpath_resolved; auto oldpath_status = CTX->relativize_fd_path(olddfd, oldname, oldpath_resolved); @@ -902,7 +895,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, return -EINVAL; } - const char* newpath_pass; + const char* newpath_pass = newname; std::string newpath_resolved; auto newpath_status = CTX->relativize_fd_path(newdfd, newname, newpath_resolved); diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 316c5e398403761b00d5f32917f153cca9c22c7b..7a8a455fe4aa63f6f3eab2bec381023dfbb888f0 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -44,6 +44,24 @@ extern "C" { #include } + +#ifdef BYPASS_SYSCALL +int (*intercept_hook_point)(long syscall_number, long arg0, long arg1, + long arg2, long arg3, long arg4, long arg5, + long* result){}; + +void (*intercept_hook_point_clone_child)(unsigned long flags, void* child_stack, + int* ptid, int* ctid, long newtls){}; + +void (*intercept_hook_point_clone_parent)(unsigned long flags, + void* child_stack, int* ptid, + int* ctid, long newtls, + long returned_pid){}; + +void (*intercept_hook_point_post_kernel)(long syscall_number, long arg0, + long arg1, long arg2, long arg3, + long arg4, long arg5, long result){}; +#endif namespace { thread_local bool reentrance_guard_flag; diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 88f9947140e070bbca5abc8f6410e7a0fd942588..830d5691ecd281c9974bc98a5ca299935fdc5b01 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -32,11 +32,7 @@ #include #include #include - -extern "C" { -#include -#include -} +#include #ifdef GKFS_ENABLE_LOGGING @@ -289,8 +285,7 @@ logger::logger(const std::string& opts, const std::string& path, , const std::string& filter, int verbosity #endif - ) - : timezone_(nullptr) { +) { /* use stderr by default */ log_fd_ = 2; @@ -335,34 +330,6 @@ logger::logger(const std::string& opts, const std::string& path, log_fd_ = fd; } - // Finding the current timezone implies accessing OS files (i.e. syscalls), - // but current_zone() doesn't actually retrieve the time zone but rather - // provides a descriptor to it that is **atomically initialized** upon its - // first use. Thus, if we don't force the initialization here, logging the - // first intercepted syscall will produce a call to date::time_zone::init() - // (under std::call_once) which internally ends up calling fopen(). Since - // fopen() ends up calling sys_open(), we will need to generate another - // timestamp for a system call log entry, which will attempt to call - // date::time_zone::init() since the prior initialization (under the same - // std::call_once) has not yet completed. - // - // Unfortunately, date::time_zone doesn't provide a function to prevent - // this lazy initialization, therefore we force it by requesting - // information from an arbitrary timepoint (January 1st 1970) which forces - // the initialization. This doesn't do any actual work and could safely be - // removed if the date API ends up providing this functionality. - try { - timezone_ = date::current_zone(); -#ifdef GKFS_DEBUG_BUILD - using namespace date; - timezone_->get_info(date::sys_days{January / 1 / 1970}); -#endif // GKFS_DEBUG_BUILD - } catch(const std::exception& ex) { - // if timezone initialization fails, setting timezone_ to nullptr - // makes format_timestamp_to() default to producing epoch timestamps - timezone_ = nullptr; - } - #ifdef GKFS_ENABLE_LOGGING const auto log_hermes_message = [](const std::string& msg, hermes::log::level l, int severity, @@ -490,9 +457,9 @@ logger::log_syscall(syscall::info info, const long syscall_number, print_syscall: - static_buffer buffer; + fmt::basic_memory_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); + detail::format_timestamp_to(buffer); detail::format_syscall_info_to(buffer, info); if(result) { @@ -501,7 +468,7 @@ print_syscall: syscall::decode(buffer, syscall_number, args); } - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), "\n"); ::syscall_no_intercept(SYS_write, log_fd_, buffer.data(), buffer.size()); } diff --git a/src/client/path.cpp b/src/client/path.cpp index 4f72b4807c3cc09da756ab769aebe89cd9871e05..85357209a24ae47a87ed8358cd8d3cad255f232a 100644 --- a/src/client/path.cpp +++ b/src/client/path.cpp @@ -39,9 +39,14 @@ #include #include +#ifndef BYPASS_SYSCALL +#include +#else +#include +#endif + extern "C" { #include -#include } using namespace std; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4de0df2c31168fe6f07a434a59e0eeb2331be27c..8f3e1e9881baa652104134771b384b4fbe6ad536 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -36,6 +36,9 @@ #include #include +#ifdef GKFS_ENABLE_CLIENT_METRICS +#include +#endif #include #include @@ -50,13 +53,13 @@ std::unique_ptr ld_network_service; // extern variable namespace { -#ifdef GKFS_ENABLE_FORWARDING +// FORWARDING pthread_t mapper; bool forwarding_running; pthread_mutex_t remap_mutex; pthread_cond_t remap_signal; -#endif +// END FORWARDING inline void exit_error_msg(int errcode, const string& msg) { @@ -67,8 +70,10 @@ exit_error_msg(int errcode, const string& msg) { // if we don't disable interception before calling ::exit() // syscall hooks may find an inconsistent in shared state // (e.g. the logger) and thus, crash - gkfs::preload::stop_interception(); - CTX->disable_interception(); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + } ::exit(errcode); } @@ -102,7 +107,6 @@ init_hermes_client() { return true; } -#ifdef GKFS_ENABLE_FORWARDING void* forwarding_mapper(void* p) { struct timespec timeout; @@ -115,7 +119,7 @@ forwarding_mapper(void* p) { try { gkfs::utils::load_forwarding_map(); - if(previous != CTX->fwd_host_id()) { + if(previous != (int64_t) CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); previous = CTX->fwd_host_id(); @@ -133,18 +137,14 @@ forwarding_mapper(void* p) { return nullptr; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } -#endif -#ifdef GKFS_ENABLE_FORWARDING void destroy_forwarding_mapper() { forwarding_running = false; @@ -153,7 +153,6 @@ destroy_forwarding_mapper() { pthread_join(mapper, NULL); } -#endif void log_prog_name() { @@ -207,31 +206,34 @@ init_environment() { } /* Setup distributor */ -#ifdef GKFS_ENABLE_FORWARDING - try { - gkfs::utils::load_forwarding_map(); + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch(std::exception& e) { - exit_error_msg( - EXIT_FAILURE, - fmt::format("Unable set the forwarding host '{}'", e.what())); - } + if(!forwarding_map_file.empty()) { + try { + gkfs::utils::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch(std::exception& e) { + exit_error_msg(EXIT_FAILURE, + fmt::format("Unable set the forwarding host '{}'", + e.what())); + } + + auto forwarder_dist = std::make_shared( + CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + } else { - auto forwarder_dist = std::make_shared( - CTX->fwd_host_id(), CTX->hosts().size()); - CTX->distributor(forwarder_dist); -#else #ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #else - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#endif - CTX->distributor(distributor); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #endif - + CTX->distributor(distributor); + } LOG(INFO, "Retrieving file system configuration..."); @@ -290,12 +292,20 @@ init_preload() { CTX->unprotect_user_fds(); -#ifdef GKFS_ENABLE_FORWARDING - init_forwarding_mapper(); -#endif + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(!forwarding_map_file.empty()) { + init_forwarding_mapper(); + } gkfs::preload::start_interception(); errno = oerrno; +#ifdef GKFS_ENABLE_CLIENT_METRICS + if(!CTX->init_metrics()) { + exit_error_msg(EXIT_FAILURE, + "Unable to initialize client metrics. Exiting..."); + } +#endif } /** @@ -304,19 +314,60 @@ init_preload() { */ void destroy_preload() { -#ifdef GKFS_ENABLE_FORWARDING - destroy_forwarding_mapper(); + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(!forwarding_map_file.empty()) { + destroy_forwarding_mapper(); + } +#ifdef GKFS_ENABLE_CLIENT_METRICS + LOG(INFO, "Flushing final metrics..."); + CTX->write_metrics()->flush_msgpack(); + CTX->read_metrics()->flush_msgpack(); + LOG(INFO, "Metrics flushed. Total flush operations: {}", + CTX->write_metrics()->flush_count()); #endif - CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); ld_network_service.reset(); LOG(DEBUG, "RPC subsystem shut down"); - gkfs::preload::stop_interception(); - CTX->disable_interception(); - LOG(DEBUG, "Syscall interception stopped"); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + LOG(DEBUG, "Syscall interception stopped"); + } LOG(INFO, "All subsystems shut down. Client shutdown complete."); } + + +/** + * @brief External functions to call linking the library + * + */ +extern "C" int +gkfs_init() { + CTX->init_logging(); + + // from here ownwards it is safe to print messages + LOG(DEBUG, "Logging subsystem initialized"); + + gkfs::preload::init_environment(); + + return 0; +} + + +extern "C" int +gkfs_end() { + CTX->clear_hosts(); + LOG(DEBUG, "Peer information deleted"); + + ld_network_service.reset(); + LOG(DEBUG, "RPC subsystem shut down"); + + LOG(INFO, "All subsystems shut down. Client shutdown complete."); + + return 0; +} diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 7101041dcdb77864a55ed671bdd014a250db5623..bbf9fab0b0224cbc5d7f927027940fc72ab0adf0 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -36,14 +36,23 @@ #include #include -#include +#ifdef GKFS_ENABLE_CLIENT_METRICS +#include +#endif +#include #include #include +#include -extern "C" { +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + +extern "C" { #include } @@ -69,6 +78,10 @@ PreloadContext::PreloadContext() std::stoi(gkfs::env::get_var(gkfs::env::NUM_REPL, "0"))); } +// Destructor set here to allow unique_ptr of forward declared classes in the +// header. T must be complete at the point of deletion. +PreloadContext::~PreloadContext() = default; + void PreloadContext::init_logging() { @@ -107,6 +120,74 @@ PreloadContext::init_logging() { ); } +bool +PreloadContext::init_metrics() { +#ifdef GKFS_ENABLE_CLIENT_METRICS + auto flush_interval = std::stoi(gkfs::env::get_var( + gkfs::env::METRICS_FLUSH_INTERVAL, + std::to_string(gkfs::config::client_metrics::flush_interval))); + if(gkfs::env::var_is_set(gkfs::env::METRICS_IP_PORT)) { + write_metrics_ = std::make_unique( + gkfs::messagepack::client_metric_io_type::write, + gkfs::messagepack::client_metric_flush_type::socket, + flush_interval); + read_metrics_ = std::make_unique( + gkfs::messagepack::client_metric_io_type::read, + gkfs::messagepack::client_metric_flush_type::socket, + flush_interval); + if(gkfs::env::var_is_set(gkfs::env::ENABLE_METRICS)) { + LOG(INFO, + "Client metrics enabled with ZeroMQ flushing. Initializing..."); + write_metrics_->enable(); + read_metrics_->enable(); + auto metrics_ip = gkfs::env::get_var(gkfs::env::METRICS_IP_PORT); + write_metrics_->zmq_connect(metrics_ip); + if(!write_metrics_->zmq_is_connected()) { + LOG(ERROR, "Client write metrics failed to connect to : {}", + metrics_ip); + return false; + } + LOG(INFO, "Client write metrics connected to : {}", metrics_ip); + read_metrics_->zmq_connect(metrics_ip); + if(!read_metrics_->zmq_is_connected()) { + LOG(ERROR, "Client read metrics failed to connect to : {}", + metrics_ip); + return false; + } + LOG(INFO, "Client read metrics connected to : {}", metrics_ip); + } + } else { + write_metrics_ = std::make_unique( + gkfs::messagepack::client_metric_io_type::write, + gkfs::messagepack::client_metric_flush_type::file, + flush_interval); + read_metrics_ = std::make_unique( + gkfs::messagepack::client_metric_io_type::read, + gkfs::messagepack::client_metric_flush_type::file, + flush_interval); + if(gkfs::env::var_is_set(gkfs::env::ENABLE_METRICS)) { + LOG(INFO, + "Client metrics enabled with file flushing. Initializing..."); + write_metrics_->enable(); + read_metrics_->enable(); + if(!gkfs::env::var_is_set(gkfs::env::METRICS_PATH)) { + LOG(WARNING, "No metrics path set. Using default path at {}", + gkfs::config::client_metrics::flush_path); + } + auto metrics_path = gkfs::env::get_var( + gkfs::env::METRICS_PATH, + gkfs::config::client_metrics::flush_path); + std::filesystem::create_directories(metrics_path); + write_metrics_->path(metrics_path, "write"); + LOG(INFO, "Client write metrics path: {}", write_metrics_->path()); + read_metrics_->path(metrics_path, "read"); + LOG(INFO, "Client read metrics path: {}", read_metrics_->path()); + } + } +#endif + return true; +} + void PreloadContext::mountdir(const std::string& path) { assert(gkfs::path::is_absolute(path)); @@ -448,7 +529,6 @@ PreloadContext::unprotect_user_fds() { internal_fds_must_relocate_ = true; } - std::string PreloadContext::get_hostname() { return hostname; @@ -464,5 +544,15 @@ PreloadContext::get_replicas() { return replicas_; } +const std::shared_ptr +PreloadContext::write_metrics() { + return write_metrics_; +} + +const std::shared_ptr +PreloadContext::read_metrics() { + return read_metrics_; +} + } // namespace preload } // namespace gkfs diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index aa09d0e0727d69e509aea64a1133063bc8db3e71..b70ddf2d9acaba9081176c30caea1ea75f0f3422 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -104,16 +104,15 @@ extract_protocol(const string& uri) { throw runtime_error(fmt::format("Invalid format for URI: '{}'", uri)); } string protocol{}; - - if(uri.find(gkfs::rpc::protocol::ofi_sockets) != string::npos) { - protocol = gkfs::rpc::protocol::ofi_sockets; - } else if(uri.find(gkfs::rpc::protocol::ofi_psm2) != string::npos) { - protocol = gkfs::rpc::protocol::ofi_psm2; - } else if(uri.find(gkfs::rpc::protocol::ofi_verbs) != string::npos) { - protocol = gkfs::rpc::protocol::ofi_verbs; + for(const auto& valid_protocol : + gkfs::rpc::protocol::all_remote_protocols) { + if(uri.find(valid_protocol) != string::npos) { + protocol = valid_protocol; + break; + } } - // check for shared memory protocol. Can be plain shared memory or real ofi - // protocol + auto_sm + // check for shared memory protocol. Can be plain shared memory or real + // ofi protocol + auto_sm if(uri.find(gkfs::rpc::protocol::na_sm) != string::npos) { if(protocol.empty()) protocol = gkfs::rpc::protocol::na_sm; @@ -286,7 +285,6 @@ metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md, return 0; } -#ifdef GKFS_ENABLE_FORWARDING map load_forwarding_map_file(const std::string& lfpath) { @@ -320,9 +318,7 @@ load_forwarding_map_file(const std::string& lfpath) { } return forwarding_map; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void load_forwarding_map() { string forwarding_map_file; @@ -359,7 +355,6 @@ load_forwarding_map() { CTX->fwd_host_id(forwarding_map[local_hostname]); } -#endif vector> read_hosts_file() { diff --git a/src/client/rpc/forward_management.cpp b/src/client/rpc/forward_management.cpp index c975842e560d15a9f107b3138ba1016b853071ed..7504b1f6b6bb869d99faab62c946a620ecace192 100644 --- a/src/client/rpc/forward_management.cpp +++ b/src/client/rpc/forward_management.cpp @@ -47,7 +47,7 @@ forward_get_fs_config() { bool found = false; size_t idx = 0; - while(!found && idx <= CTX->hosts().size()) { + while(!found && idx < CTX->hosts().size()) { try { LOG(DEBUG, "Retrieving file system configurations from daemon"); // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that diff --git a/src/client/syscalls/util.S b/src/client/syscalls/util.S new file mode 100644 index 0000000000000000000000000000000000000000..67c9b279afdcad35a0409dd800c392f079029956 --- /dev/null +++ b/src/client/syscalls/util.S @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +.global syscall_no_intercept; +.type syscall_no_intercept, @function + +.text + +syscall_no_intercept: + movq %rdi, %rax /* convert from linux ABI calling */ + movq %rsi, %rdi /* convention to syscall calling convention */ + movq %rdx, %rsi + movq %rcx, %rdx + movq %r8, %r10 + movq %r9, %r8 + movq 8(%rsp), %r9 + syscall + ret + +.size syscall_no_intercept, .-syscall_no_intercept diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 73b7348dea21e749b40a02e431bb85b1bb0bd297..5254f93f532e19c09b48782e5c8a0b3182fa5cd0 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -104,14 +104,29 @@ target_link_libraries(metadata fmt::fmt ) -add_library(path_util - STATIC - ) +add_library(path_util STATIC) set_property(TARGET path_util PROPERTY POSITION_INDEPENDENT_CODE ON) - target_sources(path_util PRIVATE path_util.cpp PUBLIC ${INCLUDE_DIR}/common/path_util.hpp ) + +if (GKFS_ENABLE_CLIENT_METRICS) + add_library(msgpack_util STATIC) + set_property(TARGET msgpack_util PROPERTY POSITION_INDEPENDENT_CODE ON) + set_target_properties(msgpack_util PROPERTIES LINKER_LANGUAGE CXX) + target_sources(msgpack_util + PUBLIC + ${INCLUDE_DIR}/common/msgpack_util.hpp + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/msgpack_util.cpp + ) + target_link_libraries(msgpack_util + PUBLIC + Msgpack::Msgpack + cppzmq + rpc_utils + ) +endif () diff --git a/src/common/env_util.cpp b/src/common/env_util.cpp index 352217f292e0846caf81902ad7606566ca7671bd..63f5c6cda7a65007359b9896e2617a30e6f73874 100644 --- a/src/common/env_util.cpp +++ b/src/common/env_util.cpp @@ -32,8 +32,18 @@ using namespace std; +namespace gkfs::env { + string -gkfs::env::get_var(const string& name, const string& default_value) { +get_var(const string& name, const string& default_value) { const char* const val = ::secure_getenv(name.c_str()); return val != nullptr ? string(val) : default_value; } + +bool +var_is_set(const string& name) { + const char* const val = ::secure_getenv(name.c_str()); + return val != nullptr; +} + +} // namespace gkfs::env diff --git a/src/common/msgpack_util.cpp b/src/common/msgpack_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..18bd3fa0d6a66944eed7158d14a6595715b9b149 --- /dev/null +++ b/src/common/msgpack_util.cpp @@ -0,0 +1,208 @@ +/* + Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +extern "C" { +#include +#include +} + +using namespace std; + +namespace gkfs::messagepack { + +ClientMetrics::ClientMetrics(client_metric_io_type io_type, + client_metric_flush_type ftype, int flush_interval) + : flush_interval_(flush_interval) { + init_t_ = std::chrono::system_clock::now(); + msgpack_data_.hostname_ = gkfs::rpc::get_my_hostname(true); + msgpack_data_.pid_ = getpid(); + if(io_type == client_metric_io_type::write) { + msgpack_data_.io_type_ = "w"; + } else { + msgpack_data_.io_type_ = "r"; + } + if(ftype == client_metric_flush_type::file) { + // set default path + flush_type_ = client_metric_flush_type::file; + flush_path_ = gkfs::config::client_metrics::flush_path; + } else { + flush_type_ = client_metric_flush_type::socket; + zmq_flush_context_ = std::make_unique(1); + zmq_flush_socket_ = + std::make_unique(*zmq_flush_context_, ZMQ_PUSH); + } + flush_thread_running_ = true; + flush_thread_ = std::thread(&ClientMetrics::flush_loop, this); +} + +ClientMetrics::~ClientMetrics() { + flush_thread_running_ = false; + flush_thread_cv_.notify_one(); + if(flush_thread_.joinable()) + flush_thread_.join(); + if(flush_type_ == client_metric_flush_type::socket) { + zmq_flush_socket_->close(); + zmq_flush_context_->close(); + } +} + +void +ClientMetrics::add_event( + size_t size, std::chrono::time_point start) { + if(!metrics_enabled_) + return; + auto end = std::chrono::system_clock::now(); + std::lock_guard const data_lock(data_mtx_); + auto start_offset = + std::chrono::duration(start - init_t_); + auto end_offset = std::chrono::duration(end - init_t_); + auto duration = std::chrono::duration(end_offset - + start_offset); + msgpack_data_.total_bytes_ += size; + // auto size_mib = size / (1024 * 1024); // in MiB + // auto duration_s = duration.count() / 1000; // in seconds + // throw away decimals + msgpack_data_.start_t_.emplace_back( + static_cast(start_offset.count())); + msgpack_data_.end_t_.emplace_back(static_cast(end_offset.count())); + msgpack_data_.req_size_.emplace_back(size); + msgpack_data_.total_iops_ += 1; +} + +void +ClientMetrics::reset_metrics() { + msgpack_data_.start_t_.clear(); + msgpack_data_.end_t_.clear(); + msgpack_data_.flush_t_ = 0; + msgpack_data_.req_size_.clear(); + msgpack_data_.total_bytes_ = 0; + msgpack_data_.total_iops_ = 0; +} + +void +ClientMetrics::flush_msgpack() { + if(!metrics_enabled_) + return; + std::lock_guard const data_lock(data_mtx_); + if(msgpack_data_.total_iops_ == 0) + return; + auto flush_t_now = std::chrono::system_clock::now(); + msgpack_data_.flush_t_ = static_cast( + std::chrono::duration(flush_t_now - init_t_) + .count()); + auto data = msgpack_data_.pack_msgpack(); + if(flush_type_ == client_metric_flush_type::file) { + auto fd = + open(flush_path_.c_str(), O_CREAT | O_WRONLY | O_APPEND, 0666); + if(fd < 0) { + // cout << "error open" << endl; + exit(1); + } + write(fd, data.data(), data.size()); + // auto written = write(fd, data.data(), data.size()); + // cout << "written: " << written << endl; + close(fd); + } else { + zmq::message_t message(data.size()); + // copy data from serialized msgpack to zmq message + memcpy(message.data(), data.data(), data.size()); + // non-blocking zmq send + if(zmq_flush_socket_->send(message, zmq::send_flags::none) == -1) { + std::cerr << "Failed to send zmq message" << std::endl; + } + } + reset_metrics(); + flush_count_++; +} + +void +ClientMetrics::flush_loop() { + while(flush_thread_running_) { + std::unique_lock wait_lock( + flush_thread_cv_mutex_); // Lock only for wait_for + flush_thread_cv_.wait_for(wait_lock, + std::chrono::seconds(flush_interval_), + [&] { return !flush_thread_running_; }); + flush_msgpack(); + } +} + +void +ClientMetrics::enable() { + metrics_enabled_ = true; +} + +void +ClientMetrics::disable() { + metrics_enabled_ = false; +} + +void +ClientMetrics::zmq_connect(const string& ip_port) { + auto address = "tcp://" + ip_port; + zmq_flush_socket_->connect(address); +} + +bool +ClientMetrics::zmq_is_connected() { + return zmq_flush_socket_->handle() != nullptr; +} + +const string& +ClientMetrics::path() const { + return flush_path_; +} +void +ClientMetrics::path(const string& path, const string prefix) { + const std::time_t t = std::chrono::system_clock::to_time_t(init_t_); + std::stringstream init_t_stream; + init_t_stream << std::put_time(std::localtime(&t), "%F_%T"); + flush_path_ = path + "/" + prefix + "_" + init_t_stream.str() + "_" + + msgpack_data_.hostname_ + "_" + + to_string(msgpack_data_.pid_) + ".msgpack"; +} +int +ClientMetrics::flush_count() const { + return flush_count_; +} + + +} // namespace gkfs::messagepack diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 95c4ade7542ead463ea3565842481a82738fa102..89af71c7624109df1d90b7c4bae1b29fd030b193 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -72,63 +72,15 @@ target_link_libraries( Threads::Threads ) +if(GKFS_ENABLE_AGIOS) + target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp) + target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS) + target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS) +endif() + if(GKFS_ENABLE_CODE_COVERAGE) target_code_coverage(gkfs_daemon AUTO) endif() install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -# ############################################################################## -# This builds the `gkfwd_daemon` executable: the daemon for GekkoFS data -# forwarding mode. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - - add_executable(gkfwd_daemon) - - # We need to add here any files that may have different compile definitions - target_sources( - gkfwd_daemon - PRIVATE ../common/rpc/rpc_util.cpp - daemon.cpp - util.cpp - ops/metadentry.cpp - ops/data.cpp - classes/fs_data.cpp - classes/rpc_data.cpp - handler/srv_metadata.cpp - handler/srv_management.cpp - handler/srv_data.cpp - ) - - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_FORWARDING) - target_link_libraries( - gkfwd_daemon - PUBLIC # internal libs - metadata - metadata_backend - storage - distributor - statistics - log_util - env_util - path_util - # external libs - CLI11::CLI11 - fmt::fmt - Mercury::Mercury - Argobots::Argobots - Margo::Margo - # others - Threads::Threads - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PRIVATE scheduler/agios.cpp) - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_AGIOS) - target_link_libraries(gkfwd_daemon PRIVATE AGIOS::AGIOS) - endif() - - install(TARGETS gkfwd_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif () diff --git a/src/daemon/backend/metadata/merge.cpp b/src/daemon/backend/metadata/merge.cpp index 1c1a2a6fc03e283112de8df70e23c99c62beca9a..84fd5c66f79a5c6d8e5681d5c10645989367a07b 100644 --- a/src/daemon/backend/metadata/merge.cpp +++ b/src/daemon/backend/metadata/merge.cpp @@ -137,6 +137,26 @@ CreateOperand::serialize_params() const { return metadata; } +UpdateTimeOperand::UpdateTimeOperand(time_t mtime) : mtime_(mtime) {} + +UpdateTimeOperand::UpdateTimeOperand(const rdb::Slice& serialized_op) { + // Parse size + size_t read = 0; + mtime_ = ::stoul(serialized_op.ToString(), &read); + // check that we consumed all the input string + assert(read == serialized_op.size()); +} + +OperandID +UpdateTimeOperand::id() const { + return OperandID::update_time; +} + +string +UpdateTimeOperand::serialize_params() const { + return ::to_string(mtime_); +} + /** * @internal * Merges all operands in chronological order for the same key. @@ -181,10 +201,6 @@ MetadataMergeOperator::FullMergeV2(const MergeOperationInput& merge_in, auto operand_id = MergeOperand::get_id(serialized_op); auto parameters = MergeOperand::get_params(serialized_op); - if constexpr(gkfs::config::metadata::use_mtime) { - md.update_mtime_now(); - } - if(operand_id == OperandID::increase_size) { auto op = IncreaseSizeOperand(parameters); if(op.append()) { @@ -204,6 +220,9 @@ MetadataMergeOperator::FullMergeV2(const MergeOperationInput& merge_in, fsize = op.size(); } else if(operand_id == OperandID::create) { continue; + } else if(operand_id == OperandID::update_time) { + auto op = UpdateTimeOperand(parameters); + md.mtime(op.mtime()); } else { throw ::runtime_error("Unrecognized merge operand ID: " + (char) operand_id); diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp index 3e8e0d687d7c7a6f610904526cfec9624f978b6e..508e7bd82df15a91806a44f68a7fb1e63a63b778 100644 --- a/src/daemon/backend/metadata/rocksdb_backend.cpp +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -34,7 +34,7 @@ #include #include #include -#include +#include extern "C" { #include } @@ -244,6 +244,15 @@ RocksDBBackend::increase_size_impl(const std::string& key, size_t io_size, if(!s.ok()) { throw_status_excpt(s); } + if constexpr(gkfs::config::metadata::use_mtime) { + // get current time and update mtime for this file + time_t now = time(nullptr); + auto t_op = UpdateTimeOperand(now); + s = db_->Merge(write_opts_, key, t_op.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } + } } return out_offset; } @@ -263,6 +272,15 @@ RocksDBBackend::decrease_size_impl(const std::string& key, size_t size) { if(!s.ok()) { throw_status_excpt(s); } + if constexpr(gkfs::config::metadata::use_mtime) { + // get current time and update mtime for this file + time_t now = time(nullptr); + auto t_op = UpdateTimeOperand(now); + s = db_->Merge(write_opts_, key, t_op.serialize()); + if(!s.ok()) { + throw_status_excpt(s); + } + } } /** diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 0d38c718c5af2b76b9c8be8414088a4239ee1c52..33b43bf5942789507cc951f737ac1da2c7188a9e 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -256,6 +256,15 @@ FsData::enable_chunkstats(bool enable_chunkstats) { FsData::enable_chunkstats_ = enable_chunkstats; } +bool +FsData::enable_forwarding() const { + return enable_forwarding_; +} + +void +FsData::enable_forwarding(bool enable_forwarding) { + FsData::enable_forwarding_ = enable_forwarding; +} bool FsData::enable_prometheus() const { return enable_prometheus_; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49a87fa604e8b6d747c6d3e9dd19b66f9ee27211..6bc229cd9b237fabe147a9221cd3bbe96c3c841f 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -261,25 +261,6 @@ init_environment() { throw; } - GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ", __func__); - try { -#ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared(); -#else - auto distributor = std::make_shared(); -#endif - RPC_DATA->distributor(distributor); - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to initialize Distributor: {}", __func__, - e.what()); - throw; - } - -#ifdef GKFS_ENABLE_FORWARDING - GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__); -#endif - #ifdef GKFS_ENABLE_AGIOS // Initialize AGIOS scheduler GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'", @@ -392,9 +373,10 @@ agios_initialize() { */ void destroy_enviroment() { - GKFS_DATA->spdlogger()->debug("{}() Removing mount directory", __func__); + // GKFS_DATA->spdlogger()->debug("{}() Removing mount directory", + // __func__); std::error_code ecode; - fs::remove_all(GKFS_DATA->mountdir(), ecode); + // fs::remove_all(GKFS_DATA->mountdir(), ecode); GKFS_DATA->spdlogger()->debug("{}() Freeing I/O executions streams", __func__); for(unsigned int i = 0; i < RPC_DATA->io_streams().size(); i++) { @@ -496,13 +478,18 @@ parse_input(const cli_options& opts, const CLI::App& desc) { auto rpc_protocol = string(gkfs::rpc::protocol::ofi_sockets); if(desc.count("--rpc-protocol")) { rpc_protocol = opts.rpc_protocol; - if(rpc_protocol != gkfs::rpc::protocol::ofi_verbs && - rpc_protocol != gkfs::rpc::protocol::ofi_sockets && - rpc_protocol != gkfs::rpc::protocol::ofi_psm2) { + auto protocol_found = false; + for(const auto& valid_protocol : + gkfs::rpc::protocol::all_remote_protocols) { + if(rpc_protocol == valid_protocol) { + protocol_found = true; + break; + } + } + if(!protocol_found) throw runtime_error(fmt::format( "Given RPC protocol '{}' not supported. Check --help for supported protocols.", rpc_protocol)); - } } auto use_auto_sm = desc.count("--auto-sm") != 0; @@ -552,10 +539,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { assert(desc.count("--rootdir")); auto rootdir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - // In forwarding mode, the backend is shared - auto rootdir_path = fs::path(rootdir); -#else auto rootdir_path = fs::path(rootdir); if(desc.count("--rootdir-suffix")) { if(opts.rootdir_suffix == gkfs::config::data::chunk_dir || @@ -570,7 +553,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { rootdir_path /= opts.rootdir_suffix; GKFS_DATA->rootdir_suffix(opts.rootdir_suffix); } -#endif if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) @@ -589,14 +571,23 @@ parse_input(const cli_options& opts, const CLI::App& desc) { fs::create_directories(rootdir_path); GKFS_DATA->rootdir(rootdir_path.native()); + if(desc.count("--enable-forwarding")) { + GKFS_DATA->enable_forwarding(true); + GKFS_DATA->spdlogger()->info("{}() Forwarding mode enabled", __func__); + } + if(desc.count("--metadir")) { auto metadir = opts.metadir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); -#else + auto metadir_path = fs::path(metadir); -#endif + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); + } + + if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...", @@ -613,13 +604,16 @@ parse_input(const cli_options& opts, const CLI::App& desc) { // use rootdir as metadata dir auto metadir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); - fs::create_directories(metadir_path); - GKFS_DATA->metadir(fs::canonical(metadir_path).native()); -#else - GKFS_DATA->metadir(GKFS_DATA->rootdir()); -#endif + + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + auto metadir_path = + fs::path(metadir) / fmt::format_int(getpid()).str(); + fs::create_directories(metadir_path); + GKFS_DATA->metadir(fs::canonical(metadir_path).native()); + } else + GKFS_DATA->metadir(GKFS_DATA->rootdir()); } if(desc.count("--dbbackend")) { @@ -787,6 +781,9 @@ main(int argc, const char* argv[]) { desc.add_option( "--output-stats", opts.stats_file, "Creates a thread that outputs the server stats each 10s to the specified file."); + desc.add_flag( + "--enable-forwarding", + "Enables forwarding mode, so the metadata is stored in a separate directory (pid)."); #ifdef GKFS_ENABLE_PROMETHEUS desc.add_flag( "--enable-prometheus", diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 9a4e1697355d95c535e74ad44a73ad924de39c9d..b59f5c551071222630fe11c439769d2a0a910d02 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -109,7 +109,7 @@ rpc_srv_write(hg_handle_t handle) { return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug( "{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", @@ -244,8 +244,6 @@ rpc_srv_write(hg_handle_t handle) { GKFS_DATA->stats()->add_write(in.path, chnk_id_file); } - GKFS_DATA->spdlogger()->error("{}() Processing at host {} -> {}", - __func__, host_id, chnk_id_file); chnk_ids_host[chnk_id_curr] = chnk_id_file; // save this id to host chunk list // offset case. Only relevant in the first iteration of the loop and if @@ -414,7 +412,7 @@ rpc_srv_read(hg_handle_t handle) { return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug( diff --git a/src/daemon/handler/srv_management.cpp b/src/daemon/handler/srv_management.cpp index 585646aeddee5c3094a5573b3fd3cd51877c1af7..ef5f8735d0fdcccb4eebda2690ce6756eca6b8d6 100644 --- a/src/daemon/handler/srv_management.cpp +++ b/src/daemon/handler/srv_management.cpp @@ -72,8 +72,9 @@ rpc_srv_get_fs_config(hg_handle_t handle) { out.blocks_state = static_cast(GKFS_DATA->blocks_state()); out.uid = getuid(); out.gid = getgid(); - GKFS_DATA->spdlogger()->debug("{}() Sending output configs back to library", - __func__); + GKFS_DATA->spdlogger()->debug( + "{}() Sending output configs back to library. mountdir '{}' rootdir '{}'", + __func__, out.mountdir, out.rootdir); auto hret = margo_respond(handle, &out); if(hret != HG_SUCCESS) { GKFS_DATA->spdlogger()->error( diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp index e68069797fc8104894399f10a38ee25d323234b8..1339170b94aa61072451c759f1f1b8b1ba849afd 100644 --- a/src/daemon/handler/srv_metadata.cpp +++ b/src/daemon/handler/srv_metadata.cpp @@ -537,7 +537,7 @@ rpc_srv_get_dirents(hg_handle_t handle) { // Retrieve size of source buffer auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ", __func__, in.path, bulk_size); @@ -696,7 +696,7 @@ rpc_srv_get_dirents_extended(hg_handle_t handle) { // Retrieve size of source buffer auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ", __func__, in.path, bulk_size); diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index bc8e0c86427bf14538ce3388b95755f78cabceac..4bc63e41d3590bae7eb7c88070bf9c7362c8b81f 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -165,7 +165,7 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): lines = f.readlines() for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] assert ion == '0' @@ -190,15 +190,16 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): ret = c00.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of read bytes - + + # log line should go to the end of the file with open(c00.log) as f: lines = f.readlines() - + ion = 0 for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] - assert ion == '1' + assert ion == '1' def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): """Write files from one client and read in the other using two daemons""" diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 9c3e97aad140f5914f37ffcd90ce6b06024f0139..03d73b6bf96bed9c2439fecae4568350eec7b912 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -70,30 +70,7 @@ add_executable(gkfs.io gkfs.io/rename.cpp ) -include(FetchContent) - -set(FETCHCONTENT_QUIET OFF) - -FetchContent_Declare(nlohmann_json - DOWNLOAD_EXTRACT_TIMESTAMP ON - URL https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz) - -FetchContent_GetProperties(nlohmann_json) - -if(NOT nlohmann_json_POPULATED) - FetchContent_Populate(nlohmann_json) - message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}") - message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}") - - # we don't really care so much about a third party library's tests to be - # run from our own project's code - set(JSON_BuildTests OFF CACHE INTERNAL "") - - # we also don't need to install it when our main project gets installed - set(JSON_Install OFF CACHE INTERNAL "") - - add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR}) -endif() +include(load_nlohmann_json) target_include_directories(gkfs.io PRIVATE ${BOOST_PREPROCESSOR_INCLUDE_DIRS} diff --git a/tests/integration/harness/gkfs.io/serialize.hpp b/tests/integration/harness/gkfs.io/serialize.hpp index 3873b2ec85d484a52f748820f150738ea9a938ec..92328fcf86dc53b3d71d3acfa63a4ec5de4ca72a 100644 --- a/tests/integration/harness/gkfs.io/serialize.hpp +++ b/tests/integration/harness/gkfs.io/serialize.hpp @@ -178,13 +178,10 @@ template <> struct adl_serializer { static void to_json(json& j, const struct ::statfs opt) { - j = json{{"f_type", opt.f_type}, - {"f_bsize", opt.f_bsize}, - {"f_blocks", opt.f_blocks}, - {"f_bfree", opt.f_bfree}, - {"f_bavail", opt.f_bavail}, - {"f_files", opt.f_files}, - {"f_ffree", opt.f_ffree}}; + j = json{{"f_type", opt.f_type}, {"f_bsize", opt.f_bsize}, + {"f_blocks", opt.f_blocks}, {"f_bfree", opt.f_bfree}, + {"f_bavail", opt.f_bavail}, {"f_files", opt.f_files}, + {"f_ffree", opt.f_ffree}}; } }; @@ -253,7 +250,8 @@ struct formatter { template auto - format(const struct ::dirent& dirent, FormatContext& ctx) { + format(const struct ::dirent& dirent, FormatContext& ctx) const + -> format_context::iterator { return format_to(ctx.out(), "struct dirent {{\n" " d_ino = {};\n" diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index ef48c404255306f5301ea029c4c189feb7e10c20..a65a171f7b7ad50f5129f2d27ebe196e7d729088 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -48,15 +48,16 @@ gkfs_client_log_level = 'all' gkfs_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' -gkfwd_daemon_cmd = 'gkfwd_daemon' +gkfwd_daemon_cmd = 'gkfs_daemon' gkfwd_client_cmd = 'gkfs.io' -gkfwd_client_lib_file = 'libgkfwd_intercept.so' +gkfwd_client_lib_file = 'libgkfs_intercept.so' gkfwd_hosts_file = 'gkfs_hosts.txt' gkfwd_forwarding_map_file = 'gkfs_forwarding.map' -gkfwd_daemon_log_file = 'gkfwd_daemon.log' +gkfwd_daemon_log_file = 'gkfs_daemon.log' gkfwd_daemon_log_level = '100' -gkfwd_client_log_file = 'gkfwd_client.log' +gkfwd_client_log_file = 'gkfs_client.log' gkfwd_client_log_level = 'all' +gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' @@ -520,7 +521,8 @@ class ShellClient: 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -732,7 +734,8 @@ class FwdDaemon: args = [ '--mountdir', self.mountdir, '--metadir', self.metadir, '--rootdir', self.rootdir, - '-l', self._address ] + '-l', self._address, + '--enable-forwarding'] logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -902,11 +905,12 @@ class FwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'LD_PRELOAD' : self._preload_library, + 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -996,7 +1000,8 @@ class ShellFwdClient: 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file), 'LIBGKFS_LOG' : gkfwd_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) diff --git a/tests/scripts/compile_dep.sh/0.9.3/agios.out b/tests/scripts/compile_dep.sh/0.9.3/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..25a1cad3a58a9c7c07c75ba87bfa2c5a6a9643ca --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/agios.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/all.out b/tests/scripts/compile_dep.sh/0.9.3/all.out new file mode 100644 index 0000000000000000000000000000000000000000..2fcd55b6f7dc5e574eeea0233a628e75ea105f4e --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/all.out @@ -0,0 +1,14 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### +Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.3/arm.out b/tests/scripts/compile_dep.sh/0.9.3/arm.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/arm.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ci.out b/tests/scripts/compile_dep.sh/0.9.3/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..9ec3c24bb12c16078d2b3869831331de15f5e305 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ci.out @@ -0,0 +1,8 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/default.out b/tests/scripts/compile_dep.sh/0.9.3/default.out new file mode 100644 index 0000000000000000000000000000000000000000..ca393e65d3bc03aa72fe6a628efab5dad85174b9 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/default.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/direct.out b/tests/scripts/compile_dep.sh/0.9.3/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..816266773b569d3d098245c7b868f7983753ff12 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/direct.out @@ -0,0 +1,6 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..ff149d28f1874e6fe4f94484c4f0b4f7abf49416 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,9 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric%verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/mogon2.out b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ngio.out b/tests/scripts/compile_dep.sh/0.9.3/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ngio.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/p9.out b/tests/scripts/compile_dep.sh/0.9.3/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/p9.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out index 7ba8e6778a49f2f7ebaf516c6143f2dd095adef8..8369fe83959bf552ffa03f984ee038b6c55baaed 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/default.out +++ b/tests/scripts/dl_dep.sh/0.8.0/default.out @@ -12,5 +12,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocks Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out index d8af0a79b2bed070d64eb40b2751a06fd5040207..d5844433bc81445c690efc0e30054a331000b4ce 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out @@ -10,4 +10,4 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ngio.out b/tests/scripts/dl_dep.sh/0.8.0/ngio.out index f0c9f79592b10c1307d92dbafc560a5914c5974d..e6cd7bdcc2d1c625d552b6c85b256500df133683 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.8.0/ngio.out @@ -8,6 +8,6 @@ Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/ngio.out b/tests/scripts/dl_dep.sh/0.9.0/ngio.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.0/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/all.out b/tests/scripts/dl_dep.sh/0.9.1/all.out index 35b6b521748a2bb789559c4df780c438d345b7b8..9cbff6342eb7dba75c4f9e617fddacfd446b13ff 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/all.out +++ b/tests/scripts/dl_dep.sh/0.9.1/all.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/ngio.out b/tests/scripts/dl_dep.sh/0.9.1/ngio.out index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.1/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/p9.out b/tests/scripts/dl_dep.sh/0.9.1/p9.out index 3a99bb523089fa9d7aaa1a36a4a62fa9d6295732..b1eee01ba618b62af568de183d0d094947586123 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/p9.out +++ b/tests/scripts/dl_dep.sh/0.9.1/p9.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/all.out b/tests/scripts/dl_dep.sh/0.9.2/all.out index 4a1ab9e6f6de6a14f9e7da7c7792bd01e6171c48..4b5a73fda0b5c2962d25cc0b7d5c786cdf5adb5b 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/all.out +++ b/tests/scripts/dl_dep.sh/0.9.2/all.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/arm.out b/tests/scripts/dl_dep.sh/0.9.2/arm.out index 2c61edb9c437445a2347de92fdd11a66ac304576..8725e5892b82e32c3e2e8b15d7f805f72c09cc88 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/arm.out +++ b/tests/scripts/dl_dep.sh/0.9.2/arm.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/ngio.out b/tests/scripts/dl_dep.sh/0.9.2/ngio.out index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.2/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out index d26b498ca41977f2e48d86f1f0437c1bc0b7a563..96e22237b1da0070350c22e2809d62a69dcd24d3 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/p9.out b/tests/scripts/dl_dep.sh/0.9.2/p9.out index 3a99bb523089fa9d7aaa1a36a4a62fa9d6295732..b1eee01ba618b62af568de183d0d094947586123 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/p9.out +++ b/tests/scripts/dl_dep.sh/0.9.2/p9.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.3/agios.out b/tests/scripts/dl_dep.sh/0.9.3/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..8d8e2486b502394977608f89057d0479487e736d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/agios.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/all.out b/tests/scripts/dl_dep.sh/0.9.3/all.out new file mode 100644 index 0000000000000000000000000000000000000000..dbd2183188053b13c59cfb0dbb93e38d96f482d6 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/all.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/arm.out b/tests/scripts/dl_dep.sh/0.9.3/arm.out new file mode 100644 index 0000000000000000000000000000000000000000..7b1d7c5fcb554b36e7e8163f4f9c751c28628534 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/arm.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[fb73c3924b502e2840a5dc8a18746e395b06a8a6]' and flags '' +Done diff --git a/tests/scripts/dl_dep.sh/0.9.3/ci.out b/tests/scripts/dl_dep.sh/0.9.3/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..0754a2860f268f916db2cc7ab121e1737b6b5705 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ci.out @@ -0,0 +1,9 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/default.out b/tests/scripts/dl_dep.sh/0.9.3/default.out new file mode 100644 index 0000000000000000000000000000000000000000..7b7330872c4578763b697f7394fb9d49f872f11d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/default.out @@ -0,0 +1,10 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/direct.out b/tests/scripts/dl_dep.sh/0.9.3/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..42a6d15450c591a23273ec6201faf713235fab58 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/direct.out @@ -0,0 +1,7 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..f2c3f6d3b2c69b9e934055bec1ef1cbfc2a20f63 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,10 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/mogon2.out b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/ngio.out b/tests/scripts/dl_dep.sh/0.9.3/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ngio.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..8ef7035e06cb383d1d826f0354d222e9bdcb48b8 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/p9.out b/tests/scripts/dl_dep.sh/0.9.3/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..1c939fc2332d564471238d6941f319b748348f7a --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/p9.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f]' and flags '' +Done diff --git a/CMake/FindDate.cmake b/tools/CMakeLists.txt similarity index 68% rename from CMake/FindDate.cmake rename to tools/CMakeLists.txt index 78b6a79d8c5cce0d27a491ba0b724881701e5f14..9c3e16519551b976c85b23601083fc1f8ee35bbe 100644 --- a/CMake/FindDate.cmake +++ b/tools/CMakeLists.txt @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -26,43 +26,13 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -find_path(DATE_INCLUDE_DIR - NAMES date/date.h -) - -find_path(TZ_INCLUDE_DIR - NAMES date/tz.h -) - -find_library(TZ_LIBRARY - NAMES tz -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( Date - DEFAULT_MSG - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) - -if(Date_FOUND) - set(DATE_INCLUDE_DIRS ${DATE_INCLUDE_DIR}) - set(TZ_INCLUDE_DIRS ${TZ_INCLUDE_DIR}) - set(TZ_LIBRARIES ${TZ_LIBRARY}) - - if(NOT TARGET Date::TZ) - add_library(Date::TZ UNKNOWN IMPORTED) - set_target_properties(Date::TZ PROPERTIES - IMPORTED_LOCATION "${TZ_LIBRARY}" - INTERFACE_COMPILE_DEFINITIONS "USE_OS_TZDB=1" - INTERFACE_INCLUDE_DIRECTORIES "${TZ_INCLUDE_DIR}" - ) - endif() -endif() - -mark_as_advanced( - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) +if (GKFS_ENABLE_CLIENT_METRICS) + include(load_nlohmann_json) + add_executable(gkfs_clientmetrics2json) + target_sources(gkfs_clientmetrics2json + PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/clientmetrics2json.cpp + ) + target_link_libraries(gkfs_clientmetrics2json PUBLIC msgpack_util nlohmann_json::nlohmann_json) + install(TARGETS gkfs_clientmetrics2json RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif () \ No newline at end of file diff --git a/tools/clientmetrics2json.cpp b/tools/clientmetrics2json.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f74667b5fa94d0100888766371da4bd1089796e5 --- /dev/null +++ b/tools/clientmetrics2json.cpp @@ -0,0 +1,139 @@ +/* + Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + +#include +#include + +#include +#include +#include +#include + +extern "C" { +#include +#include +} + +using json = nlohmann::json; + +void +report_msgpack(std::vector& buffer) { + std::error_code ec{}; + auto undata = + msgpack::unpack( + buffer, ec); + std::vector avg_thruput(undata.req_size_.size()); + for(size_t i = 0; i < avg_thruput.size(); ++i) { + auto size_mib = undata.req_size_[i] / (1024.0 * 1024.0); // in MiB + auto duration_s = (undata.end_t_[i] - undata.start_t_[i]) / 1000000.0; + avg_thruput[i] = std::round((size_mib / duration_s) * 100.0) / 100.0; + } + json json_obj; + json_obj["hostname"] = undata.hostname_; + json_obj["pid"] = undata.pid_; + json_obj["io_type"] = undata.io_type_; + json_obj["total_bytes"] = undata.total_bytes_; + json_obj["total_iops"] = undata.total_iops_; + json_obj["start_t_micro"] = undata.start_t_; + json_obj["end_t_micro"] = undata.end_t_; + json_obj["flush_t"] = undata.flush_t_; + json_obj["req_size"] = undata.req_size_; + json_obj["[extra]avg_thruput_mib"] = avg_thruput; + + std::cout << "Generated JSON:" << '\n'; + for(const auto& item : json_obj.items()) { + std::cout << item.key() << ": " << item.value().dump() << '\n'; + } +} + +void +read_file(const std::string& path) { + if(!std::filesystem::exists(path)) { + std::cerr << "Input file " << path << " does not exist" << '\n'; + return; + } + std::ifstream file(path, std::ios::binary); + if(!file.is_open()) { + std::cout << "failed to open " << '\n'; + return; + } + std::vector buffer((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + + file.close(); + report_msgpack(buffer); +} + + +int +main(int argc, char** argv) { + std::string mode{}; + if(argc != 2) { + std::cerr << "Usage: " << argv[0] + << " \n" + << " ZMQ listing must start with tcp://\n"; + return -1; + } + auto path_ip = std::string(argv[1]); + if(path_ip.find("tcp://") == 0) { + // Create a ZeroMQ context + zmq::context_t context(1); + // Create a socket to for the server to bind to + zmq::socket_t socket(context, ZMQ_PULL); + std::cout << "Binding to: " << path_ip << std::endl; + socket.bind(path_ip); + + while(true) { + // Wait for a request message + zmq::message_t request; + std::cout << "Waiting for message... " << std::endl; + if(!socket.recv(request, zmq::recv_flags::none)) { + std::cerr << "Failed to receive message" << std::endl; + continue; + } + std::cout << "Received message with size " << request.size() + << '\n'; + + std::vector requestData( + static_cast(request.data()), + static_cast(request.data()) + + request.size()); + + report_msgpack(requestData); + std::cout + << "-------------------------------------------------------------------------------------\n"; + } + } else { + read_file(path_ip); + } + // std::cout << json_obj.dump(4) << std::endl; // Use dump for pretty + // printing + + return 0; +} \ No newline at end of file