diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68eaca4ed3a2a944e32510b744c9294bf0e4d417..19c076f05914df9170c43104dda58ca13eae5890 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.2 +image: gekkofs/core:0.9.3 ################################################################################ ## Validating ################################################################################ check format: stage: lint - image: gekkofs/linter:0.9.2 + image: gekkofs/linter:0.9.3 needs: [] script: - ${SCRIPTS_DIR}/check_format.sh @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.2 + image: gekkofs/deps:0.9.3 interruptible: true needs: [] script: @@ -65,23 +65,6 @@ gkfs: - ${INSTALL_PATH} expire_in: 1 week -gkfwd: - stage: build - image: gekkofs/deps:0.9.2 - interruptible: true - needs: [] - script: - - cmake --preset ci-forwarding-coverage - - cmake --build ${BUILD_PATH} -j $(nproc) --target install - # reduce artifacts size - - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH} - artifacts: - paths: - - ${BUILD_PATH} - - ${INSTALL_PATH} - expire_in: 1 week - - ################################################################################ ## Testing ################################################################################ @@ -89,7 +72,7 @@ gkfwd: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: [] script: - mkdir -p ${BUILD_PATH}/tests/scripts @@ -105,7 +88,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true needs: ['gkfs'] parallel: @@ -154,14 +137,14 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true - needs: ['gkfwd'] + needs: ['gkfs'] parallel: matrix: - SUBTEST: [ forwarding ] - rules: - - when: never + # rules: + # - when: never script: ## run tests @@ -204,7 +187,7 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: ['gkfs'] script: ## Add path to mkfs.kreon @@ -242,7 +225,7 @@ gkfs:unit: ################################################################################ documentation: stage: docs - image: gekkofs/docs:0.9.2 + image: gekkofs/docs:0.9.3 needs: [] rules: # we only build the documentation automatically if we are on the @@ -272,9 +255,9 @@ documentation: ## == coverage baseline ==================== coverage:baseline: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true - needs: ['gkfs', 'gkfwd'] + needs: ['gkfs'] script: ## capture initial coverage information to establish a baseline @@ -298,10 +281,10 @@ coverage:baseline: coverage: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 # needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', # 'gkfs:unit' ] - needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ] + needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] script: - cd ${CI_PROJECT_DIR} - cmake diff --git a/.gitmodules b/.gitmodules index 18f937ae7b811d4f78fc6fce7f89600199f28abd..31675fc19786aac2ecbafec75dd4daf8e39a3f4b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "external/hermes"] path = external/hermes - url = https://github.com/bsc-ssrg/hermes.git + url = https://github.com/gekkofs/hermes.git [submodule "tests/scripts/bats"] path = tests/scripts/bats url = https://github.com/bats-core/bats-core.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ae0762a2456b0efd8172251cd3a3d5d079d7a8..93e0446d56f634b8b9652c0ce5839d69c6593ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). - Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. - Added reattemp support in get_fs_config to other servers, when the initial server fails. +- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable +`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). +- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)). ### New ### Changed diff --git a/CMake/FindMercury.cmake b/CMake/FindMercury.cmake index 7eb96cde545a7ea7bb863df725f9285368c47a59..451ed6d403ab2a3d4f4f2fb6cabb351179593add 100644 --- a/CMake/FindMercury.cmake +++ b/CMake/FindMercury.cmake @@ -177,7 +177,7 @@ endmacro() include(SelectLibraryConfigurations) -set(_mercury_components na mercury_util mercury_hl) +set(_mercury_components na mercury_util) # prevent repeating work if the main CMakeLists.txt already called # find_package(PkgConfig) diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake index 30da5afc387621004040ca75bfc3f43630722189..d900cdee1c3eccebb39dd75e81dcedc8c8add2f1 100644 --- a/CMake/gkfs-options.cmake +++ b/CMake/gkfs-options.cmake @@ -348,14 +348,6 @@ gkfs_define_option( # I/O forwarding ################################################################################ -## Forwarding support -gkfs_define_option( - GKFS_ENABLE_FORWARDING - HELP_TEXT "Enable I/O forwarding mode" - DEFAULT_VALUE OFF - DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer" -) - ## Scheduling in I/O forwarding mode gkfs_define_option( GKFS_ENABLE_AGIOS diff --git a/CMakeLists.txt b/CMakeLists.txt index afdda9c02ef8e0137d8bca31f8b2bab90bb8cce2..a26d68077735b69112ac945cd7006cca70c000e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,16 +32,17 @@ cmake_minimum_required(VERSION 3.13) project( GekkoFS VERSION 0.9.2 + LANGUAGES ASM CXX C ) enable_testing() -if (NOT CMAKE_COMPILER_IS_GNUCC) - message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") -endif () -if (NOT CMAKE_COMPILER_IS_GNUCXX) - message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") -endif () +#if (NOT CMAKE_COMPILER_IS_GNUCC) +# message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") +#endif () +#if (NOT CMAKE_COMPILER_IS_GNUCXX) +# message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") +#endif () set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -160,10 +161,6 @@ find_package(Margo 0.9.6 REQUIRED) message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept") find_package(Syscall_intercept REQUIRED) -### Date (required by the libgkfs_intercept.so) -message(STATUS "[${PROJECT_NAME}] Checking for Date") -find_package(Date REQUIRED) - ### AGIOS: required for scheduling I/O requests if (GKFS_ENABLE_AGIOS) message(STATUS "[${PROJECT_NAME}] Checking for Agios") @@ -225,7 +222,7 @@ include_from_source(fmt MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG f94b7364b9409f05207c3af3fa4666730e11a854 # v6.1.2 + GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0 ) # ensure that fmt is linked as PIC @@ -338,11 +335,7 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}") message(STATUS "[gekkofs] Check for forwarding tests...") - if (GKFS_ENABLE_FORWARDING) - set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - else () - set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - endif () + set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}") message(STATUS "[gekkofs] Check for guided distributor tests...") if (GKFS_USE_GUIDED_DISTRIBUTION) diff --git a/CMakePresets.json b/CMakePresets.json index 9f12e9c734ceabc71595221f384db23c67ff7bf9..da7bf79b3ef83ca3a0f51bcc014d255dccd8ece6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -27,14 +27,6 @@ "deprecated": true } }, - { - "name": "forwarding", - "inherits": "default", - "hidden": true, - "cacheVariables": { - "GKFS_ENABLE_FORWARDING": true - } - }, { "name": "release", "hidden": true, @@ -113,30 +105,6 @@ "release" ] }, - { - "name": "forwarding-debug", - "displayName": "Forwarding gekkofs (debug)", - "inherits": [ - "forwarding", - "debug" - ] - }, - { - "name": "forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage)", - "inherits": [ - "forwarding", - "coverage" - ] - }, - { - "name": "forwarding-release", - "displayName": "Forwarding gekkofs (release)", - "inherits": [ - "forwarding", - "release" - ] - }, { "name": "ci-debug", "displayName": "Default gekkofs (debug, CI flags)", @@ -171,33 +139,6 @@ "default", "release" ] - }, - { - "name": "ci-forwarding-debug", - "displayName": "Forwarding gekkofs (debug, CI flags)", - "inherits": [ - "ci", - "forwarding", - "debug" - ] - }, - { - "name": "ci-forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage, CI flags)", - "inherits": [ - "ci", - "forwarding", - "coverage" - ] - }, - { - "name": "ci-forwarding-release", - "displayName": "Forwarding gekkofs (release, CI flags)", - "inherits": [ - "ci", - "forwarding", - "release" - ] } ] } diff --git a/docker/0.9.3/core/Dockerfile b/docker/0.9.3/core/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a55434924b2cfe720195c371de6736ac049826f7 --- /dev/null +++ b/docker/0.9.3/core/Dockerfile @@ -0,0 +1,40 @@ +FROM debian:bullseye-slim + +LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + ninja-build \ + procps \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ + # Margo dependencies \ + libjson-c-dev \ + # RocksDB dependencies + liblz4-dev \ + # syscall_intercept dependencies + libcapstone-dev \ + # GekkoFS dependencies + libboost-program-options-dev \ + uuid-dev && \ + # install cmake 3.14+ since it's needed for some dependencies + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh diff --git a/docker/0.9.3/core/Makefile b/docker/0.9.3/core/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9c27a583a8ae1069865ad13cc0196214d40b9362 --- /dev/null +++ b/docker/0.9.3/core/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/core:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/core:0.9.3 . + +all: + docker buildx build -t gekkofs/core:0.9.3 . diff --git a/docker/0.9.3/deps/Dockerfile b/docker/0.9.3/deps/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ca4ecc8a646db53669fa642eaefdd2bfcc7e8804 --- /dev/null +++ b/docker/0.9.3/deps/Dockerfile @@ -0,0 +1,38 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to build GekkoFS" + +ENV GKFS_PATH /opt/gkfs +ENV GKFS_VERSION 0.9.3 + +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH /usr/local + +COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/patches ${SCRIPTS_PATH}/patches +COPY scripts/profiles ${SCRIPTS_PATH}/profiles + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python3-setuptools \ + libnuma-dev libyaml-dev libcurl4-openssl-dev \ + procps && \ + python3 -m pip install --upgrade pip && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && apt-get autoclean + +# Download and build dependencies +RUN cd ${SCRIPTS_PATH} && \ + /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \ + /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \ + cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\ + rm -rf ${DEPS_SRC_PATH} && \ + rm -rf ${SCRIPTS_PATH} && \ + rmdir ${GKFS_PATH} && \ + ldconfig diff --git a/docker/0.9.3/deps/Makefile b/docker/0.9.3/deps/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a97d3195f66a9cf7c5b44ed7b32d25abb49e6ab0 --- /dev/null +++ b/docker/0.9.3/deps/Makefile @@ -0,0 +1,23 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts remove-scripts +amd64: build-amd64 mount-scripts remove-scripts +aarch64: build-aarch64 mount-scripts remove-scripts + +copy-scripts: + cp -R $(GIT_ROOT)/scripts $(CWD)/scripts + +build: copy-scripts + docker build -t gekkofs/deps:0.9.3 . + +build-amd64: copy-scripts + docker build --platform amd64 -t gekkofs/deps:0.9.3 . + +build-aarch64: copy-scripts + docker build --platform aarch64 -t gekkofs/deps:0.9.3 . + +remove-scripts: + - rm -rf $(CWD)/scripts diff --git a/docker/0.9.3/docs/Dockerfile b/docker/0.9.3/docs/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f770f3a23de51eb7ee8f13d0f3b8ebf8623d2649 --- /dev/null +++ b/docker/0.9.3/docs/Dockerfile @@ -0,0 +1,34 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.33.1' \ + 'exhale==0.3.1' \ + 'sphinx-copybutton==0.5.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.17.0' \ + attrs && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/docs/Makefile b/docker/0.9.3/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..c6c89926af84c32ee55331a647a3d041ea6e0555 --- /dev/null +++ b/docker/0.9.3/docs/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/docs:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/docs:0.9.3 . + +all: + docker build -t gekkofs/docs:0.9.3 . \ No newline at end of file diff --git a/docker/0.9.3/linter/Dockerfile b/docker/0.9.3/linter/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f5d57118e518d36fb28f7a0f2bc95716dc99f2d0 --- /dev/null +++ b/docker/0.9.3/linter/Dockerfile @@ -0,0 +1,19 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to check the formatting of GekkoFS code" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # clang 15 deps for clang-format + lsb-release \ + wget \ + software-properties-common \ + gnupg2 && \ + # add clang-15 repos + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \ + # install clang-format + apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/linter/Makefile b/docker/0.9.3/linter/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f427eb387a7f8aebffa6a4522c428467835903e4 --- /dev/null +++ b/docker/0.9.3/linter/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/linter:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/linter:0.9.3 . + +all: + docker buildx build -t gekkofs/linter:0.9.3 . diff --git a/docker/0.9.3/testing/Dockerfile b/docker/0.9.3/testing/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..971944a86b0c5102eba84a912b33a53187078701 --- /dev/null +++ b/docker/0.9.3/testing/Dockerfile @@ -0,0 +1,32 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment to test GekkoFS" + +RUN \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + # required by lcov's genhtml + libgd-perl && \ + # install cmake 3.21+ since we need to produce JUnit XML files + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # install loguru + # (required by several of our scripts) + pip3 install loguru && \ + # install lcov_cobertura + # (required to produce Cobertura XML reports) + pip3 install lcov_cobertura && \ + # install lcov + # (required to produce partial coverage reports in parallel runs) + curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \ + tar xfz lcov-1.16.tar.gz && \ + cd lcov-1.16 && \ + make install && \ + cd .. && \ + # cleanup + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh && \ + rm -rf ./lcov-1.16.* diff --git a/docker/0.9.3/testing/Makefile b/docker/0.9.3/testing/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b5908b5399d36a19a96cc61a8f929fd9a6c4a581 --- /dev/null +++ b/docker/0.9.3/testing/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/testing:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/testing:0.9.3 . + +all: + docker build -t gekkofs/testing:0.9.3 . diff --git a/docs/sphinx/users/dependencies.rst b/docs/sphinx/users/dependencies.rst index 0d3732410f1b133cc755047a1df1ad2601fa2f0b..3490ef5d6300b3153f35f37f5ff252854252e306 100644 --- a/docs/sphinx/users/dependencies.rst +++ b/docs/sphinx/users/dependencies.rst @@ -43,4 +43,4 @@ Required Optional -------- -- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode. +- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling. diff --git a/docs/sphinx/users/forwarding.rst b/docs/sphinx/users/forwarding.rst index db0afc7b22e587f629bcacadfde6de82c62f7e80..e850b4d04d7b4649fd7b600f3dd9cd6c7dbad8f5 100644 --- a/docs/sphinx/users/forwarding.rst +++ b/docs/sphinx/users/forwarding.rst @@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes. Enabling GekkoFWD ------------------ -To enable the I/O forwarding mode of GekkoFS, the -:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when -configuring ` the build: - -.. code-block:: console - - $ cmake -DENABLE_FORWARDING:BOOL=ON +To enable the I/O forwarding mode of GekkoFS, an environment +variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided +to allow users to identify the map file on each client. +This environment variable will enable GekkoFS forwarding, then we +only need to use :code:`--enable-forwarding`` option in the servers. I/O Scheduling -------------- diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index d9e63e317b8045762757231c1196b0a973faca7a..22c9bfa99b747ad83b9185b18a20998899b2dfec 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -28,9 +28,17 @@ set (CMAKE_CXX_STANDARD 14) add_executable(sfind sfind.cpp) +add_executable(gkfs_lib_example gkfs_lib_example.cpp) + +target_link_libraries(gkfs_lib_example + PRIVATE gkfs_user_lib +) if(GKFS_INSTALL_TESTS) install(TARGETS sfind DESTINATION ${CMAKE_INSTALL_BINDIR} ) + install(TARGETS gkfs_lib_example + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) endif() diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4644d29fca2516148130210d4d4838d0b524ca05 --- /dev/null +++ b/examples/gfind/gkfs_lib_example.cpp @@ -0,0 +1,115 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +void +write_file(std::string filename) { + // Open File + int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT); + + cout << "FD open " << fd << endl; + + char* bufwrite = (char*) malloc(10); + strncpy(bufwrite, "testing", 7); + + + int size = gkfs::syscall::gkfs_write(fd, bufwrite, 7); + + cout << "Writting size " << size << endl; + + free(bufwrite); + gkfs::syscall::gkfs_close(fd); +} + + +void +read_file(std::string filename) { + int fdread = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDONLY); + if(fdread == -1) + return; + char* bufread = (char*) malloc(10); + int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); + + cout << "Reading Size: " << sizeread << " Content: " << bufread << endl; + + free(bufread); + gkfs::syscall::gkfs_close(fdread); +} + +int +main(int argc, char** argv) { + cout << "GekkoFS Client library test" << endl; + + auto res = gkfs_init(); + + cout << "Init result " << res << endl; + + write_file("/test.tmp"); + + read_file("/test.tmp"); + + + write_file("/secondfile.tmp"); + + auto f_list = gkfs::syscall::gkfs_get_file_list("/"); + + for(auto f : f_list) { + cout << "File: " << f << endl; + struct stat buf; + memset(&buf, 0, sizeof(struct stat)); + + gkfs::syscall::gkfs_stat("/" + f, &buf, true); + + cout << "Size: " << buf.st_size << " Mode: " << buf.st_mode << endl; + cout << "Atime: " << buf.st_atime << " Mtime: " << buf.st_mtime + << " Ctime: " << buf.st_ctime << endl + << " ****** " << endl; + } + + res = gkfs_end(); + + cout << "End result " << res << endl; +} diff --git a/external/hermes b/external/hermes index d8755039c82501323cdc9a4f5a2b922b9b3bcac7..a133c9dd0cefc7e8a3554badfb8995c9f2c27be6 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit d8755039c82501323cdc9a4f5a2b922b9b3bcac7 +Subproject commit a133c9dd0cefc7e8a3554badfb8995c9f2c27be6 diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 4cc56ac387e724c02d549d4e5d4774a68c62be61..1ae9e3b40e76552f804dc6c3cd4c7dd053c291a3 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -34,7 +34,3 @@ add_subdirectory(daemon) add_subdirectory(client) target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in) - -if(GKFS_ENABLE_FORWARDING) - target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in) -endif() diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index f5df6ee5c796e2dd72c909ac4e8b1b15bb3912c2..6815fc95321a27ecce55b407d10075cb53cad2ca 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -54,30 +54,30 @@ target_sources( syscalls/detail/syscall_info.h ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_intercept - PUBLIC gkfs_functions.hpp - env.hpp - hooks.hpp - intercept.hpp - logging.hpp - make_array.hpp - open_file_map.hpp - open_dir.hpp - path.hpp - preload.hpp - preload_context.hpp - preload_util.hpp - rpc/rpc_types.hpp - rpc/forward_management.hpp - rpc/forward_metadata.hpp - rpc/forward_data.hpp - syscalls/args.hpp - syscalls/decoder.hpp - syscalls/errno.hpp - syscalls/rets.hpp - syscalls/syscall.hpp - syscalls/detail/syscall_info.h - ) -endif() +target_sources( + gkfs_user_lib + PUBLIC gkfs_functions.hpp + env.hpp + hooks.hpp + intercept.hpp + logging.hpp + make_array.hpp + open_file_map.hpp + open_dir.hpp + path.hpp + preload.hpp + preload_context.hpp + preload_util.hpp + rpc/rpc_types.hpp + rpc/forward_management.hpp + rpc/forward_metadata.hpp + rpc/forward_data.hpp + syscalls/args.hpp + syscalls/decoder.hpp + syscalls/errno.hpp + syscalls/rets.hpp + syscalls/syscall.hpp + syscalls/detail/syscall_info.h + void_syscall_intercept.hpp + user_functions.hpp +) diff --git a/include/client/env.hpp b/include/client/env.hpp index 34a986facb0855c117370a2871fc36527c77d5df..5609785e8424437d6741cece32bbdb1a7bca1b5e 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -49,9 +49,8 @@ static constexpr auto LOG_PER_PROCESS = ADD_PREFIX("LOG_PER_PROCESS"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); -#ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); -#endif + static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); } // namespace gkfs::env diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 63b9b655f55d5851d780f0c5379b4b35d7225533..313df5fcb46339a309298237dc9bda8bd0b52e16 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -149,14 +149,22 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, int gkfs_rmdir(const std::string& path); +int +gkfs_close(unsigned int fd); + +std::vector +gkfs_get_file_list(const std::string& path); + #ifdef HAS_RENAME int gkfs_rename(const std::string& old_path, const std::string& new_path); #endif // HAS_RENAME + } // namespace gkfs::syscall // gkfs_getsingleserverdir is using extern "C" to demangle it for C usage extern "C" int gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp, unsigned int count, int server); + #endif // GEKKOFS_GKFS_FUNCTIONS_HPP diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index e71a6a08d68b2b0875b213b7f19e260a13c73b2b..fa9ec424bc606a941650edb253a20781af54cf8d 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -36,7 +36,13 @@ extern "C" { #include #include } + +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + /* * For PowerPC, syscall_no_intercept_wrapper() is defined in the diff --git a/include/client/logging.hpp b/include/client/logging.hpp index c99e2e4a034cf5567008ba1fa16ea6f7219105f9..9d1f464e52649e33268769b9a2cadea711b31fee 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -30,7 +30,11 @@ #ifndef LIBGKFS_LOGGING_HPP #define LIBGKFS_LOGGING_HPP +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif #include #include @@ -38,7 +42,6 @@ #include #include #include -#include #include #ifdef GKFS_DEBUG_BUILD @@ -175,6 +178,138 @@ log_buffer(int fd, const void* buffer, std::size_t length) { ::syscall_no_intercept(SYS_write, fd, buffer, length); } +/** + * @brief convert a time_t to a tm + * It is not POSIX compliant, but it dows not uses any syscall or timezone + * Converts a Unix timestamp (number of seconds since the beginning of 1970 + * CE) to a Gregorian civil date-time tuple in GMT (UTC) time zone. + * + * This conforms to C89 (and C99...) and POSIX. + * + * This implementation works, and doesn't overflow for any sizeof(time_t). + * It doesn't check for overflow/underflow in tm->tm_year output. Other than + * that, it never overflows or underflows. It assumes that that time_t is + * signed. + * + * This implements the inverse of the POSIX formula + * (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_15) + * for all time_t values, no matter the size, as long as tm->tm_year doesn't + * overflow or underflow. The formula is: tm_sec + tm_min*60 + tm_hour*3600 + * + tm_yday*86400 + (tm_year-70)*31536000 + ((tm_year-69)/4)*86400 - + * ((tm_year-1)/100)*86400 + ((tm_year+299)/400)*86400. + * + * License : GNU General Public License v2.0 from + * https://github.com/pts/minilibc686/ + * @param time_t + * @return tm + */ + +static inline struct tm* +mini_gmtime_r(const time_t* timep, struct tm* tm) { + const time_t ts = *timep; + time_t t = ts / 86400; + unsigned hms = + ts % + 86400; /* -86399 <= hms <= 86399. This needs sizeof(int) >= 4. */ + time_t c, f; + unsigned yday; /* 0 <= yday <= 426. Also fits to an `unsigned short', but + `int' is faster. */ + unsigned a; /* 0 <= a <= 2133. Also fits to an `unsigned short', but `int' + is faster. */ + if((int) hms < 0) { + --t; + hms += 86400; + } /* Fix quotient and negative remainder if ts was negative (i.e. before + year 1970 CE). */ + /* Now: -24856 <= t <= 24855. */ + tm->tm_sec = hms % 60; + hms /= 60; + tm->tm_min = hms % 60; + tm->tm_hour = hms / 60; + if(sizeof(time_t) > + 4) { /* Optimization. For int32_t, this would keep t intact, so we won't + have to do it. This produces unreachable code. */ + f = (t + 4) % 7; + if(f < 0) + f += 7; /* Fix negative remainder if (t + 4) was negative. */ + /* Now 0 <= f <= 6. */ + tm->tm_wday = f; + c = (t << 2) + 102032; + f = c / 146097; + if(c % 146097 < 0) + --f; /* Fix negative remainder if c was negative. */ + --f; + t += f; + f >>= 2; + t -= f; + f = (t << 2) + 102035; + c = f / 1461; + if(f % 1461 < 0) + --c; /* Fix negative remainder if f was negative. */ + } else { + tm->tm_wday = (t + 24861) % 7; /* t + 24861 >= 0. */ + /* Now: -24856 <= t <= 24855. */ + c = ((t << 2) + 102035) / 1461; + } + yday = t - 365 * c - (c >> 2) + 25568; + /* Now: 0 <= yday <= 425. */ + a = yday * 5 + 8; + /* Now: 8 <= a <= 2133. */ + tm->tm_mon = a / 153; + a %= 153; /* No need to fix if a < 0, because a cannot be negative here. */ + /* Now: 2 <= tm->tm_mon <= 13. */ + /* Now: 0 <= a <= 152. */ + tm->tm_mday = 1 + a / 5; /* No need to fix if a < 0, because a cannot be + negative here. */ + /* Now: 1 <= tm->tm_mday <= 31. */ + if(tm->tm_mon >= 12) { + tm->tm_mon -= 12; + /* Now: 0 <= tm->tm_mon <= 1. */ + ++c; + yday -= 366; + } else { /* Check for leap year (in c). */ + /* Now: 2 <= tm->tm_mon <= 11. */ + /* 1903: not leap; 1904: leap, 1900: not leap; 2000: leap */ + /* With sizeof(time_t) == 4, we have 1901 <= year <= 2038; of these + * years only 2000 is divisble by 100, and that's a leap year, no we + * optimize the check to `(c & 3) == 0' only. + */ + if(!((c & 3) == 0 && + (sizeof(time_t) <= 4 || c % 100 != 0 || (c + 300) % 400 == 0))) + --yday; /* These `== 0' comparisons work even if c < 0. */ + } + tm->tm_year = + c; /* This assignment may overflow or underflow, we don't check it. + Example: time_t is a huge int64_t, tm->tm_year is int32_t. */ + /* Now: 0 <= tm->tm_mon <= 11. */ + /* Now: 0 <= yday <= 365. */ + tm->tm_yday = yday; + tm->tm_isdst = 0; + return tm; +} + +static inline struct tm* +mini_gmtime(const time_t* timep) { + static struct tm tm; + return mini_gmtime_r(timep, &tm); +} + +static inline ssize_t +format_timeval(struct timeval* tv, char* buf, size_t sz) { + ssize_t written = -1; + struct tm* gm = mini_gmtime(&tv->tv_sec); + + + written = (ssize_t) strftime(buf, sz, "%Y-%m-%d %H:%M:%S", gm); + if((written > 0) && ((size_t) written < sz)) { + int w = snprintf(buf + written, sz - (size_t) written, ".%06ld", + tv->tv_usec); + written = (w > 0) ? written + w : -1; + } + + return written; +} + /** * format_timestamp_to - safely format a timestamp for logging messages * @@ -192,13 +327,10 @@ log_buffer(int fd, const void* buffer, std::size_t length) { * one thread exactly, and we pass it as an argument whenever we need to * format a timestamp. If no timezone is provided, we just format the epoch. * - * NOTE: we use the date C++ library to query the timezone database and - * to format the timestamps. */ template static inline void -format_timestamp_to(Buffer&& buffer, - const date::time_zone* const timezone = nullptr) { +format_timestamp_to(Buffer&& buffer) { struct ::timeval tv; @@ -208,17 +340,11 @@ format_timestamp_to(Buffer&& buffer, return; } - date::sys_time now{ - std::chrono::seconds{tv.tv_sec} + - std::chrono::microseconds{tv.tv_usec}}; + char buf[28]; - if(!timezone) { - fmt::format_to(buffer, "[{}] ", now.time_since_epoch().count()); - return; + if(format_timeval(&tv, buf, sizeof(buf)) > 0) { + fmt::format_to(std::back_inserter(buffer), "[{}] ", buf); } - - fmt::format_to(buffer, "[{}] ", - date::zoned_time{timezone, now}); } template @@ -226,7 +352,7 @@ static inline void format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { const auto ttid = syscall_no_intercept(SYS_gettid); - fmt::format_to(buffer, "[{}] [syscall] ", ttid); + fmt::format_to(std::back_inserter(buffer), "[{}] [syscall] ", ttid); char o; char t; @@ -256,20 +382,15 @@ format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { } const std::array tmp = {'[', o, t, ']', ' '}; - fmt::format_to(buffer, fmt::string_view(tmp.data(), tmp.size())); + fmt::format_to(std::back_inserter(buffer), + fmt::string_view(tmp.data(), tmp.size())); } } // namespace detail enum { max_buffer_size = LIBGKFS_LOG_MESSAGE_SIZE }; -struct static_buffer : public fmt::basic_memory_buffer { - -protected: - void - grow(std::size_t size) override final; -}; - +using static_buffer = fmt::basic_memory_buffer; struct logger { @@ -293,16 +414,17 @@ struct logger { } static_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); - fmt::format_to(buffer, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + detail::format_timestamp_to(buffer); + fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ", + log_process_id_, lookup_level_name(level)); if(!!(level & log::debug)) { - fmt::format_to(buffer, "<{}():{}> ", func, lineno); + fmt::format_to(std::back_inserter(buffer), "<{}():{}> ", func, + lineno); } - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(log_fd_, buffer); } @@ -337,8 +459,8 @@ struct logger { static_buffer prefix; detail::format_timestamp_to(prefix); - fmt::format_to(prefix, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + fmt::format_to(std::back_inserter(prefix), "[{}] [{}] ", + log_process_id_, lookup_level_name(level)); char buffer[max_buffer_size]; const int n = vsnprintf(buffer, sizeof(buffer), fmt, ap); @@ -387,8 +509,8 @@ struct logger { } static_buffer buffer; - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(fd, buffer); } @@ -410,8 +532,6 @@ struct logger { std::bitset<512> filtered_syscalls_; int debug_verbosity_; #endif - - const date::time_zone* timezone_; }; // the following static functions can be used to interact @@ -440,23 +560,6 @@ destroy_global_logger() { logger::global_logger().reset(); } -inline void -static_buffer::grow(std::size_t size) { - - const auto logger = get_global_logger(); - - if(logger) { - logger->log_mask_ &= ~(syscall | syscall_at_entry); - } - - std::fprintf( - stderr, - "FATAL: message too long for gkfs::log::static_buffer, increase the size of\n" - "LIBGKFS_LOG_MESSAGE_SIZE in CMake or reduce the length of the offending " - "message.\n"); - abort(); -} - } // namespace gkfs::log #define LOG(XXX, ...) LOG_##XXX(__VA_ARGS__) diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 175cd8f6edd89e441993f2055bb9eb325e611e24..afad97c684bbb5736bceef4079e67adc7b120b3c 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -31,7 +31,8 @@ #define IOINTERCEPT_PRELOAD_HPP #include - +#include +#include #define EUNKNOWN (-1) #define CTX gkfs::preload::PreloadContext::getInstance() @@ -40,11 +41,12 @@ void init_ld_env_if_needed(); } // namespace gkfs::preload +#ifndef BYPASS_SYSCALL void init_preload() __attribute__((constructor)); void destroy_preload() __attribute__((destructor)); - +#endif #endif // IOINTERCEPT_PRELOAD_HPP diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp index a3fa9b64a1104d892696af9bf5daf3c4eb674a64..4c756dec22d8fefbe3aeac29c2ed6c6bf6c6e6dd 100644 --- a/include/client/rpc/rpc_types.hpp +++ b/include/client/rpc/rpc_types.hpp @@ -85,10 +85,10 @@ struct fs_config { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3033006080; + constexpr static const uint64_t public_id = 1; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::fs_config; @@ -259,10 +259,10 @@ struct create { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 796590080; + constexpr static const uint64_t public_id = 2; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::create; @@ -375,10 +375,10 @@ struct stat { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1396244480; + constexpr static const uint64_t public_id = 3; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::stat; @@ -494,10 +494,10 @@ struct remove_metadata { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2087845888; + constexpr static const uint64_t public_id = 4; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::remove_metadata; @@ -618,10 +618,10 @@ struct decr_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1291649024; + constexpr static const uint64_t public_id = 5; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::decr_size; @@ -734,10 +734,10 @@ struct update_metadentry { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 99483648; + constexpr static const uint64_t public_id = 6; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::update_metadentry; @@ -960,10 +960,10 @@ struct get_metadentry_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3426484224; + constexpr static const uint64_t public_id = 7; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_metadentry_size; @@ -1076,10 +1076,10 @@ struct update_metadentry_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2760900608; + constexpr static const uint64_t public_id = 8; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::update_metadentry_size; @@ -1216,10 +1216,10 @@ struct mk_symlink { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3207004160; + constexpr static const uint64_t public_id = 9; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::mk_symlink; @@ -1334,10 +1334,10 @@ struct remove_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2649292800; + constexpr static const uint64_t public_id = 10; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::remove_data; @@ -1442,10 +1442,10 @@ struct write_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3716481024; + constexpr static const uint64_t public_id = 11; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::write; @@ -1627,10 +1627,10 @@ struct read_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 892207104; + constexpr static const uint64_t public_id = 12; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::read; @@ -1812,10 +1812,10 @@ struct trunc_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1850933248; + constexpr static const uint64_t public_id = 13; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::truncate; @@ -1931,10 +1931,10 @@ struct get_dirents { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 4121034752; + constexpr static const uint64_t public_id = 14; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_dirents; @@ -2055,10 +2055,10 @@ struct get_dirents_extended { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1463943168; + constexpr static const uint64_t public_id = 15; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_dirents_extended; @@ -2181,10 +2181,10 @@ struct chunk_stat { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 532742144; + constexpr static const uint64_t public_id = 16; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_chunk_stat; diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp index 05fa8296337d7ab2cfe689f394dc1d8365bdb61b..53ed726c3b1484bc7ec64b92508e26bf908ed73a 100644 --- a/include/client/syscalls/args.hpp +++ b/include/client/syscalls/args.hpp @@ -228,7 +228,7 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flag == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -239,12 +239,12 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { } if((flag == desc[i].flag_)) { - fmt::format_to(buffer, "{}", desc[i].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[i].name_); return; } } - fmt::format_to(buffer, "{:#x}", flag); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flag); } template @@ -254,7 +254,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flags == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -269,7 +269,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { } if((flags & desc[i].flag_) != 0) { - fmt::format_to(buffer, "{}{}", + fmt::format_to(std::back_inserter(buffer), "{}{}", buffer.size() != buffer_start ? "|" : "", desc[i].name_); flags &= ~desc[i].flag_; @@ -280,15 +280,15 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { if(flags != 0) { if(buffer.size() != buffer_start) { - fmt::format_to(buffer, "|"); + fmt::format_to(std::back_inserter(buffer), "|"); } - fmt::format_to(buffer, "{:#x}", flags); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flags); return; } if(buffer_start == buffer.size()) { - fmt::format_to(buffer, "0x0"); + fmt::format_to(std::back_inserter(buffer), "0x0"); } } @@ -311,7 +311,7 @@ format_whence_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SEEK_END) ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); } @@ -334,7 +334,7 @@ format_mmap_prot_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(PROT_WRITE), FLAG_ENTRY(PROT_EXEC)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; @@ -376,7 +376,7 @@ format_mmap_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { #endif ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; } @@ -420,14 +420,14 @@ format_clone_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(CLONE_NEWNET), FLAG_ENTRY(CLONE_IO)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); // the low byte in clone flags contains the number of the termination // signal sent to the parent when the child dies format_flag_set(buffer, parg.value & ~0x11l, flag_names); if((parg.value & 0x11l) != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_signum_arg_to(buffer, {"", parg.value & 0x11l}); } return; @@ -479,7 +479,7 @@ format_signum_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIGSYS)); if(std::strcmp(parg.name, "")) { - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); } format_flag(buffer, parg.value, flag_names); @@ -504,7 +504,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIG_UNBLOCK), FLAG_ENTRY(SIG_SETMASK)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, parg.value, flag_names); return; } @@ -518,7 +518,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } @@ -531,7 +531,7 @@ format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_fd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -546,11 +546,11 @@ inline void format_atfd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(static_cast(parg.value) == AT_FDCWD) { - fmt::format_to(buffer, "{}=AT_FDCWD", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=AT_FDCWD", parg.name); return; } - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -565,12 +565,12 @@ inline void format_cstr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}=\"{}\"", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}=\"{}\"", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } /** @@ -618,7 +618,7 @@ format_open_flags_to(FmtBuffer& buffer, long flags = parg.value; - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, flags & O_ACCMODE, flag_names); flags &= ~O_ACCMODE; @@ -636,7 +636,7 @@ format_open_flags_to(FmtBuffer& buffer, #endif // !O_TMPFILE if(flags != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_flag_set(buffer, flags, extra_flag_names); } } @@ -650,7 +650,7 @@ format_open_flags_to(FmtBuffer& buffer, template inline void format_octal_mode_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#04o}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#04o}", parg.name, parg.value); } /** @@ -664,12 +664,12 @@ inline void format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}={}", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } @@ -682,7 +682,7 @@ format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, parg.value); } @@ -695,7 +695,7 @@ format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -708,7 +708,7 @@ format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#x}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#x}", parg.name, parg.value); } #undef FLAG_ENTRY diff --git a/include/client/syscalls/decoder.hpp b/include/client/syscalls/decoder.hpp index eebdd6fdbc9551514b9c8d89cdc29d4a1fea6381..3172cc2df291c118dcf3de9c13cfa19f7177b10c 100644 --- a/include/client/syscalls/decoder.hpp +++ b/include/client/syscalls/decoder.hpp @@ -60,19 +60,19 @@ decode(FmtBuffer& buffer, const long syscall_number, const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); } template @@ -84,30 +84,31 @@ decode(FmtBuffer& buffer, const long syscall_number, const long argv[MAX_ARGS], const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } if(never_returns(syscall_number)) { - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); return; } if(error_code(result) != 0) { - fmt::format_to(buffer, ") = {} {} ({})", static_cast(-1), - errno_name(-result), errno_message(-result)); + fmt::format_to(std::back_inserter(buffer), ") = {} {} ({})", + static_cast(-1), errno_name(-result), + errno_message(-result)); return; } - fmt::format_to(buffer, ") = "); + fmt::format_to(std::back_inserter(buffer), ") = "); const auto& ret = sc.return_type(); ret.formatter()(buffer, result); } diff --git a/include/client/syscalls/rets.hpp b/include/client/syscalls/rets.hpp index adf77647c8928748d671effedaa73e45db1b26e8..1e40f77b02f6af0d457fd75b124f3ae14aaf85ef 100644 --- a/include/client/syscalls/rets.hpp +++ b/include/client/syscalls/rets.hpp @@ -121,7 +121,7 @@ template inline void format_none_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } template @@ -129,18 +129,18 @@ inline void format_ptr_ret_to(FmtBuffer& buffer, long val) { if(LIKELY(reinterpret_cast(val) != nullptr)) { - fmt::format_to(buffer, "{}", reinterpret_cast(val)); + fmt::format_to(std::back_inserter(buffer), "{}", reinterpret_cast(val)); return; } - fmt::format_to(buffer, "NULL"); + fmt::format_to(std::back_inserter(buffer), "NULL"); } template inline void format_dec_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "{}", val); + fmt::format_to(std::back_inserter(buffer), "{}", val); } #undef LIKELY diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp new file mode 100644 index 0000000000000000000000000000000000000000..5d1bfdfe4a64ef12474ec73f703ca66968674b64 --- /dev/null +++ b/include/client/user_functions.hpp @@ -0,0 +1,85 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + +#ifndef GEKKOFS_USER_FUNCTIONS_HPP +#define GEKKOFS_USER_FUNCTIONS_HPP +#include +#include +#include +#include + +struct linux_dirent64; + +namespace gkfs::syscall { + +int +gkfs_open(const std::string& path, mode_t mode, int flags); + +int +gkfs_create(const std::string& path, mode_t mode); + +int +gkfs_remove(const std::string& path); + +ssize_t +gkfs_write(int fd, const void* buf, size_t count); + +ssize_t +gkfs_read(int fd, void* buf, size_t count); + +int +gkfs_close(unsigned int fd); + +off64_t +gkfs_lseek(unsigned int fd, off64_t offset, unsigned int whence); + +ssize_t +gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset); + +ssize_t +gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); + +int +gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true); + +int +gkfs_remove(const std::string& path); + +std::vector +gkfs_get_file_list(const std::string& path); +} // namespace gkfs::syscall + + +extern "C" int +gkfs_init(); + +extern "C" int +gkfs_end(); + +#endif // GEKKOFS_USER_FUNCTIONS_HPP diff --git a/include/client/void_syscall_intercept.hpp b/include/client/void_syscall_intercept.hpp new file mode 100644 index 0000000000000000000000000000000000000000..16cd01b5bd59da0a1e31ac831c66ed707462e488 --- /dev/null +++ b/include/client/void_syscall_intercept.hpp @@ -0,0 +1,45 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ +#ifndef SYSCALL_BYPASS_HPP +#define SYSCALL_BYPASS_HPP + +static inline int +syscall_error_code(long result) { + if(result < 0 && result >= -0x1000) + return (int) -result; + + return 0; +} + +extern "C" { +long +syscall_no_intercept(long syscall_number, ...); +} + +#endif \ No newline at end of file diff --git a/include/common/CMakeLists.txt b/include/common/CMakeLists.txt index 9561361970c92e4ba89f02c1434508c2aaa1eb0e..38a23bc300c90a2a9f193bb440044a4890b07fa4 100644 --- a/include/common/CMakeLists.txt +++ b/include/common/CMakeLists.txt @@ -30,10 +30,3 @@ target_sources( gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp rpc/rpc_util.hpp ) - -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp - rpc/rpc_types.hpp rpc/rpc_util.hpp - ) -endif() diff --git a/include/config.hpp b/include/config.hpp index 7ec761f64e035c533adcebfa3a9da7b9c267816e..f74d0e8f419d11ed33c6e57d66cee087035e0cc9 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -39,7 +39,8 @@ namespace gkfs::config { constexpr auto hostfile_path = "./gkfs_hosts.txt"; -constexpr auto forwarding_file_path = "./gkfs_forwarding.map"; +// We do not default this, ENV variable always required. +constexpr auto forwarding_file_path = ""; namespace io { /* diff --git a/include/daemon/CMakeLists.txt b/include/daemon/CMakeLists.txt index 9cdfe06a94e699e3c0280ac3bdbe3daf512b00ee..26466ba16138af272d680b78b9f081a4f516c454 100644 --- a/include/daemon/CMakeLists.txt +++ b/include/daemon/CMakeLists.txt @@ -38,21 +38,6 @@ target_sources( handler/rpc_util.hpp ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon - PUBLIC daemon.hpp - util.hpp - ops/data.hpp - ops/metadentry.hpp - classes/fs_data.hpp - classes/rpc_data.hpp - handler/rpc_defs.hpp - handler/rpc_util.hpp - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) - endif() - +if(GKFS_ENABLE_AGIOS) + target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) endif() diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 13f8023d1aa2e4dd9c52cbb15d6c570eb37327a0..ef89b5e282e0b1f9cc7dd82f6b6c66c4287a9553 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -95,6 +95,7 @@ private: bool enable_stats_ = false; bool enable_chunkstats_ = false; bool enable_prometheus_ = false; + bool enable_forwarding_ = false; std::string stats_file_; // Prometheus @@ -246,6 +247,12 @@ public: void enable_chunkstats(bool enable_chunkstats); + bool + enable_forwarding() const; + + void + enable_forwarding(bool enable_forwarding); + bool enable_prometheus() const; diff --git a/scripts/profiles/0.9.3/agios.specs b/scripts/profiles/0.9.3/agios.specs new file mode 100644 index 0000000000000000000000000000000000000000..68921e298efc685cec9c99ac4f82213070a5442d --- /dev/null +++ b/scripts/profiles/0.9.3/agios.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies (except transport-specific and experimental)" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "agios" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/all.specs b/scripts/profiles/0.9.3/all.specs new file mode 100644 index 0000000000000000000000000000000000000000..67396b8dc377d0a5d95010d37a280a486cac4c8f --- /dev/null +++ b/scripts/profiles/0.9.3/all.specs @@ -0,0 +1,86 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.17-20230812" + ["curl"]="7.88.1" + ["prometheus-cpp"]="v1.0.0" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "agios" "curl" "prometheus-cpp" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/arm.specs b/scripts/profiles/0.9.3/arm.specs new file mode 100644 index 0000000000000000000000000000000000000000..1d5432de89d1e4cbf40c9f2137bef4d5418e082d --- /dev/null +++ b/scripts/profiles/0.9.3/arm.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="fb73c3924b502e2840a5dc8a18746e395b06a8a6" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch.arm64" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" + ["syscall_intercept"]="ARM64" +) diff --git a/scripts/profiles/0.9.3/ci.specs b/scripts/profiles/0.9.3/ci.specs new file mode 100644 index 0000000000000000000000000000000000000000..dc9fcedffa4a46fda15bbdbefc973fe98617e61e --- /dev/null +++ b/scripts/profiles/0.9.3/ci.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies required by the CI" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="8.5.4" + ["prometheus-cpp"]="v1.0.0" + ) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" + ) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "agios" "parallax" "prometheus-cpp" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/default.specs b/scripts/profiles/0.9.3/default.specs new file mode 100644 index 0000000000000000000000000000000000000000..59d8175fe8bee6a4dedc6b530323e507a583b850 --- /dev/null +++ b/scripts/profiles/0.9.3/default.specs @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" +) + +# Dependencies that must be cloned. +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/infiniband_verbs.specs b/scripts/profiles/0.9.3/infiniband_verbs.specs new file mode 100644 index 0000000000000000000000000000000000000000..abb374413860c22a20fec3f36eae13e2343489a1 --- /dev/null +++ b/scripts/profiles/0.9.3/infiniband_verbs.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Infiniband supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.5.4" + ["json-c"]="0.17-20230812" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%verbs"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading/installing +order=( + "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%verbs"]="--enable-verbs=yes" +) diff --git a/CMake/FindDate.cmake b/scripts/profiles/0.9.3/install/agios.install similarity index 64% rename from CMake/FindDate.cmake rename to scripts/profiles/0.9.3/install/agios.install index 78b6a79d8c5cce0d27a491ba0b724881701e5f14..b5ed7d7ee930fd2eb79a4113a3aaccd533f6fbe3 100644 --- a/CMake/FindDate.cmake +++ b/scripts/profiles/0.9.3/install/agios.install @@ -25,44 +25,33 @@ # # # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ +# vi: ft=bash -find_path(DATE_INCLUDE_DIR - NAMES date/date.h -) - -find_path(TZ_INCLUDE_DIR - NAMES date/tz.h -) - -find_library(TZ_LIBRARY - NAMES tz -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( Date - DEFAULT_MSG - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) - -if(Date_FOUND) - set(DATE_INCLUDE_DIRS ${DATE_INCLUDE_DIR}) - set(TZ_INCLUDE_DIRS ${TZ_INCLUDE_DIR}) - set(TZ_LIBRARIES ${TZ_LIBRARY}) +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ - if(NOT TARGET Date::TZ) - add_library(Date::TZ UNKNOWN IMPORTED) - set_target_properties(Date::TZ PROPERTIES - IMPORTED_LOCATION "${TZ_LIBRARY}" - INTERFACE_COMPILE_DEFINITIONS "USE_OS_TZDB=1" - INTERFACE_INCLUDE_DIRECTORIES "${TZ_INCLUDE_DIR}" - ) - endif() -endif() +pkg_install() { + ID="agios" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. + make install +} -mark_as_advanced( - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/argobots.install b/scripts/profiles/0.9.3/install/argobots.install new file mode 100644 index 0000000000000000000000000000000000000000..9ff03b5a09576c4e07b3fceaea6c2922ca2ee753 --- /dev/null +++ b/scripts/profiles/0.9.3/install/argobots.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/capstone.install b/scripts/profiles/0.9.3/install/capstone.install new file mode 100644 index 0000000000000000000000000000000000000000..afd7794aff4c51cfc24b9351fd875a4a8724f1dd --- /dev/null +++ b/scripts/profiles/0.9.3/install/capstone.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/curl.install b/scripts/profiles/0.9.3/install/curl.install new file mode 100644 index 0000000000000000000000000000000000000000..44fab7ce95e891e8372cbffc6ac3bd60825ab0e6 --- /dev/null +++ b/scripts/profiles/0.9.3/install/curl.install @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="curl" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + autoreconf -fi + ./configure --prefix="${INSTALL_DIR}" --without-ssl + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/json-c.install b/scripts/profiles/0.9.3/install/json-c.install new file mode 100644 index 0000000000000000000000000000000000000000..9ba99fed64b2d4c573c4e7a247be1b6d0f949a94 --- /dev/null +++ b/scripts/profiles/0.9.3/install/json-c.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/json-c" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install + # Margo doesn't search in both directories, so we make it available in both lib and lib64 + if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then + cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/ + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/libfabric%verbs.install b/scripts/profiles/0.9.3/install/libfabric%verbs.install new file mode 100644 index 0000000000000000000000000000000000000000..74a1921023aa08f5098474d1041577a2a1986dea --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric%verbs.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/libfabric.install b/scripts/profiles/0.9.3/install/libfabric.install new file mode 100644 index 0000000000000000000000000000000000000000..6fee0bf9e77375a776e91db574615aaf53988025 --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/lz4.install b/scripts/profiles/0.9.3/install/lz4.install new file mode 100644 index 0000000000000000000000000000000000000000..893a1b13bc4fa570657908c6a6ce8986d4bd1b57 --- /dev/null +++ b/scripts/profiles/0.9.3/install/lz4.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + # try to remove binaries first in case they already exist. Otherwise install fails. + LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4) + for LZ4_BIN in "${LZ4_BINS[@]}"; do + [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN" + done + make -j"${CORES}" + make DESTDIR="${INSTALL_DIR}" PREFIX="" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/margo.install b/scripts/profiles/0.9.3/install/margo.install new file mode 100644 index 0000000000000000000000000000000000000000..014621c68c61b93b0a322e64b60d2f8a332dd212 --- /dev/null +++ b/scripts/profiles/0.9.3/install/margo.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="margo" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/mercury.install b/scripts/profiles/0.9.3/install/mercury.install new file mode 100644 index 0000000000000000000000000000000000000000..e2f79ebc074cd1461c3b864ee901c507fcb0b0ff --- /dev/null +++ b/scripts/profiles/0.9.3/install/mercury.install @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + # if the profile compiles bmi, enable it + if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then + USE_BMI="-DNA_USE_BMI:BOOL=ON" + else + USE_BMI="-DNA_USE_BMI:BOOL=OFF" + fi + + # if the profile provides any flavour of libfabric, enable it + if profile_has_dependency "^libfabric.*$"; then + USE_OFI="-DNA_USE_OFI:BOOL=ON" + else + USE_OFI="-DNA_USE_OFI:BOOL=OFF" + fi + + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + ${USE_BMI} ${USE_OFI} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/parallax.install b/scripts/profiles/0.9.3/install/parallax.install new file mode 100644 index 0000000000000000000000000000000000000000..8c0cc9e95296b2553c651f4d8e1858fee8334035 --- /dev/null +++ b/scripts/profiles/0.9.3/install/parallax.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + CURR="${SOURCE_DIR}/parallax" + # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \ + -DDISABLE_LOGGING:BOOL=ON \ + .. + make -j"${CORES}" + make install + # We need to copy this file as it is not installed + cp ${CURR}/lib/include/parallax/structures.h ${INSTALL_DIR}/include/ +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/prometheus-cpp.install b/scripts/profiles/0.9.3/install/prometheus-cpp.install new file mode 100644 index 0000000000000000000000000000000000000000..154db65dfeadf13a1e4515e33203251d3ef8fead --- /dev/null +++ b/scripts/profiles/0.9.3/install/prometheus-cpp.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="prometheus-cpp" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/rocksdb.install b/scripts/profiles/0.9.3/install/rocksdb.install new file mode 100644 index 0000000000000000000000000000000000000000..7c3004fee34a937dd813fc560714ce9aac8e3009 --- /dev/null +++ b/scripts/profiles/0.9.3/install/rocksdb.install @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CXXFLAGS='' + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move -Wno-error=maybe-uninitialized' + fi + + # TODO use SSE? + CURR="${SOURCE_DIR}/rocksdb" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \ + -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \ + -DROCKSDB_BUILD_SHARED=OFF \ + -DWITH_LZ4=ON \ + -DWITH_GFLAGS=OFF \ + -DUSE_RTTI=1 \ + -DPORTABLE=1 \ + -DWITH_ALL_TESTS=OFF \ + -DWITH_BENCHMARK_TOOLS=OFF \ + -DWITH_TOOLS=OFF .. + make -j"${CORES}" install + + +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/syscall_intercept.install b/scripts/profiles/0.9.3/install/syscall_intercept.install new file mode 100644 index 0000000000000000000000000000000000000000..f69393d3a9ddb2c68f397973b26108035ea4ce4e --- /dev/null +++ b/scripts/profiles/0.9.3/install/syscall_intercept.install @@ -0,0 +1,71 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + prepare_build_dir "${CURR}" + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cd "${CURR}"/arch/aarch64/ + mkdir -p build + cd build + else + cd "${CURR}"/build + fi + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DBUILD_TESTS:BOOK=OFF .. + make -j"${CORES}" install + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cp "${CURR}"/arch/aarch64/include/libsyscall_intercept_hook_point.h ${INSTALL_DIR}/include + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/ucx.install b/scripts/profiles/0.9.3/install/ucx.install new file mode 100644 index 0000000000000000000000000000000000000000..fd90d6d3f62dd7552becf46bb02ee8c9fdc7a82c --- /dev/null +++ b/scripts/profiles/0.9.3/install/ucx.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + ID="ucx" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../contrib/configure-release --prefix=${INSTALL_DIR} + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/marenostrum4.specs b/scripts/profiles/0.9.3/marenostrum4.specs new file mode 100644 index 0000000000000000000000000000000000000000..15814ec63e526f254d74abccee0711e156599fa3 --- /dev/null +++ b/scripts/profiles/0.9.3/marenostrum4.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Marenostrum 4 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.5.4" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/0.9.3/mogon2.specs b/scripts/profiles/0.9.3/mogon2.specs new file mode 100644 index 0000000000000000000000000000000000000000..f7d9b2cadd509c0c52486d8e712902244f42d45b --- /dev/null +++ b/scripts/profiles/0.9.3/mogon2.specs @@ -0,0 +1,85 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" +# ["psm2"]="11.2.230" +# ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +# ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --enable-opx=yes" +# ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/ngio.specs b/scripts/profiles/0.9.3/ngio.specs new file mode 100644 index 0000000000000000000000000000000000000000..fd1f3a3353a462a82a6080f8193f908936f74e0c --- /dev/null +++ b/scripts/profiles/0.9.3/ngio.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for NEXTGenIO prototype cluster" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/omnipath_psm2.specs b/scripts/profiles/0.9.3/omnipath_psm2.specs new file mode 100644 index 0000000000000000000000000000000000000000..8cdac16d86074198212610762b7c38a98adb9bda --- /dev/null +++ b/scripts/profiles/0.9.3/omnipath_psm2.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Omnipath supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/p9.specs b/scripts/profiles/0.9.3/p9.specs new file mode 100644 index 0000000000000000000000000000000000000000..cf80be5a9c2b08a3ea03eb739177aa7f59e902d2 --- /dev/null +++ b/scripts/profiles/0.9.3/p9.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.4" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" + ["psm2"]="11.2.185" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" + ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/latest b/scripts/profiles/latest index f76f9131742ee389211cdafc141553a15268098f..b3ec1638fda741e3680a70960377c66e3b41c4c7 120000 --- a/scripts/profiles/latest +++ b/scripts/profiles/latest @@ -1 +1 @@ -0.9.2 \ No newline at end of file +0.9.3 \ No newline at end of file diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index b659f38fbc0f4f6a0cd194047dbf381940285f77..eec3277d6e3ae9539e67494f760b674059075f0c 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -39,7 +39,7 @@ sources=( ["argobots"]="https://github.com/pmodels/argobots/archive/v{{VERSION}}.tar.gz" ["rocksdb"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" ["rocksdb%experimental"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" - ["psm2"]="https://github.com/intel/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" + ["psm2"]="https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" ["bmi"]="https://github.com/radix-io/bmi/" ["libfabric"]="https://github.com/ofiwg/libfabric.git" ["libfabric%experimental"]="https://github.com/ofiwg/libfabric.git" @@ -53,6 +53,7 @@ sources=( ["parallax"]="https://github.com/CARV-ICS-FORTH/parallax.git" ["prometheus-cpp"]="https://github.com/jupp0r/prometheus-cpp/releases/download/{{VERSION}}/prometheus-cpp-with-submodules.tar.gz" ["curl"]="https://curl.se/download/curl-{{VERSION}}.tar.gz" + ["ucx"]="https://github.com/openucx/ucx/archive/{{VERSION}}.tar.gz" ) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0108fb9c9fa5b8a543fcf0170dca8422c9941554..a4827af37640c1142813a95182c187e23ffe5be1 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -31,13 +31,33 @@ # This builds the `libgkfs_intercept.so` library: the primary GekkoFS client # based on syscall interception. # ############################################################################## -add_library(gkfs_intercept SHARED) -target_sources( - gkfs_intercept - PRIVATE gkfs_functions.cpp +add_library (gkfs_intercept SHARED) +add_library (gkfs_user_lib SHARED) + + +target_sources(gkfs_intercept + PRIVATE gkfs_functions.cpp + intercept.cpp hooks.cpp + logging.cpp + open_file_map.cpp + open_dir.cpp + path.cpp + preload.cpp + preload_context.cpp + preload_util.cpp + rpc/rpc_types.cpp + rpc/forward_data.cpp + rpc/forward_management.cpp + rpc/forward_metadata.cpp + syscalls/detail/syscall_info.c) + +target_sources( + gkfs_user_lib + PRIVATE gkfs_functions.cpp intercept.cpp + hooks.cpp logging.cpp open_file_map.cpp open_dir.cpp @@ -49,19 +69,34 @@ target_sources( rpc/forward_data.cpp rpc/forward_management.cpp rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c + syscalls/detail/syscall_info.c syscalls/util.S ) +target_compile_definitions(gkfs_user_lib PUBLIC BYPASS_SYSCALL) + +if(GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) +endif() + target_link_libraries( gkfs_intercept PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl + PUBLIC dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads + Syscall_intercept::Syscall_intercept +) + +target_link_libraries( + gkfs_user_lib + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC dl Mercury::Mercury hermes fmt::fmt Threads::Threads - Date::TZ ) install( @@ -71,54 +106,17 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -# ############################################################################## -# This builds the `libgkfwd_intercept.so` library: the client for GekkoFS' -# forwarding mode based on syscall interception. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - add_library(gkfwd_intercept SHARED) - - target_sources( - gkfwd_intercept - PRIVATE gkfs_functions.cpp - hooks.cpp - intercept.cpp - logging.cpp - open_file_map.cpp - open_dir.cpp - path.cpp - preload.cpp - preload_context.cpp - preload_util.cpp - rpc/rpc_types.cpp - rpc/forward_data.cpp - rpc/forward_management.cpp - rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c - ) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_FORWARDING) - if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_AGIOS) - endif() +set_target_properties(gkfs_user_lib + PROPERTIES + PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" + PUBLIC_HEADER "../../include/client/user_functions.hpp" +) - target_link_libraries( - gkfwd_intercept - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads - Date::TZ - ) - install( - TARGETS gkfwd_intercept - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs - ) -endif() +install( + TARGETS gkfs_user_lib + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs +) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index b03e8eaebe906bf24ffe9fdaadec01799541a4da..865d382143ccdb080a752a07a2749af680ded3ef 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1394,6 +1394,28 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, return written; } +/** + * @brief Closes an fd. To be used externally + * + * @param fd + * @return int + */ +int +gkfs_close(unsigned int fd) { + if(CTX->file_map()->exist(fd)) { + // No call to the daemon is required + CTX->file_map()->remove(fd); + return 0; + } + + if(CTX->is_internal_fd(fd)) { + // the client application (for some reason) is trying to close an + // internal fd: ignore it + return 0; + } + + return -1; +} #ifdef HAS_SYMLINKS #ifdef GKFS_ENABLE_UNUSED_FUNCTIONS @@ -1482,6 +1504,28 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize) { #endif #endif + +std::vector +gkfs_get_file_list(const std::string& path) { + auto ret = gkfs::rpc::forward_get_dirents(path); + auto err = ret.first; + if(err) { + errno = err; + return {}; + } + + auto open_dir = ret.second; + + std::vector file_list; + unsigned int pos = 0; + + while(pos < open_dir->size()) { + auto de = open_dir->getdent(pos++); + file_list.push_back(de.name()); + } + return file_list; +} + } // namespace gkfs::syscall diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 56e1f829b3319d2a91d21504a756639a927b1a35..9a54048c544dc1bfbcf096a448b913e52114ad0b 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -91,17 +91,10 @@ hook_close(int fd) { LOG(DEBUG, "{}() called with fd: {}", __func__, fd); - if(CTX->file_map()->exist(fd)) { - // No call to the daemon is required - CTX->file_map()->remove(fd); - return 0; - } + auto ret = gkfs::syscall::gkfs_close(fd); - if(CTX->is_internal_fd(fd)) { - // the client application (for some reason) is trying to close an - // internal fd: ignore it + if(ret == 0) return 0; - } return syscall_no_intercept_wrapper(SYS_close, fd); } @@ -878,7 +871,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, "newname \"{}\", flags {}", __func__, olddfd, oldname, newdfd, newname, flags); - const char* oldpath_pass; + const char* oldpath_pass = oldname; std::string oldpath_resolved; auto oldpath_status = CTX->relativize_fd_path(olddfd, oldname, oldpath_resolved); @@ -902,7 +895,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, return -EINVAL; } - const char* newpath_pass; + const char* newpath_pass = newname; std::string newpath_resolved; auto newpath_status = CTX->relativize_fd_path(newdfd, newname, newpath_resolved); diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 316c5e398403761b00d5f32917f153cca9c22c7b..7a8a455fe4aa63f6f3eab2bec381023dfbb888f0 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -44,6 +44,24 @@ extern "C" { #include } + +#ifdef BYPASS_SYSCALL +int (*intercept_hook_point)(long syscall_number, long arg0, long arg1, + long arg2, long arg3, long arg4, long arg5, + long* result){}; + +void (*intercept_hook_point_clone_child)(unsigned long flags, void* child_stack, + int* ptid, int* ctid, long newtls){}; + +void (*intercept_hook_point_clone_parent)(unsigned long flags, + void* child_stack, int* ptid, + int* ctid, long newtls, + long returned_pid){}; + +void (*intercept_hook_point_post_kernel)(long syscall_number, long arg0, + long arg1, long arg2, long arg3, + long arg4, long arg5, long result){}; +#endif namespace { thread_local bool reentrance_guard_flag; diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 88f9947140e070bbca5abc8f6410e7a0fd942588..830d5691ecd281c9974bc98a5ca299935fdc5b01 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -32,11 +32,7 @@ #include #include #include - -extern "C" { -#include -#include -} +#include #ifdef GKFS_ENABLE_LOGGING @@ -289,8 +285,7 @@ logger::logger(const std::string& opts, const std::string& path, , const std::string& filter, int verbosity #endif - ) - : timezone_(nullptr) { +) { /* use stderr by default */ log_fd_ = 2; @@ -335,34 +330,6 @@ logger::logger(const std::string& opts, const std::string& path, log_fd_ = fd; } - // Finding the current timezone implies accessing OS files (i.e. syscalls), - // but current_zone() doesn't actually retrieve the time zone but rather - // provides a descriptor to it that is **atomically initialized** upon its - // first use. Thus, if we don't force the initialization here, logging the - // first intercepted syscall will produce a call to date::time_zone::init() - // (under std::call_once) which internally ends up calling fopen(). Since - // fopen() ends up calling sys_open(), we will need to generate another - // timestamp for a system call log entry, which will attempt to call - // date::time_zone::init() since the prior initialization (under the same - // std::call_once) has not yet completed. - // - // Unfortunately, date::time_zone doesn't provide a function to prevent - // this lazy initialization, therefore we force it by requesting - // information from an arbitrary timepoint (January 1st 1970) which forces - // the initialization. This doesn't do any actual work and could safely be - // removed if the date API ends up providing this functionality. - try { - timezone_ = date::current_zone(); -#ifdef GKFS_DEBUG_BUILD - using namespace date; - timezone_->get_info(date::sys_days{January / 1 / 1970}); -#endif // GKFS_DEBUG_BUILD - } catch(const std::exception& ex) { - // if timezone initialization fails, setting timezone_ to nullptr - // makes format_timestamp_to() default to producing epoch timestamps - timezone_ = nullptr; - } - #ifdef GKFS_ENABLE_LOGGING const auto log_hermes_message = [](const std::string& msg, hermes::log::level l, int severity, @@ -490,9 +457,9 @@ logger::log_syscall(syscall::info info, const long syscall_number, print_syscall: - static_buffer buffer; + fmt::basic_memory_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); + detail::format_timestamp_to(buffer); detail::format_syscall_info_to(buffer, info); if(result) { @@ -501,7 +468,7 @@ print_syscall: syscall::decode(buffer, syscall_number, args); } - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), "\n"); ::syscall_no_intercept(SYS_write, log_fd_, buffer.data(), buffer.size()); } diff --git a/src/client/path.cpp b/src/client/path.cpp index 4f72b4807c3cc09da756ab769aebe89cd9871e05..85357209a24ae47a87ed8358cd8d3cad255f232a 100644 --- a/src/client/path.cpp +++ b/src/client/path.cpp @@ -39,9 +39,14 @@ #include #include +#ifndef BYPASS_SYSCALL +#include +#else +#include +#endif + extern "C" { #include -#include } using namespace std; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4de0df2c31168fe6f07a434a59e0eeb2331be27c..652ef3580df7720c4103e12862a4f66265aa1217 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -50,13 +50,13 @@ std::unique_ptr ld_network_service; // extern variable namespace { -#ifdef GKFS_ENABLE_FORWARDING +// FORWARDING pthread_t mapper; bool forwarding_running; pthread_mutex_t remap_mutex; pthread_cond_t remap_signal; -#endif +// END FORWARDING inline void exit_error_msg(int errcode, const string& msg) { @@ -67,8 +67,10 @@ exit_error_msg(int errcode, const string& msg) { // if we don't disable interception before calling ::exit() // syscall hooks may find an inconsistent in shared state // (e.g. the logger) and thus, crash - gkfs::preload::stop_interception(); - CTX->disable_interception(); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + } ::exit(errcode); } @@ -102,7 +104,6 @@ init_hermes_client() { return true; } -#ifdef GKFS_ENABLE_FORWARDING void* forwarding_mapper(void* p) { struct timespec timeout; @@ -115,7 +116,7 @@ forwarding_mapper(void* p) { try { gkfs::utils::load_forwarding_map(); - if(previous != CTX->fwd_host_id()) { + if(previous != (int64_t) CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); previous = CTX->fwd_host_id(); @@ -133,18 +134,14 @@ forwarding_mapper(void* p) { return nullptr; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } -#endif -#ifdef GKFS_ENABLE_FORWARDING void destroy_forwarding_mapper() { forwarding_running = false; @@ -153,7 +150,6 @@ destroy_forwarding_mapper() { pthread_join(mapper, NULL); } -#endif void log_prog_name() { @@ -207,31 +203,34 @@ init_environment() { } /* Setup distributor */ -#ifdef GKFS_ENABLE_FORWARDING - try { - gkfs::utils::load_forwarding_map(); + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch(std::exception& e) { - exit_error_msg( - EXIT_FAILURE, - fmt::format("Unable set the forwarding host '{}'", e.what())); - } + if(!forwarding_map_file.empty()) { + try { + gkfs::utils::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch(std::exception& e) { + exit_error_msg(EXIT_FAILURE, + fmt::format("Unable set the forwarding host '{}'", + e.what())); + } + + auto forwarder_dist = std::make_shared( + CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + } else { - auto forwarder_dist = std::make_shared( - CTX->fwd_host_id(), CTX->hosts().size()); - CTX->distributor(forwarder_dist); -#else #ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #else - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #endif - CTX->distributor(distributor); -#endif - + CTX->distributor(distributor); + } LOG(INFO, "Retrieving file system configuration..."); @@ -290,9 +289,11 @@ init_preload() { CTX->unprotect_user_fds(); -#ifdef GKFS_ENABLE_FORWARDING - init_forwarding_mapper(); -#endif + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(!forwarding_map_file.empty()) { + init_forwarding_mapper(); + } gkfs::preload::start_interception(); errno = oerrno; @@ -304,19 +305,53 @@ init_preload() { */ void destroy_preload() { -#ifdef GKFS_ENABLE_FORWARDING - destroy_forwarding_mapper(); -#endif - + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(!forwarding_map_file.empty()) { + destroy_forwarding_mapper(); + } CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); ld_network_service.reset(); LOG(DEBUG, "RPC subsystem shut down"); - gkfs::preload::stop_interception(); - CTX->disable_interception(); - LOG(DEBUG, "Syscall interception stopped"); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + LOG(DEBUG, "Syscall interception stopped"); + } LOG(INFO, "All subsystems shut down. Client shutdown complete."); } + + +/** + * @brief External functions to call linking the library + * + */ +extern "C" int +gkfs_init() { + CTX->init_logging(); + + // from here ownwards it is safe to print messages + LOG(DEBUG, "Logging subsystem initialized"); + + gkfs::preload::init_environment(); + + return 0; +} + + +extern "C" int +gkfs_end() { + CTX->clear_hosts(); + LOG(DEBUG, "Peer information deleted"); + + ld_network_service.reset(); + LOG(DEBUG, "RPC subsystem shut down"); + + LOG(INFO, "All subsystems shut down. Client shutdown complete."); + + return 0; +} \ No newline at end of file diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 7101041dcdb77864a55ed671bdd014a250db5623..5dbc75c87c0944fb14190f4464578baa4c5483cc 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -37,13 +37,17 @@ #include #include #include - #include #include -extern "C" { +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + +extern "C" { #include } diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index aa09d0e0727d69e509aea64a1133063bc8db3e71..ed9e2368755485c09f7eefc8ccd5db46bfbab285 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -286,7 +286,6 @@ metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md, return 0; } -#ifdef GKFS_ENABLE_FORWARDING map load_forwarding_map_file(const std::string& lfpath) { @@ -320,9 +319,7 @@ load_forwarding_map_file(const std::string& lfpath) { } return forwarding_map; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void load_forwarding_map() { string forwarding_map_file; @@ -359,7 +356,6 @@ load_forwarding_map() { CTX->fwd_host_id(forwarding_map[local_hostname]); } -#endif vector> read_hosts_file() { diff --git a/src/client/rpc/forward_management.cpp b/src/client/rpc/forward_management.cpp index c975842e560d15a9f107b3138ba1016b853071ed..7504b1f6b6bb869d99faab62c946a620ecace192 100644 --- a/src/client/rpc/forward_management.cpp +++ b/src/client/rpc/forward_management.cpp @@ -47,7 +47,7 @@ forward_get_fs_config() { bool found = false; size_t idx = 0; - while(!found && idx <= CTX->hosts().size()) { + while(!found && idx < CTX->hosts().size()) { try { LOG(DEBUG, "Retrieving file system configurations from daemon"); // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that diff --git a/src/client/syscalls/util.S b/src/client/syscalls/util.S new file mode 100644 index 0000000000000000000000000000000000000000..67c9b279afdcad35a0409dd800c392f079029956 --- /dev/null +++ b/src/client/syscalls/util.S @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +.global syscall_no_intercept; +.type syscall_no_intercept, @function + +.text + +syscall_no_intercept: + movq %rdi, %rax /* convert from linux ABI calling */ + movq %rsi, %rdi /* convention to syscall calling convention */ + movq %rdx, %rsi + movq %rcx, %rdx + movq %r8, %r10 + movq %r9, %r8 + movq 8(%rsp), %r9 + syscall + ret + +.size syscall_no_intercept, .-syscall_no_intercept diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 95c4ade7542ead463ea3565842481a82738fa102..89af71c7624109df1d90b7c4bae1b29fd030b193 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -72,63 +72,15 @@ target_link_libraries( Threads::Threads ) +if(GKFS_ENABLE_AGIOS) + target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp) + target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS) + target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS) +endif() + if(GKFS_ENABLE_CODE_COVERAGE) target_code_coverage(gkfs_daemon AUTO) endif() install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -# ############################################################################## -# This builds the `gkfwd_daemon` executable: the daemon for GekkoFS data -# forwarding mode. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - - add_executable(gkfwd_daemon) - - # We need to add here any files that may have different compile definitions - target_sources( - gkfwd_daemon - PRIVATE ../common/rpc/rpc_util.cpp - daemon.cpp - util.cpp - ops/metadentry.cpp - ops/data.cpp - classes/fs_data.cpp - classes/rpc_data.cpp - handler/srv_metadata.cpp - handler/srv_management.cpp - handler/srv_data.cpp - ) - - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_FORWARDING) - target_link_libraries( - gkfwd_daemon - PUBLIC # internal libs - metadata - metadata_backend - storage - distributor - statistics - log_util - env_util - path_util - # external libs - CLI11::CLI11 - fmt::fmt - Mercury::Mercury - Argobots::Argobots - Margo::Margo - # others - Threads::Threads - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PRIVATE scheduler/agios.cpp) - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_AGIOS) - target_link_libraries(gkfwd_daemon PRIVATE AGIOS::AGIOS) - endif() - - install(TARGETS gkfwd_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif () diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 0d38c718c5af2b76b9c8be8414088a4239ee1c52..33b43bf5942789507cc951f737ac1da2c7188a9e 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -256,6 +256,15 @@ FsData::enable_chunkstats(bool enable_chunkstats) { FsData::enable_chunkstats_ = enable_chunkstats; } +bool +FsData::enable_forwarding() const { + return enable_forwarding_; +} + +void +FsData::enable_forwarding(bool enable_forwarding) { + FsData::enable_forwarding_ = enable_forwarding; +} bool FsData::enable_prometheus() const { return enable_prometheus_; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49a87fa604e8b6d747c6d3e9dd19b66f9ee27211..2911c69219201b1100da85418dc4b8f8a422b9ff 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -261,25 +261,6 @@ init_environment() { throw; } - GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ", __func__); - try { -#ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared(); -#else - auto distributor = std::make_shared(); -#endif - RPC_DATA->distributor(distributor); - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to initialize Distributor: {}", __func__, - e.what()); - throw; - } - -#ifdef GKFS_ENABLE_FORWARDING - GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__); -#endif - #ifdef GKFS_ENABLE_AGIOS // Initialize AGIOS scheduler GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'", @@ -552,10 +533,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { assert(desc.count("--rootdir")); auto rootdir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - // In forwarding mode, the backend is shared - auto rootdir_path = fs::path(rootdir); -#else auto rootdir_path = fs::path(rootdir); if(desc.count("--rootdir-suffix")) { if(opts.rootdir_suffix == gkfs::config::data::chunk_dir || @@ -570,7 +547,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { rootdir_path /= opts.rootdir_suffix; GKFS_DATA->rootdir_suffix(opts.rootdir_suffix); } -#endif if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) @@ -589,14 +565,23 @@ parse_input(const cli_options& opts, const CLI::App& desc) { fs::create_directories(rootdir_path); GKFS_DATA->rootdir(rootdir_path.native()); + if(desc.count("--enable-forwarding")) { + GKFS_DATA->enable_forwarding(true); + GKFS_DATA->spdlogger()->info("{}() Forwarding mode enabled", __func__); + } + if(desc.count("--metadir")) { auto metadir = opts.metadir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); -#else + auto metadir_path = fs::path(metadir); -#endif + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); + } + + if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...", @@ -613,13 +598,16 @@ parse_input(const cli_options& opts, const CLI::App& desc) { // use rootdir as metadata dir auto metadir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); - fs::create_directories(metadir_path); - GKFS_DATA->metadir(fs::canonical(metadir_path).native()); -#else - GKFS_DATA->metadir(GKFS_DATA->rootdir()); -#endif + + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + auto metadir_path = + fs::path(metadir) / fmt::format_int(getpid()).str(); + fs::create_directories(metadir_path); + GKFS_DATA->metadir(fs::canonical(metadir_path).native()); + } else + GKFS_DATA->metadir(GKFS_DATA->rootdir()); } if(desc.count("--dbbackend")) { @@ -787,6 +775,9 @@ main(int argc, const char* argv[]) { desc.add_option( "--output-stats", opts.stats_file, "Creates a thread that outputs the server stats each 10s to the specified file."); + desc.add_flag( + "--enable-forwarding", + "Enables forwarding mode, so the metadata is stored in a separate directory (pid)."); #ifdef GKFS_ENABLE_PROMETHEUS desc.add_flag( "--enable-prometheus", diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 9a4e1697355d95c535e74ad44a73ad924de39c9d..d28d8ba2c7a5cabba2f35b8f40869290f9a27179 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -109,7 +109,7 @@ rpc_srv_write(hg_handle_t handle) { return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug( "{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", @@ -414,7 +414,7 @@ rpc_srv_read(hg_handle_t handle) { return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug( diff --git a/src/daemon/handler/srv_management.cpp b/src/daemon/handler/srv_management.cpp index 585646aeddee5c3094a5573b3fd3cd51877c1af7..ef5f8735d0fdcccb4eebda2690ce6756eca6b8d6 100644 --- a/src/daemon/handler/srv_management.cpp +++ b/src/daemon/handler/srv_management.cpp @@ -72,8 +72,9 @@ rpc_srv_get_fs_config(hg_handle_t handle) { out.blocks_state = static_cast(GKFS_DATA->blocks_state()); out.uid = getuid(); out.gid = getgid(); - GKFS_DATA->spdlogger()->debug("{}() Sending output configs back to library", - __func__); + GKFS_DATA->spdlogger()->debug( + "{}() Sending output configs back to library. mountdir '{}' rootdir '{}'", + __func__, out.mountdir, out.rootdir); auto hret = margo_respond(handle, &out); if(hret != HG_SUCCESS) { GKFS_DATA->spdlogger()->error( diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp index e68069797fc8104894399f10a38ee25d323234b8..1339170b94aa61072451c759f1f1b8b1ba849afd 100644 --- a/src/daemon/handler/srv_metadata.cpp +++ b/src/daemon/handler/srv_metadata.cpp @@ -537,7 +537,7 @@ rpc_srv_get_dirents(hg_handle_t handle) { // Retrieve size of source buffer auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ", __func__, in.path, bulk_size); @@ -696,7 +696,7 @@ rpc_srv_get_dirents_extended(hg_handle_t handle) { // Retrieve size of source buffer auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ", __func__, in.path, bulk_size); diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index bc8e0c86427bf14538ce3388b95755f78cabceac..4bc63e41d3590bae7eb7c88070bf9c7362c8b81f 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -165,7 +165,7 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): lines = f.readlines() for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] assert ion == '0' @@ -190,15 +190,16 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): ret = c00.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of read bytes - + + # log line should go to the end of the file with open(c00.log) as f: lines = f.readlines() - + ion = 0 for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] - assert ion == '1' + assert ion == '1' def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): """Write files from one client and read in the other using two daemons""" diff --git a/tests/integration/harness/gkfs.io/serialize.hpp b/tests/integration/harness/gkfs.io/serialize.hpp index 3873b2ec85d484a52f748820f150738ea9a938ec..92328fcf86dc53b3d71d3acfa63a4ec5de4ca72a 100644 --- a/tests/integration/harness/gkfs.io/serialize.hpp +++ b/tests/integration/harness/gkfs.io/serialize.hpp @@ -178,13 +178,10 @@ template <> struct adl_serializer { static void to_json(json& j, const struct ::statfs opt) { - j = json{{"f_type", opt.f_type}, - {"f_bsize", opt.f_bsize}, - {"f_blocks", opt.f_blocks}, - {"f_bfree", opt.f_bfree}, - {"f_bavail", opt.f_bavail}, - {"f_files", opt.f_files}, - {"f_ffree", opt.f_ffree}}; + j = json{{"f_type", opt.f_type}, {"f_bsize", opt.f_bsize}, + {"f_blocks", opt.f_blocks}, {"f_bfree", opt.f_bfree}, + {"f_bavail", opt.f_bavail}, {"f_files", opt.f_files}, + {"f_ffree", opt.f_ffree}}; } }; @@ -253,7 +250,8 @@ struct formatter { template auto - format(const struct ::dirent& dirent, FormatContext& ctx) { + format(const struct ::dirent& dirent, FormatContext& ctx) const + -> format_context::iterator { return format_to(ctx.out(), "struct dirent {{\n" " d_ino = {};\n" diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index ef48c404255306f5301ea029c4c189feb7e10c20..a65a171f7b7ad50f5129f2d27ebe196e7d729088 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -48,15 +48,16 @@ gkfs_client_log_level = 'all' gkfs_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' -gkfwd_daemon_cmd = 'gkfwd_daemon' +gkfwd_daemon_cmd = 'gkfs_daemon' gkfwd_client_cmd = 'gkfs.io' -gkfwd_client_lib_file = 'libgkfwd_intercept.so' +gkfwd_client_lib_file = 'libgkfs_intercept.so' gkfwd_hosts_file = 'gkfs_hosts.txt' gkfwd_forwarding_map_file = 'gkfs_forwarding.map' -gkfwd_daemon_log_file = 'gkfwd_daemon.log' +gkfwd_daemon_log_file = 'gkfs_daemon.log' gkfwd_daemon_log_level = '100' -gkfwd_client_log_file = 'gkfwd_client.log' +gkfwd_client_log_file = 'gkfs_client.log' gkfwd_client_log_level = 'all' +gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' @@ -520,7 +521,8 @@ class ShellClient: 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -732,7 +734,8 @@ class FwdDaemon: args = [ '--mountdir', self.mountdir, '--metadir', self.metadir, '--rootdir', self.rootdir, - '-l', self._address ] + '-l', self._address, + '--enable-forwarding'] logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -902,11 +905,12 @@ class FwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'LD_PRELOAD' : self._preload_library, + 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -996,7 +1000,8 @@ class ShellFwdClient: 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file), 'LIBGKFS_LOG' : gkfwd_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) diff --git a/tests/scripts/compile_dep.sh/0.9.3/agios.out b/tests/scripts/compile_dep.sh/0.9.3/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..25a1cad3a58a9c7c07c75ba87bfa2c5a6a9643ca --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/agios.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/all.out b/tests/scripts/compile_dep.sh/0.9.3/all.out new file mode 100644 index 0000000000000000000000000000000000000000..2fcd55b6f7dc5e574eeea0233a628e75ea105f4e --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/all.out @@ -0,0 +1,14 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### +Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.3/arm.out b/tests/scripts/compile_dep.sh/0.9.3/arm.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/arm.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ci.out b/tests/scripts/compile_dep.sh/0.9.3/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..9ec3c24bb12c16078d2b3869831331de15f5e305 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ci.out @@ -0,0 +1,8 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/default.out b/tests/scripts/compile_dep.sh/0.9.3/default.out new file mode 100644 index 0000000000000000000000000000000000000000..ca393e65d3bc03aa72fe6a628efab5dad85174b9 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/default.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/direct.out b/tests/scripts/compile_dep.sh/0.9.3/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..816266773b569d3d098245c7b868f7983753ff12 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/direct.out @@ -0,0 +1,6 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..ff149d28f1874e6fe4f94484c4f0b4f7abf49416 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,9 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric%verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/mogon2.out b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ngio.out b/tests/scripts/compile_dep.sh/0.9.3/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ngio.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/p9.out b/tests/scripts/compile_dep.sh/0.9.3/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/p9.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out index 7ba8e6778a49f2f7ebaf516c6143f2dd095adef8..8369fe83959bf552ffa03f984ee038b6c55baaed 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/default.out +++ b/tests/scripts/dl_dep.sh/0.8.0/default.out @@ -12,5 +12,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocks Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out index d8af0a79b2bed070d64eb40b2751a06fd5040207..d5844433bc81445c690efc0e30054a331000b4ce 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out @@ -10,4 +10,4 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ngio.out b/tests/scripts/dl_dep.sh/0.8.0/ngio.out index f0c9f79592b10c1307d92dbafc560a5914c5974d..e6cd7bdcc2d1c625d552b6c85b256500df133683 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.8.0/ngio.out @@ -8,6 +8,6 @@ Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/ngio.out b/tests/scripts/dl_dep.sh/0.9.0/ngio.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.0/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/all.out b/tests/scripts/dl_dep.sh/0.9.1/all.out index 35b6b521748a2bb789559c4df780c438d345b7b8..9cbff6342eb7dba75c4f9e617fddacfd446b13ff 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/all.out +++ b/tests/scripts/dl_dep.sh/0.9.1/all.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/ngio.out b/tests/scripts/dl_dep.sh/0.9.1/ngio.out index 5185b0db8b3c3db0f8b07d19e7b32294dd9f5f69..b90a686dd2d27a1390b6ace8ede947f58359fd91 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.1/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out index e76b1788216666548e599631b3485b7348908203..2a2aaf6c2da6be99551a9eba350ae560644ebe51 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/p9.out b/tests/scripts/dl_dep.sh/0.9.1/p9.out index 3a99bb523089fa9d7aaa1a36a4a62fa9d6295732..b1eee01ba618b62af568de183d0d094947586123 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/p9.out +++ b/tests/scripts/dl_dep.sh/0.9.1/p9.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/all.out b/tests/scripts/dl_dep.sh/0.9.2/all.out index 4a1ab9e6f6de6a14f9e7da7c7792bd01e6171c48..4b5a73fda0b5c2962d25cc0b7d5c786cdf5adb5b 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/all.out +++ b/tests/scripts/dl_dep.sh/0.9.2/all.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/arm.out b/tests/scripts/dl_dep.sh/0.9.2/arm.out index 2c61edb9c437445a2347de92fdd11a66ac304576..8725e5892b82e32c3e2e8b15d7f805f72c09cc88 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/arm.out +++ b/tests/scripts/dl_dep.sh/0.9.2/arm.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/ngio.out b/tests/scripts/dl_dep.sh/0.9.2/ngio.out index 657154fdc1ec4e756f29281558459fa9662bd83d..fe6f4afecaecbda83b3ddc3a23e23a6ed5a71e8a 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.2/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out index d26b498ca41977f2e48d86f1f0437c1bc0b7a563..96e22237b1da0070350c22e2809d62a69dcd24d3 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/p9.out b/tests/scripts/dl_dep.sh/0.9.2/p9.out index 3a99bb523089fa9d7aaa1a36a4a62fa9d6295732..b1eee01ba618b62af568de183d0d094947586123 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/p9.out +++ b/tests/scripts/dl_dep.sh/0.9.2/p9.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.3/agios.out b/tests/scripts/dl_dep.sh/0.9.3/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..8d8e2486b502394977608f89057d0479487e736d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/agios.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/all.out b/tests/scripts/dl_dep.sh/0.9.3/all.out new file mode 100644 index 0000000000000000000000000000000000000000..dbd2183188053b13c59cfb0dbb93e38d96f482d6 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/all.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/arm.out b/tests/scripts/dl_dep.sh/0.9.3/arm.out new file mode 100644 index 0000000000000000000000000000000000000000..7b1d7c5fcb554b36e7e8163f4f9c751c28628534 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/arm.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[fb73c3924b502e2840a5dc8a18746e395b06a8a6]' and flags '' +Done diff --git a/tests/scripts/dl_dep.sh/0.9.3/ci.out b/tests/scripts/dl_dep.sh/0.9.3/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..0754a2860f268f916db2cc7ab121e1737b6b5705 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ci.out @@ -0,0 +1,9 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/default.out b/tests/scripts/dl_dep.sh/0.9.3/default.out new file mode 100644 index 0000000000000000000000000000000000000000..7b7330872c4578763b697f7394fb9d49f872f11d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/default.out @@ -0,0 +1,10 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/direct.out b/tests/scripts/dl_dep.sh/0.9.3/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..42a6d15450c591a23273ec6201faf713235fab58 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/direct.out @@ -0,0 +1,7 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..f2c3f6d3b2c69b9e934055bec1ef1cbfc2a20f63 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,10 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/mogon2.out b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/ngio.out b/tests/scripts/dl_dep.sh/0.9.3/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..f2bc30bb74f594cd70caf11e13398ea1eb859200 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ngio.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..8ef7035e06cb383d1d826f0354d222e9bdcb48b8 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/p9.out b/tests/scripts/dl_dep.sh/0.9.3/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..1c939fc2332d564471238d6941f319b748348f7a --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/p9.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f]' and flags '' +Done