diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68eaca4ed3a2a944e32510b744c9294bf0e4d417..19c076f05914df9170c43104dda58ca13eae5890 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.2 +image: gekkofs/core:0.9.3 ################################################################################ ## Validating ################################################################################ check format: stage: lint - image: gekkofs/linter:0.9.2 + image: gekkofs/linter:0.9.3 needs: [] script: - ${SCRIPTS_DIR}/check_format.sh @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.2 + image: gekkofs/deps:0.9.3 interruptible: true needs: [] script: @@ -65,23 +65,6 @@ gkfs: - ${INSTALL_PATH} expire_in: 1 week -gkfwd: - stage: build - image: gekkofs/deps:0.9.2 - interruptible: true - needs: [] - script: - - cmake --preset ci-forwarding-coverage - - cmake --build ${BUILD_PATH} -j $(nproc) --target install - # reduce artifacts size - - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH} - artifacts: - paths: - - ${BUILD_PATH} - - ${INSTALL_PATH} - expire_in: 1 week - - ################################################################################ ## Testing ################################################################################ @@ -89,7 +72,7 @@ gkfwd: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: [] script: - mkdir -p ${BUILD_PATH}/tests/scripts @@ -105,7 +88,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true needs: ['gkfs'] parallel: @@ -154,14 +137,14 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true - needs: ['gkfwd'] + needs: ['gkfs'] parallel: matrix: - SUBTEST: [ forwarding ] - rules: - - when: never + # rules: + # - when: never script: ## run tests @@ -204,7 +187,7 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: ['gkfs'] script: ## Add path to mkfs.kreon @@ -242,7 +225,7 @@ gkfs:unit: ################################################################################ documentation: stage: docs - image: gekkofs/docs:0.9.2 + image: gekkofs/docs:0.9.3 needs: [] rules: # we only build the documentation automatically if we are on the @@ -272,9 +255,9 @@ documentation: ## == coverage baseline ==================== coverage:baseline: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true - needs: ['gkfs', 'gkfwd'] + needs: ['gkfs'] script: ## capture initial coverage information to establish a baseline @@ -298,10 +281,10 @@ coverage:baseline: coverage: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 # needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', # 'gkfs:unit' ] - needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ] + needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] script: - cd ${CI_PROJECT_DIR} - cmake diff --git a/.gitmodules b/.gitmodules index 18f937ae7b811d4f78fc6fce7f89600199f28abd..31675fc19786aac2ecbafec75dd4daf8e39a3f4b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "external/hermes"] path = external/hermes - url = https://github.com/bsc-ssrg/hermes.git + url = https://github.com/gekkofs/hermes.git [submodule "tests/scripts/bats"] path = tests/scripts/bats url = https://github.com/bats-core/bats-core.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ae0762a2456b0efd8172251cd3a3d5d079d7a8..93e0446d56f634b8b9652c0ce5839d69c6593ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). - Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. - Added reattemp support in get_fs_config to other servers, when the initial server fails. +- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable +`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). +- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)). ### New ### Changed diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake index 30da5afc387621004040ca75bfc3f43630722189..d900cdee1c3eccebb39dd75e81dcedc8c8add2f1 100644 --- a/CMake/gkfs-options.cmake +++ b/CMake/gkfs-options.cmake @@ -348,14 +348,6 @@ gkfs_define_option( # I/O forwarding ################################################################################ -## Forwarding support -gkfs_define_option( - GKFS_ENABLE_FORWARDING - HELP_TEXT "Enable I/O forwarding mode" - DEFAULT_VALUE OFF - DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer" -) - ## Scheduling in I/O forwarding mode gkfs_define_option( GKFS_ENABLE_AGIOS diff --git a/CMakeLists.txt b/CMakeLists.txt index afdda9c02ef8e0137d8bca31f8b2bab90bb8cce2..c70d36115a34d91e11f0606ccf579efc8350c182 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,10 +160,6 @@ find_package(Margo 0.9.6 REQUIRED) message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept") find_package(Syscall_intercept REQUIRED) -### Date (required by the libgkfs_intercept.so) -message(STATUS "[${PROJECT_NAME}] Checking for Date") -find_package(Date REQUIRED) - ### AGIOS: required for scheduling I/O requests if (GKFS_ENABLE_AGIOS) message(STATUS "[${PROJECT_NAME}] Checking for Agios") @@ -225,7 +221,7 @@ include_from_source(fmt MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG f94b7364b9409f05207c3af3fa4666730e11a854 # v6.1.2 + GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0 ) # ensure that fmt is linked as PIC @@ -338,11 +334,7 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}") message(STATUS "[gekkofs] Check for forwarding tests...") - if (GKFS_ENABLE_FORWARDING) - set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - else () - set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - endif () + set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}") message(STATUS "[gekkofs] Check for guided distributor tests...") if (GKFS_USE_GUIDED_DISTRIBUTION) diff --git a/CMakePresets.json b/CMakePresets.json index 9f12e9c734ceabc71595221f384db23c67ff7bf9..da7bf79b3ef83ca3a0f51bcc014d255dccd8ece6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -27,14 +27,6 @@ "deprecated": true } }, - { - "name": "forwarding", - "inherits": "default", - "hidden": true, - "cacheVariables": { - "GKFS_ENABLE_FORWARDING": true - } - }, { "name": "release", "hidden": true, @@ -113,30 +105,6 @@ "release" ] }, - { - "name": "forwarding-debug", - "displayName": "Forwarding gekkofs (debug)", - "inherits": [ - "forwarding", - "debug" - ] - }, - { - "name": "forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage)", - "inherits": [ - "forwarding", - "coverage" - ] - }, - { - "name": "forwarding-release", - "displayName": "Forwarding gekkofs (release)", - "inherits": [ - "forwarding", - "release" - ] - }, { "name": "ci-debug", "displayName": "Default gekkofs (debug, CI flags)", @@ -171,33 +139,6 @@ "default", "release" ] - }, - { - "name": "ci-forwarding-debug", - "displayName": "Forwarding gekkofs (debug, CI flags)", - "inherits": [ - "ci", - "forwarding", - "debug" - ] - }, - { - "name": "ci-forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage, CI flags)", - "inherits": [ - "ci", - "forwarding", - "coverage" - ] - }, - { - "name": "ci-forwarding-release", - "displayName": "Forwarding gekkofs (release, CI flags)", - "inherits": [ - "ci", - "forwarding", - "release" - ] } ] } diff --git a/docker/0.9.3/core/Dockerfile b/docker/0.9.3/core/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a55434924b2cfe720195c371de6736ac049826f7 --- /dev/null +++ b/docker/0.9.3/core/Dockerfile @@ -0,0 +1,40 @@ +FROM debian:bullseye-slim + +LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + ninja-build \ + procps \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ + # Margo dependencies \ + libjson-c-dev \ + # RocksDB dependencies + liblz4-dev \ + # syscall_intercept dependencies + libcapstone-dev \ + # GekkoFS dependencies + libboost-program-options-dev \ + uuid-dev && \ + # install cmake 3.14+ since it's needed for some dependencies + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh diff --git a/docker/0.9.3/core/Makefile b/docker/0.9.3/core/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9c27a583a8ae1069865ad13cc0196214d40b9362 --- /dev/null +++ b/docker/0.9.3/core/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/core:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/core:0.9.3 . + +all: + docker buildx build -t gekkofs/core:0.9.3 . diff --git a/docker/0.9.3/deps/Dockerfile b/docker/0.9.3/deps/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ca4ecc8a646db53669fa642eaefdd2bfcc7e8804 --- /dev/null +++ b/docker/0.9.3/deps/Dockerfile @@ -0,0 +1,38 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to build GekkoFS" + +ENV GKFS_PATH /opt/gkfs +ENV GKFS_VERSION 0.9.3 + +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH /usr/local + +COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/patches ${SCRIPTS_PATH}/patches +COPY scripts/profiles ${SCRIPTS_PATH}/profiles + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python3-setuptools \ + libnuma-dev libyaml-dev libcurl4-openssl-dev \ + procps && \ + python3 -m pip install --upgrade pip && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && apt-get autoclean + +# Download and build dependencies +RUN cd ${SCRIPTS_PATH} && \ + /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \ + /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \ + cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\ + rm -rf ${DEPS_SRC_PATH} && \ + rm -rf ${SCRIPTS_PATH} && \ + rmdir ${GKFS_PATH} && \ + ldconfig diff --git a/docker/0.9.3/deps/Makefile b/docker/0.9.3/deps/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a97d3195f66a9cf7c5b44ed7b32d25abb49e6ab0 --- /dev/null +++ b/docker/0.9.3/deps/Makefile @@ -0,0 +1,23 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts remove-scripts +amd64: build-amd64 mount-scripts remove-scripts +aarch64: build-aarch64 mount-scripts remove-scripts + +copy-scripts: + cp -R $(GIT_ROOT)/scripts $(CWD)/scripts + +build: copy-scripts + docker build -t gekkofs/deps:0.9.3 . + +build-amd64: copy-scripts + docker build --platform amd64 -t gekkofs/deps:0.9.3 . + +build-aarch64: copy-scripts + docker build --platform aarch64 -t gekkofs/deps:0.9.3 . + +remove-scripts: + - rm -rf $(CWD)/scripts diff --git a/docker/0.9.3/docs/Dockerfile b/docker/0.9.3/docs/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f770f3a23de51eb7ee8f13d0f3b8ebf8623d2649 --- /dev/null +++ b/docker/0.9.3/docs/Dockerfile @@ -0,0 +1,34 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.33.1' \ + 'exhale==0.3.1' \ + 'sphinx-copybutton==0.5.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.17.0' \ + attrs && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/docs/Makefile b/docker/0.9.3/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..c6c89926af84c32ee55331a647a3d041ea6e0555 --- /dev/null +++ b/docker/0.9.3/docs/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/docs:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/docs:0.9.3 . + +all: + docker build -t gekkofs/docs:0.9.3 . \ No newline at end of file diff --git a/docker/0.9.3/linter/Dockerfile b/docker/0.9.3/linter/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f5d57118e518d36fb28f7a0f2bc95716dc99f2d0 --- /dev/null +++ b/docker/0.9.3/linter/Dockerfile @@ -0,0 +1,19 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to check the formatting of GekkoFS code" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # clang 15 deps for clang-format + lsb-release \ + wget \ + software-properties-common \ + gnupg2 && \ + # add clang-15 repos + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \ + # install clang-format + apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/linter/Makefile b/docker/0.9.3/linter/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f427eb387a7f8aebffa6a4522c428467835903e4 --- /dev/null +++ b/docker/0.9.3/linter/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/linter:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/linter:0.9.3 . + +all: + docker buildx build -t gekkofs/linter:0.9.3 . diff --git a/docker/0.9.3/testing/Dockerfile b/docker/0.9.3/testing/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..971944a86b0c5102eba84a912b33a53187078701 --- /dev/null +++ b/docker/0.9.3/testing/Dockerfile @@ -0,0 +1,32 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment to test GekkoFS" + +RUN \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + # required by lcov's genhtml + libgd-perl && \ + # install cmake 3.21+ since we need to produce JUnit XML files + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # install loguru + # (required by several of our scripts) + pip3 install loguru && \ + # install lcov_cobertura + # (required to produce Cobertura XML reports) + pip3 install lcov_cobertura && \ + # install lcov + # (required to produce partial coverage reports in parallel runs) + curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \ + tar xfz lcov-1.16.tar.gz && \ + cd lcov-1.16 && \ + make install && \ + cd .. && \ + # cleanup + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh && \ + rm -rf ./lcov-1.16.* diff --git a/docker/0.9.3/testing/Makefile b/docker/0.9.3/testing/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b5908b5399d36a19a96cc61a8f929fd9a6c4a581 --- /dev/null +++ b/docker/0.9.3/testing/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/testing:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/testing:0.9.3 . + +all: + docker build -t gekkofs/testing:0.9.3 . diff --git a/docs/sphinx/users/dependencies.rst b/docs/sphinx/users/dependencies.rst index 0d3732410f1b133cc755047a1df1ad2601fa2f0b..3490ef5d6300b3153f35f37f5ff252854252e306 100644 --- a/docs/sphinx/users/dependencies.rst +++ b/docs/sphinx/users/dependencies.rst @@ -43,4 +43,4 @@ Required Optional -------- -- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode. +- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling. diff --git a/docs/sphinx/users/forwarding.rst b/docs/sphinx/users/forwarding.rst index db0afc7b22e587f629bcacadfde6de82c62f7e80..e850b4d04d7b4649fd7b600f3dd9cd6c7dbad8f5 100644 --- a/docs/sphinx/users/forwarding.rst +++ b/docs/sphinx/users/forwarding.rst @@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes. Enabling GekkoFWD ------------------ -To enable the I/O forwarding mode of GekkoFS, the -:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when -configuring ` the build: - -.. code-block:: console - - $ cmake -DENABLE_FORWARDING:BOOL=ON +To enable the I/O forwarding mode of GekkoFS, an environment +variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided +to allow users to identify the map file on each client. +This environment variable will enable GekkoFS forwarding, then we +only need to use :code:`--enable-forwarding`` option in the servers. I/O Scheduling -------------- diff --git a/external/fmt b/external/fmt index f94b7364b9409f05207c3af3fa4666730e11a854..e57ca2e3685b160617d3d95fcd9e789c4e06ca88 160000 --- a/external/fmt +++ b/external/fmt @@ -1 +1 @@ -Subproject commit f94b7364b9409f05207c3af3fa4666730e11a854 +Subproject commit e57ca2e3685b160617d3d95fcd9e789c4e06ca88 diff --git a/external/hermes b/external/hermes index d8755039c82501323cdc9a4f5a2b922b9b3bcac7..2fa83ff4787d60e2d849c227e64d1b691f8fe0e4 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit d8755039c82501323cdc9a4f5a2b922b9b3bcac7 +Subproject commit 2fa83ff4787d60e2d849c227e64d1b691f8fe0e4 diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 4cc56ac387e724c02d549d4e5d4774a68c62be61..1ae9e3b40e76552f804dc6c3cd4c7dd053c291a3 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -34,7 +34,3 @@ add_subdirectory(daemon) add_subdirectory(client) target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in) - -if(GKFS_ENABLE_FORWARDING) - target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in) -endif() diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index f5df6ee5c796e2dd72c909ac4e8b1b15bb3912c2..2d12db14e2c3adedcb27147f6a2e166cab43a3a9 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -54,30 +54,3 @@ target_sources( syscalls/detail/syscall_info.h ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_intercept - PUBLIC gkfs_functions.hpp - env.hpp - hooks.hpp - intercept.hpp - logging.hpp - make_array.hpp - open_file_map.hpp - open_dir.hpp - path.hpp - preload.hpp - preload_context.hpp - preload_util.hpp - rpc/rpc_types.hpp - rpc/forward_management.hpp - rpc/forward_metadata.hpp - rpc/forward_data.hpp - syscalls/args.hpp - syscalls/decoder.hpp - syscalls/errno.hpp - syscalls/rets.hpp - syscalls/syscall.hpp - syscalls/detail/syscall_info.h - ) -endif() diff --git a/include/client/env.hpp b/include/client/env.hpp index 34a986facb0855c117370a2871fc36527c77d5df..5609785e8424437d6741cece32bbdb1a7bca1b5e 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -49,9 +49,8 @@ static constexpr auto LOG_PER_PROCESS = ADD_PREFIX("LOG_PER_PROCESS"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); -#ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); -#endif + static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); } // namespace gkfs::env diff --git a/include/client/logging.hpp b/include/client/logging.hpp index c99e2e4a034cf5567008ba1fa16ea6f7219105f9..2f00058d0885c8e533bbdf942c5147ab4fa78aa4 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -38,7 +38,6 @@ #include #include #include -#include #include #ifdef GKFS_DEBUG_BUILD @@ -175,6 +174,138 @@ log_buffer(int fd, const void* buffer, std::size_t length) { ::syscall_no_intercept(SYS_write, fd, buffer, length); } +/** + * @brief convert a time_t to a tm + * It is not POSIX compliant, but it dows not uses any syscall or timezone + * Converts a Unix timestamp (number of seconds since the beginning of 1970 + * CE) to a Gregorian civil date-time tuple in GMT (UTC) time zone. + * + * This conforms to C89 (and C99...) and POSIX. + * + * This implementation works, and doesn't overflow for any sizeof(time_t). + * It doesn't check for overflow/underflow in tm->tm_year output. Other than + * that, it never overflows or underflows. It assumes that that time_t is + * signed. + * + * This implements the inverse of the POSIX formula + * (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_15) + * for all time_t values, no matter the size, as long as tm->tm_year doesn't + * overflow or underflow. The formula is: tm_sec + tm_min*60 + tm_hour*3600 + * + tm_yday*86400 + (tm_year-70)*31536000 + ((tm_year-69)/4)*86400 - + * ((tm_year-1)/100)*86400 + ((tm_year+299)/400)*86400. + * + * License : GNU General Public License v2.0 from + * https://github.com/pts/minilibc686/ + * @param time_t + * @return tm + */ + +static inline struct tm* +mini_gmtime_r(const time_t* timep, struct tm* tm) { + const time_t ts = *timep; + time_t t = ts / 86400; + unsigned hms = + ts % + 86400; /* -86399 <= hms <= 86399. This needs sizeof(int) >= 4. */ + time_t c, f; + unsigned yday; /* 0 <= yday <= 426. Also fits to an `unsigned short', but + `int' is faster. */ + unsigned a; /* 0 <= a <= 2133. Also fits to an `unsigned short', but `int' + is faster. */ + if((int) hms < 0) { + --t; + hms += 86400; + } /* Fix quotient and negative remainder if ts was negative (i.e. before + year 1970 CE). */ + /* Now: -24856 <= t <= 24855. */ + tm->tm_sec = hms % 60; + hms /= 60; + tm->tm_min = hms % 60; + tm->tm_hour = hms / 60; + if(sizeof(time_t) > + 4) { /* Optimization. For int32_t, this would keep t intact, so we won't + have to do it. This produces unreachable code. */ + f = (t + 4) % 7; + if(f < 0) + f += 7; /* Fix negative remainder if (t + 4) was negative. */ + /* Now 0 <= f <= 6. */ + tm->tm_wday = f; + c = (t << 2) + 102032; + f = c / 146097; + if(c % 146097 < 0) + --f; /* Fix negative remainder if c was negative. */ + --f; + t += f; + f >>= 2; + t -= f; + f = (t << 2) + 102035; + c = f / 1461; + if(f % 1461 < 0) + --c; /* Fix negative remainder if f was negative. */ + } else { + tm->tm_wday = (t + 24861) % 7; /* t + 24861 >= 0. */ + /* Now: -24856 <= t <= 24855. */ + c = ((t << 2) + 102035) / 1461; + } + yday = t - 365 * c - (c >> 2) + 25568; + /* Now: 0 <= yday <= 425. */ + a = yday * 5 + 8; + /* Now: 8 <= a <= 2133. */ + tm->tm_mon = a / 153; + a %= 153; /* No need to fix if a < 0, because a cannot be negative here. */ + /* Now: 2 <= tm->tm_mon <= 13. */ + /* Now: 0 <= a <= 152. */ + tm->tm_mday = 1 + a / 5; /* No need to fix if a < 0, because a cannot be + negative here. */ + /* Now: 1 <= tm->tm_mday <= 31. */ + if(tm->tm_mon >= 12) { + tm->tm_mon -= 12; + /* Now: 0 <= tm->tm_mon <= 1. */ + ++c; + yday -= 366; + } else { /* Check for leap year (in c). */ + /* Now: 2 <= tm->tm_mon <= 11. */ + /* 1903: not leap; 1904: leap, 1900: not leap; 2000: leap */ + /* With sizeof(time_t) == 4, we have 1901 <= year <= 2038; of these + * years only 2000 is divisble by 100, and that's a leap year, no we + * optimize the check to `(c & 3) == 0' only. + */ + if(!((c & 3) == 0 && + (sizeof(time_t) <= 4 || c % 100 != 0 || (c + 300) % 400 == 0))) + --yday; /* These `== 0' comparisons work even if c < 0. */ + } + tm->tm_year = + c; /* This assignment may overflow or underflow, we don't check it. + Example: time_t is a huge int64_t, tm->tm_year is int32_t. */ + /* Now: 0 <= tm->tm_mon <= 11. */ + /* Now: 0 <= yday <= 365. */ + tm->tm_yday = yday; + tm->tm_isdst = 0; + return tm; +} + +static inline struct tm* +mini_gmtime(const time_t* timep) { + static struct tm tm; + return mini_gmtime_r(timep, &tm); +} + +static inline ssize_t +format_timeval(struct timeval* tv, char* buf, size_t sz) { + ssize_t written = -1; + struct tm* gm = mini_gmtime(&tv->tv_sec); + + + written = (ssize_t) strftime(buf, sz, "%Y-%m-%d %H:%M:%S", gm); + if((written > 0) && ((size_t) written < sz)) { + int w = snprintf(buf + written, sz - (size_t) written, ".%06ld", + tv->tv_usec); + written = (w > 0) ? written + w : -1; + } + + return written; +} + /** * format_timestamp_to - safely format a timestamp for logging messages * @@ -192,13 +323,10 @@ log_buffer(int fd, const void* buffer, std::size_t length) { * one thread exactly, and we pass it as an argument whenever we need to * format a timestamp. If no timezone is provided, we just format the epoch. * - * NOTE: we use the date C++ library to query the timezone database and - * to format the timestamps. */ template static inline void -format_timestamp_to(Buffer&& buffer, - const date::time_zone* const timezone = nullptr) { +format_timestamp_to(Buffer&& buffer) { struct ::timeval tv; @@ -208,17 +336,11 @@ format_timestamp_to(Buffer&& buffer, return; } - date::sys_time now{ - std::chrono::seconds{tv.tv_sec} + - std::chrono::microseconds{tv.tv_usec}}; + char buf[28]; - if(!timezone) { - fmt::format_to(buffer, "[{}] ", now.time_since_epoch().count()); - return; + if(format_timeval(&tv, buf, sizeof(buf)) > 0) { + fmt::format_to(std::back_inserter(buffer), "[{}] ", buf); } - - fmt::format_to(buffer, "[{}] ", - date::zoned_time{timezone, now}); } template @@ -226,7 +348,7 @@ static inline void format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { const auto ttid = syscall_no_intercept(SYS_gettid); - fmt::format_to(buffer, "[{}] [syscall] ", ttid); + fmt::format_to(std::back_inserter(buffer), "[{}] [syscall] ", ttid); char o; char t; @@ -256,20 +378,15 @@ format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { } const std::array tmp = {'[', o, t, ']', ' '}; - fmt::format_to(buffer, fmt::string_view(tmp.data(), tmp.size())); + fmt::format_to(std::back_inserter(buffer), + fmt::string_view(tmp.data(), tmp.size())); } } // namespace detail enum { max_buffer_size = LIBGKFS_LOG_MESSAGE_SIZE }; -struct static_buffer : public fmt::basic_memory_buffer { - -protected: - void - grow(std::size_t size) override final; -}; - +using static_buffer = fmt::basic_memory_buffer; struct logger { @@ -293,16 +410,17 @@ struct logger { } static_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); - fmt::format_to(buffer, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + detail::format_timestamp_to(buffer); + fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ", + log_process_id_, lookup_level_name(level)); if(!!(level & log::debug)) { - fmt::format_to(buffer, "<{}():{}> ", func, lineno); + fmt::format_to(std::back_inserter(buffer), "<{}():{}> ", func, + lineno); } - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(log_fd_, buffer); } @@ -337,8 +455,8 @@ struct logger { static_buffer prefix; detail::format_timestamp_to(prefix); - fmt::format_to(prefix, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + fmt::format_to(std::back_inserter(prefix), "[{}] [{}] ", + log_process_id_, lookup_level_name(level)); char buffer[max_buffer_size]; const int n = vsnprintf(buffer, sizeof(buffer), fmt, ap); @@ -387,8 +505,8 @@ struct logger { } static_buffer buffer; - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(fd, buffer); } @@ -410,8 +528,6 @@ struct logger { std::bitset<512> filtered_syscalls_; int debug_verbosity_; #endif - - const date::time_zone* timezone_; }; // the following static functions can be used to interact @@ -440,23 +556,6 @@ destroy_global_logger() { logger::global_logger().reset(); } -inline void -static_buffer::grow(std::size_t size) { - - const auto logger = get_global_logger(); - - if(logger) { - logger->log_mask_ &= ~(syscall | syscall_at_entry); - } - - std::fprintf( - stderr, - "FATAL: message too long for gkfs::log::static_buffer, increase the size of\n" - "LIBGKFS_LOG_MESSAGE_SIZE in CMake or reduce the length of the offending " - "message.\n"); - abort(); -} - } // namespace gkfs::log #define LOG(XXX, ...) LOG_##XXX(__VA_ARGS__) diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 175cd8f6edd89e441993f2055bb9eb325e611e24..6cae014b1d88b0445edd64b65321a9fe8f47717c 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -31,7 +31,8 @@ #define IOINTERCEPT_PRELOAD_HPP #include - +#include +#include #define EUNKNOWN (-1) #define CTX gkfs::preload::PreloadContext::getInstance() diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp index 05fa8296337d7ab2cfe689f394dc1d8365bdb61b..53ed726c3b1484bc7ec64b92508e26bf908ed73a 100644 --- a/include/client/syscalls/args.hpp +++ b/include/client/syscalls/args.hpp @@ -228,7 +228,7 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flag == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -239,12 +239,12 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { } if((flag == desc[i].flag_)) { - fmt::format_to(buffer, "{}", desc[i].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[i].name_); return; } } - fmt::format_to(buffer, "{:#x}", flag); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flag); } template @@ -254,7 +254,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flags == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -269,7 +269,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { } if((flags & desc[i].flag_) != 0) { - fmt::format_to(buffer, "{}{}", + fmt::format_to(std::back_inserter(buffer), "{}{}", buffer.size() != buffer_start ? "|" : "", desc[i].name_); flags &= ~desc[i].flag_; @@ -280,15 +280,15 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { if(flags != 0) { if(buffer.size() != buffer_start) { - fmt::format_to(buffer, "|"); + fmt::format_to(std::back_inserter(buffer), "|"); } - fmt::format_to(buffer, "{:#x}", flags); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flags); return; } if(buffer_start == buffer.size()) { - fmt::format_to(buffer, "0x0"); + fmt::format_to(std::back_inserter(buffer), "0x0"); } } @@ -311,7 +311,7 @@ format_whence_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SEEK_END) ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); } @@ -334,7 +334,7 @@ format_mmap_prot_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(PROT_WRITE), FLAG_ENTRY(PROT_EXEC)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; @@ -376,7 +376,7 @@ format_mmap_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { #endif ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; } @@ -420,14 +420,14 @@ format_clone_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(CLONE_NEWNET), FLAG_ENTRY(CLONE_IO)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); // the low byte in clone flags contains the number of the termination // signal sent to the parent when the child dies format_flag_set(buffer, parg.value & ~0x11l, flag_names); if((parg.value & 0x11l) != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_signum_arg_to(buffer, {"", parg.value & 0x11l}); } return; @@ -479,7 +479,7 @@ format_signum_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIGSYS)); if(std::strcmp(parg.name, "")) { - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); } format_flag(buffer, parg.value, flag_names); @@ -504,7 +504,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIG_UNBLOCK), FLAG_ENTRY(SIG_SETMASK)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, parg.value, flag_names); return; } @@ -518,7 +518,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } @@ -531,7 +531,7 @@ format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_fd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -546,11 +546,11 @@ inline void format_atfd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(static_cast(parg.value) == AT_FDCWD) { - fmt::format_to(buffer, "{}=AT_FDCWD", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=AT_FDCWD", parg.name); return; } - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -565,12 +565,12 @@ inline void format_cstr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}=\"{}\"", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}=\"{}\"", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } /** @@ -618,7 +618,7 @@ format_open_flags_to(FmtBuffer& buffer, long flags = parg.value; - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, flags & O_ACCMODE, flag_names); flags &= ~O_ACCMODE; @@ -636,7 +636,7 @@ format_open_flags_to(FmtBuffer& buffer, #endif // !O_TMPFILE if(flags != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_flag_set(buffer, flags, extra_flag_names); } } @@ -650,7 +650,7 @@ format_open_flags_to(FmtBuffer& buffer, template inline void format_octal_mode_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#04o}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#04o}", parg.name, parg.value); } /** @@ -664,12 +664,12 @@ inline void format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}={}", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } @@ -682,7 +682,7 @@ format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, parg.value); } @@ -695,7 +695,7 @@ format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -708,7 +708,7 @@ format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#x}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#x}", parg.name, parg.value); } #undef FLAG_ENTRY diff --git a/include/client/syscalls/decoder.hpp b/include/client/syscalls/decoder.hpp index eebdd6fdbc9551514b9c8d89cdc29d4a1fea6381..3172cc2df291c118dcf3de9c13cfa19f7177b10c 100644 --- a/include/client/syscalls/decoder.hpp +++ b/include/client/syscalls/decoder.hpp @@ -60,19 +60,19 @@ decode(FmtBuffer& buffer, const long syscall_number, const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); } template @@ -84,30 +84,31 @@ decode(FmtBuffer& buffer, const long syscall_number, const long argv[MAX_ARGS], const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } if(never_returns(syscall_number)) { - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); return; } if(error_code(result) != 0) { - fmt::format_to(buffer, ") = {} {} ({})", static_cast(-1), - errno_name(-result), errno_message(-result)); + fmt::format_to(std::back_inserter(buffer), ") = {} {} ({})", + static_cast(-1), errno_name(-result), + errno_message(-result)); return; } - fmt::format_to(buffer, ") = "); + fmt::format_to(std::back_inserter(buffer), ") = "); const auto& ret = sc.return_type(); ret.formatter()(buffer, result); } diff --git a/include/client/syscalls/rets.hpp b/include/client/syscalls/rets.hpp index adf77647c8928748d671effedaa73e45db1b26e8..1e40f77b02f6af0d457fd75b124f3ae14aaf85ef 100644 --- a/include/client/syscalls/rets.hpp +++ b/include/client/syscalls/rets.hpp @@ -121,7 +121,7 @@ template inline void format_none_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } template @@ -129,18 +129,18 @@ inline void format_ptr_ret_to(FmtBuffer& buffer, long val) { if(LIKELY(reinterpret_cast(val) != nullptr)) { - fmt::format_to(buffer, "{}", reinterpret_cast(val)); + fmt::format_to(std::back_inserter(buffer), "{}", reinterpret_cast(val)); return; } - fmt::format_to(buffer, "NULL"); + fmt::format_to(std::back_inserter(buffer), "NULL"); } template inline void format_dec_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "{}", val); + fmt::format_to(std::back_inserter(buffer), "{}", val); } #undef LIKELY diff --git a/include/common/CMakeLists.txt b/include/common/CMakeLists.txt index 9561361970c92e4ba89f02c1434508c2aaa1eb0e..38a23bc300c90a2a9f193bb440044a4890b07fa4 100644 --- a/include/common/CMakeLists.txt +++ b/include/common/CMakeLists.txt @@ -30,10 +30,3 @@ target_sources( gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp rpc/rpc_util.hpp ) - -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp - rpc/rpc_types.hpp rpc/rpc_util.hpp - ) -endif() diff --git a/include/config.hpp b/include/config.hpp index 7ec761f64e035c533adcebfa3a9da7b9c267816e..f74d0e8f419d11ed33c6e57d66cee087035e0cc9 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -39,7 +39,8 @@ namespace gkfs::config { constexpr auto hostfile_path = "./gkfs_hosts.txt"; -constexpr auto forwarding_file_path = "./gkfs_forwarding.map"; +// We do not default this, ENV variable always required. +constexpr auto forwarding_file_path = ""; namespace io { /* diff --git a/include/daemon/CMakeLists.txt b/include/daemon/CMakeLists.txt index 9cdfe06a94e699e3c0280ac3bdbe3daf512b00ee..26466ba16138af272d680b78b9f081a4f516c454 100644 --- a/include/daemon/CMakeLists.txt +++ b/include/daemon/CMakeLists.txt @@ -38,21 +38,6 @@ target_sources( handler/rpc_util.hpp ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon - PUBLIC daemon.hpp - util.hpp - ops/data.hpp - ops/metadentry.hpp - classes/fs_data.hpp - classes/rpc_data.hpp - handler/rpc_defs.hpp - handler/rpc_util.hpp - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) - endif() - +if(GKFS_ENABLE_AGIOS) + target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) endif() diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 13f8023d1aa2e4dd9c52cbb15d6c570eb37327a0..ef89b5e282e0b1f9cc7dd82f6b6c66c4287a9553 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -95,6 +95,7 @@ private: bool enable_stats_ = false; bool enable_chunkstats_ = false; bool enable_prometheus_ = false; + bool enable_forwarding_ = false; std::string stats_file_; // Prometheus @@ -246,6 +247,12 @@ public: void enable_chunkstats(bool enable_chunkstats); + bool + enable_forwarding() const; + + void + enable_forwarding(bool enable_forwarding); + bool enable_prometheus() const; diff --git a/scripts/profiles/0.9.3/agios.specs b/scripts/profiles/0.9.3/agios.specs new file mode 100644 index 0000000000000000000000000000000000000000..2d40731d99c027f650bd85853105960d2ca377ce --- /dev/null +++ b/scripts/profiles/0.9.3/agios.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies (except transport-specific and experimental)" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "agios" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/all.specs b/scripts/profiles/0.9.3/all.specs new file mode 100644 index 0000000000000000000000000000000000000000..fffb4c14b50890bf146bfa98ba68fbb8f67d6063 --- /dev/null +++ b/scripts/profiles/0.9.3/all.specs @@ -0,0 +1,86 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" + ["curl"]="7.82.0" + ["prometheus-cpp"]="v1.0.0" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "agios" "curl" "prometheus-cpp" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/arm.specs b/scripts/profiles/0.9.3/arm.specs new file mode 100644 index 0000000000000000000000000000000000000000..3e5c5f893ca85f87de3c1b61a102f2ff68070599 --- /dev/null +++ b/scripts/profiles/0.9.3/arm.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="fb73c3924b502e2840a5dc8a18746e395b06a8a6" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch.arm64" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" + ["syscall_intercept"]="ARM64" +) diff --git a/scripts/profiles/0.9.3/ci.specs b/scripts/profiles/0.9.3/ci.specs new file mode 100644 index 0000000000000000000000000000000000000000..deafa628e9801e8245d52fd1055888788dbe67d4 --- /dev/null +++ b/scripts/profiles/0.9.3/ci.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies required by the CI" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["prometheus-cpp"]="v1.0.0" + ) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" + ) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "agios" "parallax" "prometheus-cpp" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/default.specs b/scripts/profiles/0.9.3/default.specs new file mode 100644 index 0000000000000000000000000000000000000000..838c98145663da012a5b3177fe63ac52a2dbc580 --- /dev/null +++ b/scripts/profiles/0.9.3/default.specs @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/direct.specs b/scripts/profiles/0.9.3/direct.specs new file mode 100644 index 0000000000000000000000000000000000000000..90f4506ef762eafd7d378a4dbf41689268610fee --- /dev/null +++ b/scripts/profiles/0.9.3/direct.specs @@ -0,0 +1,76 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Direct dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/infiniband_verbs.specs b/scripts/profiles/0.9.3/infiniband_verbs.specs new file mode 100644 index 0000000000000000000000000000000000000000..0d6f865e409caed07581f2a5622f87d72b760ca8 --- /dev/null +++ b/scripts/profiles/0.9.3/infiniband_verbs.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Infiniband supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%verbs"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading/installing +order=( + "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%verbs"]="--enable-verbs=yes" +) diff --git a/CMake/FindDate.cmake b/scripts/profiles/0.9.3/install/agios.install similarity index 60% rename from CMake/FindDate.cmake rename to scripts/profiles/0.9.3/install/agios.install index 78b6a79d8c5cce0d27a491ba0b724881701e5f14..3d0392ff5305710794c19dc1e8b96985722fa5b7 100644 --- a/CMake/FindDate.cmake +++ b/scripts/profiles/0.9.3/install/agios.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -25,44 +25,33 @@ # # # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ +# vi: ft=bash -find_path(DATE_INCLUDE_DIR - NAMES date/date.h -) - -find_path(TZ_INCLUDE_DIR - NAMES date/tz.h -) - -find_library(TZ_LIBRARY - NAMES tz -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( Date - DEFAULT_MSG - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) - -if(Date_FOUND) - set(DATE_INCLUDE_DIRS ${DATE_INCLUDE_DIR}) - set(TZ_INCLUDE_DIRS ${TZ_INCLUDE_DIR}) - set(TZ_LIBRARIES ${TZ_LIBRARY}) +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ - if(NOT TARGET Date::TZ) - add_library(Date::TZ UNKNOWN IMPORTED) - set_target_properties(Date::TZ PROPERTIES - IMPORTED_LOCATION "${TZ_LIBRARY}" - INTERFACE_COMPILE_DEFINITIONS "USE_OS_TZDB=1" - INTERFACE_INCLUDE_DIRECTORIES "${TZ_INCLUDE_DIR}" - ) - endif() -endif() +pkg_install() { + ID="agios" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. + make install +} -mark_as_advanced( - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/argobots.install b/scripts/profiles/0.9.3/install/argobots.install new file mode 100644 index 0000000000000000000000000000000000000000..96ccea920eeb4493c7e6287737cf091945e6b06a --- /dev/null +++ b/scripts/profiles/0.9.3/install/argobots.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/capstone.install b/scripts/profiles/0.9.3/install/capstone.install new file mode 100644 index 0000000000000000000000000000000000000000..ff299d82b47f76389b3650553014438c65e92ae7 --- /dev/null +++ b/scripts/profiles/0.9.3/install/capstone.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/curl.install b/scripts/profiles/0.9.3/install/curl.install new file mode 100644 index 0000000000000000000000000000000000000000..d873819c4521e1ed8dce128c2699cc764356f92a --- /dev/null +++ b/scripts/profiles/0.9.3/install/curl.install @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="curl" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + autoreconf -fi + ./configure --prefix="${INSTALL_DIR}" --without-ssl + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/json-c.install b/scripts/profiles/0.9.3/install/json-c.install new file mode 100644 index 0000000000000000000000000000000000000000..7280cda81e54d91500ef5137a45ad56cad298bc1 --- /dev/null +++ b/scripts/profiles/0.9.3/install/json-c.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/json-c" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install + # Margo doesn't search in both directories, so we make it available in both lib and lib64 + if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then + cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/ + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/libfabric%verbs.install b/scripts/profiles/0.9.3/install/libfabric%verbs.install new file mode 100644 index 0000000000000000000000000000000000000000..15257eb817549eefcab8264671c5df97307e49db --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric%verbs.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/libfabric.install b/scripts/profiles/0.9.3/install/libfabric.install new file mode 100644 index 0000000000000000000000000000000000000000..7db89438e194931bb6311ed581e8d0d81576fa85 --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/lz4.install b/scripts/profiles/0.9.3/install/lz4.install new file mode 100644 index 0000000000000000000000000000000000000000..13e4e53b381f7df45aac175f3a9af84498b8abd4 --- /dev/null +++ b/scripts/profiles/0.9.3/install/lz4.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + # try to remove binaries first in case they already exist. Otherwise install fails. + LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4) + for LZ4_BIN in "${LZ4_BINS[@]}"; do + [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN" + done + make -j"${CORES}" + make DESTDIR="${INSTALL_DIR}" PREFIX="" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/margo.install b/scripts/profiles/0.9.3/install/margo.install new file mode 100644 index 0000000000000000000000000000000000000000..d2c6e4b5f2d21ff0e0f02865c53fe374d161b976 --- /dev/null +++ b/scripts/profiles/0.9.3/install/margo.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="margo" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/mercury.install b/scripts/profiles/0.9.3/install/mercury.install new file mode 100644 index 0000000000000000000000000000000000000000..601305b830c8cf1e320ca81cbdb45368b335cc3c --- /dev/null +++ b/scripts/profiles/0.9.3/install/mercury.install @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + # if the profile compiles bmi, enable it + if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then + USE_BMI="-DNA_USE_BMI:BOOL=ON" + else + USE_BMI="-DNA_USE_BMI:BOOL=OFF" + fi + + # if the profile provides any flavour of libfabric, enable it + if profile_has_dependency "^libfabric.*$"; then + USE_OFI="-DNA_USE_OFI:BOOL=ON" + else + USE_OFI="-DNA_USE_OFI:BOOL=OFF" + fi + + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + ${USE_BMI} ${USE_OFI} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/parallax.install b/scripts/profiles/0.9.3/install/parallax.install new file mode 100644 index 0000000000000000000000000000000000000000..38bd9c68961003c68762054edecd89586bcf2bda --- /dev/null +++ b/scripts/profiles/0.9.3/install/parallax.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + CURR="${SOURCE_DIR}/parallax" + # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \ + -DDISABLE_LOGGING:BOOL=ON \ + .. + make -j"${CORES}" + make install + # We need to copy this file as it is not installed + cp ${CURR}/lib/include/parallax/structures.h ${INSTALL_DIR}/include/ +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/prometheus-cpp.install b/scripts/profiles/0.9.3/install/prometheus-cpp.install new file mode 100644 index 0000000000000000000000000000000000000000..62e420217339c0217176db6aecec3a549417df6c --- /dev/null +++ b/scripts/profiles/0.9.3/install/prometheus-cpp.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="prometheus-cpp" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/rocksdb.install b/scripts/profiles/0.9.3/install/rocksdb.install new file mode 100644 index 0000000000000000000000000000000000000000..6ece050bd753ff3474c69ac1e013cd541e4d8ff7 --- /dev/null +++ b/scripts/profiles/0.9.3/install/rocksdb.install @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CXXFLAGS='' + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move -Wno-error=maybe-uninitialized' + fi + + # TODO use SSE? + CURR="${SOURCE_DIR}/rocksdb" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \ + -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \ + -DROCKSDB_BUILD_SHARED=OFF \ + -DWITH_LZ4=ON \ + -DWITH_GFLAGS=OFF \ + -DUSE_RTTI=1 \ + -DPORTABLE=1 \ + -DWITH_ALL_TESTS=OFF \ + -DWITH_BENCHMARK_TOOLS=OFF \ + -DWITH_TOOLS=OFF .. + make -j"${CORES}" install + + +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/syscall_intercept.install b/scripts/profiles/0.9.3/install/syscall_intercept.install new file mode 100644 index 0000000000000000000000000000000000000000..5c862dcb7a216c5781a198840c7def0f92931f80 --- /dev/null +++ b/scripts/profiles/0.9.3/install/syscall_intercept.install @@ -0,0 +1,71 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + prepare_build_dir "${CURR}" + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cd "${CURR}"/arch/aarch64/ + mkdir -p build + cd build + else + cd "${CURR}"/build + fi + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DBUILD_TESTS:BOOK=OFF .. + make -j"${CORES}" install + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cp "${CURR}"/arch/aarch64/include/libsyscall_intercept_hook_point.h ${INSTALL_DIR}/include + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/marenostrum4.specs b/scripts/profiles/0.9.3/marenostrum4.specs new file mode 100644 index 0000000000000000000000000000000000000000..793a5de9d0b50179f7bd795278b30a96060c83ff --- /dev/null +++ b/scripts/profiles/0.9.3/marenostrum4.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Marenostrum 4 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/0.9.3/mogon2.specs b/scripts/profiles/0.9.3/mogon2.specs new file mode 100644 index 0000000000000000000000000000000000000000..83b2242fb7f23bdf2d3bbe3a0247bed03849fbed --- /dev/null +++ b/scripts/profiles/0.9.3/mogon2.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/ngio.specs b/scripts/profiles/0.9.3/ngio.specs new file mode 100644 index 0000000000000000000000000000000000000000..ef8875716ce80de2d9abef95b46ac22cbace5799 --- /dev/null +++ b/scripts/profiles/0.9.3/ngio.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for NEXTGenIO prototype cluster" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/omnipath_psm2.specs b/scripts/profiles/0.9.3/omnipath_psm2.specs new file mode 100644 index 0000000000000000000000000000000000000000..9d18f59b33675cb19a67eb25f5ce086e900eed7d --- /dev/null +++ b/scripts/profiles/0.9.3/omnipath_psm2.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Omnipath supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/p9.specs b/scripts/profiles/0.9.3/p9.specs new file mode 100644 index 0000000000000000000000000000000000000000..8dff909a8c255b13e34b9abf102338c54308cd5c --- /dev/null +++ b/scripts/profiles/0.9.3/p9.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.10.0" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/latest b/scripts/profiles/latest index f76f9131742ee389211cdafc141553a15268098f..b3ec1638fda741e3680a70960377c66e3b41c4c7 120000 --- a/scripts/profiles/latest +++ b/scripts/profiles/latest @@ -1 +1 @@ -0.9.2 \ No newline at end of file +0.9.3 \ No newline at end of file diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0108fb9c9fa5b8a543fcf0170dca8422c9941554..c408fc7262df42000c2abe248d8f063020b930de 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -52,6 +52,11 @@ target_sources( syscalls/detail/syscall_info.c ) + + if(GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) + endif() + target_link_libraries( gkfs_intercept PRIVATE metadata distributor env_util arithmetic path_util rpc_utils @@ -61,7 +66,7 @@ target_link_libraries( hermes fmt::fmt Threads::Threads - Date::TZ + Syscall_intercept::Syscall_intercept ) install( @@ -71,54 +76,3 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -# ############################################################################## -# This builds the `libgkfwd_intercept.so` library: the client for GekkoFS' -# forwarding mode based on syscall interception. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - add_library(gkfwd_intercept SHARED) - - target_sources( - gkfwd_intercept - PRIVATE gkfs_functions.cpp - hooks.cpp - intercept.cpp - logging.cpp - open_file_map.cpp - open_dir.cpp - path.cpp - preload.cpp - preload_context.cpp - preload_util.cpp - rpc/rpc_types.cpp - rpc/forward_data.cpp - rpc/forward_management.cpp - rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c - ) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_FORWARDING) - - if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_AGIOS) - endif() - - target_link_libraries( - gkfwd_intercept - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads - Date::TZ - ) - - install( - TARGETS gkfwd_intercept - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs - ) -endif() diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 56e1f829b3319d2a91d21504a756639a927b1a35..a986d5898b12e4450a8c63d45a893f493e86cd51 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -878,7 +878,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, "newname \"{}\", flags {}", __func__, olddfd, oldname, newdfd, newname, flags); - const char* oldpath_pass; + const char* oldpath_pass = oldname; std::string oldpath_resolved; auto oldpath_status = CTX->relativize_fd_path(olddfd, oldname, oldpath_resolved); @@ -902,7 +902,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, return -EINVAL; } - const char* newpath_pass; + const char* newpath_pass = newname; std::string newpath_resolved; auto newpath_status = CTX->relativize_fd_path(newdfd, newname, newpath_resolved); diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 88f9947140e070bbca5abc8f6410e7a0fd942588..830d5691ecd281c9974bc98a5ca299935fdc5b01 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -32,11 +32,7 @@ #include #include #include - -extern "C" { -#include -#include -} +#include #ifdef GKFS_ENABLE_LOGGING @@ -289,8 +285,7 @@ logger::logger(const std::string& opts, const std::string& path, , const std::string& filter, int verbosity #endif - ) - : timezone_(nullptr) { +) { /* use stderr by default */ log_fd_ = 2; @@ -335,34 +330,6 @@ logger::logger(const std::string& opts, const std::string& path, log_fd_ = fd; } - // Finding the current timezone implies accessing OS files (i.e. syscalls), - // but current_zone() doesn't actually retrieve the time zone but rather - // provides a descriptor to it that is **atomically initialized** upon its - // first use. Thus, if we don't force the initialization here, logging the - // first intercepted syscall will produce a call to date::time_zone::init() - // (under std::call_once) which internally ends up calling fopen(). Since - // fopen() ends up calling sys_open(), we will need to generate another - // timestamp for a system call log entry, which will attempt to call - // date::time_zone::init() since the prior initialization (under the same - // std::call_once) has not yet completed. - // - // Unfortunately, date::time_zone doesn't provide a function to prevent - // this lazy initialization, therefore we force it by requesting - // information from an arbitrary timepoint (January 1st 1970) which forces - // the initialization. This doesn't do any actual work and could safely be - // removed if the date API ends up providing this functionality. - try { - timezone_ = date::current_zone(); -#ifdef GKFS_DEBUG_BUILD - using namespace date; - timezone_->get_info(date::sys_days{January / 1 / 1970}); -#endif // GKFS_DEBUG_BUILD - } catch(const std::exception& ex) { - // if timezone initialization fails, setting timezone_ to nullptr - // makes format_timestamp_to() default to producing epoch timestamps - timezone_ = nullptr; - } - #ifdef GKFS_ENABLE_LOGGING const auto log_hermes_message = [](const std::string& msg, hermes::log::level l, int severity, @@ -490,9 +457,9 @@ logger::log_syscall(syscall::info info, const long syscall_number, print_syscall: - static_buffer buffer; + fmt::basic_memory_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); + detail::format_timestamp_to(buffer); detail::format_syscall_info_to(buffer, info); if(result) { @@ -501,7 +468,7 @@ print_syscall: syscall::decode(buffer, syscall_number, args); } - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), "\n"); ::syscall_no_intercept(SYS_write, log_fd_, buffer.data(), buffer.size()); } diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4de0df2c31168fe6f07a434a59e0eeb2331be27c..6d7a0c0e9e6416b33c1d2766b61c14232da5b8c7 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -50,13 +50,13 @@ std::unique_ptr ld_network_service; // extern variable namespace { -#ifdef GKFS_ENABLE_FORWARDING +// FORWARDING pthread_t mapper; bool forwarding_running; pthread_mutex_t remap_mutex; pthread_cond_t remap_signal; -#endif +// END FORWARDING inline void exit_error_msg(int errcode, const string& msg) { @@ -102,7 +102,6 @@ init_hermes_client() { return true; } -#ifdef GKFS_ENABLE_FORWARDING void* forwarding_mapper(void* p) { struct timespec timeout; @@ -115,7 +114,7 @@ forwarding_mapper(void* p) { try { gkfs::utils::load_forwarding_map(); - if(previous != CTX->fwd_host_id()) { + if(previous != (int64_t) CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); previous = CTX->fwd_host_id(); @@ -133,18 +132,14 @@ forwarding_mapper(void* p) { return nullptr; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } -#endif -#ifdef GKFS_ENABLE_FORWARDING void destroy_forwarding_mapper() { forwarding_running = false; @@ -153,7 +148,6 @@ destroy_forwarding_mapper() { pthread_join(mapper, NULL); } -#endif void log_prog_name() { @@ -207,31 +201,34 @@ init_environment() { } /* Setup distributor */ -#ifdef GKFS_ENABLE_FORWARDING - try { - gkfs::utils::load_forwarding_map(); + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch(std::exception& e) { - exit_error_msg( - EXIT_FAILURE, - fmt::format("Unable set the forwarding host '{}'", e.what())); - } + if(!forwarding_map_file.empty()) { + try { + gkfs::utils::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch(std::exception& e) { + exit_error_msg(EXIT_FAILURE, + fmt::format("Unable set the forwarding host '{}'", + e.what())); + } + + auto forwarder_dist = std::make_shared( + CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + } else { - auto forwarder_dist = std::make_shared( - CTX->fwd_host_id(), CTX->hosts().size()); - CTX->distributor(forwarder_dist); -#else #ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #else - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#endif - CTX->distributor(distributor); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #endif - + CTX->distributor(distributor); + } LOG(INFO, "Retrieving file system configuration..."); @@ -290,9 +287,11 @@ init_preload() { CTX->unprotect_user_fds(); -#ifdef GKFS_ENABLE_FORWARDING - init_forwarding_mapper(); -#endif + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(!forwarding_map_file.empty()) { + init_forwarding_mapper(); + } gkfs::preload::start_interception(); errno = oerrno; @@ -304,10 +303,11 @@ init_preload() { */ void destroy_preload() { -#ifdef GKFS_ENABLE_FORWARDING - destroy_forwarding_mapper(); -#endif - + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(!forwarding_map_file.empty()) { + destroy_forwarding_mapper(); + } CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index aa09d0e0727d69e509aea64a1133063bc8db3e71..ed9e2368755485c09f7eefc8ccd5db46bfbab285 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -286,7 +286,6 @@ metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md, return 0; } -#ifdef GKFS_ENABLE_FORWARDING map load_forwarding_map_file(const std::string& lfpath) { @@ -320,9 +319,7 @@ load_forwarding_map_file(const std::string& lfpath) { } return forwarding_map; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void load_forwarding_map() { string forwarding_map_file; @@ -359,7 +356,6 @@ load_forwarding_map() { CTX->fwd_host_id(forwarding_map[local_hostname]); } -#endif vector> read_hosts_file() { diff --git a/src/client/rpc/forward_management.cpp b/src/client/rpc/forward_management.cpp index c975842e560d15a9f107b3138ba1016b853071ed..7504b1f6b6bb869d99faab62c946a620ecace192 100644 --- a/src/client/rpc/forward_management.cpp +++ b/src/client/rpc/forward_management.cpp @@ -47,7 +47,7 @@ forward_get_fs_config() { bool found = false; size_t idx = 0; - while(!found && idx <= CTX->hosts().size()) { + while(!found && idx < CTX->hosts().size()) { try { LOG(DEBUG, "Retrieving file system configurations from daemon"); // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 95c4ade7542ead463ea3565842481a82738fa102..89af71c7624109df1d90b7c4bae1b29fd030b193 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -72,63 +72,15 @@ target_link_libraries( Threads::Threads ) +if(GKFS_ENABLE_AGIOS) + target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp) + target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS) + target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS) +endif() + if(GKFS_ENABLE_CODE_COVERAGE) target_code_coverage(gkfs_daemon AUTO) endif() install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -# ############################################################################## -# This builds the `gkfwd_daemon` executable: the daemon for GekkoFS data -# forwarding mode. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - - add_executable(gkfwd_daemon) - - # We need to add here any files that may have different compile definitions - target_sources( - gkfwd_daemon - PRIVATE ../common/rpc/rpc_util.cpp - daemon.cpp - util.cpp - ops/metadentry.cpp - ops/data.cpp - classes/fs_data.cpp - classes/rpc_data.cpp - handler/srv_metadata.cpp - handler/srv_management.cpp - handler/srv_data.cpp - ) - - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_FORWARDING) - target_link_libraries( - gkfwd_daemon - PUBLIC # internal libs - metadata - metadata_backend - storage - distributor - statistics - log_util - env_util - path_util - # external libs - CLI11::CLI11 - fmt::fmt - Mercury::Mercury - Argobots::Argobots - Margo::Margo - # others - Threads::Threads - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PRIVATE scheduler/agios.cpp) - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_AGIOS) - target_link_libraries(gkfwd_daemon PRIVATE AGIOS::AGIOS) - endif() - - install(TARGETS gkfwd_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif () diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 0d38c718c5af2b76b9c8be8414088a4239ee1c52..33b43bf5942789507cc951f737ac1da2c7188a9e 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -256,6 +256,15 @@ FsData::enable_chunkstats(bool enable_chunkstats) { FsData::enable_chunkstats_ = enable_chunkstats; } +bool +FsData::enable_forwarding() const { + return enable_forwarding_; +} + +void +FsData::enable_forwarding(bool enable_forwarding) { + FsData::enable_forwarding_ = enable_forwarding; +} bool FsData::enable_prometheus() const { return enable_prometheus_; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49a87fa604e8b6d747c6d3e9dd19b66f9ee27211..2911c69219201b1100da85418dc4b8f8a422b9ff 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -261,25 +261,6 @@ init_environment() { throw; } - GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ", __func__); - try { -#ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared(); -#else - auto distributor = std::make_shared(); -#endif - RPC_DATA->distributor(distributor); - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to initialize Distributor: {}", __func__, - e.what()); - throw; - } - -#ifdef GKFS_ENABLE_FORWARDING - GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__); -#endif - #ifdef GKFS_ENABLE_AGIOS // Initialize AGIOS scheduler GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'", @@ -552,10 +533,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { assert(desc.count("--rootdir")); auto rootdir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - // In forwarding mode, the backend is shared - auto rootdir_path = fs::path(rootdir); -#else auto rootdir_path = fs::path(rootdir); if(desc.count("--rootdir-suffix")) { if(opts.rootdir_suffix == gkfs::config::data::chunk_dir || @@ -570,7 +547,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { rootdir_path /= opts.rootdir_suffix; GKFS_DATA->rootdir_suffix(opts.rootdir_suffix); } -#endif if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) @@ -589,14 +565,23 @@ parse_input(const cli_options& opts, const CLI::App& desc) { fs::create_directories(rootdir_path); GKFS_DATA->rootdir(rootdir_path.native()); + if(desc.count("--enable-forwarding")) { + GKFS_DATA->enable_forwarding(true); + GKFS_DATA->spdlogger()->info("{}() Forwarding mode enabled", __func__); + } + if(desc.count("--metadir")) { auto metadir = opts.metadir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); -#else + auto metadir_path = fs::path(metadir); -#endif + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); + } + + if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...", @@ -613,13 +598,16 @@ parse_input(const cli_options& opts, const CLI::App& desc) { // use rootdir as metadata dir auto metadir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); - fs::create_directories(metadir_path); - GKFS_DATA->metadir(fs::canonical(metadir_path).native()); -#else - GKFS_DATA->metadir(GKFS_DATA->rootdir()); -#endif + + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + auto metadir_path = + fs::path(metadir) / fmt::format_int(getpid()).str(); + fs::create_directories(metadir_path); + GKFS_DATA->metadir(fs::canonical(metadir_path).native()); + } else + GKFS_DATA->metadir(GKFS_DATA->rootdir()); } if(desc.count("--dbbackend")) { @@ -787,6 +775,9 @@ main(int argc, const char* argv[]) { desc.add_option( "--output-stats", opts.stats_file, "Creates a thread that outputs the server stats each 10s to the specified file."); + desc.add_flag( + "--enable-forwarding", + "Enables forwarding mode, so the metadata is stored in a separate directory (pid)."); #ifdef GKFS_ENABLE_PROMETHEUS desc.add_flag( "--enable-prometheus", diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index bc8e0c86427bf14538ce3388b95755f78cabceac..4bc63e41d3590bae7eb7c88070bf9c7362c8b81f 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -165,7 +165,7 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): lines = f.readlines() for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] assert ion == '0' @@ -190,15 +190,16 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): ret = c00.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of read bytes - + + # log line should go to the end of the file with open(c00.log) as f: lines = f.readlines() - + ion = 0 for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] - assert ion == '1' + assert ion == '1' def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): """Write files from one client and read in the other using two daemons""" diff --git a/tests/integration/harness/gkfs.io/serialize.hpp b/tests/integration/harness/gkfs.io/serialize.hpp index 3873b2ec85d484a52f748820f150738ea9a938ec..92328fcf86dc53b3d71d3acfa63a4ec5de4ca72a 100644 --- a/tests/integration/harness/gkfs.io/serialize.hpp +++ b/tests/integration/harness/gkfs.io/serialize.hpp @@ -178,13 +178,10 @@ template <> struct adl_serializer { static void to_json(json& j, const struct ::statfs opt) { - j = json{{"f_type", opt.f_type}, - {"f_bsize", opt.f_bsize}, - {"f_blocks", opt.f_blocks}, - {"f_bfree", opt.f_bfree}, - {"f_bavail", opt.f_bavail}, - {"f_files", opt.f_files}, - {"f_ffree", opt.f_ffree}}; + j = json{{"f_type", opt.f_type}, {"f_bsize", opt.f_bsize}, + {"f_blocks", opt.f_blocks}, {"f_bfree", opt.f_bfree}, + {"f_bavail", opt.f_bavail}, {"f_files", opt.f_files}, + {"f_ffree", opt.f_ffree}}; } }; @@ -253,7 +250,8 @@ struct formatter { template auto - format(const struct ::dirent& dirent, FormatContext& ctx) { + format(const struct ::dirent& dirent, FormatContext& ctx) const + -> format_context::iterator { return format_to(ctx.out(), "struct dirent {{\n" " d_ino = {};\n" diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index ef48c404255306f5301ea029c4c189feb7e10c20..a65a171f7b7ad50f5129f2d27ebe196e7d729088 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -48,15 +48,16 @@ gkfs_client_log_level = 'all' gkfs_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' -gkfwd_daemon_cmd = 'gkfwd_daemon' +gkfwd_daemon_cmd = 'gkfs_daemon' gkfwd_client_cmd = 'gkfs.io' -gkfwd_client_lib_file = 'libgkfwd_intercept.so' +gkfwd_client_lib_file = 'libgkfs_intercept.so' gkfwd_hosts_file = 'gkfs_hosts.txt' gkfwd_forwarding_map_file = 'gkfs_forwarding.map' -gkfwd_daemon_log_file = 'gkfwd_daemon.log' +gkfwd_daemon_log_file = 'gkfs_daemon.log' gkfwd_daemon_log_level = '100' -gkfwd_client_log_file = 'gkfwd_client.log' +gkfwd_client_log_file = 'gkfs_client.log' gkfwd_client_log_level = 'all' +gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' @@ -520,7 +521,8 @@ class ShellClient: 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -732,7 +734,8 @@ class FwdDaemon: args = [ '--mountdir', self.mountdir, '--metadir', self.metadir, '--rootdir', self.rootdir, - '-l', self._address ] + '-l', self._address, + '--enable-forwarding'] logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -902,11 +905,12 @@ class FwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'LD_PRELOAD' : self._preload_library, + 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -996,7 +1000,8 @@ class ShellFwdClient: 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file), 'LIBGKFS_LOG' : gkfwd_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) diff --git a/tests/scripts/compile_dep.sh/0.9.3/agios.out b/tests/scripts/compile_dep.sh/0.9.3/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..25a1cad3a58a9c7c07c75ba87bfa2c5a6a9643ca --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/agios.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/all.out b/tests/scripts/compile_dep.sh/0.9.3/all.out new file mode 100644 index 0000000000000000000000000000000000000000..2fcd55b6f7dc5e574eeea0233a628e75ea105f4e --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/all.out @@ -0,0 +1,14 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### +Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.3/arm.out b/tests/scripts/compile_dep.sh/0.9.3/arm.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/arm.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ci.out b/tests/scripts/compile_dep.sh/0.9.3/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..9ec3c24bb12c16078d2b3869831331de15f5e305 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ci.out @@ -0,0 +1,8 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/default.out b/tests/scripts/compile_dep.sh/0.9.3/default.out new file mode 100644 index 0000000000000000000000000000000000000000..ca393e65d3bc03aa72fe6a628efab5dad85174b9 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/default.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/direct.out b/tests/scripts/compile_dep.sh/0.9.3/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..816266773b569d3d098245c7b868f7983753ff12 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/direct.out @@ -0,0 +1,6 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..ff149d28f1874e6fe4f94484c4f0b4f7abf49416 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,9 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric%verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/mogon2.out b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ngio.out b/tests/scripts/compile_dep.sh/0.9.3/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..1af91c3cf3dfe4c78d6f7068f1d8d0942d1b54f1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ngio.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/p9.out b/tests/scripts/compile_dep.sh/0.9.3/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..e05da6537482068490c2b4f04ff8efa204cc6538 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/p9.out @@ -0,0 +1,11 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/latest b/tests/scripts/compile_dep.sh/latest index f76f9131742ee389211cdafc141553a15268098f..b3ec1638fda741e3680a70960377c66e3b41c4c7 120000 --- a/tests/scripts/compile_dep.sh/latest +++ b/tests/scripts/compile_dep.sh/latest @@ -1 +1 @@ -0.9.2 \ No newline at end of file +0.9.3 \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/agios.out b/tests/scripts/dl_dep.sh/0.9.3/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..a74c129fef0c92f814b278ccab9b84e1e560ca36 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/agios.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/all.out b/tests/scripts/dl_dep.sh/0.9.3/all.out new file mode 100644 index 0000000000000000000000000000000000000000..19052d3e34adef2fbba658e13445162f2ea2cab2 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/all.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/arm.out b/tests/scripts/dl_dep.sh/0.9.3/arm.out new file mode 100644 index 0000000000000000000000000000000000000000..087b7e983a49d0f1c4c05ed4946dad3658323634 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/arm.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[fb73c3924b502e2840a5dc8a18746e395b06a8a6]' and flags '' +Done diff --git a/tests/scripts/dl_dep.sh/0.9.3/ci.out b/tests/scripts/dl_dep.sh/0.9.3/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..1cbaf3e06372ce83045304dc659af4aeca567776 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ci.out @@ -0,0 +1,9 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/default.out b/tests/scripts/dl_dep.sh/0.9.3/default.out new file mode 100644 index 0000000000000000000000000000000000000000..adab1b60f8a30907c7b74735c5c79b48a8c8b76a --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/default.out @@ -0,0 +1,10 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/direct.out b/tests/scripts/dl_dep.sh/0.9.3/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..8a4d1e06338c66239dc77d38fb7ea15150435304 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/direct.out @@ -0,0 +1,7 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..34854d05b61026f25fab5714d755dfd1e4389e3c --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,10 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..addae0a621545b14df618865a3224d3ac241ca9e --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/mogon2.out b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..addae0a621545b14df618865a3224d3ac241ca9e --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/ngio.out b/tests/scripts/dl_dep.sh/0.9.3/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..addae0a621545b14df618865a3224d3ac241ca9e --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ngio.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..9267747ca2fdd47464d1a5dadcbc3cc0b3b3a7f0 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/p9.out b/tests/scripts/dl_dep.sh/0.9.3/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..68d0384dbb02acc2833f6fead9f0816558801fa8 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/p9.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.10.0.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f]' and flags '' +Done diff --git a/tests/scripts/dl_dep.sh/latest b/tests/scripts/dl_dep.sh/latest index f76f9131742ee389211cdafc141553a15268098f..b3ec1638fda741e3680a70960377c66e3b41c4c7 120000 --- a/tests/scripts/dl_dep.sh/latest +++ b/tests/scripts/dl_dep.sh/latest @@ -1 +1 @@ -0.9.2 \ No newline at end of file +0.9.3 \ No newline at end of file